_DIRECT_RUN = (__name__ == '__main__') import os import json import asyncio import requests import socket import ipaddress import subprocess import platform import threading import time import shutil import glob import uuid import re import sqlite3 import types import collections import functools from pathlib import Path from urllib.parse import quote, urljoin, urlparse from concurrent.futures import ThreadPoolExecutor, as_completed from flask import Flask, render_template, request, jsonify, redirect, send_file, send_from_directory, Response, session, g, abort from flask_socketio import SocketIO, emit, join_room, leave_room from utils.logging_config import get_logger, setup_logging from utils.async_helpers import run_async from mutagen.flac import FLAC from mutagen.mp4 import MP4 from mutagen.oggvorbis import OggVorbis # --- Core Application Imports --- # Import the same core clients and config manager used by the GUI app from config.settings import config_manager # Setup logging early to avoid any import-time logs from being swallowed _log_level = config_manager.get('logging.level', 'INFO') _log_path = config_manager.get('logging.path', 'logs/app.log') _log_dir = Path(_log_path).parent logger = setup_logging(_log_level, _log_path) # App version — single source of truth for backup metadata, system-info, update check, etc. # Semver: MAJOR.MINOR.PATCH. Bump at each dev→main release. _SOULSYNC_BASE_VERSION = "2.4.0" def _build_version_string(): """Append short commit hash to version when available (e.g. 2.35+abc1234).""" sha = os.environ.get('SOULSYNC_COMMIT_SHA', '').strip() if not sha: try: import subprocess result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.') if result.returncode == 0: sha = result.stdout.strip() except Exception: pass if sha: return f"{_SOULSYNC_BASE_VERSION}+{sha[:7]}" return _SOULSYNC_BASE_VERSION SOULSYNC_VERSION = _build_version_string() # Dedicated source reuse logger — writes alongside app.log in the configured log directory import logging as _logging import logging.handlers as _logging_handlers source_reuse_logger = _logging.getLogger("source_reuse") source_reuse_logger.setLevel(_logging.DEBUG) if not source_reuse_logger.handlers: _sr_handler = _logging_handlers.RotatingFileHandler( _log_dir / "source_reuse.log", encoding="utf-8", maxBytes=5*1024*1024, backupCount=2 ) _sr_handler.setFormatter(_logging.Formatter("%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")) source_reuse_logger.addHandler(_sr_handler) source_reuse_logger.propagate = False # Dedicated post-processing logger (failures only) — writes alongside app.log in the configured log directory pp_logger = _logging.getLogger("post_processing") pp_logger.setLevel(_logging.DEBUG) if not pp_logger.handlers: _pp_handler = _logging_handlers.RotatingFileHandler( _log_dir / "post_processing.log", encoding="utf-8", maxBytes=5*1024*1024, backupCount=2 ) _pp_handler.setFormatter(_logging.Formatter("%(asctime)s - %(message)s", datefmt="%Y-%m-%d %H:%M:%S")) pp_logger.addHandler(_pp_handler) pp_logger.propagate = False from core.spotify_client import SpotifyClient, Playlist as SpotifyPlaylist, Track as SpotifyTrack, _is_globally_rate_limited as _spotify_rate_limited from core.plex_client import PlexClient from plexapi.myplex import MyPlexAccount, MyPlexPinLogin from core.jellyfin_client import JellyfinClient from core.navidrome_client import NavidromeClient from core.soulseek_client import SoulseekClient from core.download_orchestrator import DownloadOrchestrator from core.tidal_client import TidalClient # Added import for Tidal from core.matching_engine import MusicMatchingEngine from core.database_update_worker import DatabaseUpdateWorker from core.web_scan_manager import WebScanManager from core.metadata_cache import get_metadata_cache from core.imports.context import ( get_import_clean_album, get_import_clean_title, get_import_context_album, get_import_context_artist, get_import_original_search, get_import_track_info, normalize_import_context, ) from core.imports.album import ( build_album_import_context, build_album_import_match_payload, resolve_album_artist_context, ) from core.imports.album_naming import resolve_album_group as _resolve_album_group from core.imports.filename import extract_track_number_from_filename, parse_filename_metadata from core.imports.staging import ( get_import_suggestions_cache, get_primary_source, get_staging_path, read_staging_file_metadata, refresh_import_suggestions_cache, search_import_albums, search_import_tracks, start_import_suggestions_cache, ) from core.imports.paths import build_final_path_for_track as _build_final_path_for_track from core.imports.pipeline import build_import_pipeline_runtime as _build_import_pipeline_runtime from core.metadata.common import get_file_lock from core.metadata.enrichment import build_metadata_enrichment_runtime as _build_metadata_enrichment_runtime from core.metadata.source import ( mb_release_cache, mb_release_cache_lock, mb_release_detail_cache, mb_release_detail_cache_lock, normalize_album_cache_key, ) from core.runtime_state import ( activity_feed, activity_feed_lock, add_activity_item, download_batches, download_tasks, matched_context_lock, matched_downloads_context, mark_task_completed, processed_download_ids, set_activity_toast_emitter, tasks_lock, ) from core.metadata import enrichment as metadata_enrichment from database.music_database import get_database, MusicDatabase from services.sync_service import PlaylistSyncService # --- Docker Volume Mount Guard --- # Pre-v1.3 docker-compose files mounted soulsync_database:/app/database, which overlays # the Python package with stale volume contents. Detect this after import. if not hasattr(MusicDatabase, 'get_system_automation_by_action'): logger.error( "Stale database module detected!\n" "MusicDatabase is missing required methods. This usually means your docker-compose.yml has an outdated volume mount.\n" "Fix:\n" " OLD: soulsync_database:/app/database\n" " NEW: soulsync_database:/app/data\n" "Then run: docker compose down && docker compose up -d" ) from datetime import datetime, timezone import yt_dlp from beatport_unified_scraper import BeatportUnifiedScraper from core.musicbrainz_worker import MusicBrainzWorker from core.audiodb_worker import AudioDBWorker from core.discogs_worker import DiscogsWorker from core.deezer_worker import DeezerWorker from core.spotify_worker import SpotifyWorker from core.itunes_worker import iTunesWorker from core.lastfm_worker import LastFMWorker from core.genius_worker import GeniusWorker from core.tidal_worker import TidalWorker from core.qobuz_worker import QobuzWorker from core.hydrabase_worker import HydrabaseWorker from core.hydrabase_client import HydrabaseClient from core.automation_engine import AutomationEngine # --- Flask App Setup --- base_dir = os.path.abspath(os.path.dirname(__file__)) project_root = os.path.dirname(base_dir) # Go up one level to the project root DEV_STATIC_NO_CACHE = os.environ.get('SOULSYNC_WEB_DEV_NO_CACHE', '0').lower() in ('1', 'true', 'yes', 'on') # Check for environment variable first (Docker support), then fallback to calculated path env_config_path = os.environ.get('SOULSYNC_CONFIG_PATH') if env_config_path: config_path = env_config_path logger.info(f"Using config path from environment: {config_path}") else: config_path = os.path.join(project_root, 'config', 'config.json') if os.path.exists(config_path): # Check if we need to reload or if settings.py already handled it current_loaded_path = getattr(config_manager, 'config_path', None) target_path = Path(config_path).resolve() # Resolve current loaded path if it's a Path object if isinstance(current_loaded_path, Path): current_loaded_path = current_loaded_path.resolve() if current_loaded_path == target_path and config_manager.config_data: logger.info(f"Web server configuration already loaded from: {config_path}") else: logger.info(f"Found config file at: {config_path}") # Load configuration into the existing singleton instance if hasattr(config_manager, 'load_config'): config_manager.load_config(config_path) else: # Fallback for older settings.py in Docker volumes logger.warning("Legacy configuration detected: using fallback loading method") config_manager.config_path = Path(config_path) config_manager._load_config() logger.info("Web server configuration loaded successfully.") else: logger.warning(f"config.json not found at {config_path}. Using default settings.") # Correctly point to the 'webui' directory for templates and static files app = Flask( __name__, template_folder=os.path.join(base_dir, 'webui'), static_folder=os.path.join(base_dir, 'webui', 'static') ) app.config['TEMPLATES_AUTO_RELOAD'] = DEV_STATIC_NO_CACHE app.jinja_env.auto_reload = DEV_STATIC_NO_CACHE # Static assets (library.js / style.css / etc.) get aggressive browser # caching (1 year). Safe because every static URL is bust-tagged with # `?v=static_v` (computed once per process start — see below) so each # server restart effectively invalidates every cached asset for every # user. Within a single deploy, repeat page loads hit zero round-trips # on static files — was a 304 round-trip per asset under the old # max-age=0 setting. # # In dev, DEV_STATIC_NO_CACHE flips this back to 0 so iterating on JS # / CSS doesn't require a server restart between edits. app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 if DEV_STATIC_NO_CACHE else 31536000 # Cache-bust query string for static assets — appended to every # url_for('static', ...) URL via the context processor below. Computed # once per process start so each server restart invalidates the # browser's cached copy of every JS/CSS file. This is the surefire # fix for "user has stale JS even after Ctrl+Shift+R" — the URL # itself changes, so the browser cannot reuse a previously-cached # response no matter what its Cache-Control header said. import time as _cache_bust_time _STATIC_CACHE_BUST = str(int(_cache_bust_time.time())) @app.context_processor def _inject_static_cache_bust(): return {'static_v': _STATIC_CACHE_BUST} # --- Flask Session Setup (for multi-profile support) --- import secrets as _secrets def _init_flask_secret_key(): """Load or generate a persistent secret key for Flask sessions""" try: db = get_database() key = db.get_metadata('flask_secret_key') if not key: key = _secrets.token_hex(32) db.set_metadata('flask_secret_key', key) return key except Exception: return _secrets.token_hex(32) app.secret_key = _init_flask_secret_key() # --- WebSocket (Socket.IO) Setup --- from core.socketio_cors import ( resolve_cors_origins as _resolve_socketio_cors_origins, RejectionLogger as _SocketIORejectionLogger, log_startup_status as _log_socketio_startup_status, ) _socketio_cors_origins = _resolve_socketio_cors_origins(config_manager) socketio = SocketIO(app, async_mode='threading', cors_allowed_origins=_socketio_cors_origins) _log_socketio_startup_status(_socketio_cors_origins, logger) _socketio_rejection_logger = _SocketIORejectionLogger(logger) set_activity_toast_emitter(socketio.emit) # Plex PIN auth requests stored in memory for polling _plex_pin_requests = {} _plex_pin_requests_lock = threading.Lock() @app.before_request def _log_rejected_socketio_origin(): """Hook the WS upgrade path so users see a clear log line when their Origin is about to be rejected (engineio otherwise just silently 403s the upgrade). Dedup + threading lives in `core/socketio_cors`. Note: Flask's ``before_request`` runs on every HTTP request to every endpoint — there's no path-scoped equivalent for arbitrary URL prefixes. We early-return on non-/socket.io/ paths to keep the overhead to one string compare per request. """ if not request.path.startswith('/socket.io/'): return _socketio_rejection_logger.maybe_log( _socketio_cors_origins, request.headers.get('Origin'), request.headers.get('Host', ''), request.scheme, request.headers.get('X-Forwarded-Host', ''), request.headers.get('X-Forwarded-Proto', ''), ) # --- Profile Context (before_request hook) --- @app.before_request def _set_profile_context(): """Set g.profile_id from session for every request""" g.request_start_monotonic = time.perf_counter() # Skip for profile management, static, and root routes path = request.path if (path.startswith('/api/profiles') or path.startswith('/static/') or path == '/' or path.startswith('/api/v1/')): g.profile_id = session.get('profile_id', 1) return pid = session.get('profile_id', 1) # Validate session profile still exists (handles deleted profiles) if pid != 1 and 'profile_id' in session: try: database = get_database() profile = database.get_profile(pid) if not profile: session.pop('profile_id', None) from flask import jsonify as _jsonify return _jsonify({"error": "profile_required", "message": "Profile no longer exists"}), 401 except Exception: pass # DB error — don't block requests, use the session value g.profile_id = pid @app.after_request def _log_slow_request(response): """Log slow HTTP requests so we can identify UI stall sources.""" try: path = request.path if path.startswith('/socket.io/'): return response start = getattr(g, 'request_start_monotonic', None) if start is None: return response elapsed_ms = (time.perf_counter() - start) * 1000 slow_threshold_ms = 1000.0 if elapsed_ms >= slow_threshold_ms: logger.warning( "Slow request: %s %s -> %s in %.1fms", request.method, request.full_path.rstrip('?'), response.status_code, elapsed_ms, ) except Exception: pass return response @app.after_request def _add_discover_cache_headers(response): """Browser-cache discover GETs for 5 minutes. The discover surface (hero, similar artists, recent releases, release radar, deep cuts, etc.) returns semi-stable data that's expensive to compute and not user-action-driven within a session. A short browser cache eliminates redundant fetches when the user toggles between Discover sections or navigates back. Scope: only `/api/discover/` and `/api/discovery/` paths, only GET, only successful 2xx responses. Any endpoint that explicitly sets its own Cache-Control wins (we don't override). Uses `private` not `public` because discover data is user-specific (hero artists from your watchlist, similar artists from your taste, etc.). `private` keeps it browser-only — intermediate proxies (corporate caching proxies, Cloudflare with cache rules, Nginx proxy_cache) won't store one user's response and serve it to another. """ try: if request.method != 'GET': return response if not (request.path.startswith('/api/discover/') or request.path.startswith('/api/discovery/')): return response if not (200 <= response.status_code < 300): return response if response.headers.get('Cache-Control'): return response response.headers['Cache-Control'] = 'private, max-age=300' except Exception as exc: # Don't let a header-tagging bug turn a successful response into # a 500 — log and ship the response without the cache header. logger.warning(f"[discover-cache-headers] failed for {request.path}: {exc}") return response def get_current_profile_id() -> int: """Get the current profile ID from Flask g context or default to 1""" try: return g.profile_id except AttributeError: return 1 def admin_only(view_fn): """Restrict a Flask view to the admin profile (profile_id == 1). Settings-class endpoints expose / mutate service tokens, OAuth secrets, and API keys. Non-admin profiles must not see them. NOTE on the underlying auth model: `get_current_profile_id()` defaults to 1 (admin) when no session is present, which means single-admin / no-multi-profile installs have no actual gate here — any request from the local network is treated as admin. This decorator's job is to gate non-admin profiles in MULTI-profile setups, not to authenticate the network. The "trust local network" posture is the project's existing model; tightening it (real auth on every request) is out of scope for this decorator. """ @functools.wraps(view_fn) def wrapper(*args, **kwargs): if get_current_profile_id() != 1: return jsonify({ "success": False, "error": "Admin access required", }), 403 return view_fn(*args, **kwargs) return wrapper # ── Per-profile Spotify client cache ── _profile_spotify_clients = {} # profile_id -> SpotifyClient _profile_spotify_lock = threading.Lock() def get_spotify_client_for_profile(profile_id=None): """Get the Spotify client for the current profile. If the profile has custom Spotify credentials, returns a dedicated SpotifyClient instance (cached per profile). Otherwise returns the global spotify_client. """ if profile_id is None: profile_id = get_current_profile_id() # Admin (profile 1) always uses global client if profile_id == 1: return spotify_client # Check if this profile has custom Spotify credentials try: db = get_database() creds = db.get_profile_spotify(profile_id) if not creds or not creds.get('client_id'): return spotify_client # No custom creds — use global except Exception: return spotify_client # Check cache (don't hold lock during auth check — could block on network) with _profile_spotify_lock: cached = _profile_spotify_clients.get(profile_id) if cached and cached.sp is not None: return cached # Create a new SpotifyClient for this profile try: from spotipy.oauth2 import SpotifyOAuth import spotipy auth_manager = SpotifyOAuth( client_id=creds['client_id'], client_secret=creds['client_secret'], redirect_uri=creds.get('redirect_uri', 'http://127.0.0.1:8888/callback'), scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read", cache_path=f'config/.spotify_cache_profile_{profile_id}' ) # Create a bare SpotifyClient and immediately set the profile-specific # spotipy instance (overwrites the global-config one from __init__) profile_client = SpotifyClient() profile_client.sp = spotipy.Spotify(auth_manager=auth_manager, retries=0, requests_timeout=15) profile_client.user_id = None # Will be fetched lazily with _profile_spotify_lock: _profile_spotify_clients[profile_id] = profile_client logger.info(f"Created per-profile Spotify client for profile {profile_id}") return profile_client except Exception as e: logger.error(f"Failed to create per-profile Spotify client for profile {profile_id}: {e}") return spotify_client # Fall back to global # Valid page IDs for profile permission validation VALID_PAGE_IDS = {'dashboard', 'sync', 'search', 'downloads', 'discover', 'artists', 'automations', 'library', 'import', 'settings', 'help'} def check_download_permission(): """Check if current profile has download permission. Returns error response or None if allowed.""" pid = get_current_profile_id() if pid == 1: return None # Root admin always allowed try: profile = get_database().get_profile(pid) if profile and not profile.get('can_download', True): return jsonify({'success': False, 'error': 'Downloads are disabled for this profile.'}), 403 except Exception: pass # DB error — don't block return None # --- Docker Helper Functions --- def docker_resolve_path(path_str): """ Resolve absolute paths for Docker container access In Docker, Windows drive paths (E:/) need to be mapped to WSL mount points (/mnt/e/) """ if os.path.exists('/.dockerenv') and len(path_str) >= 3 and path_str[1] == ':' and path_str[0].isalpha(): # Convert Windows path (E:/path) to WSL mount path (/mnt/e/path) drive_letter = path_str[0].lower() rest_of_path = path_str[2:].replace('\\', '/') # Remove E: and convert backslashes return f"/host/mnt/{drive_letter}{rest_of_path}" return path_str def extract_filename(full_path): """ Extract filename by working backwards from the end until we hit a separator. This is cross-platform compatible and handles both Windows and Unix path separators. Special handling for YouTube/Tidal: If the filename contains '||' (encoded format), treat it as a filename, not a path, to avoid splitting on '/' in titles. """ if not full_path: return "" # YouTube filenames are encoded as "video_id||title" and may contain '/' in the title # Don't split these on path separators if '||' in full_path: return full_path last_slash = max(full_path.rfind('/'), full_path.rfind('\\')) if last_slash != -1: return full_path[last_slash + 1:] else: return full_path def _make_context_key(username, filename): """Build a unique context key from username and full Soulseek path. Uses the full remote path (not just filename) to prevent collisions when different tracks from the same user share a filename (e.g., two albums both containing '01 - Intro.flac'). """ normalized = filename.replace('\\', '/').lstrip('/') if filename else '' return f"{username}::{normalized}" IS_SHUTTING_DOWN = False # --- Initialize Core Application Components --- # Each client is initialized independently so one failure doesn't take down everything. # Previously, a single exception set ALL clients to None, breaking the entire app. logger.info("Initializing SoulSync services for Web UI...") spotify_client = plex_client = jellyfin_client = navidrome_client = soulsync_library_client = soulseek_client = tidal_client = matching_engine = sync_service = web_scan_manager = None try: spotify_client = SpotifyClient() logger.info(" Spotify client initialized") except Exception as e: logger.error(f" Spotify client failed to initialize: {e}") try: plex_client = PlexClient() logger.info(" Plex client initialized") except Exception as e: logger.error(f" Plex client failed to initialize: {e}") try: jellyfin_client = JellyfinClient() logger.info(" Jellyfin client initialized") except Exception as e: logger.error(f" Jellyfin client failed to initialize: {e}") try: navidrome_client = NavidromeClient() logger.info(" Navidrome client initialized") except Exception as e: logger.error(f" Navidrome client failed to initialize: {e}") try: from core.soulsync_client import SoulSyncClient soulsync_library_client = SoulSyncClient() logger.info(" SoulSync library client initialized") except Exception as e: logger.error(f" SoulSync library client failed to initialize: {e}") try: soulseek_client = DownloadOrchestrator() logger.info(" Download orchestrator initialized") except Exception as e: logger.error(f" Download orchestrator failed to initialize: {e}") try: tidal_client = TidalClient() logger.info(" Tidal client initialized") except Exception as e: logger.error(f" Tidal client failed to initialize: {e}") try: matching_engine = MusicMatchingEngine() logger.info(" Matching engine initialized") except Exception as e: logger.error(f" Matching engine failed to initialize: {e}") try: sync_service = PlaylistSyncService(spotify_client, plex_client, soulseek_client, jellyfin_client, navidrome_client) logger.info(" Playlist sync service initialized") except Exception as e: logger.error(f" Playlist sync service failed to initialize: {e}") # Inject shutdown check callback into YouTube and Tidal clients (avoids circular imports) if soulseek_client: if hasattr(soulseek_client, 'youtube'): soulseek_client.youtube.set_shutdown_check(lambda: IS_SHUTTING_DOWN) logger.info(" Configured YouTube client shutdown callback") if hasattr(soulseek_client, 'tidal'): soulseek_client.tidal.set_shutdown_check(lambda: IS_SHUTTING_DOWN) logger.info(" Configured Tidal download client shutdown callback") if hasattr(soulseek_client, 'qobuz'): soulseek_client.qobuz.set_shutdown_check(lambda: IS_SHUTTING_DOWN) logger.info(" Configured Qobuz client shutdown callback") if hasattr(soulseek_client, 'hifi'): soulseek_client.hifi.set_shutdown_check(lambda: IS_SHUTTING_DOWN) logger.info(" Configured HiFi client shutdown callback") # Initialize web scan manager for automatic post-download scanning try: media_clients = { 'plex_client': plex_client, 'jellyfin_client': jellyfin_client, 'navidrome_client': navidrome_client, 'soulsync_library_client': soulsync_library_client, } web_scan_manager = WebScanManager(media_clients, delay_seconds=60) logger.info(" Web scan manager initialized") except Exception as e: logger.error(f" Web scan manager failed to initialize: {e}") logger.info("Core service initialization complete.") # --- Shared Runtime State --- # These globals are used by routes, background workers, and shutdown helpers. # A prior refactor accidentally dropped this initializer block, so several # modules and handlers were still referencing names that never got created. # Global Streaming State Management stream_state = { "status": "stopped", # States: stopped, loading, queued, ready, error "progress": 0, "track_info": None, "file_path": None, # Path to the audio file in the Stream folder "error_message": None, } stream_lock = threading.Lock() # Prevent race conditions stream_background_task = None stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time # Global OAuth State Management # Store PKCE values for Tidal OAuth flow tidal_oauth_state = { "code_verifier": None, "code_challenge": None, } tidal_oauth_lock = threading.Lock() # Sync Page Globals sync_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="SyncWorker") active_sync_workers = {} # Key: playlist_id, Value: Future object sync_states = {} # Key: playlist_id, Value: dict with progress info sync_lock = threading.Lock() # Database Update / Tool Progress State db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate") db_update_worker = None db_update_state = { "status": "idle", # idle, running, finished, error "phase": "Idle", "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "", "removed_artists": 0, "removed_albums": 0, "removed_tracks": 0, } _db_update_automation_id = None # Set when automation triggers DB update, used by callbacks db_update_lock = threading.Lock() # Quality Scanner state quality_scanner_state = { "status": "idle", # idle, running, finished, error "phase": "Ready to scan", "progress": 0, "processed": 0, "total": 0, "quality_met": 0, "low_quality": 0, "matched": 0, "error_message": "", "results": [], # List of low quality tracks with match status } quality_scanner_lock = threading.Lock() quality_scanner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="QualityScanner") # Duplicate Cleaner state duplicate_cleaner_state = { "status": "idle", # idle, running, finished, error "phase": "Ready to scan", "progress": 0, "files_scanned": 0, "total_files": 0, "duplicates_found": 0, "deleted": 0, "space_freed": 0, # in bytes "error_message": "", } duplicate_cleaner_lock = threading.Lock() duplicate_cleaner_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DuplicateCleaner") # Retag Tool Globals retag_state = { "status": "idle", "phase": "Ready", "progress": 0, "current_track": "", "total_tracks": 0, "processed": 0, "error_message": "", } retag_lock = threading.Lock() retag_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="RetagWorker") # Download Missing Tracks Modal State Management # Thread-safe state tracking for modal download functionality. # Shared task/batch state now lives in core.runtime_state. missing_download_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="MissingTrackWorker") # Automatic Wishlist / Watchlist Processing Flags # Processing state flags (guards/recovery - timers are now managed by AutomationEngine) wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 wishlist_timer_lock = threading.Lock() watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 watchlist_timer_lock = threading.Lock() # Beatport Data Cache # Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com beatport_data_cache = { 'homepage': { 'hero_tracks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'top_10_lists': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'top_10_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'new_releases': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'hype_picks': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'featured_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours 'dj_charts': {'data': None, 'timestamp': 0, 'ttl': 86400}, # 24 hours }, 'genre': { # Future expansion for genre-specific caching # 'house': {'top_10': {...}, 'releases': {...}}, # 'techno': {'top_10': {...}, 'releases': {...}} }, 'cache_lock': threading.Lock(), } # Shared Soulseek transfer cache # Keeps download status polling from hammering the API when multiple modals are open. transfer_data_cache = { 'data': {}, 'last_update': 0, 'update_lock': threading.Lock(), 'cache_duration': 0.75, } # --- Restored Web UI Helper State --- session_completed_downloads = 0 session_stats_lock = threading.Lock() # `batch_locks` lives in core/runtime_state.py (re-exported here so the existing # call sites resolve without modification). All download globals share that home. from core.runtime_state import batch_locks _orphaned_download_keys = set() _enrichment_activity_log = {} _idle_since = {} _IDLE_GRACE_SECONDS = 5 _status_cache = { 'spotify': {'connected': False, 'response_time': 0, 'source': 'itunes'}, 'media_server': {'connected': False, 'response_time': 0, 'type': None}, 'soulseek': {'connected': False, 'response_time': 0}, } _status_cache_timestamps = { 'spotify': 0, 'media_server': 0, 'soulseek': 0, } STATUS_CACHE_TTL = 120 dev_mode_enabled = False _hydrabase_ws = None _hydrabase_lock = threading.Lock() # --- Automation Engine --- try: automation_engine = AutomationEngine(get_database()) logger.info("Automation engine initialized.") except Exception as e: logger.error(f"Automation engine failed to initialize: {e}") automation_engine = None # --- Automation Progress Tracking --- _scan_library_automation_id = None def _register_automation_handlers(): """Register real SoulSync action handlers with the automation engine.""" if not automation_engine: return def _auto_process_wishlist(config): try: _process_wishlist_automatically(automation_id=config.get('_automation_id')) return {'status': 'completed'} except Exception as e: return {'status': 'error', 'error': str(e)} # Note: wishlist processing is async (batch submitted to executor), stats come via batch completion def _auto_scan_watchlist(config): try: pre_state_id = id(watchlist_scan_state) _process_watchlist_scan_automatically( automation_id=config.get('_automation_id'), profile_id=config.get('_profile_id') ) # Only report stats if a fresh scan actually ran (state dict was reassigned) if id(watchlist_scan_state) != pre_state_id: summary = watchlist_scan_state.get('summary', {}) return { 'status': 'completed', 'artists_scanned': summary.get('total_artists', 0), 'successful_scans': summary.get('successful_scans', 0), 'new_tracks_found': summary.get('new_tracks_found', 0), 'tracks_added_to_wishlist': summary.get('tracks_added_to_wishlist', 0), } return {'status': 'completed'} except Exception as e: return {'status': 'error', 'error': str(e)} def _auto_scan_library(config): global _scan_library_automation_id automation_id = config.get('_automation_id') if not web_scan_manager: return {'status': 'error', 'reason': 'Scan manager not available'} # If another automation is already tracking the scan, just forward the request if _scan_library_automation_id is not None: web_scan_manager.request_scan('Automation trigger (additional batch)') return {'status': 'skipped', 'reason': 'Scan already being tracked'} _scan_library_automation_id = automation_id try: result = web_scan_manager.request_scan('Automation trigger') scan_status_val = result.get('status', 'unknown') if scan_status_val == 'queued': _update_automation_progress(automation_id, log_line='Scan already in progress — waiting for completion', log_type='info') else: delay = result.get('delay_seconds', 60) _update_automation_progress(automation_id, log_line=f'Scan scheduled (debounce: {delay}s)', log_type='info') # Unified polling loop — handles debounce → scanning → idle transitions poll_start = time.time() scan_started = (scan_status_val == 'queued') # Already scanning if queued while time.time() - poll_start < 1800: # Max 30 min overall status = web_scan_manager.get_scan_status() st = status.get('status') if st == 'idle': break # Scan completed (or finished before we started polling) elif st == 'scheduled': elapsed = int(time.time() - poll_start) _update_automation_progress(automation_id, phase=f'Waiting for scan to start... ({elapsed}s)', progress=min(int(elapsed / 60 * 10), 14)) time.sleep(2) elif st == 'scanning': if not scan_started: scan_started = True _update_automation_progress(automation_id, progress=15, log_line='Scan triggered on media server', log_type='success') elapsed = status.get('elapsed_seconds', 0) max_time = status.get('max_time_seconds', 300) pct = min(15 + int(elapsed / max_time * 80), 95) mins, secs = divmod(elapsed, 60) _update_automation_progress(automation_id, phase=f'Library scan in progress... ({mins}m {secs}s)', progress=pct) time.sleep(5) else: time.sleep(2) # Unknown status, avoid tight loop else: # 30-min timeout reached _update_automation_progress(automation_id, status='error', phase='Timed out', log_line='Library scan timed out after 30 minutes', log_type='error') return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} elapsed = round(time.time() - poll_start, 1) _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line='Library scan completed', log_type='success') return {'status': 'completed', '_manages_own_progress': True, 'scan_duration_seconds': elapsed} except Exception as e: _update_automation_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error') return {'status': 'error', 'error': str(e), '_manages_own_progress': True} finally: _scan_library_automation_id = None # Self-guards only — prevent duplicate runs of the same operation, # but allow wishlist processing and watchlist scanning to run concurrently. # Downloads use bandwidth (Soulseek/Tidal/etc), scans use API calls — different resources. # The per-call rate limiter handles any API contention during post-processing. automation_engine.register_action_handler('process_wishlist', _auto_process_wishlist, guard_fn=lambda: is_wishlist_actually_processing()) automation_engine.register_action_handler('scan_watchlist', _auto_scan_watchlist, guard_fn=lambda: is_watchlist_actually_scanning()) automation_engine.register_action_handler('scan_library', _auto_scan_library, lambda: _scan_library_automation_id is not None) def _auto_refresh_mirrored(config): """Refresh mirrored playlist(s) from source.""" db = get_database() playlist_id = config.get('playlist_id') refresh_all = config.get('all', False) auto_id = config.get('_automation_id') if refresh_all: playlists = db.get_mirrored_playlists() elif playlist_id: p = db.get_mirrored_playlist(int(playlist_id)) playlists = [p] if p else [] else: return {'status': 'error', 'reason': 'No playlist specified'} # Filter out sources that can't be refreshed (no external API) playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] refreshed = 0 errors = [] for idx, pl in enumerate(playlists): try: source = pl.get('source', '') source_id = pl.get('source_playlist_id', '') _update_automation_progress(auto_id, progress=(idx / max(1, len(playlists))) * 100, phase=f'Refreshing: "{pl.get("name", "")}"', current_item=pl.get('name', '')) tracks = None if source == 'spotify': # Try authenticated API first, fall back to public embed scraper if spotify_client and spotify_client.is_spotify_authenticated(): playlist_obj = spotify_client.get_playlist_by_id(source_id) if playlist_obj and playlist_obj.tracks: tracks = [] for t in playlist_obj.tracks: artist_name = t.artists[0] if t.artists else '' track_dict = { 'track_name': t.name or '', 'artist_name': str(artist_name), 'album_name': t.album or '', 'duration_ms': t.duration_ms or 0, 'source_track_id': t.id or '', } # Spotify data IS official — auto-mark as discovered if t.id: _album_obj = {'name': t.album or ''} if getattr(t, 'image_url', None): _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] track_dict['extra_data'] = json.dumps({ 'discovered': True, 'provider': 'spotify', 'confidence': 1.0, 'matched_data': { 'id': t.id, 'name': t.name or '', 'artists': [{'name': str(a)} for a in (t.artists or [])], 'album': _album_obj, 'duration_ms': t.duration_ms or 0, 'image_url': getattr(t, 'image_url', None), } }) tracks.append(track_dict) # Fallback: public embed scraper (no auth needed) if tracks is None: try: from core.spotify_public_scraper import scrape_spotify_embed embed_data = scrape_spotify_embed('playlist', source_id) if embed_data and not embed_data.get('error') and embed_data.get('tracks'): embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' tracks = [] for t in embed_data['tracks']: artist_names = [a['name'] for a in t.get('artists', [])] artist_name = artist_names[0] if artist_names else '' track_dict = { 'track_name': t.get('name', ''), 'artist_name': artist_name, 'album_name': embed_album, 'duration_ms': t.get('duration_ms', 0), 'source_track_id': t.get('id', ''), } # Store Spotify track ID hint but don't mark discovered — # Discover step needs to run for proper album art if t.get('id'): track_dict['extra_data'] = json.dumps({ 'discovered': False, 'spotify_hint': { 'id': t['id'], 'name': t.get('name', ''), 'artists': t.get('artists', []), } }) tracks.append(track_dict) except Exception as e: logger.warning(f"Spotify public scraper fallback failed for {source_id}: {e}") elif source == 'spotify_public': # source_playlist_id is an MD5 hash; extract actual Spotify ID from stored description (URL) try: from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed spotify_url = pl.get('description', '') parsed = parse_spotify_url(spotify_url) if spotify_url else None # If Spotify is authenticated, use the full API (auto-discovers with album art) if parsed and parsed.get('type') == 'playlist' and spotify_client and spotify_client.is_spotify_authenticated(): playlist_obj = spotify_client.get_playlist_by_id(parsed['id']) if playlist_obj and playlist_obj.tracks: tracks = [] for t in playlist_obj.tracks: artist_name = t.artists[0] if t.artists else '' track_dict = { 'track_name': t.name or '', 'artist_name': str(artist_name), 'album_name': t.album or '', 'duration_ms': t.duration_ms or 0, 'source_track_id': t.id or '', } if t.id: _album_obj = {'name': t.album or ''} if getattr(t, 'image_url', None): _album_obj['images'] = [{'url': t.image_url, 'height': 600, 'width': 600}] track_dict['extra_data'] = json.dumps({ 'discovered': True, 'provider': 'spotify', 'confidence': 1.0, 'matched_data': { 'id': t.id, 'name': t.name or '', 'artists': [{'name': str(a)} for a in (t.artists or [])], 'album': _album_obj, 'duration_ms': t.duration_ms or 0, 'image_url': getattr(t, 'image_url', None), } }) tracks.append(track_dict) # Fallback: public embed scraper (no auth or album-type URL) if tracks is None and parsed: embed_data = scrape_spotify_embed(parsed['type'], parsed['id']) if embed_data and not embed_data.get('error') and embed_data.get('tracks'): embed_album = embed_data.get('name', '') if embed_data.get('type') == 'album' else '' tracks = [] for t in embed_data['tracks']: artist_names = [a['name'] for a in t.get('artists', [])] artist_name = artist_names[0] if artist_names else '' tracks.append({ 'track_name': t.get('name', ''), 'artist_name': artist_name, 'album_name': embed_album, 'duration_ms': t.get('duration_ms', 0), 'source_track_id': t.get('id', ''), }) # No extra_data — let preservation code keep existing discovery data except Exception as e: logger.warning(f"Spotify public playlist refresh failed for {source_id}: {e}") elif source == 'deezer': try: deezer = _get_deezer_client() playlist_data = deezer.get_playlist(source_id) if playlist_data and playlist_data.get('tracks'): tracks = [] for t in playlist_data['tracks']: artist_name = t['artists'][0] if t.get('artists') else '' tracks.append({ 'track_name': t.get('name', ''), 'artist_name': str(artist_name), 'album_name': t.get('album', ''), 'duration_ms': t.get('duration_ms', 0), 'source_track_id': str(t.get('id', '')), }) except Exception as e: logger.warning(f"Deezer playlist refresh failed for {source_id}: {e}") elif source == 'tidal': if not tidal_client or not tidal_client.is_authenticated(): logger.warning(f"Tidal not authenticated — skipping refresh for '{pl.get('name', '')}'") _update_automation_progress(auto_id, log_line=f'Skipped "{pl.get("name", "")}" — Tidal not authenticated', log_type='skip') continue full_playlist = tidal_client.get_playlist(source_id) if full_playlist and full_playlist.tracks: tracks = [] for t in full_playlist.tracks: artist_name = t.artists[0] if t.artists else '' tracks.append({ 'track_name': t.name or '', 'artist_name': str(artist_name), 'album_name': t.album or '', 'duration_ms': t.duration_ms or 0, 'source_track_id': t.id or '', }) elif source == 'youtube': # source_playlist_id is now a deterministic hash; use stored description (original URL) for refresh yt_url = pl.get('description', '') or f"https://www.youtube.com/playlist?list={source_id}" playlist_data = parse_youtube_playlist(yt_url) if playlist_data and playlist_data.get('tracks'): tracks = [] for t in playlist_data['tracks']: artist_name = t['artists'][0] if t.get('artists') else '' tracks.append({ 'track_name': t.get('name', ''), 'artist_name': str(artist_name), 'album_name': '', 'duration_ms': t.get('duration_ms', 0), 'source_track_id': t.get('id', ''), }) if tracks is not None: # Compare old vs new track IDs to detect changes old_tracks = db.get_mirrored_playlist_tracks(pl['id']) if pl.get('id') else [] old_ids = {t.get('source_track_id') for t in old_tracks if t.get('source_track_id')} new_ids = {t.get('source_track_id') for t in tracks if t.get('source_track_id')} # Preserve existing discovery extra_data for tracks that still exist old_extra_map = db.get_mirrored_tracks_extra_data_map(pl['id']) if pl.get('id') else {} for t in tracks: sid = t.get('source_track_id', '') if sid and sid in old_extra_map and 'extra_data' not in t: t['extra_data'] = old_extra_map[sid] db.mirror_playlist( source=source, source_playlist_id=source_id, name=pl['name'], tracks=tracks, profile_id=pl.get('profile_id', 1), owner=pl.get('owner'), image_url=pl.get('image_url'), ) refreshed += 1 # Emit playlist_changed if tracks actually changed if old_ids != new_ids: added_count = len(new_ids - old_ids) removed_count = len(old_ids - new_ids) logger.info(f"[AUTOMATION] Playlist changed: '{pl.get('name', '')}' — {added_count} added, {removed_count} removed (old={len(old_ids)}, new={len(new_ids)})") _update_automation_progress(auto_id, log_line=f'"{pl.get("name", "")}" — {added_count} added, {removed_count} removed', log_type='success') try: if automation_engine: automation_engine.emit('playlist_changed', { 'playlist_name': pl.get('name', ''), 'playlist_id': str(pl.get('id', '')), 'old_count': str(len(old_ids)), 'new_count': str(len(new_ids)), 'added': str(added_count), 'removed': str(removed_count), }) except Exception: pass else: logger.warning(f"[AUTOMATION] No changes: '{pl.get('name', '')}' (tracks={len(old_ids)})") _update_automation_progress(auto_id, log_line=f'No changes: "{pl.get("name", "")}"', log_type='skip') except Exception as e: errors.append(f"{pl.get('name', '?')}: {str(e)}") _update_automation_progress(auto_id, log_line=f'Error: {pl.get("name", "?")} — {str(e)}', log_type='error') return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors))} def _auto_sync_playlist(config): """Sync a mirrored playlist to media server. Uses discovered metadata when available, skips undiscovered tracks. When triggered on a schedule, skips if nothing changed since last sync.""" auto_id = config.get('_automation_id') playlist_id = config.get('playlist_id') if not playlist_id: return {'status': 'error', 'reason': 'No playlist specified'} db = get_database() pl = db.get_mirrored_playlist(int(playlist_id)) if not pl: return {'status': 'error', 'reason': 'Playlist not found'} tracks = db.get_mirrored_playlist_tracks(int(playlist_id)) if not tracks: return {'status': 'error', 'reason': 'No tracks in playlist'} # Count currently discovered tracks for smart-skip check current_discovered = 0 for t in tracks: extra = {} if t.get('extra_data'): try: extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] except (json.JSONDecodeError, TypeError): pass if extra.get('discovered') and extra.get('matched_data'): current_discovered += 1 # Convert mirrored tracks to format expected by _run_sync_task # Use discovered metadata when available, skip undiscovered tracks tracks_json = [] skipped_count = 0 for t in tracks: # Parse extra_data for discovery info extra = {} if t.get('extra_data'): try: extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] except (json.JSONDecodeError, TypeError): pass if extra.get('discovered') and extra.get('matched_data'): # Use official discovered metadata md = extra['matched_data'] album_raw = md.get('album', '') album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} _track_entry = { 'name': md.get('name', ''), 'artists': md.get('artists', [{'name': t.get('artist_name', '')}]), 'album': album_obj, 'duration_ms': md.get('duration_ms', 0), 'id': md.get('id', ''), } if md.get('track_number'): _track_entry['track_number'] = md['track_number'] if md.get('disc_number'): _track_entry['disc_number'] = md['disc_number'] tracks_json.append(_track_entry) else: # NOT discovered — try to include using available metadata so the # track can still be searched on Soulseek and added to wishlist. # Without this, failed discovery blocks the entire download pipeline. # # Priority: spotify_hint (has real Spotify ID from embed scraper) # > raw playlist fields (only if source_track_id is valid) hint = extra.get('spotify_hint', {}) # Build album object with cover art from the mirrored playlist track track_image = (t.get('image_url') or '').strip() album_obj = { 'name': (t.get('album_name') or '').strip(), 'images': [{'url': track_image, 'height': 300, 'width': 300}] if track_image else [], } if hint.get('id') and hint.get('name'): # spotify_hint has proper Spotify track ID + metadata from embed scraper hint_artists = hint.get('artists', []) if hint_artists and isinstance(hint_artists[0], str): hint_artists = [{'name': a} for a in hint_artists] elif hint_artists and isinstance(hint_artists[0], dict): pass # Already in correct format else: hint_artists = [{'name': t.get('artist_name', '')}] tracks_json.append({ 'name': hint['name'], 'artists': hint_artists, 'album': album_obj, 'duration_ms': t.get('duration_ms', 0), 'id': hint['id'], }) elif t.get('source_track_id') and (t.get('track_name') or '').strip(): # Has a valid source ID and track name — usable for wishlist tracks_json.append({ 'name': t['track_name'].strip(), 'artists': [{'name': (t.get('artist_name') or '').strip() or 'Unknown Artist'}], 'album': album_obj, 'duration_ms': t.get('duration_ms', 0), 'id': t['source_track_id'], }) else: skipped_count += 1 # No usable ID or name — truly can't process if not tracks_json: _update_automation_progress(auto_id, log_line=f'No discovered tracks — {skipped_count} need discovery first', log_type='skip') return { 'status': 'skipped', 'reason': f'No discovered tracks to sync ({skipped_count} tracks need discovery first)', 'skipped_tracks': str(skipped_count), } # Preflight: hash the track list and compare against last sync # Skip if the exact same set of tracks was already synced and all matched import hashlib track_ids_str = ','.join(sorted(t.get('id', '') for t in tracks_json)) tracks_hash = hashlib.md5(track_ids_str.encode()).hexdigest() sync_id_key = f"auto_mirror_{playlist_id}" try: sync_statuses = _load_sync_status_file() last_status = sync_statuses.get(sync_id_key, {}) last_hash = last_status.get('tracks_hash', '') last_matched = last_status.get('matched_tracks', -1) if (last_hash == tracks_hash and last_matched >= len(tracks_json)): # Exact same tracks, all matched last time — nothing to do _update_automation_progress(auto_id, log_line=f'All {len(tracks_json)} tracks unchanged since last sync — skipping', log_type='skip') return { 'status': 'skipped', 'reason': f'All {len(tracks_json)} tracks unchanged since last sync', } except Exception: pass # If we can't read last status, just run the sync _update_automation_progress(auto_id, progress=50, phase=f'Syncing "{pl["name"]}"', log_line=f'{len(tracks_json)} discovered, {skipped_count} skipped', log_type='info') sync_id = f"auto_mirror_{playlist_id}" _update_automation_progress(auto_id, progress=90, log_line=f'Starting sync: {len(tracks_json)} tracks', log_type='success') threading.Thread( target=_run_sync_task, args=(sync_id, pl['name'], tracks_json, auto_id, 1, pl.get('image_url', '')), daemon=True, name=f'auto-sync-{playlist_id}' ).start() return { 'status': 'started', 'playlist_name': pl['name'], 'discovered_tracks': str(len(tracks_json)), 'skipped_tracks': str(skipped_count), '_manages_own_progress': True, } def _auto_discover_playlist(config): """Discover official Spotify/iTunes metadata for mirrored playlist tracks.""" db = get_database() playlist_id = config.get('playlist_id') discover_all = config.get('all', False) if discover_all: playlists = db.get_mirrored_playlists() elif playlist_id: p = db.get_mirrored_playlist(int(playlist_id)) playlists = [p] if p else [] else: return {'status': 'error', 'reason': 'No playlist specified'} if not playlists: return {'status': 'error', 'reason': 'No playlists found'} threading.Thread( target=_run_playlist_discovery_worker, args=(playlists, config.get('_automation_id')), daemon=True, name='auto-discover-playlist' ).start() names = ', '.join(p['name'] for p in playlists[:3]) return {'status': 'started', 'playlist_count': str(len(playlists)), 'playlists': names, '_manages_own_progress': True} # --- Playlist Pipeline: single automation for full lifecycle --- _pipeline_running = False def _pipeline_guard(): return _pipeline_running def _auto_playlist_pipeline(config): """Full playlist lifecycle: refresh → discover → sync → wishlist. Runs all 4 phases sequentially in one automation, reporting progress throughout.""" nonlocal _pipeline_running _pipeline_running = True automation_id = config.get('_automation_id') pipeline_start = time.time() try: db = get_database() playlist_id = config.get('playlist_id') process_all = config.get('all', False) skip_wishlist = config.get('skip_wishlist', False) # Resolve playlists if process_all: playlists = db.get_mirrored_playlists() elif playlist_id: p = db.get_mirrored_playlist(int(playlist_id)) playlists = [p] if p else [] else: _pipeline_running = False return {'status': 'error', 'error': 'No playlist specified'} playlists = [pl for pl in playlists if pl.get('source', '') not in ('file', 'beatport')] if not playlists: _pipeline_running = False return {'status': 'error', 'error': 'No refreshable playlists found'} pl_names = ', '.join(p.get('name', '?') for p in playlists[:3]) if len(playlists) > 3: pl_names += f' (+{len(playlists) - 3} more)' _update_automation_progress(automation_id, progress=2, phase=f'Pipeline: {len(playlists)} playlist(s)', log_line=f'Starting pipeline for: {pl_names}', log_type='info') # ── PHASE 1: REFRESH ────────────────────────────────────────── _update_automation_progress(automation_id, progress=3, phase='Phase 1/4: Refreshing playlists...', log_line='Phase 1: Refresh', log_type='info') refresh_config = dict(config) refresh_config['_automation_id'] = None # Don't let sub-handler hijack pipeline progress refresh_result = _auto_refresh_mirrored(refresh_config) refreshed = int(refresh_result.get('refreshed', 0)) refresh_errors = int(refresh_result.get('errors', 0)) _update_automation_progress(automation_id, progress=25, phase='Phase 1/4: Refresh complete', log_line=f'Phase 1 done: {refreshed} refreshed, {refresh_errors} errors', log_type='success' if refresh_errors == 0 else 'warning') # ── PHASE 2: DISCOVER ───────────────────────────────────────── _update_automation_progress(automation_id, progress=26, phase='Phase 2/4: Discovering metadata...', log_line='Phase 2: Discover', log_type='info') # Reload playlists (refresh may have updated them) if process_all: disc_playlists = db.get_mirrored_playlists() else: disc_playlists = [db.get_mirrored_playlist(int(playlist_id))] disc_playlists = [p for p in disc_playlists if p] # Run discovery in a thread and wait for it disc_done = threading.Event() disc_result = {'discovered': 0, 'failed': 0, 'skipped': 0, 'total': 0} def _disc_wrapper(pls): try: # The worker updates automation_progress internally, # but we pass None so it doesn't conflict with our pipeline progress _run_playlist_discovery_worker(pls, automation_id=None) except Exception as e: logger.error(f"[Pipeline] Discovery error: {e}") finally: disc_done.set() threading.Thread(target=_disc_wrapper, args=(disc_playlists,), daemon=True, name='pipeline-discover').start() # Poll for completion with progress updates poll_start = time.time() while not disc_done.wait(timeout=3): elapsed = int(time.time() - poll_start) _update_automation_progress(automation_id, progress=min(26 + elapsed // 4, 54), phase=f'Phase 2/4: Discovering... ({elapsed}s)') if elapsed > 3600: # 1hr safety timeout _update_automation_progress(automation_id, log_line='Discovery timed out after 1 hour', log_type='warning') break _update_automation_progress(automation_id, progress=55, phase='Phase 2/4: Discovery complete', log_line='Phase 2 done: discovery complete', log_type='success') # ── PHASE 3: SYNC ───────────────────────────────────────────── _update_automation_progress(automation_id, progress=56, phase='Phase 3/4: Syncing to server...', log_line='Phase 3: Sync', log_type='info') total_synced = 0 total_skipped = 0 sync_errors = 0 for pl_idx, pl in enumerate(playlists): pl_id = pl.get('id') if not pl_id: continue # Build sync config for this playlist (reuse existing sync handler) sync_config = { 'playlist_id': str(pl_id), '_automation_id': None, # Don't let sync handler hijack our progress } sync_result = _auto_sync_playlist(sync_config) sync_status = sync_result.get('status', '') if sync_status == 'started': # Sync launched a background thread — wait for it sync_id = f"auto_mirror_{pl_id}" sync_poll_start = time.time() while time.time() - sync_poll_start < 600: # 10 min per playlist max if sync_id in sync_states and sync_states[sync_id].get('status') in ('finished', 'complete', 'error', 'failed'): break time.sleep(2) elapsed = int(time.time() - sync_poll_start) sub_progress = 56 + ((pl_idx + 1) / max(1, len(playlists))) * 29 _update_automation_progress(automation_id, progress=min(int(sub_progress), 84), phase=f'Phase 3/4: Syncing "{pl.get("name", "")}" ({elapsed}s)') # Check result ss = sync_states.get(sync_id, {}) ss_result = ss.get('result', ss.get('progress', {})) matched = ss_result.get('matched_tracks', 0) if isinstance(ss_result, dict) else 0 total_synced += int(matched) if matched else 0 _update_automation_progress(automation_id, log_line=f'Synced "{pl.get("name", "")}": {matched} tracks matched', log_type='success') elif sync_status == 'skipped': total_skipped += 1 reason = sync_result.get('reason', 'unchanged') _update_automation_progress(automation_id, log_line=f'Skipped "{pl.get("name", "")}": {reason}', log_type='skip') elif sync_status == 'error': sync_errors += 1 _update_automation_progress(automation_id, log_line=f'Sync error "{pl.get("name", "")}": {sync_result.get("reason", "unknown")}', log_type='error') _update_automation_progress(automation_id, progress=85, phase='Phase 3/4: Sync complete', log_line=f'Phase 3 done: {total_synced} matched, {total_skipped} skipped, {sync_errors} errors', log_type='success' if sync_errors == 0 else 'warning') # ── PHASE 4: WISHLIST ───────────────────────────────────────── wishlist_queued = 0 if not skip_wishlist: _update_automation_progress(automation_id, progress=86, phase='Phase 4/4: Processing wishlist...', log_line='Phase 4: Wishlist', log_type='info') try: if not is_wishlist_actually_processing(): _process_wishlist_automatically(automation_id=None) _update_automation_progress(automation_id, log_line='Wishlist processing triggered', log_type='success') wishlist_queued = 1 else: _update_automation_progress(automation_id, log_line='Wishlist already running — skipped', log_type='skip') except Exception as e: _update_automation_progress(automation_id, log_line=f'Wishlist error: {e}', log_type='warning') else: _update_automation_progress(automation_id, progress=86, log_line='Phase 4: Wishlist skipped (disabled)', log_type='skip') # ── COMPLETE ────────────────────────────────────────────────── duration = int(time.time() - pipeline_start) _update_automation_progress(automation_id, status='finished', progress=100, phase='Pipeline complete', log_line=f'Pipeline finished in {duration // 60}m {duration % 60}s', log_type='success') _pipeline_running = False return { 'status': 'completed', '_manages_own_progress': True, 'playlists_refreshed': str(refreshed), 'tracks_discovered': 'completed', 'tracks_synced': str(total_synced), 'sync_skipped': str(total_skipped), 'wishlist_queued': str(wishlist_queued), 'duration_seconds': str(duration), } except Exception as e: _pipeline_running = False _update_automation_progress(automation_id, status='error', progress=100, phase='Pipeline error', log_line=f'Pipeline failed: {e}', log_type='error') return {'status': 'error', 'error': str(e), '_manages_own_progress': True} automation_engine.register_action_handler('refresh_mirrored', _auto_refresh_mirrored) automation_engine.register_action_handler('sync_playlist', _auto_sync_playlist) automation_engine.register_action_handler('discover_playlist', _auto_discover_playlist) automation_engine.register_action_handler('playlist_pipeline', _auto_playlist_pipeline, _pipeline_guard) # --- Phase 3 action handlers --- def _auto_start_database_update(config): global _db_update_automation_id automation_id = config.get('_automation_id') if db_update_state.get('status') == 'running': return {'status': 'skipped', 'reason': 'Database update already running'} _db_update_automation_id = automation_id full = config.get('full_refresh', False) active_server = config_manager.get_active_media_server() with db_update_lock: db_update_state.update({ "status": "running", "phase": "Initializing...", "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" }) db_update_executor.submit(_run_db_update_task, full, active_server) # Monitor DB update progress (callbacks handle card updates, we just block until done) time.sleep(1) poll_start = time.time() last_progress_time = time.time() last_progress_val = 0 while time.time() - poll_start < 7200: # Max 2 hours time.sleep(3) with db_update_lock: status = db_update_state.get('status', 'idle') current_progress = db_update_state.get('progress', 0) if status != 'running': break # Track stall detection — if no progress change in 10 minutes, warn if current_progress != last_progress_val: last_progress_val = current_progress last_progress_time = time.time() elif time.time() - last_progress_time > 600: _update_automation_progress(automation_id, log_line='Database update appears stalled — waiting...', log_type='warning') last_progress_time = time.time() # Reset so warning repeats every 10 min else: # 2-hour timeout reached _update_automation_progress(automation_id, status='error', phase='Timed out', log_line='Database update timed out after 2 hours', log_type='error') return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} # Finished/error callback already updated the card — return matching status with db_update_lock: final_status = db_update_state.get('status', 'unknown') if final_status == 'error': return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} with db_update_lock: stats = { 'status': 'completed', 'full_refresh': str(full), '_manages_own_progress': True, 'artists': db_update_state.get('total', 0), 'albums': db_update_state.get('total_albums', 0), 'tracks': db_update_state.get('total_tracks', 0), 'removed_artists': db_update_state.get('removed_artists', 0), 'removed_albums': db_update_state.get('removed_albums', 0), 'removed_tracks': db_update_state.get('removed_tracks', 0), } return stats def _auto_deep_scan_library(config): global _db_update_automation_id automation_id = config.get('_automation_id') if db_update_state.get('status') == 'running': return {'status': 'skipped', 'reason': 'Database update already running'} _db_update_automation_id = automation_id active_server = config_manager.get_active_media_server() with db_update_lock: db_update_state.update({ "status": "running", "phase": "Deep scan: Initializing...", "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" }) db_update_executor.submit(_run_deep_scan_task, active_server) # Monitor progress (callbacks handle card updates, we just block until done) time.sleep(1) poll_start = time.time() last_progress_time = time.time() last_progress_val = 0 while time.time() - poll_start < 7200: # Max 2 hours time.sleep(3) with db_update_lock: status = db_update_state.get('status', 'idle') current_progress = db_update_state.get('progress', 0) if status != 'running': break if current_progress != last_progress_val: last_progress_val = current_progress last_progress_time = time.time() elif time.time() - last_progress_time > 600: _update_automation_progress(automation_id, log_line='Deep scan appears stalled — waiting...', log_type='warning') last_progress_time = time.time() else: _update_automation_progress(automation_id, status='error', phase='Timed out', log_line='Deep scan timed out after 2 hours', log_type='error') return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} with db_update_lock: final_status = db_update_state.get('status', 'unknown') if final_status == 'error': return {'status': 'error', 'reason': db_update_state.get('error_message', 'Unknown error'), '_manages_own_progress': True} with db_update_lock: stats = { 'status': 'completed', '_manages_own_progress': True, 'artists': db_update_state.get('total', 0), 'albums': db_update_state.get('total_albums', 0), 'tracks': db_update_state.get('total_tracks', 0), 'removed_artists': db_update_state.get('removed_artists', 0), 'removed_albums': db_update_state.get('removed_albums', 0), 'removed_tracks': db_update_state.get('removed_tracks', 0), } return stats def _auto_run_duplicate_cleaner(config): automation_id = config.get('_automation_id') if duplicate_cleaner_state.get('status') == 'running': return {'status': 'skipped', 'reason': 'Duplicate cleaner already running'} # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run with duplicate_cleaner_lock: duplicate_cleaner_state["status"] = "running" duplicate_cleaner_executor.submit(_run_duplicate_cleaner) _update_automation_progress(automation_id, log_line='Duplicate cleaner started', log_type='info') # Monitor duplicate cleaner progress (max 2 hours) time.sleep(1) # Brief pause for executor to start poll_start = time.time() while time.time() - poll_start < 7200: time.sleep(3) status = duplicate_cleaner_state.get('status', 'idle') if status not in ('running',): break phase = duplicate_cleaner_state.get('phase', 'Scanning...') progress = duplicate_cleaner_state.get('progress', 0) scanned = duplicate_cleaner_state.get('files_scanned', 0) total = duplicate_cleaner_state.get('total_files', 0) _update_automation_progress(automation_id, phase=phase, progress=progress, processed=scanned, total=total) else: # 2-hour timeout reached _update_automation_progress(automation_id, status='error', phase='Timed out', log_line='Duplicate cleaner timed out after 2 hours', log_type='error') return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} # Check actual exit status (could be 'finished' or 'error') final_status = duplicate_cleaner_state.get('status', 'idle') if final_status == 'error': err = duplicate_cleaner_state.get('error_message', 'Unknown error') _update_automation_progress(automation_id, status='error', progress=100, phase='Error', log_line=err, log_type='error') return {'status': 'error', 'reason': err, '_manages_own_progress': True} dupes = duplicate_cleaner_state.get('duplicates_found', 0) removed = duplicate_cleaner_state.get('deleted', 0) space_freed = duplicate_cleaner_state.get('space_freed', 0) scanned = duplicate_cleaner_state.get('files_scanned', 0) _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line=f'Found {dupes} duplicates, removed {removed} files', log_type='success') return { 'status': 'completed', '_manages_own_progress': True, 'files_scanned': scanned, 'duplicates_found': dupes, 'files_deleted': removed, 'space_freed_mb': round(space_freed / (1024 * 1024), 1), } def _auto_clear_quarantine(config): import shutil as _shutil automation_id = config.get('_automation_id') quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') if not os.path.exists(quarantine_path): _update_automation_progress(automation_id, log_line='No quarantine folder found', log_type='info') return {'status': 'completed', 'removed': '0'} removed = 0 for f in os.listdir(quarantine_path): fp = os.path.join(quarantine_path, f) try: if os.path.isfile(fp): os.remove(fp) removed += 1 elif os.path.isdir(fp): _shutil.rmtree(fp) removed += 1 except Exception: pass _update_automation_progress(automation_id, log_line=f'Removed {removed} quarantined items', log_type='success' if removed > 0 else 'info') return {'status': 'completed', 'removed': str(removed)} def _auto_cleanup_wishlist(config): automation_id = config.get('_automation_id') db = get_database() removed = db.remove_wishlist_duplicates(get_current_profile_id()) _update_automation_progress(automation_id, log_line=f'Removed {removed or 0} duplicate wishlist entries', log_type='success' if removed else 'info') return {'status': 'completed', 'removed': str(removed or 0)} def _auto_update_discovery_pool(config): automation_id = config.get('_automation_id') try: from core.watchlist_scanner import get_watchlist_scanner scanner = get_watchlist_scanner(spotify_client) _update_automation_progress(automation_id, log_line='Updating discovery pool...', log_type='info') scanner.update_discovery_pool_incremental(get_current_profile_id()) _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line='Discovery pool updated', log_type='success') return {'status': 'completed', '_manages_own_progress': True} except Exception as e: _update_automation_progress(automation_id, status='error', phase='Error', log_line=str(e), log_type='error') return {'status': 'error', 'reason': str(e), '_manages_own_progress': True} def _auto_start_quality_scan(config): automation_id = config.get('_automation_id') if quality_scanner_state.get('status') == 'running': return {'status': 'skipped', 'reason': 'Quality scan already running'} scope = config.get('scope', 'watchlist') # Pre-set status before submit so polling loop doesn't see stale 'finished' from last run with quality_scanner_lock: quality_scanner_state["status"] = "running" quality_scanner_executor.submit(_run_quality_scanner, scope, get_current_profile_id()) _update_automation_progress(automation_id, log_line=f'Quality scan started (scope: {scope})', log_type='info') # Monitor quality scanner progress (max 2 hours) time.sleep(1) # Brief pause for executor to start poll_start = time.time() while time.time() - poll_start < 7200: time.sleep(3) status = quality_scanner_state.get('status', 'idle') if status not in ('running',): break phase = quality_scanner_state.get('phase', 'Scanning...') progress = quality_scanner_state.get('progress', 0) processed = quality_scanner_state.get('processed', 0) total = quality_scanner_state.get('total', 0) _update_automation_progress(automation_id, phase=phase, progress=progress, processed=processed, total=total) else: # 2-hour timeout reached _update_automation_progress(automation_id, status='error', phase='Timed out', log_line='Quality scan timed out after 2 hours', log_type='error') return {'status': 'error', 'reason': 'Timed out', '_manages_own_progress': True} # Check actual exit status (could be 'finished' or 'error') final_status = quality_scanner_state.get('status', 'idle') if final_status == 'error': err = quality_scanner_state.get('error_message', 'Unknown error') _update_automation_progress(automation_id, status='error', progress=100, phase='Error', log_line=err, log_type='error') return {'status': 'error', 'reason': err, '_manages_own_progress': True} issues = quality_scanner_state.get('low_quality', 0) _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line=f'Quality scan complete — {issues} issues found', log_type='success') return { 'status': 'completed', 'scope': scope, '_manages_own_progress': True, 'tracks_scanned': quality_scanner_state.get('processed', 0), 'quality_met': quality_scanner_state.get('quality_met', 0), 'low_quality': issues, 'matched': quality_scanner_state.get('matched', 0), } def _auto_backup_database(config): import sqlite3, glob as _glob automation_id = config.get('_automation_id') db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') if not os.path.exists(db_path): return {'status': 'error', 'reason': 'Database file not found'} max_backups = 5 timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') backup_path = f"{db_path}.backup_{timestamp}" # Use SQLite backup API for safe hot-copy of active database src = sqlite3.connect(db_path) dst = sqlite3.connect(backup_path) src.backup(dst) dst.close() src.close() size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) # Rolling cleanup — keep only the newest N backups existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) while len(existing) > max_backups: try: os.remove(existing.pop(0)) except Exception: pass _update_automation_progress(automation_id, log_line=f'Backup created: {size_mb}MB ({os.path.basename(backup_path)})', log_type='success') return {'status': 'completed', 'backup_path': backup_path, 'size_mb': str(size_mb)} def _auto_refresh_beatport_cache(config): """Refresh Beatport homepage cache by calling each endpoint internally.""" automation_id = config.get('_automation_id') sections = [ ('hero_tracks', '/api/beatport/hero-tracks', 'Hero Tracks'), ('new_releases', '/api/beatport/new-releases', 'New Releases'), ('featured_charts', '/api/beatport/featured-charts', 'Featured Charts'), ('dj_charts', '/api/beatport/dj-charts', 'DJ Charts'), ('top_10_lists', '/api/beatport/homepage/top-10-lists', 'Top 10 Lists'), ('top_10_releases', '/api/beatport/homepage/top-10-releases-cards', 'Top 10 Releases'), ('hype_picks', '/api/beatport/hype-picks', 'Hype Picks'), ] # Invalidate all homepage cache timestamps so endpoints re-scrape with beatport_data_cache['cache_lock']: for key in beatport_data_cache['homepage']: beatport_data_cache['homepage'][key]['timestamp'] = 0 beatport_data_cache['homepage'][key]['data'] = None refreshed = 0 errors = [] with app.test_client() as client: for idx, (_, endpoint, label) in enumerate(sections): _update_automation_progress(automation_id, progress=(idx / len(sections)) * 100, phase=f'Scraping: {label}', current_item=label) try: resp = client.get(endpoint) if resp.status_code == 200: refreshed += 1 _update_automation_progress(automation_id, log_line=f'{label}: cached', log_type='success') else: errors.append(label) _update_automation_progress(automation_id, log_line=f'{label}: HTTP {resp.status_code}', log_type='error') except Exception as e: errors.append(label) _update_automation_progress(automation_id, log_line=f'{label}: {str(e)}', log_type='error') if idx < len(sections) - 1: time.sleep(2) _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line=f'Refreshed {refreshed}/{len(sections)} sections', log_type='success') return {'status': 'completed', 'refreshed': str(refreshed), 'errors': str(len(errors)), '_manages_own_progress': True} def _auto_clean_search_history(config): """Remove old searches from Soulseek.""" automation_id = config.get('_automation_id') # Skip if soulseek is not the active download source or in hybrid order dl_mode = config_manager.get('download_source.mode', 'hybrid') hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) soulseek_active = (dl_mode == 'soulseek' or (dl_mode == 'hybrid' and 'soulseek' in hybrid_order)) # soulseek_client is a DownloadOrchestrator; the real client lives on # .soulseek. Match the getattr pattern used at the other call sites. slskd = getattr(soulseek_client, 'soulseek', None) if soulseek_client else None if not soulseek_active or not slskd or not slskd.base_url: _update_automation_progress(automation_id, log_line='Soulseek not active — skipped', log_type='skip') return {'status': 'skipped'} if not config_manager.get('soulseek.auto_clear_searches', True): _update_automation_progress(automation_id, log_line='Auto-clear disabled in settings', log_type='skip') return {'status': 'skipped'} try: success = run_async(soulseek_client.maintain_search_history_with_buffer( keep_searches=50, trigger_threshold=200 )) if success: _update_automation_progress(automation_id, log_line='Search history maintenance completed', log_type='success') return {'status': 'completed'} else: _update_automation_progress(automation_id, log_line='No cleanup needed', log_type='skip') return {'status': 'completed'} except Exception as e: return {'status': 'error', 'reason': str(e)} def _auto_clean_completed_downloads(config): """Clear completed downloads and empty directories.""" automation_id = config.get('_automation_id') try: has_active_batches = False has_post_processing = False with tasks_lock: for batch_data in download_batches.values(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: has_active_batches = True break if not has_active_batches: for task_data in download_tasks.values(): if task_data.get('status') == 'post_processing': has_post_processing = True break if has_active_batches: _update_automation_progress(automation_id, log_line='Skipped — downloads active', log_type='skip') return {'status': 'completed'} run_async(soulseek_client.clear_all_completed_downloads()) if not has_post_processing: _sweep_empty_download_directories() _update_automation_progress(automation_id, log_line='Download cleanup completed', log_type='success') return {'status': 'completed'} except Exception as e: return {'status': 'error', 'reason': str(e)} def _auto_full_cleanup(config): """Run all cleanup tasks: quarantine, download queue, empty dirs, staging, search history.""" import shutil as _shutil automation_id = config.get('_automation_id') steps = [] # --- 1. Clear quarantine --- _update_automation_progress(automation_id, phase='Clearing quarantine...', progress=0) quarantine_path = os.path.join(docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')), 'ss_quarantine') q_removed = 0 if os.path.exists(quarantine_path): for f in os.listdir(quarantine_path): fp = os.path.join(quarantine_path, f) try: if os.path.isfile(fp): os.remove(fp) q_removed += 1 elif os.path.isdir(fp): _shutil.rmtree(fp) q_removed += 1 except Exception: pass steps.append(f'Quarantine: removed {q_removed} items') _update_automation_progress(automation_id, log_line=f'Quarantine: removed {q_removed} items', log_type='success' if q_removed else 'info') # --- 2. Clear completed/errored/cancelled downloads from Soulseek queue --- _update_automation_progress(automation_id, phase='Clearing download queue...', progress=20) has_active_batches = False has_post_processing = False with tasks_lock: for batch_data in download_batches.values(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled', None]: has_active_batches = True break if not has_active_batches: for task_data in download_tasks.values(): if task_data.get('status') == 'post_processing': has_post_processing = True break if has_active_batches: steps.append('Download queue: skipped (active batches)') _update_automation_progress(automation_id, log_line='Download queue: skipped (active batches)', log_type='skip') else: try: run_async(soulseek_client.clear_all_completed_downloads()) steps.append('Download queue: cleared') _update_automation_progress(automation_id, log_line='Download queue: cleared', log_type='success') except Exception as e: steps.append(f'Download queue: error ({e})') _update_automation_progress(automation_id, log_line=f'Download queue: error ({e})', log_type='error') # --- 3. Sweep empty download directories --- _update_automation_progress(automation_id, phase='Sweeping empty directories...', progress=40) if has_active_batches or has_post_processing: reason = 'active batches' if has_active_batches else 'post-processing active' steps.append(f'Empty directories: skipped ({reason})') _update_automation_progress(automation_id, log_line=f'Empty directories: skipped ({reason})', log_type='skip') else: dirs_removed = _sweep_empty_download_directories() steps.append(f'Empty directories: removed {dirs_removed}') _update_automation_progress(automation_id, log_line=f'Empty directories: removed {dirs_removed}', log_type='success' if dirs_removed else 'info') # --- 4. Sweep empty staging directories --- _update_automation_progress(automation_id, phase='Sweeping import folder...', progress=60) staging_path = get_staging_path() s_removed = 0 if os.path.isdir(staging_path): for dirpath, _dirnames, _filenames in os.walk(staging_path, topdown=False): if os.path.normpath(dirpath) == os.path.normpath(staging_path): continue try: entries = os.listdir(dirpath) except OSError: continue visible = [e for e in entries if not e.startswith('.')] if not visible: for hidden in entries: try: os.remove(os.path.join(dirpath, hidden)) except Exception: pass try: os.rmdir(dirpath) s_removed += 1 except OSError: pass steps.append(f'Staging: removed {s_removed} empty directories') _update_automation_progress(automation_id, log_line=f'Staging: removed {s_removed} empty directories', log_type='success' if s_removed else 'info') # --- 5. Clean search history (if enabled) --- _update_automation_progress(automation_id, phase='Cleaning search history...', progress=80) try: if not config_manager.get('soulseek.auto_clear_searches', True): steps.append('Search cleanup: disabled in settings') _update_automation_progress(automation_id, log_line='Search cleanup: disabled in settings', log_type='skip') else: run_async(soulseek_client.maintain_search_history_with_buffer( keep_searches=50, trigger_threshold=200 )) steps.append('Search history: cleaned') _update_automation_progress(automation_id, log_line='Search history: cleaned', log_type='success') except Exception as e: steps.append(f'Search history: error ({e})') _update_automation_progress(automation_id, log_line=f'Search history: error ({e})', log_type='error') total_removed = q_removed + s_removed _update_automation_progress(automation_id, status='finished', progress=100, phase='Complete', log_line=f'Full cleanup complete — {total_removed} items removed', log_type='success') return { 'status': 'completed', 'quarantine_removed': str(q_removed), 'staging_removed': str(s_removed), 'total_removed': str(total_removed), 'steps': steps, '_manages_own_progress': True, } def _auto_run_script(config): """Execute a user script from the scripts directory.""" import subprocess as _sp script_name = config.get('script_name', '') timeout = min(int(config.get('timeout', 60)), 300) automation_id = config.get('_automation_id') if not script_name: return {'status': 'error', 'error': 'No script selected'} scripts_dir = docker_resolve_path(config_manager.get('scripts.path', './scripts')) if not scripts_dir or not os.path.isdir(scripts_dir): os.makedirs(scripts_dir, exist_ok=True) return {'status': 'error', 'error': 'Scripts directory is empty. Add scripts to the scripts/ folder.'} script_path = os.path.join(scripts_dir, script_name) script_path = os.path.realpath(script_path) # Security: block path traversal if not script_path.startswith(os.path.realpath(scripts_dir)): return {'status': 'error', 'error': 'Script path traversal blocked'} if not os.path.isfile(script_path): return {'status': 'error', 'error': f'Script not found: {script_name}'} _update_automation_progress(automation_id, phase=f'Running {script_name}...', progress=10) # Build environment with SoulSync context env = os.environ.copy() event_data = config.get('_event_data') or {} env['SOULSYNC_EVENT'] = str(event_data.get('type', '')) env['SOULSYNC_AUTOMATION'] = config.get('_automation_name', '') env['SOULSYNC_SCRIPTS_DIR'] = scripts_dir try: # Determine how to run the script if script_path.endswith('.py'): cmd = ['python', script_path] elif script_path.endswith('.sh'): cmd = ['bash', script_path] else: cmd = [script_path] result = _sp.run( cmd, capture_output=True, text=True, timeout=timeout, cwd=scripts_dir, env=env ) _update_automation_progress(automation_id, phase='Script completed', progress=100) stdout = result.stdout[:2000] if result.stdout else '' stderr = result.stderr[:1000] if result.stderr else '' if result.returncode == 0: logger.info(f"Script '{script_name}' completed (exit 0)") else: logger.warning(f"Script '{script_name}' exited with code {result.returncode}") return { 'status': 'completed' if result.returncode == 0 else 'error', 'exit_code': str(result.returncode), 'stdout': stdout, 'stderr': stderr, 'script': script_name, } except _sp.TimeoutExpired: _update_automation_progress(automation_id, phase='Script timed out', progress=100) return {'status': 'error', 'error': f'Script timed out after {timeout}s', 'script': script_name} except Exception as e: return {'status': 'error', 'error': str(e), 'script': script_name} automation_engine.register_action_handler('run_script', _auto_run_script) automation_engine.register_action_handler('full_cleanup', _auto_full_cleanup) automation_engine.register_action_handler('start_database_update', _auto_start_database_update, lambda: db_update_state.get('status') == 'running') automation_engine.register_action_handler('deep_scan_library', _auto_deep_scan_library, lambda: db_update_state.get('status') == 'running') automation_engine.register_action_handler('run_duplicate_cleaner', _auto_run_duplicate_cleaner, lambda: duplicate_cleaner_state.get('status') == 'running') automation_engine.register_action_handler('clear_quarantine', _auto_clear_quarantine) automation_engine.register_action_handler('cleanup_wishlist', _auto_cleanup_wishlist) automation_engine.register_action_handler('update_discovery_pool', _auto_update_discovery_pool) automation_engine.register_action_handler('start_quality_scan', _auto_start_quality_scan, lambda: quality_scanner_state.get('status') == 'running') automation_engine.register_action_handler('backup_database', _auto_backup_database) automation_engine.register_action_handler('refresh_beatport_cache', _auto_refresh_beatport_cache) automation_engine.register_action_handler('clean_search_history', _auto_clean_search_history) automation_engine.register_action_handler('clean_completed_downloads', _auto_clean_completed_downloads) def _auto_search_and_download(config): """Search for a track and download the best match.""" automation_id = config.get('_automation_id') query = config.get('query', '').strip() # Event-triggered: pull query from event data (e.g. webhook_received) if not query: event_data = config.get('_event_data', {}) query = (event_data.get('query', '') or '').strip() if not query: if automation_id: _update_automation_progress(automation_id, log_line='No search query provided', log_type='error') return {'status': 'error', 'error': 'No search query provided'} try: if automation_id: _update_automation_progress(automation_id, phase='Searching', log_line=f'Searching: {query}', log_type='info') result = run_async(soulseek_client.search_and_download_best(query)) if result: if automation_id: _update_automation_progress(automation_id, log_line=f'Download started for: {query}', log_type='success') return {'status': 'completed', 'query': query, 'download_id': result} else: if automation_id: _update_automation_progress(automation_id, log_line=f'No match found for: {query}', log_type='warning') return {'status': 'not_found', 'query': query, 'error': 'No match found'} except Exception as e: if automation_id: _update_automation_progress(automation_id, log_line=f'Error: {e}', log_type='error') return {'status': 'error', 'query': query, 'error': str(e)} automation_engine.register_action_handler('search_and_download', _auto_search_and_download) # Register progress tracking callbacks def _progress_init(aid, name, action_type): _init_automation_progress(aid, name, action_type) def _progress_finish(aid, result): result_status = result.get('status', '') # Skip for handlers that manage their own progress lifecycle # (they call _update_automation_progress(status='finished') themselves) if result.get('_manages_own_progress'): return status = 'error' if result_status == 'error' else 'finished' msg = result.get('error', result.get('reason', result_status or 'done')) _update_automation_progress(aid, status=status, progress=100, phase='Error' if status == 'error' else 'Complete', log_line=msg, log_type='error' if status == 'error' else 'success') def _record_automation_history(aid, result): """Capture progress state into run history before cleanup clears it.""" _auto_progress.record_history(aid, result, get_database()) automation_engine.register_progress_callbacks(_progress_init, _progress_finish, _update_automation_progress, _record_automation_history) # Register permanent callback: when any scan completes, emit library_scan_completed event # This replaces the hardcoded scan_completion_callback → trigger_automatic_database_update chain if web_scan_manager: def _on_library_scan_completed(): if automation_engine: server_type = getattr(web_scan_manager, '_current_server_type', None) or 'unknown' automation_engine.emit('library_scan_completed', { 'server_type': server_type, }) web_scan_manager.add_scan_completion_callback(_on_library_scan_completed) logger.info("Automation action handlers registered") # --- Register Public REST API Blueprint (v1) --- try: from api import create_api_blueprint, limiter limiter.init_app(app) api_bp = create_api_blueprint() app.register_blueprint(api_bp, url_prefix='/api/v1') app.soulsync = { 'spotify_client': spotify_client, 'soulseek_client': soulseek_client, 'tidal_client': tidal_client, 'matching_engine': matching_engine, 'config_manager': config_manager, 'automation_engine': automation_engine, 'hydrabase_client': None, # updated after Hydrabase init 'hydrabase_worker': None, # updated after Hydrabase init } logger.info("Public REST API v1 registered at /api/v1") except Exception as e: logger.error(f"Public REST API v1 failed to register: {e}") # --- Automation Progress Tracking --- # State + helpers live in core/automation/progress.py. Re-exported here # so the existing call sites (registered as engine progress callbacks # and used inside _record_automation_history) keep resolving. from core.automation import progress as _auto_progress automation_progress_states = _auto_progress.progress_states automation_progress_lock = _auto_progress.progress_lock def _init_automation_progress(automation_id, automation_name, action_type): """Initialize progress state when an automation starts running.""" _auto_progress.init_progress(automation_id, automation_name, action_type) def _update_automation_progress(automation_id, **kwargs): """Update progress state from handler threads. Thread-safe.""" _auto_progress.update_progress(automation_id, socketio_emit=socketio.emit, **kwargs) # --- Global Matched Downloads Context Management --- # Shared with core.runtime_state so the refactored pipeline and web # server operate on the same context registry. # --- Download Missing Tracks Modal State Management --- # Thread-safe state tracking for modal download functionality with batch management def _get_max_concurrent(): """Get configured max concurrent downloads. Default 3.""" return config_manager.get('download_source.max_concurrent', 3) def _get_batch_max_concurrent(is_album=False, source=None): """Get max concurrent workers for a batch. Soulseek album downloads always use 1 (source reuse per user). Everything else uses the configured setting.""" if is_album and source in ('soulseek', None): # Check if active source is soulseek if source == 'soulseek': return 1 mode = config_manager.get('download_source.mode', 'soulseek') if mode == 'soulseek': return 1 return _get_max_concurrent() # --- Session Download Statistics --- # Track individual download completions (matches dashboard.py behavior) def _mark_task_completed(task_id, track_info=None): """ Mark a download task as completed and increment session counter. Centralizes completion logic to ensure consistent behavior. Assumes task_id exists in download_tasks (should be called within tasks_lock). """ global session_completed_downloads mark_task_completed(task_id, track_info) # Increment session counter (matches dashboard.py behavior) with session_stats_lock: session_completed_downloads += 1 # --- Automatic Wishlist Processing Infrastructure --- # Server-side timer system for automatic wishlist processing (replaces client-side JavaScript timers) # --- Automatic Wishlist/Watchlist Processing Flags --- # Processing state flags (guards/recovery — timers are now managed by AutomationEngine) # --- Shared Transfer Data Cache --- # Cache transfer data to avoid hammering the Soulseek API with multiple concurrent modals def get_cached_transfer_data(): """ Get transfer data with caching to reduce API calls when multiple modals are active. Returns a lookup dictionary for efficient transfer matching. """ current_time = time.time() with transfer_data_cache['update_lock']: # Check if cache is still valid if (current_time - transfer_data_cache['last_update']) < transfer_data_cache['cache_duration']: return transfer_data_cache['data'] # Cache expired or empty, fetch new data live_transfers_lookup = {} try: # Skip Soulseek API call if not active or known disconnected _dl_mode = config_manager.get('download_source.mode', 'hybrid') _hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) _slsk_active = (_dl_mode == 'soulseek' or (_dl_mode == 'hybrid' and 'soulseek' in _hybrid_order)) soulseek_known_down = not _slsk_active or not _status_cache.get('soulseek', {}).get('connected', True) # First, get Soulseek downloads from API transfers_data = None if not soulseek_known_down and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url: transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) if transfers_data: all_transfers = [] for user_data in transfers_data: username = user_data.get('username', 'Unknown') if 'directories' in user_data: for directory in user_data['directories']: if 'files' in directory: for file_info in directory['files']: file_info['username'] = username all_transfers.append(file_info) for transfer in all_transfers: key = _make_context_key(transfer.get('username'), transfer.get('filename', '')) live_transfers_lookup[key] = transfer # Also add non-Soulseek downloads (avoid redundant slskd call through orchestrator) try: all_downloads = [] for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz, soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr]: if _dl_client: try: all_downloads.extend(run_async(_dl_client.get_all_downloads())) except Exception: pass for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format live_transfers_lookup[key] = { 'id': download.id, 'filename': download.filename, 'username': download.username, 'state': download.state, 'percentComplete': download.progress, 'size': download.size, 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } except Exception as e: logger.error(f"Could not fetch streaming source downloads: {e}") # Update cache transfer_data_cache['data'] = live_transfers_lookup transfer_data_cache['last_update'] = current_time except Exception as e: logger.error(f"Could not fetch live transfers (cached): {e}") # Return empty dict on error, but don't update cache timestamp # This way we'll retry on the next request return {} return live_transfers_lookup # --- Beatport Data Cache --- # Cache Beatport scraping data to reduce load times and avoid hammering Beatport.com def get_cached_beatport_data(section_type, data_key, genre_slug=None): """ Get Beatport data from cache if valid, otherwise return None. Args: section_type: 'homepage' or 'genre' data_key: specific data type (e.g., 'hero_tracks', 'top_10_lists') genre_slug: only used for genre section_type Returns: Cached data if valid, None if cache miss or expired """ current_time = time.time() with beatport_data_cache['cache_lock']: try: if section_type == 'homepage': cache_entry = beatport_data_cache['homepage'].get(data_key) elif section_type == 'genre' and genre_slug: cache_entry = beatport_data_cache['genre'].get(genre_slug, {}).get(data_key) else: return None if not cache_entry: return None # Check if cache is still valid age = current_time - cache_entry['timestamp'] if age < cache_entry['ttl'] and cache_entry['data'] is not None: logger.debug(f"Cache HIT for {section_type}/{data_key} (age: {age:.1f}s)") return cache_entry['data'] else: logger.debug(f"⏰ Cache MISS for {section_type}/{data_key} (age: {age:.1f}s, ttl: {cache_entry['ttl']}s)") return None except Exception as e: logger.error(f"Cache lookup error for {section_type}/{data_key}: {e}") return None def set_cached_beatport_data(section_type, data_key, data, genre_slug=None): """ Store Beatport data in cache with current timestamp. Args: section_type: 'homepage' or 'genre' data_key: specific data type (e.g., 'hero_tracks', 'top_10_lists') data: the data to cache genre_slug: only used for genre section_type """ current_time = time.time() with beatport_data_cache['cache_lock']: try: if section_type == 'homepage': if data_key in beatport_data_cache['homepage']: beatport_data_cache['homepage'][data_key]['data'] = data beatport_data_cache['homepage'][data_key]['timestamp'] = current_time logger.info(f"Cached {section_type}/{data_key} (ttl: {beatport_data_cache['homepage'][data_key]['ttl']}s)") elif section_type == 'genre' and genre_slug: # Initialize genre cache if not exists if genre_slug not in beatport_data_cache['genre']: beatport_data_cache['genre'][genre_slug] = {} # For genre caching, we need to define TTL structure (for future use) if data_key not in beatport_data_cache['genre'][genre_slug]: beatport_data_cache['genre'][genre_slug][data_key] = { 'data': None, 'timestamp': 0, 'ttl': 600 # Default 10 minutes } beatport_data_cache['genre'][genre_slug][data_key]['data'] = data beatport_data_cache['genre'][genre_slug][data_key]['timestamp'] = current_time logger.info(f"Cached {section_type}/{genre_slug}/{data_key}") except Exception as e: logger.error(f"Cache storage error for {section_type}/{data_key}: {e}") def add_cache_headers(response, cache_duration=300): """ Add HTTP cache control headers to responses for browser-side caching. Args: response: Flask response object cache_duration: Cache duration in seconds (default: 5 minutes) """ response.headers['Cache-Control'] = f'public, max-age={cache_duration}' response.headers['Pragma'] = 'cache' return response # Enhanced-search cache + helpers live in core/search/cache.py. # Re-exported here so the existing call sites in this module still resolve. from core.search.cache import ( get_cache_key as _get_enhanced_search_cache_key_impl, get_cached_response as _get_cached_enhanced_search_response, set_cached_response as _set_cached_enhanced_search_response, ) from core.search.orchestrator import VALID_SOURCES as ENHANCED_SEARCH_VALID_SOURCES def _get_enhanced_search_cache_key(query, requested_source=None): """Thin wrapper that wires live config providers into the cache-key builder.""" return _get_enhanced_search_cache_key_impl( query, requested_source, active_server_provider=config_manager.get_active_media_server, fallback_source_provider=_get_metadata_fallback_source, hydrabase_active_provider=_is_hydrabase_active, ) # --- Background Download Monitoring (GUI Parity) --- class WebUIDownloadMonitor: """ Background monitor for download progress and retry logic, matching GUI's SyncStatusProcessingWorker. Implements identical timeout detection and automatic retry functionality. """ def __init__(self): self.monitoring = False self.monitor_thread = None self.monitored_batches = set() self._lock = threading.Lock() def start_monitoring(self, batch_id): """Start monitoring a download batch""" with self._lock: self.monitored_batches.add(batch_id) if not self.monitoring: self.monitoring = True self.monitor_thread = threading.Thread(target=self._monitor_loop, daemon=True) self.monitor_thread.start() logger.info(f"Started download monitor for batch {batch_id}") def stop_monitoring(self, batch_id): """Stop monitoring a specific batch""" with self._lock: self.monitored_batches.discard(batch_id) if not self.monitored_batches: self.monitoring = False logger.debug("Stopped download monitor (no active batches)") def shutdown(self): """Stop the monitor loop and clear active batch tracking.""" with self._lock: self.monitoring = False self.monitored_batches.clear() self.monitor_thread = None logger.info("Download monitor shutdown requested") def _monitor_loop(self): """Main monitoring loop - checks downloads every 1 second for responsive web UX""" while self.monitoring and self.monitored_batches: try: if globals().get('IS_SHUTTING_DOWN', False): self.monitoring = False break self._check_all_downloads() time.sleep(1) # 1-second polling for fast web UI updates except Exception as e: # If we get shutdown errors, stop monitoring gracefully if "interpreter shutdown" in str(e) or "cannot schedule new futures" in str(e): logger.info("Monitor detected shutdown, stopping gracefully") self.monitoring = False break logger.error(f"Download monitor error: {e}") logger.info("Download monitor loop ended") def _check_all_downloads(self): """Check all active downloads for timeouts and failures""" current_time = time.time() # Get live transfer data from slskd live_transfers_lookup = self._get_live_transfers() # Track tasks with exhausted retries to handle after releasing lock exhausted_tasks = [] # List of (batch_id, task_id) tuples # Track completed downloads to handle after releasing lock (prevents deadlock) completed_tasks = [] # List of (batch_id, task_id) tuples # Track deferred operations (network calls, nested locks) to run after releasing tasks_lock deferred_ops = [] with tasks_lock: # Check all monitored batches for timeouts and errors for batch_id in list(self.monitored_batches): if batch_id not in download_batches: self.monitored_batches.discard(batch_id) continue for task_id in download_batches[batch_id].get('queue', []): task = download_tasks.get(task_id) if not task or task['status'] not in ['downloading', 'queued']: continue # Check for timeouts and errors - retries handled directly in _should_retry_task # If _should_retry_task returns True, it means retries were exhausted retry_exhausted = self._should_retry_task(task_id, task, live_transfers_lookup, current_time, deferred_ops) # Collect exhausted tasks to handle outside lock (prevents deadlock) if retry_exhausted: exhausted_tasks.append((batch_id, task_id)) # ENHANCED: Check for successful completions (especially YouTube) task_filename = task.get('filename') or task.get('track_info', {}).get('filename') task_username = task.get('username') or task.get('track_info', {}).get('username') if task_filename and task_username: lookup_key = _make_context_key(task_username, task_filename) live_info = live_transfers_lookup.get(lookup_key) if live_info: state = live_info.get('state', '') # Trigger post-processing if download is completed successfully # slskd uses compound states like 'Completed, Succeeded' - use substring matching # Must exclude error states first (matching _build_batch_status_data's prioritized checking) has_error = ('Errored' in state or 'Failed' in state or 'Rejected' in state or 'TimedOut' in state) has_completion = ('Completed' in state or 'Succeeded' in state) # Verify bytes actually transferred before trusting state string. # slskd can report "Completed" before the full file is flushed to disk, # or on connection drops that leave a partial file. if has_completion and not has_error: expected_size = live_info.get('size', 0) transferred = live_info.get('bytesTransferred', 0) if expected_size > 0 and transferred < expected_size: if not task.get('_incomplete_warned'): logger.debug(f"Monitor: {task_id} state={state} but bytes incomplete ({transferred}/{expected_size}) — waiting") task['_incomplete_warned'] = True continue if has_completion and not has_error and task['status'] == 'downloading': task.pop('_incomplete_warned', None) # CRITICAL FIX: Transition to 'post_processing' HERE so downloads # don't depend on browser polling to trigger post-processing. # Previously, post-processing was only submitted by _build_batch_status_data # (called from browser-polled endpoints), meaning closing the browser # left tasks stuck in 'downloading' forever. task['status'] = 'post_processing' task['status_change_time'] = current_time logger.info(f"Monitor detected completed download for {task_id} ({state}) - submitting post-processing") # Collect for handling outside the lock to prevent deadlock. # _on_download_completed acquires tasks_lock which is non-reentrant. completed_tasks.append((batch_id, task_id)) # ---- All work below runs WITHOUT tasks_lock held ---- if globals().get('IS_SHUTTING_DOWN', False) or not self.monitoring: return # Execute deferred operations from _should_retry_task (network calls, nested locks) for op in deferred_ops: try: if op[0] == 'cancel_download': _, download_id, username = op logger.debug(f"[Deferred] Cancelling download: {download_id} from {username}") run_async(soulseek_client.cancel_download(download_id, username, remove=True)) logger.debug(f"[Deferred] Successfully cancelled download {download_id}") elif op[0] == 'cleanup_orphan': _, context_key = op with matched_context_lock: matched_downloads_context.pop(context_key, None) logger.debug(f"[Deferred] Cleaned up orphaned download context: {context_key}") elif op[0] == 'restart_worker': _, task_id, batch_id = op logger.debug(f"[Deferred] Restarting worker for task {task_id}") missing_download_executor.submit(_download_track_worker, task_id, batch_id) logger.debug(f"[Deferred] Successfully restarted worker for task {task_id}") except Exception as e: logger.error(f"[Deferred] Error executing deferred operation {op[0]}: {e}") # Handle completed downloads outside the lock to prevent deadlock # (_on_download_completed acquires tasks_lock internally) for batch_id, task_id in completed_tasks: try: # Submit post-processing worker (file move, tagging, AcoustID verification) # This makes batch downloads fully independent of browser polling. logger.info(f"[Monitor] Submitting post-processing worker for task {task_id}") missing_download_executor.submit(_run_post_processing_worker, task_id, batch_id) # Chain to next download in the batch queue _on_download_completed(batch_id, task_id, success=True) except Exception as e: logger.error(f"[Monitor] Error handling completed task {task_id}: {e}") # Handle exhausted retry tasks outside the lock to prevent deadlock for batch_id, task_id in exhausted_tasks: try: logger.info(f"[Monitor] Calling completion callback for exhausted task {task_id}") _on_download_completed(batch_id, task_id, success=False) except Exception as e: logger.error(f"[Monitor] Error handling exhausted task {task_id}: {e}") # ENHANCED: Add worker count validation to detect ghost workers self._validate_worker_counts() def _get_live_transfers(self): """Get current transfer status from slskd API and YouTube client""" try: # Check if we should stop due to shutdown if not self.monitoring: return {} live_transfers = {} # Only hit slskd API if soulseek is actually configured and active dl_mode = config_manager.get('download_source.mode', 'hybrid') hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) soulseek_active = (dl_mode == 'soulseek' or (dl_mode == 'hybrid' and 'soulseek' in hybrid_order)) # Get Soulseek downloads from API transfers_data = None if soulseek_active and soulseek_client and getattr(soulseek_client, 'soulseek', None) and soulseek_client.soulseek.base_url: transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) if transfers_data: for user_data in transfers_data: username = user_data.get('username', 'Unknown') if 'directories' in user_data: for directory in user_data['directories']: if 'files' in directory: for file_info in directory['files']: key = _make_context_key(username, file_info.get('filename', '')) live_transfers[key] = file_info # Also get non-Soulseek downloads (YouTube/Tidal/Qobuz/HiFi/Deezer/Lidarr) # Call each client directly to avoid redundant slskd API call through orchestrator try: all_downloads = [] for _dl_client in [soulseek_client.youtube, soulseek_client.tidal, soulseek_client.qobuz, soulseek_client.hifi, soulseek_client.deezer_dl, soulseek_client.lidarr]: if _dl_client: try: all_downloads.extend(run_async(_dl_client.get_all_downloads())) except Exception: pass for download in all_downloads: key = _make_context_key(download.username, download.filename) # Convert DownloadStatus to transfer dict format for monitor compatibility live_transfers[key] = { 'id': download.id, 'filename': download.filename, 'username': download.username, 'state': download.state, 'percentComplete': download.progress, 'size': download.size, 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } except Exception as yt_error: logger.error(f"Monitor: Could not fetch streaming source downloads: {yt_error}") return live_transfers except Exception as e: # If we get shutdown-related errors, stop monitoring immediately if ("interpreter shutdown" in str(e) or "cannot schedule new futures" in str(e) or "Event loop is closed" in str(e)): logger.info("Monitor detected shutdown, stopping immediately") self.monitoring = False return {} else: logger.error(f"Monitor: Could not fetch live transfers: {e}") return {} def _should_retry_task(self, task_id, task, live_transfers_lookup, current_time, deferred_ops): """ Determine if a task should be retried due to timeout (matches GUI logic). IMPORTANT: This runs while tasks_lock is held. All network calls (slskd API) and nested lock acquisitions (matched_context_lock) are collected into deferred_ops to be executed AFTER releasing tasks_lock. This prevents deadlocks and long lock holds. Returns True if retries are exhausted and _on_download_completed should be called outside the lock. """ ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} task_filename = task.get('filename') or ti.get('filename') task_username = task.get('username') or ti.get('username') if not task_filename or not task_username: return False lookup_key = _make_context_key(task_username, task_filename) live_info = live_transfers_lookup.get(lookup_key) if not live_info: # Task not in live transfers but status is downloading/queued - likely stuck if current_time - task.get('status_change_time', current_time) > 90: retry_count = task.get('stuck_retry_count', 0) last_retry = task.get('last_retry_time', 0) if retry_count < 3 and (current_time - last_retry) > 30: logger.warning(f"Task not in live transfers for >90s - retry {retry_count + 1}/3") task['stuck_retry_count'] = retry_count + 1 task['last_retry_time'] = current_time download_id = task.get('download_id') # Defer slskd cancel to outside the lock if task_username and download_id: deferred_ops.append(('cancel_download', download_id, task_username)) # Mark current source as used (full filename to match worker format) if task_username and task_filename: used_sources = task.get('used_sources', set()) source_key = f"{task_username}_{task_filename}" used_sources.add(source_key) task['used_sources'] = used_sources logger.warning(f"Marked missing-transfer source as used: {source_key}") # Defer orphan cleanup if task_username and task_filename: _orphaned_download_keys.add(lookup_key) deferred_ops.append(('cleanup_orphan', lookup_key)) # Clear download info and reset for retry task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) task['status'] = 'searching' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task['status_change_time'] = current_time logger.warning(f"Task {task.get('track_info', {}).get('name', 'Unknown')} reset for missing-transfer retry") batch_id = task.get('batch_id') if task_id and batch_id: deferred_ops.append(('restart_worker', task_id, batch_id)) return False elif retry_count < 3: return False else: track_label = task.get('track_info', {}).get('name', 'Unknown') tried_sources = task.get('used_sources', set()) sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else '' logger.error("Task failed after 3 retry attempts (not in live transfers)") task['status'] = 'failed' task['error_message'] = f'Download disappeared from transfer list 3 times for "{track_label}"{sources_str} — source may be unavailable' batch_id = task.get('batch_id') if batch_id: return True return False return False state_str = live_info.get('state', '') progress = live_info.get('percentComplete', 0) # IMMEDIATE ERROR RETRY: Check for errored/rejected/timed-out downloads first (no timeout needed) if 'Errored' in state_str or 'Failed' in state_str or 'Rejected' in state_str or 'TimedOut' in state_str: retry_count = task.get('error_retry_count', 0) last_retry = task.get('last_error_retry_time', 0) # Don't retry too frequently (wait at least 5 seconds between error retries) if retry_count < 3 and (current_time - last_retry) > 5: # Max 3 error retry attempts logger.error(f"Task errored (state: {state_str}) - immediate retry {retry_count + 1}/3") task['error_retry_count'] = retry_count + 1 task['last_error_retry_time'] = current_time _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} username = task.get('username') or _ti.get('username') filename = task.get('filename') or _ti.get('filename') download_id = task.get('download_id') # Defer slskd cancel to outside the lock if username and download_id: deferred_ops.append(('cancel_download', download_id, username)) # Mark current source as used to prevent retry loops # CRITICAL: Use full filename (not basename) to match worker's source_key format if username and filename: used_sources = task.get('used_sources', set()) source_key = f"{username}_{filename}" used_sources.add(source_key) task['used_sources'] = used_sources logger.error(f"Marked errored source as used: {source_key}") # Defer orphan cleanup to outside the lock (needs matched_context_lock) if username and filename: old_context_key = _make_context_key(username, filename) _orphaned_download_keys.add(old_context_key) deferred_ops.append(('cleanup_orphan', old_context_key)) # Clear download info since we cancelled it task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) # Reset task state for immediate retry task['status'] = 'searching' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task['status_change_time'] = current_time logger.error(f"Task {task.get('track_info', {}).get('name', 'Unknown')} reset for error retry") # Defer worker restart to outside the lock batch_id = task.get('batch_id') if task_id and batch_id: deferred_ops.append(('restart_worker', task_id, batch_id)) return False elif retry_count < 3: # Wait a bit before next error retry return False else: # Too many error retries, mark as failed track_label = task.get('track_info', {}).get('name', 'Unknown') tried_sources = task.get('used_sources', set()) sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else '' logger.error("Task failed after 3 error retry attempts") task['status'] = 'failed' # Tidal-specific error: check if this was a quality issue. # task['username'] is popped on error-retry (line ~2866) so we can't rely on it; # used_sources keys are formatted as "{username}_{filename}", so startswith is exact. is_tidal = any(s.startswith('tidal_') for s in tried_sources) if is_tidal: tidal_quality = config_manager.get('tidal_download.quality', 'lossless') allow_fb = config_manager.get('tidal_download.allow_fallback', True) if tidal_quality == 'hires' and not allow_fb: task['error_message'] = ( f'Tidal download failed for "{track_label}" — HiRes quality is unavailable for this track ' f'on your account or in your region. Enable "Quality Fallback" in Tidal settings to fall back to Lossless.' ) else: task['error_message'] = ( f'Tidal download failed for "{track_label}"{sources_str} — ' f'check Tidal authentication and quality settings.' ) else: task['error_message'] = f'Soulseek transfer errored 3 times for "{track_label}"{sources_str} — all sources failed or became unavailable' # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks batch_id = task.get('batch_id') if batch_id: logger.error(f"[Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}") return True # Signal that we need to call completion outside the lock return False # Check for queued timeout (90 seconds like GUI) elif 'Queued' in state_str or task['status'] == 'queued': if 'queued_start_time' not in task: task['queued_start_time'] = current_time return False else: queue_time = current_time - task['queued_start_time'] # Use context-aware timeouts like GUI: # - 15 seconds for artist album downloads (streaming context) # - 90 seconds for background playlist downloads is_streaming_context = task.get('track_info', {}).get('is_album_download', False) timeout_threshold = 15.0 if is_streaming_context else 90.0 if queue_time > timeout_threshold: # Track retry attempts to prevent rapid loops retry_count = task.get('stuck_retry_count', 0) last_retry = task.get('last_retry_time', 0) # Don't retry too frequently (wait at least 30 seconds between retries) if retry_count < 3 and (current_time - last_retry) > 30: # Max 3 retry attempts logger.warning(f"Task stuck in queue for {queue_time:.1f}s - immediate retry {retry_count + 1}/3") task['stuck_retry_count'] = retry_count + 1 task['last_retry_time'] = current_time _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} username = task.get('username') or _ti.get('username') filename = task.get('filename') or _ti.get('filename') download_id = task.get('download_id') # Defer slskd cancel to outside the lock if username and download_id: deferred_ops.append(('cancel_download', download_id, username)) # UNIFIED RETRY LOGIC: Handle timeout retry exactly like error retry # Mark current source as used to prevent retry loops # CRITICAL: Use full filename (not basename) to match worker's source_key format if username and filename: used_sources = task.get('used_sources', set()) source_key = f"{username}_{filename}" used_sources.add(source_key) task['used_sources'] = used_sources logger.error(f"Marked timeout source as used: {source_key}") # Defer orphan cleanup to outside the lock (needs matched_context_lock) if username and filename: old_context_key = _make_context_key(username, filename) _orphaned_download_keys.add(old_context_key) deferred_ops.append(('cleanup_orphan', old_context_key)) # Clear download info since we cancelled it task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) # Reset task state for immediate retry (like error retry) task['status'] = 'searching' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task['status_change_time'] = current_time logger.error(f"Task {task.get('track_info', {}).get('name', 'Unknown')} reset for timeout retry") # Defer worker restart to outside the lock batch_id = task.get('batch_id') if task_id and batch_id: deferred_ops.append(('restart_worker', task_id, batch_id)) return False elif retry_count < 3: # Wait longer before next retry return False else: # Too many retries, mark as failed track_label = task.get('track_info', {}).get('name', 'Unknown') tried_sources = task.get('used_sources', set()) sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else '' logger.error("Task failed after 3 retry attempts (queue timeout)") task['status'] = 'failed' task['error_message'] = f'Download stayed queued too long 3 times for "{track_label}"{sources_str} — peers may be offline or have full queues' # Clear timers to prevent further retry loops task.pop('queued_start_time', None) task.pop('downloading_start_time', None) # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks batch_id = task.get('batch_id') if batch_id: logger.error(f"[Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}") return True # Signal that we need to call completion outside the lock return False # Check for downloading at 0% timeout (90 seconds like GUI) elif 'InProgress' in state_str and progress < 1: if 'downloading_start_time' not in task: task['downloading_start_time'] = current_time return False else: download_time = current_time - task['downloading_start_time'] # Use context-aware timeouts like GUI: # - 15 seconds for artist album downloads (streaming context) # - 90 seconds for background playlist downloads is_streaming_context = task.get('track_info', {}).get('is_album_download', False) timeout_threshold = 15.0 if is_streaming_context else 90.0 if download_time > timeout_threshold: retry_count = task.get('stuck_retry_count', 0) last_retry = task.get('last_retry_time', 0) # Don't retry too frequently (wait at least 30 seconds between retries) if retry_count < 3 and (current_time - last_retry) > 30: # Max 3 retry attempts logger.warning(f"Task stuck at 0% for {download_time:.1f}s - immediate retry {retry_count + 1}/3") task['stuck_retry_count'] = retry_count + 1 task['last_retry_time'] = current_time _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} username = task.get('username') or _ti.get('username') filename = task.get('filename') or _ti.get('filename') download_id = task.get('download_id') # Defer slskd cancel to outside the lock if username and download_id: deferred_ops.append(('cancel_download', download_id, username)) # UNIFIED RETRY LOGIC: Handle 0% timeout retry exactly like error retry # Mark current source as used to prevent retry loops # CRITICAL: Use full filename (not basename) to match worker's source_key format if username and filename: used_sources = task.get('used_sources', set()) source_key = f"{username}_{filename}" used_sources.add(source_key) task['used_sources'] = used_sources logger.info(f"Marked 0% progress source as used: {source_key}") # Defer orphan cleanup to outside the lock (needs matched_context_lock) if username and filename: old_context_key = _make_context_key(username, filename) _orphaned_download_keys.add(old_context_key) deferred_ops.append(('cleanup_orphan', old_context_key)) # Clear download info since we cancelled it task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) # Reset task state for immediate retry (like error retry) task['status'] = 'searching' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task['status_change_time'] = current_time logger.warning(f"Task {task.get('track_info', {}).get('name', 'Unknown')} reset for 0% retry") # Defer worker restart to outside the lock batch_id = task.get('batch_id') if task_id and batch_id: deferred_ops.append(('restart_worker', task_id, batch_id)) return False elif retry_count < 3: # Wait longer before next retry return False else: track_label = task.get('track_info', {}).get('name', 'Unknown') tried_sources = task.get('used_sources', set()) sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else '' logger.error("Task failed after 3 retry attempts (0% progress timeout)") task['status'] = 'failed' task['error_message'] = f'Download stuck at 0% three times for "{track_label}"{sources_str} — peers may have connection issues' # Clear timers to prevent further retry loops task.pop('queued_start_time', None) task.pop('downloading_start_time', None) # CRITICAL: Notify batch manager so track is added to permanently_failed_tracks batch_id = task.get('batch_id') if batch_id: logger.error(f"[Retry Exhausted] Notifying batch manager of permanent failure for task {task_id}") return True # Signal that we need to call completion outside the lock return False else: # Only reset timers if actual byte progress is being made bytes_transferred = live_info.get('bytesTransferred', 0) if progress >= 1 or bytes_transferred > 0: # Real progress happening, reset timers and retry counts task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task.pop('stuck_retry_count', None) else: # Unknown state with no progress (e.g., "Requested", "Initializing") # Treat like 0% stuck — start/keep the downloading timer running if 'downloading_start_time' not in task: task['downloading_start_time'] = current_time download_time = current_time - task['downloading_start_time'] # Use context-aware timeouts is_streaming_context = task.get('track_info', {}).get('is_album_download', False) timeout_threshold = 15.0 if is_streaming_context else 90.0 if download_time > timeout_threshold: retry_count = task.get('stuck_retry_count', 0) last_retry = task.get('last_retry_time', 0) if retry_count < 3 and (current_time - last_retry) > 30: logger.warning(f"Task stuck in unknown state '{state_str}' with 0 progress for {download_time:.1f}s - retry {retry_count + 1}/3") task['stuck_retry_count'] = retry_count + 1 task['last_retry_time'] = current_time _ti = task.get('track_info') if isinstance(task.get('track_info'), dict) else {} username = task.get('username') or _ti.get('username') filename = task.get('filename') or _ti.get('filename') download_id = task.get('download_id') if username and download_id: deferred_ops.append(('cancel_download', download_id, username)) if username and filename: used_sources = task.get('used_sources', set()) source_key = f"{username}_{filename}" used_sources.add(source_key) task['used_sources'] = used_sources logger.info(f"Marked unknown-state source as used: {source_key}") if username and filename: old_context_key = _make_context_key(username, filename) _orphaned_download_keys.add(old_context_key) deferred_ops.append(('cleanup_orphan', old_context_key)) task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) task['status'] = 'searching' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) task['status_change_time'] = current_time batch_id = task.get('batch_id') if task_id and batch_id: deferred_ops.append(('restart_worker', task_id, batch_id)) return False elif retry_count >= 3: track_label = task.get('track_info', {}).get('name', 'Unknown') tried_sources = task.get('used_sources', set()) sources_str = f' (tried {len(tried_sources)} source{"s" if len(tried_sources) != 1 else ""})' if tried_sources else '' logger.error(f"Task failed after 3 retry attempts (unknown state '{state_str}')") task['status'] = 'failed' task['error_message'] = f'Download stuck in "{state_str}" state 3 times for "{track_label}"{sources_str}' task.pop('queued_start_time', None) task.pop('downloading_start_time', None) batch_id = task.get('batch_id') if batch_id: return True return False return False def _validate_worker_counts(self): """ Validate worker counts to detect and fix ghost workers or orphaned tasks. This prevents the modal from showing wrong worker counts permanently. """ try: batches_needing_workers = [] with tasks_lock: for batch_id in list(self.monitored_batches): if batch_id not in download_batches: continue batch = download_batches[batch_id] reported_active = batch['active_count'] max_concurrent = batch['max_concurrent'] queue = batch.get('queue', []) queue_index = batch.get('queue_index', 0) # Count actually active tasks based on status actually_active = 0 orphaned_tasks = [] # Tasks already processed by _on_download_completed should NOT be counted # as active, even if their status hasn't been updated yet (race condition # between stream processor calling _on_download_completed and # _run_post_processing_worker setting status to 'completed') completed_task_ids = batch.get('_completed_task_ids', set()) for task_id in queue: if task_id in download_tasks: task_status = download_tasks[task_id]['status'] if task_status in ['searching', 'downloading', 'queued', 'post_processing']: if task_id not in completed_task_ids: actually_active += 1 elif task_status in ['failed', 'completed', 'cancelled', 'not_found'] and task_id in queue[queue_index:]: # These are orphaned tasks - they're done but still in active queue orphaned_tasks.append(task_id) # Check for discrepancies if reported_active != actually_active or orphaned_tasks: logger.warning(f"[Worker Validation] Batch {batch_id}: reported={reported_active}, actual={actually_active}, orphaned={len(orphaned_tasks)}") if orphaned_tasks: logger.warning(f"[Worker Validation] Found {len(orphaned_tasks)} orphaned tasks to cleanup") # Fix the active count if it's wrong if reported_active != actually_active: old_count = batch['active_count'] batch['active_count'] = actually_active logger.info(f"[Worker Validation] Fixed active count: {old_count} → {actually_active}") # Defer starting workers to outside the lock if actually_active < max_concurrent and queue_index < len(queue): batches_needing_workers.append(batch_id) # Start replacement workers outside the lock for batch_id in batches_needing_workers: try: logger.info(f"[Worker Validation] Starting replacement workers for {batch_id}") _start_next_batch_of_downloads(batch_id) except Exception as e: logger.error(f"[Worker Validation] Error starting workers for {batch_id}: {e}") except Exception as validation_error: logger.error(f"Error in worker count validation: {validation_error}") # Global download monitor instance download_monitor = WebUIDownloadMonitor() def validate_and_heal_batch_states(): """ Periodic validation and healing of batch states to prevent permanent inconsistencies. This is the server-side equivalent of the frontend's worker count validation. """ try: if globals().get('IS_SHUTTING_DOWN', False): return import time current_time = time.time() # Collect work to do outside the lock batches_needing_workers = [] # batch_ids that need _start_next_batch_of_downloads batches_needing_completion_check = [] # batch_ids that need _check_batch_completion_v2 with tasks_lock: healed_batches = [] batches_to_cleanup = [] for batch_id, batch_data in list(download_batches.items()): active_count = batch_data.get('active_count', 0) queue = batch_data.get('queue', []) phase = batch_data.get('phase', 'unknown') # AUTO-CLEANUP: Remove completed batches after 5 minutes to prevent stale state if phase in ['complete', 'error', 'cancelled']: # Check if batch has a completion timestamp completion_time = batch_data.get('completion_time') if not completion_time: # Set completion time if not set batch_data['completion_time'] = current_time else: # Check if batch has been complete for >5 minutes time_since_completion = current_time - completion_time if time_since_completion > 300: # 5 minutes logger.warning(f"[Auto-Cleanup] Removing stale completed batch {batch_id} (completed {time_since_completion:.0f}s ago)") batches_to_cleanup.append(batch_id) continue # Skip other healing logic for this batch # Count actually active tasks actually_active = 0 orphaned_tasks = [] # Respect _on_download_completed dedup set — don't re-inflate active_count completed_task_ids = batch_data.get('_completed_task_ids', set()) for task_id in queue: if task_id in download_tasks: task_status = download_tasks[task_id]['status'] if task_status in ['searching', 'downloading', 'queued', 'post_processing']: if task_id not in completed_task_ids: actually_active += 1 elif task_status in ['failed', 'completed', 'cancelled', 'not_found']: orphaned_tasks.append(task_id) else: # Task in queue but not in download_tasks dict orphaned_tasks.append(task_id) # Check for inconsistencies if active_count != actually_active: logger.info(f"[Batch Healing] {batch_id}: fixing active count {active_count} → {actually_active}") batch_data['active_count'] = actually_active healed_batches.append(batch_id) # If we freed up slots, defer starting workers to outside the lock if actually_active < batch_data.get('max_concurrent', 3): queue_index = batch_data.get('queue_index', 0) if queue_index < len(queue): batches_needing_workers.append(batch_id) # Clean up orphaned tasks that are blocking progress if orphaned_tasks and phase == 'downloading': logger.warning(f"[Batch Healing] Found {len(orphaned_tasks)} orphaned tasks in active batch {batch_id}") batches_needing_completion_check.append(batch_id) # Cleanup stale batches inside the lock (safe - just dict mutations) for batch_id in batches_to_cleanup: task_ids_to_remove = download_batches[batch_id].get('queue', []) del download_batches[batch_id] # Clean up associated tasks for task_id in task_ids_to_remove: if task_id in download_tasks: del download_tasks[task_id] if batches_to_cleanup: logger.warning(f"[Auto-Cleanup] Removed {len(batches_to_cleanup)} stale completed batches") if healed_batches: logger.info(f"[Batch Healing] Healed {len(healed_batches)} batches: {healed_batches}") # ---- All work below runs WITHOUT tasks_lock held ---- # Start replacement workers for healed batches for batch_id in batches_needing_workers: try: logger.info(f"[Batch Healing] Starting replacement workers for {batch_id}") _start_next_batch_of_downloads(batch_id) except Exception as e: logger.error(f"[Batch Healing] Error starting workers for {batch_id}: {e}") # Trigger completion checks for batches with orphaned tasks for batch_id in batches_needing_completion_check: try: logger.warning("[Batch Healing] Triggering completion check for batch with orphaned tasks") _check_batch_completion_v2(batch_id) except Exception as e: logger.error(f"[Batch Healing] Error checking completion for {batch_id}: {e}") except Exception as healing_error: logger.error(f"[Batch Healing] Error during validation: {healing_error}") # Start periodic batch healing (every 30 seconds) import threading _batch_healing_timer = None _batch_healing_timer_lock = threading.Lock() def _schedule_batch_healing_timer(delay_seconds=30.0): """Schedule the next batch healing cycle.""" global _batch_healing_timer if globals().get('IS_SHUTTING_DOWN', False): return timer = threading.Timer(delay_seconds, start_batch_healing_timer) timer.daemon = True with _batch_healing_timer_lock: _batch_healing_timer = timer timer.start() def _cancel_batch_healing_timer(): """Cancel the current batch healing timer if one exists.""" global _batch_healing_timer with _batch_healing_timer_lock: timer = _batch_healing_timer _batch_healing_timer = None if timer: timer.cancel() def start_batch_healing_timer(): """Start periodic batch state validation and healing""" try: if globals().get('IS_SHUTTING_DOWN', False): return validate_and_heal_batch_states() except Exception as e: logger.error(f"[Batch Healing Timer] {e}") finally: # Schedule next healing cycle _schedule_batch_healing_timer(30.0) # Start the healing timer when the server starts start_batch_healing_timer() # Cleanup handler for Flask shutdown/reload import atexit import signal import sys def cleanup_monitor(): """Clean up background monitor on shutdown""" if download_monitor.monitoring: logger.info("Flask shutdown detected, stopping download monitor...") download_monitor.shutdown() # Give the thread a moment to exit cleanly time.sleep(0.5) # Clean up batch locks to prevent memory leaks try: acquired = tasks_lock.acquire(timeout=1.0) if acquired: try: batch_locks.clear() logger.info("Cleaned up batch locks") finally: tasks_lock.release() else: logger.warning("Skipped batch lock cleanup - tasks_lock busy") except Exception as e: logger.error(f"Error cleaning up batch locks: {e}") # Global shutdown flag def _shutdown_executor(executor, name): """Shut down a ThreadPoolExecutor without waiting for long-running tasks.""" if executor is None: return try: logger.info(f"Shutting down {name}...") executor.shutdown(wait=False, cancel_futures=True) except Exception as e: logger.error(f"Error shutting down {name}: {e}") def _stop_component(component, name, method_names=("stop", "shutdown")): """Call a best-effort stop method on a component if it has one.""" if component is None: return for method_name in method_names: method = getattr(component, method_name, None) if callable(method): try: logger.info(f"Stopping {name}...") method() except Exception as e: logger.error(f"Error stopping {name}: {e}") return def _stop_components_parallel(components): """Stop multiple components concurrently and wait for all stop calls to finish.""" stop_threads = [] for component, name in components: if component is None: continue thread = threading.Thread( target=_stop_component, args=(component, name), name=f"shutdown-{name.replace(' ', '-')}", ) thread.start() stop_threads.append((name, thread)) for _name, thread in stop_threads: thread.join() def _shutdown_runtime_components(): """Best-effort shutdown for timers, monitors, workers, and executors.""" global IS_SHUTTING_DOWN if IS_SHUTTING_DOWN: return IS_SHUTTING_DOWN = True _cancel_batch_healing_timer() cleanup_monitor() _stop_component(web_scan_manager, "web scan manager") _stop_component(automation_engine, "automation engine") # Persist API call history before shutting down worker pools. try: from core.api_call_tracker import api_call_tracker api_call_tracker.save() logger.info("API call history saved") except Exception as e: logger.error(f"Error saving API call history: {e}") # Stop the active DB update worker before tearing down the executor it runs on. # This lets an in-flight update observe should_stop and exit cleanly. _stop_component(db_update_worker, "db update worker") _stop_component(metadata_update_runtime_worker, "metadata update worker") # Stop long-lived worker components in parallel so shutdown waits for the # slowest worker instead of serially burning the timeout for each one. _stop_components_parallel([ (mb_worker, "musicbrainz worker"), (audiodb_worker, "audiodb worker"), (discogs_worker, "discogs worker"), (deezer_worker, "deezer worker"), (spotify_enrichment_worker, "spotify enrichment worker"), (itunes_enrichment_worker, "itunes enrichment worker"), (lastfm_worker, "lastfm worker"), (genius_worker, "genius worker"), (tidal_enrichment_worker, "tidal enrichment worker"), (qobuz_enrichment_worker, "qobuz enrichment worker"), (hydrabase_worker, "hydrabase worker"), (soulid_worker, "soulid worker"), (listening_stats_worker, "listening stats worker"), (repair_worker, "repair worker"), ]) # Shut down executor pools so their worker threads stop keeping the process alive. for executor, name in [ (stream_executor, "stream executor"), (db_update_executor, "db update executor"), (quality_scanner_executor, "quality scanner executor"), (duplicate_cleaner_executor, "duplicate cleaner executor"), (retag_executor, "retag executor"), (sync_executor, "sync executor"), (missing_download_executor, "missing download executor"), (tidal_discovery_executor, "tidal discovery executor"), (deezer_discovery_executor, "deezer discovery executor"), (spotify_public_discovery_executor, "spotify public discovery executor"), (youtube_discovery_executor, "youtube discovery executor"), (beatport_discovery_executor, "beatport discovery executor"), (listenbrainz_discovery_executor, "listenbrainz discovery executor"), (similar_artists_executor, "similar artists executor"), (metadata_update_executor, "metadata update executor"), ]: _shutdown_executor(executor, name) # Give daemon cleanup threads a moment to observe the shutdown flag. time.sleep(0.2) def signal_handler(signum, frame): """Handle SIGINT (Ctrl+C) and SIGTERM""" logger.info(f"Signal {signum} received, cleaning up...") _shutdown_runtime_components() sys.exit(0) # Register cleanup handlers def _atexit_save_history(): try: from core.api_call_tracker import api_call_tracker api_call_tracker.save() except Exception: pass def _atexit_shutdown(): try: _shutdown_runtime_components() except Exception: pass atexit.register(_atexit_save_history) atexit.register(_atexit_shutdown) atexit.register(cleanup_monitor) signal.signal(signal.SIGINT, signal_handler) signal.signal(signal.SIGTERM, signal_handler) def _handle_failed_download(batch_id, task_id, task, task_status): """Handle failed download by triggering retry logic like GUI""" try: with tasks_lock: if task_id not in download_tasks: return retry_count = task.get('retry_count', 0) task['retry_count'] = retry_count + 1 if task['retry_count'] > 2: # Max 3 attempts total (matches GUI) # All retries exhausted, mark as permanently failed logger.error(f"Task {task_id} failed after 3 retry attempts") task_status['status'] = 'failed' task['status'] = 'failed' return # Show retrying status while we process retry task_status['status'] = 'pending' # Will show as pending until retry kicks in logger.error(f"Triggering retry {task['retry_count']}/3 for failed task {task_id}") # Trigger retry with next candidate (matches GUI retry_parallel_download_with_fallback) missing_download_executor.submit(download_monitor._retry_task_with_fallback, batch_id, task_id, task) except Exception as e: logger.error(f"Error handling failed download {task_id}: {e}") task_status['status'] = 'failed' task['status'] = 'failed' def _update_task_status(task_id, new_status): """Helper to update task status and timestamp for timeout tracking""" with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = new_status download_tasks[task_id]['status_change_time'] = time.time() # --- Album Grouping State Management (Ported from GUI) --- # Thread-safe album grouping for consistent naming across tracks album_cache_lock = threading.Lock() album_groups = {} # album_key -> final_album_name album_artists = {} # album_key -> artist_name album_editions = {} # album_key -> "standard" or "deluxe" album_name_cache = {} # album_key -> cached_final_name # Regexes to strip edition suffixes for cache key normalization. # Prevents Navidrome splitting albums when tracks get different MusicBrainz release IDs. import re as _re # Strip parenthetical/bracketed editions: "Album (Deluxe Edition)" → "Album" _EDITION_PAREN_RE = _re.compile( r'\s*[\(\[]' r'[^)\]]*' r'(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' r'limited|bonus|platinum|gold|super\s*deluxe|standard|edition)' r'[^)\]]*' r'[\)\]]', _re.IGNORECASE ) # Strip trailing bare editions (no parens): "Album Deluxe Edition" → "Album" _EDITION_BARE_RE = _re.compile( r'\s+(?:-\s+)?(?:deluxe|expanded|remaster(?:ed)?|anniversary|special|collector|' r'limited|bonus|platinum|gold|super\s*deluxe|standard)' r'(?:\s+(?:edition|version))?\s*$', _re.IGNORECASE ) def _prepare_stream_task(track_data): """ Background streaming task that downloads track to Stream folder and updates global state. Enhanced version with robust error handling matching the GUI StreamingThread. """ loop = None queue_start_time = None actively_downloading = False last_progress_sent = 0.0 try: logger.info(f"Starting stream preparation for: {track_data.get('filename')}") # Update state to loading with stream_lock: stream_state.update({ "status": "loading", "progress": 0, "track_info": track_data, "file_path": None, "error_message": None }) # Get paths download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) project_root = os.path.dirname(os.path.abspath(__file__)) stream_folder = os.path.join(project_root, 'Stream') # Ensure Stream directory exists os.makedirs(stream_folder, exist_ok=True) # Clear any existing files in Stream folder (only one file at a time) for existing_file in glob.glob(os.path.join(stream_folder, '*')): try: if os.path.isfile(existing_file): os.remove(existing_file) elif os.path.isdir(existing_file): shutil.rmtree(existing_file) logger.info(f"Cleared old stream file: {existing_file}") except Exception as e: logger.error(f"Could not remove existing stream file: {e}") # Start the download using the same mechanism as regular downloads loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) try: download_result = loop.run_until_complete(soulseek_client.download( track_data.get('username'), track_data.get('filename'), track_data.get('size', 0) )) if not download_result: with stream_lock: stream_state.update({ "status": "error", "error_message": "Failed to initiate download - uploader may be offline" }) return logger.info("Download initiated for streaming") # Enhanced monitoring with queue timeout detection (matching GUI) max_wait_time = 60 # Increased timeout poll_interval = 1.5 # More frequent polling queue_timeout = 15 # Queue timeout like GUI wait_count = 0 while wait_count * poll_interval < max_wait_time: wait_count += 1 # Check download progress via orchestrator (works for Soulseek and YouTube) api_progress = None download_state = None download_status = None try: # Use orchestrator's get_all_downloads() which works for both sources all_downloads = loop.run_until_complete(soulseek_client.get_all_downloads()) download_status = _find_streaming_download_in_all_downloads(all_downloads, track_data) if download_status: api_progress = download_status.get('percentComplete', 0) download_state = download_status.get('state', '').lower() original_state = download_status.get('state', '') logger.info(f"API Download - State: {original_state}, Progress: {api_progress:.1f}%") # Track queue state timing (matching GUI logic) is_queued = ('queued' in download_state or 'initializing' in download_state) is_downloading = ('inprogress' in download_state or 'transferring' in download_state) # Verify bytes match before trusting state/progress _stream_expected = download_status.get('size', 0) _stream_transferred = download_status.get('bytesTransferred', 0) _bytes_ok = _stream_expected <= 0 or _stream_transferred >= _stream_expected is_completed = ('succeeded' in download_state or api_progress >= 100) and _bytes_ok # Handle queue state timing if is_queued and queue_start_time is None: queue_start_time = time.time() logger.info(f"Download entered queue state: {original_state}") with stream_lock: stream_state["status"] = "queued" elif is_downloading and not actively_downloading: actively_downloading = True queue_start_time = None # Reset queue timer logger.info(f"Download started actively downloading: {original_state}") with stream_lock: stream_state["status"] = "loading" # Check for queue timeout (matching GUI) if is_queued and queue_start_time: queue_elapsed = time.time() - queue_start_time if queue_elapsed > queue_timeout: logger.error(f"⏰ Queue timeout after {queue_elapsed:.1f}s - download stuck in queue") with stream_lock: stream_state.update({ "status": "error", "error_message": "Queue timeout - uploader not responding. Try another source." }) return # Update progress with stream_lock: if api_progress != last_progress_sent: stream_state["progress"] = api_progress last_progress_sent = api_progress # Check if download is complete if is_completed: logger.info(f"Download completed via API status: {original_state}") # Wait for file to stabilise on disk before moving found_file = _find_downloaded_file(download_path, track_data) if found_file: _prev_sz = -1 for _sc in range(4): try: _cur_sz = os.path.getsize(found_file) except OSError: _cur_sz = -1 if _cur_sz == _prev_sz and _cur_sz > 0: break _prev_sz = _cur_sz time.sleep(1.5) # Re-find in case it wasn't found on first try if not found_file: found_file = _find_downloaded_file(download_path, track_data) # Retry file search a few times (matching GUI logic) retry_attempts = 5 for attempt in range(retry_attempts): if found_file: break logger.warning(f"File not found yet, attempt {attempt + 1}/{retry_attempts}") time.sleep(1) found_file = _find_downloaded_file(download_path, track_data) if found_file: logger.debug(f"Found downloaded file: {found_file}") # Move file to Stream folder original_filename = extract_filename(found_file) stream_path = os.path.join(stream_folder, original_filename) try: shutil.move(found_file, stream_path) logger.debug(f"Moved file to stream folder: {stream_path}") # Clean up empty directories (matching GUI) _cleanup_empty_directories(download_path, found_file) # Update state to ready with stream_lock: stream_state.update({ "status": "ready", "progress": 100, "file_path": stream_path }) # Clean up download from slskd API try: download_id = download_status.get('id', '') if download_id and track_data.get('username'): success = loop.run_until_complete( soulseek_client.signal_download_completion( download_id, track_data.get('username'), remove=True) ) if success: logger.debug(f"Cleaned up download {download_id} from API") except Exception as e: logger.error(f"Error cleaning up download: {e}") logger.info(f"Stream file ready for playback: {stream_path}") return # Success! except Exception as e: logger.error(f"Error moving file to stream folder: {e}") with stream_lock: stream_state.update({ "status": "error", "error_message": f"Failed to prepare stream file: {e}" }) return else: logger.error("Could not find downloaded file after completion") with stream_lock: stream_state.update({ "status": "error", "error_message": "Download completed but file not found" }) return else: # No transfer found in API - may still be initializing logger.debug(f"No transfer found in API yet... (elapsed: {wait_count * poll_interval}s)") except Exception as e: logger.error(f"Error checking download progress: {e}") # Continue to next iteration if API call fails # Wait before next poll time.sleep(poll_interval) # If we get here, download timed out logger.warning(f"Download timed out after {max_wait_time}s") with stream_lock: stream_state.update({ "status": "error", "error_message": "Download timed out - try a different source" }) except asyncio.CancelledError: logger.warning("Stream task cancelled") with stream_lock: stream_state.update({ "status": "stopped", "error_message": None }) finally: if loop: try: # Clean up any pending tasks pending = asyncio.all_tasks(loop) if pending: loop.run_until_complete(asyncio.gather(*pending, return_exceptions=True)) loop.close() except Exception as e: logger.error(f"Error cleaning up streaming event loop: {e}") except Exception as e: logger.error(f"Stream preparation failed: {e}") with stream_lock: stream_state.update({ "status": "error", "error_message": f"Streaming error: {str(e)}" }) def _find_streaming_download_in_all_downloads(all_downloads, track_data): """ Find streaming download in DownloadStatus list (works for Soulseek, YouTube, and Tidal). Replaces the old _find_streaming_download_in_transfers function. """ try: if not all_downloads: return None # Look for our specific file by filename and username target_filename = extract_filename(track_data.get('filename', '')) target_username = track_data.get('username', '') for download in all_downloads: download_filename = extract_filename(download.filename) download_username = download.username if (download_filename == target_filename and download_username == target_username): # Convert DownloadStatus to dict format expected by caller return { 'percentComplete': download.progress, 'state': download.state, 'size': download.size, 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, } return None except Exception as e: logger.error(f"Error finding streaming download: {e}") return None def _find_downloaded_file(download_path, track_data): """Find the downloaded audio file in the downloads directory tree (works for Soulseek, YouTube, and Tidal)""" # Ensure path is accessible in Docker (handles E:/ -> /host/mnt/e/) download_path = docker_resolve_path(download_path) audio_extensions = {'.mp3', '.flac', '.ogg', '.aac', '.wma', '.wav', '.m4a'} target_filename = extract_filename(track_data.get('filename', '')) # YOUTUBE/TIDAL/QOBUZ/HIFI SUPPORT: Handle encoded filename format "id||title" # The file on disk will be "title.ext", not "id||title" is_youtube = track_data.get('username') == 'youtube' is_tidal = track_data.get('username') == 'tidal' is_qobuz = track_data.get('username') == 'qobuz' is_hifi = track_data.get('username') == 'hifi' is_streaming_source = is_youtube or is_tidal or is_qobuz or is_hifi target_filename_youtube = None if is_streaming_source and '||' in target_filename: _, title = target_filename.split('||', 1) if is_tidal or is_qobuz or is_hifi: # Tidal/Qobuz/HiFi files can be flac or m4a — match any audio extension safe_title = re.sub(r'[<>:"/\\|?*]', '_', title) target_filename_youtube = safe_title # Extension-less for flexible matching source_name = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else 'Tidal') logger.debug(f"[{source_name} Stream] Looking for file starting with: {target_filename_youtube}") else: # yt-dlp will create "Title.mp3" from "Title" target_filename_youtube = f"{title}.mp3" logger.debug(f"[YouTube Stream] Looking for file: {target_filename_youtube}") elif is_streaming_source: # Fallback: if streaming source but no encoded format, use as-is target_filename_youtube = target_filename logger.debug(f"[Stream] Using direct filename: {target_filename_youtube}") try: # Walk through the downloads directory to find the file best_match = None best_similarity = 0.0 for root, _dirs, files in os.walk(download_path): for file in files: # Skip non-audio files if os.path.splitext(file)[1].lower() not in audio_extensions: continue file_path = os.path.join(root, file) # Skip empty files try: if os.path.getsize(file_path) < 1024: # At least 1KB continue except: continue # Check if this is our target file if is_streaming_source and target_filename_youtube: # For YouTube/Tidal, use fuzzy matching (case-insensitive, flexible) from difflib import SequenceMatcher # For Tidal, compare without extension (file could be .flac or .m4a) compare_target = target_filename_youtube.lower() compare_file = file.lower() if is_tidal or is_qobuz or is_hifi: compare_file = os.path.splitext(compare_file)[0] similarity = SequenceMatcher(None, compare_file, compare_target).ratio() source_label = 'HiFi' if is_hifi else ('Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube')) logger.debug(f"[{source_label} Stream] Comparing: '{file}' vs '{target_filename_youtube}' = {similarity:.2f}") # Keep track of best match if similarity > best_similarity: best_similarity = similarity best_match = file_path # If we have a very good match (95%+), use it immediately if similarity >= 0.95: logger.debug(f"Found excellent match for streaming file: {file_path}") return file_path else: # For Soulseek, exact match if file == target_filename: logger.debug(f"Found streaming file: {file_path}") return file_path # For YouTube/Tidal, if we found a good enough match (80%+), use it if is_streaming_source and best_match and best_similarity >= 0.80: source_label = 'Qobuz' if is_qobuz else ('Tidal' if is_tidal else 'YouTube') logger.debug(f"Found good match ({best_similarity:.2f}) for {source_label} streaming file: {best_match}") return best_match logger.error(f"Could not find downloaded file: {target_filename}") if is_streaming_source: logger.debug(f" Looking for: {target_filename_youtube}") logger.debug(f" Best similarity: {best_similarity:.2f}") return None except Exception as e: logger.error(f"Error searching for downloaded file: {e}") return None # --- Refactored Logic from GUI Threads --- # This logic is extracted from the database update worker to be used directly by Flask. # ── Settings Connection Status Registry ── # Maps each service shown in Settings → Connections to its config requirements. # Used by _is_service_configured() to drive the green/yellow header gradient. # # Registry entry shapes: # {'required': [keys]} — green if all keys populated in config_manager.get(service) # {'always': True} — always green (no credentials required, e.g. default-storefront iTunes) # {'custom': callable} — callable(service_name) -> bool, for services with non-field checks (e.g. token file) # {'any_of': [[keys_a], [keys_b]]} — green if any one group's keys are all populated (e.g. Qobuz: email+password OR token) SERVICE_CONFIG_REGISTRY = { 'spotify': {'required': ['client_id', 'client_secret']}, 'itunes': {'always': True}, # default storefront works anon 'deezer': {'always': True}, # anon search works, premium ARL is optional 'discogs': {'required': ['token']}, 'tidal': {'custom': lambda _svc: _tidal_has_auth_token()}, 'qobuz': {'any_of': [['email', 'password'], ['token'], ['user_auth_token']]}, 'lastfm': {'required': ['api_key']}, 'genius': {'required': ['access_token']}, 'acoustid': {'required': ['api_key']}, 'listenbrainz': {'required': ['token']}, 'hydrabase': {'required': ['url', 'api_key']}, # Soulseek (slskd) needs a base URL. Used by the search source picker # to dim Soulseek and redirect to Settings when the user has no slskd # configured — clicking it would otherwise fire searches that always # fail. URL field lives on Settings → Downloads, gated behind the # download-source-mode dropdown. 'soulseek': {'required': ['slskd_url']}, } def _tidal_has_auth_token() -> bool: """Check if Tidal has a cached OAuth token. Tidal uses a token file, not config fields.""" try: return bool(tidal_client and tidal_client.is_authenticated()) except Exception: return False def _is_service_configured(service: str) -> bool: """Return True if the user has provided the required credentials for this service. Drives the green/yellow header gradient on the Connections tab. """ entry = SERVICE_CONFIG_REGISTRY.get(service) if not entry: return False if entry.get('always'): return True if 'custom' in entry: try: return bool(entry['custom'](service)) except Exception: return False service_config = config_manager.get(service, {}) or {} if 'required' in entry: return all(bool(service_config.get(key)) for key in entry['required']) if 'any_of' in entry: for key_group in entry['any_of']: if all(bool(service_config.get(key)) for key in key_group): return True return False return False def run_service_test(service, test_config): """ Performs the actual connection test for a given service. This logic is adapted from your ServiceTestThread. It temporarily modifies the config, runs the test, then restores the config. """ original_config = {} try: # 1. Save original config for the specific service original_config = config_manager.get(service, {}) # 2. Temporarily set the new config for the test (with Docker URL resolution) for key, value in test_config.items(): # Apply Docker URL resolution for URL/URI fields if isinstance(value, str) and ('url' in key.lower() or 'uri' in key.lower()): value = docker_resolve_url(value) config_manager.set(f"{service}.{key}", value) # 3. Run the test with the temporary config if service == "spotify": temp_client = SpotifyClient() # Check if Spotify credentials are configured spotify_config = config_manager.get('spotify', {}) spotify_configured = bool(spotify_config.get('client_id') and spotify_config.get('client_secret')) if temp_client.is_authenticated(): # Determine which source is active if temp_client.is_spotify_authenticated(): return True, "Spotify connection successful!" else: # Using fallback metadata source fb_src = _get_metadata_fallback_source() fallback_name = 'Deezer' if fb_src == 'deezer' else 'Discogs' if fb_src == 'discogs' else 'iTunes' if spotify_configured: return True, f"{fallback_name} connection successful! (Spotify configured but not authenticated)" else: return True, f"{fallback_name} connection successful! (Spotify not configured)" else: return False, "Music service authentication failed. Check credentials and complete OAuth flow in browser if prompted." elif service == "tidal": temp_client = TidalClient() if temp_client.is_authenticated(): user_info = temp_client.get_user_info() username = user_info.get('display_name', 'Tidal User') if user_info else 'Tidal User' return True, f"Tidal connection successful! Connected as: {username}" else: return False, "Tidal authentication failed. Please use the 'Authenticate' button and complete the flow in your browser." elif service == "plex": temp_client = PlexClient() if temp_client.is_connected(): return True, f"Successfully connected to Plex server: {temp_client.server.friendlyName}" else: return False, "Could not connect to Plex. Check URL and Token." elif service == "jellyfin": temp_client = JellyfinClient() if temp_client.is_connected(): # FIX: Check if server_info exists before accessing it. server_name = "Unknown Server" if hasattr(temp_client, 'server_info') and temp_client.server_info: server_name = temp_client.server_info.get('ServerName', 'Unknown Server') return True, f"Successfully connected to Jellyfin server: {server_name}" else: return False, "Could not connect to Jellyfin. Check URL and API Key." elif service == "navidrome": # Test Navidrome connection using Subsonic API base_url = test_config.get('base_url', '') username = test_config.get('username', '') password = test_config.get('password', '') if not all([base_url, username, password]): return False, "Missing Navidrome URL, username, or password." try: import hashlib import random import string # Generate salt and token for Subsonic API authentication salt = ''.join(random.choices(string.ascii_letters + string.digits, k=6)) token = hashlib.md5((password + salt).encode()).hexdigest() # Test ping endpoint url = f"{base_url.rstrip('/')}/rest/ping" response = requests.get(url, params={ 'u': username, 't': token, 's': salt, 'v': '1.16.1', 'c': 'soulsync', 'f': 'json' }, timeout=5) if response.status_code == 200: data = response.json() if data.get('subsonic-response', {}).get('status') == 'ok': server_version = data.get('subsonic-response', {}).get('version', 'Unknown') return True, f"Successfully connected to Navidrome server (v{server_version})" else: error = data.get('subsonic-response', {}).get('error', {}) return False, f"Navidrome authentication failed: {error.get('message', 'Unknown error')}" else: return False, f"Could not connect to Navidrome server (HTTP {response.status_code})" except Exception as e: return False, f"Navidrome connection error: {str(e)}" elif service == "soulsync": transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) if os.path.isdir(transfer_path): # Quick check — count a few audio files to confirm it's a music folder audio_exts = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav'} count = 0 found_enough = False for _root, _dirs, files in os.walk(transfer_path): for f in files: if os.path.splitext(f)[1].lower() in audio_exts: count += 1 if count >= 10: found_enough = True break if found_enough: break return True, f"SoulSync standalone ready! Output folder: {transfer_path}" + (f" ({count}+ audio files)" if count > 0 else " (empty)") else: return False, f"Output folder not found: {transfer_path}" elif service == "soulseek": if soulseek_client is None: return False, "Download orchestrator failed to initialize. Check server logs for startup errors." # Test the orchestrator's configured download source (not just Soulseek) download_mode = config_manager.get('download_source.mode', 'hybrid') if run_async(soulseek_client.check_connection()): # Success message based on active mode mode_messages = { 'soulseek': "Successfully connected to Soulseek network via slskd.", 'youtube': "YouTube download source ready.", 'tidal': "Tidal download source ready.", 'qobuz': "Qobuz download source ready.", 'hifi': "HiFi download source ready.", 'hybrid': "Download sources ready (Hybrid mode)." } message = mode_messages.get(download_mode, "Download source connected.") return True, message else: # Failure message based on active mode mode_errors = { 'soulseek': "slskd is not connected to the Soulseek network. Check slskd status and credentials.", 'youtube': "YouTube download source not available.", 'tidal': "Tidal download source not available. Check authentication.", 'qobuz': "Qobuz download source not available. Check authentication.", 'hifi': "HiFi download source not available. Public API instances may be down.", 'hybrid': "Could not connect to download sources. Check configuration." } error = mode_errors.get(download_mode, "Download source connection failed.") return False, error elif service == "listenbrainz": token = test_config.get('token', '') if not token: return False, "Missing ListenBrainz user token." try: # Test ListenBrainz API by validating the token custom_base = test_config.get('base_url', '').rstrip('/') if custom_base: if not custom_base.endswith('/1'): custom_base += '/1' lb_api_base = custom_base else: lb_api_base = "https://api.listenbrainz.org/1" url = f"{lb_api_base}/validate-token" headers = { 'Authorization': f'Token {token}' } response = requests.get(url, headers=headers, timeout=5) if response.status_code == 200: data = response.json() if data.get('valid'): username = data.get('user_name', 'Unknown') return True, f"Successfully connected to ListenBrainz! Connected as: {username}" else: return False, "Invalid ListenBrainz token." elif response.status_code == 401: return False, "Invalid ListenBrainz token (unauthorized)." else: return False, f"Could not connect to ListenBrainz (HTTP {response.status_code})" except Exception as e: return False, f"ListenBrainz connection error: {str(e)}" elif service == "acoustid": api_key = test_config.get('api_key', '') if not api_key: return False, "Missing AcoustID API key." try: from core.acoustid_client import AcoustIDClient, CHROMAPRINT_AVAILABLE, ACOUSTID_AVAILABLE, FPCALC_PATH if not ACOUSTID_AVAILABLE: return False, "pyacoustid library not installed. Run: pip install pyacoustid" client = AcoustIDClient() # Override the cached API key with the test config key client._api_key = api_key # Check chromaprint/fpcalc availability if CHROMAPRINT_AVAILABLE and FPCALC_PATH: fingerprint_status = f"fpcalc ready: {FPCALC_PATH}" elif CHROMAPRINT_AVAILABLE: fingerprint_status = "Fingerprint backend available" else: fingerprint_status = "fpcalc not found (will auto-download on first use)" # Validate API key with test request success, message = client.test_api_key() if success: return True, f"AcoustID API key is valid! {fingerprint_status}" else: return False, f"{message}. {fingerprint_status}" except Exception as e: return False, f"AcoustID test error: {str(e)}" elif service == "lastfm": api_key = test_config.get('api_key', '') if not api_key: return False, "Missing Last.fm API key." try: from core.lastfm_client import LastFMClient client = LastFMClient(api_key=api_key) if client.validate_api_key(): return True, "Successfully connected to Last.fm!" else: return False, "Invalid Last.fm API key." except Exception as e: return False, f"Last.fm connection error: {str(e)}" elif service == "genius": access_token = test_config.get('access_token', '') if not access_token: return False, "Missing Genius access token." try: from core.genius_client import GeniusClient client = GeniusClient(access_token=access_token) if client.validate_token(): return True, "Successfully connected to Genius!" else: return False, "Invalid Genius access token." except Exception as e: return False, f"Genius connection error: {str(e)}" elif service == "lidarr" or service == "lidarr_download": url = config_manager.get('lidarr_download.url', '') api_key = config_manager.get('lidarr_download.api_key', '') if not url or not api_key: return False, "Lidarr URL and API key are required." try: import requests as _req resp = _req.get(f"{url.rstrip('/')}/api/v1/system/status", headers={'X-Api-Key': api_key}, timeout=10) if resp.ok: version = resp.json().get('version', '?') return True, f"Connected to Lidarr v{version}" return False, f"Lidarr returned HTTP {resp.status_code}" except Exception as e: return False, f"Lidarr connection error: {str(e)}" elif service == "itunes": # Public API — just confirm we can reach it with a cheap search try: storefront = config_manager.get('itunes.storefront', 'US') or 'US' resp = requests.get( 'https://itunes.apple.com/search', params={'term': 'beatles', 'limit': 1, 'country': storefront, 'media': 'music'}, timeout=5, ) if resp.ok and resp.json().get('resultCount', 0) >= 0: return True, f"iTunes Search API reachable (storefront: {storefront})" return False, f"iTunes returned HTTP {resp.status_code}" except Exception as e: return False, f"iTunes connection error: {str(e)}" elif service == "deezer": # Public API — anon search works without credentials try: resp = requests.get( 'https://api.deezer.com/search/artist', params={'q': 'beatles', 'limit': 1}, timeout=5, ) if resp.ok and isinstance(resp.json(), dict): return True, "Deezer Public API reachable" return False, f"Deezer returned HTTP {resp.status_code}" except Exception as e: return False, f"Deezer connection error: {str(e)}" elif service == "discogs": token = test_config.get('token', '') or config_manager.get('discogs.token', '') if not token: return False, "Missing Discogs personal token." try: resp = requests.get( 'https://api.discogs.com/database/search', params={'q': 'beatles', 'per_page': 1}, headers={'Authorization': f'Discogs token={token}', 'User-Agent': 'SoulSync/1.0'}, timeout=10, ) if resp.ok: return True, "Discogs API reachable with provided token" if resp.status_code == 401: return False, "Discogs token rejected (HTTP 401)" return False, f"Discogs returned HTTP {resp.status_code}" except Exception as e: return False, f"Discogs connection error: {str(e)}" elif service == "qobuz": try: if qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.is_authenticated(): return True, "Qobuz client authenticated" return False, "Qobuz not authenticated. Provide email/password or user auth token." except Exception as e: return False, f"Qobuz connection error: {str(e)}" elif service == "hydrabase": try: if hydrabase_client and hydrabase_client.is_connected(): return True, "Hydrabase connected" return False, "Hydrabase not connected. Configure URL + API key and click Connect." except Exception as e: return False, f"Hydrabase connection error: {str(e)}" return False, "Unknown service." except AttributeError as e: # This specifically catches the error you reported for Jellyfin if "'JellyfinClient' object has no attribute 'server_info'" in str(e): return False, "Connection failed. Please check your Jellyfin URL and API Key." else: return False, f"An unexpected error occurred: {e}" except Exception as e: import traceback traceback.print_exc() return False, str(e) finally: # 4. CRITICAL: Restore the original config if original_config: for key, value in original_config.items(): config_manager.set(f"{service}.{key}", value) logger.debug(f"Restored original config for '{service}' after test.") def run_detection(server_type): """ Performs comprehensive network detection for a given server type (plex, jellyfin, slskd). This implements the same scanning logic as the GUI's detection threads. """ logger.info(f"Running comprehensive detection for {server_type}...") def get_network_info(): """Get comprehensive network information with subnet detection""" try: # Get local IP using socket method s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) local_ip = s.getsockname()[0] s.close() # Try to get actual subnet mask try: if platform.system() == "Windows": # Windows: Use netsh to get subnet info result = subprocess.run(['netsh', 'interface', 'ip', 'show', 'config'], capture_output=True, text=True, timeout=3) # Parse output for subnet mask (simplified) subnet_mask = "255.255.255.0" # Default fallback else: # Linux/Mac: Try to parse network interfaces result = subprocess.run(['ip', 'route', 'show'], capture_output=True, text=True, timeout=3) subnet_mask = "255.255.255.0" # Default fallback except: subnet_mask = "255.255.255.0" # Default /24 # Calculate network range network = ipaddress.IPv4Network(f"{local_ip}/{subnet_mask}", strict=False) return str(network.network_address), str(network.netmask), local_ip, network except Exception as e: # Fallback to original method s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) s.connect(("8.8.8.8", 80)) local_ip = s.getsockname()[0] s.close() # Default to /24 network network = ipaddress.IPv4Network(f"{local_ip}/24", strict=False) return str(network.network_address), "255.255.255.0", local_ip, network def test_plex_server(ip, port=32400): """Test if a Plex server is running at the given IP and port""" try: url = f"http://{ip}:{port}/web/index.html" response = requests.get(url, timeout=2, allow_redirects=True) # Check for Plex-specific indicators if response.status_code == 200: # Check if it's actually Plex if 'plex' in response.text.lower() or 'X-Plex' in str(response.headers): return f"http://{ip}:{port}" # Also try the API endpoint api_url = f"http://{ip}:{port}/identity" api_response = requests.get(api_url, timeout=1) if api_response.status_code == 200 and 'MediaContainer' in api_response.text: return f"http://{ip}:{port}" except: pass return None def test_jellyfin_server(ip, port=8096): """Test if a Jellyfin server is running at the given IP and port""" try: # Try the system info endpoint first url = f"http://{ip}:{port}/System/Info" response = requests.get(url, timeout=2, allow_redirects=True) if response.status_code == 200: # Check if response contains Jellyfin-specific content if 'jellyfin' in response.text.lower() or 'ServerName' in response.text: return f"http://{ip}:{port}" # Also try the web interface web_url = f"http://{ip}:{port}/web/index.html" web_response = requests.get(web_url, timeout=1) if web_response.status_code == 200 and 'jellyfin' in web_response.text.lower(): return f"http://{ip}:{port}" except: pass return None def test_slskd_server(ip, port=5030): """Test if a slskd server is running at the given IP and port""" try: # slskd specific API endpoint url = f"http://{ip}:{port}/api/v0/session" response = requests.get(url, timeout=2) # slskd returns 401 when not authenticated, which is still a valid response if response.status_code in [200, 401]: return f"http://{ip}:{port}" except: pass return None def test_navidrome_server(ip, port=4533): """Test if a Navidrome server is running at the given IP and port""" try: # Try Navidrome's ping endpoint (part of Subsonic API) url = f"http://{ip}:{port}/rest/ping" response = requests.get(url, timeout=2, params={ 'u': 'test', # Dummy username for ping test 'v': '1.16.1', # API version 'c': 'soulsync', # Client name 'f': 'json' # Response format }) # Navidrome should respond even with invalid credentials for ping if response.status_code in [200, 401, 403]: try: data = response.json() # Check for Subsonic/Navidrome API response structure if 'subsonic-response' in data: return f"http://{ip}:{port}" except: pass # Also try the web interface web_url = f"http://{ip}:{port}/" web_response = requests.get(web_url, timeout=2) if web_response.status_code == 200 and 'navidrome' in web_response.text.lower(): return f"http://{ip}:{port}" except: pass return None try: network_addr, netmask, local_ip, network = get_network_info() # Select the appropriate test function test_functions = { 'plex': test_plex_server, 'jellyfin': test_jellyfin_server, 'navidrome': test_navidrome_server, 'slskd': test_slskd_server } test_func = test_functions.get(server_type) if not test_func: return None # Priority 1: Test localhost first logger.debug(f"Testing localhost for {server_type}...") localhost_result = test_func("localhost") if localhost_result: logger.info(f"Found {server_type} at localhost!") return localhost_result # Priority 1.5: In Docker, try Docker host IP import os if os.path.exists('/.dockerenv'): logger.info(f"Docker detected, testing Docker host for {server_type}...") try: # Try host.docker.internal (Windows/Mac) host_result = test_func("host.docker.internal") if host_result: logger.info(f"Found {server_type} at Docker host!") return host_result.replace("host.docker.internal", "localhost") # Convert back to localhost for config # Try Docker bridge gateway (Linux) gateway_result = test_func("172.17.0.1") if gateway_result: logger.info(f"Found {server_type} at Docker gateway!") return gateway_result.replace("172.17.0.1", "localhost") # Convert back to localhost for config except Exception as e: logger.error(f"Docker host detection failed: {e}") # Priority 2: Test local IP logger.debug(f"Testing local IP {local_ip} for {server_type}...") local_result = test_func(local_ip) if local_result: logger.info(f"Found {server_type} at {local_ip}!") return local_result # Priority 3: Test common IPs (router gateway, etc.) common_ips = [ local_ip.rsplit('.', 1)[0] + '.1', # Typical gateway local_ip.rsplit('.', 1)[0] + '.2', # Alternative gateway local_ip.rsplit('.', 1)[0] + '.100', # Common static IP ] logger.debug(f"Testing common IPs for {server_type}...") for ip in common_ips: logger.info(f" Checking {ip}...") result = test_func(ip) if result: logger.info(f"Found {server_type} at {ip}!") return result # Priority 4: Scan the network range (limited to reasonable size) network_hosts = list(network.hosts()) if len(network_hosts) > 50: # Limit scan to reasonable size for performance step = max(1, len(network_hosts) // 50) network_hosts = network_hosts[::step] logger.debug(f"Scanning network range for {server_type} ({len(network_hosts)} hosts)...") # Use ThreadPoolExecutor for concurrent scanning (limited for web context) with ThreadPoolExecutor(max_workers=5) as executor: # Submit all tasks future_to_ip = {executor.submit(test_func, str(ip)): str(ip) for ip in network_hosts} try: for future in as_completed(future_to_ip): ip = future_to_ip[future] try: result = future.result() if result: logger.info(f"Found {server_type} at {ip}!") # Cancel all pending futures before returning for f in future_to_ip: if not f.done(): f.cancel() return result except Exception as e: logger.error(f"Error testing {ip}: {e}") continue except Exception as e: logger.error(f"Error in concurrent scanning: {e}") logger.warning(f"No {server_type} server found on network") return None except Exception as e: logger.error(f"Error during {server_type} detection: {e}") return None # --- Web UI Routes --- @app.route('/') def index(): return render_template('index.html') @app.route('/sw.js') def service_worker(): """Serve sw.js from root scope so the service worker can intercept fetches site-wide. A service worker only controls URLs at or below its own served path — `/static/sw.js` would scope to `/static/*` only. Serving from `/sw.js` (with the file living under static/) grants full-site scope without needing the Service-Worker-Allowed header dance. Cache-Control: no-cache so deploys that change the SW propagate on the next page load instead of being pinned by the 1-year static cache the rest of /static/ uses. """ response = send_from_directory(app.static_folder, 'sw.js', mimetype='application/javascript') response.headers['Cache-Control'] = 'no-cache' return response @app.route('/') def spa_catch_all(page): # Serve index.html for client-side routes; let Flask handle real routes first. if page.startswith(('api/', 'static/', 'auth/', 'callback', 'deezer/', 'tidal/', 'status')): abort(404) return render_template('index.html') # --- API Endpoints --- # Tracks cumulative item-processed totals over time for windowed counting. # Each entry: (timestamp, cumulative_total). Polled every ~5s, 24h = ~17280 entries. def _get_windowed_calls(key, current_total): """Record current cumulative total and return (calls_1h, calls_24h). Deque stores (timestamp, cumulative_total) in chronological order. To get calls in a window: current_total minus the oldest total within that window.""" now = time.time() history = _enrichment_activity_log.setdefault(key, collections.deque(maxlen=17300)) history.append((now, current_total)) cutoff_1h = now - 3600 cutoff_24h = now - 86400 # Forward scan: first entry with ts >= cutoff is the oldest in that window oldest_1h_total = current_total oldest_24h_total = current_total found_24h = False for ts, total in history: if not found_24h and ts >= cutoff_24h: oldest_24h_total = total found_24h = True if ts >= cutoff_1h: oldest_1h_total = total break return max(0, current_total - oldest_1h_total), max(0, current_total - oldest_24h_total) def _get_enrichment_status(): """Get lightweight status for all enrichment services (no DB queries). Reads worker properties directly to avoid expensive get_stats() calls.""" services = {} # Worker-based enrichment services: (key, display_name, worker_var) workers_info = [ ('musicbrainz', 'MusicBrainz', lambda: mb_worker), ('spotify_enrichment', 'Spotify', lambda: spotify_enrichment_worker), ('itunes_enrichment', 'iTunes', lambda: itunes_enrichment_worker), ('deezer_enrichment', 'Deezer', lambda: deezer_worker), ('tidal_enrichment', 'Tidal', lambda: tidal_enrichment_worker), ('qobuz_enrichment', 'Qobuz', lambda: qobuz_enrichment_worker), ('lastfm', 'Last.fm', lambda: lastfm_worker), ('genius', 'Genius', lambda: genius_worker), ('audiodb', 'AudioDB', lambda: audiodb_worker), ('discogs', 'Discogs', lambda: discogs_worker), ] # Config-based "configured" checks for services that need API keys/credentials configured_checks = { 'spotify_enrichment': lambda: bool(config_manager.get('spotify.client_id') and config_manager.get('spotify.client_secret')), 'tidal_enrichment': lambda: bool(tidal_client and getattr(tidal_client, 'access_token', None)), 'qobuz_enrichment': lambda: bool(qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.user_auth_token), 'lastfm': lambda: bool(config_manager.get('lastfm.api_key', '')), 'genius': lambda: bool(config_manager.get('genius.access_token', '')), } for key, name, get_worker in workers_info: worker = get_worker() if worker is not None: is_alive = worker.thread is not None and worker.thread.is_alive() try: configured = configured_checks.get(key, lambda: True)() except Exception: configured = False # Compute windowed API call counts from cumulative stats stats = worker.stats total_processed = stats.get('matched', 0) + stats.get('not_found', 0) + stats.get('errors', 0) calls_1h, calls_24h = _get_windowed_calls(key, total_processed) # Debounced idle detection — only report idle after 5s of no current_item has_item = getattr(worker, 'current_item', None) is not None if has_item: _idle_since.pop(key, None) is_idle = False else: if key not in _idle_since: _idle_since[key] = time.time() is_idle = (time.time() - _idle_since[key]) >= _IDLE_GRACE_SECONDS svc_data = { 'name': name, 'configured': configured, 'running': worker.running and is_alive and not worker.paused, 'paused': worker.paused, 'idle': is_alive and not worker.paused and is_idle, 'calls_1h': calls_1h, 'calls_24h': calls_24h, } # Spotify-specific: include daily budget info if key == 'spotify_enrichment': try: svc_data['daily_budget'] = worker._get_daily_budget_info() except Exception: pass services[key] = svc_data else: services[key] = { 'name': name, 'configured': False, 'running': False, 'paused': False, 'idle': False, 'calls_1h': 0, 'calls_24h': 0, } # Non-worker services (configured status only) services['acoustid'] = { 'name': 'AcoustID', 'configured': bool(config_manager.get('acoustid.api_key', '')), } services['listenbrainz'] = { 'name': 'ListenBrainz', 'configured': bool(config_manager.get('listenbrainz.token', '')), } return services # Status check caching to reduce unnecessary API calls @app.route('/status') def get_status(): if not config_manager: return jsonify({"error": "Server configuration manager is not initialized."}), 500 try: import time current_time = time.time() active_server = config_manager.get_active_media_server() # Test Spotify - with caching to avoid excessive API calls if current_time - _status_cache_timestamps['spotify'] > STATUS_CACHE_TTL: # Guard against spotify_client being None (partial init) is_rate_limited = spotify_client.is_rate_limited() if spotify_client else False rate_limit_info = spotify_client.get_rate_limit_info() if (spotify_client and is_rate_limited) else None cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0 # Read configured source once — no auth validation here, we do that explicitly below configured_source = config_manager.get('metadata.fallback_source', 'deezer') or 'deezer' if is_rate_limited or cooldown_remaining > 0: # During rate limit or post-ban cooldown, skip the auth probe entirely. # Probing Spotify here would reset the rate limit timer. music_source = 'deezer' if configured_source == 'spotify' else configured_source spotify_response_time = 0 else: spotify_start = time.time() spotify_connected = spotify_client.is_spotify_authenticated() if spotify_client else False spotify_response_time = (time.time() - spotify_start) * 1000 # Use configured source; fall back to deezer only if Spotify isn't authenticated if configured_source == 'spotify': music_source = 'spotify' if spotify_connected else 'deezer' else: music_source = configured_source _status_cache['spotify'] = { 'connected': True, # Always true — iTunes fallback is always available 'response_time': round(spotify_response_time, 1), 'source': music_source, 'rate_limited': is_rate_limited, 'rate_limit': rate_limit_info, 'post_ban_cooldown': cooldown_remaining if cooldown_remaining > 0 else None } _status_cache_timestamps['spotify'] = current_time # else: use cached value # Test media server - use EXISTING instances (they have internal caching) # Media server clients already cache connection checks internally if current_time - _status_cache_timestamps['media_server'] > STATUS_CACHE_TTL: media_server_start = time.time() media_server_status = False if active_server == "plex" and plex_client: # Use existing instance - has 30s internal connection cache media_server_status = plex_client.is_connected() elif active_server == "jellyfin" and jellyfin_client: # Use existing instance - has internal connection caching media_server_status = jellyfin_client.is_connected() elif active_server == "navidrome" and navidrome_client: # Use existing instance media_server_status = navidrome_client.is_connected() elif active_server == "soulsync": # Standalone mode — always connected if Transfer folder exists media_server_status = soulsync_library_client.is_connected() if soulsync_library_client else False media_server_response_time = (time.time() - media_server_start) * 1000 _status_cache['media_server'] = { 'connected': media_server_status, 'response_time': round(media_server_response_time, 1), 'type': active_server } _status_cache_timestamps['media_server'] = current_time # else: use cached value # Test Soulseek - only if it's the active source or in the hybrid order if current_time - _status_cache_timestamps['soulseek'] > STATUS_CACHE_TTL: download_mode = config_manager.get('download_source.mode', 'hybrid') hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) soulseek_relevant = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) # Serverless sources (YouTube, HiFi, Qobuz) are always available serverless_sources = ('youtube', 'hifi', 'qobuz') is_serverless = (download_mode in serverless_sources or (download_mode == 'hybrid' and hybrid_order and any(s in serverless_sources for s in hybrid_order))) # Serverless check first — avoids slow slskd timeout when YouTube/HiFi are in hybrid order if is_serverless: soulseek_status = True soulseek_response_time = 0 elif soulseek_relevant and soulseek_client: soulseek_start = time.time() try: soulseek_status = run_async(soulseek_client.check_connection()) except Exception: soulseek_status = False soulseek_response_time = (time.time() - soulseek_start) * 1000 else: soulseek_status = False soulseek_response_time = 0 _status_cache['soulseek'] = { 'connected': soulseek_status, 'response_time': round(soulseek_response_time, 1) } _status_cache_timestamps['soulseek'] = current_time # Include download source mode so frontend can update labels download_mode = config_manager.get('download_source.mode', 'hybrid') _status_cache['soulseek']['source'] = download_mode # Count active downloads for nav badge active_dl_count = 0 with tasks_lock: for t in download_tasks.values(): if t.get('status') in ('downloading', 'searching', 'post_processing', 'queued', 'pending'): active_dl_count += 1 status_data = { 'spotify': _status_cache['spotify'], 'media_server': _status_cache['media_server'], 'soulseek': _status_cache['soulseek'], 'active_media_server': active_server, 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, } return jsonify(status_data) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/fix-navidrome-urls', methods=['POST']) def fix_navidrome_urls(): """Fix Navidrome artist image URLs to use correct Subsonic format""" try: db = get_database() with db._get_connection() as conn: cursor = conn.cursor() # Get all Navidrome artists with old URL format cursor.execute('SELECT id, name, thumb_url FROM artists WHERE server_source = "navidrome" AND thumb_url LIKE "/api/artist/%"') artists = cursor.fetchall() if not artists: return jsonify({"status": "success", "message": "No URLs needed fixing", "updated": 0}) # Update URLs to new Subsonic format import re updated = 0 examples = [] for artist_id, name, old_url in artists: # Extract artist ID from old URL: /api/artist/ARTIST_ID/image match = re.search(r'/api/artist/([^/]+)/image', old_url) if match: artist_spotify_id = match.group(1) new_url = f'/rest/getCoverArt?id={artist_spotify_id}' cursor.execute('UPDATE artists SET thumb_url = ? WHERE id = ? AND server_source = "navidrome"', (new_url, artist_id)) updated += 1 if len(examples) < 3: # Show first 3 as examples examples.append(f'{name}: {old_url} -> {new_url}') conn.commit() return jsonify({ "status": "success", "message": f"Updated {updated} Navidrome artist URLs to Subsonic format", "updated": updated, "examples": examples }) except Exception as e: return jsonify({"status": "error", "message": str(e)}), 500 def _regenerate_batch_m3u(batch, tracks): """Regenerate M3U file for a completed batch using real library DB paths. Called from batch completion handler after all post-processing is done.""" try: from difflib import SequenceMatcher try: from unidecode import unidecode as _unidecode except ImportError: _unidecode = lambda x: x def _norm(text): return _unidecode(text).lower().strip() if text else '' playlist_name = batch.get('playlist_name', 'Playlist') db = get_database() active_server = config_manager.get_active_media_server() raw_base = config_manager.get('m3u_export.entry_base_path', '') or '' entry_base_path = raw_base.rstrip('/\\') # Resolve file paths from library DB from collections import defaultdict artist_groups = defaultdict(list) for idx, t in enumerate(tracks): artist_groups[t.get('artist', '') or ''].append((idx, t)) file_path_map = {} for artist, group in artist_groups.items(): if not artist: for idx, _ in group: file_path_map[idx] = None continue db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server) if not db_tracks: for idx, _ in group: file_path_map[idx] = None continue db_entries = [(_norm(t.title), t) for t in db_tracks] for idx, track in group: name = track.get('name', '') if not name: file_path_map[idx] = None continue s_norm = _norm(name) matched = None for db_n, db_t in db_entries: if s_norm == db_n or SequenceMatcher(None, s_norm, db_n).ratio() >= 0.7: matched = db_t break file_path_map[idx] = matched.file_path if matched else None # Build M3U content import datetime as _dt lines = ['#EXTM3U', f'#PLAYLIST:{playlist_name}', f'#GENERATED:{_dt.datetime.utcnow().isoformat()}Z', ''] found = 0 for idx, track in enumerate(tracks): dur_s = int((track.get('duration_ms', 0) or 0) / 1000) artist = track.get('artist', 'Unknown') name = track.get('name', 'Unknown') lines.append(f'#EXTINF:{dur_s},{artist} - {name}') fp = file_path_map.get(idx) if fp: path = f'{entry_base_path}/{fp}' if entry_base_path else fp lines.append(path) found += 1 else: lines.append(f'# MISSING: {artist} - {name}') lines.append('') if found == 0: return # Don't overwrite with an all-missing M3U m3u_content = '\n'.join(lines) transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) m3u_folder = _compute_m3u_folder(transfer_dir, 'playlist', playlist_name, '', '', '') os.makedirs(m3u_folder, exist_ok=True) safe_fn = _sanitize_filename(playlist_name) m3u_path = os.path.join(m3u_folder, f'{safe_fn}.m3u') with open(m3u_path, 'w', encoding='utf-8') as f: f.write(m3u_content) logger.info(f"[M3U] Regenerated M3U on batch complete: {m3u_path} ({found}/{len(tracks)} resolved)") except Exception as e: logger.error(f"[M3U] Error in _regenerate_batch_m3u: {e}") @app.route('/api/save-playlist-m3u', methods=['POST']) def save_playlist_m3u(): """Save M3U playlist file to the relevant download folder""" try: data = request.get_json() if not data: return jsonify({"status": "error", "message": "No data provided"}), 400 playlist_name = data.get('playlist_name', 'Playlist') m3u_content = data.get('m3u_content', '') context_type = data.get('context_type', 'playlist') artist_name = data.get('artist_name', '') album_name = data.get('album_name', '') year = data.get('year', '') force = data.get('force', False) # Check if M3U export is enabled (unless force=True from manual Export button) if not force and not config_manager.get('m3u_export.enabled', False): return jsonify({"status": "success", "message": "M3U export disabled in settings", "skipped": True}) if not m3u_content: return jsonify({"status": "error", "message": "No M3U content provided"}), 400 # Compute target folder using the template system transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) m3u_folder = _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name, album_name, year) os.makedirs(m3u_folder, exist_ok=True) # Build M3U filename from playlist or album name if context_type == 'album' and artist_name and album_name: safe_filename = _sanitize_filename(f"{artist_name} - {album_name}") else: safe_filename = _sanitize_filename(playlist_name) m3u_filename = f"{safe_filename}.m3u" m3u_path = os.path.join(m3u_folder, m3u_filename) # Write M3U file (overwrite if exists) with open(m3u_path, 'w', encoding='utf-8') as f: f.write(m3u_content) logger.info(f"Saved M3U file: {m3u_path}") return jsonify({ "status": "success", "message": f"M3U file saved: {m3u_filename}", "path": m3u_path }) except Exception as e: logger.error(f"Error saving M3U file: {e}") return jsonify({"status": "error", "message": str(e)}), 500 @app.route('/api/generate-playlist-m3u', methods=['POST']) def generate_playlist_m3u(): """Generate M3U content with real file paths resolved from the library DB. Each track entry uses its actual stored file_path rather than a synthesised Artist - Title.mp3 string, so media servers can locate the files. An optional entry_base_path prefix (from settings) is prepended to every path. """ try: data = request.get_json() if not data: return jsonify({"success": False, "error": "No data"}), 400 playlist_name = data.get('playlist_name', 'Playlist') tracks = data.get('tracks', []) # [{name, artist, duration_ms}, ...] context_type = data.get('context_type', 'playlist') artist_name_ctx = data.get('artist_name', '') album_name = data.get('album_name', '') year = data.get('year', '') save_to_disk = data.get('save_to_disk', False) force = data.get('force', False) raw_base = config_manager.get('m3u_export.entry_base_path', '') or '' entry_base_path = raw_base.rstrip('/\\') db = get_database() active_server = config_manager.get_active_media_server() # --- fuzzy matching helpers (same logic as library_check_tracks) --- import re as _re from difflib import SequenceMatcher try: from unidecode import unidecode as _unidecode except ImportError: _unidecode = lambda x: x def _norm(text): return _unidecode(text).lower().strip() if text else '' def _clean(text): s = _norm(text) s = _re.sub(r'\s*[\[\(].*?[\]\)]', '', s) s = _re.sub(r'\s*-\s*', ' ', s) s = _re.sub(r'\s*feat\..*', '', s) s = _re.sub(r'\s*featuring.*', '', s) s = _re.sub(r'\s*ft\..*', '', s) s = _re.sub(r'\s*\d{4}\s*remaster.*', '', s) s = _re.sub(r'\s*remaster(ed)?.*', '', s) return _re.sub(r'\s+', ' ', s).strip() # Group tracks by primary artist to minimise DB queries from collections import defaultdict artist_groups = defaultdict(list) for idx, t in enumerate(tracks): artist_groups[t.get('artist', '') or ''].append((idx, t)) file_path_map = {} for artist, group in artist_groups.items(): if not artist: for idx, _ in group: file_path_map[idx] = None continue db_tracks = db.search_tracks(artist=artist, limit=500, server_source=active_server) if not db_tracks: for idx, _ in group: file_path_map[idx] = None continue db_entries = [(_norm(t.title), _clean(t.title), t) for t in db_tracks] for idx, track in group: name = track.get('name', '') if not name: file_path_map[idx] = None continue s_norm, s_clean = _norm(name), _clean(name) matched = None for db_n, db_c, db_t in db_entries: if s_norm == db_n or s_clean == db_c: matched = db_t break if max(SequenceMatcher(None, s_norm, db_n).ratio(), SequenceMatcher(None, s_clean, db_c).ratio()) >= 0.7: matched = db_t break file_path_map[idx] = matched.file_path if matched else None # --- build M3U content --- import datetime as _dt found_count = 0 missing_count = 0 lines = [ '#EXTM3U', f'#PLAYLIST:{playlist_name}', f'#GENERATED:{_dt.datetime.utcnow().isoformat()}Z', '', ] for idx, track in enumerate(tracks): name = track.get('name', '') or 'Unknown' artist = track.get('artist', '') or 'Unknown Artist' dur_s = int((track.get('duration_ms') or 0) / 1000) or -1 file_path = file_path_map.get(idx) lines.append(f'#EXTINF:{dur_s},{artist} - {name}') if file_path: found_count += 1 lines.append('#STATUS:FOUND_IN_LIBRARY') entry = f'{entry_base_path}/{file_path}' if entry_base_path else file_path lines.append(entry.replace('\\', '/')) else: missing_count += 1 lines.append('#STATUS:MISSING') safe = _re.sub(r'[/\\?%*:|"<>]', '-', f'{artist} - {name}') lines.append(f'# NOT AVAILABLE: {safe}') lines.append('') lines += [ '#SUMMARY', f'#TOTAL_TRACKS:{len(tracks)}', f'#FOUND_IN_LIBRARY:{found_count}', '#DOWNLOADED:0', f'#MISSING:{missing_count}', ] m3u_content = '\n'.join(lines) # --- optionally save to disk --- saved_path = None if save_to_disk and (force or config_manager.get('m3u_export.enabled', False)): transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) m3u_folder = _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name_ctx, album_name, year) os.makedirs(m3u_folder, exist_ok=True) if context_type == 'album' and artist_name_ctx and album_name: safe_fn = _sanitize_filename(f'{artist_name_ctx} - {album_name}') else: safe_fn = _sanitize_filename(playlist_name) m3u_path = os.path.join(m3u_folder, f'{safe_fn}.m3u') with open(m3u_path, 'w', encoding='utf-8') as f: f.write(m3u_content) saved_path = m3u_path logger.info(f"Saved M3U file: {m3u_path}") return jsonify({ "success": True, "m3u_content": m3u_content, "stats": {"found": found_count, "downloaded": 0, "missing": missing_count}, "path": saved_path }) except Exception as e: logger.error(f"Error generating M3U: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _build_system_stats(): """Build system statistics dict — shared by HTTP handler and WebSocket emitter.""" import psutil import time from datetime import timedelta # Calculate uptime start_time = getattr(app, 'start_time', time.time()) uptime_seconds = time.time() - start_time uptime = str(timedelta(seconds=int(uptime_seconds))) # Get memory usage memory = psutil.virtual_memory() memory_usage = f"{memory.percent}%" # Count active downloads from download_batches (batches that are currently downloading) active_downloads = len([batch_id for batch_id, batch_data in download_batches.items() if batch_data.get('phase') == 'downloading']) # Count finished downloads (completed this session) - use session counter like dashboard.py with session_stats_lock: finished_downloads = session_completed_downloads # Calculate total download speed from active soulseek transfers # Skip the slskd API call entirely when Soulseek is not the active download # source — avoids connection timeout spam every 10 seconds for users who have # a slskd URL configured but are using YouTube/Tidal/etc. total_download_speed = 0.0 soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True) download_mode = config_manager.get('download_source.mode', 'hybrid') hybrid_order = config_manager.get('download_source.hybrid_order', ['hifi', 'youtube', 'soulseek']) soulseek_active = (download_mode == 'soulseek' or (download_mode == 'hybrid' and 'soulseek' in hybrid_order)) if soulseek_active and not soulseek_known_down: try: transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) if transfers_data: for user_data in transfers_data: if 'directories' in user_data: for directory in user_data['directories']: if 'files' in directory: for file_info in directory['files']: state = file_info.get('state', '').lower() # Only count actively downloading files if 'inprogress' in state or 'downloading' in state or 'transferring' in state: speed = file_info.get('averageSpeed', 0) if isinstance(speed, (int, float)) and speed > 0: total_download_speed += float(speed) except Exception as e: logger.error(f"Could not fetch download speeds: {e}") # Convert bytes/sec to KB/s and format if total_download_speed > 0: speed_kb_s = total_download_speed / 1024 if speed_kb_s >= 1024: speed_mb_s = speed_kb_s / 1024 download_speed_str = f"{speed_mb_s:.1f} MB/s" else: download_speed_str = f"{speed_kb_s:.1f} KB/s" else: download_speed_str = "0 KB/s" # Count active syncs (playlists currently syncing) active_syncs = 0 # Count Spotify playlist syncs for _playlist_id, sync_state in sync_states.items(): if sync_state.get('status') == 'syncing': active_syncs += 1 # Count YouTube playlist syncs for _url_hash, state in youtube_playlist_states.items(): if state.get('phase') == 'syncing': active_syncs += 1 # Count Tidal playlist syncs for _playlist_id, state in tidal_discovery_states.items(): if state.get('phase') == 'syncing': active_syncs += 1 return { 'active_downloads': active_downloads, 'finished_downloads': finished_downloads, 'download_speed': download_speed_str, 'active_syncs': active_syncs, 'uptime': uptime, 'memory_usage': memory_usage } @app.route('/api/system/stats') def get_system_stats(): """Get system statistics for dashboard""" try: return jsonify(_build_system_stats()) except Exception as e: return jsonify({'error': str(e)}), 500 def _safe_check(fn, default=False): """Safely evaluate a check function, returning default on any error.""" try: return fn() except Exception: return default @app.route('/api/debug-info') def get_debug_info(): """Collect system diagnostics for troubleshooting support requests.""" import platform import sys import psutil import time from datetime import timedelta log_lines = request.args.get('lines', 20, type=int) log_lines = max(10, min(log_lines, 500)) log_source = request.args.get('log', 'app') info = {} # App info info['version'] = SOULSYNC_VERSION info['os'] = f"{platform.system()} {platform.release()}" info['python'] = sys.version.split()[0] info['docker'] = os.path.exists('/.dockerenv') info['runner'] = 'gunicorn' if not _DIRECT_RUN else 'direct (python web_server.py)' # ffmpeg version try: import subprocess result = subprocess.run(['ffmpeg', '-version'], capture_output=True, text=True, timeout=5) first_line = result.stdout.split('\n')[0] if result.stdout else '' # e.g. "ffmpeg version 6.1.1 Copyright ..." info['ffmpeg'] = first_line.split('Copyright')[0].replace('ffmpeg version', '').strip() if first_line else 'installed (version unknown)' except FileNotFoundError: info['ffmpeg'] = 'NOT INSTALLED' except Exception: info['ffmpeg'] = 'unknown' # Uptime start_time = getattr(app, 'start_time', time.time()) uptime_seconds = time.time() - start_time info['uptime'] = str(timedelta(seconds=int(uptime_seconds))) # Paths download_path = config_manager.get('soulseek.download_path', './downloads') transfer_folder = config_manager.get('soulseek.transfer_path', './Transfer') staging_folder = config_manager.get('import.staging_path', '') info['paths'] = { 'download_path': download_path, 'download_path_exists': os.path.isdir(download_path) if download_path else False, 'download_path_writable': os.access(download_path, os.W_OK) if download_path and os.path.isdir(download_path) else False, 'transfer_folder': transfer_folder, 'transfer_folder_exists': os.path.isdir(transfer_folder) if transfer_folder else False, 'transfer_folder_writable': os.access(transfer_folder, os.W_OK) if transfer_folder and os.path.isdir(transfer_folder) else False, 'staging_folder': staging_folder, 'staging_folder_exists': os.path.isdir(staging_folder) if staging_folder else False, } # Music library paths (Settings > Library) music_paths = config_manager.get('library.music_paths', []) if isinstance(music_paths, list) and music_paths: info['paths']['music_library_paths'] = [] for p in music_paths: if p and isinstance(p, str): info['paths']['music_library_paths'].append({ 'path': p, 'exists': os.path.isdir(p), }) # Music videos directory music_videos_path = config_manager.get('library.music_videos_path', '') if music_videos_path: info['paths']['music_videos_path'] = music_videos_path info['paths']['music_videos_path_exists'] = os.path.isdir(music_videos_path) # Services from status cache spotify_cache = _status_cache.get('spotify', {}) media_server_cache = _status_cache.get('media_server', {}) soulseek_cache = _status_cache.get('soulseek', {}) info['services'] = { 'music_source': spotify_cache.get('source', 'unknown'), 'spotify_connected': spotify_cache.get('connected', False), 'spotify_rate_limited': spotify_cache.get('rate_limited', False), 'media_server_type': media_server_cache.get('type', 'none'), 'media_server_connected': media_server_cache.get('connected', False), 'soulseek_connected': soulseek_cache.get('connected', False), 'download_source': config_manager.get('download_source.mode', 'hybrid'), 'tidal_connected': _safe_check(lambda: bool(tidal_client and tidal_client.is_authenticated())), 'qobuz_connected': _safe_check(lambda: bool(qobuz_enrichment_worker and qobuz_enrichment_worker.client and qobuz_enrichment_worker.client.is_authenticated())), } # Enrichment workers workers = {} worker_names = ['musicbrainz', 'audiodb', 'deezer', 'spotify', 'itunes', 'lastfm', 'genius', 'discogs', 'tidal', 'qobuz'] for name in worker_names: paused_key = f'{name}_enrichment_paused' workers[name] = 'paused' if config_manager.get(paused_key, False) else 'active' info['enrichment_workers'] = workers # Library stats — use same method as dashboard (filters by active server) try: db = get_database() lib_stats = db.get_database_info_for_server() info['library'] = { 'artists': lib_stats.get('artists', 0), 'albums': lib_stats.get('albums', 0), 'tracks': lib_stats.get('tracks', 0), } except Exception: info['library'] = {'artists': 0, 'albums': 0, 'tracks': 0} # Watchlist count try: db = get_database() info['watchlist_count'] = db.get_watchlist_count() except Exception: info['watchlist_count'] = 0 # Wishlist pending count try: db = get_database() info['wishlist_count'] = db.get_wishlist_count() except Exception: info['wishlist_count'] = 0 # Automation count try: db = get_database() automations = db.get_automations() info['automations'] = { 'total': len(automations), 'enabled': len([a for a in automations if a.get('enabled', False)]), } except Exception: info['automations'] = {'total': 0, 'enabled': 0} # Active downloads & syncs (use list() snapshots to avoid RuntimeError from concurrent mutation) try: active_downloads = len([bid for bid, bd in list(download_batches.items()) if bd.get('phase') == 'downloading']) except Exception: active_downloads = 0 active_syncs = 0 try: for _pid, ss in list(sync_states.items()): if ss.get('status') == 'syncing': active_syncs += 1 for _uh, st in list(youtube_playlist_states.items()): if st.get('phase') == 'syncing': active_syncs += 1 for _pid, st in list(tidal_discovery_states.items()): if st.get('phase') == 'syncing': active_syncs += 1 except Exception: pass info['active_downloads'] = active_downloads info['active_syncs'] = active_syncs # Config settings relevant to troubleshooting source_mode = config_manager.get('download_source.mode', 'hybrid') info['config'] = { 'source_mode': source_mode, 'quality_profile': config_manager.get('download_source.quality_profile', 'default'), 'organization_template': config_manager.get('organization.folder_template', ''), 'post_processing_enabled': config_manager.get('post_processing.enabled', True), 'acoustid_enabled': bool(config_manager.get('acoustid.api_key', '')), 'auto_scan_enabled': config_manager.get('watchlist.auto_scan', False), 'm3u_export_enabled': config_manager.get('m3u.enabled', False), 'log_level': config_manager.get('logging.level', 'INFO'), 'primary_metadata_source': config_manager.get('metadata.fallback_source', 'deezer'), 'lossy_copy_enabled': config_manager.get('post_processing.lossy_copy.enabled', False), 'lossy_copy_format': config_manager.get('post_processing.lossy_copy.format', 'mp3'), 'lossy_copy_bitrate': config_manager.get('post_processing.lossy_copy.bitrate', 320), 'allow_duplicate_tracks': config_manager.get('library.allow_duplicate_tracks', False), 'replace_lower_quality': config_manager.get('import.replace_lower_quality', False), 'auto_import_enabled': config_manager.get('import.auto_import_enabled', False), } # Hybrid source priority order if source_mode == 'hybrid': info['config']['hybrid_sources'] = config_manager.get('download_source.hybrid_order', []) # Discogs connection status info['services']['discogs_connected'] = bool(config_manager.get('discogs.token', '')) # Download client init failures info['download_client_failures'] = [] if soulseek_client and hasattr(soulseek_client, '_init_failures'): info['download_client_failures'] = soulseek_client._init_failures elif not soulseek_client: info['download_client_failures'] = ['ALL (orchestrator failed to initialize)'] # API rate monitor — current calls/min, 24h totals, peaks, rate limit events try: from core.api_call_tracker import api_call_tracker rates = api_call_tracker.get_all_rates() info['api_rates'] = rates # Rich 24h debug summary with peaks, totals, per-endpoint breakdown, events info['api_debug_summary'] = api_call_tracker.get_debug_summary() # Spotify rate limit details if spotify_client: rl_info = spotify_client.get_rate_limit_info() if rl_info: info['spotify_rate_limit'] = { 'active': True, 'remaining_seconds': rl_info.get('remaining_seconds', 0), 'retry_after': rl_info.get('retry_after', 0), 'endpoint': rl_info.get('endpoint', ''), 'expires_at': rl_info.get('expires_at', ''), } else: info['spotify_rate_limit'] = {'active': False} except Exception: info['api_rates'] = {} info['api_debug_summary'] = {} info['spotify_rate_limit'] = {'active': False} # Database size db_path = os.path.join('database', 'music_library.db') if os.path.exists(db_path): db_size_mb = os.path.getsize(db_path) / (1024 * 1024) info['database_size'] = f"{db_size_mb:.1f} MB" else: info['database_size'] = 'not found' # Memory & CPU process = psutil.Process(os.getpid()) mem = process.memory_info() info['memory_usage'] = f"{mem.rss / (1024 * 1024):.0f} MB" info['system_memory'] = f"{psutil.virtual_memory().percent}%" try: info['cpu_percent'] = f"{process.cpu_percent(interval=0.1):.1f}%" except Exception: info['cpu_percent'] = 'unknown' info['thread_count'] = process.num_threads() # Log lines log_map = { 'app': Path(_log_path), 'acoustid': _log_dir / 'acoustid.log', 'post_processing': _log_dir / 'post_processing.log', 'source_reuse': _log_dir / 'source_reuse.log', } log_path = log_map.get(log_source, log_map['app']) info['log_source'] = log_source info['log_lines_requested'] = log_lines info['recent_logs'] = [] if os.path.exists(log_path): try: with open(log_path, 'r', encoding='utf-8', errors='replace') as f: lines = f.readlines() info['recent_logs'] = [line.rstrip() for line in lines[-log_lines:]] except Exception: info['recent_logs'] = ['(could not read log file)'] # Available log files info['available_logs'] = [] logs_dir = 'logs' if os.path.isdir(logs_dir): for fname in sorted(os.listdir(logs_dir)): if fname.endswith('.log'): fpath = os.path.join(logs_dir, fname) size_kb = os.path.getsize(fpath) / 1024 info['available_logs'].append({ 'name': fname.replace('.log', ''), 'file': fname, 'size': f"{size_kb:.0f} KB" if size_kb < 1024 else f"{size_kb/1024:.1f} MB", }) return jsonify(info) @app.route('/api/activity/feed') def get_activity_feed(): """Get recent activity feed for dashboard""" try: with activity_feed_lock: # Return last 10 activities in reverse chronological order return jsonify({'activities': activity_feed[-10:][::-1]}) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/activity/toasts') def get_recent_toasts(): """Get recent activities that should show toasts""" try: import time current_time = time.time() with activity_feed_lock: # Return activities from last 10 seconds that should show toasts recent_toasts = [ activity for activity in activity_feed if activity.get('show_toast', True) and (current_time - activity.get('timestamp', 0)) <= 10 ] return jsonify({'toasts': recent_toasts}) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/logs') def get_activity_logs(): """Get formatted activity feed for display in sync page log area""" try: with activity_feed_lock: # Get the last 50 activities (more than the dashboard shows) recent_activities = activity_feed[-50:] if len(activity_feed) > 50 else activity_feed[:] # Reverse order so newest appears at top recent_activities = recent_activities[::-1] # Format activities as readable log entries formatted_logs = [] if not recent_activities: formatted_logs = [ "No recent activity.", "Sync and download operations will appear here in real-time." ] else: for activity in recent_activities: # Format: [TIME] ICON TITLE - SUBTITLE timestamp = activity.get('time', 'Unknown') icon = activity.get('icon', '•') title = activity.get('title', 'Activity') subtitle = activity.get('subtitle', '') # Create a clean, readable log entry if subtitle: log_entry = f"[{timestamp}] {icon} {title} - {subtitle}" else: log_entry = f"[{timestamp}] {icon} {title}" formatted_logs.append(log_entry) return jsonify({'logs': formatted_logs}) except Exception as e: return jsonify({'logs': [f'Error reading activity feed: {str(e)}']}) # --- Internal API Key Management (browser-only, no auth) --- @app.route('/api/v1/api-keys-internal', methods=['GET']) def list_api_keys_internal(): """List API keys for the settings page (no auth required — same as all UI routes).""" keys = config_manager.get('api_keys', []) safe_keys = [ { "id": k.get("id"), "label": k.get("label", ""), "key_prefix": k.get("key_prefix", ""), "created_at": k.get("created_at"), "last_used_at": k.get("last_used_at"), } for k in keys ] return jsonify({"success": True, "data": {"keys": safe_keys}}) @app.route('/api/v1/api-keys-internal/generate', methods=['POST']) def generate_api_key_internal(): """Generate API key from settings page (no auth required).""" from api.auth import generate_api_key body = request.get_json(silent=True) or {} label = body.get("label", "") raw_key, record = generate_api_key(label) keys = config_manager.get('api_keys', []) keys.append(record) config_manager.set('api_keys', keys) return jsonify({"success": True, "data": { "key": raw_key, "id": record["id"], "label": record["label"], "key_prefix": record["key_prefix"], "created_at": record["created_at"], }}), 201 @app.route('/api/v1/api-keys-internal/revoke/', methods=['DELETE']) def revoke_api_key_internal(key_id): """Revoke API key from settings page (no auth required).""" keys = config_manager.get('api_keys', []) original_len = len(keys) keys = [k for k in keys if k.get("id") != key_id] if len(keys) == original_len: return jsonify({"success": False, "error": {"message": "Key not found"}}), 404 config_manager.set('api_keys', keys) return jsonify({"success": True, "data": {"message": "API key revoked"}}) @app.route('/api/settings', methods=['GET', 'POST']) @admin_only def handle_settings(): global tidal_client # Declare that we might modify the global instance if not config_manager: return jsonify({"error": "Server configuration manager is not initialized."}), 500 if request.method == 'POST': try: new_settings = request.get_json() if not new_settings: return jsonify({"success": False, "error": "No data received."}), 400 if 'active_media_server' in new_settings: config_manager.set_active_media_server(new_settings['active_media_server']) for service in ['spotify', 'plex', 'jellyfin', 'navidrome', 'soulseek', 'download_source', 'settings', 'database', 'metadata_enhancement', 'file_organization', 'playlist_sync', 'tidal', 'tidal_download', 'qobuz', 'hifi_download', 'deezer_download', 'lidarr_download', 'listenbrainz', 'acoustid', 'lastfm', 'genius', 'import', 'lossy_copy', 'listening_stats', 'ui_appearance', 'youtube', 'content_filter', 'itunes', 'm3u_export', 'musicbrainz', 'deezer', 'audiodb', 'metadata', 'hydrabase', 'security', 'discogs', 'library', 'discover', 'wishlist', 'genre_whitelist', 'post_processing']: if service in new_settings: for key, value in new_settings[service].items(): config_manager.set(f'{service}.{key}', value) logger.info("Settings saved successfully via Web UI.") # Add activity for settings save changed_services = list(new_settings.keys()) services_text = ", ".join(changed_services) add_activity_item("", "Settings Updated", f"{services_text} configuration saved", "Now") add_activity_item("", "Settings Updated", f"{services_text} configuration saved", "Now") # Reload service clients with new settings (guard against None from partial init) if spotify_client: spotify_client.reload_config() if plex_client: plex_client.server = None if jellyfin_client: jellyfin_client.reload_config() if navidrome_client: navidrome_client.reload_config() # Reload orchestrator settings (download source mode, hybrid_primary, etc.) if soulseek_client: soulseek_client.reload_settings() # Reload YouTube client settings (rate limiting, cookies) if hasattr(soulseek_client, 'youtube'): soulseek_client.youtube.reload_settings() # FIX: Re-instantiate the global tidal_client to pick up new settings try: tidal_client = TidalClient() except Exception: pass # Keep existing tidal_client if re-init fails # Reload enrichment worker clients for key-based services if lastfm_worker: lastfm_worker._init_client() if genius_worker: genius_worker._init_client() if tidal_enrichment_worker: tidal_enrichment_worker.client = tidal_client # Invalidate status cache so next poll reflects new settings (e.g. fallback source change) _status_cache_timestamps['spotify'] = 0 logger.info("Service clients re-initialized with new settings.") return jsonify({"success": True, "message": "Settings saved successfully."}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 else: # GET request try: data = dict(config_manager.config_data) # Include which download sources are configured so the UI can auto-disable unconfigured ones try: data['_source_status'] = soulseek_client.get_source_status() except Exception: pass return jsonify(data) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/dev-mode', methods=['GET', 'POST']) def handle_dev_mode(): global dev_mode_enabled if request.method == 'POST': data = request.get_json() if data.get('password') == 'hydratest': dev_mode_enabled = True logger.info("Dev mode activated") return jsonify({"success": True, "enabled": True}) return jsonify({"success": False, "error": "Invalid password"}), 401 return jsonify({"enabled": dev_mode_enabled}) # ── Hydrabase WebSocket Connection ── # ── Hydrabase Comparison Store ── import collections as _collections _hydrabase_comparisons = _collections.OrderedDict() _COMPARISON_MAX_ENTRIES = 50 _comparison_lock = threading.Lock() def _is_hydrabase_active(): """Check if Hydrabase is connected and enabled for metadata use.""" try: from core.metadata_service import is_hydrabase_enabled return is_hydrabase_enabled() except Exception: return False def _run_background_comparison(query, hydrabase_counts=None): """Run Spotify + fallback source searches in background and store for comparison. Args: query: Search query string. hydrabase_counts: Optional pre-computed dict {'tracks': N, 'artists': N, 'albums': N} from the primary search to avoid redundant Hydrabase round-trips. """ def _worker(): try: result = {'timestamp': time.time(), 'query': query} # Use pre-computed counts if available, otherwise fetch from Hydrabase if hydrabase_counts is not None: hydra_data = hydrabase_counts else: hydra_data = {'tracks': 0, 'artists': 0, 'albums': 0} if _is_hydrabase_active(): raw_t = hydrabase_client.search_raw(query, 'track') raw_ar = hydrabase_client.search_raw(query, 'artists') raw_al = hydrabase_client.search_raw(query, 'album') hydra_data = { 'tracks': len(raw_t) if raw_t else 0, 'artists': len(raw_ar) if raw_ar else 0, 'albums': len(raw_al) if raw_al else 0 } result['hydrabase'] = hydra_data # Spotify results spotify_data = {'tracks': 0, 'artists': 0, 'albums': 0} if spotify_client and spotify_client.is_authenticated(): try: s_tracks = spotify_client.search_tracks(query, limit=10) s_artists = spotify_client.search_artists(query, limit=10) s_albums = spotify_client.search_albums(query, limit=10) spotify_data = { 'tracks': len(s_tracks), 'artists': len(s_artists), 'albums': len(s_albums) } except Exception as e: logger.debug(f"Comparison Spotify search failed: {e}") result['spotify'] = spotify_data # Fallback metadata source results (iTunes or Deezer) fallback_source = _get_metadata_fallback_source() fallback_data = {'tracks': 0, 'artists': 0, 'albums': 0} try: fallback_client = _get_metadata_fallback_client() f_tracks = fallback_client.search_tracks(query, limit=10) f_artists = fallback_client.search_artists(query, limit=10) f_albums = fallback_client.search_albums(query, limit=10) fallback_data = { 'tracks': len(f_tracks), 'artists': len(f_artists), 'albums': len(f_albums) } except Exception as e: logger.debug(f"Comparison {fallback_source} search failed: {e}") result['fallback'] = fallback_data result['fallback_source'] = fallback_source with _comparison_lock: _hydrabase_comparisons[query] = result while len(_hydrabase_comparisons) > _COMPARISON_MAX_ENTRIES: _hydrabase_comparisons.popitem(last=False) logger.info(f"Background comparison stored for '{query}': H={hydra_data}, S={spotify_data}, {fallback_source.capitalize()}={fallback_data}") except Exception as e: logger.error(f"Background comparison failed for '{query}': {e}") threading.Thread(target=_worker, daemon=True).start() @app.route('/api/hydrabase/connect', methods=['POST']) def hydrabase_connect(): """Connect to a Hydrabase instance via WebSocket.""" global _hydrabase_ws data = request.get_json() url = data.get('url', '').strip() api_key = data.get('api_key', '').strip() if not url or not api_key: return jsonify({"success": False, "error": "URL and API key required"}), 400 try: import websocket with _hydrabase_lock: # Close existing connection if any if _hydrabase_ws: try: _hydrabase_ws.close() except: pass ws = websocket.create_connection( url, header={"x-api-key": api_key}, timeout=10 ) _hydrabase_ws = ws # Save credentials for auto-reconnect config_manager.set('hydrabase.url', url) config_manager.set('hydrabase.api_key', api_key) config_manager.set('hydrabase.auto_connect', True) logger.info(f"[Hydrabase] Connected to {url}") return jsonify({"success": True, "message": "Connected"}) except Exception as e: logger.error(f"[Hydrabase] Connection failed: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/hydrabase/disconnect', methods=['POST']) def hydrabase_disconnect(): """Disconnect from Hydrabase and disable dev mode.""" global _hydrabase_ws, dev_mode_enabled with _hydrabase_lock: if _hydrabase_ws: try: _hydrabase_ws.close() except: pass _hydrabase_ws = None config_manager.set('hydrabase.auto_connect', False) # Only disable dev mode if not using Hydrabase as a regular fallback source if _get_metadata_fallback_source() != 'hydrabase': dev_mode_enabled = False logger.info("[Hydrabase] Disconnected") return jsonify({"success": True}) @app.route('/api/hydrabase/status') def hydrabase_status(): """Check if connected to Hydrabase.""" try: connected = _hydrabase_ws is not None and _hydrabase_ws.connected except Exception: connected = False try: hydra_config = config_manager.get_hydrabase_config() except AttributeError: hydra_config = {} peer_count = None try: if hydrabase_client and hydrabase_client.last_peer_count is not None: peer_count = hydrabase_client.last_peer_count except NameError: pass return jsonify({ "connected": connected, "saved_url": hydra_config.get('url', ''), "saved_api_key": hydra_config.get('api_key', ''), "auto_connect": hydra_config.get('auto_connect', False), "peer_count": peer_count }) @app.route('/api/hydrabase/comparisons') def hydrabase_comparisons(): """Get recent comparison results (Hydrabase vs Spotify vs fallback source).""" if not dev_mode_enabled: return jsonify({"success": False, "error": "Dev mode not active"}), 403 with _comparison_lock: items = list(reversed(_hydrabase_comparisons.values())) return jsonify({"success": True, "comparisons": items}) @app.route('/api/hydrabase/send', methods=['POST']) def hydrabase_send(): """Send a raw JSON payload to Hydrabase and return the response.""" global _hydrabase_ws if not dev_mode_enabled: return jsonify({"success": False, "error": "Dev mode not active"}), 403 if not _hydrabase_ws or not _hydrabase_ws.connected: return jsonify({"success": False, "error": "Not connected to Hydrabase"}), 400 data = request.get_json() payload = data.get('payload') if not payload: return jsonify({"success": False, "error": "No payload provided"}), 400 try: message = json.dumps(payload) if isinstance(payload, dict) else str(payload) with _hydrabase_lock: _hydrabase_ws.send(message) response = _hydrabase_ws.recv() try: result = json.loads(response) except json.JSONDecodeError: result = response logger.info("[Hydrabase] Sent payload — got response") return jsonify({"success": True, "data": result}) except Exception as e: logger.error(f"[Hydrabase] Send failed: {e}") with _hydrabase_lock: try: _hydrabase_ws.close() except: pass _hydrabase_ws = None return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/settings/log-level', methods=['GET', 'POST']) @admin_only def handle_log_level(): """Get or set the application log level""" from utils.logging_config import set_log_level, get_current_log_level from database.music_database import MusicDatabase if request.method == 'POST': try: data = request.get_json() level = data.get('level') if not level or level.upper() not in ['DEBUG', 'INFO', 'WARNING', 'ERROR']: return jsonify({"success": False, "error": "Invalid log level. Must be DEBUG, INFO, WARNING, or ERROR"}), 400 # Change the log level dynamically success = set_log_level(level) if success: # Save to database preferences db = MusicDatabase() db.set_preference('log_level', level.upper()) logger.info(f"Log level changed to {level.upper()} via Web UI") add_activity_item("", "Log Level Changed", f"Set to {level.upper()}", "Now") return jsonify({"success": True, "level": level.upper()}) else: return jsonify({"success": False, "error": "Failed to set log level"}), 500 except Exception as e: logger.error(f"Error setting log level: {e}") return jsonify({"success": False, "error": str(e)}), 500 else: # GET request try: current_level = get_current_log_level() return jsonify({"success": True, "level": current_level}) except Exception as e: logger.error(f"Error getting log level: {e}") return jsonify({"success": False, "error": str(e)}), 500 # =========================== # LIVE LOG VIEWER API # =========================== # In-memory ring buffer for live log streaming via WebSocket _live_log_buffer = [] _live_log_buffer_lock = threading.Lock() _LIVE_LOG_BUFFER_MAX = 500 @app.route('/api/logs/tail', methods=['GET']) def get_log_tail(): """Return the last N lines from a log file, optionally filtered by level.""" log_source = request.args.get('source', 'app') lines = request.args.get('lines', 200, type=int) lines = max(10, min(lines, 1000)) level_filter = request.args.get('level', '').upper() # DEBUG, INFO, WARNING, ERROR or empty log_map = { 'app': Path(_log_path), 'acoustid': _log_dir / 'acoustid.log', 'post_processing': _log_dir / 'post_processing.log', 'source_reuse': _log_dir / 'source_reuse.log', } log_path = log_map.get(log_source, log_map['app']) search = request.args.get('search', '').lower() def _classify_log_level(line): """Classify a log line's level. Returns DEBUG/INFO/WARNING/ERROR or empty for unclassified.""" if ' - DEBUG - ' in line: return 'DEBUG' if ' - INFO - ' in line: return 'INFO' if ' - WARNING - ' in line: return 'WARNING' if ' - ERROR - ' in line or ' - CRITICAL - ' in line: return 'ERROR' # Heuristic for plain-text output and non-logger lines ll = line.lower() if 'error' in ll or 'traceback' in ll or 'exception' in ll or 'failed' in ll: return 'ERROR' if 'warning' in ll or 'warn' in ll: return 'WARNING' if 'debug' in ll: return 'DEBUG' return 'INFO' # Default unclassified lines to INFO result_lines = [] if os.path.exists(log_path): try: with open(log_path, 'r', encoding='utf-8', errors='replace') as f: all_lines = f.readlines() # Read more lines than requested so filtering has enough to work with pool_size = lines * 5 if (level_filter or search) else lines tail = all_lines[-pool_size:] for line in tail: stripped = line.rstrip() if not stripped: continue if level_filter and level_filter in ('DEBUG', 'INFO', 'WARNING', 'ERROR'): if _classify_log_level(stripped) != level_filter: continue if search and search not in stripped.lower(): continue result_lines.append(stripped) # Trim to requested count after filtering result_lines = result_lines[-lines:] except Exception as e: result_lines = [f'Error reading log file: {e}'] # Available log files available = [] logs_dir = 'logs' if os.path.isdir(logs_dir): for fname in sorted(os.listdir(logs_dir)): if fname.endswith('.log'): fpath = os.path.join(logs_dir, fname) size_kb = os.path.getsize(fpath) / 1024 available.append({ 'key': fname.replace('.log', ''), 'file': fname, 'size': f"{size_kb:.0f} KB" if size_kb < 1024 else f"{size_kb/1024:.1f} MB", }) return jsonify({ 'lines': result_lines, 'source': log_source, 'total': len(result_lines), 'available_logs': available, }) # =========================== # AUTOMATIONS API # =========================== @app.route('/api/genre-whitelist/defaults', methods=['GET']) def get_genre_whitelist_defaults(): """Return the default genre whitelist.""" from core.genre_filter import DEFAULT_GENRES return jsonify({'genres': sorted(DEFAULT_GENRES, key=str.lower)}) # Automation route bodies live in core/automation/api.py — these routes are thin handlers. from core.automation import api as _auto_api from core.automation import blocks as _auto_blocks from core.automation import signals as _auto_signals @app.route('/api/automations', methods=['GET']) def list_automations(): """List all automations for the current profile.""" try: profile_id = session.get('profile_id', 1) return jsonify(_auto_api.list_automations(get_database(), profile_id)) except Exception as e: logger.error(f"Error listing automations: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations', methods=['POST']) def create_automation(): """Create a new automation.""" try: profile_id = session.get('profile_id', 1) body, status = _auto_api.create_automation(get_database(), automation_engine, profile_id, request.get_json()) return jsonify(body), status except Exception as e: logger.error(f"Error creating automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/', methods=['GET']) def get_automation(automation_id): """Get a single automation.""" try: auto = _auto_api.get_automation(get_database(), automation_id) if auto is None: return jsonify({"error": "Automation not found"}), 404 return jsonify(auto) except Exception as e: logger.error(f"Error getting automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/', methods=['PUT']) def update_automation_endpoint(automation_id): """Update an automation.""" try: body, status = _auto_api.update_automation(get_database(), automation_engine, automation_id, request.get_json()) return jsonify(body), status except Exception as e: logger.error(f"Error updating automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/group', methods=['PUT']) def batch_update_automation_group(): """Batch update group_name for multiple automations.""" try: data = request.get_json() body, status = _auto_api.batch_update_group(get_database(), data.get('automation_ids', []), data.get('group_name')) return jsonify(body), status except Exception as e: logger.error(f"Error batch updating automation group: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/bulk-toggle', methods=['POST']) def bulk_toggle_automations(): """Bulk enable/disable multiple automations.""" try: data = request.get_json() body, status = _auto_api.bulk_toggle(get_database(), automation_engine, data.get('automation_ids', []), data.get('enabled', True)) return jsonify(body), status except Exception as e: logger.error(f"Error bulk toggling automations: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/', methods=['DELETE']) def delete_automation_endpoint(automation_id): """Delete an automation. System automations cannot be deleted.""" try: body, status = _auto_api.delete_automation(get_database(), automation_engine, automation_id) return jsonify(body), status except Exception as e: logger.error(f"Error deleting automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations//duplicate', methods=['POST']) def duplicate_automation_endpoint(automation_id): """Duplicate an automation. System automations cannot be duplicated.""" try: profile_id = session.get('profile_id', 1) body, status = _auto_api.duplicate_automation(get_database(), automation_engine, profile_id, automation_id) return jsonify(body), status except Exception as e: logger.error(f"Error duplicating automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations//toggle', methods=['POST']) def toggle_automation_endpoint(automation_id): """Toggle an automation's enabled state.""" try: body, status = _auto_api.toggle_automation(get_database(), automation_engine, automation_id) return jsonify(body), status except Exception as e: logger.error(f"Error toggling automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations//run', methods=['POST']) def run_automation_endpoint(automation_id): """Manually trigger an automation.""" try: body, status = _auto_api.run_automation(automation_engine, automation_id, get_current_profile_id()) return jsonify(body), status except Exception as e: logger.error(f"Error running automation: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/automations/progress', methods=['GET']) def get_automation_progress(): """Get current progress state for all running/recently finished automations.""" try: return jsonify(_auto_progress.get_running_progress()) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/automations//history', methods=['GET']) def get_automation_history(automation_id): """Get run history for a specific automation.""" try: limit = request.args.get('limit', 50, type=int) offset = request.args.get('offset', 0, type=int) return jsonify(_auto_api.get_history(get_database(), automation_id, limit=limit, offset=offset)) except Exception as e: logger.error(f"Error getting automation history: {e}") return jsonify({"error": str(e)}), 500 def _collect_known_signals(): """Collect all signal names used across automations (for autocomplete).""" return _auto_signals.collect_known_signals(get_database()) @app.route('/api/scripts', methods=['GET']) def list_available_scripts(): """List executable scripts in the scripts directory.""" try: scripts_dir = docker_resolve_path(config_manager.get('scripts.path', './scripts')) if not scripts_dir or not os.path.isdir(scripts_dir): return jsonify({'scripts': []}) allowed_ext = {'.sh', '.py', '.bat', '.ps1', '.rb', '.pl', '.js'} scripts = [] for fname in sorted(os.listdir(scripts_dir)): ext = os.path.splitext(fname)[1].lower() fpath = os.path.join(scripts_dir, fname) if os.path.isfile(fpath) and (ext in allowed_ext or os.access(fpath, os.X_OK)): scripts.append({ 'name': fname, 'extension': ext, 'size': os.path.getsize(fpath), }) return jsonify({'scripts': scripts}) except Exception as e: return jsonify({'scripts': [], 'error': str(e)}) @app.route('/api/automations/blocks', methods=['GET']) def get_automation_blocks(): """Return available block types for the automation builder sidebar.""" return jsonify({ 'triggers': _auto_blocks.TRIGGERS, 'actions': _auto_blocks.ACTIONS, 'notifications': _auto_blocks.NOTIFICATIONS, 'known_signals': _collect_known_signals(), }) @app.route('/api/mirrored-playlists/list', methods=['GET']) def get_mirrored_playlists_list(): """Return simple list of mirrored playlists for automation config dropdowns.""" try: database = get_database() profile_id = get_current_profile_id() playlists = database.get_mirrored_playlists(profile_id=profile_id) spotify_authed = bool(spotify_client and spotify_client.is_spotify_authenticated()) return jsonify({ "playlists": [{"id": p['id'], "name": p['name'], "source": p.get('source', '')} for p in playlists], "spotify_authenticated": spotify_authed }) except Exception as e: return jsonify({"playlists": [], "spotify_authenticated": False}), 200 @app.route('/api/setup/status', methods=['GET']) def setup_status_endpoint(): """Check if first-run setup has been completed.""" # The setup wizard sets this flag when completed. download_source.mode is only # set by user action (wizard or settings page), never by config.json defaults. setup_done = config_manager.get('setup.completed', False) download_mode = config_manager.get('download_source.mode', '') # Either the explicit flag or a user-configured download source means setup is done has_user_config = bool(setup_done) or bool(download_mode) return jsonify({ "setup_complete": has_user_config, }) @app.route('/api/setup/complete', methods=['POST']) def setup_complete_endpoint(): """Mark first-run setup as completed.""" config_manager.set('setup.completed', True) return jsonify({"success": True}) @app.route('/api/test-connection', methods=['POST']) def test_connection_endpoint(): data = request.get_json() service = data.get('service') if not service: return jsonify({"success": False, "error": "No service specified."}), 400 logger.info(f"Received test connection request for: {service}") # Get the current settings from the main config manager to test with test_config = config_manager.get(service, {}) # For media servers, the service name might be 'server' if service == 'server': active_server = config_manager.get_active_media_server() test_config = config_manager.get(active_server, {}) service = active_server # use the actual server name for the test success, message = run_service_test(service, test_config) # Update status cache immediately when test succeeds to reflect current state import time if success: current_time = time.time() if service == 'spotify': _status_cache['spotify']['connected'] = True _status_cache['spotify']['source'] = _get_metadata_fallback_source() _status_cache_timestamps['spotify'] = current_time logger.info("Updated Spotify status cache after successful test") elif service in ['plex', 'jellyfin', 'navidrome', 'soulsync']: _status_cache['media_server']['connected'] = True _status_cache['media_server']['type'] = service _status_cache_timestamps['media_server'] = current_time logger.info(f"Updated {service} status cache after successful test") elif service == 'soulseek': _status_cache['soulseek']['connected'] = True _status_cache_timestamps['soulseek'] = current_time logger.info("Updated Soulseek status cache after successful test") elif service == 'listenbrainz': logger.info("ListenBrainz test successful") # Add activity for connection test if success: add_activity_item("", "Connection Test", message, "Now") else: add_activity_item("", "Connection Test", f"{service.title()} connection failed", "Now") return jsonify({"success": success, "error": "" if success else message, "message": message if success else ""}) @app.route('/api/settings/config-status', methods=['GET']) @admin_only def settings_config_status_endpoint(): """Return per-service config state for the Settings → Connections page. Drives the green/yellow header gradient. No API calls — just config reads. """ try: return jsonify({ service: {'configured': _is_service_configured(service)} for service in SERVICE_CONFIG_REGISTRY }) except Exception as e: logger.error(f"config-status error: {e}") return jsonify({"error": str(e)}), 500 # ── Per-service verify cache ── # Stores the last verify result per service for 5 minutes to prevent # hammering external APIs when the user rapidly expands/collapses cards. _settings_verify_cache = {} # service -> {'success': bool, 'message': str, 'error': str, 'ts': float} _settings_verify_cache_lock = threading.Lock() _SETTINGS_VERIFY_TTL_SECONDS = 300 def _get_cached_verify_result(service: str): with _settings_verify_cache_lock: entry = _settings_verify_cache.get(service) if entry and (time.time() - entry['ts']) < _SETTINGS_VERIFY_TTL_SECONDS: return entry return None def _store_verify_result(service: str, success: bool, message: str): with _settings_verify_cache_lock: _settings_verify_cache[service] = { 'success': bool(success), 'message': message or '', 'error': '' if success else (message or 'Unknown error'), 'ts': time.time(), } def _run_single_verify(service: str): """Run verify for one service, reading its current saved config. Returns cached result if recent, else executes run_service_test and caches the outcome. """ if service not in SERVICE_CONFIG_REGISTRY: return {'success': False, 'error': f'Unknown service: {service}', 'cached': False} cached = _get_cached_verify_result(service) if cached: return { 'success': cached['success'], 'error': cached.get('error', ''), 'message': cached.get('message', ''), 'cached': True, } try: saved_config = config_manager.get(service, {}) or {} success, message = run_service_test(service, saved_config) _store_verify_result(service, success, message) return { 'success': bool(success), 'error': '' if success else (message or 'Verification failed'), 'message': message or '', 'cached': False, } except Exception as e: logger.error(f"verify error for {service}: {e}") _store_verify_result(service, False, str(e)) return {'success': False, 'error': str(e), 'cached': False} @app.route('/api/settings/verify', methods=['POST']) @admin_only def settings_verify_endpoint(): """Run connection verification for one or more services. Body: {"services": ["spotify", "deezer"]} — which services to check Query: ?force=true — bust cache and re-run Returns {service: {success, error, message, cached}} per requested service. Concurrency capped at 3 to avoid rate-limiting ourselves on Expand All. """ try: data = request.get_json(silent=True) or {} services = data.get('services') or [] if isinstance(services, str): services = [services] if not services: return jsonify({'error': 'No services specified'}), 400 force = (request.args.get('force') or '').strip().lower() in ('1', 'true', 'yes') if force: with _settings_verify_cache_lock: for svc in services: _settings_verify_cache.pop(svc, None) from concurrent.futures import ThreadPoolExecutor, as_completed results = {} with ThreadPoolExecutor(max_workers=3) as pool: futures = {pool.submit(_run_single_verify, svc): svc for svc in services} for fut in as_completed(futures): svc = futures[fut] try: results[svc] = fut.result() except Exception as e: results[svc] = {'success': False, 'error': str(e), 'cached': False} return jsonify(results) except Exception as e: logger.error(f"settings/verify error: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/test-dashboard-connection', methods=['POST']) def test_dashboard_connection_endpoint(): """Test connection from dashboard - creates specific dashboard activity items""" data = request.get_json() service = data.get('service') if not service: return jsonify({"success": False, "error": "No service specified."}), 400 logger.info(f"Received dashboard test connection request for: {service}") # Get the current settings from the main config manager to test with test_config = config_manager.get(service, {}) # For media servers, the service name might be 'server' if service == 'server': active_server = config_manager.get_active_media_server() test_config = config_manager.get(active_server, {}) service = active_server # use the actual server name for the test success, message = run_service_test(service, test_config) # Update status cache immediately when test succeeds to reflect current state import time if success: current_time = time.time() if service == 'spotify': _status_cache['spotify']['connected'] = True _status_cache['spotify']['source'] = _get_metadata_fallback_source() _status_cache_timestamps['spotify'] = current_time logger.info("Updated Spotify status cache after successful dashboard test") elif service in ['plex', 'jellyfin', 'navidrome', 'soulsync']: _status_cache['media_server']['connected'] = True _status_cache['media_server']['type'] = service _status_cache_timestamps['media_server'] = current_time logger.info(f"Updated {service} status cache after successful dashboard test") elif service == 'soulseek': _status_cache['soulseek']['connected'] = True _status_cache_timestamps['soulseek'] = current_time logger.info("Updated Soulseek status cache after successful dashboard test") # Add activity for dashboard connection test (different from settings test) if success: add_activity_item("", "Dashboard Test", message, "Now") else: add_activity_item("", "Dashboard Test", f"{service.title()} service check failed", "Now") return jsonify({"success": success, "error": "" if success else message, "message": message if success else ""}) @app.route('/api/detect-media-server', methods=['POST']) def detect_media_server_endpoint(): data = request.get_json() server_type = data.get('server_type') logger.info(f"Received auto-detect request for: {server_type}") # Add activity for auto-detect start add_activity_item("", "Auto-Detect Started", f"Searching for {server_type} server", "Now") found_url = run_detection(server_type) if found_url: add_activity_item("", "Auto-Detect Complete", f"{server_type} found at {found_url}", "Now") return jsonify({"success": True, "found_url": found_url}) else: add_activity_item("", "Auto-Detect Failed", f"No {server_type} server found", "Now") return jsonify({"success": False, "error": f"No {server_type} server found on common local addresses."}) @app.route('/api/plex/pin/start', methods=['POST']) def start_plex_pin_auth(): try: pinlogin = MyPlexPinLogin(oauth=False) except Exception as e: logger.error(f'Failed to start Plex PIN auth: {e}') return jsonify({"success": False, "error": str(e)}), 500 pin_code = getattr(pinlogin, 'pin', None) if not pin_code: return jsonify({"success": False, "error": 'Failed to generate Plex PIN code.'}), 500 request_id = str(uuid.uuid4()) with _plex_pin_requests_lock: _plex_pin_requests[request_id] = { 'pinlogin': pinlogin, 'created_at': time.time(), 'expires_at': getattr(pinlogin, 'expires_at', None) } expires_in = None expires_at = getattr(pinlogin, 'expires_at', None) if expires_at: try: expires_in = int((expires_at - datetime.now(timezone.utc)).total_seconds()) except Exception: expires_in = None return jsonify({ "success": True, "request_id": request_id, "code": str(pin_code), "auth_url": "https://plex.tv/link", "expires_in": expires_in }) @app.route('/api/plex/pin/status', methods=['GET']) def get_plex_pin_status(): request_id = request.args.get('request_id') if not request_id: return jsonify({"success": False, "error": 'request_id is required'}), 400 with _plex_pin_requests_lock: entry = _plex_pin_requests.get(request_id) if not entry: return jsonify({"success": False, "error": 'Invalid or expired PIN request id.'}), 400 pinlogin = entry.get('pinlogin') if not pinlogin: return jsonify({"success": False, "error": 'Invalid PIN login state.'}), 500 try: if getattr(pinlogin, 'expired', False): with _plex_pin_requests_lock: _plex_pin_requests.pop(request_id, None) return jsonify({"success": False, "expired": True, "error": 'PIN code expired.'}) if pinlogin.checkLogin(): token = getattr(pinlogin, 'token', None) if not token: raise ValueError('Plex token was not returned after authorization.') try: account = MyPlexAccount(token=token) resources = account.resources() except Exception as e: logger.error(f'Failed to fetch Plex account resources: {e}') return jsonify({"success": False, "error": f'Plex authorization succeeded but failed to resolve server resources: {e}'}), 500 server_resources = [r for r in resources if 'server' in (getattr(r, 'provides', '') or '').lower()] if not server_resources: return jsonify({"success": False, "error": 'No Plex server resources found for this account.'}), 500 local_conn = None relay_conn = None for resource in server_resources: for conn in getattr(resource, 'connections', []) or []: if getattr(conn, 'local', False): local_conn = conn break if getattr(conn, 'relay', False) and relay_conn is None: relay_conn = conn if local_conn: break chosen_conn = local_conn or relay_conn if not chosen_conn: chosen_conn = getattr(server_resources[0], 'connections', [None])[0] found_url = getattr(chosen_conn, 'uri', None) if chosen_conn else None with _plex_pin_requests_lock: _plex_pin_requests.pop(request_id, None) if not found_url: return jsonify({"success": False, "error": 'Plex authorized, but no usable server connection URI was found.'}), 500 return jsonify({ "success": True, "found_url": found_url, "token": token, "status": 'Plex authorization complete.' }) return jsonify({"success": False, "status": 'Waiting for Plex authorization. Enter the PIN on plex.tv/link.'}) except Exception as e: logger.error(f'Error checking Plex PIN status: {e}') return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/plex/clear-library', methods=['POST']) def clear_plex_library_preference(): try: from database.music_database import MusicDatabase db = MusicDatabase() db.set_preference('plex_music_library', '') if plex_client: plex_client.music_library = None return jsonify({"success": True, "message": "Plex library preference cleared."}) except Exception as e: logger.error(f"Error clearing Plex library preference: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/plex/music-libraries', methods=['GET']) def get_plex_music_libraries(): """Get list of all available music libraries from Plex""" try: libraries = plex_client.get_available_music_libraries() # Get currently selected library from database.music_database import MusicDatabase db = MusicDatabase() selected_library = db.get_preference('plex_music_library') # Get the currently active library name current_library = None if plex_client.music_library: current_library = plex_client.music_library.title return jsonify({ "success": True, "libraries": libraries, "selected": selected_library, "current": current_library }) except Exception as e: logger.error(f"Error getting Plex music libraries: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/plex/select-music-library', methods=['POST']) def select_plex_music_library(): """Set the active Plex music library""" try: data = request.get_json() library_name = data.get('library_name') if not library_name: return jsonify({"success": False, "error": "No library name provided"}), 400 success = plex_client.set_music_library_by_name(library_name) if success: add_activity_item("", "Library Selected", f"Plex music library set to: {library_name}", "Now") return jsonify({"success": True, "message": f"Music library set to: {library_name}"}) else: return jsonify({"success": False, "error": f"Library '{library_name}' not found"}), 404 except Exception as e: logger.error(f"Error setting Plex music library: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/jellyfin/users', methods=['GET']) def get_jellyfin_users(): """Get list of Jellyfin users that have music libraries""" try: users = jellyfin_client.get_available_users() # Get currently selected user from database.music_database import MusicDatabase db = MusicDatabase() selected_user = db.get_preference('jellyfin_user') # Determine the current user name from user_id current_user = None if jellyfin_client.user_id: for u in users: if u['id'] == jellyfin_client.user_id: current_user = u['name'] break return jsonify({ "success": True, "users": users, "selected": selected_user, "current": current_user }) except Exception as e: logger.error(f"Error getting Jellyfin users: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/jellyfin/select-user', methods=['POST']) def select_jellyfin_user(): """Set the active Jellyfin user""" try: data = request.get_json() username = data.get('username') if not username: return jsonify({"success": False, "error": "No username provided"}), 400 success = jellyfin_client.set_user_by_name(username) if success: add_activity_item("", "User Selected", f"Jellyfin user set to: {username}", "Now") return jsonify({"success": True, "message": f"User set to: {username}"}) else: return jsonify({"success": False, "error": f"User '{username}' not found or has no music library"}), 404 except Exception as e: logger.error(f"Error setting Jellyfin user: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/jellyfin/music-libraries', methods=['GET']) def get_jellyfin_music_libraries(): """Get list of all available music libraries from Jellyfin""" try: libraries = jellyfin_client.get_available_music_libraries() # Get currently selected library from database.music_database import MusicDatabase db = MusicDatabase() selected_library = db.get_preference('jellyfin_music_library') # Get the currently active library name (match Plex behavior) current_library = None if jellyfin_client.music_library_id: # Look up library name from ID for lib in libraries: if lib['key'] == jellyfin_client.music_library_id: current_library = lib['title'] break return jsonify({ "success": True, "libraries": libraries, "selected": selected_library, "current": current_library }) except Exception as e: logger.error(f"Error getting Jellyfin music libraries: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/jellyfin/select-music-library', methods=['POST']) def select_jellyfin_music_library(): """Set the active Jellyfin music library""" try: data = request.get_json() library_name = data.get('library_name') if not library_name: return jsonify({"success": False, "error": "No library name provided"}), 400 success = jellyfin_client.set_music_library_by_name(library_name) if success: add_activity_item("", "Library Selected", f"Jellyfin music library set to: {library_name}", "Now") return jsonify({"success": True, "message": f"Music library set to: {library_name}"}) else: return jsonify({"success": False, "error": f"Library '{library_name}' not found"}), 404 except Exception as e: logger.error(f"Error setting Jellyfin music library: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/navidrome/music-folders', methods=['GET']) def get_navidrome_music_folders(): """Get list of available music folders from Navidrome""" try: if not navidrome_client: return jsonify({"success": False, "error": "Navidrome client not configured"}), 400 folders = navidrome_client.get_music_folders() from database.music_database import MusicDatabase db = MusicDatabase() selected_folder = db.get_preference('navidrome_music_folder') current_folder = None if navidrome_client.music_folder_id: for f in folders: if f['key'] == navidrome_client.music_folder_id: current_folder = f['title'] break return jsonify({ "success": True, "folders": folders, "selected": selected_folder, "current": current_folder }) except Exception as e: logger.error(f"Error getting Navidrome music folders: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/navidrome/select-music-folder', methods=['POST']) def select_navidrome_music_folder(): """Set the active Navidrome music folder""" try: data = request.get_json() folder_name = data.get('folder_name', '') success = navidrome_client.set_music_folder_by_name(folder_name) if success: if folder_name: add_activity_item("", "Library Selected", f"Navidrome music folder set to: {folder_name}", "Now") return jsonify({"success": True, "message": f"Music folder set to: {folder_name}"}) else: add_activity_item("", "Library Selection Cleared", "Navidrome will use all libraries", "Now") return jsonify({"success": True, "message": "Music folder selection cleared — using all libraries"}) else: return jsonify({"success": False, "error": f"Folder '{folder_name}' not found"}), 404 except Exception as e: logger.error(f"Error setting Navidrome music folder: {e}") return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == QUALITY PROFILE API == # =============================== @app.route('/api/quality-profile', methods=['GET']) def get_quality_profile(): """Get current quality profile configuration""" try: from database.music_database import MusicDatabase db = MusicDatabase() profile = db.get_quality_profile() return jsonify({ "success": True, "profile": profile }) except Exception as e: logger.error(f"Error getting quality profile: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/quality-profile', methods=['POST']) def save_quality_profile(): """Save quality profile configuration""" try: from database.music_database import MusicDatabase db = MusicDatabase() data = request.get_json() if not data: return jsonify({"success": False, "error": "No profile data provided"}), 400 success = db.set_quality_profile(data) if success: add_activity_item("", "Quality Profile Updated", f"Preset: {data.get('preset', 'custom')}", "Now") return jsonify({"success": True, "message": "Quality profile saved successfully"}) else: return jsonify({"success": False, "error": "Failed to save quality profile"}), 500 except Exception as e: logger.error(f"Error saving quality profile: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/quality-profile/presets', methods=['GET']) def get_quality_presets(): """Get all available quality presets""" try: from database.music_database import MusicDatabase db = MusicDatabase() presets = { "audiophile": db.get_quality_preset("audiophile"), "balanced": db.get_quality_preset("balanced"), "space_saver": db.get_quality_preset("space_saver") } return jsonify({ "success": True, "presets": presets }) except Exception as e: logger.error(f"Error getting quality presets: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/quality-profile/preset/', methods=['POST']) def apply_quality_preset(preset_name): """Apply a predefined quality preset""" try: from database.music_database import MusicDatabase db = MusicDatabase() preset = db.get_quality_preset(preset_name) success = db.set_quality_profile(preset) if success: add_activity_item("", "Quality Preset Applied", f"Applied '{preset_name}' preset", "Now") return jsonify({ "success": True, "message": f"Applied '{preset_name}' preset", "profile": preset }) else: return jsonify({"success": False, "error": "Failed to apply preset"}), 500 except Exception as e: logger.error(f"Error applying quality preset: {e}") return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == END QUALITY PROFILE API == # =============================== @app.route('/api/detect-soulseek', methods=['POST']) def detect_soulseek_endpoint(): logger.info("Received auto-detect request for slskd") # Add activity for soulseek auto-detect start add_activity_item("", "Auto-Detect Started", "Searching for slskd server", "Now") found_url = run_detection('slskd') if found_url: add_activity_item("", "Auto-Detect Complete", f"slskd found at {found_url}", "Now") return jsonify({"success": True, "found_url": found_url}) else: add_activity_item("", "Auto-Detect Failed", "No slskd server found", "Now") return jsonify({"success": False, "error": "No slskd server found on common local addresses."}) # --- Authentication Routes --- @app.route('/auth/spotify') def auth_spotify(): """ Initiates Spotify OAuth authentication flow. Supports per-profile auth via ?profile_id= query param. """ try: profile_id = request.args.get('profile_id', '') # Per-profile auth: use profile's own credentials if profile_id and profile_id != '1': try: profile_id_int = int(profile_id) db = get_database() creds = db.get_profile_spotify(profile_id_int) if creds and creds.get('client_id'): from spotipy.oauth2 import SpotifyOAuth redirect_uri = creds.get('redirect_uri') or config_manager.get_spotify_config().get('redirect_uri', 'http://127.0.0.1:8888/callback') auth_manager = SpotifyOAuth( client_id=creds['client_id'], client_secret=creds['client_secret'], redirect_uri=redirect_uri, scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read", cache_path=f'config/.spotify_cache_profile_{profile_id_int}', state=f'profile_{profile_id_int}' ) auth_url = auth_manager.get_authorize_url() logger.info(f"Per-profile Spotify auth initiated for profile {profile_id_int}") return redirect(auth_url) except (ValueError, Exception) as e: logger.error(f"Per-profile Spotify auth failed, falling back to global: {e}") # Global auth (admin or fallback) temp_spotify_client = SpotifyClient() if temp_spotify_client.sp and temp_spotify_client.sp.auth_manager: # Get the authorization URL auth_url = temp_spotify_client.sp.auth_manager.get_authorize_url() configured_uri = config_manager.get_spotify_config().get('redirect_uri', 'http://127.0.0.1:8888/callback') logger.info(f"Spotify auth initiated — redirect_uri: {configured_uri}") add_activity_item("", "Spotify Auth Started", "Please complete OAuth in browser", "Now") # Detect if accessing remotely host = request.host.split(':')[0] is_remote = host not in ['127.0.0.1', 'localhost'] is_docker = os.path.exists('/.dockerenv') # If in Docker and accessing via 127.0.0.1, recommend localhost if is_docker and host == '127.0.0.1': host = 'localhost' # Check if the redirect_uri uses port 8008 (main app) vs the standalone callback server. # A localhost URI (127.0.0.1/localhost) on a non-8008 port → standalone callback server. # A custom domain or port 8008 → main Flask app (or reverse proxy to it). uses_main_port = ':8008' in configured_uri or ( '127.0.0.1' not in configured_uri and 'localhost' not in configured_uri ) if is_remote or is_docker: # Show instructions for remote/docker access if uses_main_port: # redirect_uri already points to port 8008 or a custom domain — # callback will come through the main Flask app, no manual steps needed return f'''

Spotify Authentication

Click the link below to authenticate with Spotify:

Authenticate with Spotify

Redirect URI: {configured_uri}
After authorizing, Spotify will redirect back automatically. Make sure this URL matches your Spotify Dashboard redirect URI.

After authentication completes, you can close this window and return to SoulSync.

''' else: # redirect_uri points to the standalone callback server — show manual steps AND suggest switching import re as _re _port_match = _re.search(r':(\d+)/', configured_uri) callback_server_port = _port_match.group(1) if _port_match else str(os.environ.get('SOULSYNC_SPOTIFY_CALLBACK_PORT', '8888')) return f'''

Spotify Authentication (Remote/Docker)

Using a reverse proxy? Your redirect URI is set to {configured_uri} which uses port {callback_server_port}. If you're behind a reverse proxy (Caddy, Nginx, Traefik), change the redirect URI in SoulSync settings to use your proxy URL on the main port instead, e.g.:
https://{host}/callback
Then update the same URI in your Spotify Dashboard. This avoids the need for manual URL editing below.

Step 1: Click the link below to authenticate with Spotify

{auth_url}

Step 2: After authorizing, you'll see a blank page. The URL will look like:

http://127.0.0.1:{callback_server_port}/callback?code=...

Step 3: Change 127.0.0.1 to {host} and press Enter:

http://{host}:{callback_server_port}/callback?code=...

Authentication will then complete!

''' else: # Local access - simple message return f'

Spotify Authentication

Click the link below to authenticate:

{auth_url}

After authentication, return to the app.

' else: return "

Spotify Authentication Failed

Could not initialize Spotify client. Check your credentials.

", 400 except Exception as e: logger.error(f"Error starting Spotify auth: {e}") return f"

Spotify Authentication Error

{str(e)}

", 500 @app.route('/auth/tidal') def auth_tidal(): """ Initiates Tidal OAuth authentication flow """ logger.info("TIDAL AUTH ROUTE CALLED ") try: # Create a fresh tidal client to get OAuth URL from core.tidal_client import TidalClient temp_tidal_client = TidalClient() if not temp_tidal_client.client_id: return "

Tidal Authentication Failed

Tidal client ID not configured. Check your credentials.

", 400 # Generate PKCE challenge and store globally temp_tidal_client._generate_pkce_challenge() # Store PKCE values globally for callback use global tidal_oauth_state with tidal_oauth_lock: tidal_oauth_state["code_verifier"] = temp_tidal_client.code_verifier tidal_oauth_state["code_challenge"] = temp_tidal_client.code_challenge # Use the user's configured redirect_uri from settings — don't override # with request.host, which in Docker returns the container hostname configured_redirect = config_manager.get('tidal.redirect_uri', '') if configured_redirect: temp_tidal_client.redirect_uri = configured_redirect logger.info(f"Using configured Tidal redirect_uri: {configured_redirect}") else: # Fallback: dynamically set based on request host (non-Docker local access) request_host = request.host.split(':')[0] if request_host not in ('127.0.0.1', 'localhost'): dynamic_redirect = f"http://{request_host}:8889/tidal/callback" temp_tidal_client.redirect_uri = dynamic_redirect logger.info(f"Tidal redirect_uri set from request host: {dynamic_redirect}") # Store PKCE + redirect_uri for callback to use the same values with tidal_oauth_lock: tidal_oauth_state["redirect_uri"] = temp_tidal_client.redirect_uri logger.info(f"Stored PKCE - verifier: {temp_tidal_client.code_verifier[:20]}... challenge: {temp_tidal_client.code_challenge[:20]}...") # Store profile_id for per-profile auth profile_id = request.args.get('profile_id', '') with tidal_oauth_lock: tidal_oauth_state["profile_id"] = profile_id if profile_id and profile_id != '1' else None # Create OAuth URL import urllib.parse params = { 'response_type': 'code', 'client_id': temp_tidal_client.client_id, 'redirect_uri': temp_tidal_client.redirect_uri, 'scope': 'user.read playlists.read', 'code_challenge': temp_tidal_client.code_challenge, 'code_challenge_method': 'S256' } auth_url = f"{temp_tidal_client.auth_url}?" + urllib.parse.urlencode(params) logger.info(f"Generated Tidal OAuth URL: {auth_url}") logger.info(f"Redirect URI in URL: {params['redirect_uri']}") add_activity_item("", "Tidal Auth Started", "Please complete OAuth in browser", "Now") # Detect if accessing remotely (copied from Spotify auth logic) host = request.host.split(':')[0] is_remote = host not in ['127.0.0.1', 'localhost'] is_docker = os.path.exists('/.dockerenv') # If in Docker and accessing via 127.0.0.1, recommend localhost if is_docker and host == '127.0.0.1': host = 'localhost' if is_remote or is_docker: # Show instructions for remote/docker access page_title = "Tidal Authentication (Remote/Docker)" step_1_text = "Click the link below to authenticate with Tidal" return f'''

{page_title}

Step 1: {step_1_text}

{auth_url}

Step 2: After authorizing, you'll see a blank page or an error. The URL will look like:

http://127.0.0.1:8888/tidal/callback?code=...

Step 3: Change 127.0.0.1 to {host} and press Enter:

http://{host}:8888/tidal/callback?code=...

Authentication will then complete!

''' else: return f'

Tidal Authentication

Please visit this URL to authenticate:

{auth_url}

After authentication, return to the app.

' except Exception as e: logger.error(f"Error starting Tidal auth: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") return f"

Tidal Authentication Error

{str(e)}

", 500 @app.route('/callback') def spotify_callback(): """ Handles Spotify OAuth callback via the main Flask app (port 8008). This is the recommended callback for reverse proxy / Docker setups. The dedicated HTTPServer on port 8888 continues to work for direct/local access. """ global spotify_client auth_code = request.args.get('code') if not auth_code: error = request.args.get('error') if error: logger.error(f"Spotify OAuth error on port 8008: Spotify returned error: {error}") add_activity_item("", "Spotify Auth Failed", f"Spotify returned error: {error}", "Now") return f"

Spotify Authentication Failed

Spotify returned error: {error}

", 400 # No code AND no error — check if query params were stripped if request.args: logger.info(f"Spotify callback on port 8008 received unexpected params: {dict(request.args)}") else: # Completely empty — likely a healthcheck or spurious request pass return '', 204 logger.info("Spotify callback received on port 8008 with authorization code") # Check for per-profile state parameter state = request.args.get('state', '') profile_id_from_state = None if state and state.startswith('profile_'): try: profile_id_from_state = int(state.replace('profile_', '')) logger.info(f"Per-profile callback detected for profile {profile_id_from_state}") except ValueError: pass try: from core.spotify_client import SpotifyClient from spotipy.oauth2 import SpotifyOAuth from config.settings import config_manager # Per-profile callback: use profile's credentials if profile_id_from_state and profile_id_from_state != 1: db = get_database() creds = db.get_profile_spotify(profile_id_from_state) if creds and creds.get('client_id'): redirect_uri = creds.get('redirect_uri') or config_manager.get_spotify_config().get('redirect_uri', 'http://127.0.0.1:8888/callback') auth_manager = SpotifyOAuth( client_id=creds['client_id'], client_secret=creds['client_secret'], redirect_uri=redirect_uri, scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read", cache_path=f'config/.spotify_cache_profile_{profile_id_from_state}', state=f'profile_{profile_id_from_state}' ) token_info = auth_manager.get_access_token(auth_code) if token_info: # Invalidate cached profile client so it gets recreated with new tokens with _profile_spotify_lock: _profile_spotify_clients.pop(profile_id_from_state, None) add_activity_item("", "Spotify Auth Complete", f"Profile {profile_id_from_state} authenticated with Spotify", "Now") return "

Spotify Authentication Successful!

Your personal Spotify account is now connected. You can close this window.

" else: raise Exception("Failed to exchange authorization code for access token") # Global callback (admin) config = config_manager.get_spotify_config() configured_uri = config.get('redirect_uri', "http://127.0.0.1:8888/callback") logger.info(f"Using redirect_uri for token exchange: {configured_uri}") auth_manager = SpotifyOAuth( client_id=config['client_id'], client_secret=config['client_secret'], redirect_uri=configured_uri, scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read", cache_path='config/.spotify_cache' ) token_info = auth_manager.get_access_token(auth_code) if token_info: # CRITICAL: update the GLOBAL spotify_client, not a local variable global spotify_client spotify_client = SpotifyClient() if spotify_client.is_spotify_authenticated(): # Clear any active rate limit ban and post-ban cooldown # so Spotify is immediately usable after re-auth from core.spotify_client import _clear_rate_limit _clear_rate_limit() spotify_client._invalidate_auth_cache() # Invalidate status cache so next poll picks up the new connection _status_cache_timestamps['spotify'] = 0 # Refresh enrichment worker's client so it picks up new auth if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'): spotify_enrichment_worker.client.reload_config() spotify_enrichment_worker.client._invalidate_auth_cache() add_activity_item("", "Spotify Auth Complete", "Successfully authenticated with Spotify", "Now") return "

Spotify Authentication Successful!

You can close this window.

" else: raise Exception("Token exchange succeeded but authentication validation failed") else: raise Exception("Failed to exchange authorization code for access token") except Exception as e: logger.error(f"Spotify OAuth callback error on port 8008: {e}") add_activity_item("", "Spotify Auth Failed", f"Token processing failed: {str(e)}", "Now") return f"

Spotify Authentication Failed

{str(e)}

", 400 @app.route('/api/spotify/disconnect', methods=['POST']) def spotify_disconnect(): """Disconnect Spotify and fall back to iTunes/Apple Music""" global spotify_client try: # Pause enrichment worker before disconnecting to prevent it from hammering API if spotify_enrichment_worker: spotify_enrichment_worker.pause() spotify_client.disconnect() # Immediately update status cache so UI reflects the change fallback_src = _get_metadata_fallback_source() _status_cache['spotify'] = { 'connected': True, # Fallback source is always available 'response_time': 0, 'source': fallback_src, 'rate_limited': False, 'rate_limit': None } _status_cache_timestamps['spotify'] = time.time() fallback_label = 'Deezer' if fallback_src == 'deezer' else 'Discogs' if fallback_src == 'discogs' else 'iTunes' add_activity_item("", "Spotify Disconnected", f"Switched to {fallback_label} metadata source", "Now") return jsonify({'success': True, 'message': f'Spotify disconnected. Now using {fallback_label}.'}) except Exception as e: logger.error(f"Error disconnecting Spotify: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/spotify/rate-limit-status', methods=['GET']) def spotify_rate_limit_status(): """Get Spotify rate limit ban details""" try: info = spotify_client.get_rate_limit_info() if info: return jsonify({ 'rate_limited': True, 'remaining_seconds': info['remaining_seconds'], 'retry_after': info['retry_after'], 'endpoint': info['endpoint'], 'expires_at': info['expires_at'] }) return jsonify({'rate_limited': False}) except Exception as e: logger.error(f"Error getting Spotify rate limit status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/tidal/callback') def tidal_callback(): """ Handles the callback from Tidal after the user authorizes the application. It receives an authorization code, exchanges it for an access token, and saves the token. """ global tidal_client # We will re-initialize the global client auth_code = request.args.get('code') if not auth_code: error = request.args.get('error', 'Unknown error') error_description = request.args.get('error_description', 'No description provided.') return f"

Tidal Authentication Failed

Error: {error}

{error_description}

Please close this window and try again.

", 400 try: # Create a temporary client for the token exchange temp_tidal_client = TidalClient() # Restore PKCE values, redirect_uri, and profile_id from the auth request profile_id_for_tidal = None with tidal_oauth_lock: temp_tidal_client.code_verifier = tidal_oauth_state["code_verifier"] temp_tidal_client.code_challenge = tidal_oauth_state["code_challenge"] if "redirect_uri" in tidal_oauth_state: temp_tidal_client.redirect_uri = tidal_oauth_state["redirect_uri"] profile_id_for_tidal = tidal_oauth_state.get("profile_id") success = temp_tidal_client.fetch_token_from_code(auth_code) if success: # Per-profile: store tokens on profile, don't touch global client if profile_id_for_tidal: try: profile_id_int = int(profile_id_for_tidal) db = get_database() # Store Tidal tokens on the profile from config.settings import config_manager as _cm enc_access = _cm._encrypt_value(temp_tidal_client.access_token) if temp_tidal_client.access_token else None enc_refresh = _cm._encrypt_value(temp_tidal_client.refresh_token) if temp_tidal_client.refresh_token else None with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" UPDATE profiles SET tidal_access_token = ?, tidal_refresh_token = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ? """, (enc_access, enc_refresh, profile_id_int)) conn.commit() add_activity_item("", "Tidal Auth Complete", f"Profile {profile_id_int} authenticated with Tidal", "Now") return "

Tidal Authentication Successful!

Your personal Tidal account is now connected. You can close this window.

" except Exception as profile_err: logger.error(f"Per-profile Tidal auth failed, falling back to global: {profile_err}") # Global: Re-initialize the main global tidal_client instance with the new token tidal_client = TidalClient() if tidal_enrichment_worker: tidal_enrichment_worker.client = tidal_client return "

Tidal Authentication Successful!

You can now close this window and return to the SoulSync application.

" else: return "

Tidal Authentication Failed

Could not exchange authorization code for a token. Please try again.

", 400 except Exception as e: logger.error(f"Error during Tidal token exchange: {e}") return f"

An Error Occurred

An unexpected error occurred during the authentication process: {e}

", 500 # --- Deezer OAuth --- @app.route('/auth/deezer') def auth_deezer(): """Initialize Deezer OAuth flow. Redirects user to Deezer authorization page.""" try: app_id = config_manager.get('deezer.app_id', '') redirect_uri = config_manager.get('deezer.redirect_uri', 'http://127.0.0.1:8008/deezer/callback') if not app_id: return "

Deezer App ID not configured

Go to Settings → Connections and enter your Deezer App ID first.

", 400 perms = 'basic_access,email,offline_access,manage_library,listening_history' import urllib.parse auth_url = f"https://connect.deezer.com/oauth/auth.php?app_id={app_id}&redirect_uri={urllib.parse.quote(redirect_uri)}&perms={perms}" host = request.host.split(':')[0] return f"""

Deezer Authorization

Click the link below to authorize SoulSync with your Deezer account:

Authorize on Deezer →

If running remotely, replace 127.0.0.1 in the redirect URI with {host}

""" except Exception as e: return f"

Error

{e}

", 500 @app.route('/deezer/callback') def deezer_callback(): """Handle Deezer OAuth callback — exchange code for access token.""" auth_code = request.args.get('code') error_reason = request.args.get('error_reason', '') if not auth_code: return f"

Deezer Authentication Failed

{error_reason or 'No authorization code received.'}

", 400 try: app_id = config_manager.get('deezer.app_id', '') app_secret = config_manager.get('deezer.app_secret', '') if not app_id or not app_secret: return "

Missing Credentials

Deezer App ID or Secret not configured.

", 400 # Exchange code for token — simple GET request (Deezer's unique approach) token_url = f"https://connect.deezer.com/oauth/access_token.php?app_id={app_id}&secret={app_secret}&code={auth_code}" resp = requests.get(token_url, timeout=15) if resp.status_code != 200: return f"

Token Exchange Failed

Deezer returned status {resp.status_code}

", 400 # Deezer returns: access_token=TOKEN&expires=SECONDS (URL-encoded, not JSON) import urllib.parse token_data = dict(urllib.parse.parse_qsl(resp.text)) access_token = token_data.get('access_token') if not access_token: # Try JSON format (some Deezer API versions) try: json_data = resp.json() access_token = json_data.get('access_token') except Exception: pass if not access_token: return f"

No Access Token

Deezer response: {resp.text[:200]}

", 400 # Save token to config (encrypted at rest) config_manager.set('deezer.access_token', access_token) # Reload the global deezer client to pick up the token deezer_client = _get_deezer_client() deezer_client.reload_config() add_activity_item("", "Deezer Auth Complete", "Deezer account connected via OAuth", "Now") logger.info("Deezer OAuth authentication successful") return """

Deezer Authentication Successful!

Your Deezer account is now connected. You can close this window.

""" except Exception as e: logger.error(f"Deezer OAuth callback error: {e}") return f"

Error

{e}

", 500 # --- Beatport Data API --- @app.route('/api/beatport/hero-tracks') def get_beatport_hero_tracks(): """Get fresh tracks from Beatport hero slideshow for the rebuild slider""" try: logger.info("Fetching Beatport hero tracks...") # Check cache first cached_data = get_cached_beatport_data('homepage', 'hero_tracks') if cached_data: logger.info("Returning cached hero tracks data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh hero tracks data...") # Initialize scraper scraper = BeatportUnifiedScraper() # Get tracks from hero slideshow (increased limit to capture all slides) tracks = scraper.scrape_new_on_beatport_hero(limit=15) # SMART FILTERING - Remove duplicates and invalid tracks valid_tracks = [] seen_urls = set() filtered_reasons = collections.Counter() for _i, track in enumerate(tracks): # Extract and clean basic data title = track.get('title', '').strip() artist = track.get('artist', '').strip() url = track.get('url', '').strip() image_url = track.get('image_url', '').strip() # Apply text cleaning for proper spacing if title: title = clean_beatport_text(title) if artist: artist = clean_beatport_text(artist) # Validation filters is_valid = True skip_reasons = [] # Filter 1: Must have title (artist can be fallback) if not title or title in ['No title', 'MISSING', 'Unknown Title']: is_valid = False skip_reasons.append("Missing/invalid title") # If no artist, use fallback based on URL or default if not artist or artist in ['No artist', 'MISSING', 'Unknown Artist', 'NO_ARTIST']: if url and '/release/' in url: artist = 'Various Artists' # Release pages often have multiple artists else: artist = 'Unknown Artist' # Filter 2: Must have valid URL and image if not url or not image_url: is_valid = False skip_reasons.append("Missing URL or image") # Filter 3: URL must be a track/release page (not promotional pages) if url and not any(pattern in url for pattern in ['/release/', '/track/']): is_valid = False skip_reasons.append("URL is not a track/release page") # Filter 4: Deduplication by URL (most reliable method) if url in seen_urls: is_valid = False skip_reasons.append("Duplicate URL") if not is_valid: filtered_reasons.update(skip_reasons) continue # Mark URL as seen for deduplication seen_urls.add(url) # Clean up title title = title.replace(" t ", "'t ").replace("(Extended)DJ", "(Extended)") # Clean up artist names if 'SyrossianHappy' in artist: artist = 'Darius Syrossian' if 'Carroll,' in artist: artist = 'Ron Carroll' if artist.endswith('DJ') and ' ' not in artist[-4:]: artist = artist[:-2].strip() # Create clean track data track_data = { 'title': title, 'artist': artist, 'url': url, 'image_url': image_url, 'genre': 'Electronic', # Default genre 'year': datetime.now().year } # Determine genre based on artist genre_mapping = { 'thakzin': 'Afro House', 'yaya': 'Tech House', 'darius syrossian': 'Techno', 'ron carroll': 'House', 'dj minx': 'House', 'durante': 'Progressive House' } for artist_key, mapped_genre in genre_mapping.items(): if artist_key in artist.lower(): track_data['genre'] = mapped_genre break valid_tracks.append(track_data) sample_titles = [f"{t['title']} - {t['artist']}" for t in valid_tracks[:3]] logger.debug( "Beatport smart filter summary: raw=%s valid=%s filtered=%s reasons=%s sample=%s", len(tracks), len(valid_tracks), len(tracks) - len(valid_tracks), dict(filtered_reasons), sample_titles, ) logger.info(f"Retrieved {len(valid_tracks)} valid unique Beatport tracks (SMART FILTERING)") # Prepare response data response_data = { 'success': True, 'tracks': valid_tracks, 'count': len(valid_tracks), 'timestamp': datetime.now().isoformat() } # Cache the successful response set_cached_beatport_data('homepage', 'hero_tracks', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error fetching Beatport tracks: {str(e)}") return jsonify({ 'success': False, 'error': str(e), 'tracks': [] }), 500 @app.route('/api/beatport/new-releases') def get_beatport_new_releases(): """Get new releases from Beatport for the rebuild slider grid""" try: logger.info("🆕 Fetching Beatport new releases...") # Check cache first cached_data = get_cached_beatport_data('homepage', 'new_releases') if cached_data: logger.info("🆕 Returning cached new releases data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh new releases data...") # Initialize scraper scraper = BeatportUnifiedScraper() # Get page and extract releases soup = scraper.get_page(scraper.base_url) if not soup: raise Exception("Could not fetch Beatport homepage") # Find New Releases GridSlider container by section heading # Use partial class match to avoid brittle hashed class names gridsliders = soup.select('[class*="GridSlider-style__Wrapper"]') releases_container = None for container in gridsliders: h2 = container.select_one('h2') if h2: title = h2.get_text(strip=True).lower() if 'new release' in title: releases_container = container logger.info(f"🆕 FOUND NEW RELEASES: '{h2.get_text(strip=True)}'") break # Fallback: try ReleaseCard partial class match on whole page if releases_container: release_cards = releases_container.select('[class*="ReleaseCard-style__Wrapper"]') else: logger.warning("No New Releases GridSlider found, trying page-wide ReleaseCard search") release_cards = soup.select('[class*="ReleaseCard-style__Wrapper"]') releases = [] logger.info(f"Found {len(release_cards)} release cards") for _i, card in enumerate(release_cards[:100]): # Limit to 100 for 10 slides release_data = {} # Extract title from Meta section title_elem = card.select_one('[class*="ReleaseCard-style__Meta"] a[href*="/release/"]') if not title_elem: title_elem = card.select_one('[class*="title"], [class*="Title"], h3, h4, h5, h6') if title_elem: title_text = title_elem.get_text(strip=True) if title_text and len(title_text) > 2 and title_text not in ['New Releases', 'Buy', 'Play']: release_data['title'] = title_text # Extract artist artist_elem = card.select_one('a[href*="/artist/"]') if artist_elem: artist_text = artist_elem.get_text(strip=True) if artist_text and len(artist_text) > 1: release_data['artist'] = artist_text # Extract label label_elem = card.select_one('a[href*="/label/"]') if label_elem: label_text = label_elem.get_text(strip=True) if label_text and len(label_text) > 1: release_data['label'] = label_text # Extract URL url_link = card.select_one('a[href*="/release/"]') if url_link: href = url_link.get('href') if href: release_data['url'] = urljoin(scraper.base_url, href) # Extract image img = card.select_one('img') if img: src = img.get('src') or img.get('data-src') or img.get('data-lazy-src') if src: release_data['image_url'] = src # URL fallback for title if not release_data.get('title') and release_data.get('url'): url_parts = release_data['url'].split('/release/') if len(url_parts) > 1: slug = url_parts[1].split('/')[0] release_data['title'] = slug.replace('-', ' ').title() # Only add if we have essential data if release_data.get('title') and release_data.get('url'): # Add fallbacks for missing data if not release_data.get('artist'): release_data['artist'] = 'Various Artists' if not release_data.get('label'): release_data['label'] = 'Unknown Label' releases.append(release_data) logger.info(f"Successfully extracted {len(releases)} new releases") # Prepare response data response_data = { 'success': True, 'releases': releases, 'count': len(releases), 'slides': (len(releases) + 9) // 10, # Calculate number of slides needed 'timestamp': datetime.now().isoformat() } # Cache the successful response set_cached_beatport_data('homepage', 'new_releases', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error fetching new releases: {str(e)}") return jsonify({ 'success': False, 'error': str(e), 'releases': [] }), 500 @app.route('/api/beatport/featured-charts') def get_beatport_featured_charts(): """Get featured charts from Beatport for the charts slider grid using GridSlider approach""" try: logger.info("Fetching Beatport featured charts...") # Check cache first cached_data = get_cached_beatport_data('homepage', 'featured_charts') if cached_data: logger.info("Returning cached featured charts data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh featured charts data...") # Initialize scraper scraper = BeatportUnifiedScraper() # Get page and extract charts soup = scraper.get_page(scraper.base_url) if not soup: raise Exception("Could not fetch Beatport homepage") # Find Featured Charts GridSlider container (like New Releases) gridsliders = soup.select('[class*="GridSlider-style__Wrapper"]') featured_container = None logger.debug(f"Checking {len(gridsliders)} GridSlider containers for featured charts...") for container in gridsliders: h2 = container.select_one('h2') if h2: title = h2.get_text(strip=True).lower() logger.debug(f"Found section: '{h2.get_text(strip=True)}'") if 'featured' in title and 'chart' in title: featured_container = container logger.debug(f"FOUND FEATURED CHARTS: '{h2.get_text(strip=True)}'") break if not featured_container: logger.warning("No Featured Charts GridSlider container found") return jsonify({ 'success': False, 'error': 'Featured Charts section not found', 'charts': [] }) # Extract charts from the container using chart links charts = [] chart_links = featured_container.select('a[href*="/chart/"]') logger.debug(f"Found {len(chart_links)} chart links in Featured Charts section") for _i, link in enumerate(chart_links[:100]): # Limit to 100 for 10 slides chart_data = {} # Extract chart name from link text or nearby elements name_elem = link.select_one('h3, h4, h5, h6, [class*="title"], [class*="Title"], [class*="name"], [class*="Name"]') if name_elem: name_text = name_elem.get_text(strip=True) else: name_text = link.get_text(strip=True) if name_text and len(name_text) > 2 and name_text.lower() not in ['featured charts', 'buy', 'play']: chart_data['name'] = name_text # Extract creator using the specific CSS class pattern from chart cards creator = 'Beatport' # Default # Look for the ChartCard Name class that contains the creator creator_elem = link.select_one('[class*="ChartCard-style__Name"]') if creator_elem: creator_text = creator_elem.get_text(strip=True) if creator_text and len(creator_text) > 1 and creator_text.lower() not in ['by', 'chart', 'featured', 'beatport']: creator = creator_text elif creator_text.lower() == 'beatport': creator = 'Beatport' else: # Fallback: look for other creator indicators parent = link.parent if parent: fallback_selectors = [ '[class*="artist"]', '[class*="Artist"]', '[class*="creator"]', '[class*="Creator"]', '[class*="author"]', '[class*="Author"]' ] for selector in fallback_selectors: fallback_elem = parent.select_one(selector) if fallback_elem: fallback_text = fallback_elem.get_text(strip=True) if fallback_text and len(fallback_text) > 1 and fallback_text.lower() not in ['by', 'chart', 'featured']: creator = fallback_text break chart_data['creator'] = creator # Extract URL href = link.get('href', '') if href: if href.startswith('/'): chart_data['url'] = f"https://www.beatport.com{href}" else: chart_data['url'] = href # Extract image img_elem = link.select_one('img') or (link.parent.select_one('img') if link.parent else None) if img_elem: src = img_elem.get('src', '') or img_elem.get('data-src', '') if src: if src.startswith('//'): src = f"https:{src}" elif src.startswith('/'): src = f"https://www.beatport.com{src}" chart_data['image'] = src # Only add if we have meaningful data if 'name' in chart_data and 'url' in chart_data: charts.append(chart_data) logger.debug(f"Chart {len(charts)}: {chart_data['name']} by {chart_data['creator']}") logger.info(f"Successfully extracted {len(charts)} featured charts") # Prepare response data response_data = { 'success': True, 'charts': charts, 'count': len(charts), 'slides': (len(charts) + 9) // 10, # Calculate number of slides needed 'timestamp': datetime.now().isoformat() } # Cache the successful response set_cached_beatport_data('homepage', 'featured_charts', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error fetching featured charts: {str(e)}") return jsonify({ 'success': False, 'error': str(e), 'charts': [] }), 500 @app.route('/api/beatport/dj-charts') def get_beatport_dj_charts(): """Get DJ charts from Beatport for the DJ charts slider using Carousel approach""" try: logger.info("Fetching Beatport DJ charts...") # Check cache first cached_data = get_cached_beatport_data('homepage', 'dj_charts') if cached_data: logger.info("Returning cached DJ charts data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh DJ charts data...") # Initialize scraper scraper = BeatportUnifiedScraper() # Get page and extract charts soup = scraper.get_page(scraper.base_url) if not soup: raise Exception("Could not fetch Beatport homepage") # Find all Carousel containers carousels = soup.select('[class*="Carousel-style__Wrapper"]') dj_container = None logger.debug(f"Checking {len(carousels)} Carousel containers for DJ charts...") # Based on test results, DJ charts are in the second carousel (index 1) with ~9 chart links for i, container in enumerate(carousels): chart_links = container.select('a[href*="/chart/"]') logger.debug(f"Carousel {i+1}: {len(chart_links)} chart links") # DJ charts container typically has 8-12 chart links (not 99+ like featured charts) if 5 <= len(chart_links) <= 15: dj_container = container logger.info(f"FOUND DJ CHARTS: Carousel {i+1} with {len(chart_links)} charts") break if not dj_container: logger.warning("No DJ Charts Carousel container found") return jsonify({ 'success': False, 'error': 'DJ Charts section not found', 'charts': [] }) # Extract charts from the container using chart links charts = [] chart_links = dj_container.select('a[href*="/chart/"]') logger.info(f"Found {len(chart_links)} DJ chart links") for _i, link in enumerate(chart_links): chart_data = {} # Extract chart name from link text or nearby elements name_elem = link.select_one('h3, h4, h5, h6, [class*="title"], [class*="Title"], [class*="name"], [class*="Name"]') if name_elem: name_text = name_elem.get_text(strip=True) else: name_text = link.get_text(strip=True) if name_text and len(name_text) > 2: chart_data['name'] = name_text # Extract creator - for DJ charts, the chart name might be the artist name creator = name_text # Use chart name as creator for DJ charts # Look for additional creator info in parent elements parent = link.parent if parent: creator_selectors = [ '[class*="artist"]', '[class*="Artist"]', '[class*="creator"]', '[class*="Creator"]', '[class*="author"]', '[class*="Author"]' ] for selector in creator_selectors: creator_elem = parent.select_one(selector) if creator_elem: creator_text = creator_elem.get_text(strip=True) if creator_text and len(creator_text) > 1 and creator_text != name_text: creator = creator_text break chart_data['creator'] = creator # Extract URL href = link.get('href', '') if href: if href.startswith('/'): chart_data['url'] = f"https://www.beatport.com{href}" else: chart_data['url'] = href # Extract image img_elem = link.select_one('img') or (link.parent.select_one('img') if link.parent else None) if img_elem: src = img_elem.get('src', '') or img_elem.get('data-src', '') if src: if src.startswith('//'): src = f"https:{src}" elif src.startswith('/'): src = f"https://www.beatport.com{src}" chart_data['image'] = src # Only add if we have meaningful data if 'name' in chart_data and 'url' in chart_data: charts.append(chart_data) logger.info(f"DJ Chart {len(charts)}: {chart_data['name']} by {chart_data['creator']}") logger.info(f"Successfully extracted {len(charts)} DJ charts") # Prepare response data response_data = { 'success': True, 'charts': charts, 'count': len(charts), 'slides': max(1, (len(charts) + 2) // 3), # 3 cards per slide 'timestamp': datetime.now().isoformat() } # Cache the successful response set_cached_beatport_data('homepage', 'dj_charts', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error fetching DJ charts: {str(e)}") return jsonify({ 'success': False, 'error': str(e), 'charts': [] }), 500 # --- Placeholder API Endpoints for Other Pages --- @app.route('/api/activity') def get_activity(): # Placeholder: returns mock activity data mock_activity = [ {"time": "1 min ago", "text": "Service status checked."}, {"time": "5 min ago", "text": "Application server started."} ] return jsonify({"activities": mock_activity}) @app.route('/api/playlists') def get_playlists(): # Placeholder: returns mock playlist data if spotify_client and spotify_client.is_authenticated(): # In a real implementation, you would call spotify_client.get_user_playlists() mock_playlists = [ {"id": "1", "name": "Chill Vibes"}, {"id": "2", "name": "Workout Mix"}, {"id": "3", "name": "Liked Songs"} ] return jsonify({"playlists": mock_playlists}) return jsonify({"playlists": [], "error": "Spotify not authenticated."}) @app.route('/api/sync', methods=['POST']) def start_sync(): # Placeholder: simulates starting a sync return jsonify({"success": True, "message": "Sync process started."}) # Search route bodies live in core/search/* — these routes are thin handlers. from core.search import basic as _search_basic from core.search import library_check as _search_library_check from core.search import orchestrator as _search_orchestrator from core.search import stream as _search_stream def _build_search_deps(): """Build the SearchDeps bundle from this module's globals on each request. Constructed per-request so config / client state is always live. """ return _search_orchestrator.SearchDeps( database=get_database(), config_manager=config_manager, spotify_client=spotify_client, hydrabase_client=hydrabase_client, hydrabase_worker=hydrabase_worker, soulseek_client=soulseek_client, fix_artist_image_url=fix_artist_image_url, is_hydrabase_active=_is_hydrabase_active, get_metadata_fallback_source=_get_metadata_fallback_source, get_metadata_fallback_client=_get_metadata_fallback_client, get_itunes_client=_get_itunes_client, get_deezer_client=_get_deezer_client, get_discogs_client=_get_discogs_client, run_background_comparison=_run_background_comparison, run_async=run_async, dev_mode_enabled_provider=lambda: dev_mode_enabled, ) @app.route('/api/search', methods=['POST']) def search_music(): """Basic Soulseek file search.""" data = request.get_json() query = data.get('query') if not query: return jsonify({"error": "No search query provided."}), 400 logger.info(f"Web UI Search initiated for: '{query}'") add_activity_item("", "Search Started", f"'{query}'", "Now") try: results = _search_basic.run_basic_soulseek_search(query, soulseek_client, run_async) add_activity_item("", "Search Complete", f"'{query}' - {len(results)} results", "Now") return jsonify({"results": results}) except Exception as e: logger.error(f"Search error: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/enhanced-search', methods=['POST']) def enhanced_search(): """Unified metadata search across configured sources + local DB artists. Optional `source` body param ("spotify"|"itunes"|"deezer"|"discogs"| "hydrabase"|"musicbrainz"|"auto"|"") forces a single-source search and bypasses the fan-out. Otherwise picks a primary source per the user's configuration and lists alternates for the frontend to fetch async. """ data = request.get_json() query = data.get('query', '').strip() requested_source = (data.get('source') or '').strip().lower() if requested_source == 'auto': requested_source = '' if requested_source and requested_source not in ENHANCED_SEARCH_VALID_SOURCES: return jsonify({"error": f"Unknown source: {requested_source}"}), 400 if not query: return jsonify(_search_orchestrator.empty_response()) cache_key = _get_enhanced_search_cache_key(query, requested_source) cached = _get_cached_enhanced_search_response(cache_key) if cached is not None: logger.info(f"Enhanced search cache hit for: '{query}'") return jsonify(cached) logger.info(f"Enhanced search initiated for: '{query}' (source={requested_source or 'auto'})") try: deps = _build_search_deps() response_data = _search_orchestrator.run_enhanced_search(query, requested_source, deps) _set_cached_enhanced_search_response(cache_key, response_data) return jsonify(response_data) except Exception as e: logger.error(f"Enhanced search error: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/enhanced-search/source/', methods=['POST']) def enhanced_search_source(source_name): """Streaming NDJSON search for one alternate metadata source. One line per search-kind (artists, albums, tracks) as it completes, plus a final `{"type":"done"}` marker. `youtube_videos` yields a single `videos` chunk via yt-dlp instead. When the requested source's client isn't available (Spotify unauthed, Discogs missing token, Hydrabase disconnected, MusicBrainz import failure, soulseek_client.youtube missing), returns plain JSON `{"artists":[],"albums":[],"tracks":[],"available":false}` to match the original endpoint contract. """ if source_name not in _search_orchestrator.VALID_STREAM_SOURCES: return jsonify({"error": f"Unknown source: {source_name}"}), 400 data = request.get_json() query = (data.get('query', '') if data else '').strip() if not query: return jsonify({"artists": [], "albums": [], "tracks": [], "available": False}) deps = _build_search_deps() if source_name == 'youtube_videos': youtube_client = _search_orchestrator.resolve_youtube_videos_client(deps) if youtube_client is None: return jsonify({"videos": [], "available": False}) try: return app.response_class( _search_orchestrator.stream_youtube_videos(query, youtube_client, run_async), mimetype='application/x-ndjson', ) except Exception as e: return jsonify({"error": str(e)}), 500 try: client, _available = _search_orchestrator.resolve_client(source_name, deps) if client is None: return jsonify({"artists": [], "albums": [], "tracks": [], "available": False}) return app.response_class( _search_orchestrator.stream_metadata_source(source_name, query, client), mimetype='application/x-ndjson', ) except Exception as e: logger.error(f"Enhanced search source ({source_name}) error: {e}") return jsonify({"artists": [], "albums": [], "tracks": [], "available": False}) @app.route('/api/enhanced-search/library-check', methods=['POST']) def enhanced_search_library_check(): """Batch-check which albums/tracks from search results are already in the library. Called async after search renders so badges fade in without blocking results. """ try: data = request.get_json() or {} result = _search_library_check.check_library_presence( database=get_database(), plex_client=plex_client, config_manager=config_manager, profile_id=get_current_profile_id(), albums=data.get('albums', []), tracks=data.get('tracks', []), ) return jsonify(result) except Exception as e: logger.debug(f"Library check error: {e}") return jsonify({'albums': [], 'tracks': []}) @app.route('/api/enhanced-search/stream-track', methods=['POST']) def stream_enhanced_search_track(): """Single-track preview search — finds the best Soulseek/stream-source match. Uses a multi-query retry strategy to work around Soulseek keyword filtering. """ data = request.get_json() track_name = data.get('track_name', '').strip() artist_name = data.get('artist_name', '').strip() album_name = data.get('album_name', '').strip() duration_ms = data.get('duration_ms', 0) if not track_name or not artist_name: return jsonify({"error": "Track name and artist name are required"}), 400 logger.info(f"Enhanced search stream request: '{track_name}' by '{artist_name}'") try: result = _search_stream.stream_search_track( track_name=track_name, artist_name=artist_name, album_name=album_name, duration_ms=duration_ms, config_manager=config_manager, soulseek_client=soulseek_client, matching_engine=matching_engine, run_async=run_async, ) if result is None: return jsonify({ "success": False, "error": "No suitable track found after trying multiple search strategies", }), 404 return jsonify({"success": True, "result": result}) except Exception as e: logger.error(f"Error streaming enhanced search track: {e}", exc_info=True) return jsonify({"error": str(e)}), 500 # ============================================================================= # MUSIC VIDEO DOWNLOADS # ============================================================================= _music_video_downloads = {} # {video_id: {status, progress, path, error}} @app.route('/api/music-video/download', methods=['POST']) def download_music_video(): """Download a YouTube video as a music video file to the configured music videos folder.""" data = request.get_json() if not data: return jsonify({"error": "No data"}), 400 video_id = data.get('video_id', '') video_url = data.get('url', '') raw_title = data.get('title', '') raw_channel = data.get('channel', '') if not video_id or not video_url: return jsonify({"error": "Missing video_id or url"}), 400 # Check if already downloading if video_id in _music_video_downloads and _music_video_downloads[video_id].get('status') == 'downloading': return jsonify({"error": "Already downloading"}), 409 # Get and validate music videos path music_videos_path = config_manager.get('library.music_videos_path', '') or '' if not music_videos_path.strip(): return jsonify({"error": "Music Videos directory not configured. Set it in Settings > Downloads."}), 400 music_videos_path = docker_resolve_path(music_videos_path) try: os.makedirs(music_videos_path, exist_ok=True) # Quick write test test_file = os.path.join(music_videos_path, '.soulsync_write_test') with open(test_file, 'w') as f: f.write('test') os.remove(test_file) except (OSError, PermissionError) as e: return jsonify({"error": f"Music Videos directory is not writable: {e}"}), 400 # Initialize download state _music_video_downloads[video_id] = {'status': 'searching', 'progress': 0, 'path': None, 'error': None} def _do_download(): try: # Step 1: Try to match against primary metadata source for clean artist/title _music_video_downloads[video_id]['status'] = 'matching' artist_name = raw_channel track_title = raw_title year = '' # Strip common YouTube suffixes for cleaner search import re as _re clean_search = _re.sub(r'\s*[\(\[](official\s*(music\s*)?video|official\s*lyric\s*video|official\s*audio|official\s*hd|hd|4k|remastered|lyric\s*video|visualizer|audio)[\)\]]', '', raw_title, flags=_re.IGNORECASE).strip() clean_search = _re.sub(r'\s*-\s*$', '', clean_search).strip() try: fallback_client = _get_metadata_fallback_client() results = fallback_client.search_tracks(clean_search, limit=5) if results: from difflib import SequenceMatcher best = None best_score = 0 for r in results: name_sim = SequenceMatcher(None, clean_search.lower(), r.name.lower()).ratio() if r.artists: artist_sim = SequenceMatcher(None, raw_channel.lower(), r.artists[0].lower()).ratio() name_sim = (name_sim * 0.6) + (artist_sim * 0.4) if name_sim > best_score: best_score = name_sim best = r if best and best_score >= 0.5: artist_name = best.artists[0] if best.artists else raw_channel track_title = best.name if hasattr(best, 'release_date') and best.release_date: year = str(best.release_date)[:4] logger.info(f"[Music Video] Matched to: {artist_name} - {track_title} (confidence: {best_score:.2f})") else: # Parse artist from video title: "Artist - Title" pattern if ' - ' in raw_title: parts = raw_title.split(' - ', 1) artist_name = parts[0].strip() track_title = _re.sub(r'\s*[\(\[].*?[\)\]]', '', parts[1]).strip() logger.warning(f"[Music Video] No metadata match, using parsed: {artist_name} - {track_title}") except Exception as e: logger.error(f"[Music Video] Metadata lookup failed: {e}") if ' - ' in raw_title: parts = raw_title.split(' - ', 1) artist_name = parts[0].strip() track_title = _re.sub(r'\s*[\(\[].*?[\)\]]', '', parts[1]).strip() # Sanitize for filesystem def _sanitize(s): return _re.sub(r'[<>:"/\\|?*]', '_', s).strip().rstrip('.') # Apply video path template video_template = config_manager.get('file_organization.templates', {}).get('video_path', '$artist/$title-video') if not video_template or not video_template.strip(): video_template = '$artist/$title-video' safe_artist = _sanitize(artist_name) video_path = video_template video_path = video_path.replace('$artistletter', safe_artist[0].upper() if safe_artist else 'A') video_path = video_path.replace('$artist', safe_artist) video_path = video_path.replace('$title', _sanitize(track_title)) video_path = video_path.replace('$year', str(year) if year else '') # Clean up empty segments from missing variables video_path = _re.sub(r'//+', '/', video_path).strip('/') # Split into folder and filename path_parts = video_path.rsplit('/', 1) if len(path_parts) == 2: folder_part, file_part = path_parts else: folder_part, file_part = '', path_parts[0] output_dir = os.path.join(music_videos_path, folder_part) if folder_part else music_videos_path os.makedirs(output_dir, exist_ok=True) output_path = os.path.join(output_dir, file_part) # Step 2: Download _music_video_downloads[video_id]['status'] = 'downloading' _music_video_downloads[video_id]['artist'] = artist_name _music_video_downloads[video_id]['title'] = track_title def _progress(pct): _music_video_downloads[video_id]['progress'] = round(pct, 1) final_path = soulseek_client.youtube.download_music_video(video_url, output_path, progress_callback=_progress) if final_path and os.path.exists(final_path): _music_video_downloads[video_id]['status'] = 'completed' _music_video_downloads[video_id]['progress'] = 100 _music_video_downloads[video_id]['path'] = final_path logger.info(f"[Music Video] Downloaded: {artist_name} - {track_title} → {final_path}") add_activity_item("", "Music Video Downloaded", f"{artist_name} - {track_title}", "Now") else: _music_video_downloads[video_id]['status'] = 'error' _music_video_downloads[video_id]['error'] = 'Download failed — file not found' logger.error(f"[Music Video] Download failed for: {artist_name} - {track_title}") except Exception as e: _music_video_downloads[video_id]['status'] = 'error' _music_video_downloads[video_id]['error'] = str(e) logger.error(f"[Music Video] {e}") # Run in background thread import threading threading.Thread(target=_do_download, daemon=True, name=f'music-video-{video_id}').start() return jsonify({"success": True, "video_id": video_id}) @app.route('/api/music-video/status/', methods=['GET']) def get_music_video_status(video_id): """Get download status for a music video.""" status = _music_video_downloads.get(video_id) if not status: return jsonify({"status": "unknown"}) return jsonify(status) @app.route('/api/download', methods=['POST']) def start_download(): """Simple download route""" dl_err = check_download_permission() if dl_err: return dl_err data = request.get_json() if not data: return jsonify({"error": "No download data provided."}), 400 try: result_type = data.get('result_type', 'track') if result_type == 'album': tracks = data.get('tracks', []) if not tracks: return jsonify({"error": "No tracks found in album."}), 400 started_downloads = 0 for track_data in tracks: try: username = track_data.get('username') filename = track_data.get('filename') file_size = track_data.get('size', 0) download_id = run_async(soulseek_client.download( username, filename, file_size )) if download_id: # Register download for post-processing (simple transfer to /Transfer) context_key = _make_context_key(username, filename) with matched_context_lock: matched_downloads_context[context_key] = { 'search_result': { 'username': username, 'filename': filename, 'size': file_size, 'title': track_data.get('title', 'Unknown'), 'artist': track_data.get('artist', 'Unknown'), 'quality': track_data.get('quality', 'Unknown'), 'is_simple_download': True # Flag for simple processing }, 'spotify_artist': None, # No Spotify metadata 'spotify_album': None, 'track_info': None } started_downloads += 1 except Exception as e: logger.error(f"Failed to start track download: {e}") continue # Add activity for album download start album_name = data.get('album_name', 'Unknown Album') logger.info(f"Starting simple album download: '{album_name}' with {started_downloads}/{len(tracks)} tracks") add_activity_item("", "Album Download Started", f"'{album_name}' - {started_downloads} tracks", "Now") return jsonify({ "success": True, "message": f"Started {started_downloads} downloads from album" }) else: # Single track download username = data.get('username') filename = data.get('filename') file_size = data.get('size', 0) logger.info(f"Download request - Username: {username}, Filename: {filename[:50]}...") if not username or not filename: return jsonify({"error": "Missing username or filename."}), 400 download_id = run_async(soulseek_client.download(username, filename, file_size)) logger.info(f"Download ID returned: {download_id}") if download_id: # Register download for post-processing (simple transfer to /Transfer) context_key = _make_context_key(username, filename) is_streaming_source = username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr') with matched_context_lock: matched_downloads_context[context_key] = { 'search_result': { 'username': username, 'filename': filename, 'size': file_size, 'title': data.get('title', 'Unknown'), 'artist': data.get('artist', 'Unknown'), 'quality': data.get('quality', 'Unknown'), 'is_simple_download': True # Flag for simple processing }, 'spotify_artist': None, # No Spotify metadata 'spotify_album': None, 'track_info': None } source_label = username.title() if is_streaming_source else 'Soulseek' logger.info(f"[{source_label}] Registered simple download for post-processing: {context_key}") # Extract track name from filename for activity track_name = filename.split('/')[-1] if '/' in filename else filename.split('\\')[-1] if '\\' in filename else filename logger.info(f"Starting simple track download: '{track_name}'") add_activity_item("", "Track Download Started", f"'{track_name}'", "Now") return jsonify({"success": True, "message": "Download started"}) else: logger.error(f"Failed to start download for: {filename}") return jsonify({"error": "Failed to start download"}), 500 except Exception as e: logger.error(f"Download error: {e}") return jsonify({"error": str(e)}), 500 def _find_completed_file_robust(download_dir, api_filename, transfer_dir=None): """ Robustly finds a completed file on disk, accounting for name variations and unexpected subdirectories. This version uses the superior normalization logic from the GUI's matching_engine.py to ensure consistency. First searches in download_dir, then optionally searches in transfer_dir if provided. Returns tuple (file_path, location) where location is 'downloads' or 'transfer'. """ import re import os from difflib import SequenceMatcher from unidecode import unidecode # YOUTUBE/TIDAL SUPPORT: Handle encoded filename format "id||title" # Extract just the title part for file matching if '||' in api_filename: _, title = api_filename.split('||', 1) api_filename = title # Use just the title for file searching def normalize_for_finding(text: str) -> str: """A powerful normalization function adapted from matching_engine.py.""" if not text: return "" text = unidecode(text).lower() # Replace common separators with spaces to preserve word boundaries text = re.sub(r'[._/]', ' ', text) # Keep alphanumeric, spaces, and hyphens. Remove brackets/parentheses content. text = re.sub(r'[\[\(].*?[\]\)]', '', text) text = re.sub(r'[^a-z0-9\s-]', '', text) # Consolidate multiple spaces return ' '.join(text.split()).strip() def _path_matches_api_dirs(file_path): """Check if ALL api directory components appear in the file's path.""" path_parts = set(p.lower() for p in file_path.replace('\\', '/').split('/')) return all(d in path_parts for d in api_dir_parts) def search_in_directory(search_dir, location_name): """Search for the file in a specific directory.""" best_fuzzy_path = None highest_fuzzy_similarity = 0.0 exact_matches = [] # Walk through the entire directory for root, dirs, files in os.walk(search_dir): # Skip quarantine folder — contains known-wrong files from AcoustID verification dirs[:] = [d for d in dirs if d != 'ss_quarantine'] for file in files: # Direct basename match if os.path.basename(file) == target_basename: file_path = os.path.join(root, file) # Fast path: if path aligns with expected directory structure, return now if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed match in {location_name}: {file_path}") return file_path, 1.0 if not api_dir_parts: # No directory info to disambiguate — return first match (original behavior) logger.info(f"Found exact match in {location_name}: {file_path}") return file_path, 1.0 exact_matches.append(file_path) continue # Check for slskd dedup suffix (e.g. "Song_639067852665564677.flac") # slskd appends _ when a file with the same name already exists file_stem, file_ext_part = os.path.splitext(file) stripped_stem = re.sub(r'_\d{10,}$', '', file_stem) if stripped_stem != file_stem and stripped_stem + file_ext_part == target_basename: file_path = os.path.join(root, file) if api_dir_parts and _path_matches_api_dirs(file_path): logger.info(f"Found path-confirmed dedup match in {location_name}: {file_path}") return file_path, 1.0 if not api_dir_parts: logger.info(f"Found dedup-suffix match in {location_name}: {file_path}") return file_path, 1.0 exact_matches.append(file_path) continue # Fuzzy matching for variations normalized_file = normalize_for_finding(file) similarity = SequenceMatcher(None, normalized_target, normalized_file).ratio() if similarity > highest_fuzzy_similarity: highest_fuzzy_similarity = similarity best_fuzzy_path = os.path.join(root, file) # Return best exact match (disambiguated by path), or fall back to fuzzy if exact_matches: if len(exact_matches) == 1: logger.info(f"Found exact match in {location_name}: {exact_matches[0]}") return exact_matches[0], 1.0 # Multiple files share the basename — pick the one whose path best # matches the expected directory structure from the Soulseek remote path best = exact_matches[0] best_score = -1 for m in exact_matches: m_parts = set(p.lower() for p in m.replace('\\', '/').split('/')) score = sum(1 for d in api_dir_parts if d in m_parts) if score > best_score: best_score = score best = m logger.info(f"Found {len(exact_matches)} files named '{target_basename}' in {location_name}, picked best path match: {best}") return best, 1.0 return best_fuzzy_path, highest_fuzzy_similarity # Extract filename using the helper function target_basename = extract_filename(api_filename) normalized_target = normalize_for_finding(target_basename) # Extract directory components from the API path for disambiguation. # When multiple downloads produce the same basename (e.g., "01 - Silent Night.flac" # from different albums/users), these let us pick the correct file on disk. api_path_normalized = api_filename.replace('\\', '/') if api_filename else '' api_dir_parts = [p.lower() for p in api_path_normalized.split('/')[:-1] if p] # First search in downloads directory best_downloads_path, downloads_similarity = search_in_directory(download_dir, 'downloads') # Use a high confidence threshold for fuzzy matches to prevent false positives if downloads_similarity > 0.85: location = 'downloads' if downloads_similarity < 1.0: logger.info(f"Found fuzzy match in downloads ({downloads_similarity:.2f}): {best_downloads_path}") return (best_downloads_path, location) # If not found in downloads and transfer_dir is provided, search there transfer_similarity = 0.0 # Initialize transfer_similarity if transfer_dir and os.path.exists(transfer_dir): best_transfer_path, transfer_similarity = search_in_directory(transfer_dir, 'transfer') if transfer_similarity > 0.85: location = 'transfer' if transfer_similarity < 1.0: logger.info(f"Found fuzzy match in transfer ({transfer_similarity:.2f}): {best_transfer_path}") return (best_transfer_path, location) # Don't spam logs - file not found is common for completed/processed downloads return (None, None) @app.route('/api/downloads/status') def get_download_status(): """ A robust status checker that correctly finds completed files by searching the entire download directory with fuzzy matching, mirroring the logic from downloads.py. """ if not soulseek_client: return jsonify({"transfers": []}) try: global processed_download_ids # Skip slskd API call if Soulseek is already known to be disconnected soulseek_known_down = not _status_cache.get('soulseek', {}).get('connected', True) transfers_data = None if not soulseek_known_down: transfers_data = run_async(soulseek_client._make_request('GET', 'transfers/downloads')) # Don't return early if no Soulseek transfers - YouTube/Tidal downloads need to be checked too! all_transfers = [] completed_matched_downloads = [] # Track files already claimed this poll cycle to prevent two contexts from # grabbing the same physical file when downloads share a basename (e.g., # "07 - Aurora.flac" from two different albums/artists). _files_claimed_this_cycle = set() if not transfers_data: # No Soulseek transfers, but continue to check YouTube/Tidal downloads below pass else: # This logic now correctly processes the nested structure from the slskd API for user_data in transfers_data: username = user_data.get('username', 'Unknown') if 'directories' in user_data: for directory in user_data['directories']: if 'files' in directory: for file_info in directory['files']: file_info['username'] = username all_transfers.append(file_info) state = file_info.get('state', '').lower() # Check for completion state if ('succeeded' in state or 'completed' in state) and 'errored' not in state and 'rejected' not in state: # Verify bytes actually transferred before trusting state _fi_size = file_info.get('size', 0) _fi_transferred = file_info.get('bytesTransferred', 0) if _fi_size > 0 and _fi_transferred < _fi_size: continue # Not truly complete yet filename_from_api = file_info.get('filename') if not filename_from_api: continue # Check if this completed download has a matched context context_key = _make_context_key(username, filename_from_api) # Check if this is an orphaned download that completed after retry if context_key in _orphaned_download_keys: # Safety check: if a new context exists for this key, the retry # re-claimed the same source — treat it as active, not orphaned with matched_context_lock: has_active_context = context_key in matched_downloads_context if has_active_context: logger.warning(f"Orphaned key {context_key} has active context — retry re-used same source, treating as active") _orphaned_download_keys.discard(context_key) # Fall through to normal processing below else: download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) found_result = _find_completed_file_robust(download_dir, filename_from_api) found_path = found_result[0] if found_result and found_result[0] else None orphan_cleaned = False if found_path: try: os.remove(found_path) logger.warning(f"Deleted orphaned download: {os.path.basename(found_path)}") orphan_cleaned = True except Exception as e: logger.error(f"Failed to delete orphaned file (will retry next poll): {e}") else: # File not on disk (already gone or never written) — nothing to clean orphan_cleaned = True if orphan_cleaned: # Remove transfer from slskd transfer_id = file_info.get('id') if transfer_id: try: run_async(soulseek_client.cancel_download(str(transfer_id), username, remove=True)) except Exception: pass _orphaned_download_keys.discard(context_key) continue # Skip normal post-processing either way # Skip downloads we've already processed (prevents log spam) if context_key in processed_download_ids: continue with matched_context_lock: context = matched_downloads_context.get(context_key) available_keys = list(matched_downloads_context.keys())[:5] if not context else None if context: logger.info(f"[Context Lookup] Found context for key: {context_key}") elif context_key not in _stale_transfer_keys: # Only log once per stale key to avoid spamming every poll cycle logger.warning(f"[Context Lookup] No context found for key: {context_key}") logger.info(f" Available keys: {available_keys}...") _stale_transfer_keys.add(context_key) if context: download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) # Use the new robust file finder (only search downloads for post-processing candidates) found_result = _find_completed_file_robust(download_dir, filename_from_api) found_path = found_result[0] if found_result and found_result[0] else None if found_path: # Prevent two contexts from claiming the same physical file _norm_path = os.path.normpath(found_path) if _norm_path in _files_claimed_this_cycle: logger.info(f"File already claimed by another context this cycle: {os.path.basename(found_path)} — deferring to next poll") else: _files_claimed_this_cycle.add(_norm_path) logger.info(f"Found completed matched file on disk: {found_path}") completed_matched_downloads.append((context_key, context, found_path)) # Don't add to processed_download_ids yet - wait until thread starts successfully # Clean up retry tracking if file was found after retries with _download_retry_lock: if context_key in _download_retry_attempts: retry_count = _download_retry_attempts[context_key]['count'] elapsed = time.time() - _download_retry_attempts[context_key]['first_attempt'] logger.warning(f"File found after {retry_count} retry attempt(s) ({elapsed:.1f}s): {os.path.basename(filename_from_api)}") del _download_retry_attempts[context_key] else: # File not found yet - implement retry logic instead of immediate give-up # This fixes race condition where slskd reports completion before file is written to disk with _download_retry_lock: if context_key not in _download_retry_attempts: # First retry attempt _download_retry_attempts[context_key] = { 'count': 1, 'first_attempt': time.time() } logger.warning(f"File not found yet: '{os.path.basename(filename_from_api)}' - Will retry (attempt 1/{_download_retry_max})") else: # Increment retry count _download_retry_attempts[context_key]['count'] += 1 retry_count = _download_retry_attempts[context_key]['count'] elapsed = time.time() - _download_retry_attempts[context_key]['first_attempt'] if retry_count >= _download_retry_max: # Max retries reached, give up logger.error(f"CRITICAL: Could not find '{os.path.basename(filename_from_api)}' after {retry_count} attempts over {elapsed:.1f}s. Giving up.") processed_download_ids.add(context_key) # Clean up retry tracking del _download_retry_attempts[context_key] else: logger.warning(f"File not found yet: '{os.path.basename(filename_from_api)}' - Will retry (attempt {retry_count}/{_download_retry_max}, elapsed: {elapsed:.1f}s)") # If we found completed matched downloads, start processing them in background threads if completed_matched_downloads: def process_completed_downloads(): for context_key, context, found_path in completed_matched_downloads: try: logger.info(f"Starting post-processing thread for: {context_key}") # Use verification wrapper if context has task tracking IDs, # otherwise call directly (race guard flag still gets set on context) _pp_task_id = context.get('task_id') _pp_batch_id = context.get('batch_id') if _pp_task_id and _pp_batch_id: _pp_target = _post_process_matched_download_with_verification _pp_args = (context_key, context, found_path, _pp_task_id, _pp_batch_id) else: _pp_target = _post_process_matched_download _pp_args = (context_key, context, found_path) thread = threading.Thread(target=_pp_target, args=_pp_args) thread.daemon = True thread.start() # Only mark as processed AFTER thread starts successfully processed_download_ids.add(context_key) logger.info(f"Marked as processed: {context_key}") # DON'T remove context immediately - verification worker needs it # Context will be cleaned up by verification worker after both processors complete logger.info(f"Keeping context for verification worker: {context_key}") except Exception as e: logger.error(f"Error starting post-processing thread for {context_key}: {e}") # Don't add to processed set if thread failed to start logger.warning(f"Will retry {context_key} on next check") # Start a single thread to manage the launching of all processing threads processing_thread = threading.Thread(target=process_completed_downloads) processing_thread.daemon = True processing_thread.start() # Also include YouTube/Tidal downloads in the response try: all_streaming_downloads = run_async(soulseek_client.get_all_downloads()) for download in all_streaming_downloads: if download.username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): source_label = download.username.title() # Convert DownloadStatus to transfer format that frontend expects streaming_transfer = { 'id': download.id, 'filename': download.filename, 'username': download.username, 'state': download.state, 'percentComplete': download.progress, 'size': download.size, 'bytesTransferred': download.transferred, 'averageSpeed': download.speed, 'direction': 'Download', # Required by frontend } all_transfers.append(streaming_transfer) # Check if download is completed and needs post-processing # Verify bytes match before trusting state string _st_bytes_ok = download.size <= 0 or download.transferred >= download.size if _st_bytes_ok and download.state and ('succeeded' in download.state.lower() or 'completed' in download.state.lower()): context_key = _make_context_key(download.username, download.filename) with matched_context_lock: context = matched_downloads_context.get(context_key) if context and context_key not in processed_download_ids: download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) found_result = _find_completed_file_robust(download_dir, download.filename) found_path = found_result[0] if found_result and found_result[0] else None if found_path: # Prevent two contexts from claiming the same physical file _st_norm = os.path.normpath(found_path) if _st_norm in _files_claimed_this_cycle: logger.info(f"[{source_label}] File already claimed this cycle: {os.path.basename(found_path)} — deferring") continue _files_claimed_this_cycle.add(_st_norm) logger.info(f"[{source_label}] Found completed matched file on disk: {found_path}") # Start post-processing thread def process_streaming_download(_ctx_key=context_key, _ctx=context, _path=found_path, _label=source_label): try: logger.info(f"[{_label}] Starting post-processing thread for: {_ctx_key}") # Use verification wrapper if context has task tracking IDs _st_task_id = _ctx.get('task_id') _st_batch_id = _ctx.get('batch_id') if _st_task_id and _st_batch_id: _st_target = _post_process_matched_download_with_verification _st_args = (_ctx_key, _ctx, _path, _st_task_id, _st_batch_id) else: _st_target = _post_process_matched_download _st_args = (_ctx_key, _ctx, _path) thread = threading.Thread(target=_st_target, args=_st_args) thread.daemon = True thread.start() processed_download_ids.add(_ctx_key) logger.info(f"[{_label}] Marked as processed: {_ctx_key}") except Exception as e: logger.error(f"[{_label}] Error starting post-processing thread for {_ctx_key}: {e}") processing_thread = threading.Thread(target=process_streaming_download) processing_thread.daemon = True processing_thread.start() else: # File not found - likely already processed and moved to library # Mark as processed to prevent infinite checking processed_download_ids.add(context_key) except Exception as streaming_error: import traceback logger.error(f"Could not fetch YouTube/Tidal downloads for status: {streaming_error}") traceback.print_exc() # Enrich transfers with metadata from download context (artist, album, artwork) with matched_context_lock: for transfer in all_transfers: ctx_key = _make_context_key(transfer.get('username', ''), transfer.get('filename', '')) ctx = matched_downloads_context.get(ctx_key) if ctx: _sp_artist = ctx.get('spotify_artist') or {} _sp_album = ctx.get('spotify_album') or {} _sp_track = ctx.get('track_info') or {} # Cover art can live in a few places depending on the # origin path: album.images[0].url (Spotify-shaped), # album.image_url (context-built / wishlist / fixes), or # track_info.image_url (some discovery flows). Try them # all so fixed-discovery tracks render their artwork. _sp_images = _sp_album.get('images') or [] _art_url = '' if _sp_images and isinstance(_sp_images[0], dict): _art_url = _sp_images[0].get('url', '') or '' if not _art_url: _art_url = _sp_album.get('image_url') or '' if not _art_url: _art_url = _sp_track.get('image_url') or '' transfer['_meta'] = { 'artist': _sp_artist.get('name', ''), 'album': _sp_album.get('name', ''), 'artwork_url': _art_url, 'track_number': _sp_track.get('track_number'), 'quality': ctx.get('_audio_quality', ''), } return jsonify({"transfers": all_transfers}) except Exception as e: logger.error(f"Error fetching download status: {e}") return jsonify({"error": str(e)}), 500 # Cancel + clear logic lives in core/downloads/cancel.py — these routes are thin handlers. from core.downloads import cancel as _downloads_cancel @app.route('/api/downloads/cancel', methods=['POST']) def cancel_download(): """Cancel a specific download transfer, matching GUI functionality.""" data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided."}), 400 download_id = data.get('download_id') username = data.get('username') if not all([download_id, username]): return jsonify({"success": False, "error": "Missing download_id or username."}), 400 try: success = _downloads_cancel.cancel_single_download(soulseek_client, run_async, download_id, username) if success: return jsonify({"success": True, "message": "Download cancelled."}) return jsonify({"success": False, "error": "Failed to cancel download via slskd."}), 500 except Exception as e: logger.error(f"Error cancelling download: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/downloads/cancel-all', methods=['POST']) def cancel_all_downloads(): """Cancel all active downloads from slskd, then clear completed ones.""" try: success, msg = _downloads_cancel.cancel_all_active( soulseek_client, run_async, _sweep_empty_download_directories, ) if success: return jsonify({"success": True, "message": msg}) return jsonify({"success": False, "error": msg}), 500 except Exception as e: logger.error(f"Error cancelling all downloads: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/downloads/clear-finished', methods=['POST']) def clear_finished_downloads(): """Clear all terminal (completed, cancelled, failed) downloads from slskd.""" try: success = _downloads_cancel.clear_finished_active( soulseek_client, run_async, _sweep_empty_download_directories, ) if success: return jsonify({"success": True, "message": "Finished downloads cleared."}) return jsonify({"success": False, "error": "Backend failed to clear downloads."}), 500 except Exception as e: logger.error(f"Error clearing finished downloads: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/downloads/task//candidates', methods=['GET']) def get_task_candidates(task_id): """Returns the cached search candidates for a download task so the UI can show what was found.""" try: with tasks_lock: task = download_tasks.get(task_id) if not task: return jsonify({"error": "Task not found"}), 404 candidates = task.get('cached_candidates', []) track_info = task.get('track_info', {}) error_message = task.get('error_message', '') serialized = [] for c in candidates: if hasattr(c, '__dict__'): serialized.append({ 'username': getattr(c, 'username', ''), 'filename': getattr(c, 'filename', ''), 'size': getattr(c, 'size', 0), 'bitrate': getattr(c, 'bitrate', None), 'duration': getattr(c, 'duration', None), 'quality': getattr(c, 'quality', ''), 'free_upload_slots': getattr(c, 'free_upload_slots', 0), 'upload_speed': getattr(c, 'upload_speed', 0), 'queue_length': getattr(c, 'queue_length', 0), 'artist': getattr(c, 'artist', None), 'title': getattr(c, 'title', None), 'album': getattr(c, 'album', None), }) elif isinstance(c, dict): serialized.append(c) return jsonify({ "task_id": task_id, "track_info": { "name": track_info.get('name', 'Unknown') if isinstance(track_info, dict) else 'Unknown', "artist": _get_track_artist_name(track_info) if isinstance(track_info, dict) else 'Unknown', }, "error_message": error_message, "candidates": serialized, "candidate_count": len(serialized), }) except Exception as e: logger.error(f"[Candidates] Error fetching candidates for task {task_id}: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/downloads/task//download-candidate', methods=['POST']) def download_selected_candidate(task_id): """Restart a not_found/failed task by downloading a user-selected candidate.""" dl_err = check_download_permission() if dl_err: return dl_err try: data = request.get_json() if not data or not data.get('username') or not data.get('filename'): return jsonify({"error": "Missing username or filename"}), 400 username = data['username'] filename = data['filename'] size = data.get('size', 0) with tasks_lock: task = download_tasks.get(task_id) if not task: return jsonify({"error": "Task not found"}), 404 if task['status'] not in ('not_found', 'failed'): return jsonify({"error": f"Task is {task['status']}, not eligible for retry"}), 400 batch_id = task.get('batch_id') track_info = task.get('track_info', {}) # Reset task state task['status'] = 'downloading' task['error_message'] = None task['status_change_time'] = time.time() task.pop('download_id', None) task.pop('username', None) task.pop('filename', None) # Clear the selected candidate from used_sources so it won't be skipped used_sources = task.get('used_sources', set()) source_key = f"{username}_{filename}" used_sources.discard(source_key) # Reset batch tracking for this task if batch_id and batch_id in download_batches: batch = download_batches[batch_id] # Remove from completed set so _on_download_completed can fire again completed_set = batch.get('_completed_task_ids', set()) completed_set.discard(task_id) # Remove from permanently_failed_tracks track_index = task.get('track_index') batch['permanently_failed_tracks'] = [ t for t in batch.get('permanently_failed_tracks', []) if t.get('table_index') != track_index and t.get('download_index') != track_index ] # Restore worker slot batch['active_count'] = batch.get('active_count', 0) + 1 # Build a TrackResult-like candidate object from core.soulseek_client import TrackResult candidate = TrackResult( username=username, filename=filename, size=size, bitrate=data.get('bitrate'), duration=data.get('duration'), quality=data.get('quality', 'unknown'), free_upload_slots=data.get('free_upload_slots', 0), upload_speed=data.get('upload_speed', 0), queue_length=data.get('queue_length', 0), artist=data.get('artist'), title=data.get('title'), album=data.get('album'), ) candidate.confidence = 1.0 # Required by _attempt_download_with_candidates sort # Reconstruct Track object from task's track_info from core.itunes_client import Track artists = track_info.get('artists', []) artist_names = [] for a in (artists if isinstance(artists, list) else []): if isinstance(a, dict): artist_names.append(a.get('name', 'Unknown')) elif isinstance(a, str): artist_names.append(a) if not artist_names: artist_names = [track_info.get('artist', 'Unknown')] track = Track( id=track_info.get('id', ''), name=track_info.get('name', 'Unknown'), artists=artist_names, album=track_info.get('album', {}).get('name', '') if isinstance(track_info.get('album'), dict) else track_info.get('album', ''), duration_ms=track_info.get('duration_ms', 0), popularity=0, ) # Submit to thread pool — don't block the request def _run_manual_download(): success = _attempt_download_with_candidates(task_id, [candidate], track, batch_id) if not success: with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'failed' download_tasks[task_id]['error_message'] = 'Manual download failed to start — user may be offline' if batch_id: _on_download_completed(batch_id, task_id, success=False) missing_download_executor.submit(_run_manual_download) track_name = track_info.get('name', 'Unknown') logger.info(f"[Manual Download] User selected candidate for '{track_name}' from {username}") return jsonify({"success": True, "message": f"Download initiated for '{track_name}'"}) except Exception as e: logger.error(f"[Manual Download] {e}") import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/quarantine/clear', methods=['POST']) def clear_quarantine(): """Delete all files and folders inside the ss_quarantine directory.""" import shutil try: download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) quarantine_path = os.path.join(download_path, 'ss_quarantine') if not os.path.isdir(quarantine_path): return jsonify({"success": True, "message": "Quarantine folder is already empty."}) removed_files = 0 for entry in os.listdir(quarantine_path): entry_path = os.path.join(quarantine_path, entry) try: if os.path.isfile(entry_path): os.remove(entry_path) removed_files += 1 elif os.path.isdir(entry_path): shutil.rmtree(entry_path) removed_files += 1 except Exception as e: logger.error(f"[Quarantine] Failed to remove {entry}: {e}") logger.info(f"[Quarantine] Cleared {removed_files} item(s) from quarantine folder") return jsonify({"success": True, "message": f"Quarantine cleared ({removed_files} item{'s' if removed_files != 1 else ''} removed)."}) except Exception as e: logger.error(f"[Quarantine] Error clearing quarantine: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/scan/request', methods=['POST']) def request_media_scan(): """ Request a media library scan. Database update is handled automatically by the 'Auto-Update Database After Scan' system automation. """ try: if not web_scan_manager: return jsonify({"success": False, "error": "Scan manager not initialized"}), 500 data = request.get_json() or {} reason = data.get('reason', 'Web UI download completed') result = web_scan_manager.request_scan(reason=reason) add_activity_item("", "Media Scan", f"Scan requested: {reason}", "Now") return jsonify({ "success": True, "scan_info": result, }) except Exception as e: logger.error(f"Error requesting media scan: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/scan/status', methods=['GET']) def get_scan_status(): """ Get current media scan status. """ try: if not web_scan_manager: return jsonify({"success": False, "error": "Scan manager not initialized"}), 500 status = web_scan_manager.get_scan_status() return jsonify({"success": True, "status": status}) except Exception as e: logger.error(f"Error getting scan status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/incremental-update', methods=['POST']) def request_incremental_database_update(): """ Request an incremental database update with prerequisites checking. """ try: data = request.get_json() or {} reason = data.get('reason', 'Web UI manual request') # Check prerequisites (similar to GUI logic) db = get_database() # Check if database has enough content for incremental updates track_count = db.execute("SELECT COUNT(*) FROM tracks").fetchone()[0] if track_count < 100: return jsonify({ "success": False, "error": f"Database has only {track_count} tracks - insufficient for incremental updates (minimum 100)", "track_count": track_count }), 400 # Check if there's been a previous full refresh last_refresh = db.execute( "SELECT value FROM system_info WHERE key = 'last_full_refresh'" ).fetchone() if not last_refresh: return jsonify({ "success": False, "error": "No previous full refresh found - incremental updates require established database", "suggestion": "Run a full refresh first" }), 400 # Start incremental update if db_update_state.get('status') == 'running': return jsonify({"success": False, "error": "Database update already running"}), 409 active_server = config_manager.get_active_media_server() with db_update_lock: db_update_state.update({ "status": "running", "phase": "Initializing...", "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" }) db_update_executor.submit(_run_db_update_task, False, active_server) add_activity_item("", "Database Update", f"Incremental update started: {reason}", "Now") return jsonify({ "success": True, "message": "Incremental database update started", "track_count": track_count, "last_refresh": last_refresh[0] if last_refresh else None, "reason": reason }) except Exception as e: logger.error(f"Error requesting incremental database update: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/test/automation', methods=['POST']) def test_automation_workflow(): """ Test endpoint to verify the automatic workflow functionality. """ try: data = request.get_json() or {} test_type = data.get('test_type', 'full') results = {} # Test 1: Scan manager status if web_scan_manager: scan_status = web_scan_manager.get_scan_status() results['scan_manager'] = {'status': 'available', 'current_status': scan_status} else: results['scan_manager'] = {'status': 'unavailable'} # Test 2: Database prerequisites try: db = get_database() track_count = db.execute("SELECT COUNT(*) FROM tracks").fetchone()[0] last_refresh = db.execute( "SELECT value FROM system_info WHERE key = 'last_full_refresh'" ).fetchone() results['database'] = { 'track_count': track_count, 'meets_minimum': track_count >= 100, 'has_previous_refresh': last_refresh is not None, 'last_refresh': last_refresh[0] if last_refresh else None } except Exception as e: results['database'] = {'error': str(e)} # Test 3: Media client connections active_server = config_manager.get_active_media_server() results['media_clients'] = {'active_server': active_server} for client_name, client in [ ('plex', plex_client), ('jellyfin', jellyfin_client), ('navidrome', navidrome_client) ]: try: is_connected = client.is_connected() if client else False results['media_clients'][client_name] = { 'available': client is not None, 'connected': is_connected } except Exception as e: results['media_clients'][client_name] = { 'available': client is not None, 'connected': False, 'error': str(e) } # Test 4: If requested, actually test the scan request if test_type == 'full' and web_scan_manager: try: scan_result = web_scan_manager.request_scan( reason="Automation test", callback=None ) results['scan_test'] = {'success': True, 'result': scan_result} except Exception as e: results['scan_test'] = {'success': False, 'error': str(e)} return jsonify({ "success": True, "test_results": results, "automation_ready": ( results.get('scan_manager', {}).get('status') == 'available' and results.get('database', {}).get('meets_minimum', False) and results.get('database', {}).get('has_previous_refresh', False) ) }) except Exception as e: logger.error(f"Error in automation test: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/searches/clear-all', methods=['POST']) def clear_all_searches(): """ Clear all searches from slskd search history. """ try: success = run_async(soulseek_client.clear_all_searches()) if success: add_activity_item("", "Search Cleanup", "All search history cleared manually", "Now") return jsonify({"success": True, "message": "All searches cleared."}) else: return jsonify({"success": False, "error": "Backend failed to clear searches."}), 500 except Exception as e: logger.error(f"Error clearing searches: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/searches/maintain', methods=['POST']) def maintain_search_history(): """ Maintain search history by keeping only recent searches. """ try: data = request.get_json() or {} keep_searches = data.get('keep_searches', 50) trigger_threshold = data.get('trigger_threshold', 200) success = run_async(soulseek_client.maintain_search_history_with_buffer( keep_searches=keep_searches, trigger_threshold=trigger_threshold )) if success: add_activity_item("", "Search Maintenance", f"Search history maintained (keeping {keep_searches} searches)", "Now") return jsonify({"success": True, "message": f"Search history maintained (keeping {keep_searches} searches)."}) else: return jsonify({"success": False, "error": "Backend failed to maintain search history."}), 500 except Exception as e: logger.error(f"Error maintaining search history: {e}") return jsonify({"success": False, "error": str(e)}), 500 def fix_artist_image_url(thumb_url): """Convert media-server image URLs into browser-safe URLs.""" if not thumb_url: return None try: # Check if it's a localhost URL or relative path that needs fixing needs_fixing = ( thumb_url.startswith('http://localhost:') or thumb_url.startswith('https://localhost:') or thumb_url.startswith('http://127.0.0.1:') or thumb_url.startswith('https://127.0.0.1:') or thumb_url.startswith('http://host.docker.internal:') or thumb_url.startswith('https://host.docker.internal:') or (thumb_url.startswith('http://') and _is_internal_image_host(thumb_url)) or thumb_url.startswith('/library/') or # Plex relative paths thumb_url.startswith('/Items/') or # Jellyfin relative paths thumb_url.startswith('/api/') or # Old Navidrome API paths thumb_url.startswith('/rest/') # Navidrome Subsonic API paths ) if needs_fixing: active_server = config_manager.get_active_media_server() logger.debug(f"Fixing URL: {thumb_url}, Active server: {active_server}") if active_server == 'plex': plex_config = config_manager.get_plex_config() plex_base_url = plex_config.get('base_url', '') plex_token = plex_config.get('token', '') logger.info(f"Plex config - base_url: {plex_base_url}, token: {plex_token[:10]}...") if plex_base_url and plex_token: # Extract the path from URL if thumb_url.startswith('/library/'): # Already a path path = thumb_url else: # Full localhost URL, extract path from urllib.parse import urlparse parsed = urlparse(thumb_url) path = parsed.path # Construct proper Plex URL with token fixed_url = f"{plex_base_url.rstrip('/')}{path}?X-Plex-Token={plex_token}" logger.info(f"Fixed URL: {fixed_url}") return _browser_safe_image_url(fixed_url) elif active_server == 'jellyfin': jellyfin_config = config_manager.get_jellyfin_config() jellyfin_base_url = jellyfin_config.get('base_url', '') jellyfin_token = jellyfin_config.get('api_key', '') logger.info(f"Jellyfin config - base_url: {jellyfin_base_url}, token: {jellyfin_token[:10] if jellyfin_token else 'None'}...") if jellyfin_base_url: # Extract the path from URL if thumb_url.startswith('/Items/') or thumb_url.startswith('/api/'): # Already a path path = thumb_url else: # Full localhost URL, extract path from urllib.parse import urlparse parsed = urlparse(thumb_url) path = parsed.path # Construct proper Jellyfin URL with token if jellyfin_token: separator = '&' if '?' in path else '?' fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}{separator}X-Emby-Token={jellyfin_token}" else: fixed_url = f"{jellyfin_base_url.rstrip('/')}{path}" logger.info(f"Fixed URL: {fixed_url}") return _browser_safe_image_url(fixed_url) elif active_server == 'navidrome': navidrome_config = config_manager.get_navidrome_config() navidrome_base_url = navidrome_config.get('base_url', '') navidrome_username = navidrome_config.get('username', '') navidrome_password = navidrome_config.get('password', '') logger.info(f"Navidrome config - base_url: {navidrome_base_url}, username: {navidrome_username}") if navidrome_base_url and navidrome_username and navidrome_password: # Extract the path from URL if thumb_url.startswith('/rest/'): # Already a Subsonic API path path = thumb_url else: # Full localhost URL, extract path from urllib.parse import urlparse parsed = urlparse(thumb_url) path = parsed.path # Generate Subsonic API authentication import hashlib import secrets salt = secrets.token_hex(6) token = hashlib.md5((navidrome_password + salt).encode()).hexdigest() # Add authentication parameters to the URL separator = '&' if '?' in path else '?' auth_params = f"u={navidrome_username}&t={token}&s={salt}&v=1.16.1&c=SoulSync&f=json" # Construct proper Navidrome Subsonic URL fixed_url = f"{navidrome_base_url.rstrip('/')}{path}{separator}{auth_params}" logger.info(f"Fixed URL: {fixed_url}") return _browser_safe_image_url(fixed_url) logger.warning(f"No configuration found for {active_server} or unsupported server type") # Return a browser-safe URL even if no server-specific rebuild was possible. return _browser_safe_image_url(thumb_url) except Exception as e: logger.error(f"Error fixing image URL '{thumb_url}': {e}") return _browser_safe_image_url(thumb_url) def _is_internal_image_host(url: str) -> bool: """Return True when an image URL points at a host the browser likely cannot reach directly.""" try: parsed = urlparse(url) host = (parsed.hostname or '').strip('[]').lower() if not host: return False if host in {'localhost', '127.0.0.1', '::1', 'host.docker.internal'}: return True # Single-label hosts are usually Docker service names or local LAN aliases. if '.' not in host: return True try: ip = ipaddress.ip_address(host) return ip.is_loopback or ip.is_private or ip.is_link_local or ip.is_reserved except ValueError: return False except Exception: return False def _browser_safe_image_url(url: str) -> str: """Return a browser-safe image URL, proxying internal hosts through SoulSync.""" if not url: return url if url.startswith('/api/image-proxy?url='): return url if url.startswith('http://') or url.startswith('https://'): if _is_internal_image_host(url): return f"/api/image-proxy?url={quote(url, safe='')}" return url # Relative media-server paths should already have been expanded before this point. return url @app.route('/api/library/history') def get_library_history(): """Get persistent library history (downloads and server imports).""" try: event_type = request.args.get('type', None) if event_type and event_type not in ('download', 'import'): event_type = None page = max(1, int(request.args.get('page', 1))) limit = min(200, max(1, int(request.args.get('limit', 50)))) db = get_database() entries, total = db.get_library_history(event_type=event_type, page=page, limit=limit) stats = db.get_library_history_stats() return jsonify({ 'success': True, 'entries': entries, 'total': total, 'page': page, 'limit': limit, 'stats': stats }) except Exception as e: logger.error(f"Error fetching library history: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/library/artists') def get_library_artists(): """Get artists for the library page with search, filtering, and pagination""" try: # Get query parameters search_query = request.args.get('search', '') letter = request.args.get('letter', 'all') page = int(request.args.get('page', 1)) limit = int(request.args.get('limit', 75)) watchlist_filter = request.args.get('watchlist', 'all') source_filter = request.args.get('source_filter', '') # Get database instance database = get_database() # Get artists from database result = database.get_library_artists( search_query=search_query, letter=letter, page=page, limit=limit, watchlist_filter=watchlist_filter, profile_id=get_current_profile_id(), source_filter=source_filter ) # Fix image URLs for all artists for artist in result['artists']: if artist.get('image_url'): artist['image_url'] = fix_artist_image_url(artist['image_url']) return jsonify({ "success": True, **result }) except Exception as e: logger.error(f"Error fetching library artists: {e}") import traceback traceback.print_exc() return jsonify({ "success": False, "error": str(e), "artists": [], "pagination": { "page": 1, "limit": 75, "total_count": 0, "total_pages": 0, "has_prev": False, "has_next": False } }), 500 @app.route('/api/test-artist/') def test_artist_endpoint(artist_id): """Simple test endpoint""" return jsonify({ "success": True, "message": f"Test endpoint working for artist ID: {artist_id}" }) from core.artist_source_lookup import ( SOURCE_ID_FIELD as _SOURCE_ID_FIELD, SOURCE_ONLY_ARTIST_SOURCES as _SOURCE_ONLY_ARTIST_SOURCES, find_library_artist_for_source as _core_find_library_artist_for_source, ) def _find_library_artist_for_source(database, source, source_artist_id, artist_name=None): """Thin wrapper that injects the active-server context for the core lookup.""" try: active_server = config_manager.get_active_media_server() except Exception: active_server = None return _core_find_library_artist_for_source( database, source, source_artist_id, artist_name, active_server=active_server ) def _build_source_only_artist_detail(artist_id, artist_name, source): """Thin wrapper around ``core.artist_source_detail.build_source_only_artist_detail``. Builds the per-source client bag from web_server's module globals (each source's module-level client + Last.fm api key), forwards to the pure implementation in ``core/``, and wraps the (dict, status) return in ``jsonify``. """ from core.artist_source_detail import build_source_only_artist_detail # Resolve the per-source clients defensively — the original inline code # wrapped the whole source-side lookup in try/except so a failing # client helper (e.g. Spotify auth probe during a rate-limit ban, # Discogs client init error) would degrade gracefully to empty # enrichment instead of 500-ing the request. Preserve that. sp = None dz = None it = None dc = None try: if spotify_client and spotify_client.is_spotify_authenticated(): sp = spotify_client except Exception as e: logger.debug(f"Spotify client resolution failed: {e}") try: dz = _get_deezer_client() except Exception as e: logger.debug(f"Deezer client resolution failed: {e}") try: it = _get_itunes_client() except Exception as e: logger.debug(f"iTunes client resolution failed: {e}") try: discogs_token = config_manager.get('discogs.token', '') or '' if discogs_token: dc = _get_discogs_client(discogs_token) except Exception as e: logger.debug(f"Discogs client resolution failed: {e}") try: lastfm_api_key = config_manager.get('lastfm.api_key', '') or None except Exception: lastfm_api_key = None payload, status = build_source_only_artist_detail( artist_id, artist_name, source, spotify_client=sp, deezer_client=dz, itunes_client=it, discogs_client=dc, lastfm_api_key=lastfm_api_key, ) return jsonify(payload), status @app.route('/api/artist-detail/') def get_artist_detail(artist_id): """Get artist detail data. For library artists, `artist_id` is the local DB primary key and the full library-aware path runs (owned releases + merged source discography + per- service enrichment coverage). For source artists (Spotify/Deezer/iTunes/etc. that aren't in the library yet), pass `?source=&name=` and the endpoint synthesizes a response directly from the metadata source — no owned releases, just name + image + discography so the artist-detail page can still render. """ try: source_param = (request.args.get('source', '') or '').strip().lower() artist_name_arg = (request.args.get('name', '') or '').strip() logger.info( f"Getting artist detail for ID: {artist_id} " f"(source={source_param or 'library'})" ) # Get database instance database = get_database() # Get artist discography from database db_result = database.get_artist_discography(artist_id) # Library upgrade: if direct ID lookup missed AND we have a source hint, # check whether the user already owns this artist in the library under # a different ID (e.g. clicking a Deezer search result for an artist # they have indexed in Plex). Prefer the library record so they get # all their owned releases + enrichment instead of a bare source view. if not db_result.get('success') and source_param in _SOURCE_ONLY_ARTIST_SOURCES: library_pk = _find_library_artist_for_source( database, source_param, artist_id, artist_name_arg ) if library_pk: logger.info( f"Source-id {source_param}:{artist_id} matched library artist " f"PK={library_pk} — upgrading to library response" ) db_result = database.get_artist_discography(library_pk) if not db_result.get('success'): # Library lookup still failed. If a metadata source was specified, # fall back to a source-only response so the page can render a # non-library artist. if source_param in _SOURCE_ONLY_ARTIST_SOURCES: return _build_source_only_artist_detail( artist_id, artist_name_arg, source_param ) logger.error(f"Database returned error: {db_result}") return jsonify({ "success": False, "error": db_result.get('error', 'Artist not found') }), 404 artist_info = db_result['artist'] owned_releases = db_result['owned_releases'] logger.info(f"Found artist: {artist_info['name']} with {len(owned_releases['albums'])} albums") # Fix artist image URL logger.info(f"Artist image before fix: '{artist_info.get('image_url')}'") if artist_info.get('image_url'): artist_info['image_url'] = fix_artist_image_url(artist_info['image_url']) logger.info(f"Artist image after fix: '{artist_info['image_url']}'") else: logger.warning(f"No artist image URL found for {artist_info['name']}") # Debug final artist data being sent logger.info(f"Final artist data being sent: {artist_info}") # Fix image URLs for all albums for album in owned_releases['albums']: if album.get('image_url'): album['image_url'] = fix_artist_image_url(album['image_url']) # Fix image URLs for EPs and singles (currently empty but for future use) for ep in owned_releases['eps']: if ep.get('image_url'): ep['image_url'] = fix_artist_image_url(ep['image_url']) for single in owned_releases['singles']: if single.get('image_url'): single['image_url'] = fix_artist_image_url(single['image_url']) # Get source-priority discography for proper categorization and missing releases artist_detail_discography = None try: from core.metadata_service import MetadataLookupOptions, get_artist_detail_discography as _get_artist_detail_discography artist_source_ids = { 'spotify': artist_info.get('spotify_artist_id'), 'deezer': artist_info.get('deezer_id'), 'itunes': artist_info.get('itunes_artist_id'), 'discogs': artist_info.get('discogs_id'), 'hydrabase': artist_info.get('soul_id'), } artist_detail_discography = _get_artist_detail_discography( artist_id, artist_name=artist_info['name'], options=MetadataLookupOptions( allow_fallback=True, skip_cache=False, max_pages=0, limit=50, artist_source_ids=artist_source_ids, ), ) if artist_detail_discography['success']: logger.debug( "Source-priority discography found - " f"Albums: {len(artist_detail_discography['albums'])}, " f"EPs: {len(artist_detail_discography['eps'])}, " f"Singles: {len(artist_detail_discography['singles'])}" ) merged_discography = artist_detail_discography else: logger.debug(f"Source-priority discography not found: {artist_detail_discography.get('error', 'Unknown error')}") merged_discography = owned_releases except Exception as detail_error: logger.error(f"Error fetching source-priority discography: {detail_error}") merged_discography = owned_releases spotify_artist_data = None if artist_info.get('spotify_artist_id'): spotify_artist_data = { 'spotify_artist_id': artist_info.get('spotify_artist_id'), 'spotify_artist_name': artist_info.get('name'), 'artist_image': artist_info.get('image_url') } # Compute per-artist track enrichment coverage enrichment_coverage = {} try: with database._get_connection() as conn: cursor = conn.cursor() artist_name = artist_info['name'] server_source = artist_info.get('server_source', '') cursor.execute(""" SELECT COUNT(*) FROM tracks t JOIN albums al ON al.id = t.album_id JOIN artists ar ON ar.id = al.artist_id WHERE ar.name = ? AND ar.server_source = ? """, (artist_name, server_source)) total = (cursor.fetchone() or [0])[0] if total > 0: for svc, col in [('spotify', 'spotify_track_id'), ('musicbrainz', 'musicbrainz_recording_id'), ('deezer', 'deezer_id'), ('lastfm', 'lastfm_url'), ('itunes', 'itunes_track_id'), ('audiodb', 'audiodb_id'), ('genius', 'genius_id'), ('tidal', 'tidal_id'), ('qobuz', 'qobuz_id')]: try: cursor.execute(f""" SELECT COUNT(*) FROM tracks t JOIN albums al ON al.id = t.album_id JOIN artists ar ON ar.id = al.artist_id WHERE ar.name = ? AND ar.server_source = ? AND t.{col} IS NOT NULL AND t.{col} != '' """, (artist_name, server_source)) matched = (cursor.fetchone() or [0])[0] enrichment_coverage[svc] = round(matched / total * 100, 1) except Exception: enrichment_coverage[svc] = 0 enrichment_coverage['total_tracks'] = total except Exception: pass response_data = { "success": True, "artist": artist_info, "discography": merged_discography, "enrichment_coverage": enrichment_coverage } # Add Spotify artist data if available if spotify_artist_data: response_data["spotify_artist"] = spotify_artist_data return jsonify(response_data) except Exception as e: logger.error(f"Error in get_artist_detail: {e}") import traceback traceback.print_exc() return jsonify({ "success": False, "error": str(e) }), 500 @app.route('/api/library/debug-photos') def debug_library_photos(): """Debug endpoint to check artist photo URLs""" try: database = get_database() with database._get_connection() as conn: cursor = conn.cursor() # Get first 10 artists with their photo URLs cursor.execute(""" SELECT name, thumb_url, server_source FROM artists WHERE thumb_url IS NOT NULL AND thumb_url != '' LIMIT 10 """) artists_with_photos = cursor.fetchall() # Get first 10 artists without photos cursor.execute(""" SELECT name, thumb_url, server_source FROM artists WHERE thumb_url IS NULL OR thumb_url = '' LIMIT 10 """) artists_without_photos = cursor.fetchall() return jsonify({ "artists_with_photos": [dict(row) for row in artists_with_photos], "artists_without_photos": [dict(row) for row in artists_without_photos], "total_with_photos": len(artists_with_photos), "total_without_photos": len(artists_without_photos) }) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/artist/similar//stream') def get_similar_artists_stream(artist_name): """Stream MusicMap similar artists using source-priority metadata matching.""" from core.metadata_service import iter_musicmap_similar_artist_events def generate(): logger.info(f"Streaming similar artists for: {artist_name}") for event in iter_musicmap_similar_artist_events(artist_name, limit=20): yield f"data: {json.dumps(event)}\n\n" if event.get('artist'): time.sleep(0.1) return Response(generate(), mimetype='text/event-stream') @app.route('/api/artist/similar/') def get_similar_artists(artist_name): """Get MusicMap similar artists using source-priority metadata matching.""" from core.metadata_service import get_musicmap_similar_artists try: logger.info(f"Getting similar artists for: {artist_name}") result = get_musicmap_similar_artists(artist_name, limit=20) if not result.get('success'): error = result.get('error', 'Failed to fetch similar artists') status_code = int(result.get('status_code') or 500) return jsonify({ "success": False, "error": error }), status_code return jsonify({ "success": True, "artist": artist_name, "similar_artists": result.get('similar_artists', []), "total_found": result.get('total_found', 0), "total_matched": result.get('total_matched', 0), "source_priority": result.get('source_priority', []), }) except Exception as e: logger.error(f"Error getting similar artists: {e}") import traceback traceback.print_exc() return jsonify({ "success": False, "error": str(e) }), 500 @app.route('/api/artist//image', methods=['GET']) def get_artist_image(artist_id): """Get an artist image URL using source-aware metadata resolution.""" try: from core.metadata_service import get_artist_image_url as _get_artist_image_url source_override = request.args.get('source', '').strip().lower() or None plugin = request.args.get('plugin', '').strip().lower() or None # `name` is optional but required for sources that don't store # artist images directly (MusicBrainz) — the resolver falls back # to searching iTunes/Deezer by name. artist_name = request.args.get('name', '').strip() or None image_url = _get_artist_image_url( artist_id, source_override=source_override, plugin=plugin, artist_name=artist_name, ) return jsonify({"success": True, "image_url": image_url}) except Exception as e: logger.error(f"Error fetching artist image: {e}") return jsonify({"success": False, "image_url": None, "error": str(e)}) @app.route('/api/artist//discography', methods=['GET']) def get_artist_discography(artist_id): """Get an artist's complete discography (albums and singles)""" try: # Get optional artist name for fallback searches artist_name = request.args.get('artist_name', '').strip() # Optional source override from multi-source search tabs source_override = request.args.get('source', '').strip().lower() # Mirror to Hydrabase P2P network if hydrabase_worker and dev_mode_enabled and artist_name: hydrabase_worker.enqueue(artist_name, 'artist.albums') effective_override_source = source_override if source_override == 'hydrabase': plugin = request.args.get('plugin', '').strip().lower() if plugin == 'deezer': effective_override_source = 'deezer' elif plugin == 'itunes' or artist_id.isdigit(): effective_override_source = 'itunes' else: effective_override_source = 'spotify' from core.metadata_service import MetadataLookupOptions, get_artist_discography as _get_artist_discography discography = _get_artist_discography( artist_id, artist_name=artist_name, options=MetadataLookupOptions( source_override=effective_override_source, allow_fallback=True, skip_cache=False, max_pages=0, limit=50, ), ) album_list = discography['albums'] singles_list = discography['singles'] active_source = discography['source'] source_priority = discography['source_priority'] # Gather artist enrichment info from cache + library artist_info = {} try: cache = get_metadata_cache() cache_sources = [] if active_source: cache_sources.append(active_source) for source in source_priority: if source not in cache_sources: cache_sources.append(source) # Try metadata cache for genres, image, followers cached = None for src in cache_sources: cached = cache.get_entity(src, 'artist', artist_id) if cached: break if not cached and artist_name: # Try by name across all sources for src in cache_sources: db_tmp = get_database() conn_tmp = db_tmp._get_connection() try: cur = conn_tmp.cursor() cur.execute(""" SELECT genres, image_url, followers, popularity, external_urls FROM metadata_cache_entities WHERE entity_type = 'artist' AND name COLLATE NOCASE = ? AND source = ? LIMIT 1 """, (artist_name, src)) row = cur.fetchone() if row: cached = dict(row) break finally: conn_tmp.close() if cached: try: artist_info['genres'] = json.loads(cached.get('genres', '[]')) if isinstance(cached.get('genres'), str) else (cached.get('genres') or []) except Exception: artist_info['genres'] = [] artist_info['image_url'] = cached.get('image_url') artist_info['followers'] = cached.get('followers') artist_info['popularity'] = cached.get('popularity') try: artist_info['external_urls'] = json.loads(cached.get('external_urls', '{}')) if isinstance(cached.get('external_urls'), str) else (cached.get('external_urls') or {}) except Exception: artist_info['external_urls'] = {} # Try library for full enrichment (Last.fm bio, stats, service IDs) if artist_name: db_lib = get_database() conn_lib = db_lib._get_connection() try: cur_lib = conn_lib.cursor() cur_lib.execute(""" SELECT id, summary, genres, thumb_url, spotify_artist_id, musicbrainz_id, deezer_id, itunes_artist_id, audiodb_id, discogs_id, tidal_id, qobuz_id, genius_id, soul_id, lastfm_bio, lastfm_listeners, lastfm_playcount, lastfm_tags, lastfm_url, genius_url, style, mood, label FROM artists WHERE name COLLATE NOCASE = ? LIMIT 1 """, (artist_name,)) lib_row = cur_lib.fetchone() if lib_row: lib = dict(lib_row) artist_info['library_id'] = lib['id'] # Image fallback if not artist_info.get('image_url') and lib['thumb_url']: artist_info['image_url'] = fix_artist_image_url(lib['thumb_url']) # Genres fallback if not artist_info.get('genres') and lib['genres']: try: artist_info['genres'] = json.loads(lib['genres']) except Exception: pass # Last.fm enrichment if lib.get('lastfm_bio'): artist_info['lastfm_bio'] = lib['lastfm_bio'] if lib.get('lastfm_listeners'): artist_info['lastfm_listeners'] = lib['lastfm_listeners'] if lib.get('lastfm_playcount'): artist_info['lastfm_playcount'] = lib['lastfm_playcount'] if lib.get('lastfm_tags'): try: artist_info['lastfm_tags'] = json.loads(lib['lastfm_tags']) if isinstance(lib['lastfm_tags'], str) else lib['lastfm_tags'] except Exception: pass if lib.get('lastfm_url'): artist_info['lastfm_url'] = lib['lastfm_url'] if lib.get('genius_url'): artist_info['genius_url'] = lib['genius_url'] # Service IDs for badges for key in ['spotify_artist_id', 'musicbrainz_id', 'deezer_id', 'itunes_artist_id', 'audiodb_id', 'discogs_id', 'tidal_id', 'qobuz_id', 'genius_id', 'soul_id']: if lib.get(key): artist_info[key] = lib[key] # Bio fallback from summary if not artist_info.get('lastfm_bio') and lib.get('summary'): artist_info['bio'] = lib['summary'] finally: conn_lib.close() except Exception as e: logger.debug(f"Artist info enrichment failed (non-fatal): {e}") return jsonify({ "albums": album_list, "singles": singles_list, "source": active_source or (source_priority[0] if source_priority else "unknown"), "artist_info": artist_info, }) except Exception as e: logger.exception("Error fetching artist discography for %s", artist_id) return jsonify({"error": str(e)}), 500 @app.route('/api/album//tracks', methods=['GET']) def get_album_tracks(album_id): """Get tracks for specific album formatted for download missing tracks modal""" try: album_name = request.args.get('name', '').strip() artist_name = request.args.get('artist', '').strip() source_override = request.args.get('source', '').strip().lower() if source_override == 'hydrabase': plugin = request.args.get('plugin', '').strip().lower() if plugin in ('itunes', 'deezer'): source_override = plugin elif album_id.isdigit(): source_override = 'itunes' else: source_override = 'spotify' from core.metadata_service import get_artist_album_tracks as _get_artist_album_tracks result = _get_artist_album_tracks( album_id, artist_name=artist_name, album_name=album_name, source_override=source_override or None, ) if not result.get('success'): return jsonify({"error": result.get('error', 'Album not found')}), result.get('status_code', 404) logger.info( "Successfully formatted %s tracks for album %s", len(result.get('tracks', [])), result.get('album', {}).get('name', album_name or album_id), ) return jsonify({ 'success': True, 'album': result['album'], 'tracks': result['tracks'], 'source': result.get('source'), 'source_priority': result.get('source_priority', []), 'resolved_album_id': result.get('resolved_album_id'), }) except Exception as e: logger.exception("Error fetching album tracks for album %s", album_id) return jsonify({"error": str(e)}), 500 @app.route('/api/artist//download-discography', methods=['POST']) def download_discography(artist_id): """Add selected albums from an artist's discography to the wishlist.""" try: data = request.get_json() if not data or 'album_ids' not in data: return jsonify({"success": False, "error": "album_ids required"}), 400 album_ids = data['album_ids'] artist_name = data.get('artist_name', 'Unknown Artist') from database.music_database import MusicDatabase db = MusicDatabase() profile_id = get_current_profile_id() active_server = config_manager.get_active_media_server() # Resolve metadata client client = None if spotify_client and spotify_client.is_authenticated(): client = spotify_client else: fallback_src = _get_metadata_fallback_source() if fallback_src == 'itunes': client = _get_itunes_client() elif fallback_src == 'deezer': client = _get_deezer_client() if not client: return jsonify({"success": False, "error": "No metadata source available"}), 500 total_added = 0 total_skipped = 0 def generate_ndjson(): nonlocal total_added, total_skipped for album_id in album_ids: try: album_data = client.get_album(album_id) if not album_data: yield json.dumps({"album_id": album_id, "status": "error", "message": "Album not found"}) + '\n' continue album_name = album_data.get('name', 'Unknown') album_images = album_data.get('images', []) album_type = album_data.get('album_type', 'album') release_date = album_data.get('release_date', '') album_artists = album_data.get('artists', []) tracks = album_data.get('tracks', {}).get('items', []) if not tracks: tracks_data = client.get_album_tracks(album_id) if tracks_data and 'items' in tracks_data: tracks = tracks_data['items'] if not tracks: yield json.dumps({"album_id": album_id, "name": album_name, "status": "error", "message": "No tracks"}) + '\n' continue added = 0 skipped = 0 for track in tracks: track_name = track.get('name', '') track_artists = track.get('artists', []) track_id = track.get('id', '') if not track_name: continue spotify_track_data = { 'id': track_id, 'name': track_name, 'artists': track_artists if isinstance(track_artists, list) else [{'name': str(track_artists)}], 'album': { 'id': str(album_id), 'name': album_name, 'artists': album_artists, 'images': album_images, 'album_type': album_type, 'release_date': release_date, 'total_tracks': len(tracks) }, 'duration_ms': track.get('duration_ms', 0), 'explicit': track.get('explicit', False), 'track_number': track.get('track_number', 0), 'disc_number': track.get('disc_number', 1), 'uri': track.get('uri', ''), 'preview_url': track.get('preview_url'), 'external_urls': track.get('external_urls', {}), 'is_local': False } try: was_added = db.add_to_wishlist( spotify_track_data=spotify_track_data, failure_reason="Added via Download Discography", source_type="discography", source_info=json.dumps({ 'artist_name': artist_name, 'album_name': album_name, 'album_type': album_type }), profile_id=profile_id ) if was_added: added += 1 else: skipped += 1 except Exception: skipped += 1 total_added += added total_skipped += skipped logger.warning(f"[Discography] {album_name}: {added} added, {skipped} skipped") yield json.dumps({ "album_id": album_id, "name": album_name, "status": "done", "tracks_added": added, "tracks_skipped": skipped, "tracks_total": len(tracks) }) + '\n' except Exception as album_err: yield json.dumps({"album_id": album_id, "status": "error", "message": str(album_err)}) + '\n' logger.warning(f"[Discography] Complete for {artist_name}: {total_added} tracks added, {total_skipped} skipped across {len(album_ids)} albums") yield json.dumps({"status": "complete", "total_added": total_added, "total_skipped": total_skipped, "total_albums": len(album_ids)}) + '\n' return app.response_class(generate_ndjson(), mimetype='application/x-ndjson', headers={'X-Accel-Buffering': 'no'}) except Exception as e: logger.error(f"Error in download discography: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/artist//completion', methods=['POST']) def check_artist_discography_completion(artist_id): """Check completion status for artist's albums and singles""" try: data = request.get_json() if not data or 'discography' not in data: return jsonify({"error": "Missing discography data"}), 400 from core.metadata_service import check_artist_discography_completion as _check_artist_discography_completion discography = data['discography'] source_override = (data.get('source') or '').strip().lower() or None result = _check_artist_discography_completion( discography, artist_name=data.get('artist_name', 'Unknown Artist'), source_override=source_override, ) return jsonify(result) except Exception as e: logger.error(f"Error checking discography completion: {e}") import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/artist//completion-stream', methods=['POST']) def check_artist_discography_completion_stream(artist_id): """Stream completion status for artist's albums and singles one by one""" # Capture request data BEFORE the generator function try: data = request.get_json() if not data or 'discography' not in data: return jsonify({"error": "Missing discography data"}), 400 except Exception as e: return jsonify({"error": "Invalid request data"}), 400 # Extract data for the generator discography = data['discography'] artist_name = data.get('artist_name', 'Unknown Artist') source_override = (data.get('source') or '').strip().lower() or None from core.metadata_service import iter_artist_discography_completion_events def generate_completion_stream(): try: logger.info(f"Starting streaming completion check for artist: {artist_name}") for event in iter_artist_discography_completion_events( discography, artist_name=artist_name, source_override=source_override, ): yield f"data: {json.dumps(event)}\n\n" if event.get('type') in ('album_completion', 'single_completion'): # Small delay to make the streaming effect visible time.sleep(0.1) # 100ms delay between items except Exception as e: logger.error(f"Error in streaming completion check: {e}") import traceback traceback.print_exc() yield f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n" return Response( generate_completion_stream(), content_type='text/event-stream', headers={ 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Headers': 'Cache-Control' } ) @app.route('/api/library/completion-stream', methods=['POST']) def library_completion_stream(): """Stream completion status for library artist detail view - checks ownership per release via SSE""" try: data = request.get_json() if not data or 'artist_name' not in data: return jsonify({"error": "Missing artist_name"}), 400 except Exception as e: return jsonify({"error": "Invalid request data"}), 400 artist_name = data['artist_name'] def generate(): try: from core.metadata_service import check_album_completion, check_single_completion db = get_database() source_override = (data.get('source') or '').strip().lower() or None categories = ['albums', 'eps', 'singles'] all_items = [] for cat in categories: for item in data.get(cat, []): all_items.append((cat, item)) # Pre-fetch the artist's library albums AND tracks ONCE so per-item # matching runs in-memory instead of firing per-item SQL searches. # Turns N*K queries into 2 broad fetches + N track-count lookups. from config.settings import config_manager as _cm_cs _active_server = _cm_cs.get_active_media_server() candidate_albums = None candidate_tracks = None _t0 = time.perf_counter() try: candidate_albums = db.get_candidate_albums_for_artist(artist_name, server_source=_active_server) except Exception as _cand_err: print(f"[completion-stream] Failed to pre-fetch album candidates for '{artist_name}': {_cand_err}") candidate_albums = None _t1 = time.perf_counter() print(f"[completion-stream] Pre-fetched {len(candidate_albums) if candidate_albums is not None else 0} library albums for '{artist_name}' in {(_t1 - _t0) * 1000:.0f}ms") if candidate_albums: _t2 = time.perf_counter() try: candidate_tracks = db.get_candidate_tracks_for_albums([a.id for a in candidate_albums]) except Exception as _tr_err: print(f"[completion-stream] Failed to pre-fetch track candidates for '{artist_name}': {_tr_err}") candidate_tracks = None _t3 = time.perf_counter() print(f"[completion-stream] Pre-fetched {len(candidate_tracks) if candidate_tracks is not None else 0} library tracks in {(_t3 - _t2) * 1000:.0f}ms") yield f"data: {json.dumps({'type': 'start', 'total_items': len(all_items)})}\n\n" _loop_start = time.perf_counter() for _i, (category, item) in enumerate(all_items): try: # Map Library field names to helper field names mapped = { 'id': item['id'], 'name': item['title'], 'total_tracks': item.get('track_count', 0), 'album_type': item.get('album_type', 'album') } if category == 'singles': result = check_single_completion(db, mapped, artist_name, source_override=source_override, candidate_albums=candidate_albums, candidate_tracks=candidate_tracks) else: result = check_album_completion(db, mapped, artist_name, source_override=source_override, candidate_albums=candidate_albums) result['id'] = item['id'] result['category'] = category result['type'] = 'completion' yield f"data: {json.dumps(result)}\n\n" except Exception as e: yield f"data: {json.dumps({'type': 'completion', 'category': category, 'id': item['id'], 'status': 'error', 'owned_tracks': 0, 'expected_tracks': item.get('track_count', 0), 'completion_percentage': 0, 'confidence': 0.0, 'error': str(e)})}\n\n" time.sleep(0.05) # 50ms between items for visible streaming _loop_elapsed = time.perf_counter() - _loop_start _sleep_floor = 0.05 * len(all_items) print(f"[completion-stream] Processed {len(all_items)} items for '{artist_name}' in {_loop_elapsed * 1000:.0f}ms (sleep floor: {_sleep_floor * 1000:.0f}ms)") yield f"data: {json.dumps({'type': 'complete', 'processed_count': len(all_items)})}\n\n" except Exception as e: logger.error(f"Error in library completion stream: {e}") import traceback traceback.print_exc() yield f"data: {json.dumps({'type': 'error', 'error': str(e)})}\n\n" return Response( generate(), content_type='text/event-stream', headers={ 'Cache-Control': 'no-cache', 'Connection': 'keep-alive', 'Access-Control-Allow-Origin': '*', 'Access-Control-Allow-Headers': 'Cache-Control' } ) @app.route('/api/library/check-tracks', methods=['POST']) def library_check_tracks(): """Check which tracks from a list are already owned in the library. Uses a single batch DB query + in-memory fuzzy matching for speed.""" try: data = request.get_json() if not data or 'artist_name' not in data or 'tracks' not in data: return jsonify({"success": False, "error": "Missing artist_name or tracks"}), 400 artist_name = data['artist_name'] tracks = data['tracks'] from database.music_database import MusicDatabase db = MusicDatabase() active_server = config_manager.get_active_media_server() # Single query: get ALL tracks by this artist from the DB db_tracks = db.search_tracks(artist=artist_name, limit=500, server_source=active_server) if not db_tracks: # No tracks by this artist in DB — none owned owned_map = {t.get('name', ''): {"owned": False} for t in tracks if t.get('name')} return jsonify({"success": True, "owned_tracks": owned_map}) # Pre-normalize all DB track titles for fast in-memory comparison from difflib import SequenceMatcher try: from unidecode import unidecode except ImportError: unidecode = lambda x: x def _normalize(text): if not text: return "" return unidecode(text).lower().strip() def _clean_title(text): import re cleaned = _normalize(text) # Remove parenthetical/bracket content, dashes, feat/ft, remaster tags cleaned = re.sub(r'\s*[\[\(].*?[\]\)]', '', cleaned) cleaned = re.sub(r'\s*-\s*', ' ', cleaned) cleaned = re.sub(r'\s*feat\..*', '', cleaned) cleaned = re.sub(r'\s*featuring.*', '', cleaned) cleaned = re.sub(r'\s*ft\..*', '', cleaned) cleaned = re.sub(r'\s*\d{4}\s*remaster.*', '', cleaned) cleaned = re.sub(r'\s*remaster(ed)?.*', '', cleaned) cleaned = re.sub(r'\s+', ' ', cleaned).strip() return cleaned # Pre-compute normalized DB titles once (keep reference to db_track for metadata) db_title_entries = [(_normalize(t.title), _clean_title(t.title), t) for t in db_tracks] # Album-aware matching: if album_name provided, prefer tracks from that album target_album = data.get('album_name', '') target_album_norm = _normalize(target_album) if target_album else '' def _match_title(search_norm, search_clean, candidates): """Find best matching track from a list of (norm, clean, db_track) candidates.""" for db_norm, db_clean, db_track in candidates: if search_norm == db_norm or search_clean == db_clean: return db_track sim = max( SequenceMatcher(None, search_norm, db_norm).ratio(), SequenceMatcher(None, search_clean, db_clean).ratio() ) if sim >= 0.7: return db_track return None # Split DB tracks by album if album-aware matching is active album_entries = [] other_entries = [] if target_album_norm: for entry in db_title_entries: db_album = _normalize(getattr(entry[2], 'album_title', '') or '') if db_album and SequenceMatcher(None, target_album_norm, db_album).ratio() >= 0.7: album_entries.append(entry) else: other_entries.append(entry) else: other_entries = db_title_entries owned_map = {} for track in tracks: track_name = track.get('name', '') if not track_name: continue search_norm = _normalize(track_name) search_clean = _clean_title(track_name) # When album context provided, only match within that album — # prevents false positives where "Thriller" on Album A shows as owned # because it exists on Album B. Without album context, search all tracks. if target_album_norm: matched_db_track = _match_title(search_norm, search_clean, album_entries) else: matched_db_track = _match_title(search_norm, search_clean, other_entries) if matched_db_track: import os file_ext = os.path.splitext(matched_db_track.file_path or '')[1].lstrip('.').upper() or None owned_map[track_name] = { "owned": True, "format": file_ext, "bitrate": matched_db_track.bitrate, "album": getattr(matched_db_track, 'album_title', None) } else: owned_map[track_name] = {"owned": False} return jsonify({"success": True, "owned_tracks": owned_map}) except Exception as e: logger.error(f"Error checking track ownership: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # ==================== Enhanced Library Management Endpoints ==================== @app.route('/api/library/artist//enhanced') def get_artist_enhanced_detail(artist_id): """Get full artist detail with all albums and tracks for enhanced library view.""" try: database = get_database() result = database.get_artist_full_detail(artist_id) if not result.get('success'): return jsonify(result), 404 # Fix image URLs for artist and all albums (resolve Plex/Jellyfin relative paths) if result.get('artist', {}).get('thumb_url'): result['artist']['thumb_url'] = fix_artist_image_url(result['artist']['thumb_url']) for album in result.get('albums', []): if album.get('thumb_url'): album['thumb_url'] = fix_artist_image_url(album['thumb_url']) # Include server type for sync option active_server = config_manager.get_active_media_server() server_connected = False if active_server == 'plex': server_connected = plex_client.is_connected() elif active_server == 'jellyfin': server_connected = jellyfin_client.is_connected() elif active_server == 'navidrome': server_connected = navidrome_client.is_connected() result['server_type'] = active_server if server_connected else None return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/artist//quality-analysis') def get_artist_quality_analysis(artist_id): """Analyze track quality for an artist — returns tier classification for each track.""" try: database = get_database() result = database.get_artist_full_detail(artist_id) if not result.get('success'): return jsonify(result), 404 artist = result.get('artist', {}) albums = result.get('albums', []) # Get user's quality profile to determine min acceptable tier quality_profile = database.get_quality_profile() preferred_qualities = quality_profile.get('qualities', {}) min_acceptable_tier = 999 tier_map = {'flac': 'lossless', 'mp3_320': 'low_lossy', 'mp3_256': 'low_lossy', 'mp3_192': 'low_lossy'} for qname, qconfig in preferred_qualities.items(): if qconfig.get('enabled', False): tname = tier_map.get(qname) if tname and tname in QUALITY_TIERS: min_acceptable_tier = min(min_acceptable_tier, QUALITY_TIERS[tname]['tier']) tracks = [] summary = {'total': 0, 'lossless': 0, 'high_lossy': 0, 'standard_lossy': 0, 'low_lossy': 0, 'unknown': 0} for album in albums: album_title = album.get('title', 'Unknown Album') album_id = album.get('id') for track in album.get('tracks', []): file_path = track.get('file_path') if not file_path: continue tier_name, tier_num = _get_quality_tier_from_extension(file_path) ext = os.path.splitext(file_path)[1].lstrip('.').upper() # Use filename as fallback when title is empty title = track.get('title', '') or '' if not title.strip(): title = os.path.splitext(os.path.basename(file_path))[0] tracks.append({ 'track_id': track.get('id'), 'title': title, 'album_title': album_title, 'album_id': album_id, 'file_path': file_path, 'format': ext or 'Unknown', 'tier_name': tier_name, 'tier_num': tier_num, 'bitrate': track.get('bitrate'), 'spotify_track_id': track.get('spotify_track_id'), 'track_number': track.get('track_number'), 'disc_number': track.get('disc_number'), }) summary['total'] += 1 if tier_name in summary: summary[tier_name] += 1 else: summary['unknown'] += 1 return jsonify({ 'success': True, 'artist_name': artist.get('name', 'Unknown Artist'), 'artist_id': artist_id, 'tracks': tracks, 'quality_summary': summary, 'min_acceptable_tier': min_acceptable_tier }) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/artist//enhance', methods=['POST']) def enhance_artist_quality(artist_id): """Add selected tracks to wishlist for quality enhancement re-download.""" try: data = request.get_json() or {} track_ids = data.get('track_ids', []) if not track_ids: return jsonify({"success": False, "error": "No track IDs provided"}), 400 database = get_database() from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() profile_id = get_current_profile_id() # Get artist info artist_result = database.get_artist_full_detail(artist_id) if not artist_result.get('success'): return jsonify({"success": False, "error": "Artist not found"}), 404 artist_name = artist_result.get('artist', {}).get('name', 'Unknown Artist') # Build lookup of all tracks for this artist track_lookup = {} for album in artist_result.get('albums', []): album_title = album.get('title', '') for track in album.get('tracks', []): tid = str(track.get('id', '')) track['_album_title'] = album_title track['_album_id'] = album.get('id') track_lookup[tid] = track enhanced_count = 0 failed_count = 0 failed_tracks = [] for track_id in track_ids: track_id_str = str(track_id) track = track_lookup.get(track_id_str) if not track: failed_count += 1 failed_tracks.append({'track_id': track_id, 'reason': 'Track not found'}) continue file_path = track.get('file_path') if not file_path: failed_count += 1 failed_tracks.append({'track_id': track_id, 'reason': 'No file path'}) continue tier_name, tier_num = _get_quality_tier_from_extension(file_path) title = track.get('title', '') or '' if not title.strip(): title = os.path.splitext(os.path.basename(file_path))[0] spotify_tid = track.get('spotify_track_id') # Build Spotify track data for wishlist matched_track_data = None if spotify_tid and spotify_client: # Direct lookup via stored Spotify ID — raw_data has full Spotify API format try: track_details = spotify_client.get_track_details(spotify_tid) if track_details and track_details.get('raw_data'): matched_track_data = track_details['raw_data'] elif track_details: # Enhanced format — rebuild with images for wishlist compatibility album_data = track_details.get('album', {}) album_images = [] # Try to get album art from a full album lookup if album_data.get('id'): try: full_album = spotify_client.get_album(album_data['id']) if full_album and full_album.get('images'): album_images = full_album['images'] except Exception: pass matched_track_data = { 'id': spotify_tid, 'name': track_details.get('name', title), 'artists': [{'name': a} for a in track_details.get('artists', [artist_name])], 'album': { 'id': album_data.get('id', ''), 'name': album_data.get('name', track.get('_album_title', '')), 'album_type': album_data.get('album_type', 'album'), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', 1), 'artists': [{'name': a} for a in album_data.get('artists', [artist_name])], 'images': album_images, }, 'duration_ms': track_details.get('duration_ms', track.get('duration', 0)), 'track_number': track_details.get('track_number', track.get('track_number', 1)), 'disc_number': track_details.get('disc_number', 1), 'popularity': 0, 'preview_url': None, 'external_urls': {}, } except Exception as e: logger.error(f"[Enhance] Spotify lookup failed for {spotify_tid}: {e}") if not matched_track_data and spotify_client: # Fallback: Spotify search matching — need full track data for wishlist try: temp_track = type('TempTrack', (), { 'name': title, 'artists': [artist_name], 'album': track.get('_album_title', '') })() search_queries = matching_engine.generate_download_queries(temp_track) best_match = None best_match_raw = None best_confidence = 0.0 for search_query in search_queries[:3]: # Limit queries try: results = spotify_client.search_tracks(search_query, limit=5) if not results: continue for sp_track in results: artist_conf = max( (matching_engine.similarity_score( matching_engine.normalize_string(artist_name), matching_engine.normalize_string(a) ) for a in (sp_track.artists or [artist_name])), default=0 ) title_conf = matching_engine.similarity_score( matching_engine.normalize_string(title), matching_engine.normalize_string(sp_track.name) ) combined = artist_conf * 0.5 + title_conf * 0.5 # Small bonus for album tracks over singles _at = getattr(sp_track, 'album_type', None) or '' if _at == 'album': combined += 0.02 elif _at == 'ep': combined += 0.01 if combined > best_confidence and combined >= 0.7: best_confidence = combined best_match = sp_track if best_confidence >= 0.9: break except Exception: continue if best_match: # Fetch full track data from Spotify for proper wishlist format try: full_details = spotify_client.get_track_details(best_match.id) if full_details and full_details.get('raw_data'): matched_track_data = full_details['raw_data'] else: raise ValueError("No raw_data from get_track_details") except Exception: # Build from Track dataclass with image album_images = [{'url': best_match.image_url}] if best_match.image_url else [] matched_track_data = { 'id': best_match.id, 'name': best_match.name, 'artists': [{'name': a} for a in best_match.artists], 'album': { 'name': best_match.album, 'artists': [{'name': a} for a in best_match.artists], 'album_type': 'album', 'release_date': getattr(best_match, 'release_date', '') or '', 'images': album_images, }, 'duration_ms': best_match.duration_ms, 'popularity': best_match.popularity or 0, 'preview_url': best_match.preview_url, 'external_urls': best_match.external_urls or {}, } except Exception as e: logger.error(f"[Enhance] Search match failed for {title}: {e}") # Fallback source when Spotify unavailable or no match found if not matched_track_data: try: fallback_client = _get_metadata_fallback_client() itunes_best = None itunes_best_conf = 0.0 itunes_queries = matching_engine.generate_download_queries( type('TempTrack', (), { 'name': title, 'artists': [artist_name], 'album': track.get('_album_title', '') })() ) for search_query in itunes_queries[:3]: try: itunes_results = fallback_client.search_tracks(search_query, limit=5) if not itunes_results: continue for it_track in itunes_results: artist_conf = max( (matching_engine.similarity_score( matching_engine.normalize_string(artist_name), matching_engine.normalize_string(a) ) for a in (it_track.artists or [artist_name])), default=0 ) title_conf = matching_engine.similarity_score( matching_engine.normalize_string(title), matching_engine.normalize_string(it_track.name) ) combined = artist_conf * 0.5 + title_conf * 0.5 # Small bonus for album tracks over singles _at = getattr(it_track, 'album_type', None) or '' if _at == 'album': combined += 0.02 elif _at == 'ep': combined += 0.01 if combined > itunes_best_conf and combined >= 0.7: itunes_best_conf = combined itunes_best = it_track if itunes_best_conf >= 0.9: break except Exception: continue if itunes_best: album_images = [{'url': itunes_best.image_url, 'height': 600, 'width': 600}] if itunes_best.image_url else [] matched_track_data = { 'id': itunes_best.id, 'name': itunes_best.name, 'artists': [{'name': a} for a in itunes_best.artists], 'album': { 'name': itunes_best.album, 'artists': [{'name': a} for a in itunes_best.artists], 'album_type': 'album', 'images': album_images, 'release_date': itunes_best.release_date or '', 'total_tracks': 1, }, 'duration_ms': itunes_best.duration_ms, 'track_number': itunes_best.track_number or 1, 'disc_number': itunes_best.disc_number or 1, 'popularity': itunes_best.popularity or 0, 'preview_url': itunes_best.preview_url, 'external_urls': itunes_best.external_urls or {}, } logger.warning(f"[Enhance] Fallback match for {title}: {itunes_best.artists[0]} - {itunes_best.name} (conf: {itunes_best_conf:.3f})") except Exception as e: logger.error(f"[Enhance] Fallback source failed for {title}: {e}") if not matched_track_data: failed_count += 1 failed_tracks.append({'track_id': track_id, 'title': title, 'reason': 'No Spotify or fallback match'}) continue # Add to wishlist with enhance source source_context = { 'enhance': True, 'original_file_path': file_path, 'original_format': tier_name, 'original_bitrate': track.get('bitrate'), 'original_tier': tier_num, 'artist_name': artist_name, } success = wishlist_service.add_spotify_track_to_wishlist( spotify_track_data=matched_track_data, failure_reason=f"Quality enhance - upgrading from {tier_name.replace('_', ' ').title()}", source_type='enhance', source_context=source_context, profile_id=profile_id ) if success: enhanced_count += 1 logger.info(f"[Enhance] Queued for upgrade: {artist_name} - {title} ({tier_name})") else: failed_count += 1 failed_tracks.append({'track_id': track_id, 'title': title, 'reason': 'Wishlist add failed'}) return jsonify({ 'success': True, 'enhanced_count': enhanced_count, 'failed_count': failed_count, 'failed_tracks': failed_tracks }) except Exception as e: logger.error(f"[Enhance] {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/artist/', methods=['PUT']) def update_library_artist(artist_id): """Update artist metadata fields.""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 result = database.update_artist_fields(artist_id, data) if not result.get('success'): return jsonify(result), 400 return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/album/', methods=['PUT']) def update_library_album(album_id): """Update album metadata fields.""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 result = database.update_album_fields(album_id, data) if not result.get('success'): return jsonify(result), 400 return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/track/', methods=['PUT']) def update_library_track(track_id): """Update track metadata fields.""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 result = database.update_track_fields(track_id, data) if not result.get('success'): return jsonify(result), 400 return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/tracks/batch', methods=['PUT']) def batch_update_library_tracks(): """Batch update multiple tracks with the same field values.""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 track_ids = data.get('track_ids', []) updates = data.get('updates', {}) if not track_ids or not updates: return jsonify({"success": False, "error": "track_ids and updates are required"}), 400 result = database.batch_update_tracks(track_ids, updates) if not result.get('success'): return jsonify(result), 400 return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # ── Write Tags to File endpoints ── @app.route('/api/library/track//tag-preview', methods=['GET']) def get_track_tag_preview(track_id): """Read current file tags and compare against DB metadata for a single track.""" try: from core.tag_writer import read_file_tags, build_tag_diff database = get_database() # Get track + album + artist data from DB conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT t.*, a.name as artist_name, al.title as album_title, al.year, al.genres as album_genres, al.track_count, al.thumb_url as album_thumb_url, a.thumb_url as artist_thumb_url FROM tracks t JOIN artists a ON t.artist_id = a.id JOIN albums al ON t.album_id = al.id WHERE t.id = ? """, (str(track_id),)) row = cursor.fetchone() if not row: return jsonify({"success": False, "error": "Track not found"}), 404 track_data = dict(row) file_path = track_data.get('file_path') # Resolve path if needed resolved_path = _resolve_library_file_path(file_path) if not resolved_path: return jsonify({"success": False, "error": _get_file_not_found_error(file_path), "file_path": file_path}), 404 # Read current file tags file_tags = read_file_tags(resolved_path) if file_tags.get('error'): return jsonify({"success": False, "error": file_tags['error']}), 400 # Parse album genres for diff album_genres = [] if track_data.get('album_genres'): try: import json as _json parsed = _json.loads(track_data['album_genres']) album_genres = parsed if isinstance(parsed, list) else [str(parsed)] except (ValueError, TypeError): album_genres = [g.strip() for g in track_data['album_genres'].split(',') if g.strip()] # Build DB metadata dict for comparison db_data = { 'title': track_data.get('title'), 'artist_name': track_data.get('artist_name'), 'track_artist': track_data.get('track_artist'), 'album_title': track_data.get('album_title'), 'year': track_data.get('year'), 'genres': album_genres, 'track_number': track_data.get('track_number'), 'disc_number': track_data.get('disc_number'), 'bpm': track_data.get('bpm'), 'track_count': track_data.get('track_count'), 'thumb_url': track_data.get('album_thumb_url') or track_data.get('artist_thumb_url'), } diff = build_tag_diff(file_tags, db_data) has_changes = any(d['changed'] for d in diff) # Include server type so frontend can offer server sync option active_server = config_manager.get_active_media_server() server_connected = False if active_server == 'plex': server_connected = plex_client.is_connected() elif active_server == 'jellyfin': server_connected = jellyfin_client.is_connected() elif active_server == 'navidrome': server_connected = navidrome_client.is_connected() return jsonify({ "success": True, "file_path": resolved_path, "file_tags": file_tags, "db_data": db_data, "diff": diff, "has_changes": has_changes, "server_type": active_server if server_connected else None, }) except Exception as e: logger.error(f"Tag preview error for track {track_id}: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/tracks/tag-preview-batch', methods=['POST']) def get_batch_tag_preview(): """Read current file tags and compare against DB metadata for multiple tracks.""" try: from core.tag_writer import read_file_tags, build_tag_diff data = request.get_json() track_ids = data.get('track_ids', []) if not track_ids: return jsonify({"success": False, "error": "No track IDs provided"}), 400 database = get_database() conn = database._get_connection() cursor = conn.cursor() placeholders = ','.join('?' for _ in track_ids) cursor.execute(f""" SELECT t.*, a.name as artist_name, al.title as album_title, al.year, al.genres as album_genres, al.track_count, al.thumb_url as album_thumb_url, a.thumb_url as artist_thumb_url FROM tracks t JOIN artists a ON t.artist_id = a.id JOIN albums al ON t.album_id = al.id WHERE t.id IN ({placeholders}) """, [str(tid) for tid in track_ids]) rows = [dict(r) for r in cursor.fetchall()] results = [] for track_data in rows: track_id = track_data['id'] file_path = track_data.get('file_path') resolved_path = _resolve_library_file_path(file_path) entry = { 'track_id': track_id, 'title': track_data.get('title', 'Unknown'), 'track_number': track_data.get('track_number'), } if not resolved_path: entry['error'] = _get_file_not_found_error(file_path) results.append(entry) continue try: file_tags = read_file_tags(resolved_path) if file_tags.get('error'): entry['error'] = file_tags['error'] results.append(entry) continue album_genres = [] if track_data.get('album_genres'): try: parsed = json.loads(track_data['album_genres']) album_genres = parsed if isinstance(parsed, list) else [str(parsed)] except (ValueError, TypeError): album_genres = [g.strip() for g in track_data['album_genres'].split(',') if g.strip()] db_data = { 'title': track_data.get('title'), 'artist_name': track_data.get('artist_name'), 'album_title': track_data.get('album_title'), 'year': track_data.get('year'), 'genres': album_genres, 'track_number': track_data.get('track_number'), 'disc_number': track_data.get('disc_number'), 'bpm': track_data.get('bpm'), 'track_count': track_data.get('track_count'), 'thumb_url': track_data.get('album_thumb_url') or track_data.get('artist_thumb_url'), } diff = build_tag_diff(file_tags, db_data) has_changes = any(d['changed'] for d in diff) changed_fields = [d for d in diff if d['changed']] entry['diff'] = diff entry['has_changes'] = has_changes entry['changed_count'] = len(changed_fields) except Exception as e: entry['error'] = str(e) results.append(entry) # Server type info active_server = config_manager.get_active_media_server() server_connected = False if active_server == 'plex': server_connected = plex_client.is_connected() elif active_server == 'jellyfin': server_connected = jellyfin_client.is_connected() elif active_server == 'navidrome': server_connected = navidrome_client.is_connected() return jsonify({ "success": True, "tracks": results, "server_type": active_server if server_connected else None, }) except Exception as e: logger.error(f"Batch tag preview error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/track//write-tags', methods=['POST']) def write_track_tags(track_id): """Write DB metadata into the audio file tags for a single track.""" try: from core.tag_writer import write_tags_to_file database = get_database() data = request.get_json() or {} embed_cover = data.get('embed_cover', True) # Get full track data conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT t.*, a.name as artist_name, al.title as album_title, al.year, al.genres as album_genres, al.track_count, al.thumb_url as album_thumb_url, a.thumb_url as artist_thumb_url FROM tracks t JOIN artists a ON t.artist_id = a.id JOIN albums al ON t.album_id = al.id WHERE t.id = ? """, (str(track_id),)) row = cursor.fetchone() if not row: return jsonify({"success": False, "error": "Track not found"}), 404 track_data = dict(row) file_path = track_data.get('file_path') resolved_path = _resolve_library_file_path(file_path) if not resolved_path: return jsonify({"success": False, "error": _get_file_not_found_error(file_path)}), 404 # Parse genres album_genres = [] if track_data.get('album_genres'): try: import json as _json parsed = _json.loads(track_data['album_genres']) album_genres = parsed if isinstance(parsed, list) else [str(parsed)] except (ValueError, TypeError): album_genres = [g.strip() for g in track_data['album_genres'].split(',') if g.strip()] # Build data for writer db_data = { 'title': track_data.get('title'), 'artist_name': track_data.get('artist_name'), 'album_title': track_data.get('album_title'), 'year': track_data.get('year'), 'genres': album_genres, 'track_number': track_data.get('track_number'), 'disc_number': track_data.get('disc_number'), 'bpm': track_data.get('bpm'), 'track_count': track_data.get('track_count'), } # Resolve cover URL cover_url = None if embed_cover: thumb = track_data.get('album_thumb_url') or track_data.get('artist_thumb_url') if thumb and thumb.startswith('http'): cover_url = thumb # Use file lock for thread safety file_lock = get_file_lock(resolved_path) with file_lock: result = write_tags_to_file(resolved_path, db_data, embed_cover=embed_cover, cover_url=cover_url) # Sync to media server if requested and write succeeded sync_result = None if result.get('success') and data.get('sync_to_server'): server_type = config_manager.get_active_media_server() sync_result = _sync_tracks_to_server([track_data], server_type) result['server_sync'] = sync_result return jsonify(result) except Exception as e: logger.error(f"Write tags error for track {track_id}: {e}") return jsonify({"success": False, "error": str(e)}), 500 _write_tags_batch_state = { 'status': 'idle', # idle | running | done 'total': 0, 'processed': 0, 'written': 0, 'failed': 0, 'current_track': '', 'errors': [], 'sync_phase': None, # None | 'syncing' | 'done' 'sync_server': None, 'sync_synced': 0, 'sync_failed': 0, } _write_tags_batch_lock = threading.Lock() @app.route('/api/library/tracks/write-tags-batch', methods=['POST']) def write_tracks_tags_batch(): """Write DB metadata into audio file tags for multiple tracks (runs in background).""" try: with _write_tags_batch_lock: if _write_tags_batch_state['status'] == 'running': return jsonify({"success": False, "error": "A batch tag write is already in progress"}), 409 database = get_database() data = request.get_json() if not data or not data.get('track_ids'): return jsonify({"success": False, "error": "track_ids required"}), 400 track_ids = data['track_ids'] embed_cover = data.get('embed_cover', True) # Fetch all track data upfront (in the request thread, fast DB query) conn = database._get_connection() cursor = conn.cursor() placeholders = ','.join('?' * len(track_ids)) cursor.execute(f""" SELECT t.*, a.name as artist_name, al.title as album_title, al.year, al.genres as album_genres, al.track_count, al.thumb_url as album_thumb_url, a.thumb_url as artist_thumb_url FROM tracks t JOIN artists a ON t.artist_id = a.id JOIN albums al ON t.album_id = al.id WHERE t.id IN ({placeholders}) """, [str(tid) for tid in track_ids]) rows = [dict(r) for r in cursor.fetchall()] sync_to_server = data.get('sync_to_server', False) # Initialize state with _write_tags_batch_lock: _write_tags_batch_state.update({ 'status': 'running', 'total': len(track_ids), 'processed': 0, 'written': 0, 'failed': 0, 'current_track': '', 'errors': [], 'sync_phase': None, 'sync_server': None, 'sync_synced': 0, 'sync_failed': 0, }) # Count missing DB rows found_ids = {str(r['id']) for r in rows} missing = [tid for tid in track_ids if str(tid) not in found_ids] if missing: with _write_tags_batch_lock: _write_tags_batch_state['failed'] += len(missing) _write_tags_batch_state['processed'] += len(missing) for tid in missing: _write_tags_batch_state['errors'].append({'track_id': tid, 'error': 'Track not found in database'}) # Run the actual writes in a background thread def _run_batch(): try: from core.tag_writer import write_tags_to_file, download_cover_art written_tracks = [] # Track dicts that were successfully written (for server sync) # Pre-download cover art once per unique album URL cover_cache = {} # url → (bytes, mime) or None if embed_cover: unique_urls = set() for td in rows: thumb = td.get('album_thumb_url') or td.get('artist_thumb_url') if thumb and thumb.startswith('http'): unique_urls.add(thumb) if unique_urls: with _write_tags_batch_lock: _write_tags_batch_state['current_track'] = f'Downloading cover art ({len(unique_urls)} album{"s" if len(unique_urls) != 1 else ""})...' for url in unique_urls: cover_cache[url] = download_cover_art(url) for track_data in rows: file_path = track_data.get('file_path') resolved_path = _resolve_library_file_path(file_path) track_title = track_data.get('title', 'Unknown') with _write_tags_batch_lock: _write_tags_batch_state['current_track'] = track_title if not resolved_path: with _write_tags_batch_lock: _write_tags_batch_state['failed'] += 1 _write_tags_batch_state['processed'] += 1 _write_tags_batch_state['errors'].append({'track_id': track_data['id'], 'error': _get_file_not_found_error(file_path)}) continue # Parse genres album_genres = [] if track_data.get('album_genres'): try: parsed = json.loads(track_data['album_genres']) album_genres = parsed if isinstance(parsed, list) else [str(parsed)] except (ValueError, TypeError): album_genres = [g.strip() for g in track_data['album_genres'].split(',') if g.strip()] db_data = { 'title': track_data.get('title'), 'artist_name': track_data.get('artist_name'), 'album_title': track_data.get('album_title'), 'year': track_data.get('year'), 'genres': album_genres, 'track_number': track_data.get('track_number'), 'disc_number': track_data.get('disc_number'), 'bpm': track_data.get('bpm'), 'track_count': track_data.get('track_count'), } # Get pre-downloaded cover art for this track's album art_data = None if embed_cover: thumb = track_data.get('album_thumb_url') or track_data.get('artist_thumb_url') if thumb and thumb.startswith('http'): art_data = cover_cache.get(thumb) file_lock = get_file_lock(resolved_path) with file_lock: write_result = write_tags_to_file( resolved_path, db_data, embed_cover=embed_cover, cover_data=art_data ) with _write_tags_batch_lock: _write_tags_batch_state['processed'] += 1 if write_result.get('success'): _write_tags_batch_state['written'] += 1 written_tracks.append(track_data) else: _write_tags_batch_state['failed'] += 1 _write_tags_batch_state['errors'].append({ 'track_id': track_data['id'], 'error': write_result.get('error', 'Unknown') }) # Server sync phase if sync_to_server and written_tracks: server_type = config_manager.get_active_media_server() with _write_tags_batch_lock: _write_tags_batch_state['sync_phase'] = 'syncing' _write_tags_batch_state['sync_server'] = server_type _write_tags_batch_state['current_track'] = f'Syncing to {server_type.title()}...' sync_result = _sync_tracks_to_server(written_tracks, server_type) with _write_tags_batch_lock: _write_tags_batch_state['sync_phase'] = 'done' _write_tags_batch_state['sync_synced'] = sync_result['synced'] _write_tags_batch_state['sync_failed'] = sync_result['failed'] except Exception as e: logger.error(f"Batch write tags background error: {e}") finally: with _write_tags_batch_lock: _write_tags_batch_state['status'] = 'done' _write_tags_batch_state['current_track'] = '' thread = threading.Thread(target=_run_batch, daemon=True, name="WriteTagsBatch") thread.start() return jsonify({"success": True, "message": "Batch tag write started", "total": len(track_ids)}) except Exception as e: logger.error(f"Batch write tags error: {e}") with _write_tags_batch_lock: _write_tags_batch_state['status'] = 'idle' return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/tracks/write-tags-batch/status', methods=['GET']) def get_write_tags_batch_status(): """Poll the status of a running batch tag write.""" with _write_tags_batch_lock: state = dict(_write_tags_batch_state) state['errors'] = list(_write_tags_batch_state['errors']) # snapshot to avoid mutation during serialize return jsonify(state) # ── ReplayGain Analysis endpoints ── from core.replaygain import ( analyze_track as _rg_analyze_track, write_replaygain_tags as _rg_write_tags, is_ffmpeg_available as _rg_ffmpeg_available, RG_REFERENCE_LUFS as _RG_REFERENCE_LUFS, ) # State machine for album-level ReplayGain jobs _rg_album_state = { 'status': 'idle', # idle | running | done 'album_id': None, 'total': 0, 'processed': 0, 'analyzed': 0, 'failed': 0, 'current_track': '', 'errors': [], } _rg_album_lock = threading.Lock() # State machine for selected-tracks batch ReplayGain jobs _rg_batch_state = { 'status': 'idle', 'total': 0, 'processed': 0, 'analyzed': 0, 'failed': 0, 'current_track': '', 'errors': [], } _rg_batch_lock = threading.Lock() @app.route('/api/library/track//analyze-replaygain', methods=['POST']) def analyze_track_replaygain(track_id): """ Analyze a single track and write ReplayGain track-level tags immediately. Synchronous — runs FFmpeg inline (typically 1–3 s per track). """ if not _rg_ffmpeg_available(): return jsonify({'success': False, 'error': 'ffmpeg not found on PATH'}), 500 database = get_database() conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT * FROM tracks WHERE id = ?", (str(track_id),)) row = cursor.fetchone() if not row: return jsonify({'success': False, 'error': 'Track not found'}), 404 file_path = _resolve_library_file_path(dict(row).get('file_path')) if not file_path: return jsonify({'success': False, 'error': 'File not found on disk'}), 404 try: lufs, peak_dbfs = _rg_analyze_track(file_path) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 track_gain_db = _RG_REFERENCE_LUFS - lufs file_lock = get_file_lock(file_path) with file_lock: ok = _rg_write_tags(file_path, track_gain_db, peak_dbfs) if not ok: return jsonify({'success': False, 'error': 'Failed to write tags to file'}), 500 return jsonify({ 'success': True, 'track_gain': f"{track_gain_db:+.2f} dB", 'track_peak': f"{10 ** (peak_dbfs / 20.0):.6f}", 'lufs': round(lufs, 2), }) @app.route('/api/library/album//analyze-replaygain', methods=['POST']) def analyze_album_replaygain(album_id): """ Analyze all tracks in an album and write both track-level and album-level ReplayGain tags. Runs in a background thread — poll /status for progress. """ with _rg_album_lock: if _rg_album_state['status'] == 'running': return jsonify({'success': False, 'error': 'An album ReplayGain job is already running'}), 409 if not _rg_ffmpeg_available(): return jsonify({'success': False, 'error': 'ffmpeg not found on PATH'}), 500 database = get_database() conn = database._get_connection() cursor = conn.cursor() cursor.execute( "SELECT * FROM tracks WHERE album_id = ? ORDER BY track_number, title", (album_id,) ) tracks = [dict(r) for r in cursor.fetchall()] if not tracks: return jsonify({'success': False, 'error': 'No tracks found for this album'}), 404 with _rg_album_lock: _rg_album_state.update({ 'status': 'running', 'album_id': album_id, 'total': len(tracks), 'processed': 0, 'analyzed': 0, 'failed': 0, 'current_track': '', 'errors': [], }) def _run_album(): lufs_values = [] peak_values = [] track_results = [] # (file_path, track_gain_db, peak_dbfs) # Pass 1: analyze every track for track in tracks: file_path = _resolve_library_file_path(track.get('file_path')) title = track.get('title') or track.get('file_path') or '' with _rg_album_lock: _rg_album_state['current_track'] = title if not file_path: with _rg_album_lock: _rg_album_state['failed'] += 1 _rg_album_state['errors'].append({'track': title, 'error': 'File not found'}) _rg_album_state['processed'] += 1 track_results.append(None) continue try: lufs, peak_dbfs = _rg_analyze_track(file_path) lufs_values.append(lufs) peak_values.append(peak_dbfs) track_gain_db = _RG_REFERENCE_LUFS - lufs track_results.append((file_path, track_gain_db, peak_dbfs)) with _rg_album_lock: _rg_album_state['analyzed'] += 1 _rg_album_state['processed'] += 1 except Exception as e: with _rg_album_lock: _rg_album_state['failed'] += 1 _rg_album_state['errors'].append({'track': title, 'error': str(e)}) _rg_album_state['processed'] += 1 track_results.append(None) # Compute album gain from tracks that analyzed successfully album_gain_db = None album_peak_dbfs = None if lufs_values: mean_lufs = sum(lufs_values) / len(lufs_values) album_gain_db = _RG_REFERENCE_LUFS - mean_lufs album_peak_dbfs = max(peak_values) # Pass 2: write tags to every successfully analyzed track for i, track in enumerate(tracks): entry = track_results[i] if entry is None: continue file_path, track_gain_db, peak_dbfs = entry try: file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs, album_gain_db, album_peak_dbfs) except Exception as e: with _rg_album_lock: _rg_album_state['failed'] += 1 _rg_album_state['errors'].append({'track': track.get('title', ''), 'error': str(e)}) with _rg_album_lock: _rg_album_state['status'] = 'done' _rg_album_state['current_track'] = '' threading.Thread(target=_run_album, daemon=True, name='RgAlbum').start() return jsonify({'success': True}) @app.route('/api/library/album//analyze-replaygain/status', methods=['GET']) def get_album_replaygain_status(album_id): """Poll the status of a running album ReplayGain job.""" with _rg_album_lock: state = dict(_rg_album_state) state['errors'] = list(_rg_album_state['errors']) return jsonify(state) @app.route('/api/library/tracks/analyze-replaygain-batch', methods=['POST']) def analyze_tracks_replaygain_batch(): """ Analyze a set of selected tracks and write track-level ReplayGain tags. No album gain is computed (tracks may span multiple albums). Runs in a background thread — poll /status for progress. """ with _rg_batch_lock: if _rg_batch_state['status'] == 'running': return jsonify({'success': False, 'error': 'A batch ReplayGain job is already running'}), 409 if not _rg_ffmpeg_available(): return jsonify({'success': False, 'error': 'ffmpeg not found on PATH'}), 500 data = request.get_json() or {} track_ids = data.get('track_ids', []) if not track_ids: return jsonify({'success': False, 'error': 'No track IDs provided'}), 400 database = get_database() conn = database._get_connection() cursor = conn.cursor() placeholders = ','.join('?' for _ in track_ids) cursor.execute( f"SELECT * FROM tracks WHERE id IN ({placeholders})", [str(tid) for tid in track_ids] ) tracks = [dict(r) for r in cursor.fetchall()] if not tracks: return jsonify({'success': False, 'error': 'No valid tracks found'}), 404 with _rg_batch_lock: _rg_batch_state.update({ 'status': 'running', 'total': len(tracks), 'processed': 0, 'analyzed': 0, 'failed': 0, 'current_track': '', 'errors': [], }) def _run_batch(): for track in tracks: file_path = _resolve_library_file_path(track.get('file_path')) title = track.get('title') or track.get('file_path') or '' with _rg_batch_lock: _rg_batch_state['current_track'] = title if not file_path: with _rg_batch_lock: _rg_batch_state['failed'] += 1 _rg_batch_state['errors'].append({'track': title, 'error': 'File not found'}) _rg_batch_state['processed'] += 1 continue try: lufs, peak_dbfs = _rg_analyze_track(file_path) track_gain_db = _RG_REFERENCE_LUFS - lufs file_lock = get_file_lock(file_path) with file_lock: _rg_write_tags(file_path, track_gain_db, peak_dbfs) with _rg_batch_lock: _rg_batch_state['analyzed'] += 1 _rg_batch_state['processed'] += 1 except Exception as e: with _rg_batch_lock: _rg_batch_state['failed'] += 1 _rg_batch_state['errors'].append({'track': title, 'error': str(e)}) _rg_batch_state['processed'] += 1 with _rg_batch_lock: _rg_batch_state['status'] = 'done' _rg_batch_state['current_track'] = '' threading.Thread(target=_run_batch, daemon=True, name='RgBatch').start() return jsonify({'success': True}) @app.route('/api/library/tracks/analyze-replaygain-batch/status', methods=['GET']) def get_tracks_replaygain_batch_status(): """Poll the status of a running batch ReplayGain job.""" with _rg_batch_lock: state = dict(_rg_batch_state) state['errors'] = list(_rg_batch_state['errors']) return jsonify(state) # ── Reorganize Album Files endpoints ── # # Reorganize requests flow through ``core.reorganize_queue`` — a FIFO # queue with a single background worker. The endpoints here are thin # enqueue / snapshot / cancel wrappers; the heavy lifting is in # :mod:`core.library_reorganize`. @app.route('/api/library/reorganize/sources', methods=['GET']) def reorganize_sources_global(): """List metadata sources the user has authed on this instance. Used by the bulk "Reorganize All" modal where per-album ID coverage varies. No network calls.""" try: from core.library_reorganize import authed_sources return jsonify({"success": True, "sources": authed_sources()}) except Exception as e: logger.error(f"Reorganize sources (global) error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/album//reorganize/sources', methods=['GET']) def reorganize_album_sources(album_id): """List metadata sources the user can pick for this album's reorganize — every entry has both a stored album ID on the local row AND an authenticated client. No network calls.""" try: from core.library_reorganize import available_sources_for_album, load_album_and_tracks album_data, _tracks = load_album_and_tracks(get_database(), album_id) if album_data is None: return jsonify({"success": False, "error": "Album not found"}), 404 return jsonify({"success": True, "sources": available_sources_for_album(album_data)}) except Exception as e: logger.error(f"Reorganize sources error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/album//reorganize/preview', methods=['POST']) def reorganize_album_preview(album_id): """Preview file reorganization for an album — returns current vs proposed paths without moving anything. Implementation lives in :mod:`core.library_reorganize` and shares the planning logic with the apply endpoint, so the preview is guaranteed to match what apply would actually produce. Optional body param ``source``: when provided, only that metadata source is queried (no fallback chain).""" try: from core.library_reorganize import preview_album_reorganize data = request.get_json() or {} chosen_source = data.get('source') or None transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) result = preview_album_reorganize( album_id=album_id, db=get_database(), transfer_dir=transfer_dir, resolve_file_path_fn=_resolve_library_file_path, build_final_path_fn=_build_final_path_for_track, primary_source=chosen_source, strict_source=bool(chosen_source), ) if result.get('status') == 'no_album': return jsonify({"success": False, "error": "Album not found"}), 404 if result.get('status') == 'no_tracks': return jsonify({"success": False, "error": "No tracks found for this album"}), 404 return jsonify(result) except Exception as e: logger.error(f"Reorganize preview error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/album//reorganize', methods=['POST']) def reorganize_album_files(album_id): """Enqueue an album for reorganize. Returns immediately — the queue worker processes items FIFO. Repeat clicks for an album that's already queued or running are deduped (returns ``{queued: false, reason: 'already_queued'}``). Body params: source (optional): per-album source pick (Spotify / iTunes / Deezer / Discogs / Hydrabase). When omitted, the orchestrator uses the configured primary with fallback. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None # Capture display fields at enqueue time so the status panel # can render them without a DB lookup later. meta = get_database().get_album_display_meta(album_id) if meta is None: return jsonify({"success": False, "error": "Album not found"}), 404 result = get_queue().enqueue( album_id=str(album_id), album_title=meta['album_title'], artist_id=meta['artist_id'], artist_name=meta['artist_name'], source=chosen_source, ) return jsonify({"success": True, **result}) except Exception as e: logger.error(f"Reorganize enqueue error: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/artist//reorganize-all', methods=['POST']) def reorganize_all_artist_albums(artist_id): """Enqueue every album for an artist. Replaces the old frontend bulk-loop. Each album becomes its own queue item, processed FIFO. Albums already queued or running are deduped silently. Body params: source (optional): same pick applied to every album. Per-album overrides aren't supported here — use the per-album modal for that. """ try: from core.reorganize_queue import get_queue data = request.get_json() or {} chosen_source = data.get('source') or None albums = get_database().get_artist_albums_for_reorganize(artist_id) if not albums: return jsonify({"success": False, "error": "No albums found for this artist"}), 404 # Apply the user's chosen source to every album, then hand off # to the queue's bulk-enqueue helper which owns the loop+tally. for album in albums: album['source'] = chosen_source result = get_queue().enqueue_many(albums) return jsonify({ "success": True, "enqueued": result['enqueued'], "already_queued": result['already_queued'], "total_albums": result['total'], }) except Exception as e: logger.error(f"Reorganize-all enqueue error: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/reorganize/queue', methods=['GET']) def reorganize_queue_snapshot(): """Snapshot of the reorganize queue — what's running, what's queued, recent completions. Polled by the status panel.""" try: from core.reorganize_queue import get_queue return jsonify({"success": True, **get_queue().snapshot()}) except Exception as e: logger.error(f"Reorganize queue snapshot error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/reorganize/queue//cancel', methods=['POST']) def reorganize_queue_cancel(queue_id): """Cancel a queued item (running items can't be cleanly cancelled — see the queue module's design rules).""" try: from core.reorganize_queue import get_queue result = get_queue().cancel(queue_id) status_code = 200 if result.get('cancelled') else 409 return jsonify({"success": result.get('cancelled', False), **result}), status_code except Exception as e: logger.error(f"Reorganize cancel error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/reorganize/queue/clear', methods=['POST']) def reorganize_queue_clear(): """Cancel all queued items at once (the running item continues).""" try: from core.reorganize_queue import get_queue cancelled = get_queue().clear_queued() return jsonify({"success": True, "cancelled": cancelled}) except Exception as e: logger.error(f"Reorganize clear error: {e}") return jsonify({"success": False, "error": str(e)}), 500 # Wire the reorganize queue worker to its runner at module load. The # runner factory lives in :mod:`core.reorganize_runner` so this monolith # stays small. Config (paths) is read **per run** inside the closure, # so changing your download path in Settings takes effect on the next # reorganize without a server restart. # # The injected callables are wrapped in lambdas because the underlying # helpers (``_resolve_library_file_path`` etc.) are defined LATER in # this file. Lambdas defer name resolution to call time so module-load # import order works regardless of definition order. try: from core.reorganize_queue import get_queue as _get_reorganize_queue from core.reorganize_runner import build_runner as _build_reorganize_runner _get_reorganize_queue().set_runner(_build_reorganize_runner( get_database=get_database, resolve_file_path_fn=lambda p: _resolve_library_file_path(p), post_process_fn=lambda *a, **kw: _post_process_matched_download(*a, **kw), cleanup_empty_directories_fn=lambda *a, **kw: _cleanup_empty_directories(*a, **kw), is_shutting_down_fn=lambda: bool(IS_SHUTTING_DOWN), get_download_path=lambda: docker_resolve_path( config_manager.get('soulseek.download_path', './downloads') ), get_transfer_path=lambda: docker_resolve_path( config_manager.get('soulseek.transfer_path', './Transfer') ), )) except Exception as _runner_init_err: logger.error(f"Failed to register reorganize queue runner: {_runner_init_err}") # ── Library Issues endpoints ── @app.route('/api/issues', methods=['GET']) def list_issues(): """List issues. Admin sees all; non-admin sees own only.""" try: database = get_database() profile_id = request.headers.get('X-Profile-Id', '1') try: profile_id = int(profile_id) except (ValueError, TypeError): profile_id = 1 # Determine admin status profile = database.get_profile(profile_id) is_admin = profile.get('is_admin', False) if profile else False status = request.args.get('status') category = request.args.get('category') entity_type = request.args.get('entity_type') try: limit = min(200, max(1, int(request.args.get('limit', 100)))) except (ValueError, TypeError): limit = 100 try: offset = max(0, int(request.args.get('offset', 0))) except (ValueError, TypeError): offset = 0 result = database.get_issues( profile_id=profile_id, status=status, category=category, entity_type=entity_type, limit=limit, offset=offset, is_admin=is_admin, ) # Fix Plex/Jellyfin relative thumb URLs in stored snapshots for issue in result.get('issues', []): snap = issue.get('snapshot_data') if isinstance(snap, dict): for key in ('thumb_url', 'artist_thumb', 'album_thumb'): if snap.get(key): snap[key] = fix_artist_image_url(snap[key]) or snap[key] return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/issues', methods=['POST']) def create_issue(): """Create a new library issue.""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 # Use header for profile_id (not body) to prevent spoofing profile_id = request.headers.get('X-Profile-Id', '1') try: profile_id = int(profile_id) except (ValueError, TypeError): profile_id = 1 entity_type = data.get('entity_type') entity_id = data.get('entity_id') category = data.get('category') title = data.get('title', '').strip() description = data.get('description', '').strip() priority = data.get('priority', 'normal') if not entity_type or not entity_id or not category or not title: return jsonify({"success": False, "error": "entity_type, entity_id, category, and title are required"}), 400 valid_types = ('artist', 'album', 'track') if entity_type not in valid_types: return jsonify({"success": False, "error": f"entity_type must be one of: {', '.join(valid_types)}"}), 400 valid_categories = ('wrong_track', 'wrong_metadata', 'wrong_cover', 'duplicate_tracks', 'missing_tracks', 'audio_quality', 'wrong_artist', 'wrong_album', 'incomplete_album', 'other') if category not in valid_categories: return jsonify({"success": False, "error": f"Invalid category: {category}"}), 400 # Build snapshot of the entity's current state snapshot = _build_issue_snapshot(database, entity_type, str(entity_id)) result = database.create_issue( profile_id=profile_id, entity_type=entity_type, entity_id=str(entity_id), category=category, title=title, description=description, snapshot_data=snapshot, priority=priority, ) return jsonify(result), 201 if result.get('success') else 400 except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/issues/', methods=['GET']) def get_issue(issue_id): """Get a single issue.""" try: database = get_database() issue = database.get_issue(issue_id) if not issue: return jsonify({"success": False, "error": "Issue not found"}), 404 # Fix Plex/Jellyfin relative thumb URLs in stored snapshot snap = issue.get('snapshot_data') if isinstance(snap, dict): for key in ('thumb_url', 'artist_thumb', 'album_thumb'): if snap.get(key): snap[key] = fix_artist_image_url(snap[key]) or snap[key] return jsonify({"success": True, "issue": issue}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/issues/', methods=['PUT']) def update_issue(issue_id): """Update an issue (admin: respond/resolve; user: edit own description).""" try: database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 profile_id = request.headers.get('X-Profile-Id', '1') try: profile_id = int(profile_id) except (ValueError, TypeError): profile_id = 1 profile = database.get_profile(profile_id) is_admin = profile.get('is_admin', False) if profile else False # Non-admin can only edit their own issue's title/description if not is_admin: issue = database.get_issue(issue_id) if not issue: return jsonify({"success": False, "error": "Issue not found"}), 404 if issue['profile_id'] != profile_id: return jsonify({"success": False, "error": "Not authorized"}), 403 data = {k: v for k, v in data.items() if k in ('title', 'description')} # If resolving, stamp resolved_by and resolved_at if data.get('status') in ('resolved', 'dismissed') and is_admin: data['resolved_by'] = profile_id from datetime import datetime data['resolved_at'] = datetime.utcnow().strftime('%Y-%m-%d %H:%M:%S') # If reopening, clear resolution metadata elif data.get('status') in ('open', 'in_progress') and is_admin: data['resolved_by'] = None data['resolved_at'] = None result = database.update_issue(issue_id, data) return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/issues/', methods=['DELETE']) def delete_issue(issue_id): """Delete an issue (admin or issue owner).""" try: database = get_database() profile_id = request.headers.get('X-Profile-Id', '1') try: profile_id = int(profile_id) except (ValueError, TypeError): profile_id = 1 profile = database.get_profile(profile_id) is_admin = profile.get('is_admin', False) if profile else False if not is_admin: issue = database.get_issue(issue_id) if not issue: return jsonify({"success": False, "error": "Issue not found"}), 404 if issue['profile_id'] != profile_id: return jsonify({"success": False, "error": "Not authorized"}), 403 result = database.delete_issue(issue_id) return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/issues/counts', methods=['GET']) def get_issue_counts(): """Get issue counts by status for badge display.""" try: database = get_database() profile_id = request.headers.get('X-Profile-Id', '1') try: profile_id = int(profile_id) except (ValueError, TypeError): profile_id = 1 profile = database.get_profile(profile_id) is_admin = profile.get('is_admin', False) if profile else False counts = database.get_issue_counts(is_admin=is_admin, profile_id=profile_id) return jsonify({"success": True, "counts": counts}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 def _build_issue_snapshot(database, entity_type, entity_id): """Capture current state of the entity for the issue report.""" snapshot = {} try: conn = database._get_connection() cursor = conn.cursor() if entity_type == 'track': cursor.execute(""" SELECT t.id, t.title, t.track_number, t.duration, t.file_path, t.bitrate, t.bpm, t.spotify_track_id, t.musicbrainz_recording_id, t.deezer_id as track_deezer_id, a.name as artist_name, a.id as artist_id, a.spotify_artist_id, a.musicbrainz_id as artist_musicbrainz_id, a.deezer_id as artist_deezer_id, a.tidal_id as artist_tidal_id, a.qobuz_id as artist_qobuz_id, a.thumb_url as artist_thumb, al.title as album_title, al.year, al.thumb_url as album_thumb, al.id as album_id, al.spotify_album_id, al.musicbrainz_release_id, al.deezer_id as album_deezer_id, al.tidal_id as album_tidal_id, al.qobuz_id as album_qobuz_id, al.label, al.record_type, al.track_count as album_track_count FROM tracks t JOIN artists a ON t.artist_id = a.id JOIN albums al ON t.album_id = al.id WHERE t.id = ? """, (entity_id,)) row = cursor.fetchone() if row: d = dict(row) # Add format info if file exists resolved = _resolve_library_file_path(d.get('file_path')) if resolved: ext = os.path.splitext(resolved)[1].lower().lstrip('.') d['format'] = ext.upper() d['quality'] = _get_audio_quality_string(resolved) # Fix Plex/Jellyfin relative thumb URLs if d.get('artist_thumb'): d['artist_thumb'] = fix_artist_image_url(d['artist_thumb']) or d['artist_thumb'] if d.get('album_thumb'): d['album_thumb'] = fix_artist_image_url(d['album_thumb']) or d['album_thumb'] snapshot = d elif entity_type == 'album': cursor.execute(""" SELECT al.id, al.title, al.year, al.track_count, al.thumb_url, al.genres, al.label, al.record_type, al.duration, al.spotify_album_id, al.musicbrainz_release_id, al.deezer_id as album_deezer_id, al.tidal_id as album_tidal_id, al.qobuz_id as album_qobuz_id, al.upc, a.name as artist_name, a.id as artist_id, a.spotify_artist_id, a.musicbrainz_id as artist_musicbrainz_id, a.deezer_id as artist_deezer_id, a.tidal_id as artist_tidal_id, a.qobuz_id as artist_qobuz_id, a.thumb_url as artist_thumb FROM albums al JOIN artists a ON al.artist_id = a.id WHERE al.id = ? """, (entity_id,)) row = cursor.fetchone() if row: d = dict(row) # Fix Plex/Jellyfin relative thumb URLs if d.get('thumb_url'): d['thumb_url'] = fix_artist_image_url(d['thumb_url']) or d['thumb_url'] if d.get('artist_thumb'): d['artist_thumb'] = fix_artist_image_url(d['artist_thumb']) or d['artist_thumb'] # Parse genres if d.get('genres'): try: d['genres'] = json.loads(d['genres']) except (json.JSONDecodeError, TypeError): pass # Get track listing with enriched data cursor.execute(""" SELECT id, title, track_number, duration, file_path, bitrate, spotify_track_id, bpm FROM tracks WHERE album_id = ? ORDER BY track_number """, (entity_id,)) tracks_list = [] for r in cursor.fetchall(): td = dict(r) # Add format from file extension if td.get('file_path'): resolved = _resolve_library_file_path(td['file_path']) if resolved: ext = os.path.splitext(resolved)[1].lower().lstrip('.') td['format'] = ext.upper() tracks_list.append(td) d['tracks'] = tracks_list snapshot = d elif entity_type == 'artist': cursor.execute(""" SELECT id, name, thumb_url, genres, summary, spotify_artist_id, musicbrainz_id as artist_musicbrainz_id, deezer_id as artist_deezer_id, tidal_id as artist_tidal_id, qobuz_id as artist_qobuz_id FROM artists WHERE id = ? """, (entity_id,)) row = cursor.fetchone() if row: d = dict(row) # Fix Plex/Jellyfin relative thumb URL if d.get('thumb_url'): d['thumb_url'] = fix_artist_image_url(d['thumb_url']) or d['thumb_url'] if d.get('genres'): try: d['genres'] = json.loads(d['genres']) except (json.JSONDecodeError, TypeError): pass snapshot = d except Exception as e: logger.error(f"Error building issue snapshot: {e}") snapshot['_snapshot_error'] = str(e) return snapshot def _sync_tracks_to_server(track_rows, server_type): """Sync metadata for tracks to the active media server after writing file tags. Args: track_rows: list of track dicts (must include 'id', 'title', 'artist_name', 'album_title', 'year', 'server_source') server_type: 'plex', 'jellyfin', or 'navidrome' Returns: dict with 'synced', 'failed', 'skipped' counts and 'errors' list """ result = {'synced': 0, 'failed': 0, 'skipped': 0, 'errors': []} if server_type == 'navidrome': # Navidrome auto-detects file tag changes, no action needed result['synced'] = len(track_rows) return result if server_type == 'plex': for track_data in track_rows: # Only sync tracks that came from this server if track_data.get('server_source') and track_data['server_source'] != 'plex': result['skipped'] += 1 continue try: metadata = {} if track_data.get('title'): metadata['title'] = track_data['title'] if track_data.get('artist_name'): metadata['artist'] = track_data['artist_name'] if track_data.get('album_title'): metadata['album'] = track_data['album_title'] if track_data.get('year'): metadata['year'] = track_data['year'] if metadata: success = plex_client.update_track_metadata(str(track_data['id']), metadata) if success: result['synced'] += 1 else: result['failed'] += 1 result['errors'].append({'track_id': track_data['id'], 'error': 'Plex update returned false'}) else: result['skipped'] += 1 except Exception as e: result['failed'] += 1 result['errors'].append({'track_id': track_data['id'], 'error': str(e)}) elif server_type == 'jellyfin': # Jellyfin: just trigger a library scan once after all file writes try: success = jellyfin_client.trigger_library_scan() if success: result['synced'] = len(track_rows) else: result['failed'] = len(track_rows) result['errors'].append({'error': 'Jellyfin library scan failed'}) except Exception as e: result['failed'] = len(track_rows) result['errors'].append({'error': f'Jellyfin scan error: {e}'}) return result def _resolve_library_file_path(file_path): """Resolve a library file path to an actual file on disk.""" if not file_path: return None if os.path.exists(file_path): return file_path # Try resolving server-side paths to local directories # Check transfer path, download path, and media server library paths transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) download_dir = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) # Also check the media server's music library path (handles Docker↔host path mismatch) library_dirs = set() try: if plex_client and plex_client.server and plex_client.music_library: for loc in plex_client.music_library.locations: library_dirs.add(loc) except Exception: pass # Check user-configured music library paths (Settings > Library) try: music_paths = config_manager.get('library.music_paths', []) if isinstance(music_paths, list): for p in music_paths: if p and isinstance(p, str): resolved_p = docker_resolve_path(p.strip()) if resolved_p: library_dirs.add(resolved_p) except Exception: pass path_parts = file_path.replace('\\', '/').split('/') # Try progressively shorter path suffixes against each candidate directory # (skip index 0 to avoid drive letter issues) for base_dir in [transfer_dir, download_dir] + list(library_dirs): if not base_dir or not os.path.isdir(base_dir): continue for i in range(1, len(path_parts)): candidate = os.path.join(base_dir, *path_parts[i:]) if os.path.exists(candidate): return candidate return None def _get_file_not_found_error(file_path): """Return a helpful error message when a library file can't be found.""" active_server = config_manager.get_active_media_server() if active_server == 'navidrome': # Check if path looks like a Navidrome fake path (no real filesystem root) # Fake paths look like: "Artist/Album/01 - Track.flac" or just "Track.flac" if file_path and ('/' not in file_path or not file_path.startswith('/')): return ('File not found — Navidrome may be sending virtual paths. ' 'Go to Navidrome → Profile → Players → select SoulSync → enable "Report Real Path", ' 'then run a full database refresh in SoulSync.') return ('File not found on disk — check that your Navidrome music folder ' 'is mounted in the SoulSync container and that "Report Real Path" is enabled ' 'in Navidrome\'s player settings.') return 'File not found on disk' @app.route('/api/library/play', methods=['POST']) def library_play_track(): """Start playing a track directly from the user's library (no download needed).""" try: data = request.get_json() if not data or not data.get('file_path'): return jsonify({"success": False, "error": "file_path is required"}), 400 file_path = data['file_path'] # Resolve server-side paths (e.g. /mnt/musicBackup/...) to local transfer path resolved = _resolve_library_file_path(file_path) if resolved: file_path = resolved else: return jsonify({"success": False, "error": _get_file_not_found_error(file_path)}), 404 logger.info(f"Library play request: {os.path.basename(file_path)}") # Set stream state to ready with the library file path directly with stream_lock: stream_state.update({ "status": "ready", "progress": 100, "track_info": { "title": data.get('title', os.path.basename(file_path)), "artist": data.get('artist', 'Unknown Artist'), "album": data.get('album', 'Unknown Album'), }, "file_path": file_path, "error_message": None, "is_library": True }) return jsonify({"success": True, "message": "Library track ready for playback"}) except Exception as e: logger.error(f"Error playing library track: {e}") return jsonify({"success": False, "error": str(e)}), 500 _enrichment_locks = {svc: threading.Lock() for svc in ('audiodb', 'deezer', 'musicbrainz', 'spotify', 'itunes', 'lastfm', 'genius', 'tidal', 'qobuz', 'discogs')} @app.route('/api/library/enrich', methods=['POST']) def library_enrich_entity(): """Trigger enrichment of a specific entity from a single service. Body: { entity_type: 'artist'|'album'|'track', entity_id: str, service: str, name: str, artist_name: str? } service: 'audiodb', 'deezer', 'musicbrainz', 'spotify', 'itunes', 'lastfm', 'genius', 'tidal', 'qobuz' """ try: data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 entity_type = data.get('entity_type') # artist, album, track entity_id = data.get('entity_id') service = data.get('service') name = data.get('name', '') artist_name = data.get('artist_name', '') if not entity_type or not entity_id or not service: return jsonify({"success": False, "error": "entity_type, entity_id, and service are required"}), 400 if entity_type not in ('artist', 'album', 'track'): return jsonify({"success": False, "error": "entity_type must be artist, album, or track"}), 400 valid_services = ('audiodb', 'deezer', 'musicbrainz', 'spotify', 'itunes', 'lastfm', 'genius', 'tidal', 'qobuz', 'discogs') if service not in valid_services: return jsonify({"success": False, "error": f"service must be one of: {', '.join(valid_services)}"}), 400 # Per-service lock to avoid thread-safety issues with shared worker clients lock = _enrichment_locks.get(service) if not lock: return jsonify({"success": False, "error": f"Unknown service: {service}"}), 400 acquired = lock.acquire(blocking=False) if not acquired: return jsonify({"success": False, "error": f"{service} enrichment already in progress. Please wait."}), 429 try: results = {} try: result = _run_single_enrichment(service, entity_type, entity_id, name, artist_name) results[service] = result except Exception as e: results[service] = {"success": False, "error": str(e)} finally: lock.release() # Re-fetch updated data to return fresh state database = get_database() updated = database.get_artist_full_detail( data.get('artist_id', entity_id) if entity_type == 'artist' else _get_artist_id_for_entity(database, entity_type, entity_id) ) # Fix image URLs in updated data if updated.get('success'): if updated.get('artist', {}).get('thumb_url'): updated['artist']['thumb_url'] = fix_artist_image_url(updated['artist']['thumb_url']) for album in updated.get('albums', []): if album.get('thumb_url'): album['thumb_url'] = fix_artist_image_url(album['thumb_url']) return jsonify({ "success": True, "results": results, "updated_data": updated if updated.get('success') else None }) except Exception as e: logger.error(f"Error enriching entity: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _get_artist_id_for_entity(database, entity_type, entity_id): """Look up the artist_id for an album or track.""" try: with database._get_connection() as conn: cursor = conn.cursor() if entity_type == 'album': cursor.execute("SELECT artist_id FROM albums WHERE id = ?", (entity_id,)) elif entity_type == 'track': cursor.execute("SELECT artist_id FROM tracks WHERE id = ?", (entity_id,)) else: return entity_id row = cursor.fetchone() return row['artist_id'] if row else entity_id except Exception: return entity_id def _run_single_enrichment(service, entity_type, entity_id, name, artist_name): """Run a single enrichment service on a single entity.""" if service == 'audiodb': if not audiodb_worker: return {"success": False, "error": "AudioDB worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name} if entity_type in ('album', 'track'): item['artist'] = artist_name audiodb_worker._process_item(item) return {"success": True, "message": f"AudioDB lookup complete for {entity_type}"} elif service == 'deezer': if not deezer_worker: return {"success": False, "error": "Deezer worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name, 'artist': artist_name} if entity_type == 'artist': deezer_worker._process_artist(entity_id, name) elif entity_type == 'album': deezer_worker._process_album(entity_id, name, artist_name, item) elif entity_type == 'track': deezer_worker._process_track(entity_id, name, artist_name, item) return {"success": True, "message": f"Deezer lookup complete for {entity_type}"} elif service == 'musicbrainz': if not mb_worker: return {"success": False, "error": "MusicBrainz worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name} if entity_type in ('album', 'track'): item['artist'] = artist_name mb_worker._process_item(item) return {"success": True, "message": f"MusicBrainz lookup complete for {entity_type}"} elif service == 'spotify': if not spotify_enrichment_worker: return {"success": False, "error": "Spotify worker not initialized"} if entity_type == 'artist': spotify_enrichment_worker._process_artist({'type': 'artist', 'id': entity_id, 'name': name}) elif entity_type == 'album': spotify_enrichment_worker._process_album_individual({'type': 'album_individual', 'id': entity_id, 'name': name, 'artist': artist_name}) elif entity_type == 'track': spotify_enrichment_worker._process_track_individual({'type': 'track_individual', 'id': entity_id, 'name': name, 'artist': artist_name}) return {"success": True, "message": f"Spotify lookup complete for {entity_type}"} elif service == 'itunes': if not itunes_enrichment_worker: return {"success": False, "error": "iTunes worker not initialized"} if entity_type == 'artist': itunes_enrichment_worker._process_artist({'type': 'artist', 'id': entity_id, 'name': name}) elif entity_type == 'album': itunes_enrichment_worker._process_album_individual({'type': 'album_individual', 'id': entity_id, 'name': name, 'artist': artist_name}) elif entity_type == 'track': itunes_enrichment_worker._process_track_individual({'type': 'track_individual', 'id': entity_id, 'name': name, 'artist': artist_name}) return {"success": True, "message": f"iTunes lookup complete for {entity_type}"} elif service == 'lastfm': if not lastfm_worker: return {"success": False, "error": "Last.fm worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name} if entity_type in ('album', 'track'): item['artist'] = artist_name lastfm_worker._process_item(item) return {"success": True, "message": f"Last.fm lookup complete for {entity_type}"} elif service == 'genius': if not genius_worker: return {"success": False, "error": "Genius worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name} if entity_type == 'track': item['artist'] = artist_name elif entity_type == 'album': return {"success": False, "error": "Genius does not support album enrichment"} genius_worker._process_item(item) return {"success": True, "message": f"Genius lookup complete for {entity_type}"} elif service == 'tidal': if not tidal_enrichment_worker: return {"success": False, "error": "Tidal worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name, 'artist': artist_name} if entity_type == 'artist': tidal_enrichment_worker._process_artist(entity_id, name) elif entity_type == 'album': tidal_enrichment_worker._process_album(entity_id, name, artist_name, item) elif entity_type == 'track': tidal_enrichment_worker._process_track(entity_id, name, artist_name, item) return {"success": True, "message": f"Tidal lookup complete for {entity_type}"} elif service == 'qobuz': if not qobuz_enrichment_worker: return {"success": False, "error": "Qobuz worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name, 'artist': artist_name} if entity_type == 'artist': qobuz_enrichment_worker._process_artist(entity_id, name) elif entity_type == 'album': qobuz_enrichment_worker._process_album(entity_id, name, artist_name, item) elif entity_type == 'track': qobuz_enrichment_worker._process_track(entity_id, name, artist_name, item) return {"success": True, "message": f"Qobuz lookup complete for {entity_type}"} elif service == 'discogs': if not discogs_worker: return {"success": False, "error": "Discogs worker not initialized"} item = {'type': entity_type, 'id': entity_id, 'name': name} if entity_type in ('album', 'track'): item['artist'] = artist_name if entity_type == 'track': return {"success": False, "error": "Discogs does not support track-level enrichment"} discogs_worker._process_item(item) return {"success": True, "message": f"Discogs lookup complete for {entity_type}"} else: return {"success": False, "error": f"Unknown service: {service}"} @app.route('/api/library/search-service', methods=['POST']) def library_search_service(): """Search a specific service for matching entities (for manual matching). Body: { service: str, entity_type: str, query: str } Returns normalized list of results. """ try: data = request.get_json() if not data: return jsonify({"success": False, "error": "Invalid or missing JSON body"}), 400 service = data.get('service', '') entity_type = data.get('entity_type', '') query = data.get('query', '').strip() if not service or not entity_type or not query: return jsonify({"success": False, "error": "service, entity_type, and query are required"}), 400 results = _search_service(service, entity_type, query) return jsonify({"success": True, "results": results}) except Exception as e: logger.error(f"Error searching service: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _detect_provider(items, client): """Detect actual provider from result IDs. Spotify IDs are alphanumeric; iTunes/Deezer IDs are purely numeric. If the results have numeric IDs, they came from the fallback source, not Spotify.""" if items and str(items[0].id).isdigit(): return client._fallback_source return 'spotify' def _search_service(service, entity_type, query): """Search a service and return normalized results.""" import requests as req_lib if service == 'spotify': if not spotify_enrichment_worker or not spotify_enrichment_worker.client: raise ValueError("Spotify worker not initialized") client = spotify_enrichment_worker.client if entity_type == 'artist': items = client.search_artists(query, limit=8) # Detect actual provider from result IDs — Spotify IDs are alphanumeric, # iTunes/Deezer IDs are purely numeric. Prevents storing wrong IDs. provider = _detect_provider(items, client) return [{'id': a.id, 'name': a.name, 'image': a.image_url, 'extra': ', '.join(a.genres[:3]) if a.genres else '', 'provider': provider} for a in items] elif entity_type == 'album': items = client.search_albums(query, limit=8) provider = _detect_provider(items, client) return [{'id': a.id, 'name': a.name, 'image': a.image_url, 'extra': f"{', '.join(a.artists)} · {a.release_date or ''}", 'provider': provider} for a in items] elif entity_type == 'track': items = client.search_tracks(query, limit=8) provider = _detect_provider(items, client) return [{'id': t.id, 'name': t.name, 'image': t.image_url, 'extra': f"{', '.join(t.artists)} · {t.album or ''}", 'provider': provider} for t in items] elif service == 'itunes': if not itunes_enrichment_worker or not itunes_enrichment_worker.client: raise ValueError("iTunes worker not initialized") client = itunes_enrichment_worker.client if entity_type == 'artist': items = client.search_artists(query, limit=8) return [{'id': a.id, 'name': a.name, 'image': a.image_url, 'extra': ', '.join(a.genres[:3]) if a.genres else ''} for a in items] elif entity_type == 'album': items = client.search_albums(query, limit=8) return [{'id': a.id, 'name': a.name, 'image': a.image_url, 'extra': f"{', '.join(a.artists)} · {a.release_date or ''}"} for a in items] elif entity_type == 'track': items = client.search_tracks(query, limit=8) return [{'id': t.id, 'name': t.name, 'image': t.image_url, 'extra': f"{', '.join(t.artists)} · {t.album or ''}"} for t in items] elif service == 'musicbrainz': if not mb_worker or not mb_worker.mb_service: raise ValueError("MusicBrainz worker not initialized") mb_client = mb_worker.mb_service.mb_client if entity_type == 'artist': items = mb_client.search_artist(query, limit=8) return [{'id': a['id'], 'name': a.get('name', ''), 'image': None, 'extra': f"Score: {a.get('score', '')} · {a.get('disambiguation', '') or a.get('country', '')}"} for a in items] elif entity_type == 'album': items = mb_client.search_release(query, limit=8) results = [] for r in items: artists = ', '.join(ac.get('name', '') for ac in r.get('artist-credit', []) if isinstance(ac, dict)) # Cover Art Archive provides album art by release MBID cover_url = f"https://coverartarchive.org/release/{r['id']}/front-250" if r.get('id') else None results.append({'id': r['id'], 'name': r.get('title', ''), 'image': cover_url, 'extra': f"{artists} · {r.get('date', '')} · Score: {r.get('score', '')}"}) return results elif entity_type == 'track': items = mb_client.search_recording(query, limit=8) results = [] for r in items: artists = ', '.join(ac.get('name', '') for ac in r.get('artist-credit', []) if isinstance(ac, dict)) results.append({'id': r['id'], 'name': r.get('title', ''), 'image': None, 'extra': f"{artists} · Score: {r.get('score', '')}"}) return results elif service == 'deezer': # Deezer client only returns single results, so hit the API directly for multiple type_map = {'artist': 'artist', 'album': 'album', 'track': 'track'} deezer_type = type_map.get(entity_type, 'track') try: resp = req_lib.get(f'https://api.deezer.com/search/{deezer_type}', params={'q': query, 'limit': 8}, timeout=10) data = resp.json().get('data', []) except Exception: data = [] results = [] for item in data: if entity_type == 'artist': results.append({'id': str(item.get('id', '')), 'name': item.get('name', ''), 'image': item.get('picture_medium'), 'extra': f"{item.get('nb_fan', 0)} fans"}) elif entity_type == 'album': artist_name = item.get('artist', {}).get('name', '') if isinstance(item.get('artist'), dict) else '' results.append({'id': str(item.get('id', '')), 'name': item.get('title', ''), 'image': item.get('cover_medium'), 'extra': artist_name}) elif entity_type == 'track': artist_name = item.get('artist', {}).get('name', '') if isinstance(item.get('artist'), dict) else '' album_name = item.get('album', {}).get('title', '') if isinstance(item.get('album'), dict) else '' results.append({'id': str(item.get('id', '')), 'name': item.get('title', ''), 'image': item.get('album', {}).get('cover_medium') if isinstance(item.get('album'), dict) else None, 'extra': f"{artist_name} · {album_name}"}) return results elif service == 'lastfm': if not lastfm_worker or not lastfm_worker.client: raise ValueError("Last.fm worker not initialized") client = lastfm_worker.client if entity_type == 'artist': result = client.search_artist(query) if result: image = client.get_best_image(result.get('image', [])) return [{'id': result.get('url', ''), 'name': result.get('name', ''), 'image': image, 'extra': f"{result.get('listeners', '0')} listeners"}] elif entity_type == 'album': result = client.search_album(query, '') if result: image = client.get_best_image(result.get('image', [])) return [{'id': result.get('url', ''), 'name': result.get('name', ''), 'image': image, 'extra': result.get('artist', '')}] elif entity_type == 'track': # search_track takes separate artist/track params parts = query.split(' - ', 1) if ' - ' in query else ['', query] result = client.search_track(parts[0], parts[1]) if result: artist_name = result.get('artist', '') return [{'id': result.get('url', ''), 'name': result.get('name', ''), 'image': None, 'extra': f"{artist_name} · {result.get('listeners', '0')} listeners"}] return [] elif service == 'genius': if not genius_worker or not genius_worker.client: raise ValueError("Genius worker not initialized") client = genius_worker.client if entity_type == 'artist': artists = client.search_artists(query, limit=8) return [{'id': str(a.get('id', '')), 'name': a.get('name', ''), 'image': a.get('image_url'), 'extra': a.get('url', '')} for a in artists] elif entity_type == 'track': # Search with broader results for manual matching hits = client.search(f"{query}", per_page=10) results = [] seen_ids = set() for hit in hits: r = hit.get('result', {}) rid = r.get('id') if rid and rid not in seen_ids: seen_ids.add(rid) results.append({'id': str(rid), 'name': r.get('title', ''), 'image': r.get('song_art_image_url'), 'extra': r.get('artist_names', '')}) return results return [] elif service == 'tidal': if not tidal_enrichment_worker or not tidal_enrichment_worker.client: raise ValueError("Tidal worker not initialized") client = tidal_enrichment_worker.client if entity_type == 'artist': result = client.search_artist(query) if result: thumb = result.get('picture', '') if isinstance(thumb, list) and thumb: thumb = thumb[0].get('url', '') if isinstance(thumb[0], dict) else str(thumb[0]) return [{'id': str(result.get('id', '')), 'name': result.get('name', ''), 'image': thumb if isinstance(thumb, str) else None, 'extra': ''}] elif entity_type == 'album': result = client.search_album('', query) if result: return [{'id': str(result.get('id', '')), 'name': result.get('title', ''), 'image': None, 'extra': result.get('artist', {}).get('name', '') if isinstance(result.get('artist'), dict) else ''}] elif entity_type == 'track': result = client.search_track('', query) if result: artist_name = result.get('artist', {}).get('name', '') if isinstance(result.get('artist'), dict) else '' return [{'id': str(result.get('id', '')), 'name': result.get('title', ''), 'image': None, 'extra': artist_name}] return [] elif service == 'qobuz': if not qobuz_enrichment_worker or not qobuz_enrichment_worker.client: raise ValueError("Qobuz worker not initialized") client = qobuz_enrichment_worker.client if entity_type == 'artist': result = client.search_artist(query) if result: image = result.get('image', {}) thumb = image.get('large', image.get('medium', '')) if isinstance(image, dict) else '' return [{'id': str(result.get('id', '')), 'name': result.get('name', ''), 'image': thumb, 'extra': ''}] elif entity_type == 'album': result = client.search_album('', query) if result: artist_name = result.get('artist', {}).get('name', '') if isinstance(result.get('artist'), dict) else '' image = result.get('image', {}) thumb = image.get('large', image.get('medium', '')) if isinstance(image, dict) else '' return [{'id': str(result.get('id', '')), 'name': result.get('title', ''), 'image': thumb, 'extra': artist_name}] elif entity_type == 'track': result = client.search_track('', query) if result: artist_name = result.get('performer', {}).get('name', '') if isinstance(result.get('performer'), dict) else '' if not artist_name: artist_name = result.get('artist', {}).get('name', '') if isinstance(result.get('artist'), dict) else '' return [{'id': str(result.get('id', '')), 'name': result.get('title', ''), 'image': None, 'extra': artist_name}] return [] elif service == 'discogs': if not discogs_worker or not discogs_worker.client: raise ValueError("Discogs worker not initialized") client = discogs_worker.client if entity_type == 'artist': items = client.search_artists(query, limit=8) return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, 'extra': ', '.join(a.genres[:3]) if a.genres else ''} for a in items] elif entity_type == 'album': items = client.search_albums(query, limit=8) return [{'id': str(a.id), 'name': a.name, 'image': a.image_url, 'extra': f"{', '.join(a.artists)} · {a.release_date or ''}"} for a in items] elif entity_type == 'track': items = client.search_tracks(query, limit=8) return [{'id': str(t.id), 'name': t.name, 'image': t.image_url, 'extra': f"{', '.join(t.artists)} · {t.album or ''}"} for t in items] return [] elif service == 'audiodb': if not audiodb_worker or not audiodb_worker.client: raise ValueError("AudioDB worker not initialized") client = audiodb_worker.client result = None if entity_type == 'artist': result = client.search_artist(query) elif entity_type == 'album': # AudioDB album search needs artist + album, try query as-is parts = query.split(' - ', 1) if ' - ' in query else [query, ''] result = client.search_album(parts[0], parts[1] if len(parts) > 1 else query) elif entity_type == 'track': parts = query.split(' - ', 1) if ' - ' in query else [query, ''] result = client.search_track(parts[0], parts[1] if len(parts) > 1 else query) if result: if entity_type == 'artist': return [{'id': str(result.get('idArtist', '')), 'name': result.get('strArtist', ''), 'image': result.get('strArtistThumb'), 'extra': result.get('strGenre', '')}] elif entity_type == 'album': return [{'id': str(result.get('idAlbum', '')), 'name': result.get('strAlbum', ''), 'image': result.get('strAlbumThumb'), 'extra': f"{result.get('strArtist', '')} · {result.get('intYearReleased', '')}"}] elif entity_type == 'track': return [{'id': str(result.get('idTrack', '')), 'name': result.get('strTrack', ''), 'image': None, 'extra': f"{result.get('strArtist', '')} · {result.get('strAlbum', '')}"}] return [] return [] # Column name mappings for manual matching _SERVICE_ID_COLUMNS = { 'spotify': {'artist': 'spotify_artist_id', 'album': 'spotify_album_id', 'track': 'spotify_track_id'}, 'musicbrainz': {'artist': 'musicbrainz_id', 'album': 'musicbrainz_release_id', 'track': 'musicbrainz_recording_id'}, 'deezer': {'artist': 'deezer_id', 'album': 'deezer_id', 'track': 'deezer_id'}, 'audiodb': {'artist': 'audiodb_id', 'album': 'audiodb_id', 'track': 'audiodb_id'}, 'discogs': {'artist': 'discogs_id', 'album': 'discogs_id'}, 'itunes': {'artist': 'itunes_artist_id', 'album': 'itunes_album_id', 'track': 'itunes_track_id'}, 'lastfm': {'artist': 'lastfm_url', 'album': 'lastfm_url', 'track': 'lastfm_url'}, 'genius': {'artist': 'genius_id', 'track': 'genius_id'}, 'tidal': {'artist': 'tidal_id', 'album': 'tidal_id', 'track': 'tidal_id'}, 'qobuz': {'artist': 'qobuz_id', 'album': 'qobuz_id', 'track': 'qobuz_id'}, } @app.route('/api/library/manual-match', methods=['PUT']) def library_manual_match(): """Manually set a service ID for an entity. Body: { entity_type: str, entity_id: str, service: str, service_id: str, artist_id: str } """ try: data = request.get_json() entity_type = data.get('entity_type') entity_id = data.get('entity_id') service = data.get('service') service_id = data.get('service_id') if not all([entity_type, entity_id, service, service_id]): return jsonify({"success": False, "error": "entity_type, entity_id, service, and service_id are required"}), 400 id_col = _SERVICE_ID_COLUMNS.get(service, {}).get(entity_type) if not id_col: return jsonify({"success": False, "error": "Invalid service/entity_type combination"}), 400 status_col = f"{service}_match_status" attempted_col = f"{service}_last_attempted" table = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}[entity_type] database = get_database() with database._get_connection() as conn: cursor = conn.cursor() cursor.execute(f""" UPDATE {table} SET {id_col} = ?, {status_col} = 'matched', {attempted_col} = CURRENT_TIMESTAMP WHERE id = ? """, (service_id, entity_id)) conn.commit() if cursor.rowcount == 0: return jsonify({"success": False, "error": "Entity not found"}), 404 # Re-fetch fresh data artist_id = data.get('artist_id', entity_id) if entity_type != 'artist': artist_id = _get_artist_id_for_entity(database, entity_type, entity_id) updated = database.get_artist_full_detail(artist_id) if updated.get('success'): if updated.get('artist', {}).get('thumb_url'): updated['artist']['thumb_url'] = fix_artist_image_url(updated['artist']['thumb_url']) for album in updated.get('albums', []): if album.get('thumb_url'): album['thumb_url'] = fix_artist_image_url(album['thumb_url']) return jsonify({ "success": True, "message": f"Manually matched {entity_type} to {service} ID: {service_id}", "updated_data": updated if updated.get('success') else None }) except Exception as e: logger.error(f"Error manual matching: {e}") @app.route('/api/library/clear-match', methods=['PUT']) def library_clear_match(): """Clear a service ID match for an entity, reverting it to not_found. Body: { entity_type: str, entity_id: str, service: str } """ try: data = request.get_json() entity_type = data.get('entity_type') entity_id = data.get('entity_id') service = data.get('service') if not all([entity_type, entity_id, service]): return jsonify({"success": False, "error": "entity_type, entity_id, and service are required"}), 400 id_col = _SERVICE_ID_COLUMNS.get(service, {}).get(entity_type) if not id_col: return jsonify({"success": False, "error": "Invalid service/entity_type combination"}), 400 status_col = f"{service}_match_status" attempted_col = f"{service}_last_attempted" table = {'artist': 'artists', 'album': 'albums', 'track': 'tracks'}.get(entity_type) if not table: return jsonify({"success": False, "error": "Invalid entity_type"}), 400 database = get_database() with database._get_connection() as conn: cursor = conn.cursor() cursor.execute(f""" UPDATE {table} SET {id_col} = NULL, {status_col} = 'not_found', {attempted_col} = NULL WHERE id = ? """, (entity_id,)) conn.commit() if cursor.rowcount == 0: return jsonify({"success": False, "error": "Entity not found"}), 404 # Re-fetch fresh data artist_id = data.get('artist_id', entity_id) if entity_type != 'artist': artist_id = _get_artist_id_for_entity(database, entity_type, entity_id) updated = database.get_artist_full_detail(artist_id) if updated.get('success'): if updated.get('artist', {}).get('thumb_url'): updated['artist']['thumb_url'] = fix_artist_image_url(updated['artist']['thumb_url']) for album in updated.get('albums', []): if album.get('thumb_url'): album['thumb_url'] = fix_artist_image_url(album['thumb_url']) return jsonify({ "success": True, "message": f"Cleared {service} match for {entity_type}", "updated_data": updated if updated.get('success') else None }) except Exception as e: logger.error(f"Error clearing match: {e}") return jsonify({"success": False, "error": str(e)}), 500 import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/track/', methods=['DELETE']) def library_delete_track(track_id): """Delete a track from the database, optionally deleting the file and blacklisting the source.""" try: delete_file = request.args.get('delete_file', 'false').lower() == 'true' add_blacklist = request.args.get('blacklist', 'false').lower() == 'true' database = get_database() file_deleted = False blacklisted = False with database._get_connection() as conn: cursor = conn.cursor() # Get track info before deleting (for file removal + blacklist) track_info = None if delete_file or add_blacklist: cursor.execute(""" SELECT t.file_path, t.title, ar.name AS artist_name FROM tracks t JOIN artists ar ON t.artist_id = ar.id WHERE t.id = ? """, (track_id,)) track_info = cursor.fetchone() # Delete file from disk if requested file_error = None if delete_file and track_info and track_info['file_path']: resolved = _resolve_library_file_path(track_info['file_path']) if resolved and os.path.exists(resolved): try: os.remove(resolved) file_deleted = True logger.info(f"Deleted file from disk: {resolved}") # Clean up sidecar files (.lrc, .txt lyrics, cover.jpg) base_no_ext = os.path.splitext(resolved)[0] for sidecar_ext in ('.lrc', '.txt'): sidecar = base_no_ext + sidecar_ext if os.path.exists(sidecar): try: os.remove(sidecar) logger.info(f"Deleted sidecar file: {sidecar}") except Exception: pass except Exception as e: logger.warning(f"Failed to delete file: {e}") file_error = str(e) else: logger.warning(f"Could not resolve file path for deletion: {track_info['file_path']} (resolved={resolved})") file_error = _get_file_not_found_error(track_info['file_path']) # Add to blacklist if requested if add_blacklist and track_info and track_info['file_path']: # Extract username and filename from the file path for blacklisting # Soulseek paths are stored as: username/path/to/file.ext or just the local path fp = track_info['file_path'].replace('\\', '/') database.add_to_blacklist( track_title=track_info['title'], track_artist=track_info['artist_name'], blocked_filename=os.path.basename(fp), blocked_username='', # Local files don't have a username reason='user_rejected' ) blacklisted = True # Delete DB record cursor.execute("DELETE FROM tracks WHERE id = ?", (track_id,)) conn.commit() if cursor.rowcount == 0: return jsonify({"success": False, "error": "Track not found"}), 404 result = {"success": True, "deleted_count": cursor.rowcount, "file_deleted": file_deleted, "blacklisted": blacklisted} if delete_file and not file_deleted and file_error: result["file_error"] = file_error return jsonify(result) except Exception as e: logger.error(f"Error deleting track {track_id}: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # ================================================================================== # DOWNLOAD BLACKLIST API # ================================================================================== @app.route('/api/library/blacklist', methods=['GET']) def get_blacklist(): """Get all blacklisted download sources.""" try: database = get_database() entries = database.get_blacklist(limit=200) return jsonify({"success": True, "entries": entries}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/blacklist', methods=['POST']) def add_to_blacklist(): """Add a download source to the blacklist.""" try: data = request.get_json() database = get_database() result = database.add_to_blacklist( track_title=data.get('track_title', ''), track_artist=data.get('track_artist', ''), blocked_filename=data.get('blocked_filename', ''), blocked_username=data.get('blocked_username', ''), reason=data.get('reason', 'user_rejected'), ) return jsonify({"success": result}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/blacklist/', methods=['DELETE']) def remove_from_blacklist(blacklist_id): """Remove an entry from the blacklist.""" try: database = get_database() result = database.remove_from_blacklist(blacklist_id) return jsonify({"success": result}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # ================================================================================== # TRACK SOURCE INFO & PROVENANCE # ================================================================================== @app.route('/api/library/track//source-info', methods=['GET']) def get_track_source_info(track_id): """Get download provenance info for a library track.""" try: database = get_database() downloads = database.get_track_downloads(str(track_id)) if not downloads: # Try matching by file path as fallback (exact, then filename suffix) conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT file_path FROM tracks WHERE id = ?", (track_id,)) row = cursor.fetchone() conn.close() if row and row['file_path']: dl = database.get_download_by_file_path(row['file_path']) if dl: downloads = [dl] else: # Path format mismatch (e.g. Plex path vs local Windows path) — # fall back to filename-only match and back-link the track_id import os as _os fname = _os.path.basename(row['file_path'].replace('\\', '/')) if fname: dl = database.get_download_by_filename(fname, link_track_id=track_id) if dl: downloads = [dl] return jsonify({"success": True, "downloads": downloads}) except Exception as e: logger.error(f"Error getting track source info: {e}") return jsonify({"success": False, "error": str(e)}), 500 # ================================================================================== # TRACK REDOWNLOAD — Search metadata, search download sources, start redownload # ================================================================================== @app.route('/api/library/track//redownload/search-metadata', methods=['POST']) def redownload_search_metadata(track_id): """Search all available metadata sources for a track to find the correct version.""" try: database = get_database() conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT t.id, t.title, t.file_path, t.bitrate, t.duration, ar.name AS artist_name, al.title AS album_title, al.thumb_url AS album_thumb_url, t.spotify_track_id, t.deezer_id FROM tracks t JOIN artists ar ON t.artist_id = ar.id JOIN albums al ON t.album_id = al.id WHERE t.id = ? """, (track_id,)) row = cursor.fetchone() conn.close() if not row: return jsonify({"success": False, "error": "Track not found"}), 404 track_title = row['title'] artist_name = row['artist_name'] # Clean the title for better search results — strip version/edition suffixes import re as _re clean_title = _re.sub(r'\s*[\(\[](single version|album version|remaster|deluxe|bonus|explicit|clean|radio edit)[\)\]]', '', track_title, flags=_re.IGNORECASE).strip() query = f"{artist_name} {clean_title}" # Resolve file info file_path = row['file_path'] or '' ext = os.path.splitext(file_path)[1].lstrip('.').upper() if file_path else '' fmt = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else '' # Resolve album thumb URL (may be relative Plex path) thumb_url = row['album_thumb_url'] or '' if thumb_url and not thumb_url.startswith('http'): _ab = '' _at = '' if plex_client and plex_client.server: _ab = getattr(plex_client.server, '_baseurl', '') or '' _at = getattr(plex_client.server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') _at = _at or _pc.get('token', '') if _ab and thumb_url.startswith('/'): thumb_url = f"{_ab}{thumb_url}?X-Plex-Token={_at}" if _at else f"{_ab}{thumb_url}" current_track = { 'id': row['id'], 'title': track_title, 'artist': artist_name, 'album': row['album_title'], 'duration_ms': row['duration'] or 0, 'file_path': file_path, 'format': fmt, 'bitrate': row['bitrate'] or 0, 'spotify_track_id': row['spotify_track_id'], 'deezer_id': row['deezer_id'], 'thumb_url': thumb_url, } # Search all available metadata sources in parallel from concurrent.futures import ThreadPoolExecutor, as_completed from difflib import SequenceMatcher def _score_metadata_match(result): """Score a metadata result against the current track.""" title_sim = SequenceMatcher(None, track_title.lower(), (result.get('name') or '').lower()).ratio() artist_sim = SequenceMatcher(None, artist_name.lower(), (result.get('artist') or '').lower()).ratio() dur_diff = abs((row['duration'] or 0) - (result.get('duration_ms') or 0)) dur_score = max(0, 1 - dur_diff / 30000) if row['duration'] else 0.5 return round((title_sim * 0.5 + artist_sim * 0.35 + dur_score * 0.15), 3) metadata_results = {} sources_to_search = [] if spotify_client and spotify_client.is_authenticated(): sources_to_search.append(('spotify', spotify_client)) try: sources_to_search.append(('itunes', _get_itunes_client())) except Exception as e: logger.debug(f"iTunes client not available for redownload search: {e}") try: sources_to_search.append(('deezer', _get_deezer_client())) except Exception as e: logger.debug(f"Deezer client not available for redownload search: {e}") # Build source-optimized queries deezer_query = f'artist:"{artist_name}" track:"{clean_title}"' def _search_source(source_name, client): try: # Deezer works best with structured artist:/track: queries search_q = deezer_query if source_name == 'deezer' else query logger.info(f"[Redownload] Searching {source_name} for: {search_q}") track_objs = client.search_tracks(search_q, limit=10) # If no results, try plain query as fallback if not track_objs and search_q != query: track_objs = client.search_tracks(query, limit=10) # Last resort: title only if not track_objs and clean_title != query: track_objs = client.search_tracks(clean_title, limit=10) logger.info(f"[Redownload] {source_name} returned {len(track_objs)} results") results = [] for t in track_objs: r = { 'id': str(t.id), 'name': t.name, 'artist': ', '.join(t.artists) if t.artists else '', 'album': t.album or '', 'duration_ms': t.duration_ms or 0, 'image_url': t.image_url or '', 'is_current_match': False, } # Flag current match if source_name == 'spotify' and row['spotify_track_id'] and str(t.id) == str(row['spotify_track_id']): r['is_current_match'] = True elif source_name == 'deezer' and row['deezer_id'] and str(t.id) == str(row['deezer_id']): r['is_current_match'] = True r['match_score'] = _score_metadata_match(r) results.append(r) results.sort(key=lambda x: (-int(x['is_current_match']), -x['match_score'])) return source_name, results except Exception as e: logger.error(f"[Redownload] Metadata search failed for {source_name}: {e}", exc_info=True) return source_name, [] with ThreadPoolExecutor(max_workers=3) as pool: futures = {pool.submit(_search_source, name, client): name for name, client in sources_to_search} for future in as_completed(futures): source_name, results = future.result() metadata_results[source_name] = results # Find best overall match best_match = None for source, results in metadata_results.items(): if results: top = results[0] if not best_match or top['match_score'] > best_match['score']: best_match = {'source': source, 'index': 0, 'score': top['match_score']} return jsonify({ "success": True, "current_track": current_track, "metadata_results": metadata_results, "best_match": best_match, }) except Exception as e: logger.error(f"Error in redownload metadata search: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/track//redownload/search-sources', methods=['POST']) def redownload_search_sources(track_id): """Search all active download sources for a track using the selected metadata.""" try: data = request.get_json() metadata = data.get('metadata', {}) if not metadata.get('name'): return jsonify({"success": False, "error": "metadata with name required"}), 400 # Build a track-like object for query generation from core.itunes_client import Track as MetaTrack track_obj = MetaTrack( id=metadata.get('id', ''), name=metadata['name'], artists=[metadata.get('artist', '')], album=metadata.get('album', ''), duration_ms=metadata.get('duration_ms', 0), popularity=0, ) # Generate search queries search_queries = matching_engine.generate_download_queries(track_obj) if not search_queries: search_queries = [f"{metadata.get('artist', '')} {metadata['name']}".strip()] # Use first 2 queries for speed search_queries = search_queries[:2] # Search ALL configured download sources individually (not through hybrid which stops at first hit) candidates = [] database = get_database() # Get all available download source clients download_clients = {} try: orch = soulseek_client # The download orchestrator if hasattr(orch, 'soulseek') and orch.soulseek: if not (hasattr(orch.soulseek, 'is_configured') and not orch.soulseek.is_configured()): download_clients['soulseek'] = orch.soulseek if hasattr(orch, 'youtube') and orch.youtube: if not (hasattr(orch.youtube, 'is_configured') and not orch.youtube.is_configured()): download_clients['youtube'] = orch.youtube if hasattr(orch, 'tidal') and orch.tidal: if not (hasattr(orch.tidal, 'is_configured') and not orch.tidal.is_configured()): download_clients['tidal'] = orch.tidal if hasattr(orch, 'qobuz') and orch.qobuz: if not (hasattr(orch.qobuz, 'is_configured') and not orch.qobuz.is_configured()): download_clients['qobuz'] = orch.qobuz if hasattr(orch, 'hifi') and orch.hifi: if not (hasattr(orch.hifi, 'is_configured') and not orch.hifi.is_configured()): download_clients['hifi'] = orch.hifi if hasattr(orch, 'deezer_dl') and orch.deezer_dl: if not (hasattr(orch.deezer_dl, 'is_configured') and not orch.deezer_dl.is_configured()): download_clients['deezer_dl'] = orch.deezer_dl except Exception as e: logger.warning(f"[Redownload] Error getting download clients: {e}") if not download_clients: # Fallback: use orchestrator directly download_clients = {'default': soulseek_client} logger.info(f"[Redownload] Streaming search across {len(download_clients)} sources: {list(download_clients.keys())}") def _search_one_source(source_name, client): """Search a single download source and return formatted candidates.""" source_candidates = [] for _qi, q in enumerate(search_queries): try: tracks_result, _ = run_async(client.search(q, timeout=20)) if not tracks_result: continue valid = get_valid_candidates(tracks_result, track_obj, q) for candidate in valid: is_bl = database.is_blacklisted(candidate.username, candidate.filename) display_name = os.path.basename(candidate.filename.replace('\\', '/')) ext = os.path.splitext(display_name)[1].lstrip('.').upper() quality = ext if ext in ('FLAC', 'MP3', 'OPUS', 'OGG', 'M4A', 'WAV') else candidate.quality or '' svc = source_name if source_name != 'default' else 'hybrid' uname = candidate.username if uname in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): svc = uname source_candidates.append({ 'username': uname, 'filename': candidate.filename, 'display_name': display_name, 'size': candidate.size or 0, 'size_display': f"{(candidate.size or 0) / 1048576:.1f} MB", 'bitrate': candidate.bitrate or 0, 'quality': quality, 'duration': candidate.duration or 0, 'confidence': round(getattr(candidate, 'confidence', 0), 3), 'source_service': svc, 'source_query': q, 'blacklisted': is_bl, 'free_upload_slots': getattr(candidate, 'free_upload_slots', 0), 'upload_speed': getattr(candidate, 'upload_speed', 0), 'queue_length': getattr(candidate, 'queue_length', 0), }) except Exception as e: logger.debug(f"[Redownload] {source_name} search failed for query '{q}': {e}") # Deduplicate within source seen = set() unique = [] for c in source_candidates: key = f"{c['username']}|{c['filename']}" if key not in seen: seen.add(key) unique.append(c) unique.sort(key=lambda c: (-int(not c['blacklisted']), -c['confidence'])) return unique # Stream NDJSON — one line per source as it completes from concurrent.futures import ThreadPoolExecutor, as_completed def generate_stream(): with ThreadPoolExecutor(max_workers=4) as pool: futures = {pool.submit(_search_one_source, name, client): name for name, client in download_clients.items()} for future in as_completed(futures): source_name = futures[future] try: results = future.result() yield json.dumps({'source': source_name, 'candidates': results}) + '\n' except Exception as e: yield json.dumps({'source': source_name, 'candidates': [], 'error': str(e)}) + '\n' yield json.dumps({'done': True}) + '\n' return app.response_class(generate_stream(), mimetype='application/x-ndjson', headers={'X-Accel-Buffering': 'no'}) except Exception as e: logger.error(f"Error in redownload source search: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/track//redownload/start', methods=['POST']) def redownload_start(track_id): """Start downloading a specific track from a selected source to replace the current file.""" try: data = request.get_json() metadata = data.get('metadata', {}) candidate = data.get('candidate', {}) delete_old = data.get('delete_old_file', True) if not candidate.get('username') or not candidate.get('filename'): return jsonify({"success": False, "error": "candidate with username and filename required"}), 400 # Get current track info for old file path database = get_database() conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT file_path FROM tracks WHERE id = ?", (track_id,)) row = cursor.fetchone() conn.close() old_file_path = None if row and row['file_path'] and delete_old: old_file_path = _resolve_library_file_path(row['file_path']) task_id = f"redownload_{track_id}_{int(time.time())}" batch_id = f"redownload_batch_{track_id}" # Fetch full track details from the metadata source for pipeline parity # This gives us track_number, disc_number, full album data meta_source = metadata.get('_source', '') meta_id = metadata.get('id', '') full_track_details = None full_album_data = None if meta_id: try: if meta_source == 'spotify' and spotify_client and spotify_client.is_authenticated(): full_track_details = spotify_client.get_track_details(meta_id) if full_track_details and full_track_details.get('album', {}).get('id'): full_album_data = spotify_client.get_album(full_track_details['album']['id']) elif meta_source == 'itunes': _it = _get_itunes_client() results = _it._lookup(id=meta_id, entity='song') if results: for r in results: if r.get('wrapperType') == 'track': full_track_details = r break elif meta_source == 'deezer': _dz = _get_deezer_client() full_track_details = _dz._api_get(f'track/{meta_id}') except Exception as e: logger.debug(f"[Redownload] Could not fetch full track details: {e}") # Build track data with full metadata for pipeline parity track_number = None disc_number = 1 album_data = {'name': metadata.get('album', '')} if full_track_details: if meta_source == 'spotify': track_number = full_track_details.get('track_number') disc_number = full_track_details.get('disc_number', 1) album_raw = full_track_details.get('album', {}) if album_raw: album_images = album_raw.get('images', []) album_data = { 'id': album_raw.get('id', ''), 'name': album_raw.get('name', metadata.get('album', '')), 'release_date': album_raw.get('release_date', ''), 'album_type': album_raw.get('album_type', 'album'), 'total_tracks': album_raw.get('total_tracks', 0), 'images': album_images, 'image_url': album_images[0]['url'] if album_images else '', } elif meta_source == 'itunes': track_number = full_track_details.get('trackNumber') disc_number = full_track_details.get('discNumber', 1) elif meta_source == 'deezer': track_number = full_track_details.get('track_position') disc_number = full_track_details.get('disk_number', 1) track_data = { 'id': meta_id, 'name': metadata.get('name', ''), 'artists': [{'name': metadata.get('artist', '')}], 'album': album_data, 'duration_ms': metadata.get('duration_ms', 0), 'track_number': track_number, 'disc_number': disc_number, '_is_explicit_album_download': bool(full_album_data or (album_data.get('id'))), } # Build explicit context if we have full album data if full_album_data or album_data.get('id'): track_data['_explicit_album_context'] = full_album_data if isinstance(full_album_data, dict) else album_data track_data['_explicit_artist_context'] = {'name': metadata.get('artist', ''), 'id': '', 'genres': []} # Create batch with tasks_lock: download_batches[batch_id] = { 'queue': [task_id], 'queue_index': 1, # Already past the first (only) item 'active_count': 1, # One worker is about to start 'max_concurrent': 1, 'playlist_id': f'redownload_{track_id}', 'playlist_name': f"Redownload: {metadata.get('artist', '')} - {metadata.get('name', '')}", 'phase': 'downloading', 'total_tracks': 1, 'completed_count': 0, 'failed_count': 0, 'cancelled_tracks': set(), 'permanently_failed_tracks': [], 'force_download': True, 'auto_initiated': False, } download_tasks[task_id] = { 'status': 'queued', 'track_info': track_data, 'playlist_id': f'redownload_{track_id}', 'batch_id': batch_id, 'track_index': 0, 'download_id': None, 'username': None, 'filename': None, 'retry_count': 0, 'cached_candidates': [], 'used_sources': set(), 'status_change_time': time.time(), 'metadata_enhanced': False, 'error_message': None, '_redownload_context': { 'library_track_id': track_id, 'old_file_path': old_file_path, 'delete_old_file': delete_old, }, } # Build a TrackResult-like candidate and submit to download def _run_redownload(): try: from core.soulseek_client import TrackResult from core.itunes_client import Track as MetaTrack tr = TrackResult( username=candidate['username'], filename=candidate['filename'], size=candidate.get('size', 0), bitrate=candidate.get('bitrate', 0), duration=candidate.get('duration', 0), quality=candidate.get('quality', ''), free_upload_slots=candidate.get('free_upload_slots', 0), upload_speed=candidate.get('upload_speed', 0), queue_length=candidate.get('queue_length', 0), ) tr.artist = metadata.get('artist', '') tr.title = metadata.get('name', '') tr.album = metadata.get('album', '') tr.confidence = candidate.get('confidence', 1.0) # Build a proper Track object (not a dict) — _attempt_download_with_candidates # accesses track.artists, track.album etc. as attributes artist_name = metadata.get('artist', '') track_obj = MetaTrack( id=metadata.get('id', ''), name=metadata.get('name', ''), artists=[artist_name] if artist_name else ['Unknown'], album=metadata.get('album', ''), duration_ms=metadata.get('duration_ms', 0), popularity=0, ) _attempt_download_with_candidates(task_id, [tr], track_obj, batch_id) except Exception as e: logger.error(f"Redownload failed: {e}", exc_info=True) with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'failed' download_tasks[task_id]['error_message'] = str(e) missing_download_executor.submit(_run_redownload) return jsonify({ "success": True, "task_id": task_id, "batch_id": batch_id, "message": "Redownload started", }) except Exception as e: logger.error(f"Error starting redownload: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/artist//sync', methods=['POST']) def sync_artist_library(artist_id): """Bidirectional sync: pull new content from media server AND remove stale entries.""" try: database = get_database() with database._get_connection() as conn: cursor = conn.cursor() # Resolve artist_id: could be a DB integer ID or a source artist ID (Spotify/iTunes/Deezer) db_artist_id = None # Try direct ID match first (works for both integer and text IDs) cursor.execute("SELECT id FROM artists WHERE id = ?", (artist_id,)) row = cursor.fetchone() if row: db_artist_id = row['id'] # Also try as integer (legacy integer PKs) if not db_artist_id: try: candidate = int(artist_id) cursor.execute("SELECT id FROM artists WHERE id = ?", (candidate,)) row = cursor.fetchone() if row: db_artist_id = row['id'] except (ValueError, TypeError): pass # Try source-specific ID columns if not db_artist_id: for col in ('spotify_artist_id', 'itunes_artist_id', 'deezer_id', 'discogs_id'): cursor.execute(f"SELECT id FROM artists WHERE {col} = ?", (artist_id,)) row = cursor.fetchone() if row: db_artist_id = row['id'] break if not db_artist_id: return jsonify({"success": False, "error": "Artist not found"}), 404 cursor.execute("SELECT name, server_source FROM artists WHERE id = ?", (db_artist_id,)) artist_row = cursor.fetchone() artist_name = artist_row['name'] if artist_row else f'ID {db_artist_id}' server_source = artist_row['server_source'] if artist_row else None # ── Phase 1: Pull new content from media server ── new_albums = 0 new_tracks = 0 name_updated = False if server_source: media_client = None if server_source == 'plex' and plex_client and plex_client.server: media_client = plex_client elif server_source == 'jellyfin' and jellyfin_client: media_client = jellyfin_client elif server_source == 'navidrome' and navidrome_client: media_client = navidrome_client if media_client: try: from core.database_update_worker import DatabaseUpdateWorker worker = DatabaseUpdateWorker( media_client=media_client, full_refresh=False, server_type=server_source, force_sequential=True, ) worker.database = database # Use existing DB instance instead of creating new one # Fetch the artist object from the server server_artist = None logger.info(f"[Artist Sync] Fetching artist {db_artist_id} from {server_source}...") if server_source == 'plex' and hasattr(media_client, 'server'): try: server_artist = media_client.server.fetchItem(int(db_artist_id)) logger.info(f"[Artist Sync] Plex returned: {getattr(server_artist, 'title', 'None')}") except Exception as e: logger.error(f"[Artist Sync] Plex fetchItem failed: {e}") elif hasattr(media_client, 'get_artist_by_id'): try: server_artist = media_client.get_artist_by_id(str(db_artist_id)) logger.info(f"[Artist Sync] Server returned: {getattr(server_artist, 'title', None) or server_artist}") except Exception as e: logger.error(f"[Artist Sync] get_artist_by_id failed: {e}") else: logger.warning(f"[Artist Sync] No get_artist_by_id method on {type(media_client).__name__}") if not server_artist: logger.error("[Artist Sync] Could not fetch artist from server — skipping pull phase") if server_artist: # Check for name change new_name = getattr(server_artist, 'title', None) if new_name and new_name != artist_name: database.execute_query( "UPDATE artists SET name = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?", (new_name, db_artist_id) ) logger.info(f"[Artist Sync] Name updated: '{artist_name}' → '{new_name}'") artist_name = new_name name_updated = True # Process artist content (deep scan mode — skip existing, preserve enrichment) success, details, new_albums, new_tracks = worker._process_artist_with_content( server_artist, skip_existing_tracks=True ) logger.info(f"[Artist Sync] Server pull for {artist_name}: {details}") except Exception as e: logger.error(f"[Artist Sync] Server pull failed for {artist_name}: {e}") # ── Phase 2: Remove stale entries (files no longer on disk) ── stale_removed = 0 empty_albums_removed = 0 with database._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT id, file_path FROM tracks WHERE artist_id = ?", (db_artist_id,)) tracks = cursor.fetchall() stale_ids = [] for track in tracks: fp = track['file_path'] if not fp: stale_ids.append(track['id']) continue resolved = _resolve_library_file_path(fp) if not resolved or not os.path.exists(resolved): stale_ids.append(track['id']) if stale_ids: placeholders = ','.join('?' for _ in stale_ids) cursor.execute(f"DELETE FROM tracks WHERE id IN ({placeholders})", stale_ids) stale_removed = len(stale_ids) cursor.execute(""" DELETE FROM albums WHERE artist_id = ? AND id NOT IN (SELECT DISTINCT album_id FROM tracks) """, (db_artist_id,)) empty_albums_removed = cursor.rowcount cursor.execute(""" UPDATE albums SET track_count = ( SELECT COUNT(*) FROM tracks WHERE tracks.album_id = albums.id ) WHERE artist_id = ? """, (db_artist_id,)) conn.commit() logger.warning(f"[Artist Sync] {artist_name}: +{new_albums} albums, +{new_tracks} tracks, " f"-{stale_removed} stale, -{empty_albums_removed} empty albums") return jsonify({ "success": True, "artist_name": artist_name, "name_updated": name_updated, "new_albums": new_albums, "new_tracks": new_tracks, "stale_removed": stale_removed, "empty_albums_removed": empty_albums_removed, }) except Exception as e: logger.error(f"Error syncing artist {artist_id}: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/album/', methods=['DELETE']) def library_delete_album(album_id): """Delete an album and all its tracks from the database, optionally deleting files on disk.""" try: delete_files = request.args.get('delete_files', 'false').lower() == 'true' database = get_database() files_deleted = 0 files_failed = 0 with database._get_connection() as conn: cursor = conn.cursor() # If deleting files, resolve and remove each track's file first if delete_files: cursor.execute("SELECT id, file_path FROM tracks WHERE album_id = ?", (album_id,)) track_rows = cursor.fetchall() for row in track_rows: fp = row['file_path'] if not fp: continue resolved = _resolve_library_file_path(fp) if resolved and os.path.exists(resolved): try: os.remove(resolved) files_deleted += 1 # Clean up sidecar files (.lrc, .txt lyrics) base_no_ext = os.path.splitext(resolved)[0] for sidecar_ext in ('.lrc', '.txt'): sidecar = base_no_ext + sidecar_ext if os.path.exists(sidecar): try: os.remove(sidecar) except Exception: pass except Exception as e: logger.warning(f"Failed to delete track file: {e}") files_failed += 1 else: files_failed += 1 # Try to remove the album folder if it's now empty if track_rows: first_fp = track_rows[0]['file_path'] if first_fp: resolved_first = _resolve_library_file_path(first_fp) if resolved_first: album_dir = os.path.dirname(resolved_first) try: if os.path.isdir(album_dir) and not os.listdir(album_dir): os.rmdir(album_dir) logger.info(f"Removed empty album directory: {album_dir}") except Exception: pass # Delete all tracks belonging to this album cursor.execute("DELETE FROM tracks WHERE album_id = ?", (album_id,)) tracks_deleted = cursor.rowcount # Delete the album itself cursor.execute("DELETE FROM albums WHERE id = ?", (album_id,)) if cursor.rowcount == 0: conn.rollback() return jsonify({"success": False, "error": "Album not found"}), 404 conn.commit() return jsonify({ "success": True, "deleted_count": 1, "tracks_deleted": tracks_deleted, "files_deleted": files_deleted, "files_failed": files_failed }) except Exception as e: logger.error(f"Error deleting album {album_id}: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/tracks/delete-batch', methods=['POST']) def library_delete_tracks_batch(): """Delete multiple track records from the database (does NOT delete files on disk). Body: { track_ids: [int] } """ try: data = request.get_json() track_ids = data.get('track_ids', []) if not track_ids or not isinstance(track_ids, list): return jsonify({"success": False, "error": "track_ids array is required"}), 400 database = get_database() with database._get_connection() as conn: cursor = conn.cursor() placeholders = ','.join('?' for _ in track_ids) cursor.execute(f"DELETE FROM tracks WHERE id IN ({placeholders})", track_ids) conn.commit() return jsonify({"success": True, "deleted_count": cursor.rowcount}) except Exception as e: logger.error(f"Error batch deleting tracks: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/radio') def library_radio(): """Get a smart queue of similar tracks for radio mode auto-play.""" try: track_id = request.args.get('track_id') if not track_id: return jsonify({"success": False, "error": "track_id is required"}), 400 limit = request.args.get('limit', 20, type=int) exclude_raw = request.args.get('exclude', '') exclude_ids = [eid.strip() for eid in exclude_raw.split(',') if eid.strip()] if exclude_raw else None database = get_database() result = database.get_radio_tracks(track_id, limit=limit, exclude_ids=exclude_ids) if not result.get('success'): return jsonify(result), 404 # Fix image URLs (DB stores server-relative paths that need base URL + auth) for track in result.get('tracks', []): if track.get('image_url'): track['image_url'] = fix_artist_image_url(track['image_url']) return jsonify(result) except Exception as e: logger.error(f"Error getting radio tracks: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # ==================== End Enhanced Library Management ==================== @app.route('/api/stream/start', methods=['POST']) def stream_start(): """Start streaming a track in the background""" global stream_background_task data = request.get_json() if not data: return jsonify({"success": False, "error": "No track data provided"}), 400 logger.info(f"Web UI Stream request for: {data.get('filename')}") try: # Stop any existing streaming task if stream_background_task and not stream_background_task.done(): stream_background_task.cancel() # Reset stream state with stream_lock: stream_state.update({ "status": "stopped", "progress": 0, "track_info": None, "file_path": None, "error_message": None, "is_library": False }) # Start new background streaming task stream_background_task = stream_executor.submit(_prepare_stream_task, data) return jsonify({"success": True, "message": "Streaming started"}) except Exception as e: logger.error(f"Error starting stream: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/stream/status') def stream_status(): """Get current streaming status and progress""" try: with stream_lock: # Return copy of current stream state return jsonify({ "status": stream_state["status"], "progress": stream_state["progress"], "track_info": stream_state["track_info"], "error_message": stream_state["error_message"] }) except Exception as e: logger.error(f"Error getting stream status: {e}") return jsonify({ "status": "error", "progress": 0, "track_info": None, "error_message": str(e) }), 500 @app.route('/stream/audio') def stream_audio(): """Serve the audio file from the Stream folder with range request support""" try: with stream_lock: if stream_state["status"] != "ready" or not stream_state["file_path"]: return jsonify({"error": "No audio file ready for streaming"}), 404 file_path = stream_state["file_path"] if not os.path.exists(file_path): return jsonify({"error": "Audio file not found"}), 404 logger.info(f"Serving audio file: {os.path.basename(file_path)}") # Determine MIME type based on file extension file_ext = os.path.splitext(file_path)[1].lower() mime_types = { '.mp3': 'audio/mpeg', '.flac': 'audio/flac', '.ogg': 'audio/ogg', '.aac': 'audio/aac', '.m4a': 'audio/mp4', '.wav': 'audio/wav', '.opus': 'audio/ogg', '.webm': 'audio/webm', '.wma': 'audio/x-ms-wma' } mimetype = mime_types.get(file_ext, 'audio/mpeg') # Get file size file_size = os.path.getsize(file_path) # Handle range requests (important for HTML5 audio seeking) range_header = request.headers.get('Range', None) if range_header: byte_start = 0 byte_end = file_size - 1 # Parse range header (format: "bytes=start-end") try: range_match = re.match(r'bytes=(\d*)-(\d*)', range_header) if range_match: start_str, end_str = range_match.groups() if start_str: byte_start = int(start_str) if end_str: byte_end = int(end_str) else: # If no end specified, serve from start to end of file byte_end = file_size - 1 except (ValueError, AttributeError): # Invalid range header, serve full file pass # Ensure valid range byte_start = max(0, byte_start) byte_end = min(file_size - 1, byte_end) content_length = byte_end - byte_start + 1 # Create response with partial content def generate(): with open(file_path, 'rb') as f: f.seek(byte_start) remaining = content_length while remaining: chunk_size = min(8192, remaining) # 8KB chunks chunk = f.read(chunk_size) if not chunk: break remaining -= len(chunk) yield chunk response = Response(generate(), status=206, # Partial Content mimetype=mimetype, direct_passthrough=True) # Set range headers response.headers.add('Content-Range', f'bytes {byte_start}-{byte_end}/{file_size}') response.headers.add('Accept-Ranges', 'bytes') response.headers.add('Content-Length', str(content_length)) response.headers.add('Cache-Control', 'no-cache') return response else: # No range request, serve entire file response = send_file(file_path, as_attachment=False, mimetype=mimetype) response.headers.add('Accept-Ranges', 'bytes') response.headers.add('Content-Length', str(file_size)) # Override the default static-cache max-age — streaming media # bypasses caching (range requests, mid-track seeks). response.headers['Cache-Control'] = 'no-cache' return response except Exception as e: logger.error(f"Error serving audio file: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/stream/stop', methods=['POST']) def stream_stop(): """Stop streaming and clean up""" global stream_background_task try: # Cancel background task if stream_background_task and not stream_background_task.done(): stream_background_task.cancel() # Only clear Stream folder if NOT playing a library file with stream_lock: is_library = stream_state.get("is_library", False) if not is_library: project_root = os.path.dirname(os.path.abspath(__file__)) stream_folder = os.path.join(project_root, 'Stream') if os.path.exists(stream_folder): for filename in os.listdir(stream_folder): file_path = os.path.join(stream_folder, filename) if os.path.isfile(file_path): os.remove(file_path) logger.info(f"Removed stream file: {filename}") else: logger.info("Library playback stopped - skipping file deletion") # Reset stream state with stream_lock: stream_state.update({ "status": "stopped", "progress": 0, "track_info": None, "file_path": None, "error_message": None, "is_library": False }) return jsonify({"success": True, "message": "Stream stopped"}) except Exception as e: logger.error(f"Error stopping stream: {e}") return jsonify({"success": False, "error": str(e)}), 500 # --- Matched Downloads API Endpoints --- def _generate_artist_suggestions(search_result, is_album=False, album_result=None): """ Port of ArtistSuggestionThread.generate_artist_suggestions() from GUI Generate artist suggestions using multiple strategies """ if not spotify_client or not matching_engine: return [] try: logger.info(f"Generating artist suggestions for: {search_result.get('artist', '')} - {search_result.get('title', '')}") suggestions = [] # Special handling for albums - use album title to find artist if is_album and album_result and album_result.get('album_title'): logger.info("Album mode detected - using album title for artist search") album_title = album_result.get('album_title', '') # Clean album title (remove year prefixes like "(2005)") import re clean_album_title = re.sub(r'^\(\d{4}\)\s*', '', album_title).strip() logger.info(f" clean_album_title: '{clean_album_title}'") # Search tracks using album title to find the artist tracks = spotify_client.search_tracks(clean_album_title, limit=10) logger.info(f"Found {len(tracks)} tracks from album search") # Collect unique artists and their associated tracks/albums unique_artists = {} # artist_name -> list of (track, album) tuples for track in tracks: for artist_name in track.artists: if artist_name not in unique_artists: unique_artists[artist_name] = [] unique_artists[artist_name].append((track, track.album)) # Batch fetch artist objects for speed from concurrent.futures import ThreadPoolExecutor, as_completed artist_objects = {} # artist_name -> Artist object def fetch_artist(artist_name): try: matches = spotify_client.search_artists(artist_name, limit=1) if matches: return artist_name, matches[0] except Exception as e: logger.error(f"Error fetching artist '{artist_name}': {e}") return artist_name, None # Use limited concurrency to respect rate limits with ThreadPoolExecutor(max_workers=3) as executor: future_to_artist = {executor.submit(fetch_artist, name): name for name in unique_artists.keys()} for future in as_completed(future_to_artist): artist_name, artist_obj = future.result() if artist_obj: artist_objects[artist_name] = artist_obj # Calculate confidence scores for each artist artist_scores = {} for artist_name, track_album_pairs in unique_artists.items(): if artist_name not in artist_objects: continue artist = artist_objects[artist_name] best_confidence = 0 # Find the best confidence score across all albums for this artist for _track, album in track_album_pairs: confidence = matching_engine.similarity_score( matching_engine.normalize_string(clean_album_title), matching_engine.normalize_string(album) ) if confidence > best_confidence: best_confidence = confidence artist_scores[artist_name] = (artist, best_confidence) # Create suggestions from top matches for _artist_name, (artist, confidence) in sorted(artist_scores.items(), key=lambda x: x[1][1], reverse=True)[:8]: suggestions.append({ "artist": { "id": artist.id, "name": artist.name, "image_url": getattr(artist, 'image_url', None), "genres": getattr(artist, 'genres', []), "popularity": getattr(artist, 'popularity', 0) }, "confidence": confidence }) else: # Single track mode - search by artist name search_artist = search_result.get('artist', '') if not search_artist: return [] logger.info(f"Single track mode - searching for artist: '{search_artist}'") # Search for artists directly artist_matches = spotify_client.search_artists(search_artist, limit=10) for artist in artist_matches: # Calculate confidence based on artist name similarity confidence = matching_engine.similarity_score( matching_engine.normalize_string(search_artist), matching_engine.normalize_string(artist.name) ) suggestions.append({ "artist": { "id": artist.id, "name": artist.name, "image_url": getattr(artist, 'image_url', None), "genres": getattr(artist, 'genres', []), "popularity": getattr(artist, 'popularity', 0) }, "confidence": confidence }) # Sort by confidence and return top results suggestions.sort(key=lambda x: x['confidence'], reverse=True) return suggestions[:4] except Exception as e: logger.error(f"Error generating artist suggestions: {e}") return [] def _generate_album_suggestions(selected_artist, search_result): """ Port of AlbumSuggestionThread logic from GUI Generate album suggestions for a selected artist """ if not spotify_client or not matching_engine: return [] try: logger.info(f"Generating album suggestions for artist: {selected_artist['name']}") # Determine target album name from search result target_album_name = search_result.get('album', '') or search_result.get('album_title', '') if not target_album_name: logger.warning("No album name found in search result") return [] # Clean target album name import re clean_target = re.sub(r'^\(\d{4}\)\s*', '', target_album_name).strip() logger.info(f" target_album: '{clean_target}'") # Get artist's albums from Spotify artist_albums = spotify_client.get_artist_albums(selected_artist['id']) logger.info(f"Found {len(artist_albums)} albums for artist") album_matches = [] for album in artist_albums: # Calculate confidence based on album name similarity confidence = matching_engine.similarity_score( matching_engine.normalize_string(clean_target), matching_engine.normalize_string(album.name) ) album_matches.append({ "album": { "id": album.id, "name": album.name, "release_date": getattr(album, 'release_date', ''), "album_type": getattr(album, 'album_type', 'album'), "image_url": getattr(album, 'image_url', None), "total_tracks": getattr(album, 'total_tracks', 0) }, "confidence": confidence }) # Sort by confidence and return top results album_matches.sort(key=lambda x: x['confidence'], reverse=True) return album_matches[:4] except Exception as e: logger.error(f"Error generating album suggestions: {e}") return [] @app.route('/api/match/suggestions', methods=['POST']) def get_match_suggestions(): """Get AI-powered suggestions for artist or album matching""" try: data = request.get_json() search_result = data.get('search_result', {}) context = data.get('context', 'artist') # 'artist' or 'album' if context == 'artist': is_album = data.get('is_album', False) album_result = data.get('album_result', None) if is_album else None suggestions = _generate_artist_suggestions(search_result, is_album, album_result) elif context == 'album': selected_artist = data.get('selected_artist', {}) suggestions = _generate_album_suggestions(selected_artist, search_result) else: return jsonify({"error": "Invalid context. Must be 'artist' or 'album'"}), 400 return jsonify({"suggestions": suggestions}) except Exception as e: logger.error(f"Error in match suggestions: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/match/search', methods=['POST']) def search_match(): """Manual search for artists or albums""" try: data = request.get_json() query = data.get('query', '').strip() context = data.get('context', 'artist') # 'artist' or 'album' if not query: return jsonify({"results": []}) use_hydrabase = _is_hydrabase_active() # Mirror to Hydrabase P2P network (fire-and-forget when not primary) if not use_hydrabase and hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, context) if context == 'artist': # Search for artists — title-case the query for better API results # (some metadata APIs return fewer results for all-lowercase queries) search_query = query.title() if query == query.lower() else query if use_hydrabase: artist_matches = hydrabase_client.search_artists(search_query, limit=8) provider = 'hydrabase' else: artist_matches = _get_metadata_fallback_client().search_artists(search_query, limit=8) provider = _get_metadata_fallback_source() results = [] for artist in artist_matches: # Calculate confidence based on search similarity confidence = matching_engine.similarity_score( matching_engine.normalize_string(query), matching_engine.normalize_string(artist.name) ) results.append({ "artist": { "id": artist.id, "name": artist.name, "image_url": getattr(artist, 'image_url', None), "genres": getattr(artist, 'genres', []), "popularity": getattr(artist, 'popularity', 0), "followers": getattr(artist, 'followers', 0) }, "confidence": confidence }) return jsonify({"results": results, "provider": provider}) elif context == 'album': # Search for albums by specific artist artist_id = data.get('artist_id') if use_hydrabase: # Hydrabase: search albums by query directly album_matches = hydrabase_client.search_albums(query, limit=20) provider = 'hydrabase' else: if not artist_id: return jsonify({"error": "Artist ID required for album search"}), 400 # Get artist's albums and filter by query album_matches = _get_metadata_fallback_client().get_artist_albums(artist_id) provider = _get_metadata_fallback_source() results = [] for album in album_matches: # Calculate confidence based on query similarity confidence = matching_engine.similarity_score( matching_engine.normalize_string(query), matching_engine.normalize_string(album.name) ) # Only include results with reasonable similarity if confidence > 0.3: results.append({ "album": { "id": album.id, "name": album.name, "release_date": getattr(album, 'release_date', ''), "album_type": getattr(album, 'album_type', 'album'), "image_url": getattr(album, 'image_url', None), "total_tracks": getattr(album, 'total_tracks', 0) }, "confidence": confidence }) # Sort by confidence results.sort(key=lambda x: x['confidence'], reverse=True) return jsonify({"results": results[:8], "provider": provider}) else: return jsonify({"error": "Invalid context. Must be 'artist' or 'album'"}), 400 except Exception as e: logger.error(f"Error in match search: {e}") return jsonify({"error": str(e)}), 500 def _is_explicit_blocked(track_data): """Check if a track should be blocked by the explicit content filter. Returns True if the track is explicit and explicit content is disabled.""" if config_manager.get('content_filter.allow_explicit', True): return False # Check direct explicit field if track_data.get('explicit', False): return True # Check nested spotify_data (wishlist tracks) sp_data = track_data.get('spotify_data', {}) if isinstance(sp_data, str): try: sp_data = json.loads(sp_data) except Exception: sp_data = {} return sp_data.get('explicit', False) def _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_artist, spotify_album): """ Download album tracks that have been matched to Spotify with full track metadata. This provides the best possible metadata enhancement and organization. """ logger.info(f"Processing enhanced album download for '{spotify_album['name']}' with {len(enhanced_tracks)} matched tracks") # Compute total_discs for multi-disc album subfolder support total_discs = max((t['spotify_track'].get('disc_number', 1) for t in enhanced_tracks), default=1) spotify_album['total_discs'] = total_discs started_count = 0 # PREFLIGHT: Pre-populate MusicBrainz release cache so all tracks get the same release try: mb_svc = mb_worker.mb_service if mb_worker else None if mb_svc and spotify_album.get('name') and spotify_artist.get('name'): from core.album_consistency import _find_best_release _pf_count = len(enhanced_tracks) + len(unmatched_tracks) _pf_release = _find_best_release(spotify_album['name'], spotify_artist['name'], _pf_count, mb_svc) if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with mb_release_cache_lock: mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with mb_release_detail_cache_lock: mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: logger.error(f"[Preflight] MB release preflight failed: {pf_err}") # Process matched tracks with full Spotify metadata for matched_item in enhanced_tracks: try: slskd_track = matched_item['slskd_track'] spotify_track = matched_item['spotify_track'] if _is_explicit_blocked(spotify_track): logger.info(f"[Content Filter] Skipping explicit track: '{spotify_track.get('name')}'") continue username = slskd_track.get('username') filename = slskd_track.get('filename') size = slskd_track.get('size', 0) if not username or not filename: logger.warning(f"Skipping track with missing username or filename: {slskd_track}") continue # Start download download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) with matched_context_lock: # Create context with FULL Spotify track metadata (like Download Missing Tracks modal) matched_downloads_context[context_key] = { "spotify_artist": spotify_artist, "spotify_album": spotify_album, "track_info": spotify_track, # Full Spotify track object! "original_search_result": { 'username': username, 'filename': filename, 'size': size, 'title': spotify_track['name'], # Use Spotify title 'artist': spotify_artist['name'], 'album': spotify_album['name'], 'track_number': spotify_track['track_number'], # Use Spotify track number 'disc_number': spotify_track.get('disc_number', 1), 'spotify_clean_title': spotify_track['name'] # For filename generation }, "is_album_download": True, "has_full_spotify_metadata": True # Flag for robust processing } logger.info( "Queued matched track: title=%r track_number=%s", spotify_track['name'], spotify_track['track_number'], ) started_count += 1 else: logger.error("Failed to queue track: filename=%s", filename) except Exception as e: logger.error(f"Error processing matched track: {e}") continue # Process unmatched tracks with basic cleanup for slskd_track in unmatched_tracks: try: username = slskd_track.get('username') filename = slskd_track.get('filename') size = slskd_track.get('size', 0) if not username or not filename: continue download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) with matched_context_lock: # Basic context for unmatched tracks (simple cleanup) matched_downloads_context[context_key] = { 'search_result': { 'username': username, 'filename': filename, 'size': size, 'is_simple_download': True # Falls back to simple transfer }, 'spotify_artist': None, 'spotify_album': None, 'track_info': None } logger.info(f"Queued unmatched track (basic cleanup): {filename}") started_count += 1 except Exception as e: logger.error(f"Error processing unmatched track: {e}") continue return started_count def _start_album_download_tasks(album_result, spotify_artist, spotify_album): """ This final version now fetches the official Spotify tracklist and uses it to match and correct the metadata for each individual track before downloading, ensuring perfect tagging and naming. """ logger.info(f"Processing matched album download for '{spotify_album['name']}' with {len(album_result.get('tracks', []))} tracks.") tracks_to_download = album_result.get('tracks', []) if not tracks_to_download: logger.warning("Album result contained no tracks. Aborting.") return 0 # --- THIS IS THE NEW LOGIC --- # Fetch the official tracklist from Spotify ONCE for the entire album. official_spotify_tracks = _get_spotify_album_tracks(spotify_album) if not official_spotify_tracks: logger.error("Could not fetch official tracklist from Spotify. Metadata may be inaccurate.") # --- END OF NEW LOGIC --- # Compute total_discs for multi-disc album subfolder support if official_spotify_tracks: total_discs = max((t.get('disc_number', 1) for t in official_spotify_tracks), default=1) else: total_discs = 1 spotify_album['total_discs'] = total_discs # PREFLIGHT: Pre-populate MusicBrainz release cache so all tracks get the same release try: mb_svc = mb_worker.mb_service if mb_worker else None if mb_svc and spotify_album.get('name') and spotify_artist.get('name'): from core.album_consistency import _find_best_release _pf_release = _find_best_release(spotify_album['name'], spotify_artist['name'], len(tracks_to_download), mb_svc) if _pf_release and _pf_release.get('id'): _pf_mbid = _pf_release['id'] _pf_artist_key = spotify_artist['name'].lower().strip() with mb_release_cache_lock: mb_release_cache[(normalize_album_cache_key(spotify_album['name']), _pf_artist_key)] = _pf_mbid mb_release_cache[(spotify_album['name'].lower().strip(), _pf_artist_key)] = _pf_mbid with mb_release_detail_cache_lock: mb_release_detail_cache[_pf_mbid] = _pf_release logger.info(f"[Preflight] Pre-cached MB release for '{spotify_album['name']}': " f"'{_pf_release.get('title', '')}' ({_pf_mbid[:8]}...)") except Exception as pf_err: logger.error(f"[Preflight] MB release preflight failed: {pf_err}") started_count = 0 for track_data in tracks_to_download: try: username = track_data.get('username') or album_result.get('username') filename = track_data.get('filename') size = track_data.get('size', 0) if not username or not filename: continue # Pre-parse the filename to get a baseline for metadata parsed_meta = _parse_filename_metadata(filename) # --- THIS IS THE CRITICAL MATCHING STEP --- # Match the parsed metadata against the official Spotify tracklist corrected_meta = _match_track_to_spotify_title(parsed_meta, official_spotify_tracks) # --- END OF CRITICAL STEP --- if _is_explicit_blocked(corrected_meta): logger.info(f"[Content Filter] Skipping explicit track: '{corrected_meta.get('title')}'") continue # Create a clean context object using the CORRECTED metadata individual_track_context = { 'username': username, 'filename': filename, 'size': size, 'title': corrected_meta.get('title'), 'artist': corrected_meta.get('artist') or spotify_artist['name'], 'album': spotify_album['name'], 'track_number': corrected_meta.get('track_number'), 'disc_number': corrected_meta.get('disc_number', 1) } download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) with matched_context_lock: # Enhanced context storage with Spotify clean titles (GUI parity) enhanced_context = individual_track_context.copy() enhanced_context['spotify_clean_title'] = individual_track_context.get('title', '') matched_downloads_context[context_key] = { "spotify_artist": spotify_artist, "spotify_album": spotify_album, "original_search_result": enhanced_context, # Contains corrected data + clean title "is_album_download": True } logger.info( "Queued track: filename=%s matched_title=%r", filename, corrected_meta.get('title'), ) started_count += 1 else: logger.error("Failed to queue track: filename=%s", filename) except Exception as e: logger.error(f"Error processing track in album batch: {track_data.get('filename')}: {e}") continue return started_count @app.route('/api/download/matched', methods=['POST']) def start_matched_download(): """ Starts a matched download. Supports: 1. Enhanced album downloads with full Spotify track metadata 2. Regular album downloads (fallback) 3. Single track downloads """ dl_err = check_download_permission() if dl_err: return dl_err try: data = request.get_json() download_payload = data.get('search_result', {}) spotify_artist = data.get('spotify_artist', {}) spotify_album = data.get('spotify_album', None) enhanced_tracks = data.get('enhanced_tracks', []) # Album: Matched tracks with Spotify data unmatched_tracks = data.get('unmatched_tracks', []) # Album: Tracks that didn't match spotify_track = data.get('spotify_track', None) # Single: Full Spotify track object is_single_track = data.get('is_single_track', False) # Single: Flag for single track if not download_payload or not spotify_artist: return jsonify({"success": False, "error": "Missing download payload or artist data"}), 400 # NEW: Enhanced single track with full Spotify metadata if is_single_track and spotify_track: if _is_explicit_blocked(spotify_track): return jsonify({"success": False, "error": "Explicit content is disabled in settings", "explicit_blocked": True}), 403 logger.info(f"Starting enhanced single track download: '{spotify_track['name']}' by {spotify_artist['name']}") username = download_payload.get('username') filename = download_payload.get('filename') size = download_payload.get('size', 0) if not username or not filename: return jsonify({"success": False, "error": "Missing username or filename"}), 400 download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) with matched_context_lock: # Create context with FULL Spotify track metadata (like Download Missing Tracks modal) matched_downloads_context[context_key] = { "spotify_artist": spotify_artist, "spotify_album": spotify_track.get('album'), # Single's album from Spotify "track_info": spotify_track, # Full Spotify track object! "original_search_result": { 'username': username, 'filename': filename, 'size': size, 'title': spotify_track['name'], 'artist': spotify_artist['name'], 'album': spotify_track.get('album', {}).get('name', 'Unknown Album'), 'track_number': spotify_track.get('track_number', 1), 'spotify_clean_title': spotify_track['name'] }, "is_album_download": False, # It's a single "has_full_spotify_metadata": True # Flag for robust processing } logger.info(f"Queued enhanced single track: '{spotify_track['name']}'") return jsonify({"success": True, "message": "Enhanced single track download started"}) else: return jsonify({"success": False, "error": "Failed to start download via slskd"}), 500 # NEW: Enhanced album download with track-to-track matching if enhanced_tracks: logger.info(f"Starting enhanced album download: {len(enhanced_tracks)} matched tracks, {len(unmatched_tracks)} unmatched") started_count = _start_enhanced_album_download(enhanced_tracks, unmatched_tracks, spotify_artist, spotify_album) if started_count > 0: return jsonify({"success": True, "message": f"Queued {started_count} tracks with full Spotify metadata."}) else: return jsonify({"success": False, "error": "Failed to queue any tracks from the album."}), 500 # Regular album download (fallback if matching fails) is_full_album_download = bool(spotify_album and download_payload.get('result_type') == 'album') if is_full_album_download: started_count = _start_album_download_tasks(download_payload, spotify_artist, spotify_album) if started_count > 0: return jsonify({"success": True, "message": f"Queued {started_count} tracks for matched album download."}) else: return jsonify({"success": False, "error": "Failed to queue any tracks from the album."}), 500 else: # This block handles BOTH regular singles AND individual tracks from an album card. username = download_payload.get('username') filename = download_payload.get('filename') size = download_payload.get('size', 0) if not username or not filename: return jsonify({"success": False, "error": "Missing username or filename"}), 400 parsed_meta = _parse_filename_metadata(filename) download_payload['title'] = parsed_meta.get('title') or download_payload.get('title') download_payload['artist'] = parsed_meta.get('artist') or download_payload.get('artist') download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: context_key = _make_context_key(username, filename) with matched_context_lock: # THE FIX: We preserve the spotify_album context if it was provided. # For a regular single, spotify_album will be None. # For an album track, it will contain the album's data. # Enhanced context storage with Spotify clean titles (GUI parity) enhanced_payload = download_payload.copy() enhanced_payload['spotify_clean_title'] = download_payload.get('title', '') matched_downloads_context[context_key] = { "spotify_artist": spotify_artist, "spotify_album": spotify_album, # PRESERVE album context "original_search_result": enhanced_payload, "is_album_download": False # It's a single track download, not a full album job. } return jsonify({"success": True, "message": "Matched download started"}) else: return jsonify({"success": False, "error": "Failed to start download via slskd"}), 500 except Exception as e: import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _parse_filename_metadata(filename: str) -> dict: """ A direct port of the metadata parsing logic from the GUI's soulseek_client.py. This is the crucial missing step that cleans filenames BEFORE Spotify matching. """ return parse_filename_metadata(filename) def _read_staging_file_metadata(full_path: str, filename: str) -> dict: """Read metadata from a staging file — tags first, filename parsing as fallback. Returns dict with: title, artist, albumartist, album, track_number, disc_number. Only falls back to filename parsing when BOTH title AND artist tags are empty. """ return read_staging_file_metadata(full_path, filename) # =================================================================== # NEW POST-PROCESSING HELPERS (Ported from downloads.py) # =================================================================== def _sanitize_filename(filename: str) -> str: """Sanitize filename for file system compatibility.""" import re sanitized = re.sub(r'[<>:"/\\|?*]', '_', filename) sanitized = re.sub(r'\s+', ' ', sanitized).strip() # Windows forbids trailing dots/spaces on files and folders. # Artists like "Fred again.." would create mangled 8.3 short names. sanitized = sanitized.rstrip('. ') or '_' # Windows reserved device names (CON, PRN, AUX, NUL, COM1-9, LPT1-9) # can't be used as file or folder names even with extensions. if re.match(r'^(CON|PRN|AUX|NUL|COM[1-9]|LPT[1-9])(\.|$)', sanitized, re.IGNORECASE): sanitized = '_' + sanitized return sanitized[:200] def _sanitize_context_values(context: dict) -> dict: """ Sanitize all string values in context dict for path safety. Prevents characters like '/' in artist names (e.g., 'AC/DC') from being interpreted as path separators during template substitution. Args: context: Dictionary with metadata values Returns: New dictionary with sanitized string values """ sanitized = {} for key, value in context.items(): if isinstance(value, str) and value: sanitized[key] = _sanitize_filename(value) else: sanitized[key] = value return sanitized def _clean_track_title(track_title: str, artist_name: str) -> str: """Clean up track title by removing artist prefix and other noise.""" import re original = track_title.strip() cleaned = original cleaned = re.sub(r'^\d{1,2}[\.\s\-]+', '', cleaned) artist_pattern = re.escape(artist_name) + r'\s*-\s*' cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE) cleaned = re.sub(r'^[A-Za-z0-9\.]+\s*-\s*\d{1,2}\s*-\s*', '', cleaned) quality_patterns = [r'\s*[\[\(][0-9]+\s*kbps[\]\)]\s*', r'\s*[\[\(]flac[\]\)]\s*', r'\s*[\[\(]mp3[\]\)]\s*'] for pattern in quality_patterns: cleaned = re.sub(pattern, '', cleaned, flags=re.IGNORECASE) cleaned = re.sub(r'^[-\s\.]+', '', cleaned) cleaned = re.sub(r'[-\s\.]+$', '', cleaned) cleaned = re.sub(r'\s+', ' ', cleaned).strip() return cleaned if cleaned else original def _extract_track_number_from_filename(filename: str, title: str = None) -> int: """Extract track number from filename, returns 1 if not found. Requires a separator after digits to avoid matching artist names like '50 Cent'.""" import re import os basename = os.path.splitext(os.path.basename(filename))[0].strip() # Match: "01 - Song", "01. Song", "01-Song", "1.Song", "(01) Song", "[01] Song" match = re.match(r'^\(?(\d{1,3})\)?\s*[\-\.)\]]\s*', basename) if match: num = int(match.group(1)) if 1 <= num <= 999: return num # Match: "1-03 Song" (disc-track format) match = re.match(r'^\d[\-\.](\d{1,2})\s*[\-\.]\s*', basename) if match: num = int(match.group(1)) if 1 <= num <= 99: return num return 1 def _search_track_in_album_context(original_search: dict, artist: dict) -> dict: """ Searches for a track within its album context to avoid matching promotional singles. This is a direct port from downloads.py for web server use. """ try: album_name = original_search.get('album') track_title = original_search.get('title') if not all([album_name, track_title, artist]): return None clean_album = _clean_track_title(album_name, artist['name']) # Use track cleaner for album too clean_track = _clean_track_title(track_title, artist['name']) album_query = f"album:\"{clean_album}\" artist:\"{artist['name']}\"" albums = spotify_client.search_albums(album_query, limit=1) if not albums: return None spotify_album = albums[0] album_tracks_data = spotify_client.get_album_tracks(spotify_album.id) if not album_tracks_data or 'items' not in album_tracks_data: return None for track_data in album_tracks_data['items']: similarity = matching_engine.similarity_score( matching_engine.normalize_string(clean_track), matching_engine.normalize_string(track_data['name']) ) if similarity > 0.7: logger.info(f"Found track in album context: '{track_data['name']}'") return { 'is_album': True, 'album_name': spotify_album.name, 'track_number': track_data['track_number'], 'clean_track_name': track_data['name'], 'album_image_url': spotify_album.image_url } return None except Exception as e: logger.error(f"Error in _search_track_in_album_context: {e}") return None def _cleanup_empty_directories(download_path, moved_file_path): """Cleans up empty directories after a file move, ignoring hidden files.""" import os try: current_dir = os.path.dirname(moved_file_path) while current_dir != download_path and current_dir.startswith(download_path): is_empty = not any(not f.startswith('.') for f in os.listdir(current_dir)) if is_empty: logger.warning(f"Removing empty directory: {current_dir}") os.rmdir(current_dir) current_dir = os.path.dirname(current_dir) else: break except Exception as e: logger.error(f"An error occurred during directory cleanup: {e}") def _sweep_empty_download_directories(): """ Walk the download directory bottom-up and remove ALL empty directories. Called periodically when no downloads or post-processing are active. Handles the edge case where per-file cleanup misses folders that become empty only after all sibling downloads in a batch have been processed. """ import os try: download_path = docker_resolve_path(config_manager.get('soulseek.download_path', './downloads')) if not os.path.isdir(download_path): return 0 removed = 0 # os.walk bottom-up: deepest directories first so parents become empty after children removed for dirpath, _dirnames, _filenames in os.walk(download_path, topdown=False): # Never remove the root download directory itself if os.path.normpath(dirpath) == os.path.normpath(download_path): continue # Re-read actual contents — os.walk's lists are stale after child removal try: entries = os.listdir(dirpath) except OSError: continue visible = [e for e in entries if not e.startswith('.')] if not visible: try: # Remove any leftover hidden files (e.g. .DS_Store) before rmdir for hidden in entries: try: os.remove(os.path.join(dirpath, hidden)) except Exception: pass os.rmdir(dirpath) removed += 1 except OSError: pass # Directory not actually empty or locked — skip silently if removed > 0: logger.warning(f"[Folder Cleanup] Removed {removed} empty director{'y' if removed == 1 else 'ies'} from downloads folder") return removed except Exception as e: logger.error(f"[Folder Cleanup] Error sweeping empty directories: {e}") return 0 # =================================================================== # ALBUM GROUPING SYSTEM (Ported from GUI downloads.py) # =================================================================== def _get_base_album_name(album_name: str) -> str: """ Extract the base album name without edition indicators. E.g., 'good kid, m.A.A.d city (Deluxe Edition)' -> 'good kid, m.A.A.d city' 'Battle Hymns (MMXI Special Edition)' -> 'Battle Hymns' """ import re # Remove common edition suffixes base_name = album_name # Remove edition indicators in parentheses or brackets # Allow any prefix before the keyword (e.g. "MMXI Special Edition", "20th Anniversary Edition") base_name = re.sub(r'\s*[\[\(][^)\]]*\b(deluxe|special|expanded|extended|bonus|remaster(?:ed)?|anniversary|collectors?|limited|silver|gold|platinum)\b[^)\]]*[\]\)]\s*$', '', base_name, flags=re.IGNORECASE) # Generic: any parenthesized/bracketed text ending with "edition" base_name = re.sub(r'\s*[\[\(][^)\]]*\bedition\b[^)\]]*[\]\)]\s*$', '', base_name, flags=re.IGNORECASE) # Remove standalone edition words at the end base_name = re.sub(r'\s+(deluxe|special|expanded|extended|bonus|remastered|anniversary|collectors?|limited|silver|gold|platinum)\s*(edition)?\s*$', '', base_name, flags=re.IGNORECASE) return base_name.strip() def _detect_deluxe_edition(album_name: str) -> bool: """ Detect if an album name indicates a deluxe/special edition. Returns True if it's a deluxe variant, False for standard. """ if not album_name: return False album_lower = album_name.lower() # Check for deluxe indicators deluxe_indicators = [ 'deluxe', 'deluxe edition', 'special edition', 'expanded edition', 'extended edition', 'bonus', 'remastered', 'anniversary', 'collectors edition', 'limited edition', 'silver edition', 'gold edition', 'platinum edition', ] for indicator in deluxe_indicators: if indicator in album_lower: logger.info(f"Detected deluxe edition: '{album_name}' contains '{indicator}'") return True return False def _normalize_base_album_name(base_album: str, artist_name: str) -> str: """ Normalize the base album name to handle case variations and known corrections. """ import re # Apply known album corrections for consistent naming normalized_lower = base_album.lower().strip() # Handle common album title variations known_corrections = { # Add specific album name corrections here as needed # Example: "good kid maad city": "good kid, m.A.A.d city" } # Check for exact matches in our corrections for variant, correction in known_corrections.items(): if normalized_lower == variant.lower(): logger.info(f"Album correction applied: '{base_album}' -> '{correction}'") return correction # Handle punctuation variations normalized = base_album # Normalize common punctuation patterns normalized = re.sub(r'\s*&\s*', ' & ', normalized) # Standardize & spacing normalized = re.sub(r'\s+', ' ', normalized) # Clean multiple spaces normalized = normalized.strip() logger.info(f"Album variant normalization: '{base_album}' -> '{normalized}'") return normalized def _clean_album_title_web(album_title: str, artist_name: str) -> str: """Clean up album title by removing common prefixes, suffixes, and artist redundancy""" import re # Start with the original title original = album_title.strip() cleaned = original logger.info(f"Album Title Cleaning: '{original}' (artist: '{artist_name}')") # Remove "Album - " prefix cleaned = re.sub(r'^Album\s*-\s*', '', cleaned, flags=re.IGNORECASE) # Remove artist name prefix if it appears at the beginning # This handles cases like "Kendrick Lamar - good kid, m.A.A.d city" artist_pattern = re.escape(artist_name) + r'\s*-\s*' cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE) # Remove common Soulseek suffixes in square brackets and parentheses # Examples: [Deluxe Edition] [2012] [320 Kbps] [Album+iTunes+Bonus Tracks] [F10] # (Deluxe Edition) (2012) (320 Kbps) etc. # Remove year patterns like [2012], (2020), etc. cleaned = re.sub(r'\s*[\[\(]\d{4}[\]\)]\s*', ' ', cleaned) # Remove quality/format indicators quality_patterns = [ r'\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*', r'\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*', r'\s*[\[\(].*?flac.*?[\]\)]\s*', r'\s*[\[\(].*?mp3.*?[\]\)]\s*', r'\s*[\[\(].*?itunes.*?[\]\)]\s*', r'\s*[\[\(].*?web.*?[\]\)]\s*', r'\s*[\[\(].*?cd.*?[\]\)]\s*' ] for pattern in quality_patterns: cleaned = re.sub(pattern, ' ', cleaned, flags=re.IGNORECASE) # Remove common edition indicators (but preserve them for deluxe detection above) # This happens AFTER deluxe detection to avoid interfering with that logic # Clean up spacing cleaned = re.sub(r'\s+', ' ', cleaned).strip() # Remove leading/trailing punctuation cleaned = re.sub(r'^[-\s]+|[-\s]+$', '', cleaned) logger.info(f"Album Title Result: '{original}' -> '{cleaned}'") return cleaned if cleaned else original def _search_track_in_album_context_web(context: dict, spotify_artist: dict) -> dict: """ Search for a track within its album context to avoid promotional single confusion. (Ported from GUI downloads.py) """ try: from core.matching_engine import MusicMatchingEngine matching_engine = MusicMatchingEngine() # Get album and track info from context original_search = context.get("original_search_result", {}) album_name = original_search.get("album") track_title = original_search.get("title") artist_name = spotify_artist["name"] if not album_name or not track_title: logger.error(f"Album-aware search failed: Missing album ({album_name}) or track ({track_title})") return None logger.info(f"Album-aware search: '{track_title}' in album '{album_name}' by '{artist_name}'") # Clean the album name for better search results clean_album = _clean_album_title_web(album_name, artist_name) clean_track = _clean_track_title_web(track_title, artist_name) # Search for the specific album first album_query = f"album:{clean_album} artist:{artist_name}" logger.info(f"Searching albums: {album_query}") albums = spotify_client.search_albums(album_query, limit=5) if not albums: logger.warning(f"No albums found for query: {album_query}") return None # Check each album to see if our track is in it for album in albums: logger.info(f"Checking album: '{album.name}' ({album.total_tracks} tracks)") # Get tracks from this album album_tracks_data = spotify_client.get_album_tracks(album.id) if not album_tracks_data or 'items' not in album_tracks_data: logger.error(f"Could not get tracks for album: {album.name}") continue # Check if our track is in this album for track_data in album_tracks_data['items']: track_name = track_data['name'] track_number = track_data['track_number'] # Calculate similarity between our track and this album track similarity = matching_engine.similarity_score( matching_engine.normalize_string(clean_track), matching_engine.normalize_string(track_name) ) # Use higher threshold for remix matching to ensure precision (GUI PARITY) is_remix = any(word in clean_track.lower() for word in ['remix', 'mix', 'edit', 'version']) threshold = 0.9 if is_remix else 0.65 # Lower threshold to favor album matches over singles if similarity > threshold: logger.info(f"FOUND: '{track_name}' (track #{track_number}) matches '{clean_track}' (similarity: {similarity:.2f})") # Classify as album vs single using the shared detect_album_info_web helper ctx_album_type = getattr(album, 'album_type', 'album') or 'album' ctx_total_tracks = getattr(album, 'total_tracks', 1) or 1 ctx_is_album = ( ctx_album_type == 'album' and ctx_total_tracks > 1 and matching_engine.normalize_string(album.name) != matching_engine.normalize_string(clean_track) and matching_engine.normalize_string(album.name) != matching_engine.normalize_string(artist_name) ) logger.info(f"Album context classification: is_album={ctx_is_album} (type={ctx_album_type}, tracks={ctx_total_tracks})") return { 'is_album': ctx_is_album, 'album_name': album.name, 'track_number': track_number, 'clean_track_name': clean_track, # Use the ORIGINAL download title, not the database match 'album_image_url': album.image_url, 'confidence': similarity, 'source': 'album_context_search' } logger.warning(f"Track '{clean_track}' not found in album '{album.name}'") logger.warning(f"Track '{clean_track}' not found in any matching albums") return None except Exception as e: logger.error(f"Error in album-aware search: {e}") return None def _clean_track_title_web(track_title: str, artist_name: str) -> str: """Clean up track title by removing artist prefix and common patterns""" import re # Start with the original title original = track_title.strip() cleaned = original logger.info(f"Track Title Cleaning: '{original}' (artist: '{artist_name}')") # Remove artist name prefix if it appears at the beginning # This handles cases like "Kendrick Lamar - HUMBLE." artist_pattern = re.escape(artist_name) + r'\s*-\s*' cleaned = re.sub(f'^{artist_pattern}', '', cleaned, flags=re.IGNORECASE) # Remove common prefixes cleaned = re.sub(r'^Track\s*\d*\s*-\s*', '', cleaned, flags=re.IGNORECASE) cleaned = re.sub(r'^\d+\.\s*', '', cleaned) # Remove track numbers like "01. " # Remove quality/format indicators quality_patterns = [ r'\s*[\[\(].*?320.*?kbps.*?[\]\)]\s*', r'\s*[\[\(].*?256.*?kbps.*?[\]\)]\s*', r'\s*[\[\(].*?flac.*?[\]\)]\s*', r'\s*[\[\(].*?mp3.*?[\]\)]\s*', r'\s*[\[\(].*?explicit.*?[\]\)]\s*' ] for pattern in quality_patterns: cleaned = re.sub(pattern, ' ', cleaned, flags=re.IGNORECASE) # Clean up spacing cleaned = re.sub(r'\s+', ' ', cleaned).strip() # Remove leading/trailing punctuation cleaned = re.sub(r'^[-\s]+|[-\s]+$', '', cleaned) logger.info(f"Track Title Result: '{original}' -> '{cleaned}'") return cleaned if cleaned else original # =================================================================== # YOUTUBE TRACK CLEANING FUNCTIONS (Ported from GUI sync.py) # =================================================================== def clean_youtube_track_title(title, artist_name=None): """ Aggressively clean YouTube track titles by removing video noise and extracting clean track names Examples: 'No Way Jose (Official Music Video)' → 'No Way Jose' 'bbno$ - mary poppins (official music video)' → 'mary poppins' 'Beyond (From "Moana 2") (Official Video) ft. Rachel House' → 'Beyond' 'Temporary (feat. Skylar Grey) [Official Music Video]' → 'Temporary' 'ALL MY LOVE (Directors\' Cut)' → 'ALL MY LOVE' 'Espresso Macchiato | Estonia 🇪🇪 | Official Music Video | #Eurovision2025' → 'Espresso Macchiato' """ import re if not title: return title original_title = title # FIRST: Try to extract track name from "Artist - Track" or "Track - Artist" format artist_removed = False if artist_name and '-' in title: # Check if artist is at the start: "Artist - Track" or "Artist & Others - Track" # Handle collaborations: "Artist1 & Artist2 - Track" or "Artist, Artist2 - Track" artist_pattern = r'^' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*[-–—]\s*' cleaned_title = re.sub(artist_pattern, '', title, flags=re.IGNORECASE).strip() if cleaned_title != title: logger.info(f"Removed artist from start: '{title}' -> '{cleaned_title}' (artist: '{artist_name}')") title = cleaned_title artist_removed = True else: # Artist not at start, check if format is "Track - Artist" by looking for artist at end # Only remove trailing artist if it comes after a dash artist_end_pattern = r'\s*[-–—]\s*' + re.escape(artist_name.strip()) + r'(?:\s*[&,x]\s*[^-]+)?\s*$' cleaned_title = re.sub(artist_end_pattern, '', title, flags=re.IGNORECASE).strip() if cleaned_title != title: logger.info(f"Removed artist from end: '{title}' -> '{cleaned_title}' (artist: '{artist_name}')") title = cleaned_title artist_removed = True # Remove content in brackets/braces BEFORE removing dashes title = re.sub(r'【[^】]*】', '', title) # Japanese brackets title = re.sub(r'\s*\([^)]*\)', '', title) # Parentheses - removes everything after first ( title = re.sub(r'\s*\(.*$', '', title) # Remove everything after lone ( (unmatched parentheses) title = re.sub(r'\[[^\]]*\]', '', title) # Square brackets title = re.sub(r'\{[^}]*\}', '', title) # Curly braces title = re.sub(r'<[^>]*>', '', title) # Angle brackets # ONLY remove trailing dashes with garbage if artist was already removed # This prevents "Artist1, Artist2 - Song" from becoming "Artist1, Artist2" if artist_removed: # Safe to remove any remaining trailing dash content (likely album/extra info) title = re.sub(r'\s*-\s*.*$', '', title) # Remove everything after pipes (|) - often used for additional context title = re.split(r'\s*\|\s*', title)[0].strip() # Remove common video/platform noise noise_patterns = [ r'\bapple\s+music\b', r'\bfull\s+video\b', r'\bmusic\s+video\b', r'\bofficial\s+video\b', r'\bofficial\s+music\s+video\b', r'\bofficial\b', r'\bcensored\s+version\b', r'\buncensored\s+version\b', r'\bexplicit\s+version\b', r'\blive\s+version\b', r'\bversion\b', r'\btopic\b', r'\baudio\b', r'\blyrics?\b', r'\blyric\s+video\b', r'\bwith\s+lyrics?\b', r'\bvisuali[sz]er\b', r'\bmv\b', r'\bdirectors?\s+cut\b', r'\bremaster(ed)?\b', r'\bremix\b' ] for pattern in noise_patterns: title = re.sub(pattern, '', title, flags=re.IGNORECASE) # Only remove artist name if it's standalone (not part of "Artist1 & Artist2") # Skip this if the title contains collaboration indicators near the artist name if artist_name: # Check if artist appears with collaboration indicators (& or ,) collab_pattern = rf'\b{re.escape(artist_name)}\s*[&,]\s*\w+|[\w\s]+[&,]\s*{re.escape(artist_name)}\b' has_collab = re.search(collab_pattern, title, flags=re.IGNORECASE) if not has_collab: # Safe to remove artist - it's standalone title = re.sub(rf'\b{re.escape(artist_name)}\b', '', title, flags=re.IGNORECASE) title = re.sub(rf'\b{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) title = re.sub(rf'^{re.escape(artist_name)}\s*[-–—:]\s*', '', title, flags=re.IGNORECASE) else: logger.info(f"Skipping artist removal - collaboration detected: '{title}'") # Remove "prod. Producer" patterns title = re.sub(r'\s+prod\.?\s+\S+', '', title, flags=re.IGNORECASE) # Remove all quotes and other punctuation title = re.sub(r'["\'''""„‚‛‹›«»]', '', title) # Remove featured artist patterns (after removing parentheses) feat_patterns = [ r'\s+feat\.?\s+.+$', # " feat Artist" at end r'\s+ft\.?\s+.+$', # " ft Artist" at end r'\s+featuring\s+.+$', # " featuring Artist" at end r'\s+with\s+.+$', # " with Artist" at end ] for pattern in feat_patterns: title = re.sub(pattern, '', title, flags=re.IGNORECASE).strip() # Clean up whitespace and punctuation title = re.sub(r'\s+', ' ', title).strip() title = re.sub(r'^[-–—:,.\s]+|[-–—:,.\s]+$', '', title).strip() # If we cleaned too much, return original if not title.strip() or len(title.strip()) < 2: title = original_title if title != original_title: logger.info(f"YouTube title cleaned: '{original_title}' → '{title}'") return title def clean_youtube_artist(artist_string): """ Clean YouTube artist strings to get primary artist name Examples: 'Yung Gravy, bbno$ (BABY GRAVY)' → 'Yung Gravy' 'Y2K, bbno$' → 'Y2K' 'LITTLE BIG' → 'LITTLE BIG' 'Artist "Nickname" Name' → 'Artist Nickname Name' 'ArtistVEVO' → 'Artist' """ import re if not artist_string: return artist_string original_artist = artist_string # Remove all quotes - they're usually not part of artist names artist_string = artist_string.replace('"', '').replace("'", '').replace(''', '').replace(''', '').replace('"', '').replace('"', '') # Remove anything in parentheses (often group/label names) artist_string = re.sub(r'\s*\([^)]*\)', '', artist_string).strip() # Remove anything in brackets (often additional info) artist_string = re.sub(r'\s*\[[^\]]*\]', '', artist_string).strip() # Remove common YouTube channel suffixes channel_suffixes = [ r'\s*-\s*Topic\s*$', # YouTube auto-generated "Topic" channels (e.g. "Koven - Topic") r'\s*VEVO\s*$', r'\s*Music\s*$', r'\s*Official\s*$', r'\s*Records\s*$', r'\s*Entertainment\s*$', r'\s*TV\s*$', r'\s*Channel\s*$' ] for suffix in channel_suffixes: artist_string = re.sub(suffix, '', artist_string, flags=re.IGNORECASE).strip() # Split on common separators and take the first artist separators = [',', '&', ' and ', ' x ', ' X ', ' feat.', ' ft.', ' featuring', ' with', ' vs ', ' vs.'] for sep in separators: if sep in artist_string: parts = artist_string.split(sep) artist_string = parts[0].strip() break # Clean up extra whitespace and punctuation artist_string = re.sub(r'\s+', ' ', artist_string).strip() artist_string = re.sub(r'^\-\s*|\s*\-$', '', artist_string).strip() # Remove leading/trailing dashes artist_string = re.sub(r'^,\s*|\s*,$', '', artist_string).strip() # Remove leading/trailing commas # If we cleaned too much, return original if not artist_string.strip(): artist_string = original_artist if artist_string != original_artist: logger.info(f"YouTube artist cleaned: '{original_artist}' → '{artist_string}'") return artist_string def parse_youtube_playlist(url): """ Parse a YouTube Music playlist URL and extract track information using yt-dlp Uses flat playlist extraction to avoid rate limits and get all tracks Returns a list of track dictionaries compatible with our Track structure """ try: # Configure yt-dlp options for flat playlist extraction (avoids rate limits) ydl_opts = { 'quiet': True, 'no_warnings': True, 'extract_flat': 'in_playlist', # Only extract basic info, no individual video metadata 'skip_download': True, # Don't download, just extract IDs and basic info 'lazy_playlist': False, # Force full playlist resolution (prevents ~100 entry cap) } tracks = [] with yt_dlp.YoutubeDL(ydl_opts) as ydl: # Extract playlist info playlist_info = ydl.extract_info(url, download=False) if not playlist_info: logger.error("Could not extract playlist information") return None playlist_name = playlist_info.get('title', 'Unknown Playlist') playlist_id = playlist_info.get('id', 'unknown_id') entries = list(playlist_info.get('entries', []) or []) logger.info(f"Found YouTube playlist: '{playlist_name}' with {len(entries)} entries") for entry in entries: if not entry: continue # Extract basic information from flat extraction raw_title = entry.get('title', 'Unknown Track') raw_uploader = entry.get('uploader', 'Unknown Artist') duration = entry.get('duration', 0) video_id = entry.get('id', '') # Clean the track title and artist using our cleaning functions cleaned_artist = clean_youtube_artist(raw_uploader) cleaned_title = clean_youtube_track_title(raw_title, cleaned_artist) # Create track object matching GUI structure track_data = { 'id': video_id, 'name': cleaned_title, 'artists': [cleaned_artist], 'duration_ms': duration * 1000 if duration else 0, 'raw_title': raw_title, # Keep original for reference 'raw_artist': raw_uploader, # Keep original for reference 'url': f"https://www.youtube.com/watch?v={video_id}" } tracks.append(track_data) # Create playlist object matching GUI structure playlist_data = { 'id': playlist_id, 'name': playlist_name, 'tracks': tracks, 'track_count': len(tracks), 'url': url, 'source': 'youtube', 'image_url': playlist_info.get('thumbnail', '') or '', } logger.info(f"Successfully parsed YouTube playlist: {len(tracks)} tracks extracted") return playlist_data except Exception as e: logger.error(f"Error parsing YouTube playlist: {e}") return None # =================================================================== # FILE ORGANIZATION TEMPLATE ENGINE # =================================================================== def _compute_m3u_folder(transfer_dir, context_type, playlist_name, artist_name='', album_name='', year=''): """ Compute the target folder for an M3U file using the template system. For playlists: uses playlist_path template, extracts folder portion. For albums: uses album_path template, extracts folder portion. Returns: absolute folder path """ if context_type == 'album' and artist_name and album_name: template_context = { 'artist': artist_name, 'albumartist': artist_name, 'album': album_name, 'title': 'placeholder', 'track_number': 1, 'disc_number': 1, 'year': year, 'quality': '' } folder_path, _ = _get_file_path_from_template(template_context, 'album_path') if folder_path: return os.path.join(transfer_dir, folder_path) # Fallback artist_sanitized = _sanitize_filename(artist_name) album_sanitized = _sanitize_filename(album_name) return os.path.join(transfer_dir, artist_sanitized, f"{artist_sanitized} - {album_sanitized}") else: template_context = { 'artist': 'placeholder', 'albumartist': 'placeholder', 'album': 'placeholder', 'title': 'placeholder', 'playlist_name': playlist_name, 'track_number': 1, 'disc_number': 1, 'year': '', 'quality': '' } folder_path, _ = _get_file_path_from_template(template_context, 'playlist_path') if folder_path: return os.path.join(transfer_dir, folder_path) # Fallback playlist_sanitized = _sanitize_filename(playlist_name) return os.path.join(transfer_dir, playlist_sanitized) def _get_file_path_from_template_raw(template: str, context: dict) -> tuple: """ Build file path using a user-provided template string directly. Unlike _get_file_path_from_template, this bypasses config lookup and the file_organization.enabled check — used by the reorganize feature where the user supplies the template explicitly. Args: template: Template string like "$artist/$album/$track - $title" context: Dict with all track/album metadata Returns: (folder_path, filename_base) tuple — no file extension included """ import re full_path = _apply_path_template(template, context) quality_value = context.get('quality', '') disc_value = f"{context.get('disc_number', 1):02d}" disc_value_raw = str(context.get('disc_number', 1)) path_parts = full_path.split('/') if len(path_parts) > 1: folder_parts = path_parts[:-1] filename_base = path_parts[-1] # $discnum before $disc — longer match first cleaned_folders = [] for part in folder_parts: part = part.replace('$quality', '') part = part.replace('$discnum', '') part = part.replace('$disc', '') part = re.sub(r'\s*\[\s*\]', '', part) part = re.sub(r'\s*\(\s*\)', '', part) part = re.sub(r'\s*\{\s*\}', '', part) part = re.sub(r'\s*-\s*$', '', part) part = re.sub(r'^\s*-\s*', '', part) part = re.sub(r'\s+', ' ', part).strip() if part: cleaned_folders.append(part) filename_base = filename_base.replace('$quality', quality_value) filename_base = filename_base.replace('$discnum', disc_value_raw) filename_base = filename_base.replace('$disc', disc_value) filename_base = re.sub(r'\s*\[\s*\]', '', filename_base) filename_base = re.sub(r'\s*\(\s*\)', '', filename_base) filename_base = re.sub(r'\s*\{\s*\}', '', filename_base) filename_base = re.sub(r'\s*-\s*$', '', filename_base) filename_base = re.sub(r'\s+', ' ', filename_base).strip() sanitized_folders = [_sanitize_filename(part) for part in cleaned_folders] folder_path = os.path.join(*sanitized_folders) if sanitized_folders else '' return folder_path, _sanitize_filename(filename_base) else: full_path = full_path.replace('$quality', quality_value) full_path = full_path.replace('$discnum', disc_value_raw) full_path = full_path.replace('$disc', disc_value) full_path = re.sub(r'\s*\[\s*\]', '', full_path) full_path = re.sub(r'\s*\(\s*\)', '', full_path) full_path = re.sub(r'\s*\{\s*\}', '', full_path) full_path = re.sub(r'\s*-\s*$', '', full_path) full_path = re.sub(r'\s+', ' ', full_path).strip() return '', _sanitize_filename(full_path) def _get_audio_quality_string(file_path): """ Read audio file and return a quality descriptor string. Returns strings like 'FLAC 16bit', 'MP3-320', 'M4A-256', 'OGG-192'. Returns empty string on any error. """ try: ext = os.path.splitext(file_path)[1].lower() if ext == '.flac': audio = FLAC(file_path) bits = audio.info.bits_per_sample return f"FLAC {bits}bit" elif ext == '.mp3': from mutagen.mp3 import MP3, BitrateMode audio = MP3(file_path) bitrate_kbps = audio.info.bitrate // 1000 if audio.info.bitrate_mode == BitrateMode.VBR: return "MP3-VBR" return f"MP3-{bitrate_kbps}" elif ext in ('.m4a', '.aac', '.mp4'): audio = MP4(file_path) bitrate_kbps = audio.info.bitrate // 1000 return f"M4A-{bitrate_kbps}" elif ext == '.ogg': audio = OggVorbis(file_path) bitrate_kbps = audio.info.bitrate // 1000 return f"OGG-{bitrate_kbps}" elif ext == '.opus': from mutagen.oggopus import OggOpus audio = OggOpus(file_path) bitrate_kbps = audio.info.bitrate // 1000 return f"OPUS-{bitrate_kbps}" return '' except Exception as e: logger.debug(f"Could not determine audio quality for {file_path}: {e}") return '' def _get_album_type_display(raw_type, track_count) -> str: """ Return the display form of an album's type for the $albumtype template variable. Mirrors the inference used in the download pipeline so reorganize output matches initial placement. Values: 'Album', 'Single', 'EP', 'Compilation'. """ raw = (raw_type or '').strip().lower() try: tc = int(track_count or 0) except (TypeError, ValueError): tc = 0 # Deezer's raw API returns 'compile' (only mapped to 'compilation' in the # Album dataclass path); the Deezer enrichment worker writes the raw value, # so both need to match here. if raw in ('compilation', 'compile'): return 'Compilation' if raw == 'album': return 'Album' if raw in ('single', 'ep'): # Match download-pipeline logic: Spotify labels both singles and EPs # as 'single', so final classification is by track count. Applying the # same rule to explicit 'ep' keeps reorganize consistent with where # the files were first placed. if tc <= 3: return 'Single' if tc <= 6: return 'EP' return 'Album' # Unknown/missing — infer from track count if tc <= 0: return 'Album' if tc <= 3: return 'Single' if tc <= 6: return 'EP' return 'Album' def _apply_path_template(template: str, context: dict) -> str: """ Apply template to build file path. Args: template: Template string like "$artist/$album/$track - $title" context: Dict with values like {'artist': 'Drake', 'album': 'Scorpion', ...} Returns: Processed path string """ # Sanitize context values BEFORE template substitution # This prevents '/' in metadata from creating unintended path components clean_context = _sanitize_context_values(context) result = template # Replace variables in order from longest to shortest to avoid partial replacements # (e.g., $albumartist must be replaced before $album to prevent "Scorpionartist" from typo "$albumartis") # Longest variables first album_artist_value = clean_context.get('albumartist', clean_context.get('artist', 'Unknown Artist')) # Collaborative album artist handling: "first" uses only the primary artist for folder names collab_mode = config_manager.get('file_organization.collab_artist_mode', 'first') if collab_mode == 'first' and album_artist_value: # Use structured artists list to safely extract first artist # Only splits when we have multiple distinct artist objects (Spotify provides these) # Avoids string splitting which breaks names like "Tyler, the Creator" or "Simon & Garfunkel" artists_list = context.get('_artists_list') if artists_list and len(artists_list) > 1: # Multiple artist objects (Spotify) — use first first = artists_list[0] album_artist_value = first.get('name', first) if isinstance(first, dict) else str(first) elif artists_list and len(artists_list) == 1: # Single artist string — could be a combined name from iTunes # Resolve via artistId if available (safe: "Simon & Garfunkel" ID → "Simon & Garfunkel") itunes_artist_id = context.get('_itunes_artist_id') if itunes_artist_id and (',' in album_artist_value or ' & ' in album_artist_value): try: resolved = _get_itunes_client().resolve_primary_artist(itunes_artist_id) if resolved and resolved != album_artist_value: album_artist_value = resolved except Exception: pass # $cdnum — smart CD label for multi-disc filenames. Produces "CD01" / # "CD02" etc. when the album has 2+ discs, empty string otherwise. # Empty output collapses gracefully via the trailing double-dash cleanup # regex, so single-disc albums don't end up with "CD01" in every name. _total_discs = int(clean_context.get('total_discs', 1) or 1) _disc_number = int(clean_context.get('disc_number', 1) or 1) cdnum_value = f"CD{_disc_number:02d}" if _total_discs > 1 else '' # Support ${var} delimited syntax (e.g. ${albumtype}s → Albums) # Must run before $var replacements to prevent partial matching _bracket_map = { 'albumartist': album_artist_value, 'albumtype': clean_context.get('albumtype', 'Album'), 'playlist': clean_context.get('playlist_name', ''), 'artistletter': (clean_context.get('artist', 'U') or 'U')[0].upper(), 'artist': clean_context.get('artist', 'Unknown Artist'), 'album': clean_context.get('album', 'Unknown Album'), 'title': clean_context.get('title', 'Unknown Track'), 'track': f"{clean_context.get('track_number', 1):02d}", 'cdnum': cdnum_value, 'disc': str(clean_context.get('disc_number', 1)), 'discnum': str(clean_context.get('disc_number', 1)), 'year': str(clean_context.get('year', '')), 'quality': clean_context.get('quality', ''), } for var_name, val in _bracket_map.items(): result = result.replace('${' + var_name + '}', val) result = result.replace('$albumartist', album_artist_value) result = result.replace('$albumtype', clean_context.get('albumtype', 'Album')) result = result.replace('$playlist', clean_context.get('playlist_name', '')) # Medium length variables result = result.replace('$artistletter', (clean_context.get('artist', 'U') or 'U')[0].upper()) result = result.replace('$artist', clean_context.get('artist', 'Unknown Artist')) result = result.replace('$album', clean_context.get('album', 'Unknown Album')) result = result.replace('$title', clean_context.get('title', 'Unknown Track')) # $cdnum must replace before $track to avoid conflict with variables that # start with "$c" — no such variable exists today but this ordering # mirrors the "longest first" rule used throughout this function. result = result.replace('$cdnum', cdnum_value) result = result.replace('$track', f"{clean_context.get('track_number', 1):02d}") result = result.replace('$year', str(clean_context.get('year', ''))) # Empty string instead of 'Unknown' # Clean up extra spaces that might result from empty variables import re result = re.sub(r'\s+', ' ', result) # Multiple spaces to single space result = re.sub(r'\s*-\s*-\s*', ' - ', result) # Clean up double dashes result = result.strip() # Remove leading/trailing spaces return result def _get_file_path_from_template(context: dict, template_type: str = 'album_path') -> tuple: """ Build complete file path using configured templates. Args: context: Dict with all track/album metadata template_type: 'album_path', 'single_path', 'compilation_path', 'playlist_path' Returns: (folder_path, filename) tuple """ # Check if template system is enabled if not config_manager.get('file_organization.enabled', True): # Fallback to hardcoded structure return None, None # Get template from config templates = config_manager.get('file_organization.templates', {}) template = templates.get(template_type) if not template: # Fallback templates if config missing default_templates = { 'album_path': '$albumartist/$albumartist - $album/$track - $title', 'single_path': '$artist/$artist - $title/$title', 'compilation_path': 'Compilations/$album/$track - $artist - $title', 'playlist_path': '$playlist/$artist - $title' } template = default_templates.get(template_type, '$artist/$album/$track - $title') # Apply template full_path = _apply_path_template(template, context) # Split into folder and filename path_parts = full_path.split('/') # Handle $quality and $disc: only substituted in the filename (last component). # In folder components they become empty string to prevent album splits # when tracks arrive in mixed qualities or disc numbers in folder names. import re quality_value = context.get('quality', '') disc_value = f"{context.get('disc_number', 1):02d}" disc_value_raw = str(context.get('disc_number', 1)) if len(path_parts) > 1: folder_parts = path_parts[:-1] filename_base = path_parts[-1] # Strip $quality, $discnum, $disc from folder parts and clean up artifacts # $discnum before $disc — longer match first to prevent partial replacement cleaned_folders = [] for part in folder_parts: part = part.replace('$quality', '') part = part.replace('$discnum', '') part = part.replace('$disc', '') part = re.sub(r'\s*\[\s*\]', '', part) # empty [] part = re.sub(r'\s*\(\s*\)', '', part) # empty () part = re.sub(r'\s*\{\s*\}', '', part) # empty {} part = re.sub(r'\s*-\s*$', '', part) # trailing dash part = re.sub(r'^\s*-\s*', '', part) # leading dash part = re.sub(r'\s+', ' ', part).strip() if part: cleaned_folders.append(part) # Substitute $quality, $discnum, $disc in filename only # $discnum before $disc — longer match first filename_base = filename_base.replace('$quality', quality_value) filename_base = filename_base.replace('$discnum', disc_value_raw) filename_base = filename_base.replace('$disc', disc_value) # Clean up empty brackets/parens from any variable that resolved to empty filename_base = re.sub(r'\s*\[\s*\]', '', filename_base) filename_base = re.sub(r'\s*\(\s*\)', '', filename_base) filename_base = re.sub(r'\s*\{\s*\}', '', filename_base) filename_base = re.sub(r'\s*-\s*$', '', filename_base) # Leading dash cleanup — lets $cdnum (and other optional vars) sit at # the start of the filename without leaving a stray "- " when empty. filename_base = re.sub(r'^\s*-\s*', '', filename_base) filename_base = re.sub(r'\s+', ' ', filename_base).strip() # Sanitize each folder component sanitized_folders = [_sanitize_filename(part) for part in cleaned_folders] folder_path = os.path.join(*sanitized_folders) if sanitized_folders else '' # Sanitize filename filename = _sanitize_filename(filename_base) return folder_path, filename else: # Single component, treat as filename — substitute $quality, $discnum, $disc full_path = full_path.replace('$quality', quality_value) full_path = full_path.replace('$discnum', disc_value_raw) full_path = full_path.replace('$disc', disc_value) full_path = re.sub(r'\s*\[\s*\]', '', full_path) full_path = re.sub(r'\s*\(\s*\)', '', full_path) full_path = re.sub(r'\s*\{\s*\}', '', full_path) full_path = re.sub(r'\s*-\s*$', '', full_path) full_path = re.sub(r'^\s*-\s*', '', full_path) full_path = re.sub(r'\s+', ' ', full_path).strip() return '', _sanitize_filename(full_path) # METADATA & COVER ART HELPERS (Ported from downloads.py) # =================================================================== from mutagen import File as MutagenFile from mutagen.id3 import ID3, TIT2, TPE1, TALB, TDRC, TRCK, TCON, TPE2, TPOS, TXXX, APIC, UFID, TSRC, TBPM, TCOP, TPUB, TMED, TDOR from mutagen.apev2 import APEv2, APENoHeaderError import urllib.request def _wipe_source_tags(file_path: str) -> bool: return metadata_enrichment.wipe_source_tags(file_path) def _enhance_file_metadata(file_path: str, context: dict, artist: dict, album_info: dict, metadata_runtime=None) -> bool: return metadata_enrichment.enhance_file_metadata( file_path, context, artist, album_info, runtime=metadata_runtime or _build_metadata_enrichment_runtime(), ) def _download_cover_art(album_info: dict, target_dir: str, context: dict = None): return metadata_enrichment.download_cover_art( album_info, target_dir, context, ) def _get_spotify_album_tracks(spotify_album: dict) -> list: """Fetches all tracks for a given Spotify album ID.""" if not spotify_album or not spotify_album.get('id'): return [] try: tracks_data = spotify_client.get_album_tracks(spotify_album['id']) if tracks_data and 'items' in tracks_data: return [{ 'name': item.get('name'), 'track_number': item.get('track_number'), 'disc_number': item.get('disc_number', 1), 'id': item.get('id'), 'explicit': item.get('explicit', False) } for item in tracks_data['items']] return [] except Exception as e: logger.error(f"Error fetching Spotify album tracks: {e}") return [] def _match_track_to_spotify_title(slsk_track_meta: dict, spotify_tracks: list) -> dict: """ Intelligently matches a Soulseek track to a track from the official Spotify tracklist using track numbers and title similarity. Returns the matched Spotify track data. """ if not spotify_tracks: return slsk_track_meta # Return original if no list to match against # Priority 1: Match by track number if slsk_track_meta.get('track_number'): track_num = slsk_track_meta['track_number'] for sp_track in spotify_tracks: if sp_track.get('track_number') == track_num: logger.info(f"Matched track by number ({track_num}): '{slsk_track_meta['title']}' -> '{sp_track['name']}'") # Return a new dict with the corrected title and number return { 'title': sp_track['name'], 'artist': slsk_track_meta.get('artist'), 'album': slsk_track_meta.get('album'), 'track_number': sp_track['track_number'], 'disc_number': sp_track.get('disc_number', 1), 'explicit': sp_track.get('explicit', False) } # Priority 2: Match by title similarity (if track number fails) best_match = None best_score = 0.6 # Require a decent similarity for sp_track in spotify_tracks: score = matching_engine.similarity_score( matching_engine.normalize_string(slsk_track_meta.get('title', '')), matching_engine.normalize_string(sp_track.get('name', '')) ) if score > best_score: best_score = score best_match = sp_track if best_match: logger.info(f"Matched track by title similarity ({best_score:.2f}): '{slsk_track_meta['title']}' -> '{best_match['name']}'") return { 'title': best_match['name'], 'artist': slsk_track_meta.get('artist'), 'album': slsk_track_meta.get('album'), 'track_number': best_match['track_number'], 'disc_number': best_match.get('disc_number', 1), 'explicit': best_match.get('explicit', False) } logger.error(f"Could not confidently match track '{slsk_track_meta['title']}'. Using original metadata.") return slsk_track_meta # Fallback to original def _post_process_matched_download_with_verification(context_key, context, file_path, task_id, batch_id): """ NEW VERIFICATION WORKFLOW: Enhanced post-processing with file verification. Only sets task status to 'completed' after successful file verification and move operation. """ from core.imports.pipeline import post_process_matched_download_with_verification return post_process_matched_download_with_verification( context_key, context, file_path, task_id, batch_id, _build_import_pipeline_runtime(), _build_metadata_enrichment_runtime(), ) def _safe_move_file(src, dst): """ Safely move a file across different filesystems/volumes. Handles Docker volume mount issues where shutil.move() fails on metadata preservation. Args: src: Source file path (str or Path) dst: Destination file path (str or Path) """ import shutil from pathlib import Path src = Path(src) dst = Path(dst) # Ensure parent directory exists dst.parent.mkdir(parents=True, exist_ok=True) # If source doesn't exist, check if it was already moved to destination # This happens when a retry or parallel thread already transferred the file if not src.exists(): if dst.exists(): logger.info(f"Source gone but destination exists, file already transferred: {dst.name}") return else: raise FileNotFoundError(f"Source file not found and destination does not exist: {src}") # On Windows, shutil.move fails with FileExistsError if destination exists. # Remove it first, retrying briefly for Windows file locks (e.g. Plex scanning). if dst.exists(): for _attempt in range(3): try: dst.unlink() break except PermissionError: if _attempt < 2: time.sleep(1) else: logger.warning(f"Could not remove locked destination after 3 attempts: {dst.name}") except Exception: break try: # Try standard move first (works if same filesystem) shutil.move(str(src), str(dst)) return except FileNotFoundError: # Source vanished between our exists() check and the move - another thread got it first # If destination now exists, the other thread completed the transfer successfully if dst.exists(): logger.info(f"Source moved by another thread, destination exists: {dst.name}") return raise except (OSError, PermissionError) as e: error_msg = str(e).lower() # shutil.move may have already copied the file successfully but failed # to delete the source (e.g. permission denied on slskd-owned downloads). # If destination exists with content, treat as success. if dst.exists() and dst.stat().st_size > 0: logger.warning(f"Move raised {type(e).__name__} but destination exists, treating as success: {e}") # Try to clean up source, but don't fail if we can't try: src.unlink() except Exception: logger.info(f"Could not delete source file (may be owned by another process): {src}") return # Cross-device link error — do manual binary copy if "cross-device" in error_msg or "operation not permitted" in error_msg or "permission denied" in error_msg: logger.warning(f"Cross-device move detected, using fallback copy method: {e}") try: # Simple copy without metadata preservation (avoids permission errors) with open(src, 'rb') as f_src: with open(dst, 'wb') as f_dst: shutil.copyfileobj(f_src, f_dst) f_dst.flush() os.fsync(f_dst.fileno()) # Delete source after successful copy try: src.unlink() except PermissionError: logger.info(f"Could not delete source file (may be owned by another process): {src}") logger.info(f"Successfully moved file using fallback method: {src} -> {dst}") return except Exception as fallback_error: logger.error(f"Fallback copy also failed: {fallback_error}") raise else: # Re-raise if it's a different error raise def _post_process_matched_download(context_key, context, file_path): """ This is the final, corrected post-processing function. It now mirrors the GUI's logic by trusting the pre-matched context for album downloads, which solves the track numbering issue. Also handles simple downloads (from search page "Download" button) which just move files to /Transfer without metadata enhancement. """ from core.imports.pipeline import post_process_matched_download return post_process_matched_download( context_key, context, file_path, _build_import_pipeline_runtime(), metadata_runtime=_build_metadata_enrichment_runtime(), ) # Track stale transfer keys (completed in slskd but no context — e.g., from before app restart) # so we only log the warning once per key instead of spamming every poll cycle _stale_transfer_keys = set() # --- File Discovery Retry System --- # Prevents race condition where slskd reports completion before file is written to disk # Tracks retry attempts per download to give files time to finish writing _download_retry_attempts = {} # {context_key: {'count': N, 'first_attempt': timestamp}} _download_retry_max = 10 # Max retries before giving up (10 seconds with 1s poll interval) _download_retry_lock = threading.Lock() def _execute_retag(group_id, album_id): """Execute a retag operation: re-tag files in a group with metadata from a new album match.""" global retag_state from database.music_database import get_database try: with retag_lock: retag_state.update({ "status": "running", "phase": "Fetching album metadata...", "progress": 0, "current_track": "", "total_tracks": 0, "processed": 0, "error_message": "" }) # 1. Fetch new album metadata from Spotify/iTunes album_data = spotify_client.get_album(album_id) if not album_data: raise ValueError(f"Could not fetch album data for ID: {album_id}") album_tracks_response = spotify_client.get_album_tracks(album_id) if not album_tracks_response: raise ValueError(f"Could not fetch album tracks for ID: {album_id}") album_tracks_items = album_tracks_response.get('items', []) # Extract artist info album_artists = album_data.get('artists', []) new_artist = album_artists[0] if album_artists else {'name': 'Unknown Artist', 'id': ''} # Ensure artist is a dict with expected fields if not isinstance(new_artist, dict): new_artist = {'name': str(new_artist), 'id': ''} new_album_name = album_data.get('name', 'Unknown Album') new_images = album_data.get('images', []) new_image_url = new_images[0]['url'] if new_images else None new_release_date = album_data.get('release_date', '') total_tracks = album_data.get('total_tracks', len(album_tracks_items)) # Build spotify track list spotify_tracks = [] for item in album_tracks_items: track_artists = item.get('artists', []) spotify_tracks.append({ 'name': item.get('name', ''), 'track_number': item.get('track_number', 1), 'disc_number': item.get('disc_number', 1), 'id': item.get('id', ''), 'artists': track_artists, 'duration_ms': item.get('duration_ms', 0) }) total_discs = max((t['disc_number'] for t in spotify_tracks), default=1) # 2. Load existing tracks for this group db = get_database() existing_tracks = db.get_retag_tracks(group_id) if not existing_tracks: raise ValueError(f"No tracks found for retag group {group_id}") with retag_lock: retag_state['total_tracks'] = len(existing_tracks) retag_state['phase'] = "Matching tracks..." # 3. Match existing files to new tracklist matched_pairs = [] for existing_track in existing_tracks: best_match = None best_score = 0 # Priority 1: Match by track number for st in spotify_tracks: if (st['track_number'] == existing_track.get('track_number') and st['disc_number'] == existing_track.get('disc_number', 1)): best_match = st best_score = 1.0 break # Priority 2: Match by title similarity if not best_match: from difflib import SequenceMatcher existing_title = (existing_track.get('title') or '').lower().strip() for st in spotify_tracks: st_title = (st.get('name') or '').lower().strip() score = SequenceMatcher(None, existing_title, st_title).ratio() if score > best_score and score > 0.6: best_score = score best_match = st if best_match: matched_pairs.append((existing_track, best_match)) else: logger.warning(f"[Retag] No match found for track: '{existing_track.get('title')}'") matched_pairs.append((existing_track, None)) with retag_lock: retag_state['phase'] = "Retagging files..." # 4. Retag each matched track for existing_track, matched_spotify in matched_pairs: current_file_path = existing_track.get('file_path', '') track_title = matched_spotify['name'] if matched_spotify else existing_track.get('title', 'Unknown') with retag_lock: retag_state['current_track'] = track_title if not matched_spotify: with retag_lock: retag_state['processed'] += 1 retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100) continue # Verify file exists if not os.path.exists(current_file_path): logger.warning(f"[Retag] File not found, skipping: {current_file_path}") with retag_lock: retag_state['processed'] += 1 retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100) continue # Build synthetic context for _enhance_file_metadata track_artists = matched_spotify.get('artists', []) context = { 'original_search_result': { 'spotify_clean_title': matched_spotify['name'], 'spotify_clean_album': new_album_name, 'track_number': matched_spotify['track_number'], 'disc_number': matched_spotify.get('disc_number', 1), 'artists': track_artists, 'title': matched_spotify['name'] }, 'spotify_album': { 'id': album_id, 'name': new_album_name, 'release_date': new_release_date, 'total_tracks': total_tracks, 'image_url': new_image_url, 'total_discs': total_discs }, 'track_info': {'id': matched_spotify['id']}, 'spotify_artist': new_artist, '_audio_quality': _get_audio_quality_string(current_file_path) or '' } album_info = { 'is_album': total_tracks > 1, 'album_name': new_album_name, 'track_number': matched_spotify['track_number'], 'disc_number': matched_spotify.get('disc_number', 1), 'clean_track_name': matched_spotify['name'], 'album_image_url': new_image_url } # Re-write metadata tags try: _enhance_file_metadata(current_file_path, context, new_artist, album_info) logger.info(f"[Retag] Re-tagged: '{track_title}'") except Exception as meta_err: logger.error(f"[Retag] Metadata write failed for '{track_title}': {meta_err}") # Compute new path and move if different file_ext = os.path.splitext(current_file_path)[1] try: new_path, _ = _build_final_path_for_track(context, new_artist, album_info, file_ext) if os.path.normpath(current_file_path) != os.path.normpath(new_path): logger.info(f"[Retag] Moving '{os.path.basename(current_file_path)}' -> '{new_path}'") old_dir = os.path.dirname(current_file_path) os.makedirs(os.path.dirname(new_path), exist_ok=True) _safe_move_file(current_file_path, new_path) # Move lyrics sidecar file alongside audio file if it exists for lyrics_ext in ('.lrc', '.txt'): old_lyrics = os.path.splitext(current_file_path)[0] + lyrics_ext if os.path.exists(old_lyrics): new_lyrics = os.path.splitext(new_path)[0] + lyrics_ext try: _safe_move_file(old_lyrics, new_lyrics) logger.info(f"[Retag] Moved {lyrics_ext} file alongside audio") except Exception as lrc_err: logger.error(f"[Retag] Failed to move {lyrics_ext} file: {lrc_err}") # Remove old cover.jpg if directory changed and old dir is now empty of audio new_dir = os.path.dirname(new_path) if os.path.normpath(old_dir) != os.path.normpath(new_dir): old_cover = os.path.join(old_dir, 'cover.jpg') if os.path.exists(old_cover): # Check if any audio files remain in old directory audio_exts = {'.flac', '.mp3', '.m4a', '.ogg', '.opus', '.wav', '.aac'} remaining_audio = [f for f in os.listdir(old_dir) if os.path.splitext(f)[1].lower() in audio_exts] if not remaining_audio: try: os.remove(old_cover) logger.warning("[Retag] Removed orphaned cover.jpg from old directory") except Exception: pass # Cleanup old empty directories transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) _cleanup_empty_directories(transfer_dir, current_file_path) # Update DB record db.update_retag_track_path(existing_track['id'], str(new_path)) current_file_path = new_path else: logger.warning(f"[Retag] Path unchanged for '{track_title}', no move needed") except Exception as move_err: logger.error(f"[Retag] Path/move failed for '{track_title}': {move_err}") # Download cover art to album directory try: _download_cover_art(album_info, os.path.dirname(current_file_path), context) except Exception as cover_err: logger.error(f"[Retag] Cover art download failed: {cover_err}") with retag_lock: retag_state['processed'] += 1 retag_state['progress'] = int(retag_state['processed'] / retag_state['total_tracks'] * 100) # 5. Update the retag group record with new metadata update_kwargs = { 'artist_name': new_artist.get('name', 'Unknown Artist'), 'album_name': new_album_name, 'image_url': new_image_url, 'total_tracks': total_tracks, 'release_date': new_release_date } # Set the correct ID field based on Spotify vs iTunes if str(album_id).isdigit(): update_kwargs['itunes_album_id'] = album_id update_kwargs['spotify_album_id'] = None else: update_kwargs['spotify_album_id'] = album_id update_kwargs['itunes_album_id'] = None db.update_retag_group(group_id, **update_kwargs) with retag_lock: retag_state.update({ "status": "finished", "phase": "Retag complete!", "progress": 100, "current_track": "" }) logger.info(f"[Retag] Retag operation complete for group {group_id}") except Exception as e: import traceback logger.error(f"[Retag] Error during retag: {e}") logger.error(traceback.format_exc()) with retag_lock: retag_state.update({ "status": "error", "phase": "Error", "error_message": str(e) }) def _check_and_remove_from_wishlist(context): """ Check if a successfully downloaded track should be removed from wishlist. Extracts Spotify track data from download context and removes from wishlist if found. """ try: from core.wishlist_service import get_wishlist_service from core.imports.context import get_import_source, get_import_source_ids wishlist_service = get_wishlist_service() # Try to extract a source-aware track ID from the context spotify_track_id = None # Populated lazily by Method 3 or Method 4. Initialized here so Method 4's # `if not wishlist_tracks` guard doesn't UnboundLocalError when Methods 1/2 # found nothing and Method 3 never ran (no wishlist_id in track_info). wishlist_tracks = [] # Method 1: Source-specific track lookup from track_info / source_ids track_info = context.get('track_info', {}) source = get_import_source(context) source_ids = get_import_source_ids(context) source_label = { 'spotify': 'Spotify', 'itunes': 'iTunes', 'deezer': 'Deezer', 'discogs': 'Discogs', 'hydrabase': 'Hydrabase', }.get(source, 'Source') if source == 'spotify' and source_ids.get('track_id'): spotify_track_id = source_ids['track_id'] logger.info(f"[Wishlist] Found {source_label} track ID from source_ids: {spotify_track_id}") # Method 2: Fallback to the original search result for source-specific IDs elif source == 'spotify' and context.get('original_search_result', {}).get('id'): spotify_track_id = context['original_search_result']['id'] logger.info(f"[Wishlist] Found {source_label} track ID from original_search_result: {spotify_track_id}") # Method 3: Check if this is a wishlist download (context has wishlist_id) elif 'wishlist_id' in track_info: wishlist_id = track_info['wishlist_id'] logger.info(f"[Wishlist] Found wishlist_id in context: {wishlist_id}") # Get the track ID from the wishlist entry (search all profiles) database = get_database() all_profiles = database.get_all_profiles() wishlist_tracks = [] for p in all_profiles: wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id'])) for wl_track in wishlist_tracks: if wl_track.get('wishlist_id') == wishlist_id: spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id') logger.info(f"[Wishlist] Found track ID from wishlist entry: {spotify_track_id}") break # Method 4: Try to construct a track ID from metadata for fuzzy matching if not spotify_track_id: track_name = track_info.get('name') or context.get('original_search_result', {}).get('title', '') artist_name = _get_track_artist_name(track_info) or _get_track_artist_name(context.get('original_search_result', {})) if track_name and artist_name: logger.warning(f"[Wishlist] No track ID found, checking for fuzzy match: '{track_name}' by '{artist_name}'") # Get all wishlist tracks and find potential matches (search all profiles) if not wishlist_tracks: database = get_database() all_profiles = database.get_all_profiles() wishlist_tracks = [] for p in all_profiles: wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id'])) for wl_track in wishlist_tracks: wl_name = wl_track.get('name', '').lower() wl_artists = wl_track.get('artists', []) wl_artist_name = '' # Extract artist name from wishlist track if wl_artists: if isinstance(wl_artists[0], dict): wl_artist_name = wl_artists[0].get('name', '').lower() else: wl_artist_name = str(wl_artists[0]).lower() # Simple fuzzy matching if (wl_name == track_name.lower() and wl_artist_name == artist_name.lower()): spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id') logger.info(f"[Wishlist] Found fuzzy match - track ID: {spotify_track_id}") break # If we found a track ID, remove it from wishlist if spotify_track_id: logger.info(f"[Wishlist] Attempting to remove track from wishlist: {spotify_track_id}") removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) if removed: logger.info(f"[Wishlist] Successfully removed track from wishlist: {spotify_track_id}") else: logger.warning(f"ℹ️ [Wishlist] Track not found in wishlist or already removed: {spotify_track_id}") else: logger.warning("ℹ️ [Wishlist] No track ID found for wishlist removal check") except Exception as e: logger.error(f"[Wishlist] Error in wishlist removal check: {e}") import traceback traceback.print_exc() def _check_and_remove_track_from_wishlist_by_metadata(track_data): """ Check if a track found during database analysis should be removed from wishlist. Uses track metadata (name, artists, id) to find and remove from wishlist. """ try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() # Extract track info track_name = track_data.get('name', '') track_id = track_data.get('id', '') artists = track_data.get('artists', []) logger.info(f"[Analysis] Checking if track should be removed from wishlist: '{track_name}' (ID: {track_id})") # Method 1: Direct Spotify ID match if track_id: removed = wishlist_service.mark_track_download_result(track_id, success=True) if removed: logger.info(f"[Analysis] Removed track from wishlist via direct ID match: {track_id}") return True # Method 2: Fuzzy matching by name and artist if no direct ID match if track_name and artists: # Extract primary artist name primary_artist = '' if isinstance(artists[0], dict) and 'name' in artists[0]: primary_artist = artists[0]['name'] elif isinstance(artists[0], str): primary_artist = artists[0] else: primary_artist = str(artists[0]) logger.warning(f"[Analysis] No direct ID match, trying fuzzy match: '{track_name}' by '{primary_artist}'") # Get all wishlist tracks and find matches (search all profiles) database = get_database() all_profiles = database.get_all_profiles() wishlist_tracks = [] for p in all_profiles: wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id'])) for wl_track in wishlist_tracks: wl_name = wl_track.get('name', '').lower() wl_artists = wl_track.get('artists', []) wl_artist_name = '' # Extract artist name from wishlist track if wl_artists: if isinstance(wl_artists[0], dict): wl_artist_name = wl_artists[0].get('name', '').lower() else: wl_artist_name = str(wl_artists[0]).lower() # Fuzzy matching - normalize strings for comparison if (wl_name == track_name.lower() and wl_artist_name == primary_artist.lower()): spotify_track_id = wl_track.get('spotify_track_id') or wl_track.get('id') if spotify_track_id: removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) if removed: logger.info(f"[Analysis] Removed track from wishlist via fuzzy match: {spotify_track_id}") return True logger.warning(f"ℹ️ [Analysis] Track not found in wishlist or already removed: '{track_name}'") return False except Exception as e: logger.error(f"[Analysis] Error checking wishlist removal by metadata: {e}") import traceback traceback.print_exc() return False # Auto-wishlist cleanup logic lives in core/downloads/cleanup.py. from core.downloads import cleanup as _downloads_cleanup def _automatic_wishlist_cleanup_after_db_update(): """Automatic wishlist cleanup that runs after database updates.""" _downloads_cleanup.cleanup_wishlist_after_db_update(config_manager) # ── Update detection ───────────────────────────────────────────── _GITHUB_REPO = "Nezreka/SoulSync" _update_cache = {'latest_sha': None, 'last_check': 0, 'error': None} _UPDATE_CHECK_INTERVAL = 3600 # 1 hour def _get_current_commit_sha(): """Get the commit SHA of the running instance (env var for Docker, git for local).""" # Docker: baked in at build time via COMMIT_SHA build arg sha = os.environ.get('SOULSYNC_COMMIT_SHA', '').strip() if sha: return sha # Local dev: read from git try: import subprocess result = subprocess.run(['git', 'rev-parse', 'HEAD'], capture_output=True, text=True, cwd=os.path.dirname(__file__) or '.') if result.returncode == 0: return result.stdout.strip() except Exception: pass return None _current_commit_sha = _get_current_commit_sha() def _check_for_updates(): """Check GitHub for the latest commit SHA on main branch.""" import time as _time now = _time.time() if now - _update_cache['last_check'] < _UPDATE_CHECK_INTERVAL: return # Still fresh _update_cache['last_check'] = now try: import urllib.request import json as _json req = urllib.request.Request( f"https://api.github.com/repos/{_GITHUB_REPO}/commits/main", headers={'Accept': 'application/vnd.github.v3+json', 'User-Agent': 'SoulSync-UpdateCheck'} ) with urllib.request.urlopen(req, timeout=10) as resp: data = _json.loads(resp.read().decode()) _update_cache['latest_sha'] = data.get('sha') _update_cache['error'] = None except Exception as e: _update_cache['error'] = str(e) logger.debug(f"Update check failed: {e}") @app.route('/api/update-check', methods=['GET']) def check_for_update(): """Check if a newer version is available on GitHub.""" _check_for_updates() current = _current_commit_sha latest = _update_cache.get('latest_sha') update_available = bool(current and latest and current != latest) return jsonify({ 'update_available': update_available, 'current_sha': current[:8] if current else None, 'latest_sha': latest[:8] if latest else None, 'is_docker': os.path.exists('/.dockerenv'), }) def _simple_monitor_task(): """The actual monitoring task that runs in the background thread. Search cleanup and download cleanup are now handled by system automations.""" logger.info("Simple background monitor started") while not globals().get('IS_SHUTTING_DOWN', False): try: with matched_context_lock: pending_count = len(matched_downloads_context) if pending_count > 0: # Use app_context to safely call endpoint logic from a thread with app.app_context(): get_download_status() # Cleanup stale retry attempts (older than 60 seconds) # This prevents memory leaks from stuck/failed downloads with _download_retry_lock: current_time = time.time() stale_keys = [ key for key, data in _download_retry_attempts.items() if current_time - data['first_attempt'] > 60 ] for key in stale_keys: logger.warning(f"Cleaning up stale retry attempt: {key}") del _download_retry_attempts[key] time.sleep(1) except Exception as e: logger.error(f"Simple monitor error: {e}") time.sleep(10) logger.info("Simple background monitor stopped") def start_simple_background_monitor(): """Starts the simple background monitor thread.""" monitor_thread = threading.Thread(target=_simple_monitor_task) monitor_thread.daemon = True monitor_thread.start() # =============================== # == AUTOMATIC WISHLIST PROCESSING == # =============================== def _sanitize_track_data_for_processing(track_data): """ Sanitizes track data from wishlist service to ensure consistent format. Preserves album dict to retain full metadata (images, id, etc.) and normalizes artist field. """ if not isinstance(track_data, dict): logger.info(f"[Sanitize] Unexpected track data type: {type(track_data)}") return track_data # Create a copy to avoid modifying original data sanitized = track_data.copy() # Handle album field - preserve dict format to retain full metadata (images, id, etc.) # Downstream code already handles both dict and string formats defensively raw_album = sanitized.get('album', '') if not isinstance(raw_album, (dict, str)): sanitized['album'] = str(raw_album) # Handle artists field - ensure it's a list of strings raw_artists = sanitized.get('artists', []) if isinstance(raw_artists, list): processed_artists = [] for artist in raw_artists: if isinstance(artist, str): processed_artists.append(artist) elif isinstance(artist, dict) and 'name' in artist: processed_artists.append(artist['name']) else: processed_artists.append(str(artist)) sanitized['artists'] = processed_artists else: logger.info(f"[Sanitize] Unexpected artists format: {type(raw_artists)}") sanitized['artists'] = [str(raw_artists)] if raw_artists else [] return sanitized def check_and_recover_stuck_flags(): """ Check if wishlist_auto_processing or watchlist_auto_scanning flags are stuck. If a flag has been True for more than 2 hours (7200 seconds), reset it. This prevents indefinite blocking when processes crash without cleanup. """ global wishlist_auto_processing, wishlist_auto_processing_timestamp global watchlist_auto_scanning, watchlist_auto_scanning_timestamp import time current_time = time.time() stuck_timeout = 900 # 15 minutes in seconds (reduced from 2 hours for faster recovery) # Check wishlist flag if wishlist_auto_processing: time_stuck = current_time - wishlist_auto_processing_timestamp if time_stuck > stuck_timeout: stuck_minutes = time_stuck / 60 logger.info(f"[Stuck Detection] Wishlist auto-processing flag has been stuck for {stuck_minutes:.1f} minutes - RESETTING") with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return True # Check watchlist flag if watchlist_auto_scanning: time_stuck = current_time - watchlist_auto_scanning_timestamp if time_stuck > stuck_timeout: stuck_minutes = time_stuck / 60 logger.info(f"[Stuck Detection] Watchlist auto-scanning flag has been stuck for {stuck_minutes:.1f} minutes - RESETTING") with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 return True return False def is_wishlist_actually_processing(): """ Check if wishlist is truly processing (not just flag stuck). Returns True only if flag is set AND timestamp is recent (< 15 minutes). """ global wishlist_auto_processing, wishlist_auto_processing_timestamp if not wishlist_auto_processing: return False import time current_time = time.time() time_since_start = current_time - wishlist_auto_processing_timestamp # If more than 15 minutes, flag is stuck - auto-recover and return False if time_since_start > 900: # 15 minutes stuck_minutes = time_since_start / 60 logger.warning(f"[Stuck Detection] Wishlist flag stuck for {stuck_minutes:.1f} minutes - auto-recovering") check_and_recover_stuck_flags() return False return True def is_watchlist_actually_scanning(): """ Check if watchlist is truly scanning (not just flag stuck). Returns True only if flag is set AND timestamp is recent (< 15 minutes). """ global watchlist_auto_scanning, watchlist_auto_scanning_timestamp if not watchlist_auto_scanning: return False import time current_time = time.time() time_since_start = current_time - watchlist_auto_scanning_timestamp # If more than 15 minutes, flag is stuck - auto-recover and return False if time_since_start > 900: # 15 minutes stuck_minutes = time_since_start / 60 logger.warning(f"[Stuck Detection] Watchlist flag stuck for {stuck_minutes:.1f} minutes - auto-recovering") check_and_recover_stuck_flags() return False return True def _classify_wishlist_track(track): """Classify a wishlist track as 'singles' or 'albums'. Uses Spotify's album_type as the primary signal (most authoritative), falls back to total_tracks heuristic, defaults to 'albums'. Returns: 'singles' or 'albums' """ spotify_data = track.get('spotify_data', {}) if isinstance(spotify_data, str): try: import json spotify_data = json.loads(spotify_data) except Exception: spotify_data = {} album_data = spotify_data.get('album') or {} if not isinstance(album_data, dict): album_data = {} total_tracks = album_data.get('total_tracks') album_type = album_data.get('album_type', '').lower() # Prioritize Spotify's album_type classification (most accurate) if album_type in ('single', 'ep'): return 'singles' if album_type in ('album', 'compilation'): return 'albums' # Fallback: track count heuristic if total_tracks is not None and total_tracks > 0: return 'singles' if total_tracks < 6 else 'albums' # No classification data — default to albums return 'albums' def _process_wishlist_automatically(automation_id=None): """Main automatic processing logic that runs in background thread.""" global wishlist_auto_processing, wishlist_auto_processing_timestamp logger.info("[Auto-Wishlist] Timer triggered - starting automatic wishlist processing...") try: # CRITICAL FIX: Use smart stuck detection BEFORE acquiring lock # This prevents deadlock and handles stuck flags (2-hour timeout) if is_wishlist_actually_processing(): logger.info("[Auto-Wishlist] Already processing (verified with stuck detection), skipping.") return # Check conditions and set flag should_skip_already_running = False with wishlist_timer_lock: # Re-check inside lock to handle race conditions if wishlist_auto_processing: logger.info("[Auto-Wishlist] Already processing (race condition check), skipping.") should_skip_already_running = True else: # Set flag and timestamp import time wishlist_auto_processing = True wishlist_auto_processing_timestamp = time.time() logger.info(f"[Auto-Wishlist] Flag set at timestamp {wishlist_auto_processing_timestamp}") if should_skip_already_running: return # Use app context for database operations with app.app_context(): from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() # Check if wishlist has tracks across all profiles database = get_database() all_profiles = database.get_all_profiles() count = sum(wishlist_service.get_wishlist_count(profile_id=p['id']) for p in all_profiles) logger.info(f"[Auto-Wishlist] Wishlist count check: {count} tracks found across {len(all_profiles)} profiles") _update_automation_progress(automation_id, progress=10, phase='Checking wishlist', log_line=f'{count} tracks across {len(all_profiles)} profiles', log_type='info') if count == 0: logger.warning("ℹ️ [Auto-Wishlist] No tracks in wishlist for auto-processing.") with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return logger.info(f"[Auto-Wishlist] Found {count} tracks in wishlist, starting automatic processing...") # Check if wishlist processing is already active (auto or manual) playlist_id = "wishlist" with tasks_lock: for _batch_id, batch_data in download_batches.items(): batch_playlist_id = batch_data.get('playlist_id') # Check for both auto ('wishlist') and manual ('wishlist_manual') batches if (batch_playlist_id in ['wishlist', 'wishlist_manual'] and batch_data.get('phase') not in ['complete', 'error', 'cancelled']): logger.info(f"Wishlist processing already active in another batch ({batch_playlist_id}), skipping automatic start") with wishlist_timer_lock: wishlist_auto_processing = False return # CRITICAL: Clean duplicates BEFORE fetching tracks to prevent count mismatches # This prevents the "11 tracks shown but 12 counted" bug from database.music_database import MusicDatabase db = MusicDatabase() logger.warning("[Auto-Wishlist] Cleaning duplicate tracks before processing...") for p in all_profiles: duplicates_removed = db.remove_wishlist_duplicates(profile_id=p['id']) if duplicates_removed > 0: logger.warning(f"[Auto-Wishlist] Removed {duplicates_removed} duplicate tracks from profile {p['id']}") # CLEANUP: Remove tracks from wishlist that already exist in library # This prevents wasting bandwidth on tracks we already have logger.debug("[Auto-Wishlist] Checking wishlist against library for already-owned tracks...") active_server = config_manager.get_active_media_server() cleanup_tracks = [] for p in all_profiles: cleanup_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id'])) cleanup_removed = 0 for track in cleanup_tracks: track_name = track.get('name', '') artists = track.get('artists', []) spotify_track_id = track.get('spotify_track_id') or track.get('id') track_album = track.get('album', {}).get('name') if isinstance(track.get('album'), dict) else track.get('album') if not track_name or not artists or not spotify_track_id: continue # Check if track exists in library found_in_db = False for artist in artists: if isinstance(artist, str): artist_name = artist elif isinstance(artist, dict) and 'name' in artist: artist_name = artist['name'] else: artist_name = str(artist) try: db_track, confidence = db.check_track_exists( track_name, artist_name, confidence_threshold=0.7, server_source=active_server, album=track_album ) if db_track and confidence >= 0.7: found_in_db = True break except Exception as db_error: continue # Remove from wishlist if found in library if found_in_db: try: removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) if removed: cleanup_removed += 1 logger.info(f"[Auto-Wishlist] Removed already-owned track: '{track_name}' by {artist_name}") except Exception as remove_error: logger.error(f"[Auto-Wishlist] Error removing track from wishlist: {remove_error}") if cleanup_removed > 0: logger.info(f"[Auto-Wishlist] Cleaned up {cleanup_removed} already-owned tracks from wishlist") _update_automation_progress(automation_id, progress=25, phase='Cleaned up duplicates', log_line=f'Removed {cleanup_removed} already-owned tracks', log_type='success') else: _update_automation_progress(automation_id, progress=25, phase='Cleanup done', log_line='No duplicates or already-owned tracks found', log_type='skip') # Get wishlist tracks for processing (after cleanup) - combine all profiles raw_wishlist_tracks = [] for p in all_profiles: raw_wishlist_tracks.extend(wishlist_service.get_wishlist_tracks_for_download(profile_id=p['id'])) if not raw_wishlist_tracks: logger.warning("No tracks returned from wishlist service.") with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return # SANITIZE: Ensure consistent data format from wishlist service wishlist_tracks = [] seen_track_ids_sanitation = set() # Deduplicate during sanitization duplicates_found = 0 for track in raw_wishlist_tracks: sanitized_track = _sanitize_track_data_for_processing(track) spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id') # Skip duplicates during sanitization if spotify_track_id and spotify_track_id in seen_track_ids_sanitation: duplicates_found += 1 continue wishlist_tracks.append(sanitized_track) if spotify_track_id: seen_track_ids_sanitation.add(spotify_track_id) if duplicates_found > 0: logger.warning(f"[Auto-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization") logger.info(f"[Auto-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service") # CYCLE FILTERING: Get current cycle and filter tracks by category with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'") row = cursor.fetchone() if row: current_cycle = row['value'] else: # Default to albums on first run current_cycle = 'albums' cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('wishlist_cycle', 'albums', CURRENT_TIMESTAMP) """) conn.commit() # Filter tracks by current cycle category filtered_tracks = [] seen_track_ids_filtering = set() # Deduplicate during filtering for track in wishlist_tracks: track_category = _classify_wishlist_track(track) spotify_track_id = track.get('spotify_track_id') or track.get('id') matches_category = (current_cycle == track_category) # Only add if matches category AND not a duplicate if matches_category: # Only deduplicate if track has a valid ID if spotify_track_id: if spotify_track_id not in seen_track_ids_filtering: filtered_tracks.append(track) seen_track_ids_filtering.add(spotify_track_id) else: # No ID - can't deduplicate safely, always add filtered_tracks.append(track) logger.info(f"[Auto-Wishlist] Current cycle: {current_cycle}") logger.info(f"[Auto-Wishlist] Filtered {len(filtered_tracks)}/{len(wishlist_tracks)} tracks for '{current_cycle}' category") _update_automation_progress(automation_id, progress=40, phase=f'Processing {current_cycle}', log_line=f'Cycle: {current_cycle} — {len(filtered_tracks)} tracks to process', log_type='info') # If no tracks in this category, skip to next cycle immediately if len(filtered_tracks) == 0: logger.warning(f"ℹ️ [Auto-Wishlist] No {current_cycle} tracks in wishlist, toggling cycle and scheduling next run") # Toggle cycle next_cycle = 'singles' if current_cycle == 'albums' else 'albums' with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP) """, (next_cycle,)) conn.commit() logger.info(f"[Auto-Wishlist] Cycle toggled: {current_cycle} → {next_cycle}") with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return # Use filtered tracks for processing — stamp original index wishlist_tracks = filtered_tracks for i, track in enumerate(wishlist_tracks): track['_original_index'] = i # Create batch for automatic processing batch_id = str(uuid.uuid4()) playlist_name = f"Wishlist (Auto - {current_cycle.capitalize()})" # Create task queue - convert wishlist tracks to expected format with tasks_lock: download_batches[batch_id] = { 'phase': 'analysis', 'playlist_id': playlist_id, 'playlist_name': playlist_name, 'queue': [], 'active_count': 0, 'max_concurrent': _get_batch_max_concurrent(), # Wishlist always does single-track downloads, not folder grabs 'queue_index': 0, 'analysis_total': len(wishlist_tracks), 'analysis_processed': 0, 'analysis_results': [], # Track state management (replicating sync.py) 'permanently_failed_tracks': [], 'cancelled_tracks': set(), # Wishlist tracks are already known-missing — skip the expensive library check 'force_download_all': True, # Mark as auto-initiated 'auto_initiated': True, 'auto_processing_timestamp': time.time(), # Store current cycle for toggling after completion 'current_cycle': current_cycle, # Profile context for failed track wishlist re-adds (auto = profile 1 default) 'profile_id': 1 } logger.info(f"Starting automatic wishlist batch {batch_id} with {len(wishlist_tracks)} tracks") _update_automation_progress(automation_id, progress=50, phase=f'Downloading {len(wishlist_tracks)} tracks', log_line=f'Started batch: {len(wishlist_tracks)} {current_cycle}', log_type='success') # Submit the wishlist processing job using existing infrastructure missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks) # Don't mark auto_processing as False here - let completion handler do it except Exception as e: logger.error(f"Error in automatic wishlist processing: {e}") import traceback traceback.print_exc() _update_automation_progress(automation_id, log_line=f'Error: {str(e)}', log_type='error') with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 raise # re-raise so automation wrapper returns error status # =============================== # == DATABASE UPDATER API == # =============================== def _db_update_progress_callback(current_item, processed, total, percentage): logger.info(f"[DB Progress] {current_item} - {processed}/{total} ({percentage:.1f}%)") with db_update_lock: db_update_state.update({ "current_item": current_item, "processed": processed, "total": total, "progress": percentage }) _update_automation_progress(_db_update_automation_id, progress=percentage, processed=processed, total=total, current_item=current_item) def _db_update_phase_callback(phase): logger.info(f"[DB Phase] {phase}") with db_update_lock: db_update_state["phase"] = phase _update_automation_progress(_db_update_automation_id, phase=phase) def _db_update_artist_callback(artist_name, success, details, album_count, track_count): if success: # Use the details string from the worker — it includes context like "0 new tracks (150 existing updated)" log_msg = f'{artist_name} — {details}' if details else f'{artist_name} — {album_count} albums, {track_count} tracks' _update_automation_progress(_db_update_automation_id, log_line=log_msg, log_type='success') else: _update_automation_progress(_db_update_automation_id, log_line=f'{artist_name} — {details}', log_type='error') def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed): global _db_update_automation_id # Check for removal results from the worker removed_artists = 0 removed_albums = 0 removed_tracks = 0 if db_update_worker: removed_artists = getattr(db_update_worker, 'removed_artists', 0) removed_albums = getattr(db_update_worker, 'removed_albums', 0) removed_tracks = getattr(db_update_worker, 'removed_tracks', 0) removal_msg = "" if removed_artists > 0 or removed_albums > 0: removal_msg = f" | Removed: {removed_artists} artists, {removed_albums} albums" if removed_tracks > 0: removal_msg += f", {removed_tracks} tracks" # Build a clear summary message # For deep scans: total_tracks = new tracks only, successful = artists processed # Include skipped/existing count when available for clarity skipped_tracks = 0 if db_update_worker: skipped_tracks = getattr(db_update_worker, '_total_skipped', 0) # Calculate from processed counts if not tracked directly if not skipped_tracks: total_processed = getattr(db_update_worker, 'processed_tracks', 0) if total_processed == 0 and total_tracks == 0 and successful > 0: # Deep scan with nothing new — show artists scanned skipped_tracks = getattr(db_update_worker, 'processed_albums', 0) if total_tracks > 0: phase_msg = f"Completed: {total_artists} artists, {total_albums} albums, {total_tracks} new tracks{removal_msg}." elif successful > 0: phase_msg = f"Completed: {successful} artists scanned, library up to date{removal_msg}." else: phase_msg = f"Completed: {successful} successful, {failed} failed{removal_msg}." with db_update_lock: db_update_state["status"] = "finished" db_update_state["phase"] = phase_msg db_update_state["total_albums"] = total_albums db_update_state["total_tracks"] = total_tracks db_update_state["removed_artists"] = removed_artists db_update_state["removed_albums"] = removed_albums db_update_state["removed_tracks"] = removed_tracks # Finalize automation progress auto_summary = f"{total_tracks} tracks, {total_albums} albums from {total_artists} artists" if removed_artists > 0 or removed_albums > 0: auto_summary += f" | Removed {removed_artists} artists, {removed_albums} albums" _update_automation_progress(_db_update_automation_id, status='finished', progress=100, phase='Complete', log_line=auto_summary, log_type='success') _db_update_automation_id = None # Resume enrichment workers now that scan is done _resume_workers_after_scan() # Add activity for database update completion summary = f"{total_tracks} tracks, {total_albums} albums, {total_artists} artists processed" if removed_artists > 0 or removed_albums > 0: summary += f" | {removed_artists} artists, {removed_albums} albums removed" add_activity_item("", "Database Update Complete", summary, "Now") try: if automation_engine: automation_engine.emit('database_update_completed', { 'total_artists': str(total_artists), 'total_albums': str(total_albums), 'total_tracks': str(total_tracks), }) except Exception: pass # Invalidate sync match cache (track IDs may have changed) try: inv_db = get_database() cleared = inv_db.invalidate_sync_match_cache() if cleared: logger.info(f"Cleared {cleared} sync match cache entries after database update") except Exception: pass # WISHLIST CLEANUP: Automatically clean up wishlist after database update try: logger.info("[DB Update] Database update completed, starting automatic wishlist cleanup...") # Run cleanup in background to avoid blocking the UI missing_download_executor.submit(_automatic_wishlist_cleanup_after_db_update) except Exception as cleanup_error: logger.error(f"[DB Update] Error starting automatic wishlist cleanup: {cleanup_error}") def _db_update_error_callback(error_message): global _db_update_automation_id with db_update_lock: db_update_state["status"] = "error" db_update_state["error_message"] = error_message # Resume enrichment workers even on error _resume_workers_after_scan() _update_automation_progress(_db_update_automation_id, status='error', phase='Error', log_line=error_message, log_type='error') _db_update_automation_id = None # Add activity for database update error add_activity_item("", "Database Update Failed", error_message, "Now") _workers_paused_by_scan = set() # Track which workers WE paused (don't resume manually-paused ones) def _pause_workers_for_scan(): """Pause all enrichment and maintenance workers during database scans to reduce lock contention.""" global _workers_paused_by_scan _workers_paused_by_scan = set() workers = { 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, 'repair': repair_worker, 'soulid': soulid_worker, } for name, w in workers.items(): if w and hasattr(w, 'pause') and not getattr(w, 'paused', True): w.pause() _workers_paused_by_scan.add(name) if _workers_paused_by_scan: logger.warning(f"Paused {len(_workers_paused_by_scan)} workers during database scan: {', '.join(_workers_paused_by_scan)}") def _resume_workers_after_scan(): """Resume only the workers that WE paused (don't resume manually-paused ones).""" global _workers_paused_by_scan workers = { 'mb': mb_worker, 'spotify': spotify_enrichment_worker, 'itunes': itunes_enrichment_worker, 'deezer': deezer_worker, 'audiodb': audiodb_worker, 'discogs': discogs_worker, 'lastfm': lastfm_worker, 'genius': genius_worker, 'tidal': tidal_enrichment_worker, 'qobuz': qobuz_enrichment_worker, 'repair': repair_worker, 'soulid': soulid_worker, } resumed = 0 for name, w in workers.items(): if name in _workers_paused_by_scan and w and hasattr(w, 'resume'): w.resume() resumed += 1 if resumed: logger.info(f"Resumed {resumed} workers after database scan") _workers_paused_by_scan = set() def _run_soulsync_full_refresh(): """Full refresh for SoulSync standalone — wipe all soulsync records, re-scan output folder, rebuild library from file tags.""" try: from core.soulsync_client import _read_tags, _stable_id transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) if not os.path.isdir(transfer_path): _db_update_error_callback(f"Output folder not found: {transfer_path}") return logger.info(f"[SoulSync Full Refresh] Starting — clearing soulsync data, re-scanning: {transfer_path}") _db_update_phase_callback('Clearing library...') db = get_database() db.clear_server_data('soulsync') # Collect all audio files _db_update_phase_callback('Scanning output folder...') audio_exts = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} audio_files = [] for root, _dirs, files in os.walk(transfer_path): for fname in files: if os.path.splitext(fname)[1].lower() in audio_exts: audio_files.append(os.path.join(root, fname)) total = len(audio_files) logger.info(f"[SoulSync Full Refresh] Found {total} audio files, rebuilding library...") if total == 0: _db_update_finished_callback(0, 0, 0, 0, 0) return # Group files by artist → album using tags _db_update_phase_callback(f'Reading tags from {total} files...') artists_map = {} # artist_name → { albums_map: { album_name → [tracks] } } processed = 0 for file_path in audio_files: tags = _read_tags(file_path) artist_name = tags.get('album_artist') or tags.get('artist') or 'Unknown Artist' album_name = tags.get('album') or 'Unknown Album' if artist_name not in artists_map: artists_map[artist_name] = {} if album_name not in artists_map[artist_name]: artists_map[artist_name][album_name] = [] artists_map[artist_name][album_name].append((file_path, tags)) processed += 1 if processed % 50 == 0: _db_update_phase_callback(f'Reading tags... {processed}/{total}') # Write to DB _db_update_phase_callback('Writing to database...') successful = 0 failed = 0 try: with db._get_connection() as conn: cursor = conn.cursor() for artist_name, albums in artists_map.items(): artist_id = _stable_id(artist_name.lower()) + '::soulsync' # Insert artist try: cursor.execute(""" INSERT OR IGNORE INTO artists (id, name, server_source, created_at, updated_at) VALUES (?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (artist_id, artist_name)) except Exception: pass for album_name, tracks in albums.items(): album_key = f"{artist_name.lower()}::{album_name.lower()}" album_id = _stable_id(album_key) + '::soulsync' # Get year from first track with a year year = '' for _, t in tracks: if t.get('year'): year = t['year'] break # Insert album try: cursor.execute(""" INSERT OR IGNORE INTO albums (id, artist_id, title, year, track_count, server_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (album_id, artist_id, album_name, year, len(tracks))) except Exception: pass # Insert tracks for file_path, tags in tracks: track_id = _stable_id(file_path) + '::soulsync' try: cursor.execute(""" INSERT OR IGNORE INTO tracks (id, album_id, artist_id, title, track_number, disc_number, duration, file_path, bitrate, year, server_source, created_at, updated_at) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, 'soulsync', CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) """, (track_id, album_id, artist_id, tags['title'], tags['track_number'], tags['disc_number'], tags['duration_ms'], file_path, tags['bitrate'], tags.get('year', ''))) successful += 1 except Exception as e: failed += 1 logger.error(f"[SoulSync Full Refresh] Track insert error: {e}") conn.commit() except Exception as e: logger.error(f"[SoulSync Full Refresh] DB error: {e}") _db_update_error_callback(f"Database error: {e}") return artist_count = len(artists_map) album_count = sum(len(albums) for albums in artists_map.values()) summary = f"Full refresh complete: {successful} tracks from {album_count} albums by {artist_count} artists" if failed > 0: summary += f" ({failed} failed)" logger.info(f"[SoulSync Full Refresh] {summary}") add_activity_item("", "SoulSync Full Refresh", summary, "Now") _db_update_finished_callback(artist_count, album_count, total, successful, failed) except Exception as e: logger.error(f"[SoulSync Full Refresh] {e}") import traceback traceback.print_exc() _db_update_error_callback(f"Full refresh failed: {e}") def _run_soulsync_deep_scan(): """Deep scan for SoulSync standalone mode. 1. Scans the output folder for all audio files 2. Compares against soulsync DB records (by file_path) 3. Untracked files → moved to import folder for auto-import processing 4. Stale DB records (file gone) → removed from DB """ try: import shutil transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) staging_path = docker_resolve_path(config_manager.get('import.staging_path', './Staging')) if not os.path.isdir(transfer_path): _db_update_error_callback(f"Output folder not found: {transfer_path}") return logger.info(f"[SoulSync Deep Scan] Starting — Transfer: {transfer_path}") _db_update_phase_callback('scanning') # Phase 1: Collect all audio files in Transfer audio_extensions = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} transfer_files = set() for root, _dirs, files in os.walk(transfer_path): for filename in files: if os.path.splitext(filename)[1].lower() in audio_extensions: transfer_files.add(os.path.join(root, filename)) logger.info(f"[SoulSync Deep Scan] Found {len(transfer_files)} audio files in Transfer") # Phase 2: Get all soulsync file paths from DB db = get_database() db_paths = set() try: with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT file_path FROM tracks WHERE server_source = 'soulsync' AND file_path IS NOT NULL") for row in cursor.fetchall(): if row['file_path']: db_paths.add(row['file_path']) except Exception as e: logger.error(f"[SoulSync Deep Scan] Error reading DB paths: {e}") logger.info(f"[SoulSync Deep Scan] {len(db_paths)} tracks in soulsync DB") # Phase 3: Find untracked files (in Transfer but not in DB) untracked = transfer_files - db_paths # Also check with normalized paths (Windows vs Unix separators) if untracked: db_paths_normalized = {p.replace('\\', '/') for p in db_paths} untracked = {f for f in untracked if f.replace('\\', '/') not in db_paths_normalized} # Phase 4: Move untracked files to Staging for auto-import moved_count = 0 if untracked and os.path.isdir(staging_path): _db_update_phase_callback('moving_untracked') for file_path in untracked: try: # Preserve relative folder structure from Transfer rel_path = os.path.relpath(file_path, transfer_path) dest_path = os.path.join(staging_path, rel_path) os.makedirs(os.path.dirname(dest_path), exist_ok=True) shutil.move(file_path, dest_path) moved_count += 1 except Exception as e: logger.error(f"[SoulSync Deep Scan] Could not move {os.path.basename(file_path)}: {e}") # Clean up empty directories in Transfer after moving files for root, dirs, _files in os.walk(transfer_path, topdown=False): for d in dirs: dir_path = os.path.join(root, d) try: if not os.listdir(dir_path): os.rmdir(dir_path) except OSError: pass # Phase 5: Find stale DB records (in DB but file gone from disk) _db_update_phase_callback('cleanup') stale_count = 0 stale_track_ids = [] for db_path in db_paths: if not os.path.exists(db_path): stale_track_ids.append(db_path) stale_count += 1 # Remove stale records if stale_track_ids: try: with db._get_connection() as conn: cursor = conn.cursor() for fp in stale_track_ids: cursor.execute("DELETE FROM tracks WHERE file_path = ? AND server_source = 'soulsync'", (fp,)) conn.commit() # Clean up orphaned albums (no tracks left) cursor.execute(""" DELETE FROM albums WHERE server_source = 'soulsync' AND id NOT IN (SELECT DISTINCT album_id FROM tracks WHERE server_source = 'soulsync') """) orphan_albums = cursor.rowcount # Clean up orphaned artists (no albums left) cursor.execute(""" DELETE FROM artists WHERE server_source = 'soulsync' AND id NOT IN (SELECT DISTINCT artist_id FROM albums WHERE server_source = 'soulsync') """) orphan_artists = cursor.rowcount conn.commit() if orphan_albums > 0 or orphan_artists > 0: logger.warning(f"[SoulSync Deep Scan] Cleaned up {orphan_albums} orphaned albums, {orphan_artists} orphaned artists") except Exception as e: logger.error(f"[SoulSync Deep Scan] Error cleaning stale records: {e}") summary = f"Deep scan complete: {len(transfer_files)} files scanned" if moved_count > 0: summary += f", {moved_count} untracked files moved to Staging" if stale_count > 0: summary += f", {stale_count} stale records removed" if moved_count == 0 and stale_count == 0: summary += " — library is clean" logger.info(f"[SoulSync Deep Scan] {summary}") add_activity_item("", "SoulSync Deep Scan", summary, "Now") _db_update_finished_callback(0, 0, len(transfer_files), moved_count + stale_count, 0) except Exception as e: logger.error(f"[SoulSync Deep Scan] {e}") import traceback traceback.print_exc() _db_update_error_callback(f"Deep scan failed: {e}") def _run_db_update_task(full_refresh, server_type): """The actual function that runs in the background thread.""" global db_update_worker # SoulSync standalone if server_type == "soulsync": if full_refresh: _run_soulsync_full_refresh() else: # Incremental: library updates at download/import time, nothing to do logger.warning("[SoulSync Standalone] Incremental scan skipped — library updates at download time. Use Deep Scan or Full Refresh.") _db_update_finished_callback(0, 0, 0, 0, 0) return media_client = None if server_type == "plex": media_client = plex_client elif server_type == "jellyfin": media_client = jellyfin_client elif server_type == "navidrome": media_client = navidrome_client if not media_client: _db_update_error_callback(f"Media client for '{server_type}' not available.") return # Pause enrichment workers to reduce DB lock contention during scan _pause_workers_for_scan() with db_update_lock: db_update_worker = DatabaseUpdateWorker( media_client=media_client, full_refresh=full_refresh, server_type=server_type, force_sequential=True # Force sequential processing in web server mode ) # Connect signals to callbacks (handle both Qt and headless modes) try: # Try Qt signal connection first db_update_worker.progress_updated.connect(_db_update_progress_callback) db_update_worker.phase_changed.connect(_db_update_phase_callback) db_update_worker.artist_processed.connect(_db_update_artist_callback) db_update_worker.finished.connect(_db_update_finished_callback) db_update_worker.error.connect(_db_update_error_callback) except AttributeError: # Headless mode - use callback system db_update_worker.connect_callback('progress_updated', _db_update_progress_callback) db_update_worker.connect_callback('phase_changed', _db_update_phase_callback) db_update_worker.connect_callback('artist_processed', _db_update_artist_callback) db_update_worker.connect_callback('finished', _db_update_finished_callback) db_update_worker.connect_callback('error', _db_update_error_callback) # This is a blocking call that runs the worker logic db_update_worker.run() def _run_deep_scan_task(server_type): """Run a deep library scan in the background thread.""" global db_update_worker media_client = None if server_type == "plex": media_client = plex_client elif server_type == "jellyfin": media_client = jellyfin_client elif server_type == "navidrome": media_client = navidrome_client elif server_type == "soulsync": # SoulSync standalone deep scan: find untracked files → move to Staging, # remove stale DB records where files no longer exist on disk _run_soulsync_deep_scan() return if not media_client: _db_update_error_callback(f"Media client for '{server_type}' not available.") return # Pause enrichment workers to reduce DB lock contention during deep scan _pause_workers_for_scan() with db_update_lock: db_update_worker = DatabaseUpdateWorker( media_client=media_client, full_refresh=False, server_type=server_type, force_sequential=True ) try: db_update_worker.progress_updated.connect(_db_update_progress_callback) db_update_worker.phase_changed.connect(_db_update_phase_callback) db_update_worker.artist_processed.connect(_db_update_artist_callback) db_update_worker.finished.connect(_db_update_finished_callback) db_update_worker.error.connect(_db_update_error_callback) except AttributeError: db_update_worker.connect_callback('progress_updated', _db_update_progress_callback) db_update_worker.connect_callback('phase_changed', _db_update_phase_callback) db_update_worker.connect_callback('artist_processed', _db_update_artist_callback) db_update_worker.connect_callback('finished', _db_update_finished_callback) db_update_worker.connect_callback('error', _db_update_error_callback) # Run deep scan instead of normal run() db_update_worker.run_deep_scan() @app.route('/api/database/stats', methods=['GET']) def get_database_stats(): """Endpoint to get current database statistics.""" try: # This endpoint returns the same stats shape the UI expects. db = get_database() stats = db.get_database_info_for_server() return jsonify(stats) except Exception as e: logger.error(f"Error getting database stats: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/process', methods=['POST']) def process_wishlist_api(): """Trigger wishlist processing via API. Processes pending wishlist tracks in the background.""" try: if wishlist_auto_processing: return jsonify({"success": False, "error": "Wishlist processing already in progress"}), 409 # Run in background thread (same as automation trigger) import threading thread = threading.Thread(target=_process_wishlist_automatically, daemon=True) thread.start() return jsonify({"success": True, "message": "Wishlist processing started"}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/count', methods=['GET']) def get_wishlist_count(): """Endpoint to get current wishlist count.""" try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() count = wishlist_service.get_wishlist_count(profile_id=get_current_profile_id()) return jsonify({"count": count}) except Exception as e: logger.error(f"Error getting wishlist count: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/stats', methods=['GET']) def get_wishlist_stats(): """ Get wishlist statistics broken down by category. Returns: { "singles": int, # Count of singles + EPs "albums": int, # Count of album tracks "total": int # Total count } """ try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() raw_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id()) singles_count = 0 albums_count = 0 seen_ids = set() for track in raw_tracks: # Deduplicate by ID (same as tracks endpoint) so counts match track_id = track.get('spotify_track_id') or track.get('id') if track_id: if track_id in seen_ids: continue seen_ids.add(track_id) category = _classify_wishlist_track(track) if category == 'singles': singles_count += 1 else: albums_count += 1 total_count = singles_count + albums_count # Calculate time until next auto-processing and get processing state next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('process_wishlist') if automation_engine else 0 # Use smart function with stuck detection (not raw flag) is_processing = is_wishlist_actually_processing() # Get current cycle (albums or singles) from database.music_database import MusicDatabase db = MusicDatabase() try: with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'") row = cursor.fetchone() current_cycle = row['value'] if row else 'albums' except Exception: current_cycle = 'albums' # Safe fallback return jsonify({ "singles": singles_count, "albums": albums_count, "total": total_count, "next_run_in_seconds": next_run_in_seconds, "is_auto_processing": is_processing, "current_cycle": current_cycle }) except Exception as e: logger.error(f"Error getting wishlist stats: {e}") import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/cycle', methods=['GET']) def get_wishlist_cycle(): """ Get the current wishlist processing cycle. Returns: {"cycle": "albums" | "singles"} """ try: from database.music_database import MusicDatabase db = MusicDatabase() # Get cycle from metadata table with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT value FROM metadata WHERE key = 'wishlist_cycle'") row = cursor.fetchone() if row: cycle = row['value'] else: # Default to albums on first run cycle = 'albums' cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('wishlist_cycle', 'albums', CURRENT_TIMESTAMP) """) conn.commit() return jsonify({"cycle": cycle}) except Exception as e: logger.error(f"Error getting wishlist cycle: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/cycle', methods=['POST']) def set_wishlist_cycle(): """ Set the current wishlist processing cycle. Body: {"cycle": "albums" | "singles"} """ try: data = request.get_json() cycle = data.get('cycle') if cycle not in ['albums', 'singles']: return jsonify({"error": "Invalid cycle. Must be 'albums' or 'singles'"}), 400 from database.music_database import MusicDatabase db = MusicDatabase() # Store cycle in metadata table with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP) """, (cycle,)) conn.commit() logger.info(f"Wishlist cycle set to: {cycle}") return jsonify({"success": True, "cycle": cycle}) except Exception as e: logger.error(f"Error setting wishlist cycle: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery/lookback-period', methods=['GET']) def get_discovery_lookback_period(): """ Get the discovery pool lookback period setting. Returns: {"period": "7" | "30" | "90" | "180" | "all"} """ try: from database.music_database import MusicDatabase db = MusicDatabase() # Get lookback period from metadata table with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT value FROM metadata WHERE key = 'discovery_lookback_period'") row = cursor.fetchone() if row: period = row['value'] else: # Default to 30 days on first access period = '30' cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('discovery_lookback_period', '30', CURRENT_TIMESTAMP) """) conn.commit() return jsonify({"period": period}) except Exception as e: logger.error(f"Error getting discovery lookback period: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery/lookback-period', methods=['POST']) def set_discovery_lookback_period(): """ Set the discovery pool lookback period setting. Body: {"period": "7" | "30" | "90" | "180" | "all"} """ try: data = request.get_json() period = data.get('period') valid_periods = ['7', '30', '90', '180', 'all'] if period not in valid_periods: return jsonify({"error": f"Invalid period. Must be one of: {', '.join(valid_periods)}"}), 400 from database.music_database import MusicDatabase db = MusicDatabase() # Store lookback period in metadata table with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('discovery_lookback_period', ?, CURRENT_TIMESTAMP) """, (period,)) # Set a one-time rescan cutoff so the next scan cycle uses the new # lookback window for artists that were already scanned under the old setting. # This avoids wiping last_scan_timestamp (which is needed for UI display). if period == 'all': # 'all' means no cutoff — store empty to signal "scan everything" rescan_value = '' else: from datetime import datetime, timedelta, timezone cutoff = datetime.now(timezone.utc) - timedelta(days=int(period)) rescan_value = cutoff.isoformat() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('watchlist_rescan_cutoff', ?, CURRENT_TIMESTAMP) """, (rescan_value,)) conn.commit() logger.info(f"Discovery lookback period set to: {period}") return jsonify({"success": True, "period": period}) except Exception as e: logger.error(f"Error setting discovery lookback period: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery/hemisphere', methods=['GET']) def get_hemisphere(): """Get the hemisphere setting for seasonal content.""" try: db = get_database() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT value FROM metadata WHERE key = 'hemisphere'") row = cursor.fetchone() value = 'northern' if row: val = row[0] if isinstance(row, tuple) else row['value'] if val in ('northern', 'southern'): value = val return jsonify({"hemisphere": value}) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/discovery/hemisphere', methods=['POST']) def set_hemisphere(): """Set the hemisphere for seasonal content (northern or southern).""" try: data = request.get_json() hemisphere = data.get('hemisphere', '').lower() if hemisphere not in ('northern', 'southern'): return jsonify({"error": "Must be 'northern' or 'southern'"}), 400 db = get_database() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('hemisphere', ?, CURRENT_TIMESTAMP) """, (hemisphere,)) conn.commit() logger.info("Hemisphere set to: %s", hemisphere) return jsonify({"success": True, "hemisphere": hemisphere}) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/tracks', methods=['GET']) def get_wishlist_tracks(): """ Endpoint to get wishlist tracks for display in modal. Supports category filtering via query parameter. Query Parameters: category (optional): 'singles' or 'albums' - filters tracks by album type limit (optional): Maximum number of tracks to return (for performance) """ try: from core.wishlist_service import get_wishlist_service from database.music_database import MusicDatabase # Get category filter and limit from query params category = request.args.get('category', None) # None = all tracks limit = request.args.get('limit', type=int, default=None) # None = no limit # Clean duplicates ONLY if no active wishlist download is running # This prevents count mismatches during active downloads with tasks_lock: wishlist_batch_active = any( batch.get('playlist_id') == 'wishlist' and batch.get('phase') in ['analysis', 'downloading'] for batch in download_batches.values() ) if not wishlist_batch_active: db = MusicDatabase() duplicates_removed = db.remove_wishlist_duplicates(profile_id=get_current_profile_id()) if duplicates_removed > 0: logger.warning(f"Cleaned {duplicates_removed} duplicate tracks from wishlist") else: logger.warning("Skipping wishlist duplicate cleanup - download in progress") wishlist_service = get_wishlist_service() raw_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id()) # SANITIZE: Ensure consistent data format for frontend sanitized_tracks = [] seen_track_ids_sanitation = set() # Deduplicate during sanitization duplicates_found = 0 for track in raw_tracks: sanitized_track = _sanitize_track_data_for_processing(track) spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id') # Skip duplicates during sanitization if spotify_track_id and spotify_track_id in seen_track_ids_sanitation: duplicates_found += 1 continue sanitized_tracks.append(sanitized_track) if spotify_track_id: seen_track_ids_sanitation.add(spotify_track_id) if duplicates_found > 0: logger.warning(f"[API-Wishlist-Tracks] Found and removed {duplicates_found} duplicate tracks during sanitization") # FILTER by category if specified if category: filtered_tracks = [] seen_track_ids_filtering = set() # Deduplicate during filtering for track in sanitized_tracks: track_category = _classify_wishlist_track(track) spotify_track_id = track.get('spotify_track_id') or track.get('id') matches_category = (category == track_category) # Only add if matches category AND not a duplicate if matches_category: # Only deduplicate if track has a valid ID if spotify_track_id: if spotify_track_id not in seen_track_ids_filtering: filtered_tracks.append(track) seen_track_ids_filtering.add(spotify_track_id) else: # No ID - can't deduplicate safely, always add filtered_tracks.append(track) # Apply limit early for performance if limit and len(filtered_tracks) >= limit: break # Count total in category (quick scan — no heavy processing, just classification) total_in_category = sum(1 for t in sanitized_tracks if _classify_wishlist_track(t) == category) logger.info(f"Wishlist filter: {len(filtered_tracks)}/{total_in_category} tracks in '{category}' category (limit: {limit or 'none'})") return jsonify({"tracks": filtered_tracks, "category": category, "total": total_in_category}) # Apply limit to non-filtered results total_count = len(sanitized_tracks) result_tracks = sanitized_tracks[:limit] if limit else sanitized_tracks return jsonify({"tracks": result_tracks, "total": total_count}) except Exception as e: logger.error(f"Error getting wishlist tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/wishlist/download_missing', methods=['POST']) def start_wishlist_missing_downloads(): """ This endpoint fetches wishlist tracks and manages them with batch processing identical to playlist processing, maintaining exactly 3 concurrent downloads. """ dl_err = check_download_permission() if dl_err: return dl_err try: # Check if auto-processing is currently running (prevent concurrent wishlist access) if is_wishlist_actually_processing(): return jsonify({ "error": "Wishlist auto-processing is currently running. Please wait for it to complete.", "retry_after": 30 }), 409 data = request.get_json() or {} force_download_all = data.get('force_download_all', False) category = data.get('category') # Get category filter (albums or singles) track_ids = data.get('track_ids') # NEW: Get specific track IDs from frontend from core.wishlist_service import get_wishlist_service from database.music_database import MusicDatabase wishlist_service = get_wishlist_service() # CRITICAL: Clean duplicates BEFORE fetching tracks to prevent count mismatches # This prevents the "11 tracks shown but 12 counted" bug logger.warning("[Manual-Wishlist] Cleaning duplicate tracks before download...") db = MusicDatabase() manual_profile_id = get_current_profile_id() duplicates_removed = db.remove_wishlist_duplicates(profile_id=manual_profile_id) if duplicates_removed > 0: logger.warning(f"[Manual-Wishlist] Removed {duplicates_removed} duplicate tracks") # CLEANUP: Remove tracks from wishlist that already exist in library # This prevents wasting bandwidth on tracks we already have logger.info("[Manual-Wishlist] Checking wishlist against library for already-owned tracks...") active_server = config_manager.get_active_media_server() cleanup_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=manual_profile_id) cleanup_removed = 0 for track in cleanup_tracks: # BYPASS: Don't remove enhance tracks — they intentionally re-download existing library tracks if track.get('source_type') == 'enhance': continue track_name = track.get('name', '') artists = track.get('artists', []) spotify_track_id = track.get('spotify_track_id') or track.get('id') track_album = track.get('album', {}).get('name') if isinstance(track.get('album'), dict) else track.get('album') if not track_name or not artists or not spotify_track_id: continue # Check if track exists in library found_in_db = False for artist in artists: if isinstance(artist, str): artist_name = artist elif isinstance(artist, dict) and 'name' in artist: artist_name = artist['name'] else: artist_name = str(artist) try: db_track, confidence = db.check_track_exists( track_name, artist_name, confidence_threshold=0.7, server_source=active_server, album=track_album ) if db_track and confidence >= 0.7: found_in_db = True break except Exception as db_error: continue # Remove from wishlist if found in library if found_in_db: try: removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) if removed: cleanup_removed += 1 logger.info(f"[Manual-Wishlist] Removed already-owned track: '{track_name}' by {artist_name}") except Exception as remove_error: logger.error(f"[Manual-Wishlist] Error removing track from wishlist: {remove_error}") if cleanup_removed > 0: logger.info(f"[Manual-Wishlist] Cleaned up {cleanup_removed} already-owned tracks from wishlist") # Get wishlist tracks formatted for download modal (after cleanup) raw_wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=manual_profile_id) if not raw_wishlist_tracks: return jsonify({"success": False, "error": "No tracks in wishlist"}), 400 # SANITIZE: Ensure consistent data format from wishlist service wishlist_tracks = [] seen_track_ids_sanitation = set() # Deduplicate during sanitization duplicates_found = 0 for track in raw_wishlist_tracks: sanitized_track = _sanitize_track_data_for_processing(track) spotify_track_id = sanitized_track.get('spotify_track_id') or sanitized_track.get('id') # Skip duplicates during sanitization if spotify_track_id and spotify_track_id in seen_track_ids_sanitation: duplicates_found += 1 continue wishlist_tracks.append(sanitized_track) if spotify_track_id: seen_track_ids_sanitation.add(spotify_track_id) if duplicates_found > 0: logger.warning(f"[Manual-Wishlist] Found and removed {duplicates_found} duplicate tracks during sanitization") logger.info(f"[Manual-Wishlist] Sanitized {len(wishlist_tracks)} tracks from wishlist service") # FILTER BY TRACK IDs if specified (prioritized - prevents race conditions) if track_ids: # Build a lookup by track ID for O(1) access track_lookup = {} for track in wishlist_tracks: spotify_track_id = track.get('spotify_track_id') or track.get('id') if spotify_track_id and spotify_track_id not in track_lookup: track_lookup[spotify_track_id] = track # Iterate in track_ids order (matches frontend display order) # Stamp each track with its original position in the track_ids array # so track_index matches the modal row even if cleanup removed some tracks filtered_tracks = [] seen_track_ids = set() for frontend_index, tid in enumerate(track_ids): if tid in track_lookup and tid not in seen_track_ids: track = track_lookup[tid] track['_original_index'] = frontend_index # Preserve frontend table position filtered_tracks.append(track) seen_track_ids.add(tid) wishlist_tracks = filtered_tracks logger.info(f"[Manual-Wishlist] Filtered to {len(wishlist_tracks)} specific tracks by ID (preserving frontend display order)") # FILTER BY CATEGORY if specified and no track_ids (backward compatibility) elif category: import json filtered_tracks = [] seen_track_ids = set() # Track IDs we've already added to prevent duplicates for track in wishlist_tracks: # Extract track count from spotify_data spotify_data = track.get('spotify_data', {}) if isinstance(spotify_data, str): try: spotify_data = json.loads(spotify_data) except: spotify_data = {} album_data = spotify_data.get('album', {}) if not isinstance(album_data, dict): album_data = {} total_tracks = album_data.get('total_tracks') album_type = album_data.get('album_type', 'album').lower() # Categorize by track count if available, otherwise use album_type # Single: 1 track, EP: 2-5 tracks, Album: 6+ tracks is_single_or_ep = False is_album = False if total_tracks is not None and total_tracks > 0: # Use track count (most accurate) is_single_or_ep = total_tracks < 6 is_album = total_tracks >= 6 else: # Fall back to Spotify's album_type is_single_or_ep = album_type in ['single', 'ep'] is_album = album_type == 'album' spotify_track_id = track.get('spotify_track_id') or track.get('id') matches_category = (category == 'singles' and is_single_or_ep) or (category == 'albums' and is_album) # Only add if matches category AND not a duplicate if matches_category: # Only deduplicate if track has a valid ID if spotify_track_id: if spotify_track_id not in seen_track_ids: filtered_tracks.append(track) seen_track_ids.add(spotify_track_id) else: # No ID - can't deduplicate safely, always add filtered_tracks.append(track) wishlist_tracks = filtered_tracks logger.info(f"[Manual-Wishlist] Filtered to {len(wishlist_tracks)} tracks for category: {category}") # Stamp original index on each track so task indices match frontend row order for i, track in enumerate(wishlist_tracks): track['_original_index'] = i # Add activity for wishlist download start add_activity_item("", "Wishlist Download Started", f"{len(wishlist_tracks)} tracks", "Now") batch_id = str(uuid.uuid4()) # Use "wishlist" as the playlist_id for consistency in the modal system playlist_id = "wishlist" playlist_name = "Wishlist" # Create task queue for this batch - convert wishlist tracks to the expected format task_queue = [] with tasks_lock: download_batches[batch_id] = { 'phase': 'analysis', 'playlist_id': playlist_id, 'playlist_name': playlist_name, 'queue': task_queue, 'active_count': 0, 'max_concurrent': _get_batch_max_concurrent(), # Wishlist always does single-track downloads, not folder grabs 'queue_index': 0, 'analysis_total': len(wishlist_tracks), 'analysis_processed': 0, 'analysis_results': [], # Track state management (replicating sync.py) 'permanently_failed_tracks': [], 'cancelled_tracks': set(), # Wishlist tracks are already known-missing — always skip the library check 'force_download_all': True, # Profile context for failed track wishlist re-adds 'profile_id': manual_profile_id } # Submit the wishlist processing job using the same processing function missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, wishlist_tracks) return jsonify({ "success": True, "batch_id": batch_id }) except Exception as e: logger.error(f"Error starting wishlist download process: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/clear', methods=['POST']) def clear_wishlist(): """Endpoint to clear all tracks from the wishlist. Also cancels any active wishlist download batch so cleared tracks don't keep downloading.""" try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() success = wishlist_service.clear_wishlist(profile_id=get_current_profile_id()) if success: # Cancel any active wishlist download batch cancelled_count = 0 with tasks_lock: for _batch_id, batch_data in download_batches.items(): if batch_data.get('playlist_id') == 'wishlist' and batch_data.get('phase') not in ('complete', 'error', 'cancelled'): batch_data['phase'] = 'cancelled' for task_id in batch_data.get('queue', []): if task_id in download_tasks and download_tasks[task_id]['status'] not in ('completed', 'failed', 'not_found', 'cancelled'): download_tasks[task_id]['status'] = 'cancelled' cancelled_count += 1 # Reset wishlist auto-processing flag global wishlist_auto_processing, wishlist_auto_processing_timestamp with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 if cancelled_count > 0: logger.warning(f"[Wishlist Clear] Cancelled {cancelled_count} active wishlist downloads") add_activity_item("", "Wishlist Cleared", f"Wishlist cleared and {cancelled_count} downloads cancelled", "Now") return jsonify({"success": True, "message": "Wishlist cleared successfully", "cancelled_downloads": cancelled_count}) else: return jsonify({"success": False, "error": "Failed to clear wishlist"}), 500 except Exception as e: logger.error(f"Error clearing wishlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/cleanup', methods=['POST']) def cleanup_wishlist(): """Endpoint to remove tracks from wishlist that already exist in the database.""" try: from core.wishlist_service import get_wishlist_service from database.music_database import MusicDatabase wishlist_service = get_wishlist_service() db = MusicDatabase() active_server = config_manager.get_active_media_server() logger.info("[Wishlist Cleanup] Starting wishlist cleanup process...") # Get wishlist tracks for current profile cleanup_profile_id = get_current_profile_id() wishlist_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=cleanup_profile_id) if not wishlist_tracks: return jsonify({"success": True, "message": "No tracks in wishlist to clean up", "removed_count": 0}) logger.info(f"[Wishlist Cleanup] Found {len(wishlist_tracks)} tracks in wishlist") removed_count = 0 processed_count = 0 for track in wishlist_tracks: processed_count += 1 track_name = track.get('name', '') artists = track.get('artists', []) spotify_track_id = track.get('spotify_track_id') or track.get('id') track_album = track.get('album', {}).get('name') if isinstance(track.get('album'), dict) else track.get('album') # Skip if no essential data if not track_name or not artists or not spotify_track_id: continue logger.info(f"[Wishlist Cleanup] Checking track {processed_count}/{len(wishlist_tracks)}: '{track_name}'") # Check each artist found_in_db = False for artist in artists: # Handle both string format and dict format if isinstance(artist, str): artist_name = artist elif isinstance(artist, dict) and 'name' in artist: artist_name = artist['name'] else: artist_name = str(artist) try: db_track, confidence = db.check_track_exists( track_name, artist_name, confidence_threshold=0.7, server_source=active_server, album=track_album ) if db_track and confidence >= 0.7: found_in_db = True logger.info(f"[Wishlist Cleanup] Track found in database: '{track_name}' by {artist_name} (confidence: {confidence:.2f})") break except Exception as db_error: logger.error(f"[Wishlist Cleanup] Error checking database for track '{track_name}': {db_error}") continue # If found in database, remove from wishlist if found_in_db: try: removed = wishlist_service.mark_track_download_result(spotify_track_id, success=True) if removed: removed_count += 1 logger.info(f"[Wishlist Cleanup] Removed track from wishlist: '{track_name}' ({spotify_track_id})") else: logger.error(f"[Wishlist Cleanup] Failed to remove track from wishlist: '{track_name}' ({spotify_track_id})") except Exception as remove_error: logger.error(f"[Wishlist Cleanup] Error removing track from wishlist: {remove_error}") logger.info(f"[Wishlist Cleanup] Completed cleanup: {removed_count} tracks removed from wishlist") return jsonify({ "success": True, "message": f"Wishlist cleanup completed: {removed_count} tracks removed", "removed_count": removed_count, "processed_count": processed_count }) except Exception as e: logger.error(f"Error in wishlist cleanup: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/remove-track', methods=['POST']) def remove_track_from_wishlist(): """Endpoint to remove a single track from the wishlist.""" try: from core.wishlist_service import get_wishlist_service data = request.get_json() spotify_track_id = data.get('spotify_track_id') if not spotify_track_id: return jsonify({"success": False, "error": "No spotify_track_id provided"}), 400 wishlist_service = get_wishlist_service() success = wishlist_service.remove_track_from_wishlist(spotify_track_id, profile_id=get_current_profile_id()) if success: logger.info(f"Successfully removed track from wishlist: {spotify_track_id}") return jsonify({"success": True, "message": "Track removed from wishlist"}) else: logger.warning(f"Failed to remove track from wishlist: {spotify_track_id}") return jsonify({"success": False, "error": "Track not found in wishlist"}), 404 except Exception as e: logger.error(f"Error removing track from wishlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/remove-album', methods=['POST']) def remove_album_from_wishlist(): """Endpoint to remove all tracks from an album from the wishlist.""" try: from core.wishlist_service import get_wishlist_service import json data = request.get_json() album_id = data.get('album_id') album_name_filter = data.get('album_name') if not album_id and not album_name_filter: return jsonify({"success": False, "error": "No album_id or album_name provided"}), 400 wishlist_service = get_wishlist_service() all_tracks = wishlist_service.get_wishlist_tracks_for_download(profile_id=get_current_profile_id()) # Find all tracks that belong to this album tracks_to_remove = [] for track in all_tracks: spotify_data = track.get('spotify_data', {}) if isinstance(spotify_data, str): try: spotify_data = json.loads(spotify_data) except: spotify_data = {} # Get album ID - safely handle null album data album_data = spotify_data.get('album') or {} if not isinstance(album_data, dict): album_data = {} track_album_id = album_data.get('id') if not track_album_id: # Create custom ID matching frontend logic exactly # album_data is guaranteed to be a dict from above check album_name = album_data.get('name', 'Unknown Album') artists = spotify_data.get('artists', []) if artists and isinstance(artists[0], dict): artist_name = artists[0].get('name', 'Unknown Artist') elif artists and isinstance(artists[0], str): artist_name = artists[0] else: artist_name = 'Unknown Artist' custom_id = f"{album_name}_{artist_name}" # Match frontend regex exactly: # 1. Remove all special chars except spaces, underscores, hyphens: /[^a-zA-Z0-9\s_-]/g # 2. Replace consecutive whitespace with single underscore: /\s+/g track_album_id = re.sub(r'[^a-zA-Z0-9\s_-]', '', custom_id) # Remove special chars track_album_id = re.sub(r'\s+', '_', track_album_id).lower() # Replace spaces & lowercase # Match by album ID or album name matched = False if album_id and track_album_id == album_id: matched = True elif album_name_filter: track_album_name = album_data.get('name', '') if isinstance(spotify_data.get('album'), str): track_album_name = spotify_data['album'] if track_album_name and track_album_name.lower().strip() == album_name_filter.lower().strip(): matched = True if matched: spotify_track_id = track.get('spotify_track_id') or track.get('id') if spotify_track_id: tracks_to_remove.append(spotify_track_id) # Remove all matching tracks removed_count = 0 album_remove_pid = get_current_profile_id() for spotify_track_id in tracks_to_remove: if wishlist_service.remove_track_from_wishlist(spotify_track_id, profile_id=album_remove_pid): removed_count += 1 if removed_count > 0: logger.info(f"Successfully removed {removed_count} tracks from album {album_id}") return jsonify({ "success": True, "message": f"Removed {removed_count} track(s) from wishlist", "removed_count": removed_count }) else: logger.warning(f"No tracks found for album {album_id}") return jsonify({"success": False, "error": "No tracks found for this album"}), 404 except Exception as e: logger.error(f"Error removing album from wishlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/wishlist/remove-batch', methods=['POST']) def remove_batch_from_wishlist(): """Endpoint to remove multiple tracks from the wishlist.""" try: from core.wishlist_service import get_wishlist_service data = request.get_json() spotify_track_ids = data.get('spotify_track_ids', []) if not spotify_track_ids or not isinstance(spotify_track_ids, list): return jsonify({"success": False, "error": "Missing or invalid spotify_track_ids"}), 400 wishlist_service = get_wishlist_service() removed = 0 pid = get_current_profile_id() for track_id in spotify_track_ids: if wishlist_service.remove_track_from_wishlist(track_id, profile_id=pid): removed += 1 logger.info(f"Batch removed {removed} track(s) from wishlist") return jsonify({ "success": True, "removed": removed, "message": f"Removed {removed} track{'s' if removed != 1 else ''} from wishlist" }) except Exception as e: logger.error(f"Error batch removing from wishlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/add-album-to-wishlist', methods=['POST']) def add_album_track_to_wishlist(): """Endpoint to add a single track from an album to the wishlist.""" try: from core.wishlist_service import get_wishlist_service data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 track = data.get('track') artist = data.get('artist') album = data.get('album') source_type = data.get('source_type', 'album') source_context = data.get('source_context', {}) if not track or not artist or not album: return jsonify({"success": False, "error": "Missing required fields: track, artist, album"}), 400 # Create Spotify track data format expected by wishlist service # Handle both formats: Spotify API format (images array) and library format (image_url string) album_images = [] if 'images' in album and album.get('images'): # Spotify API format with images array album_images = album['images'] elif 'image_url' in album and album.get('image_url'): # Library format with single image_url - convert to Spotify format album_images = [{'url': album['image_url'], 'height': 640, 'width': 640}] spotify_track_data = { 'id': track.get('id'), 'name': track.get('name'), 'artists': track.get('artists', []), 'album': { 'id': album.get('id'), 'name': album.get('name'), 'artists': album.get('artists', []), 'images': album_images, 'album_type': album.get('album_type', 'album'), 'release_date': album.get('release_date', ''), 'total_tracks': album.get('total_tracks', 1) }, 'duration_ms': track.get('duration_ms', 0), 'track_number': track.get('track_number', 1), 'disc_number': track.get('disc_number', 1), 'explicit': track.get('explicit', False), 'popularity': track.get('popularity', 0), 'preview_url': track.get('preview_url'), 'external_urls': track.get('external_urls', {}) } # Add source context information enhanced_source_context = { **source_context, 'artist_id': artist.get('id'), 'artist_name': artist.get('name'), 'album_id': album.get('id'), 'album_name': album.get('name'), 'added_via': 'library_wishlist_modal' } # Get wishlist service and add track wishlist_service = get_wishlist_service() success = wishlist_service.add_spotify_track_to_wishlist( spotify_track_data=spotify_track_data, failure_reason="Added from library (incomplete album)", source_type=source_type, source_context=enhanced_source_context, profile_id=get_current_profile_id() ) if success: logger.info(f"Added track '{track.get('name')}' by '{artist.get('name')}' to wishlist") return jsonify({ "success": True, "message": f"Added '{track.get('name')}' to wishlist" }) else: logger.error(f"Failed to add track '{track.get('name')}' to wishlist") return jsonify({ "success": False, "error": "Failed to add track to wishlist" }) except Exception as e: logger.error(f"Error adding track to wishlist: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/update', methods=['POST']) def start_database_update(): """Endpoint to start the database update process.""" global db_update_worker with db_update_lock: if db_update_state["status"] == "running": return jsonify({"success": False, "error": "An update is already in progress."}), 409 data = request.get_json() full_refresh = data.get('full_refresh', False) deep_scan = data.get('deep_scan', False) active_server = config_manager.get_active_media_server() scan_type = "Deep scan" if deep_scan else ("Full" if full_refresh else "Incremental") db_update_state.update({ "status": "running", "phase": f"{scan_type}: Initializing...", "progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": "" }) # Add activity for database update start server_name = active_server.capitalize() add_activity_item("", "Database Update", f"Starting {scan_type.lower()} update from {server_name}...", "Now") # Submit the appropriate worker if deep_scan: db_update_executor.submit(_run_deep_scan_task, active_server) else: db_update_executor.submit(_run_db_update_task, full_refresh, active_server) return jsonify({"success": True, "message": "Database update started."}) @app.route('/api/database/update/status', methods=['GET']) def get_database_update_status(): """Endpoint to poll for the current update status.""" with db_update_lock: # Debug: Log current state occasionally if db_update_state["status"] == "running": logger.info(f"[Status Check] {db_update_state['processed']}/{db_update_state['total']} ({db_update_state['progress']:.1f}%) - {db_update_state['phase']}") return jsonify(db_update_state) @app.route('/api/database/update/stop', methods=['POST']) def stop_database_update(): """Endpoint to stop the current database update.""" global db_update_worker with db_update_lock: if db_update_worker and db_update_state["status"] == "running": db_update_worker.stop() db_update_state["status"] = "finished" db_update_state["phase"] = "Update stopped by user." return jsonify({"success": True, "message": "Stop request sent."}) else: return jsonify({"success": False, "error": "No update is currently running."}), 404 _BACKUP_FILENAME_RE = re.compile(r'^music_library\.db\.backup_\d{8}_\d{6}$') @app.route('/api/database/backup', methods=['POST']) def backup_database_endpoint(): """Create a rolling backup of the database (max 5).""" try: import sqlite3, glob as _glob db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') if not os.path.exists(db_path): return jsonify({"success": False, "error": "Database file not found"}), 404 max_backups = 5 timestamp = datetime.now().strftime('%Y%m%d_%H%M%S') backup_path = f"{db_path}.backup_{timestamp}" src = sqlite3.connect(db_path) dst = sqlite3.connect(backup_path) src.backup(dst) dst.close() src.close() size_mb = round(os.path.getsize(backup_path) / (1024 * 1024), 1) # Write version metadata sidecar meta_path = backup_path + '.meta.json' try: with open(meta_path, 'w') as mf: json.dump({"version": SOULSYNC_VERSION, "created": timestamp}, mf) except Exception: pass # Non-critical — backup still works without metadata # Rolling cleanup existing = sorted(_glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime) # Filter out .meta.json files from the backup list existing = [f for f in existing if not f.endswith('.meta.json')] while len(existing) > max_backups: try: removed = existing.pop(0) os.remove(removed) # Also remove sidecar if present if os.path.exists(removed + '.meta.json'): os.remove(removed + '.meta.json') except Exception: pass return jsonify({"success": True, "backup_path": backup_path, "size_mb": size_mb, "version": SOULSYNC_VERSION}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/backups', methods=['GET']) def list_backups_endpoint(): """List all database backups with metadata.""" try: import glob as _glob db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') backup_files = sorted( _glob.glob(f"{db_path}.backup_*"), key=os.path.getmtime, reverse=True ) backups = [] for fp in backup_files: fname = os.path.basename(fp) if not _BACKUP_FILENAME_RE.match(fname): continue stat = os.stat(fp) entry = { 'filename': fname, 'size_mb': round(stat.st_size / (1024 * 1024), 2), 'created': datetime.utcfromtimestamp(stat.st_mtime).isoformat() } # Read version from sidecar metadata if available meta_path = fp + '.meta.json' if os.path.exists(meta_path): try: with open(meta_path, 'r') as mf: meta = json.load(mf) entry['version'] = meta.get('version') except Exception: pass backups.append(entry) db_size_mb = round(os.path.getsize(db_path) / (1024 * 1024), 2) if os.path.exists(db_path) else 0 return jsonify({ 'success': True, 'backups': backups, 'count': len(backups), 'db_size_mb': db_size_mb }) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/backups/', methods=['DELETE']) def delete_backup_endpoint(filename): """Delete a specific database backup.""" try: if not _BACKUP_FILENAME_RE.match(filename) or '/' in filename or '\\' in filename or '..' in filename: return jsonify({"success": False, "error": "Invalid backup filename"}), 400 db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') backup_path = os.path.join(os.path.dirname(db_path), filename) if not os.path.exists(backup_path): return jsonify({"success": False, "error": "Backup not found"}), 404 os.remove(backup_path) # Also remove sidecar metadata if present meta_path = backup_path + '.meta.json' if os.path.exists(meta_path): try: os.remove(meta_path) except Exception: pass return jsonify({"success": True, "deleted": filename}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/backups//restore', methods=['POST']) def restore_backup_endpoint(filename): """Restore the database from a specific backup.""" try: import sqlite3 if not _BACKUP_FILENAME_RE.match(filename) or '/' in filename or '\\' in filename or '..' in filename: return jsonify({"success": False, "error": "Invalid backup filename"}), 400 db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') db_dir = os.path.dirname(db_path) backup_path = os.path.join(db_dir, filename) if not os.path.exists(backup_path): return jsonify({"success": False, "error": "Backup not found"}), 404 # Check version compatibility backup_version = None meta_path = backup_path + '.meta.json' if os.path.exists(meta_path): try: with open(meta_path, 'r') as mf: meta = json.load(mf) backup_version = meta.get('version') except Exception: pass version_warning = None # Compare base versions only (strip +commit suffix) to avoid false mismatches _backup_base = backup_version.split('+')[0] if backup_version else None _current_base = SOULSYNC_VERSION.split('+')[0] if _backup_base and _backup_base != _current_base: # Allow restore but warn — the caller must pass force=true to confirm force = request.json.get('force', False) if request.is_json else False if not force: return jsonify({ "success": False, "version_mismatch": True, "backup_version": backup_version, "current_version": SOULSYNC_VERSION, "error": f"This backup was created on SoulSync v{backup_version}, but you're running v{SOULSYNC_VERSION}. Restoring may cause issues. Send force=true to proceed." }), 409 version_warning = f"Restored from v{backup_version} backup (current: v{SOULSYNC_VERSION})" # Create safety backup of current DB before restoring safety_ts = datetime.now().strftime('%Y%m%d_%H%M%S') safety_filename = f"music_library.db.backup_{safety_ts}" safety_path = os.path.join(db_dir, safety_filename) src_conn = sqlite3.connect(db_path) dst_conn = sqlite3.connect(safety_path) src_conn.backup(dst_conn) dst_conn.close() src_conn.close() # Write version metadata for the safety backup too try: with open(safety_path + '.meta.json', 'w') as mf: json.dump({"version": SOULSYNC_VERSION, "created": safety_ts}, mf) except Exception: pass # Restore using SQLite backup API (handles concurrent access safely) from database.music_database import close_database, get_database close_database() src_restore = sqlite3.connect(backup_path) dst_restore = sqlite3.connect(db_path) src_restore.backup(dst_restore) dst_restore.close() src_restore.close() # Reinitialize database and verify db = get_database() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("SELECT COUNT(*) FROM artists") artist_count = cursor.fetchone()[0] result = { "success": True, "restored_from": filename, "safety_backup": safety_filename, "artist_count": artist_count } if backup_version: result["backup_version"] = backup_version if version_warning: result["version_warning"] = version_warning return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/database/backups//download', methods=['GET']) def download_backup_endpoint(filename): """Download a specific database backup file.""" try: if not _BACKUP_FILENAME_RE.match(filename) or '/' in filename or '\\' in filename or '..' in filename: return jsonify({"success": False, "error": "Invalid backup filename"}), 400 db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') backup_path = os.path.join(os.path.dirname(db_path), filename) if not os.path.exists(backup_path): return jsonify({"success": False, "error": "Backup not found"}), 404 # Override the default static-cache max-age — this is a sensitive # DB backup, browsers should never cache it. response = send_file(backup_path, as_attachment=True, download_name=filename) response.headers['Cache-Control'] = 'no-store' return response except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == DATABASE MAINTENANCE == # =============================== @app.route('/api/database/maintenance/info', methods=['GET']) def database_maintenance_info(): """Get database size, free pages, and auto_vacuum mode.""" try: import sqlite3 db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') conn = sqlite3.connect(db_path) c = conn.cursor() c.execute('PRAGMA page_count'); total_pages = c.fetchone()[0] c.execute('PRAGMA freelist_count'); free_pages = c.fetchone()[0] c.execute('PRAGMA page_size'); page_size = c.fetchone()[0] c.execute('PRAGMA auto_vacuum'); auto_vacuum = c.fetchone()[0] conn.close() total_bytes = total_pages * page_size free_bytes = free_pages * page_size auto_vacuum_labels = {0: 'None', 1: 'Full', 2: 'Incremental'} return jsonify({ 'success': True, 'total_size': total_bytes, 'total_size_display': f'{total_bytes / 1024 / 1024:.1f} MB', 'free_pages': free_pages, 'free_size': free_bytes, 'free_size_display': f'{free_bytes / 1024 / 1024:.1f} MB', 'bloat_percent': round(free_pages / total_pages * 100, 1) if total_pages > 0 else 0, 'auto_vacuum': auto_vacuum, 'auto_vacuum_label': auto_vacuum_labels.get(auto_vacuum, 'Unknown'), }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/database/maintenance/vacuum', methods=['POST']) def database_vacuum(): """Run VACUUM to compact the database. Locks DB during operation.""" try: import sqlite3, time db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') # Get size before size_before = os.path.getsize(db_path) conn = sqlite3.connect(db_path) start = time.time() conn.execute('VACUUM') elapsed = time.time() - start conn.close() size_after = os.path.getsize(db_path) saved = size_before - size_after logger.info(f"Database VACUUM completed in {elapsed:.1f}s — saved {saved / 1024 / 1024:.1f} MB") return jsonify({ 'success': True, 'elapsed_seconds': round(elapsed, 1), 'size_before': size_before, 'size_after': size_after, 'saved_bytes': saved, 'saved_display': f'{saved / 1024 / 1024:.1f} MB', }) except Exception as e: logger.error(f"Database VACUUM failed: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/database/maintenance/enable-incremental-vacuum', methods=['POST']) def enable_incremental_vacuum(): """Enable incremental auto_vacuum. Requires a full VACUUM to activate.""" try: import sqlite3, time db_path = os.environ.get('DATABASE_PATH', 'database/music_library.db') conn = sqlite3.connect(db_path) c = conn.cursor() c.execute('PRAGMA auto_vacuum') current = c.fetchone()[0] if current == 2: conn.close() return jsonify({'success': True, 'message': 'Incremental vacuum is already enabled', 'already_enabled': True}) size_before = os.path.getsize(db_path) # Set incremental mode and VACUUM to activate it c.execute('PRAGMA auto_vacuum = INCREMENTAL') start = time.time() conn.execute('VACUUM') elapsed = time.time() - start conn.close() size_after = os.path.getsize(db_path) saved = size_before - size_after logger.info(f"Incremental auto_vacuum enabled in {elapsed:.1f}s — saved {saved / 1024 / 1024:.1f} MB") return jsonify({ 'success': True, 'message': 'Incremental vacuum enabled', 'elapsed_seconds': round(elapsed, 1), 'saved_display': f'{saved / 1024 / 1024:.1f} MB', }) except Exception as e: logger.error(f"Failed to enable incremental vacuum: {e}") return jsonify({'success': False, 'error': str(e)}), 500 # =============================== # == METADATA CACHE API == # =============================== @app.route('/api/metadata-cache/stats', methods=['GET']) def metadata_cache_stats(): """Get metadata cache statistics.""" try: cache = get_metadata_cache() stats = cache.get_stats() return jsonify(stats) except Exception as e: logger.error(f"Error getting metadata cache stats: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/browse', methods=['GET']) def metadata_cache_browse(): """Browse cached metadata entities with filtering, search, sorting, and pagination.""" try: cache = get_metadata_cache() entity_type = request.args.get('type', 'artist') source = request.args.get('source') search = request.args.get('search') sort = request.args.get('sort', 'last_accessed_at') sort_dir = request.args.get('sort_dir', 'desc') offset = int(request.args.get('offset', 0)) limit = int(request.args.get('limit', 48)) result = cache.browse( entity_type=entity_type, source=source if source else None, search=search if search else None, sort=sort, sort_dir=sort_dir, offset=offset, limit=limit ) return jsonify(result) except Exception as e: logger.error(f"Error browsing metadata cache: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/entity///', methods=['GET']) def metadata_cache_entity_detail(source, entity_type, entity_id): """Get detailed view of a single cached entity.""" try: cache = get_metadata_cache() detail = cache.get_entity_detail(source, entity_type, entity_id) if detail is None: return jsonify({"error": "Entity not found"}), 404 return jsonify(detail) except Exception as e: logger.error(f"Error getting metadata cache entity: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/browse-musicbrainz', methods=['GET']) def metadata_cache_browse_musicbrainz(): """Browse MusicBrainz cache entries in the same format as metadata cache browse.""" try: entity_type = request.args.get('entity_type', 'artist') search = request.args.get('search', '').strip() page = int(request.args.get('page', 1)) limit = int(request.args.get('limit', 48)) offset = (page - 1) * limit database = get_database() conn = database._get_connection() try: cursor = conn.cursor() where_parts = [] params = [] if entity_type: where_parts.append("entity_type = ?") params.append(entity_type) if search: where_parts.append("LOWER(entity_name) LIKE LOWER(?)") params.append(f"%{search}%") where_clause = f"WHERE {' AND '.join(where_parts)}" if where_parts else "" cursor.execute(f"SELECT COUNT(*) FROM musicbrainz_cache {where_clause}", params) total = cursor.fetchone()[0] cursor.execute(f""" SELECT * FROM musicbrainz_cache {where_clause} ORDER BY last_updated DESC LIMIT ? OFFSET ? """, params + [limit, offset]) items = [] for row in cursor.fetchall(): r = dict(row) matched = r.get('musicbrainz_id') is not None items.append({ 'entity_id': r.get('musicbrainz_id') or f"mb-{r.get('entity_type','')}-{r.get('entity_name','')}", 'source': 'musicbrainz', 'name': r.get('entity_name', ''), 'artist_name': r.get('artist_name', ''), 'image_url': None, 'popularity': int((r.get('match_confidence') or 0) * 100), 'access_count': 1, 'last_accessed_at': r.get('last_updated', ''), 'created_at': r.get('last_updated', ''), '_mb_matched': matched, '_mb_id': r.get('musicbrainz_id', ''), }) return jsonify({'items': items, 'total': total, 'offset': offset}) finally: conn.close() except Exception as e: logger.error(f"Error browsing MusicBrainz cache: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/clear', methods=['DELETE']) def metadata_cache_clear(): """Clear cached metadata. Optional query params: source, type.""" try: cache = get_metadata_cache() source = request.args.get('source') entity_type = request.args.get('type') cleared = cache.clear( source=source if source else None, entity_type=entity_type if entity_type else None ) return jsonify({"success": True, "cleared": cleared}) except Exception as e: logger.error(f"Error clearing metadata cache: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata-cache/evict', methods=['POST']) def metadata_cache_evict(): """Evict expired entries from the metadata cache.""" try: cache = get_metadata_cache() evicted = cache.evict_expired() return jsonify({"success": True, "evicted": evicted}) except Exception as e: logger.error(f"Error evicting metadata cache: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata-cache/clear-musicbrainz', methods=['DELETE']) def metadata_cache_clear_musicbrainz(): """Clear MusicBrainz cache entries. Optional query param: failed_only=true.""" try: cache = get_metadata_cache() failed_only = request.args.get('failed_only', '').lower() == 'true' cleared = cache.clear_musicbrainz(failed_only=failed_only) return jsonify({"success": True, "cleared": cleared}) except Exception as e: logger.error(f"Error clearing MusicBrainz cache: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata-cache/failed-mb-lookups', methods=['GET']) def metadata_cache_failed_mb_lookups(): """Get all failed MusicBrainz lookups with pagination and filtering.""" try: entity_type = request.args.get('entity_type', '') search = request.args.get('search', '').strip() page = int(request.args.get('page', 1)) limit = int(request.args.get('limit', 50)) offset = (page - 1) * limit # Only fetch type_counts on first load (page 1, no filters) — frontend caches them include_counts = request.args.get('counts', '').lower() == 'true' database = get_database() conn = database._get_connection() try: cursor = conn.cursor() where_parts = ["musicbrainz_id IS NULL"] params = [] if entity_type: where_parts.append("entity_type = ?") params.append(entity_type) if search: where_parts.append("(entity_name LIKE ? COLLATE NOCASE OR artist_name LIKE ? COLLATE NOCASE)") params.extend([f"%{search}%", f"%{search}%"]) where_clause = f"WHERE {' AND '.join(where_parts)}" # Single query: fetch items + use SQL window for total count cursor.execute(f""" SELECT id, entity_type, entity_name, artist_name, match_confidence, last_updated, COUNT(*) OVER() as _total FROM musicbrainz_cache {where_clause} ORDER BY last_updated DESC LIMIT ? OFFSET ? """, params + [limit, offset]) rows = cursor.fetchall() total = rows[0]['_total'] if rows else 0 items = [{ 'id': r['id'], 'entity_type': r['entity_type'], 'entity_name': r['entity_name'], 'artist_name': r['artist_name'] or '', 'confidence': r['match_confidence'] or 0, 'last_updated': r['last_updated'] or '', } for r in rows] result = {'items': items, 'total': total, 'page': page} # Type counts only when requested (avoids full table scan on every tab switch) if include_counts: cursor.execute(""" SELECT entity_type, COUNT(*) as cnt FROM musicbrainz_cache WHERE musicbrainz_id IS NULL GROUP BY entity_type """) result['type_counts'] = {row['entity_type']: row['cnt'] for row in cursor.fetchall()} return jsonify(result) finally: conn.close() except Exception as e: logger.error(f"Error getting failed MB lookups: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/mb-entry/', methods=['DELETE']) def metadata_cache_delete_mb_entry(entry_id): """Delete a single MusicBrainz cache entry by ID.""" try: database = get_database() conn = database._get_connection() try: cursor = conn.cursor() cursor.execute("DELETE FROM musicbrainz_cache WHERE id = ?", (entry_id,)) conn.commit() return jsonify({"success": True, "deleted": cursor.rowcount}) finally: conn.close() except Exception as e: logger.error(f"Error deleting MB cache entry: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/musicbrainz/search', methods=['GET']) def musicbrainz_search_api(): """Search MusicBrainz for manual matching. Returns raw results.""" try: entity_type = request.args.get('type', 'artist') # artist, release, recording query = request.args.get('q', '').strip() artist = request.args.get('artist', '').strip() limit = min(int(request.args.get('limit', 10)), 20) if not query: return jsonify({"error": "Missing query parameter 'q'"}), 400 mb_svc = mb_worker.mb_service if mb_worker else None if not mb_svc: return jsonify({"error": "MusicBrainz service not available"}), 503 mb_client = mb_svc.mb_client results = [] if entity_type == 'artist': raw = mb_client.search_artist(query, limit=limit) for r in raw: results.append({ 'mbid': r.get('id', ''), 'name': r.get('name', ''), 'disambiguation': r.get('disambiguation', ''), 'score': r.get('score', 0), 'type': r.get('type', ''), 'country': r.get('country', ''), }) elif entity_type == 'release': raw = mb_client.search_release(query, artist_name=artist or None, limit=limit) for r in raw: artist_credit = ', '.join(a.get('name', '') for a in r.get('artist-credit', []) if isinstance(a, dict)) results.append({ 'mbid': r.get('id', ''), 'name': r.get('title', ''), 'artist': artist_credit, 'disambiguation': r.get('disambiguation', ''), 'score': r.get('score', 0), 'date': r.get('date', ''), 'country': r.get('country', ''), 'track_count': r.get('track-count', 0), }) elif entity_type == 'recording': raw = mb_client.search_recording(query, artist_name=artist or None, limit=limit) for r in raw: artist_credit = ', '.join(a.get('name', '') for a in r.get('artist-credit', []) if isinstance(a, dict)) releases = r.get('releases', []) first_release = releases[0].get('title', '') if releases else '' results.append({ 'mbid': r.get('id', ''), 'name': r.get('title', ''), 'artist': artist_credit, 'disambiguation': r.get('disambiguation', ''), 'score': r.get('score', 0), 'album': first_release, 'length': r.get('length', 0), }) else: return jsonify({"error": f"Unknown entity type: {entity_type}"}), 400 return jsonify({"results": results, "total": len(results)}) except Exception as e: logger.error(f"Error searching MusicBrainz: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata-cache/mb-match', methods=['POST']) def metadata_cache_save_mb_match(): """Save a manual MusicBrainz match for a failed lookup.""" try: data = request.get_json() entry_id = data.get('entry_id') mbid = data.get('mbid', '').strip() mb_name = data.get('mb_name', '').strip() if not entry_id or not mbid: return jsonify({"success": False, "error": "Missing entry_id or mbid"}), 400 database = get_database() conn = database._get_connection() try: cursor = conn.cursor() # Update the failed entry with the user-selected MBID cursor.execute(""" UPDATE musicbrainz_cache SET musicbrainz_id = ?, match_confidence = 100, metadata_json = ?, last_updated = CURRENT_TIMESTAMP WHERE id = ? """, (mbid, json.dumps({'name': mb_name, 'manual_match': True}), entry_id)) conn.commit() if cursor.rowcount == 0: return jsonify({"success": False, "error": "Entry not found"}), 404 logger.info(f"Manual MB match: entry {entry_id} → {mbid} ({mb_name})") return jsonify({"success": True}) finally: conn.close() except Exception as e: logger.error(f"Error saving MB match: {e}") return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == QUALITY SCANNER == # =============================== # Quality tier mappings QUALITY_TIERS = { 'lossless': { 'extensions': ['.flac', '.ape', '.wav', '.alac', '.dsf', '.dff', '.aiff', '.aif'], 'tier': 1 }, 'high_lossy': { 'extensions': ['.opus', '.ogg'], 'tier': 2 }, 'standard_lossy': { 'extensions': ['.m4a', '.aac'], 'tier': 3 }, 'low_lossy': { 'extensions': ['.mp3', '.wma'], 'tier': 4 } } def _get_quality_tier_from_extension(file_path): """Determine quality tier from file extension""" if not file_path: return ('unknown', 999) ext = os.path.splitext(file_path)[1].lower() for tier_name, tier_data in QUALITY_TIERS.items(): if ext in tier_data['extensions']: return (tier_name, tier_data['tier']) return ('unknown', 999) def _run_quality_scanner(scope='watchlist', profile_id=1): """Main quality scanner worker function""" from core.wishlist_service import get_wishlist_service from database.music_database import MusicDatabase try: with quality_scanner_lock: quality_scanner_state["status"] = "running" quality_scanner_state["phase"] = "Initializing scan..." quality_scanner_state["progress"] = 0 quality_scanner_state["processed"] = 0 quality_scanner_state["total"] = 0 quality_scanner_state["quality_met"] = 0 quality_scanner_state["low_quality"] = 0 quality_scanner_state["matched"] = 0 quality_scanner_state["results"] = [] quality_scanner_state["error_message"] = "" logger.info(f"[Quality Scanner] Starting scan with scope: {scope}") # Get database instance db = MusicDatabase() # Get quality profile to determine preferred quality quality_profile = db.get_quality_profile() preferred_qualities = quality_profile.get('qualities', {}) # Determine minimum acceptable tier based on enabled qualities min_acceptable_tier = 999 for quality_name, quality_config in preferred_qualities.items(): if quality_config.get('enabled', False): # Map quality profile names to tier names tier_map = { 'flac': 'lossless', 'mp3_320': 'low_lossy', 'mp3_256': 'low_lossy', 'mp3_192': 'low_lossy' } tier_name = tier_map.get(quality_name) if tier_name: tier_num = QUALITY_TIERS[tier_name]['tier'] min_acceptable_tier = min(min_acceptable_tier, tier_num) logger.info(f"[Quality Scanner] Minimum acceptable tier: {min_acceptable_tier}") # Get tracks to scan based on scope with quality_scanner_lock: quality_scanner_state["phase"] = "Loading tracks from database..." if scope == 'watchlist': # Get watchlist artists watchlist_artists = db.get_watchlist_artists(profile_id=profile_id) if not watchlist_artists: with quality_scanner_lock: quality_scanner_state["status"] = "finished" quality_scanner_state["phase"] = "No watchlist artists found" quality_scanner_state["error_message"] = "Please add artists to watchlist first" logger.warning("[Quality Scanner] No watchlist artists found") return # Get artist names from watchlist artist_names = [artist.artist_name for artist in watchlist_artists] logger.info(f"[Quality Scanner] Scanning {len(artist_names)} watchlist artists") # Get all tracks for these artists by name conn = db._get_connection() placeholders = ','.join(['?' for _ in artist_names]) tracks_to_scan = conn.execute( f"SELECT t.id, t.title, t.artist_id, t.album_id, t.file_path, t.bitrate, a.name as artist_name, al.title as album_title " f"FROM tracks t " f"JOIN artists a ON t.artist_id = a.id " f"JOIN albums al ON t.album_id = al.id " f"WHERE a.name IN ({placeholders}) AND t.file_path IS NOT NULL", artist_names ).fetchall() conn.close() else: # Scan all library tracks with quality_scanner_lock: quality_scanner_state["phase"] = "Loading all library tracks..." conn = db._get_connection() tracks_to_scan = conn.execute( "SELECT t.id, t.title, t.artist_id, t.album_id, t.file_path, t.bitrate, a.name as artist_name, al.title as album_title " "FROM tracks t " "JOIN artists a ON t.artist_id = a.id " "JOIN albums al ON t.album_id = al.id " "WHERE t.file_path IS NOT NULL" ).fetchall() conn.close() total_tracks = len(tracks_to_scan) logger.info(f"[Quality Scanner] Found {total_tracks} tracks to scan") with quality_scanner_lock: quality_scanner_state["total"] = total_tracks quality_scanner_state["phase"] = f"Scanning {total_tracks} tracks..." # Use the module-level spotify_client (already authenticated with cached token) if not spotify_client or not spotify_client.is_spotify_authenticated(): with quality_scanner_lock: quality_scanner_state["status"] = "error" quality_scanner_state["phase"] = "Spotify not authenticated" quality_scanner_state["error_message"] = "Please authenticate with Spotify first" logger.info("[Quality Scanner] Spotify not authenticated") return wishlist_service = get_wishlist_service() # Scan each track for idx, track_row in enumerate(tracks_to_scan, 1): # Check for stop request if quality_scanner_state.get('status') != 'running': logger.info(f"[Quality Scanner] Stop requested, halting at track {idx}/{total_tracks}") break try: track_id, title, artist_id, album_id, file_path, bitrate, artist_name, album_title = track_row # Check quality tier tier_name, tier_num = _get_quality_tier_from_extension(file_path) # Update progress with quality_scanner_lock: quality_scanner_state["processed"] = idx quality_scanner_state["progress"] = (idx / total_tracks) * 100 quality_scanner_state["phase"] = f"Scanning: {artist_name} - {title}" # Check if meets quality standards if tier_num <= min_acceptable_tier: # Quality met with quality_scanner_lock: quality_scanner_state["quality_met"] += 1 continue # Low quality track found with quality_scanner_lock: quality_scanner_state["low_quality"] += 1 logger.info(f"[Quality Scanner] Low quality: {artist_name} - {title} ({tier_name}, {file_path})") # Attempt to match to Spotify using matching_engine matched = False matched_track_data = None try: # Generate search queries using matching engine temp_track = type('TempTrack', (), { 'name': title, 'artists': [artist_name], 'album': album_title })() search_queries = matching_engine.generate_download_queries(temp_track) logger.info(f"[Quality Scanner] Generated {len(search_queries)} search queries for {artist_name} - {title}") # Find best match using confidence scoring best_match = None best_confidence = 0.0 min_confidence = 0.7 # Match existing standard for _query_idx, search_query in enumerate(search_queries): try: spotify_matches = spotify_client.search_tracks(search_query, limit=5) time.sleep(0.5) # Rate limit Spotify API calls if not spotify_matches: continue # Score each result using matching engine for spotify_track in spotify_matches: try: # Calculate artist confidence artist_confidence = 0.0 if spotify_track.artists: for result_artist in spotify_track.artists: artist_sim = matching_engine.similarity_score( matching_engine.normalize_string(artist_name), matching_engine.normalize_string(result_artist) ) artist_confidence = max(artist_confidence, artist_sim) # Calculate title confidence title_confidence = matching_engine.similarity_score( matching_engine.normalize_string(title), matching_engine.normalize_string(spotify_track.name) ) # Combined confidence (50% artist + 50% title) combined_confidence = (artist_confidence * 0.5 + title_confidence * 0.5) # Small bonus for album tracks over singles _at = getattr(spotify_track, 'album_type', None) or '' if _at == 'album': combined_confidence += 0.02 elif _at == 'ep': combined_confidence += 0.01 logger.info(f"[Quality Scanner] Candidate: '{spotify_track.artists[0]}' - '{spotify_track.name}' (confidence: {combined_confidence:.3f})") # Update best match if this is better if combined_confidence > best_confidence and combined_confidence >= min_confidence: best_confidence = combined_confidence best_match = spotify_track logger.info(f"[Quality Scanner] New best match: {spotify_track.artists[0]} - {spotify_track.name} (confidence: {combined_confidence:.3f})") except Exception as e: logger.error(f"[Quality Scanner] Error scoring result: {e}") continue # If we found a very high confidence match, stop searching if best_confidence >= 0.9: logger.info(f"[Quality Scanner] High confidence match found ({best_confidence:.3f}), stopping search") break except Exception as e: logger.debug(f"[Quality Scanner] Error searching with query '{search_query}': {e}") continue # Process best match if best_match: matched = True logger.info(f"[Quality Scanner] Final match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})") # Build full Spotify track data for wishlist matched_track_data = { 'id': best_match.id, 'name': best_match.name, 'artists': [{'name': artist} for artist in best_match.artists], 'album': { 'name': best_match.album, 'artists': [{'name': artist} for artist in best_match.artists], 'album_type': 'album', # Default to 'album' for quality scanner matches 'release_date': getattr(best_match, 'release_date', '') or '' }, 'duration_ms': best_match.duration_ms, 'popularity': best_match.popularity, 'preview_url': best_match.preview_url, 'external_urls': best_match.external_urls or {} } # Add to wishlist source_context = { 'quality_scanner': True, 'original_file_path': file_path, 'original_format': tier_name, 'original_bitrate': bitrate, 'match_confidence': best_confidence, 'scan_date': datetime.now().isoformat() } success = wishlist_service.add_spotify_track_to_wishlist( spotify_track_data=matched_track_data, failure_reason=f"Low quality - {tier_name.replace('_', ' ').title()} format", source_type='quality_scanner', source_context=source_context, profile_id=profile_id ) if success: with quality_scanner_lock: quality_scanner_state["matched"] += 1 logger.info(f"[Quality Scanner] Matched and added to wishlist: {artist_name} - {title}") else: logger.error(f"[Quality Scanner] Failed to add to wishlist: {artist_name} - {title}") else: logger.warning(f"[Quality Scanner] No suitable match found (best confidence: {best_confidence:.3f}, required: {min_confidence:.3f})") except Exception as matching_error: logger.error(f"[Quality Scanner] Matching error for {artist_name} - {title}: {matching_error}") # Store result result_entry = { 'track_id': track_id, 'title': title, 'artist': artist_name, 'album': album_title, 'file_path': file_path, 'current_format': tier_name, 'bitrate': bitrate, 'matched': matched, 'spotify_id': matched_track_data['id'] if matched_track_data else None } with quality_scanner_lock: quality_scanner_state["results"].append(result_entry) if not matched: logger.warning(f"[Quality Scanner] No Spotify match found for: {artist_name} - {title}") except Exception as track_error: logger.error(f"[Quality Scanner] Error processing track: {track_error}") continue # Scan complete (don't overwrite if already stopped by user) with quality_scanner_lock: was_stopped = quality_scanner_state["status"] != "running" quality_scanner_state["status"] = "finished" quality_scanner_state["progress"] = 100 if not was_stopped: quality_scanner_state["phase"] = "Scan complete" logger.info(f"[Quality Scanner] Scan {'stopped' if was_stopped else 'complete'}: {quality_scanner_state['processed']} processed, " f"{quality_scanner_state['low_quality']} low quality, {quality_scanner_state['matched']} matched to Spotify") # Add activity add_activity_item("", "Quality Scan Complete", f"{quality_scanner_state['matched']} tracks added to wishlist", "Now") try: if automation_engine: automation_engine.emit('quality_scan_completed', { 'quality_met': str(quality_scanner_state.get('quality_met', 0)), 'low_quality': str(quality_scanner_state.get('low_quality', 0)), 'total_scanned': str(quality_scanner_state.get('processed', 0)), }) except Exception: pass except Exception as e: logger.error(f"[Quality Scanner] Critical error: {e}") import traceback traceback.print_exc() with quality_scanner_lock: quality_scanner_state["status"] = "error" quality_scanner_state["error_message"] = str(e) quality_scanner_state["phase"] = f"Error: {str(e)}" def _run_duplicate_cleaner(): """Main duplicate cleaner worker function - scans Transfer folder for duplicate files""" import os import shutil from collections import defaultdict from pathlib import Path try: with duplicate_cleaner_lock: duplicate_cleaner_state["status"] = "running" duplicate_cleaner_state["phase"] = "Initializing scan..." duplicate_cleaner_state["progress"] = 0 duplicate_cleaner_state["files_scanned"] = 0 duplicate_cleaner_state["total_files"] = 0 duplicate_cleaner_state["duplicates_found"] = 0 duplicate_cleaner_state["deleted"] = 0 duplicate_cleaner_state["space_freed"] = 0 duplicate_cleaner_state["error_message"] = "" logger.warning("[Duplicate Cleaner] Starting duplicate scan...") # Get Transfer folder path from config transfer_folder = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) if not transfer_folder or not os.path.exists(transfer_folder): with duplicate_cleaner_lock: duplicate_cleaner_state["status"] = "error" duplicate_cleaner_state["phase"] = "Output folder not configured or does not exist" duplicate_cleaner_state["error_message"] = "Please configure output folder in settings" logger.warning(f"[Duplicate Cleaner] Transfer folder not found: {transfer_folder}") return # Create deleted folder if it doesn't exist deleted_folder = os.path.join(transfer_folder, 'deleted') os.makedirs(deleted_folder, exist_ok=True) logger.warning(f"[Duplicate Cleaner] Deleted folder: {deleted_folder}") # Phase 1: Count total files for progress tracking with duplicate_cleaner_lock: duplicate_cleaner_state["phase"] = "Counting files..." total_files = 0 for _root, dirs, files in os.walk(transfer_folder): # Skip the deleted folder itself if 'deleted' in dirs: dirs.remove('deleted') total_files += len(files) logger.warning(f"[Duplicate Cleaner] Found {total_files} total files to scan") with duplicate_cleaner_lock: duplicate_cleaner_state["total_files"] = total_files duplicate_cleaner_state["phase"] = f"Scanning {total_files} files..." # Phase 2: Scan and group files by directory and filename # Structure: {directory_path: {filename_without_ext: [full_file_paths]}} files_by_dir_and_name = defaultdict(lambda: defaultdict(list)) files_scanned = 0 # Audio file extensions to consider audio_extensions = {'.flac', '.mp3', '.m4a', '.aac', '.opus', '.ogg', '.wav', '.ape', '.wma', '.alac', '.aiff', '.aif', '.dsf', '.dff'} for root, dirs, files in os.walk(transfer_folder): # Skip the deleted folder if 'deleted' in dirs: dirs.remove('deleted') for file in files: files_scanned += 1 # Update progress with duplicate_cleaner_lock: duplicate_cleaner_state["files_scanned"] = files_scanned duplicate_cleaner_state["progress"] = (files_scanned / total_files) * 100 if total_files > 0 else 0 duplicate_cleaner_state["phase"] = f"Scanning: {file}" # Get file extension file_path = os.path.join(root, file) file_name, file_ext = os.path.splitext(file) file_ext_lower = file_ext.lower() # Only process audio files if file_ext_lower not in audio_extensions: continue # Group by directory and filename (without extension) files_by_dir_and_name[root][file_name].append({ 'full_path': file_path, 'extension': file_ext_lower, 'size': os.path.getsize(file_path) }) # Phase 3: Process duplicates with duplicate_cleaner_lock: duplicate_cleaner_state["phase"] = "Processing duplicates..." # Quality priority: FLAC > OPUS/OGG > M4A/AAC > MP3/WMA format_priority = { '.flac': 1, '.ape': 1, '.wav': 1, '.alac': 1, '.aiff': 1, '.aif': 1, '.dsf': 1, '.dff': 1, # Lossless '.opus': 2, '.ogg': 2, # High quality lossy '.m4a': 3, '.aac': 3, # Standard lossy '.mp3': 4, '.wma': 4 # Lower quality lossy } duplicates_found = 0 deleted_count = 0 space_freed = 0 for directory, files_by_name in files_by_dir_and_name.items(): for filename, file_versions in files_by_name.items(): # Only process if we have duplicates (more than one version) if len(file_versions) <= 1: continue duplicates_found += len(file_versions) - 1 # Count all but the one we keep logger.warning(f"[Duplicate Cleaner] Found {len(file_versions)} versions of '{filename}' in {directory}") # Sort by priority: best format first, then largest size def sort_key(f): priority = format_priority.get(f['extension'], 999) size = f['size'] return (priority, -size) # Negative size for descending order sorted_versions = sorted(file_versions, key=sort_key) # Keep the first one (best quality), delete the rest best_version = sorted_versions[0] logger.warning(f"[Duplicate Cleaner] Keeping: {os.path.basename(best_version['full_path'])} " f"({best_version['extension']}, {best_version['size']} bytes)") for duplicate_file in sorted_versions[1:]: try: # Move to deleted folder with relative path preserved relative_path = os.path.relpath(duplicate_file['full_path'], transfer_folder) deleted_path = os.path.join(deleted_folder, relative_path) # Create subdirectories in deleted folder if needed os.makedirs(os.path.dirname(deleted_path), exist_ok=True) # Move the file shutil.move(duplicate_file['full_path'], deleted_path) # Track stats deleted_count += 1 space_freed += duplicate_file['size'] logger.warning(f"[Duplicate Cleaner] Moved to deleted: {os.path.basename(duplicate_file['full_path'])} " f"({duplicate_file['extension']}, {duplicate_file['size']} bytes)") # Update stats with duplicate_cleaner_lock: duplicate_cleaner_state["deleted"] = deleted_count duplicate_cleaner_state["space_freed"] = space_freed duplicate_cleaner_state["duplicates_found"] = duplicates_found except Exception as e: logger.error(f"[Duplicate Cleaner] Error moving file {duplicate_file['full_path']}: {e}") continue # Scan complete with duplicate_cleaner_lock: duplicate_cleaner_state["status"] = "finished" duplicate_cleaner_state["progress"] = 100 duplicate_cleaner_state["phase"] = "Cleaning complete" space_mb = space_freed / (1024 * 1024) logger.warning(f"[Duplicate Cleaner] Scan complete: {files_scanned} files scanned, " f"{duplicates_found} duplicates found, {deleted_count} files moved to deleted folder, " f"{space_mb:.2f} MB freed") # Add activity add_activity_item("", "Duplicate Cleaner Complete", f"{deleted_count} files removed, {space_mb:.1f} MB freed", "Now") try: if automation_engine: automation_engine.emit('duplicate_scan_completed', { 'files_scanned': str(files_scanned), 'duplicates_found': str(duplicates_found), 'space_freed': f"{space_mb:.1f} MB", }) except Exception: pass except Exception as e: logger.error(f"[Duplicate Cleaner] Critical error: {e}") import traceback traceback.print_exc() with duplicate_cleaner_lock: duplicate_cleaner_state["status"] = "error" duplicate_cleaner_state["error_message"] = str(e) duplicate_cleaner_state["phase"] = f"Error: {str(e)}" @app.route('/api/quality-scanner/start', methods=['POST']) def start_quality_scan(): """Start the quality scanner""" with quality_scanner_lock: if quality_scanner_state["status"] == "running": return jsonify({"success": False, "error": "A scan is already in progress"}), 409 data = request.get_json() or {} scope = data.get('scope', 'watchlist') # 'watchlist' or 'all' logger.info(f"[Quality Scanner API] Starting scan with scope: {scope}") # Reset state quality_scanner_state["status"] = "running" quality_scanner_state["phase"] = "Initializing..." quality_scanner_state["progress"] = 0 quality_scanner_state["processed"] = 0 quality_scanner_state["total"] = 0 quality_scanner_state["quality_met"] = 0 quality_scanner_state["low_quality"] = 0 quality_scanner_state["matched"] = 0 quality_scanner_state["results"] = [] quality_scanner_state["error_message"] = "" # Submit worker (capture profile_id before thread) scan_profile_id = get_current_profile_id() quality_scanner_executor.submit(_run_quality_scanner, scope, scan_profile_id) add_activity_item("", "Quality Scan Started", f"Scanning {scope} tracks", "Now") return jsonify({"success": True, "message": "Quality scan started"}) @app.route('/api/quality-scanner/status', methods=['GET']) def get_quality_scanner_status(): """Get current quality scanner status""" with quality_scanner_lock: return jsonify(quality_scanner_state) @app.route('/api/quality-scanner/stop', methods=['POST']) def stop_quality_scan(): """Stop the quality scanner (sets a stop flag)""" with quality_scanner_lock: if quality_scanner_state["status"] == "running": quality_scanner_state["status"] = "finished" quality_scanner_state["phase"] = "Scan stopped by user" return jsonify({"success": True, "message": "Stop request sent"}) else: return jsonify({"success": False, "error": "No scan is currently running"}), 404 @app.route('/api/duplicate-cleaner/start', methods=['POST']) def start_duplicate_cleaner(): """Start the duplicate cleaner""" with duplicate_cleaner_lock: if duplicate_cleaner_state["status"] == "running": return jsonify({"success": False, "error": "A scan is already in progress"}), 409 logger.warning("[Duplicate Cleaner API] Starting duplicate cleaner...") # Reset state duplicate_cleaner_state["status"] = "running" duplicate_cleaner_state["phase"] = "Initializing..." duplicate_cleaner_state["progress"] = 0 duplicate_cleaner_state["files_scanned"] = 0 duplicate_cleaner_state["total_files"] = 0 duplicate_cleaner_state["duplicates_found"] = 0 duplicate_cleaner_state["deleted"] = 0 duplicate_cleaner_state["space_freed"] = 0 duplicate_cleaner_state["error_message"] = "" # Submit worker duplicate_cleaner_executor.submit(_run_duplicate_cleaner) add_activity_item("", "Duplicate Cleaner Started", "Scanning Transfer folder", "Now") return jsonify({"success": True, "message": "Duplicate cleaner started"}) @app.route('/api/duplicate-cleaner/status', methods=['GET']) def get_duplicate_cleaner_status(): """Get current duplicate cleaner status""" with duplicate_cleaner_lock: # Convert space_freed from bytes to MB for display state_copy = duplicate_cleaner_state.copy() state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024) return jsonify(state_copy) @app.route('/api/duplicate-cleaner/stop', methods=['POST']) def stop_duplicate_cleaner(): """Stop the duplicate cleaner (sets a stop flag)""" with duplicate_cleaner_lock: if duplicate_cleaner_state["status"] == "running": duplicate_cleaner_state["status"] = "finished" duplicate_cleaner_state["phase"] = "Scan stopped by user" return jsonify({"success": True, "message": "Stop request sent"}) else: return jsonify({"success": False, "error": "No scan is currently running"}), 404 # =============================== # == RETAG TOOL ENDPOINTS == # =============================== @app.route('/api/retag/stats', methods=['GET']) def get_retag_stats(): """Get retag tool statistics for the dashboard card.""" from database.music_database import get_database db = get_database() stats = db.get_retag_stats() return jsonify({"success": True, **stats}) @app.route('/api/retag/groups', methods=['GET']) def get_retag_groups(): """Get all retag groups sorted by artist name.""" from database.music_database import get_database db = get_database() groups = db.get_retag_groups() return jsonify({"success": True, "groups": groups}) @app.route('/api/retag/groups//tracks', methods=['GET']) def get_retag_group_tracks(group_id): """Get tracks for a specific retag group.""" from database.music_database import get_database db = get_database() tracks = db.get_retag_tracks(group_id) return jsonify({"success": True, "tracks": tracks}) @app.route('/api/retag/search', methods=['GET']) def search_retag_albums(): """Search for albums to use for retagging (uses Spotify/iTunes fallback).""" query = request.args.get('q', '').strip() if not query: return jsonify({"success": False, "error": "Query parameter 'q' is required"}), 400 limit = min(int(request.args.get('limit', 12)), 50) try: results = spotify_client.search_albums(query, limit=limit) albums = [] for a in results: albums.append({ 'id': str(a.id), 'name': a.name, 'artist': ', '.join(a.artists) if a.artists else 'Unknown Artist', 'release_date': a.release_date or '', 'total_tracks': a.total_tracks, 'image_url': a.image_url, 'album_type': a.album_type or 'album' }) return jsonify({"success": True, "albums": albums}) except Exception as e: logger.error(f"[Retag] Album search error: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/retag/execute', methods=['POST']) def execute_retag(): """Start a retag operation for a group with a new album match.""" data = request.get_json() if not data: return jsonify({"success": False, "error": "JSON body required"}), 400 group_id = data.get('group_id') album_id = data.get('album_id') if not group_id or not album_id: return jsonify({"success": False, "error": "group_id and album_id are required"}), 400 with retag_lock: if retag_state["status"] == "running": return jsonify({"success": False, "error": "A retag operation is already running"}), 409 retag_executor.submit(_execute_retag, group_id, str(album_id)) return jsonify({"success": True, "message": "Retag operation started"}) @app.route('/api/retag/status', methods=['GET']) def get_retag_status(): """Get the current retag operation status.""" with retag_lock: return jsonify(dict(retag_state)) @app.route('/api/retag/groups/', methods=['DELETE']) def delete_retag_group(group_id): """Delete a retag group (files are NOT deleted).""" from database.music_database import get_database db = get_database() success = db.delete_retag_group(group_id) if success: return jsonify({"success": True}) else: return jsonify({"success": False, "error": "Group not found"}), 404 @app.route('/api/retag/groups/delete-batch', methods=['POST']) def delete_retag_groups_batch(): """Delete multiple retag groups at once.""" from database.music_database import get_database data = request.get_json() or {} group_ids = data.get('group_ids', []) if not group_ids: return jsonify({"success": False, "error": "No group IDs provided"}), 400 db = get_database() removed = 0 for gid in group_ids: if db.delete_retag_group(int(gid)): removed += 1 return jsonify({"success": True, "removed": removed}) @app.route('/api/retag/groups/clear-all', methods=['POST']) def clear_all_retag_groups(): """Delete all retag groups.""" from database.music_database import get_database db = get_database() count = db.delete_all_retag_groups() return jsonify({"success": True, "removed": count}) # =============================== # == DOWNLOAD MISSING TRACKS == # =============================== def get_valid_candidates(results, spotify_track, query): """ This function is a direct port from sync.py. It scores and filters Soulseek search results against a Spotify track to find the best, most accurate download candidates. """ if not results: return [] # Streaming sources (YouTube, Tidal, Qobuz, HiFi, Deezer) return structured API results # with proper artist/title metadata — score using the same matching engine as Soulseek _streaming_sources = ("youtube", "tidal", "qobuz", "hifi", "deezer_dl") if results[0].username in _streaming_sources: source_label = results[0].username.replace('_dl', '').title() expected_artists = spotify_track.artists if spotify_track else [] expected_title = spotify_track.name if spotify_track else '' expected_duration = spotify_track.duration_ms if spotify_track else 0 # Detect if the expected track is a specific version (live, remix, acoustic, etc.) expected_title_lower = (expected_title or '').lower() _version_keywords = ['remix', 'live', 'acoustic', 'instrumental', 'radio edit', 'extended', 'slowed', 'sped up', 'reverb', 'karaoke'] expected_is_version = any(kw in expected_title_lower for kw in _version_keywords) scored = [] for r in results: # Score using matching engine's generic scorer (same weights as Soulseek) confidence, match_type = matching_engine.score_track_match( source_title=expected_title, source_artists=expected_artists, source_duration_ms=expected_duration, candidate_title=r.title or '', candidate_artists=[r.artist] if r.artist else [], candidate_duration_ms=r.duration or 0, ) # Version detection penalty — reject live/remix/acoustic when expecting original r_title_lower = (r.title or '').lower() is_wrong_version = False if not expected_is_version: # Expecting original — penalize versions for kw in _version_keywords: if kw in r_title_lower and kw not in expected_title_lower: confidence *= 0.4 # Heavy penalty is_wrong_version = True break else: # Expecting specific version — penalize results that don't have it for kw in _version_keywords: if kw in expected_title_lower and kw not in r_title_lower: confidence *= 0.5 is_wrong_version = True break # Artist gate — streaming APIs (Tidal/Qobuz/HiFi/Deezer) have reliable metadata, # so "My Will" by "B. Starr" should never match expected "B小町". # Skip for YouTube — artist is parsed from video titles and often unreliable. if r.username != 'youtube': from difflib import SequenceMatcher import re as _re _cand_artist_raw = r.artist or '' _cand_artist = matching_engine.normalize_string(_cand_artist_raw) _best_artist = 0.0 for _ea in expected_artists: _ea_norm = matching_engine.normalize_string(_ea) if not _ea_norm: continue # For short normalized names (e.g. "B小町"→"b"), containment is useless. # Compare original Unicode strings directly via similarity instead. if len(_ea_norm) <= 2: _best_artist = max(_best_artist, SequenceMatcher(None, _ea.lower(), _cand_artist_raw.lower()).ratio()) elif _re.search(r'\b' + _re.escape(_ea_norm) + r'\b', _cand_artist): _best_artist = 1.0 break elif _ea_norm == _cand_artist: _best_artist = 1.0 break else: _best_artist = max(_best_artist, SequenceMatcher(None, _ea_norm, _cand_artist).ratio()) # Raised from 0.4 → 0.5 to close a fencepost bug: SequenceMatcher # returns exactly 0.400 for "maduk" vs "tom walker" (5 chars vs # 10 chars with 2 coincidental char matches), which bypassed the # strict `< 0.4` check and let Tom Walker through as a candidate # for a Maduk track. The word-boundary containment check above # already short-circuits legitimate formatting variations # ("Beatles"/"The Beatles", "Maduk"/"Maduk feat. X") to sim=1.0, # so falling to SequenceMatcher means the strings are genuinely # different. 0.5 gives a safer buffer without blocking real # matches that would have scored above 0.85 anyway. if _best_artist < 0.5 and confidence < 0.85: continue r.confidence = confidence r.version_type = 'wrong_version' if is_wrong_version else match_type if confidence >= 0.60: scored.append(r) if scored: # Sort by confidence (best match first) scored.sort(key=lambda x: x.confidence, reverse=True) best = scored[0] logger.info(f"[{source_label}] {len(scored)}/{len(results)} candidates passed validation " f"(best: {best.confidence:.2f} '{best.artist} - {best.title}')") return scored else: if results[0].username == 'youtube': logger.warning(f"[{source_label}] No streaming results passed validation — falling through to filename matching") # YouTube artist data is unreliable, allow fallback to filename-based matching else: logger.warning(f"[{source_label}] No streaming results passed validation (threshold: 0.60, artist gate: 0.50) — rejecting all candidates") return [] # Tidal/Qobuz/HiFi/Deezer have structured metadata; don't fall back to filename matching # Uses the existing, powerful matching engine for scoring (Soulseek P2P results) _max_q = config_manager.get('soulseek.max_peer_queue', 0) or 0 initial_candidates = matching_engine.find_best_slskd_matches_enhanced(spotify_track, results, max_peer_queue=_max_q) if not initial_candidates: return [] # Skip quality filtering for streaming source results that somehow got here is_streaming_source = initial_candidates[0].username in _streaming_sources if initial_candidates else False if is_streaming_source: source_label = initial_candidates[0].username.title() logger.info(f"[{source_label}] Skipping quality filter - streaming source handles quality internally") quality_filtered_candidates = initial_candidates else: # Filter by user's quality profile before artist verification (Soulseek only) # Use existing soulseek_client to avoid re-initializing (which accesses download_path filesystem) quality_filtered_candidates = soulseek_client.soulseek.filter_results_by_quality_preference(initial_candidates) # IMPORTANT: Respect empty results from quality filter # If user has strict quality requirements (e.g., FLAC-only with fallback disabled), # and no results match, we should fail the download rather than force a fallback. # The quality filter already has its own fallback logic controlled by the user's settings. if not quality_filtered_candidates: logger.error("[Quality Filter] No candidates match quality profile - download will fail per user preferences") return [] verified_candidates = [] spotify_artists = spotify_track.artists if spotify_track.artists else [] # Pre-normalize all artist names into word sets using the matching engine # This handles Cyrillic, accents, special chars ($), separators, etc. artist_word_sets = [] for artist_name in spotify_artists: normalized = matching_engine.normalize_string(artist_name) words = set(normalized.split()) if words: artist_word_sets.append(words) for candidate in quality_filtered_candidates: # Skip artist check for streaming results (title matching is sufficient as processed by matching engine) if is_streaming_source: verified_candidates.append(candidate) continue # No artist info available — can't verify, accept candidate if not artist_word_sets: verified_candidates.append(candidate) continue # Split the Soulseek path into segments (folders + filename) and check each one. # This prevents false positives where a short artist name like "Sia" accidentally # matches inside a folder name like "Enthusiastic" — by checking words within # individual segments rather than a flat substring of the entire path. path_segments = re.split(r'[/\\]', candidate.filename) artist_found = False for segment in path_segments: if not segment: continue seg_words = set(matching_engine.normalize_string(segment).split()) if not seg_words: continue # Check if ANY artist's words are ALL present in this segment for artist_words in artist_word_sets: if artist_words.issubset(seg_words): artist_found = True break if artist_found: break if artist_found: verified_candidates.append(candidate) return verified_candidates def _recover_worker_slot(batch_id, task_id): """ Emergency worker slot recovery function for when normal completion callback fails. This prevents permanent worker slot leaks that cause modal to show wrong worker counts. """ try: logger.warning(f"[Worker Recovery] Attempting to recover worker slot for batch {batch_id}, task {task_id}") # Acquire lock with timeout to prevent deadlock lock_acquired = tasks_lock.acquire(timeout=3.0) if not lock_acquired: logger.error("[Worker Recovery] FATAL: Could not acquire lock for recovery - worker slot LEAKED") return False try: # Verify batch still exists if batch_id not in download_batches: logger.warning(f"[Worker Recovery] Batch {batch_id} not found - nothing to recover") return True batch = download_batches[batch_id] old_active = batch['active_count'] # Only decrement if there are active workers to prevent negative counts if old_active > 0: batch['active_count'] -= 1 new_active = batch['active_count'] logger.warning(f"[Worker Recovery] Recovered worker slot - Active count: {old_active} → {new_active}") # Try to start next worker if queue isn't empty if batch['queue_index'] < len(batch['queue']) and new_active < batch['max_concurrent']: logger.warning("[Worker Recovery] Attempting to start replacement worker") # Release lock temporarily to avoid deadlock in _start_next_batch_of_downloads tasks_lock.release() try: _start_next_batch_of_downloads(batch_id) finally: # Re-acquire lock for final cleanup tasks_lock.acquire(timeout=2.0) return True else: logger.warning("[Worker Recovery] Active count already 0 - no recovery needed") return True finally: tasks_lock.release() except Exception as recovery_error: logger.error(f"[Worker Recovery] FATAL ERROR in recovery: {recovery_error}") return False def _get_batch_lock(batch_id): """Get or create a lock for a specific batch to prevent race conditions""" with tasks_lock: if batch_id not in batch_locks: batch_locks[batch_id] = threading.Lock() return batch_locks[batch_id] # Batch lifecycle logic lives in core/downloads/lifecycle.py. from core.downloads import lifecycle as _downloads_lifecycle def _build_lifecycle_deps(): """Build LifecycleDeps bundle from web_server.py globals on each call.""" return _downloads_lifecycle.LifecycleDeps( config_manager=config_manager, automation_engine=automation_engine, download_monitor=download_monitor, repair_worker=repair_worker, mb_worker=mb_worker, is_shutting_down=lambda: IS_SHUTTING_DOWN, get_batch_lock=_get_batch_lock, submit_download_track_worker=lambda task_id, batch_id: missing_download_executor.submit( _download_track_worker, task_id, batch_id, ), submit_failed_to_wishlist=lambda batch_id: missing_download_executor.submit( _process_failed_tracks_to_wishlist_exact, batch_id, ), submit_failed_to_wishlist_with_auto_completion=lambda batch_id: missing_download_executor.submit( _process_failed_tracks_to_wishlist_exact_with_auto_completion, batch_id, ), process_failed_to_wishlist=_process_failed_tracks_to_wishlist_exact, process_failed_to_wishlist_with_auto_completion=_process_failed_tracks_to_wishlist_exact_with_auto_completion, ensure_spotify_track_format=_ensure_spotify_track_format, get_track_artist_name=_get_track_artist_name, check_and_remove_from_wishlist=_check_and_remove_from_wishlist, regenerate_batch_m3u=_regenerate_batch_m3u, youtube_playlist_states=youtube_playlist_states, tidal_discovery_states=tidal_discovery_states, deezer_discovery_states=deezer_discovery_states, spotify_public_discovery_states=spotify_public_discovery_states, ) def _start_next_batch_of_downloads(batch_id): """Start the next batch of downloads up to the concurrent limit.""" _downloads_lifecycle.start_next_batch_of_downloads(batch_id, _build_lifecycle_deps()) def _get_track_artist_name(track_info): """Extract artist name from track info, handling different data formats (replicating sync.py)""" if not track_info: return "Unknown Artist" # Handle Spotify API format with artists array artists = track_info.get('artists', []) if artists and len(artists) > 0: if isinstance(artists[0], dict) and 'name' in artists[0]: return artists[0]['name'] elif isinstance(artists[0], str): return artists[0] # Fallback to single artist field artist = track_info.get('artist') if artist: return artist return "Unknown Artist" def _ensure_spotify_track_format(track_info): """ Ensure track_info has proper Spotify track structure for wishlist service. Converts webui track format to match sync.py's spotify_track format. """ if not track_info: return {} # If it already has the proper Spotify structure, return as-is if isinstance(track_info.get('artists'), list) and len(track_info.get('artists', [])) > 0: first_artist = track_info['artists'][0] if isinstance(first_artist, dict) and 'name' in first_artist: # Already has proper Spotify format return track_info # Convert to proper Spotify format artists_list = [] # Handle different artist formats from webui artists = track_info.get('artists', []) if artists: if isinstance(artists, list): for artist in artists: if isinstance(artist, dict) and 'name' in artist: artists_list.append({'name': artist['name']}) elif isinstance(artist, str): artists_list.append({'name': artist}) else: artists_list.append({'name': str(artist)}) else: # Single artist as string artists_list.append({'name': str(artists)}) else: # Fallback: try single artist field artist = track_info.get('artist') if artist: artists_list.append({'name': str(artist)}) else: artists_list.append({'name': 'Unknown Artist'}) # Build album object — preserve ALL fields (id, release_date, total_tracks, # album_type, images, etc.) so wishlist tracks retain full album context # for correct folder placement, multi-disc support, and classification album_data = track_info.get('album', {}) if isinstance(album_data, dict): album = dict(album_data) # Copy all fields album.setdefault('name', 'Unknown Album') else: album = { 'name': str(album_data) if album_data else track_info.get('name', 'Unknown Album'), 'album_type': 'single', 'total_tracks': 1, 'release_date': '', } album.setdefault('images', []) album.setdefault('album_type', 'album') album.setdefault('total_tracks', 0) # Build proper Spotify track structure spotify_track = { 'id': track_info.get('id', f"webui_{hash(str(track_info))}"), 'name': track_info.get('name', 'Unknown Track'), 'artists': artists_list, # Proper Spotify format 'album': album, 'duration_ms': track_info.get('duration_ms', 0), 'track_number': track_info.get('track_number', 1), 'disc_number': track_info.get('disc_number', 1), 'preview_url': track_info.get('preview_url'), 'external_urls': track_info.get('external_urls', {}), 'popularity': track_info.get('popularity', 0), 'source': 'webui_modal' # Mark as coming from webui } return spotify_track def _process_failed_tracks_to_wishlist_exact(batch_id): """ Process failed and cancelled tracks to wishlist - EXACT replication of sync.py's on_all_downloads_complete() logic. This matches sync.py's behavior precisely. """ try: from core.wishlist_service import get_wishlist_service from datetime import datetime logger.info(f"[Wishlist Processing] Starting wishlist processing for batch {batch_id}") with tasks_lock: if batch_id not in download_batches: logger.warning(f"[Wishlist Processing] Batch {batch_id} not found") return {'tracks_added': 0, 'errors': 0} batch = download_batches[batch_id] # Wing It mode — skip wishlist entirely for failed tracks if batch.get('wing_it'): failed_count = len(batch.get('permanently_failed_tracks', [])) logger.error(f"[Wing It] Skipping wishlist for {failed_count} failed tracks (wing it mode)") return {'tracks_added': 0, 'errors': 0} permanently_failed_tracks = batch.get('permanently_failed_tracks', []) cancelled_tracks = batch.get('cancelled_tracks', set()) # STEP 0: Remove completed tracks from wishlist (THIS WAS MISSING!) logger.info("[Wishlist Processing] Checking completed tracks for wishlist removal") for task_id in batch.get('queue', []): if task_id in download_tasks: task = download_tasks[task_id] if task.get('status') == 'completed': try: track_info = task.get('track_info', {}) context = {'track_info': track_info, 'original_search_result': track_info} _check_and_remove_from_wishlist(context) except Exception as e: logger.error(f"[Wishlist Processing] Error removing completed track from wishlist: {e}") # STEP 1: Add cancelled tracks that were missing to permanently_failed_tracks (replicating sync.py) # This matches sync.py's logic for adding cancelled missing tracks to the failed list if cancelled_tracks: logger.warning(f"[Wishlist Processing] Processing {len(cancelled_tracks)} cancelled tracks") # Process cancelled tracks with safeguard to prevent infinite loops processed_count = 0 max_process = 100 # Safety limit with tasks_lock: for task_id in batch.get('queue', [])[:max_process]: # Limit processing if task_id in download_tasks: task = download_tasks[task_id] track_index = task.get('track_index', 0) if track_index in cancelled_tracks: # Check if track was actually missing (not successfully downloaded) task_status = task.get('status', 'unknown') if task_status != 'completed': # Build cancelled track info matching sync.py format original_track_info = task.get('track_info', {}) spotify_track_data = _ensure_spotify_track_format(original_track_info) cancelled_track_info = { 'download_index': track_index, 'table_index': track_index, 'track_name': original_track_info.get('name', 'Unknown Track'), 'artist_name': _get_track_artist_name(original_track_info), 'retry_count': 0, 'spotify_track': spotify_track_data, # Properly formatted spotify track 'failure_reason': 'Download cancelled', 'candidates': task.get('cached_candidates', []) } # Check if not already in permanently_failed_tracks (sync.py does this check) if not any(t.get('table_index') == track_index for t in permanently_failed_tracks): permanently_failed_tracks.append(cancelled_track_info) processed_count += 1 logger.error(f"[Wishlist Processing] Added cancelled missing track {cancelled_track_info['track_name']} to failed list for wishlist") logger.warning(f"[Wishlist Processing] Processed {processed_count} cancelled tracks") # STEP 1.5: Recover any failed/not_found tasks not captured in permanently_failed_tracks. # Stuck detection (in _on_download_completed, _check_batch_completion_v2, and the Safety Valve) # can force-mark tasks as not_found/failed without adding them to permanently_failed_tracks, # causing them to silently skip wishlist processing. with tasks_lock: for task_id in batch.get('queue', []): if task_id in download_tasks: task = download_tasks[task_id] if task['status'] in ('failed', 'not_found'): track_index = task.get('track_index', 0) if not any(t.get('table_index') == track_index for t in permanently_failed_tracks): original_track_info = task.get('track_info', {}) spotify_track_data = _ensure_spotify_track_format(original_track_info) recovered_track_info = { 'download_index': track_index, 'table_index': track_index, 'track_name': original_track_info.get('name', 'Unknown Track'), 'artist_name': _get_track_artist_name(original_track_info), 'retry_count': task.get('retry_count', 0), 'spotify_track': spotify_track_data, 'failure_reason': task.get('error_message', 'Download failed'), 'candidates': task.get('cached_candidates', []) } permanently_failed_tracks.append(recovered_track_info) logger.error(f"[Wishlist Processing] Recovered uncaptured failed track for wishlist: {recovered_track_info['track_name']}") # STEP 2: Add permanently failed tracks to wishlist (exact sync.py logic) failed_count = len(permanently_failed_tracks) wishlist_added_count = 0 error_count = 0 logger.error(f"[Wishlist Processing] Processing {failed_count} failed tracks for wishlist") if permanently_failed_tracks: try: wishlist_service = get_wishlist_service() # Create source_context identical to sync.py source_context = { 'playlist_name': batch.get('playlist_name', 'Unknown Playlist'), 'playlist_id': batch.get('playlist_id', None), 'added_from': 'webui_modal', # Distinguish from sync_page_modal 'timestamp': datetime.now().isoformat() } # Process each failed track (matching sync.py's loop) with safety limit max_failed_tracks = min(len(permanently_failed_tracks), 50) # Safety limit wing_it_skipped = 0 for i, failed_track_info in enumerate(permanently_failed_tracks[:max_failed_tracks]): try: track_name = failed_track_info.get('track_name', f'Track {i+1}') # Skip wing-it fallback tracks — they had no real metadata match, # so adding them to wishlist would just retry with the same raw data. # Check the track ID prefix since _ensure_spotify_track_format overwrites source. sp_track = failed_track_info.get('spotify_track', {}) sp_id = sp_track.get('id', '') if isinstance(sp_track, dict) else '' if str(sp_id).startswith('wing_it_'): wing_it_skipped += 1 logger.info(f"[Wishlist Processing] Skipping wing-it track: {track_name}") continue logger.error(f"[Wishlist Processing] Adding track {i+1}/{max_failed_tracks}: {track_name}") success = wishlist_service.add_failed_track_from_modal( track_info=failed_track_info, source_type='playlist', source_context=source_context, profile_id=batch.get('profile_id', 1) ) if success: wishlist_added_count += 1 logger.info(f"[Wishlist Processing] Added {track_name} to wishlist") try: if automation_engine: automation_engine.emit('wishlist_item_added', { 'artist': failed_track_info.get('artist_name', ''), 'title': track_name, 'reason': failed_track_info.get('failure_reason', ''), }) except Exception: pass else: logger.error(f"[Wishlist Processing] Failed to add {track_name} to wishlist") except Exception as e: error_count += 1 logger.error(f"[Wishlist Processing] Exception adding track to wishlist: {e}") if wing_it_skipped: logger.warning(f"[Wishlist Processing] Skipped {wing_it_skipped} wing-it fallback tracks") logger.error(f"[Wishlist Processing] Added {wishlist_added_count}/{failed_count} failed tracks to wishlist (errors: {error_count})") except Exception as e: error_count = len(permanently_failed_tracks) logger.error(f"[Wishlist Processing] Critical error adding failed tracks to wishlist: {e}") import traceback traceback.print_exc() else: logger.error("ℹ️ [Wishlist Processing] No failed tracks to add to wishlist") # Store completion summary in batch for API response (matching sync.py pattern) completion_summary = { 'tracks_added': wishlist_added_count, 'errors': error_count, 'total_failed': failed_count } with tasks_lock: if batch_id in download_batches: download_batches[batch_id]['wishlist_summary'] = completion_summary download_batches[batch_id]['wishlist_processing_complete'] = True # Phase already set to 'complete' in _on_download_completed logger.info(f"[Wishlist Processing] Completed wishlist processing for batch {batch_id}") # Auto-cleanup: Clear completed downloads from slskd try: logger.info(f"[Auto-Cleanup] Clearing completed downloads from slskd after batch {batch_id}") run_async(soulseek_client.clear_all_completed_downloads()) logger.info("[Auto-Cleanup] Completed downloads cleared from slskd") except Exception as cleanup_error: logger.warning(f"[Auto-Cleanup] Failed to clear completed downloads: {cleanup_error}") # Sweep empty directories left behind by this batch's downloads try: _sweep_empty_download_directories() except Exception as sweep_error: logger.warning(f"[Auto-Cleanup] Failed to sweep empty directories: {sweep_error}") return completion_summary except Exception as e: logger.error(f"[Wishlist Processing] CRITICAL ERROR in wishlist processing: {e}") import traceback traceback.print_exc() # Mark batch as complete even with errors to prevent infinite loops try: with tasks_lock: if batch_id in download_batches: download_batches[batch_id]['phase'] = 'complete' download_batches[batch_id]['completion_time'] = time.time() # Track for auto-cleanup download_batches[batch_id]['wishlist_summary'] = { 'tracks_added': 0, 'errors': 1, 'total_failed': 0, 'error_message': str(e) } download_batches[batch_id]['wishlist_processing_complete'] = True except Exception as lock_error: logger.error(f"[Wishlist Processing] Failed to update batch after error: {lock_error}") return {'tracks_added': 0, 'errors': 1, 'total_failed': 0} def _process_failed_tracks_to_wishlist_exact_with_auto_completion(batch_id): """ Process failed tracks to wishlist for auto-initiated batches and handle auto-processing completion. This extends the standard processing with automatic scheduling of the next cycle. """ global wishlist_auto_processing try: logger.info(f"[Auto-Wishlist] Processing completion for auto-initiated batch {batch_id}") # Run standard wishlist processing completion_summary = _process_failed_tracks_to_wishlist_exact(batch_id) # Log auto-processing completion tracks_added = completion_summary.get('tracks_added', 0) total_failed = completion_summary.get('total_failed', 0) logger.error(f"[Auto-Wishlist] Background processing complete: {tracks_added} added to wishlist, {total_failed} failed") # Add activity for wishlist processing if tracks_added > 0: add_activity_item("", "Wishlist Updated", f"{tracks_added} failed tracks added to wishlist", "Now") # TOGGLE CYCLE: Switch to next category for next run try: with tasks_lock: if batch_id in download_batches: current_cycle = download_batches[batch_id].get('current_cycle', 'albums') else: current_cycle = 'albums' # Default fallback next_cycle = 'singles' if current_cycle == 'albums' else 'albums' from database.music_database import MusicDatabase db = MusicDatabase() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" INSERT OR REPLACE INTO metadata (key, value, updated_at) VALUES ('wishlist_cycle', ?, CURRENT_TIMESTAMP) """, (next_cycle,)) conn.commit() logger.info(f"[Auto-Wishlist] Cycle toggled after completion: {current_cycle} → {next_cycle}") except Exception as cycle_error: logger.error(f"[Auto-Wishlist] Error toggling cycle: {cycle_error}") # Mark auto-processing as complete and reset timestamp with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 try: if automation_engine: automation_engine.emit('wishlist_processing_completed', { 'tracks_processed': str(total_failed), 'tracks_found': str(tracks_added), 'tracks_failed': str(total_failed - tracks_added), }) except Exception: pass return completion_summary except Exception as e: logger.error(f"[Auto-Wishlist] Error in auto-completion processing: {e}") import traceback traceback.print_exc() # Ensure auto-processing flag is reset even on error and reset timestamp with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return {'tracks_added': 0, 'errors': 1, 'total_failed': 0} def _on_download_completed(batch_id, task_id, success=True): """Called when a download completes to start the next one in queue.""" _downloads_lifecycle.on_download_completed(batch_id, task_id, success, _build_lifecycle_deps()) # Master worker for the missing tracks pipeline lives in core/downloads/master.py. from core.downloads import master as _downloads_master def _build_master_deps(): """Build the MasterDeps bundle from web_server.py globals on each call.""" def _reset_wishlist_auto_processing(): global wishlist_auto_processing, wishlist_auto_processing_timestamp with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 return _downloads_master.MasterDeps( config_manager=config_manager, soulseek_client=soulseek_client, run_async=run_async, mb_worker=mb_worker, mb_release_cache=mb_release_cache, mb_release_cache_lock=mb_release_cache_lock, mb_release_detail_cache=mb_release_detail_cache, mb_release_detail_cache_lock=mb_release_detail_cache_lock, normalize_album_cache_key=normalize_album_cache_key, check_and_remove_track_from_wishlist_by_metadata=_check_and_remove_track_from_wishlist_by_metadata, is_explicit_blocked=_is_explicit_blocked, youtube_playlist_states=youtube_playlist_states, tidal_discovery_states=tidal_discovery_states, deezer_discovery_states=deezer_discovery_states, spotify_public_discovery_states=spotify_public_discovery_states, missing_download_executor=missing_download_executor, process_failed_tracks_to_wishlist_exact_with_auto_completion=_process_failed_tracks_to_wishlist_exact_with_auto_completion, source_reuse_logger=source_reuse_logger, download_monitor=download_monitor, start_next_batch_of_downloads=_start_next_batch_of_downloads, reset_wishlist_auto_processing=_reset_wishlist_auto_processing, ) def _run_full_missing_tracks_process(batch_id, playlist_id, tracks_json): return _downloads_master.run_full_missing_tracks_process( batch_id, playlist_id, tracks_json, _build_master_deps() ) # Post-processing verification worker logic lives in core/downloads/post_processing.py. from core.downloads import post_processing as _downloads_post_processing def _build_post_processing_deps(): """Build the PostProcessDeps bundle from web_server.py globals on each call.""" return _downloads_post_processing.PostProcessDeps( config_manager=config_manager, soulseek_client=soulseek_client, run_async=run_async, docker_resolve_path=docker_resolve_path, extract_filename=extract_filename, make_context_key=_make_context_key, find_completed_file=_find_completed_file_robust, enhance_file_metadata=_enhance_file_metadata, wipe_source_tags=_wipe_source_tags, post_process_with_verification=_post_process_matched_download_with_verification, mark_task_completed=_mark_task_completed, on_download_completed=_on_download_completed, ) def _run_post_processing_worker(task_id, batch_id): """Post-processing verification worker — see core/downloads/post_processing.py.""" _downloads_post_processing.run_post_processing_worker(task_id, batch_id, _build_post_processing_deps()) # Per-task download worker logic lives in core/downloads/task_worker.py. from core.downloads import task_worker as _downloads_task_worker def _build_task_worker_deps(): """Build TaskWorkerDeps bundle from web_server.py globals on each call.""" return _downloads_task_worker.TaskWorkerDeps( soulseek_client=soulseek_client, matching_engine=matching_engine, run_async=run_async, try_source_reuse=_try_source_reuse, store_batch_source=_store_batch_source, try_staging_match=_try_staging_match, get_valid_candidates=get_valid_candidates, attempt_download_with_candidates=_attempt_download_with_candidates, on_download_completed=lambda b, t, success: _on_download_completed(b, t, success=success), recover_worker_slot=_recover_worker_slot, ) def _download_track_worker(task_id, batch_id=None): """Per-task download worker — see core/downloads/task_worker.py.""" _downloads_task_worker.download_track_worker(task_id, batch_id, _build_task_worker_deps()) def _attempt_download_with_candidates(task_id, candidates, track, batch_id=None): """ Attempts to download with fallback candidate logic (matches GUI's retry_parallel_download_with_fallback). Returns True if successful, False if all candidates fail. """ # Sort candidates by confidence (best first) candidates.sort(key=lambda r: r.confidence, reverse=True) with tasks_lock: task = download_tasks.get(task_id) if not task: return False used_sources = task.get('used_sources', set()) # Try each candidate until one succeeds (like GUI's fallback logic) for candidate_index, candidate in enumerate(candidates): # Check cancellation before each attempt with tasks_lock: if task_id not in download_tasks: logger.info(f"[Modal Worker] Task {task_id} was deleted during candidate {candidate_index + 1}") return False if download_tasks[task_id]['status'] == 'cancelled': logger.warning(f"[Modal Worker] Task {task_id} cancelled during candidate {candidate_index + 1}") # Don't call _on_download_completed for cancelled tasks as it can stop monitoring return False download_tasks[task_id]['current_candidate_index'] = candidate_index # Create source key to avoid duplicate attempts (like GUI) source_key = f"{candidate.username}_{candidate.filename}" if source_key in used_sources: logger.info(f"[Modal Worker] Skipping already tried source: {source_key}") continue # Blacklist check — skip sources the user has flagged as bad matches try: _bl_db = get_database() if _bl_db.is_blacklisted(candidate.username, candidate.filename): logger.info(f"[Modal Worker] Skipping blacklisted source: {source_key}") continue except Exception: pass # CRITICAL: Add source to used_sources IMMEDIATELY to prevent race conditions # This must happen BEFORE starting download to prevent multiple retries from picking same source with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['used_sources'].add(source_key) logger.info(f"[Modal Worker] Marked source as used before download attempt: {source_key}") logger.info(f"[Modal Worker] Trying candidate {candidate_index + 1}/{len(candidates)}: {candidate.filename} (Confidence: {candidate.confidence:.2f})") try: # Update task status to downloading _update_task_status(task_id, 'downloading') # Prepare download - check if we have explicit album context from artist page track_info = {} with tasks_lock: if task_id in download_tasks: raw_track_info = download_tasks[task_id].get('track_info') track_info = raw_track_info if isinstance(raw_track_info, dict) else {} # Use explicit album/artist context if available (from artist album downloads) has_explicit_context = track_info and track_info.get('_is_explicit_album_download', False) if has_explicit_context: # Use the real Spotify album/artist data from the UI explicit_album = track_info.get('_explicit_album_context', {}) explicit_artist = track_info.get('_explicit_artist_context', {}) # Normalize artist context if it's a plain string (e.g. from wishlist spotify_data) if isinstance(explicit_artist, str): explicit_artist = {'name': explicit_artist} spotify_artist_context = { 'id': explicit_artist.get('id', 'explicit_artist'), 'name': explicit_artist.get('name', track.artists[0] if track.artists else 'Unknown'), 'genres': explicit_artist.get('genres', []) } # Handle both image_url formats (direct string or images array) album_image_url = None if explicit_album.get('image_url'): # Backend API returns image_url as direct string album_image_url = explicit_album.get('image_url') elif explicit_album.get('images'): # Fallback: images array format from Spotify API album_image_url = explicit_album.get('images', [{}])[0].get('url') spotify_album_context = { 'id': explicit_album.get('id', 'explicit_album'), 'name': explicit_album.get('name', track.album), 'release_date': explicit_album.get('release_date', ''), 'image_url': album_image_url, 'total_tracks': explicit_album.get('total_tracks', 0), 'total_discs': explicit_album.get('total_discs', 1), 'album_type': explicit_album.get('album_type', 'album'), 'artists': explicit_album.get('artists', [{'name': spotify_artist_context.get('name', '')}]) } logger.info(f"[Explicit Context] Using real album data: '{spotify_album_context['name']}' ({spotify_album_context['album_type']}, {spotify_album_context['total_discs']} disc(s))") else: # Fallback to generic context for playlists/wishlists # Extract album metadata from track_info if available (discovery enriches tracks with full album objects) fallback_album = track_info.get('album', {}) if track_info else {} if isinstance(fallback_album, str): fallback_album = {'name': fallback_album} elif not isinstance(fallback_album, dict): fallback_album = {} fallback_image_url = None fallback_images = fallback_album.get('images', []) if fallback_album.get('image_url'): fallback_image_url = fallback_album['image_url'] elif fallback_images and isinstance(fallback_images, list) and len(fallback_images) > 0: fallback_image_url = fallback_images[0].get('url') if isinstance(fallback_images[0], dict) else None spotify_artist_context = {'id': 'from_sync_modal', 'name': track.artists[0] if track.artists else 'Unknown', 'genres': []} # Preserve album-level artists for consistent folder naming _fallback_album_artists = fallback_album.get('artists', []) if not _fallback_album_artists: _fallback_album_artists = [{'name': track.artists[0]}] if track.artists else [] spotify_album_context = { 'id': fallback_album.get('id', 'from_sync_modal'), 'name': fallback_album.get('name', '') or track.album, 'release_date': fallback_album.get('release_date', ''), 'image_url': fallback_image_url, 'album_type': fallback_album.get('album_type', 'album'), 'total_tracks': fallback_album.get('total_tracks', 0), 'total_discs': fallback_album.get('total_discs', 1), 'artists': _fallback_album_artists } download_payload = candidate.__dict__ username = download_payload.get('username') filename = download_payload.get('filename') size = download_payload.get('size', 0) if not username or not filename: logger.error("[Modal Worker] Invalid candidate data: missing username or filename") continue # PROTECTION: Check if there's already an active download for this task current_download_id = None with tasks_lock: if task_id in download_tasks: current_download_id = download_tasks[task_id].get('download_id') if current_download_id: logger.info(f"[Modal Worker] Task {task_id} already has active download {current_download_id} - skipping new download attempt") logger.info("[Modal Worker] This prevents race condition where multiple retries start overlapping downloads") continue # Initiate download logger.info(f"[Modal Worker] Starting download: {username} / {os.path.basename(filename)}") download_id = run_async(soulseek_client.download(username, filename, size)) if download_id: # Store context for post-processing with complete Spotify metadata (GUI PARITY) context_key = _make_context_key(username, filename) with matched_context_lock: # Create WebUI equivalent of GUI's SpotifyBasedSearchResult data structure enhanced_payload = download_payload.copy() # Extract clean Spotify metadata from track object (same as GUI) has_clean_spotify_data = track and hasattr(track, 'name') and hasattr(track, 'album') if has_clean_spotify_data: # Use clean Spotify metadata (matches GUI's SpotifyBasedSearchResult) enhanced_payload['spotify_clean_title'] = track.name enhanced_payload['spotify_clean_album'] = track.album enhanced_payload['spotify_clean_artist'] = track.artists[0] if track.artists else enhanced_payload.get('artist', '') # Preserve all artists for metadata tagging enhanced_payload['artists'] = [{'name': artist} for artist in track.artists] if track.artists else [] logger.info(f"[Context] Using clean Spotify metadata - Album: '{track.album}', Title: '{track.name}'") # Get track_number and disc_number — prefer track data we already have, # fall back to detailed API call only if needed got_track_number = False # 1. Try track_info (from frontend, has album track data) tn = track_info.get('track_number', 0) if isinstance(track_info, dict) else 0 dn = track_info.get('disc_number', 1) if isinstance(track_info, dict) else 1 if tn and tn > 0: enhanced_payload['track_number'] = tn enhanced_payload['disc_number'] = dn got_track_number = True logger.info(f"[Context] Added track_number from track_info: {tn}, disc_number: {dn}") # 2. Try the track object itself (from album tracks response) if not got_track_number and hasattr(track, 'track_number') and track.track_number: enhanced_payload['track_number'] = track.track_number enhanced_payload['disc_number'] = getattr(track, 'disc_number', 1) or 1 got_track_number = True logger.info(f"[Context] Added track_number from track object: {track.track_number}, disc_number: {enhanced_payload['disc_number']}") # 3. Last resort — fetch from metadata source API if not got_track_number and hasattr(track, 'id') and track.id: try: detailed_track = spotify_client.get_track_details(track.id) if detailed_track and detailed_track.get('track_number'): enhanced_payload['track_number'] = detailed_track['track_number'] enhanced_payload['disc_number'] = detailed_track.get('disc_number', 1) got_track_number = True logger.info(f"[Context] Added track_number from API: {detailed_track['track_number']}, disc_number: {enhanced_payload['disc_number']}") # Backfill album metadata from detailed track when context # has incomplete data (missing release_date, total_tracks, etc.) if isinstance(detailed_track.get('album'), dict): dt_album = detailed_track['album'] if not spotify_album_context.get('release_date') and dt_album.get('release_date'): spotify_album_context['release_date'] = dt_album['release_date'] logger.info(f"[Context] Backfilled release_date from API: {dt_album['release_date']}") if not spotify_album_context.get('album_type') and dt_album.get('album_type'): spotify_album_context['album_type'] = dt_album['album_type'] if not spotify_album_context.get('total_tracks') and dt_album.get('total_tracks'): spotify_album_context['total_tracks'] = dt_album['total_tracks'] if not spotify_album_context.get('id') and dt_album.get('id'): spotify_album_context['id'] = dt_album['id'] if not spotify_album_context.get('image_url') and dt_album.get('images'): spotify_album_context['image_url'] = dt_album['images'][0].get('url', '') except Exception as e: logger.error(f"[Context] API track details failed: {e}") if not got_track_number: enhanced_payload.setdefault('track_number', 0) enhanced_payload.setdefault('disc_number', 1) logger.warning("[Context] No track_number found from any source") # Determine if this should be treated as album download # First check if we have explicit album context from artist page if has_explicit_context: is_album_context = True logger.info("[Context] Using explicit album context flag from artist page") else: # Fall back to guessing based on clean data is_album_context = ( track.album and track.album.strip() and track.album != "Unknown Album" and track.album.lower() != track.name.lower() # Album different from track ) else: # Fallback to original data enhanced_payload['spotify_clean_title'] = enhanced_payload.get('title', '') enhanced_payload['spotify_clean_album'] = enhanced_payload.get('album', '') enhanced_payload['spotify_clean_artist'] = enhanced_payload.get('artist', '') # Preserve existing artists array if available, otherwise create from single artist if 'artists' not in enhanced_payload and enhanced_payload.get('artist'): enhanced_payload['artists'] = [{'name': enhanced_payload['artist']}] enhanced_payload['track_number'] = track_info.get('track_number', 1) # Fallback when no clean Spotify data is_album_context = False logger.warning(f"[Context] Using fallback data - no clean Spotify metadata available, track_number={enhanced_payload['track_number']}") matched_downloads_context[context_key] = { "spotify_artist": spotify_artist_context, "spotify_album": spotify_album_context, "original_search_result": enhanced_payload, "is_album_download": is_album_context, # Critical fix: Use actual album context "has_clean_spotify_data": has_clean_spotify_data, # Flag for post-processing "task_id": task_id, # Add task_id for completion callbacks "batch_id": batch_id, # Add batch_id for completion callbacks "track_info": track_info, # Add track_info for playlist folder mode "_download_username": username, # Source username for AcoustID skip logic } logger.info(f"[Context] Set is_album_download: {is_album_context} (has clean data: {has_clean_spotify_data})") logger.debug(f"[Debug] Context creation - track_info: {track_info is not None}, playlist_folder_mode: {track_info.get('_playlist_folder_mode', False) if track_info else False}") # Update task with successful download info with tasks_lock: if task_id in download_tasks: # PHASE 3: Final cancellation check after download started (GUI PARITY) if download_tasks[task_id]['status'] == 'cancelled': logger.warning(f"[Modal Worker] Task {task_id} cancelled after download {download_id} started - attempting to cancel download") # Try to cancel the download immediately try: run_async(soulseek_client.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled active download {download_id}") except Exception as cancel_error: logger.error(f"Failed to cancel active download {download_id}: {cancel_error}") # Free worker slot if batch_id: _on_download_completed(batch_id, task_id, success=False) return False # Store download information - use real download ID from soulseek_client # CRITICAL FIX: Trust the download ID returned by soulseek_client.download() download_tasks[task_id]['download_id'] = download_id download_tasks[task_id]['username'] = username download_tasks[task_id]['filename'] = filename logger.info(f"[Modal Worker] Download started successfully for '{filename}'. Download ID: {download_id}") return True # Success! else: logger.error(f"[Modal Worker] Failed to start download for '{filename}'") # Reset status back to searching for next attempt with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'searching' continue except Exception as e: import traceback logger.error(f"[Modal Worker] Error attempting download for '{candidate.filename}': {e}") traceback.print_exc() # Reset status back to searching for next attempt with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'searching' continue # All candidates failed logger.error(f"[Modal Worker] All {len(candidates)} candidates failed for '{track.name}'") return False # ── Staging folder match cache (per-batch, avoids re-scanning for every track) ── _staging_cache = {} # batch_id -> list of {full_path, title, artist, album, extension} _staging_cache_lock = threading.Lock() STAGING_AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif'} def _get_staging_file_cache(batch_id): """Scan staging folder once per batch and cache the results.""" with _staging_cache_lock: if batch_id in _staging_cache: return _staging_cache[batch_id] staging_path = get_staging_path() if not os.path.isdir(staging_path): with _staging_cache_lock: _staging_cache[batch_id] = [] return [] files = [] for root, _dirs, filenames in os.walk(staging_path): for fname in filenames: ext = os.path.splitext(fname)[1].lower() if ext not in STAGING_AUDIO_EXTENSIONS: continue full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) meta = _read_staging_file_metadata(full_path, rel_path) files.append({ 'full_path': full_path, 'title': meta['title'] or '', 'artist': meta['albumartist'] or meta['artist'] or '', 'album': meta['album'] or '', 'extension': ext, }) logger.info(f"[Staging] Scanned {len(files)} audio files in staging folder") with _staging_cache_lock: _staging_cache[batch_id] = files return files def _try_staging_match(task_id, batch_id, track): """Check if a matching file exists in the staging folder before downloading. Returns True if a match was found and the file was moved to the transfer folder. Returns False to fall through to normal download. """ staging_files = _get_staging_file_cache(batch_id or task_id) if not staging_files: return False track_title = track.name or '' track_artist = track.artists[0] if track.artists else '' if not track_title: return False from difflib import SequenceMatcher normalize = matching_engine.normalize_string norm_title = normalize(track_title) norm_artist = normalize(track_artist) best_match = None best_score = 0.0 for sf in staging_files: sf_norm_title = normalize(sf['title']) sf_norm_artist = normalize(sf['artist']) if not sf_norm_title: continue # Title similarity (primary) title_sim = SequenceMatcher(None, norm_title, sf_norm_title).ratio() if title_sim < 0.80: continue # Artist similarity (secondary) artist_sim = 0.0 if norm_artist and sf_norm_artist: artist_sim = SequenceMatcher(None, norm_artist, sf_norm_artist).ratio() elif not norm_artist and not sf_norm_artist: artist_sim = 0.5 # Both unknown — neutral elif norm_artist and not sf_norm_artist: artist_sim = 0.3 # Staging file lacks artist — partial credit if title is strong elif sf_norm_artist and not norm_artist: artist_sim = 0.3 # Track lacks artist — same partial credit # Combined score: title-weighted (these are user-curated staging files) # If artist info is available, require it to match. If not, lean on title. if norm_artist and sf_norm_artist: combined = (title_sim * 0.55) + (artist_sim * 0.45) else: combined = (title_sim * 0.80) + (artist_sim * 0.20) if combined > best_score: best_score = combined best_match = sf # Require high confidence to avoid false positives if not best_match or best_score < 0.75: return False logger.info(f"[Staging] Match found for '{track_title}' by '{track_artist}': " f"{os.path.basename(best_match['full_path'])} (score: {best_score:.2f})") # Copy the file to the transfer folder try: transfer_dir = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) dest_filename = os.path.basename(best_match['full_path']) dest_path = os.path.join(transfer_dir, dest_filename) os.makedirs(transfer_dir, exist_ok=True) # Don't overwrite existing files if os.path.exists(dest_path): base, ext = os.path.splitext(dest_filename) dest_path = os.path.join(transfer_dir, f"{base}_staging{ext}") import shutil shutil.copy2(best_match['full_path'], dest_path) logger.info(f"[Staging] Copied to transfer: {dest_path}") # Mark task as completed with staging context with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'post_processing' download_tasks[task_id]['filename'] = dest_path download_tasks[task_id]['username'] = 'staging' download_tasks[task_id]['staging_match'] = True # Run post-processing (tagging, AcoustID verification, path building) context_key = f"staging_{task_id}" with tasks_lock: track_info = download_tasks.get(task_id, {}).get('track_info', {}) if not isinstance(track_info, dict): track_info = {} # Build spotify_artist / spotify_album context so post-processing can apply # the path template. Without these, _post_process_matched_download returns # early and the file stays at the transfer root with its original filename. # Mirror the context-building logic from the sync modal worker. has_explicit_context = track_info.get('_is_explicit_album_download', False) if has_explicit_context: explicit_artist = track_info.get('_explicit_artist_context', {}) if isinstance(explicit_artist, str): explicit_artist = {'name': explicit_artist} elif not isinstance(explicit_artist, dict): explicit_artist = {} spotify_artist_ctx = { 'id': explicit_artist.get('id', 'staging'), 'name': explicit_artist.get('name', track_artist), 'genres': explicit_artist.get('genres', []) } explicit_album = track_info.get('_explicit_album_context', {}) if not isinstance(explicit_album, dict): explicit_album = {} _album_image_url = explicit_album.get('image_url') if not _album_image_url and explicit_album.get('images'): _imgs = explicit_album['images'] if isinstance(_imgs, list) and _imgs: _album_image_url = _imgs[0].get('url') if isinstance(_imgs[0], dict) else None spotify_album_ctx = { 'id': explicit_album.get('id', 'staging'), 'name': explicit_album.get('name', getattr(track, 'album', '') or ''), 'release_date': explicit_album.get('release_date', ''), 'image_url': _album_image_url, 'album_type': explicit_album.get('album_type', 'album'), 'total_tracks': explicit_album.get('total_tracks', 0), 'total_discs': explicit_album.get('total_discs', 1), 'artists': explicit_album.get('artists', [{'name': spotify_artist_ctx.get('name', '')}]) } is_album_ctx = True has_clean_data = True else: fallback_album = track_info.get('album', {}) if isinstance(fallback_album, str): fallback_album = {'name': fallback_album} elif not isinstance(fallback_album, dict): fallback_album = {} track_album_name = getattr(track, 'album', '') or fallback_album.get('name', '') or '' spotify_artist_ctx = { 'id': 'staging', 'name': track_artist or 'Unknown', 'genres': [] } spotify_album_ctx = { 'id': 'staging', 'name': track_album_name, 'release_date': fallback_album.get('release_date', ''), 'image_url': fallback_album.get('image_url'), 'album_type': fallback_album.get('album_type', 'album'), 'total_tracks': fallback_album.get('total_tracks', 0), 'total_discs': fallback_album.get('total_discs', 1), 'artists': [{'name': track_artist}] if track_artist else [] } is_album_ctx = bool( track_album_name and track_album_name.strip() and track_album_name.lower() not in ('unknown album', '') and track_album_name.lower() != track_title.lower() ) has_clean_data = bool(track_title and track_artist and track_album_name) track_number = ( track_info.get('track_number', 0) or getattr(track, 'track_number', 0) or 0 ) disc_number = ( track_info.get('disc_number', 1) or getattr(track, 'disc_number', 1) or 1 ) context = { 'track_info': track_info, 'spotify_artist': spotify_artist_ctx, 'spotify_album': spotify_album_ctx, 'original_search_result': { 'title': track_title, 'artist': track_artist, 'spotify_clean_title': track_title, 'spotify_clean_album': spotify_album_ctx.get('name', ''), 'spotify_clean_artist': track_artist, 'track_number': track_number, 'disc_number': disc_number, }, 'is_album_download': is_album_ctx, 'has_clean_spotify_data': has_clean_data, 'staging_source': True, } # Store context in the matched downloads context store (used by post-processing) with matched_context_lock: matched_downloads_context[context_key] = context # Trigger post-processing which handles tagging, path building, and DB insertion _post_process_matched_download_with_verification(context_key, context, dest_path, task_id, batch_id) return True except Exception as e: logger.error(f"[Staging] Failed to use staging file: {e}") return False def _try_source_reuse(task_id, batch_id, track): """ Check batch's last_good_source for the current track before searching. Returns True if source reuse succeeded, False to fall through to normal search. """ _sr = source_reuse_logger _sr.info(f"_try_source_reuse called: task={task_id}, batch={batch_id}, track={track.name}") if not batch_id: _sr.info("Skipped — no batch_id") return False with tasks_lock: batch = download_batches.get(batch_id) if not batch: _sr.info(f"Skipped — batch {batch_id} not found") return False # Gate: album/EP downloads only is_album = batch.get('is_album_download', False) is_wishlist = batch.get('playlist_id', '') == 'wishlist' if not is_album and not is_wishlist: _sr.info(f"Skipped — not album ({is_album}) and not wishlist ({is_wishlist})") return False source_tracks = batch.get('source_folder_tracks') last_source = batch.get('last_good_source') _sr.info(f"Batch state: last_good_source={last_source}, source_folder_tracks={'None' if source_tracks is None else f'{len(source_tracks)} tracks'}") if not source_tracks or not last_source: _sr.info("Skipped — no source_tracks or no last_source") return False if last_source.get('username') in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): _sr.info(f"Skipped — {last_source.get('username')} source (no folder-based reuse)") return False source_username = last_source.get('username') source_folder = last_source.get('folder_path', '') source_key = f"{source_username}:{source_folder}" # Check if this source+folder has already failed for this batch with tasks_lock: batch = download_batches.get(batch_id, {}) failed_sources = batch.get('failed_sources', set()) if source_key in failed_sources: _sr.info(f"Source {source_key} already in failed_sources — skipping") with tasks_lock: if batch_id in download_batches: download_batches[batch_id]['last_good_source'] = None download_batches[batch_id]['source_folder_tracks'] = None return False # Detect retry: if this task already tried the stored source and failed (monitor resubmitted), # the task will still have the previous username/download_id from the failed attempt with tasks_lock: task = download_tasks.get(task_id, {}) prev_username = task.get('username') prev_download_id = task.get('download_id') if prev_username and prev_download_id and prev_username == source_username: _sr.info(f"Task {task_id} already failed from source {source_key} — marking as failed") with tasks_lock: if batch_id in download_batches: if 'failed_sources' not in download_batches[batch_id]: download_batches[batch_id]['failed_sources'] = set() download_batches[batch_id]['failed_sources'].add(source_key) download_batches[batch_id]['last_good_source'] = None download_batches[batch_id]['source_folder_tracks'] = None return False _sr.info(f"Checking reused source for task {task_id}: {source_key}") # Score each folder track against current track candidates = [] for folder_track in source_tracks: confidence = matching_engine.calculate_slskd_match_confidence(track, folder_track) _sr.info(f" Match '{track.name}' vs '{folder_track.filename}' → confidence={confidence:.3f}") if confidence >= 0.70: folder_track.confidence = confidence candidates.append(folder_track) if not candidates: _sr.info(f"No folder tracks matched above 0.70 for task {task_id}") return False # Sort by confidence, filter by quality preference candidates.sort(key=lambda c: c.confidence, reverse=True) _sr.info(f"Found {len(candidates)} candidates above 0.70, best={candidates[0].confidence:.3f} ({candidates[0].filename})") slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client filtered = slsk.filter_results_by_quality_preference(candidates) if not filtered: _sr.info(f"Quality filter rejected all candidates for task {task_id}") return False _sr.info(f"After quality filter: {len(filtered)} candidates remain") # Artist verification artist_name = track.artists[0].lower() if track.artists else '' verified = [c for c in filtered if artist_name and artist_name in c.filename.lower().replace('\\', '/')] final_candidates = verified if verified else filtered[:1] _sr.info(f"Artist verification: artist='{artist_name}', verified={len(verified)}, using={len(final_candidates)} candidates") # Initialize task state for download attempt # IMPORTANT: Preserve used_sources from previous attempts (e.g. monitor error retries) # so that _attempt_download_with_candidates skips sources that already failed. with tasks_lock: if task_id in download_tasks: download_tasks[task_id]['status'] = 'searching' download_tasks[task_id]['current_query_index'] = 0 download_tasks[task_id]['current_candidate_index'] = 0 download_tasks[task_id]['retry_count'] = 0 download_tasks[task_id]['candidates'] = [] # Don't reset used_sources — the download monitor marks failed sources here # Attempt download success = _attempt_download_with_candidates(task_id, final_candidates, track, batch_id) if success: _sr.info(f"SUCCESS — Downloaded from reused source for task {task_id}") return True # Source failed — mark as failed so it's never tried again for this batch with tasks_lock: if batch_id in download_batches: if 'failed_sources' not in download_batches[batch_id]: download_batches[batch_id]['failed_sources'] = set() download_batches[batch_id]['failed_sources'].add(source_key) download_batches[batch_id]['last_good_source'] = None download_batches[batch_id]['source_folder_tracks'] = None _sr.info(f"FAILED — Source {source_key} failed for task {task_id}, added to failed_sources") return False def _store_batch_source(batch_id, username, filename): """Browse the successful download's folder and store results on the batch for reuse.""" _sr = source_reuse_logger _sr.info(f"_store_batch_source called: batch={batch_id}, user={username}, file={filename}") if not batch_id or username in ('youtube', 'tidal', 'qobuz', 'hifi', 'deezer_dl', 'lidarr'): _sr.info(f"Skipped — no batch_id or streaming source ({username})") return with tasks_lock: batch = download_batches.get(batch_id) if not batch: _sr.info("Skipped — batch not found") return is_album = batch.get('is_album_download', False) is_wishlist = batch.get('playlist_id', '') == 'wishlist' if not is_album and not is_wishlist: _sr.info(f"Skipped — not album ({is_album}) and not wishlist ({is_wishlist})") return # Don't store a source+folder that already failed for this batch failed_sources = batch.get('failed_sources', set()) # Extract folder path from filename — preserve original separators for slskd API if '\\' in filename: folder_path = filename.rsplit('\\', 1)[0] elif '/' in filename: folder_path = filename.rsplit('/', 1)[0] else: _sr.info(f"Skipped — no folder separator in filename: {filename}") return # Check failed_sources with username:folder key source_key = f"{username}:{folder_path}" if source_key in failed_sources: _sr.info(f"Not storing source {source_key} — already in failed_sources") return try: # Access SoulseekClient directly (soulseek_client is DownloadOrchestrator) slsk = soulseek_client.soulseek if hasattr(soulseek_client, 'soulseek') else soulseek_client _sr.info(f"Browsing {username}:{folder_path}...") files = run_async(slsk.browse_user_directory(username, folder_path)) if not files: _sr.info(f"Browse returned no files for {username}:{folder_path}") return _sr.info(f"Browse returned {len(files)} raw files") tracks = slsk.parse_browse_results_to_tracks(username, files, directory=folder_path) if not tracks: _sr.info(f"No audio tracks after parsing for {username}:{folder_path}") return _sr.info(f"Parsed {len(tracks)} audio tracks from {username}:{folder_path}") with tasks_lock: if batch_id in download_batches: download_batches[batch_id]['last_good_source'] = { 'username': username, 'folder_path': folder_path } download_batches[batch_id]['source_folder_tracks'] = tracks _sr.info(f"STORED {len(tracks)} tracks from {username}:{folder_path} as last_good_source") except Exception as e: _sr.info(f"EXCEPTION browsing source folder: {e}") import traceback _sr.info(traceback.format_exc()) @app.route('/api/playlists//download_missing', methods=['POST']) def start_playlist_missing_downloads(playlist_id): """ This endpoint receives the list of missing tracks and manages them with batch processing like the GUI, maintaining exactly 3 concurrent downloads. """ dl_err = check_download_permission() if dl_err: return dl_err data = request.get_json() missing_tracks = data.get('missing_tracks', []) if not missing_tracks: return jsonify({"success": False, "error": "No missing tracks provided"}), 400 # Filter explicit tracks if content filter is enabled if not config_manager.get('content_filter.allow_explicit', True): before_count = len(missing_tracks) missing_tracks = [t for t in missing_tracks if not _is_explicit_blocked(t.get('track', t))] skipped = before_count - len(missing_tracks) if skipped > 0: logger.warning(f"[Content Filter] Filtered out {skipped} explicit track(s) from playlist download") if not missing_tracks: return jsonify({"success": False, "error": "All tracks were filtered by explicit content setting"}), 400 # Add activity for playlist download missing start playlist_name = data.get('playlist_name', f'Playlist {playlist_id}') add_activity_item("", "Missing Tracks Download Started", f"'{playlist_name}' - {len(missing_tracks)} tracks", "Now") try: batch_id = str(uuid.uuid4()) # Create task queue for this batch task_queue = [] with tasks_lock: # Initialize batch management download_batches[batch_id] = { 'queue': [], 'active_count': 0, 'max_concurrent': _get_max_concurrent(), 'queue_index': 0, # Track state management (replicating sync.py) 'permanently_failed_tracks': [], 'cancelled_tracks': set(), # Profile context for failed track wishlist re-adds 'profile_id': get_current_profile_id() } for i, track_entry in enumerate(missing_tracks): task_id = str(uuid.uuid4()) # Extract track data and original track index from frontend track_data = track_entry.get('track', track_entry) # Support both old and new format original_track_index = track_entry.get('track_index', i) # Use original index or fallback to enumeration download_tasks[task_id] = { 'status': 'pending', 'track_info': track_data, 'playlist_id': playlist_id, 'batch_id': batch_id, 'track_index': original_track_index, # Use original playlist track index 'download_id': None, 'username': None, 'filename': None, # Retry-related fields (GUI parity) 'retry_count': 0, 'cached_candidates': [], 'used_sources': set(), 'status_change_time': time.time() } # Add to batch queue instead of submitting immediately download_batches[batch_id]['queue'].append(task_id) # Start background monitoring for timeouts and retries (GUI parity) download_monitor.start_monitoring(batch_id) # Start the first batch of downloads (up to 3) _start_next_batch_of_downloads(batch_id) return jsonify({"success": True, "batch_id": batch_id, "message": f"Queued {len(missing_tracks)} downloads for processing."}) except Exception as e: logger.error(f"Error starting missing downloads: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/active-processes', methods=['GET']) def get_active_processes(): """ Returns all active processes for frontend rehydration: - Download batch processes (Spotify playlists) - YouTube discovery/sync processes (non-fresh phases) """ active_processes = [] # Add active download batch processes with tasks_lock: for batch_id, batch_data in download_batches.items(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled']: process_info = { "type": "batch", "playlist_id": batch_data.get('playlist_id'), "playlist_name": batch_data.get('playlist_name'), "batch_id": batch_id, "phase": batch_data.get('phase') } # Enhanced wishlist information for better frontend state management if batch_data.get('playlist_id') == 'wishlist': process_info.update({ "auto_initiated": batch_data.get('auto_initiated', False), "auto_processing_timestamp": batch_data.get('auto_processing_timestamp'), "should_show_modal": True, # Wishlist processes should always be visible "is_background_process": batch_data.get('auto_initiated', False), "current_cycle": batch_data.get('current_cycle') # Pass category filter to frontend }) # Add current auto-processing state for frontend awareness with wishlist_timer_lock: process_info["auto_processing_active"] = wishlist_auto_processing active_processes.append(process_info) # Add YouTube playlists in non-fresh phases for rehydration for url_hash, state in youtube_playlist_states.items(): # Include playlists that have progressed beyond fresh phase if state['phase'] != 'fresh': active_processes.append({ "type": "youtube_playlist", "url_hash": url_hash, "url": state['url'], "playlist_name": state['playlist']['name'], "phase": state['phase'], "status": state['status'], "discovery_progress": state['discovery_progress'], "spotify_matches": state['spotify_matches'], "spotify_total": state['spotify_total'], "converted_spotify_playlist_id": state.get('converted_spotify_playlist_id'), "download_process_id": state.get('download_process_id') # batch_id for download modal rehydration }) logger.info(f"Active processes check: {len([p for p in active_processes if p['type'] == 'batch'])} download batches, {len([p for p in active_processes if p['type'] == 'youtube_playlist'])} YouTube playlists") return jsonify({"active_processes": active_processes}) # Status payload helpers live in core/downloads/status.py. from core.downloads import status as _downloads_status def _build_status_deps(): """Build StatusDeps bundle from web_server.py globals on each call.""" return _downloads_status.StatusDeps( config_manager=config_manager, docker_resolve_path=docker_resolve_path, find_completed_file=_find_completed_file_robust, make_context_key=_make_context_key, submit_post_processing=lambda task_id, batch_id: missing_download_executor.submit( _run_post_processing_worker, task_id, batch_id ), get_cached_transfer_data=get_cached_transfer_data, ) def _build_batch_status_data(batch_id, batch, live_transfers_lookup): """Helper function to build status data for a single batch. Extracted from get_batch_download_status for reuse in batched endpoint. """ return _downloads_status.build_batch_status_data(batch_id, batch, live_transfers_lookup, _build_status_deps()) @app.route('/api/playlists//download_status', methods=['GET']) def get_batch_download_status(batch_id): """Returns real-time status for a single batch.""" try: body, status = _downloads_status.build_single_batch_status(batch_id, _build_status_deps()) return jsonify(body), status except Exception as e: import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/download_status/batch', methods=['GET']) def get_batched_download_statuses(): """Returns status for multiple download batches in one request.""" try: requested_batch_ids = request.args.getlist('batch_ids') return jsonify(_downloads_status.build_batched_status(requested_batch_ids, _build_status_deps())) except Exception as e: import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/downloads/all', methods=['GET']) def get_all_downloads_unified(): """Unified downloads list for the centralized Downloads page.""" try: limit = int(request.args.get('limit', 200)) return jsonify(_downloads_status.build_unified_downloads_response(limit, _build_status_deps())) except Exception as e: logger.error(f"Error getting unified downloads: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/downloads/batch-history', methods=['GET']) def get_batch_history(): """Return completed batch summaries from the last N days for the batch panel history section.""" try: days = int(request.args.get('days', 7)) limit = int(request.args.get('limit', 50)) database = get_database() history = database.get_recent_batch_history(days=days, limit=limit) return jsonify({'success': True, 'history': history}) except Exception as e: logger.error(f"Error getting batch history: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/downloads/clear-completed', methods=['POST']) def clear_completed_downloads(): """Remove completed/failed/cancelled tasks from the download tracker.""" try: cleared = _downloads_cancel.clear_completed_local() return jsonify({'success': True, 'cleared': cleared}) except Exception as e: logger.error(f"Error clearing completed downloads: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/downloads/cancel_task', methods=['POST']) def cancel_download_task(): """ Cancels a single, specific download task. This version is now identical to the GUI, adding the cancelled track to the wishlist for future automatic retries. """ data = request.get_json() task_id = data.get('task_id') if not task_id: return jsonify({"success": False, "error": "Missing task_id"}), 400 try: with tasks_lock: if task_id not in download_tasks: return jsonify({"success": False, "error": "Task not found"}), 404 task = download_tasks[task_id] # Log current task state for debugging current_status = task.get('status', 'unknown') download_id = task.get('download_id') username = task.get('username') logger.info(f"[Cancel Debug] Task {task_id} - Current status: '{current_status}', download_id: {download_id}, username: {username}") # Immediately mark as cancelled to prevent race conditions task['status'] = 'cancelled' # IMPROVED WORKER SLOT MANAGEMENT: Use batch state validation instead of task status batch_id = task.get('batch_id') worker_slot_freed = False if batch_id: try: # Check if we need to free a worker slot by examining batch state with tasks_lock: if batch_id in download_batches: batch = download_batches[batch_id] active_count = batch['active_count'] # Free worker slot if there are active workers and task was actively running # This is more reliable than checking task status which can be inconsistent if active_count > 0 and current_status in ['pending', 'searching', 'downloading', 'queued']: logger.info(f"[Cancel] Task {task_id} (status: {current_status}) - freeing worker slot for batch {batch_id}") logger.info(f"[Cancel] Active count before: {active_count}") # Use the completion callback with error handling _on_download_completed(batch_id, task_id, success=False) worker_slot_freed = True # Verify slot was actually freed new_active = download_batches[batch_id]['active_count'] logger.info(f"[Cancel] Active count after: {new_active}") elif active_count == 0: logger.warning(f"[Cancel] Task {task_id} - no active workers to free") else: logger.warning(f"[Cancel] Task {task_id} (status: {current_status}) - not actively running, no slot to free") else: logger.warning(f"[Cancel] Task {task_id} - batch {batch_id} not found") except Exception as slot_error: logger.error(f"[Cancel] Error managing worker slot for {task_id}: {slot_error}") # Attempt emergency recovery if normal completion failed if not worker_slot_freed: try: logger.warning("[Cancel] Attempting emergency worker slot recovery") _recover_worker_slot(batch_id, task_id) except Exception as recovery_error: logger.error(f"[Cancel] FATAL: Emergency recovery failed: {recovery_error}") else: logger.warning(f"[Cancel] Task {task_id} cancelled (no batch_id - likely already completed)") # Optionally try to cancel the Soulseek download (don't block worker progression) if download_id and username: try: # This is an async call, so we run it and wait run_async(soulseek_client.cancel_download(download_id, username, remove=True)) logger.warning(f"Successfully cancelled Soulseek download {download_id} for task {task_id}") except Exception as e: logger.error(f"Failed to cancel download on slskd, but worker already moved on: {e}") ### NEW LOGIC START: Add cancelled track to wishlist ### try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() # The task dictionary contains all the necessary info track_info = task.get('track_info', {}) # The wishlist service expects a dictionary with specific keys # We need to properly format the artists to avoid nested structures artists_data = track_info.get('artists', []) formatted_artists = [] for artist in artists_data: if isinstance(artist, str): # Already a string, use as-is formatted_artists.append({'name': artist}) elif isinstance(artist, dict): # Check if it's already in the correct format if 'name' in artist and isinstance(artist['name'], str): # Already properly formatted formatted_artists.append(artist) elif 'name' in artist and isinstance(artist['name'], dict) and 'name' in artist['name']: # Nested structure, extract the inner name formatted_artists.append({'name': artist['name']['name']}) else: # Fallback: convert to string formatted_artists.append({'name': str(artist)}) else: # Fallback for any other type formatted_artists.append({'name': str(artist)}) # Build album data - preserve all fields (including artists) for correct folder placement album_raw = track_info.get('album', {}) if isinstance(album_raw, dict): album_data = dict(album_raw) # Copy all fields including artists album_data.setdefault('name', 'Unknown Album') album_data.setdefault('album_type', track_info.get('album_type', 'album')) else: album_data = { 'name': str(album_raw) if album_raw else 'Unknown Album', 'album_type': track_info.get('album_type', 'album') } spotify_track_data = { 'id': track_info.get('id'), 'name': track_info.get('name'), 'artists': formatted_artists, 'album': album_data, 'duration_ms': track_info.get('duration_ms') } source_context = { 'playlist_name': task.get('playlist_name', 'Unknown Playlist'), 'playlist_id': task.get('playlist_id'), 'added_from': 'modal_cancellation' } # Add to wishlist, treating cancellation as a failure # Pass the spotify data directly instead of creating a fake Track object success = wishlist_service.add_spotify_track_to_wishlist( spotify_track_data=spotify_track_data, failure_reason="Download cancelled by user", source_type="playlist", source_context=source_context, profile_id=get_current_profile_id() ) if success: logger.warning(f"Added cancelled track '{track_info.get('name')}' to wishlist.") else: logger.error(f"Failed to add cancelled track '{track_info.get('name')}' to wishlist.") except Exception as e: logger.error(f"CRITICAL ERROR adding cancelled track to wishlist: {e}") ### NEW LOGIC END ### return jsonify({"success": True, "message": "Task cancelled and added to wishlist for retry."}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # =============================== # NEW ATOMIC CANCEL SYSTEM V2 # =============================== def _find_task_by_playlist_track(playlist_id, track_index): """ Find task_id by playlist_id and track_index. This enables the new v2 API to work without requiring task_id from frontend. """ for task_id, task in download_tasks.items(): if (task.get('playlist_id') == playlist_id and task.get('track_index') == track_index): return task_id, task return None, None def _atomic_cancel_task(playlist_id, track_index): """ Atomically cancel a single task with proper worker slot management. This is the core of the new cancel system - everything in one transaction. Returns: (success: bool, message: str, task_info: dict) """ try: # Find the task to cancel task_id, task = _find_task_by_playlist_track(playlist_id, track_index) if not task_id: return False, f"Task not found for playlist {playlist_id}, track {track_index}", None # Check if already cancelled if task.get('status') == 'cancelled': return False, "Task already cancelled", {'task_id': task_id, 'status': 'cancelled'} current_status = task.get('status', 'unknown') original_status = current_status # Store original status before changing it batch_id = task.get('batch_id') logger.info(f"[Atomic Cancel] Starting atomic cancel: playlist={playlist_id}, track={track_index}, task={task_id}, status={current_status}") # Mark task as cancelled immediately (within same lock context) task['status'] = 'cancelled' task['cancel_requested'] = True task['cancel_timestamp'] = __import__('time').time() task['ui_state'] = 'cancelled' # Ensure task has persistent identifiers for V2 system if 'playlist_id' not in task: task['playlist_id'] = playlist_id # Handle worker slot management worker_slot_freed = False if batch_id and batch_id in download_batches: batch = download_batches[batch_id] active_count = batch['active_count'] # Free worker slot if task was consuming one # More precise check: only free if task was actually running if active_count > 0 and current_status in ['pending', 'searching', 'downloading', 'queued']: logger.info(f"[Atomic Cancel] Freeing worker slot for {task_id} (was {current_status})") # CRITICAL: Direct worker slot management to prevent _on_download_completed race old_active = batch['active_count'] batch['active_count'] = max(0, old_active - 1) # Prevent negative counts worker_slot_freed = True logger.info(f"[Atomic Cancel] Worker count: {old_active} → {batch['active_count']}") # Try to start next task if available (still within lock) if (batch['queue_index'] < len(batch['queue']) and batch['active_count'] < batch['max_concurrent']): logger.info("[Atomic Cancel] Starting next task in queue") # Call the existing function to start next downloads # Note: This will be called outside the lock to prevent deadlock else: logger.warning(f"[Atomic Cancel] No next task to start (queue_index: {batch['queue_index']}/{len(batch['queue'])}, active: {batch['active_count']}/{batch['max_concurrent']})") # Build result info task_info = { 'task_id': task_id, 'status': 'cancelled', 'original_status': original_status, # Pass original status for slskd cancellation 'track_name': task.get('track_info', {}).get('name', 'Unknown'), 'playlist_id': playlist_id, 'track_index': track_index, 'worker_slot_freed': worker_slot_freed } logger.warning(f"[Atomic Cancel] Successfully cancelled task {task_id}") return True, "Task cancelled successfully", task_info except Exception as e: logger.error(f"[Atomic Cancel] Error in atomic cancel: {e}") import traceback traceback.print_exc() return False, f"Internal error: {str(e)}", None @app.route('/api/downloads/cancel_task_v2', methods=['POST']) def cancel_task_v2(): """ NEW ATOMIC CANCEL SYSTEM V2 Accepts playlist_id and track_index instead of task_id. Performs atomic cancellation with proper worker slot management. No race conditions, no dual state management. """ data = request.get_json() playlist_id = data.get('playlist_id') track_index = data.get('track_index') if not playlist_id or track_index is None: return jsonify({ "success": False, "error": "Missing playlist_id or track_index" }), 400 try: # Everything in one atomic operation within the lock with tasks_lock: success, message, task_info = _atomic_cancel_task(playlist_id, track_index) if not success: return jsonify({"success": False, "error": message}), 400 # Handle post-cancel operations (outside the lock to prevent deadlock) task_id = task_info['task_id'] task = download_tasks.get(task_id) # Try to start next batch of downloads (this may start new workers) if task and task.get('batch_id'): batch_id = task['batch_id'] # Call existing function to manage batch progression try: _start_next_batch_of_downloads(batch_id) except Exception as e: logger.error(f"[Atomic Cancel] Could not start next downloads: {e}") # CRITICAL: Check for batch completion after V2 cancel # V2 system bypasses _on_download_completed, so we need to check completion manually try: _check_batch_completion_v2(batch_id) except Exception as e: logger.error(f"[Atomic Cancel] Could not check batch completion: {e}") # Cancel Soulseek download if active (non-blocking) if task: download_id = task.get('download_id') username = task.get('username') current_status = task.get('status') original_status = task_info.get('original_status', current_status) # Get original status from task_info logger.info(f"[Atomic Cancel] Task {task_id} state: status='{current_status}', original_status='{original_status}', download_id='{download_id}', username='{username}'") logger.info(f"[Atomic Cancel] Download ID type: {type(download_id)}, length: {len(str(download_id)) if download_id else 0}") backslash = '\\' logger.info(f"[Atomic Cancel] Download ID looks like filename: {download_id and ('/' in str(download_id) or backslash in str(download_id))}") if download_id and username: # Route through the DownloadOrchestrator's dispatch (same code # path /api/downloads/cancel uses). It picks the right client by # username: youtube/tidal/qobuz/hifi/deezer_dl/lidarr go to # their streaming clients, anything else goes to Soulseek. # # Replaces an older block that assumed soulseek_client was a # raw SoulseekClient and accessed .base_url / ._make_request # directly — crashed with AttributeError on the orchestrator # and silently left streaming downloads running in background. try: logger.info(f"[Atomic Cancel] Dispatching cancel to orchestrator: username={username} download_id={download_id}") cancel_success = run_async( soulseek_client.cancel_download(download_id, username, remove=True) ) if cancel_success: logger.info(f"[Atomic Cancel] Orchestrator cancelled download: {download_id}") else: # Non-fatal: task is already marked cancelled in the DB. # Streaming workers also poll status='cancelled' and bail. logger.warning(f"[Atomic Cancel] Orchestrator could not cancel {download_id} (likely already finished or not yet started)") except Exception as e: logger.error(f"[Atomic Cancel] Exception cancelling download {download_id}: {e}") import traceback logger.error(f"[Atomic Cancel] Cancel error traceback: {traceback.format_exc()}") else: logger.warning("ℹ️ [Atomic Cancel] No download_id or username available - skipping cancel dispatch") # Add to wishlist (non-blocking, best effort) try: _add_cancelled_task_to_wishlist(task) except Exception as e: logger.error(f"[Atomic Cancel] Could not add to wishlist: {e}") return jsonify({ "success": True, "message": message, "task_info": { 'task_id': task_info['task_id'], 'track_name': task_info['track_name'], 'status': 'cancelled' } }) except Exception as e: logger.error(f"[Cancel V2] Unexpected error: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _check_batch_completion_v2(batch_id): """V2 SYSTEM: Check if batch is complete after worker slot changes.""" return _downloads_lifecycle.check_batch_completion_v2(batch_id, _build_lifecycle_deps()) def _add_cancelled_task_to_wishlist(task): """ Helper function to add cancelled task to wishlist. Separated for clarity and error isolation. """ if not task: return try: from core.wishlist_service import get_wishlist_service wishlist_service = get_wishlist_service() track_info = task.get('track_info', {}) artists_data = track_info.get('artists', []) formatted_artists = [] for artist in artists_data: if isinstance(artist, str): formatted_artists.append({'name': artist}) elif isinstance(artist, dict): if 'name' in artist and isinstance(artist['name'], str): formatted_artists.append(artist) elif 'name' in artist and isinstance(artist['name'], dict) and 'name' in artist['name']: formatted_artists.append({'name': artist['name']['name']}) else: formatted_artists.append({'name': str(artist)}) else: formatted_artists.append({'name': str(artist)}) # Build album data - preserve all fields (including artists) for correct folder placement album_raw = track_info.get('album', {}) if isinstance(album_raw, dict): album_data = dict(album_raw) # Copy all fields including artists album_data.setdefault('name', 'Unknown Album') album_data.setdefault('album_type', track_info.get('album_type', 'album')) # Add images fallback if not present if 'images' not in album_data and track_info.get('album_image_url'): album_data['images'] = [{'url': track_info.get('album_image_url')}] else: # album is a string (album name) album_data = { 'name': str(album_raw) if album_raw else 'Unknown Album', 'album_type': track_info.get('album_type', 'album') } # Add album image if available if track_info.get('album_image_url'): album_data['images'] = [{'url': track_info.get('album_image_url')}] spotify_track_data = { 'id': track_info.get('id'), 'name': track_info.get('name'), 'artists': formatted_artists, 'album': album_data, 'duration_ms': track_info.get('duration_ms') } source_context = { 'playlist_name': task.get('playlist_name', 'Unknown Playlist'), 'playlist_id': task.get('playlist_id'), 'added_from': 'modal_cancellation_v2' } success = wishlist_service.add_spotify_track_to_wishlist( spotify_track_data=spotify_track_data, failure_reason="Download cancelled by user (v2)", source_type="playlist", source_context=source_context, profile_id=get_current_profile_id() ) if success: logger.info(f"[Atomic Cancel] Added '{track_info.get('name')}' to wishlist") else: logger.error(f"[Atomic Cancel] Failed to add '{track_info.get('name')}' to wishlist") except Exception as e: logger.error(f"[Atomic Cancel] Critical error adding to wishlist: {e}") @app.route('/api/playlists//cancel_batch', methods=['POST']) def cancel_batch(batch_id): """ Cancels an entire batch - useful for cancelling during analysis phase or cancelling all downloads at once. """ try: with tasks_lock: if batch_id not in download_batches: return jsonify({"success": False, "error": "Batch not found"}), 404 # Mark batch as cancelled download_batches[batch_id]['phase'] = 'cancelled' # Get playlist_id before doing resets playlist_id = download_batches[batch_id].get('playlist_id') # Reset wishlist auto-processing flag if this is a wishlist batch (auto-initiated only) # Manual wishlist downloads don't set the flag, so only reset if auto-initiated if playlist_id == 'wishlist': auto_initiated = download_batches[batch_id].get('auto_initiated', False) if auto_initiated: global wishlist_auto_processing, wishlist_auto_processing_timestamp with wishlist_timer_lock: wishlist_auto_processing = False wishlist_auto_processing_timestamp = 0 logger.warning("[Wishlist Cancel] Reset wishlist auto-processing flag for cancelled auto-batch") else: logger.warning("ℹ️ [Wishlist Cancel] Manual wishlist batch cancelled (no flag reset needed)") # Reset YouTube playlist phase to 'discovered' if this is a YouTube playlist if playlist_id and playlist_id.startswith('youtube_'): url_hash = playlist_id.replace('youtube_', '') if url_hash in youtube_playlist_states: youtube_playlist_states[url_hash]['phase'] = 'discovered' logger.warning(f"Reset YouTube playlist {url_hash} to discovered phase (batch cancelled)") # Cancel all individual tasks in the batch cancelled_count = 0 for task_id in download_batches[batch_id].get('queue', []): if task_id in download_tasks: task = download_tasks[task_id] if task['status'] not in ['completed', 'failed', 'not_found', 'cancelled']: task['status'] = 'cancelled' cancelled_count += 1 # Add activity for batch cancellation playlist_name = download_batches[batch_id].get('playlist_name', 'Unknown Playlist') add_activity_item("", "Batch Cancelled", f"'{playlist_name}' - {cancelled_count} downloads cancelled", "Now") logger.warning(f"Cancelled batch {batch_id} with {cancelled_count} tasks") return jsonify({"success": True, "cancelled_tasks": cancelled_count}) except Exception as e: logger.error(f"Error cancelling batch {batch_id}: {e}") return jsonify({"success": False, "error": str(e)}), 500 # NEW ENDPOINT: Add this function to web_server.py @app.route('/api/playlists/cleanup_batch', methods=['POST']) def cleanup_batch(): """ Cleans up a completed or cancelled batch from the server's in-memory state. This is called by the client after the user closes a finished modal. """ data = request.get_json() batch_id = data.get('batch_id') if not batch_id: return jsonify({"success": False, "error": "Missing batch_id"}), 400 try: with tasks_lock: # Check if the batch exists before trying to delete if batch_id in download_batches: batch = download_batches[batch_id] # CRITICAL: Don't allow cleanup if wishlist processing is in progress # This prevents a race condition where cleanup deletes the batch before # the wishlist processing thread can access it if batch.get('wishlist_processing_started') and not batch.get('wishlist_processing_complete'): logger.info(f"[Cleanup] Batch {batch_id} cleanup deferred - wishlist processing in progress") return jsonify({ "success": False, "error": "Batch cleanup deferred - wishlist processing in progress", "deferred": True }), 202 # 202 = Accepted but not yet processed # Get the list of task IDs before deleting the batch task_ids_to_remove = batch.get('queue', []) # Delete the batch record del download_batches[batch_id] # Clean up the associated tasks from the tasks dictionary for task_id in task_ids_to_remove: if task_id in download_tasks: del download_tasks[task_id] logger.info(f"Cleaned up batch '{batch_id}' and its associated tasks from server state.") return jsonify({"success": True, "message": f"Batch {batch_id} cleaned up."}) else: # It's not an error if the batch is already gone logger.info(f"Cleanup requested for non-existent batch '{batch_id}'. Already cleaned up?") return jsonify({"success": True, "message": "Batch already cleaned up."}) except Exception as e: logger.error(f"Error during batch cleanup for '{batch_id}': {e}") return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == SYNC HISTORY API == # =============================== @app.route('/api/sync/history', methods=['GET']) def get_sync_history(): """Get paginated sync history.""" try: page = int(request.args.get('page', 1)) limit = int(request.args.get('limit', 20)) source = request.args.get('source') or None db = MusicDatabase() entries, total = db.get_sync_history(source=source, page=page, limit=limit) stats = db.get_sync_history_stats() # Parse artist/album names from JSON context for display for entry in entries: if entry.get('artist_context'): try: ac = json.loads(entry['artist_context']) entry['artist_name'] = ac.get('name', '') except: entry['artist_name'] = '' else: entry['artist_name'] = '' if entry.get('album_context'): try: alc = json.loads(entry['album_context']) entry['album_name'] = alc.get('name', '') except: entry['album_name'] = '' else: entry['album_name'] = '' # Remove raw JSON from list response entry.pop('artist_context', None) entry.pop('album_context', None) return jsonify({"success": True, "entries": entries, "total": total, "page": page, "limit": limit, "stats": stats}) except Exception as e: logger.error(f"Error getting sync history: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/sync/history/', methods=['GET']) def get_sync_history_entry(entry_id): """Get a single sync history entry with full cached data for re-trigger.""" try: db = MusicDatabase() entry = db.get_sync_history_entry(entry_id) if not entry: return jsonify({"success": False, "error": "Entry not found"}), 404 # Parse JSON fields entry['tracks'] = json.loads(entry['tracks_json']) if entry.get('tracks_json') else [] entry['artist_context'] = json.loads(entry['artist_context']) if entry.get('artist_context') else None entry['album_context'] = json.loads(entry['album_context']) if entry.get('album_context') else None entry['track_results'] = json.loads(entry['track_results']) if entry.get('track_results') else None entry.pop('tracks_json', None) return jsonify({"success": True, "entry": entry}) except Exception as e: logger.error(f"Error getting sync history entry: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/sync/history/', methods=['DELETE']) def delete_sync_history_entry_api(entry_id): """Delete a sync history entry.""" try: db = MusicDatabase() deleted = db.delete_sync_history_entry(entry_id) return jsonify({"success": deleted}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/sync/history/names', methods=['GET']) def get_sync_history_playlist_names(): """Return distinct playlist names ever synced, for server playlist cross-reference.""" try: db = MusicDatabase() names = db.get_sync_history_playlist_names() return jsonify(names) except Exception as e: return jsonify([]) # =============================== # == UNIFIED MISSING TRACKS API == # =============================== # Sync history recording lives in core/downloads/history.py. # Re-exported here as thin wrappers so existing call sites still resolve. from core.downloads import history as _downloads_history def _detect_sync_source(playlist_id): """Derive the sync source from the playlist_id prefix.""" return _downloads_history.detect_sync_source(playlist_id) def _record_sync_history_start(batch_id, playlist_id, playlist_name, tracks, is_album_download, album_context, artist_context, playlist_folder_mode, source_page=None): """Record a sync start to the database.""" _downloads_history.record_sync_history_start( MusicDatabase(), batch_id, playlist_id, playlist_name, tracks, is_album_download, album_context, artist_context, playlist_folder_mode, source_page=source_page, ) def _record_sync_history_completion(batch_id, batch): """Update sync history with completion stats and per-track results. NOTE: Called from within tasks_lock context — do NOT acquire tasks_lock here.""" _downloads_history.record_sync_history_completion(MusicDatabase(), batch_id, batch) # =============================== # == SERVER PLAYLIST MANAGER == # =============================== @app.route('/api/server/playlists', methods=['GET']) def get_server_playlists(): """Get all playlists from the active media server.""" try: active_server = config_manager.get_active_media_server() logger.info(f"[ServerPlaylists] Active server: {active_server}") if not active_server: return jsonify({"success": False, "error": "No media server configured"}), 400 playlists_data = [] if active_server == 'plex' and plex_client and plex_client.is_connected(): # Use raw Plex API to get playlist metadata without fetching all tracks try: raw_playlists = plex_client.server.playlists() logger.info(f"[ServerPlaylists] Plex returned {len(raw_playlists)} total playlists") for playlist in raw_playlists: if getattr(playlist, 'playlistType', None) == 'audio': playlists_data.append({ 'id': str(playlist.ratingKey), 'name': playlist.title, 'track_count': playlist.leafCount, }) logger.info(f"[ServerPlaylists] Found {len(playlists_data)} audio playlists") except Exception as e: logger.error(f"[ServerPlaylists] Error fetching Plex playlists: {e}", exc_info=True) return jsonify({"success": False, "error": f"Plex error: {str(e)}"}), 500 elif active_server == 'jellyfin' and jellyfin_client and jellyfin_client.is_connected(): for pl in jellyfin_client.get_all_playlists(): playlists_data.append({ 'id': pl.id, 'name': pl.title, 'track_count': pl.leaf_count, }) elif active_server == 'navidrome' and navidrome_client and navidrome_client.is_connected(): for pl in navidrome_client.get_all_playlists(): playlists_data.append({ 'id': pl.id, 'name': pl.title, 'track_count': pl.leaf_count, }) else: logger.warning(f"[ServerPlaylists] Server '{active_server}' not connected. plex_client={plex_client is not None}, jellyfin_client={jellyfin_client is not None}, navidrome_client={navidrome_client is not None}") return jsonify({"success": False, "error": f"{active_server} not connected"}), 400 return jsonify({"success": True, "server_type": active_server, "playlists": playlists_data}) except Exception as e: logger.error(f"Error getting server playlists: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/server/playlist//tracks', methods=['GET']) def get_server_playlist_tracks(playlist_id): """Get tracks from a server playlist with source match info from sync history.""" try: active_server = config_manager.get_active_media_server() playlist_name = request.args.get('name', '') # Get tracks from server server_tracks = [] if active_server == 'plex' and plex_client: try: # Try by ID first, fall back to name lookup (ID changes when playlist is recreated) raw_playlist = None try: raw_playlist = plex_client.server.fetchItem(int(playlist_id)) except Exception: pass if not raw_playlist and playlist_name: try: raw_playlist = plex_client.server.playlist(playlist_name) except Exception: pass if raw_playlist: if not playlist_name: playlist_name = raw_playlist.title plex_base = getattr(plex_client.server, '_baseurl', '') or '' plex_token = getattr(plex_client.server, '_token', '') or '' if not plex_base: # Fallback: get from config _pc = config_manager.get_plex_config() plex_base = (_pc.get('base_url', '') or '').rstrip('/') plex_token = plex_token or _pc.get('token', '') logger.debug(f"[ServerPlaylistTracks] Plex base URL: {plex_base}") for item in raw_playlist.items(): grandparent = getattr(item, 'grandparentTitle', '') or '' parent = getattr(item, 'parentTitle', '') or '' # Build full thumb URL from Plex relative path thumb = '' raw_thumb = getattr(item, 'thumb', '') or getattr(item, 'parentThumb', '') or '' if raw_thumb and plex_base and plex_token: thumb = f"{plex_base}{raw_thumb}?X-Plex-Token={plex_token}" server_tracks.append({ 'id': str(item.ratingKey), 'title': item.title, 'artist': grandparent, 'album': parent, 'duration': item.duration or 0, 'thumb': thumb, }) except Exception as e: logger.error(f"[ServerPlaylistTracks] Plex error: {e}", exc_info=True) elif active_server == 'jellyfin' and jellyfin_client: tracks = jellyfin_client.get_playlist_tracks(playlist_id) jf_base = jellyfin_client.base_url or '' for t in (tracks or []): raw = t._data if hasattr(t, '_data') else {} artists = raw.get('Artists', []) # Jellyfin image: /Items/{Id}/Images/Primary album_id = raw.get('AlbumId', '') thumb = f"{jf_base}/Items/{album_id}/Images/Primary?maxHeight=100" if album_id and jf_base else '' server_tracks.append({ 'id': str(t.ratingKey), 'title': t.title, 'artist': artists[0] if artists else raw.get('AlbumArtist', ''), 'album': raw.get('Album', ''), 'duration': t.duration, 'thumb': thumb, }) elif active_server == 'navidrome' and navidrome_client: tracks = navidrome_client.get_playlist_tracks(playlist_id) for t in (tracks or []): raw = t._data if hasattr(t, '_data') else {} # Navidrome cover art via Subsonic API cover_id = raw.get('coverArt', '') or raw.get('albumId', '') thumb = f"/api/navidrome/cover/{cover_id}" if cover_id else '' server_tracks.append({ 'id': str(t.ratingKey), 'title': t.title, 'artist': raw.get('artist', ''), 'album': raw.get('album', ''), 'duration': t.duration, 'thumb': thumb, }) # Get source tracks — prefer mirrored playlist, fall back to sync history source_tracks = [] mirrored_id = request.args.get('mirrored_playlist_id') if mirrored_id: db = get_database() raw_tracks = db.get_mirrored_playlist_tracks(int(mirrored_id)) # Build server art URL prefix for resolving relative thumb paths _art_prefix = '' _art_suffix = '' if active_server == 'plex' and plex_client and plex_client.server: _ab = getattr(plex_client.server, '_baseurl', '') or '' _at = getattr(plex_client.server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') _at = _at or _pc.get('token', '') _art_prefix = _ab _art_suffix = f"?X-Plex-Token={_at}" if _at else '' def _resolve_thumb(url): """Make relative server thumb URLs absolute.""" if not url: return '' if url.startswith('http'): return url if url.startswith('/') and _art_prefix: return f"{_art_prefix}{url}{_art_suffix}" return url # Build art lookup from server tracks we already fetched (no extra DB queries) _server_art_map = {} for svr in server_tracks: if svr.get('thumb'): key = f"{(svr.get('artist') or '').lower().strip()}|{svr['title'].lower().strip()}" _server_art_map[key] = svr['thumb'] # Also store by title-only as fallback _server_art_map[svr['title'].lower().strip()] = svr['thumb'] for t in raw_tracks: img = t.get('image_url') or '' if not img: # Try artist+title first, fall back to title-only key = f"{(t.get('artist_name') or '').lower().strip()}|{(t.get('track_name') or '').lower().strip()}" img = _server_art_map.get(key, '') or _server_art_map.get((t.get('track_name') or '').lower().strip(), '') source_tracks.append({ 'name': t.get('track_name', ''), 'artist': t.get('artist_name', ''), 'album': t.get('album_name', ''), 'image_url': img, 'duration_ms': t.get('duration_ms', 0), 'position': t.get('position', 0), }) elif playlist_name: # Legacy fallback: cross-reference with sync history db = get_database() entries, _ = db.get_sync_history(page=1, limit=50) for entry in entries: if entry.get('playlist_name', '').lower() == playlist_name.lower(): full_entry = db.get_sync_history_entry(entry['id']) if full_entry: try: tr = json.loads(full_entry.get('track_results') or '[]') source_tracks = tr if isinstance(tr, list) else [] except (json.JSONDecodeError, TypeError): pass if not source_tracks: try: source_tracks = json.loads(full_entry.get('tracks_json') or '[]') except (json.JSONDecodeError, TypeError): pass break # Build combined view with two-pass matching (exact then fuzzy) import re as _re from difflib import SequenceMatcher def _norm_title(t): """Strip feat./ft., remaster, and edition qualifiers for comparison only.""" # feat./ft. — e.g. (feat. Artist), [ft. Artist] t = _re.sub(r'\s*[\(\[](?:feat|ft)\.?[^\)\]]*[\)\]]', '', t, flags=_re.IGNORECASE) # Remaster/Remastered — e.g. (2019 Remaster), (Remastered), (2019 Remastered Version) t = _re.sub(r'\s*[\(\[](?:\d{4}\s+)?remaster(?:ed)?(?:\s+version)?\s*[\)\]]', '', t, flags=_re.IGNORECASE) # Edition qualifiers — e.g. (Deluxe Edition), (Special Edition), [Anniversary Edition] t = _re.sub(r'\s*[\(\[](?:deluxe|special|anniversary|legacy|expanded|limited)(?:\s+edition)?\s*[\)\]]', '', t, flags=_re.IGNORECASE) return t.lower().strip() combined = [] used_server_indices = set() unmatched_source = [] # (index_in_combined, src_dict) for fuzzy second pass # Pass 1: Exact title match (normalized — strips feat./ft. qualifiers) for i, src in enumerate(source_tracks): src_name = src.get('name', '') src_artist = src.get('artist', '') if not src_artist and src.get('artists'): a = src['artists'][0] if src['artists'] else '' src_artist = a.get('name', a) if isinstance(a, dict) else str(a) src_entry = { 'name': src_name, 'artist': src_artist, 'album': src.get('album', ''), 'image_url': src.get('image_url', ''), 'duration_ms': src.get('duration_ms', 0), 'position': src.get('position', i), } src_norm = _norm_title(src_name) best_idx = -1 for j, svr in enumerate(server_tracks): if j in used_server_indices: continue if _norm_title(svr['title']) == src_norm: best_idx = j break if best_idx >= 0: used_server_indices.add(best_idx) combined.append({ 'source_track': src_entry, 'server_track': server_tracks[best_idx], 'match_status': 'matched', 'confidence': 1.0, }) else: idx = len(combined) combined.append({ 'source_track': src_entry, 'server_track': None, 'match_status': 'missing', 'confidence': 0.0, }) unmatched_source.append((idx, src_entry)) # Pass 2: Fuzzy match on remaining unmatched source tracks (normalized keys) for combo_idx, src_entry in unmatched_source: src_key = f"{src_entry['artist']} {_norm_title(src_entry['name'])}".strip() best_score = 0.0 best_j = -1 for j, svr in enumerate(server_tracks): if j in used_server_indices: continue svr_key = f"{svr['artist']} {_norm_title(svr['title'])}".strip().lower() score = SequenceMatcher(None, src_key.lower(), svr_key).ratio() if score > best_score and score >= 0.75: best_score = score best_j = j if best_j >= 0: used_server_indices.add(best_j) combined[combo_idx] = { 'source_track': src_entry, 'server_track': server_tracks[best_j], 'match_status': 'matched', 'confidence': round(best_score, 3), } # Add server tracks that aren't in the source (extra tracks on server) for j, svr in enumerate(server_tracks): if j not in used_server_indices: combined.append({ 'source_track': None, 'server_track': svr, 'match_status': 'extra', 'confidence': 0.0, }) return jsonify({ "success": True, "server_type": active_server, "playlist_name": playlist_name, "tracks": combined, "server_track_count": len(server_tracks), "source_track_count": len(source_tracks), }) except Exception as e: logger.error(f"Error getting server playlist tracks: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/server/playlist//replace-track', methods=['POST']) def server_playlist_replace_track(playlist_id): """Replace a track in a server playlist. Rebuilds the playlist with the swap.""" try: data = request.get_json() old_track_id = data.get('old_track_id') new_track_id = data.get('new_track_id') playlist_name = data.get('playlist_name', '') if not old_track_id or not new_track_id: return jsonify({"success": False, "error": "old_track_id and new_track_id required"}), 400 if not playlist_name: return jsonify({"success": False, "error": "playlist_name required"}), 400 active_server = config_manager.get_active_media_server() if active_server == 'plex' and plex_client: # Use raw Plex API - fetch playlist directly try: raw_playlist = plex_client.server.playlist(playlist_name) except Exception: return jsonify({"success": False, "error": "Playlist not found on server"}), 404 # Build new track list with replacement new_tracks = [] replaced = False for item in raw_playlist.items(): if str(item.ratingKey) == str(old_track_id) and not replaced: new_item = plex_client.server.fetchItem(int(new_track_id)) if new_item: new_tracks.append(new_item) replaced = True else: new_tracks.append(item) else: new_tracks.append(item) if replaced: # Delete old and recreate directly (avoid update_playlist's backup logic) raw_playlist.delete() from plexapi.playlist import Playlist new_pl = Playlist.create(plex_client.server, playlist_name, items=new_tracks) return jsonify({"success": True, "message": "Track replaced", "new_playlist_id": str(new_pl.ratingKey)}) else: return jsonify({"success": False, "error": "Old track not found in playlist"}), 404 elif active_server == 'jellyfin' and jellyfin_client: current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) new_track_ids = [] replaced = False for t in (current_tracks or []): tid = str(t.ratingKey) if tid == str(old_track_id) and not replaced: new_track_ids.append(new_track_id) replaced = True else: new_track_ids.append(tid) if replaced: new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_track_ids] jellyfin_client.update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track replaced"}) return jsonify({"success": False, "error": "Old track not found"}), 404 elif active_server == 'navidrome' and navidrome_client: current_tracks = navidrome_client.get_playlist_tracks(playlist_id) new_track_ids = [] replaced = False for t in (current_tracks or []): tid = str(t.ratingKey) if tid == str(old_track_id) and not replaced: new_track_ids.append(new_track_id) replaced = True else: new_track_ids.append(tid) if replaced: new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_track_ids] navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track replaced"}) return jsonify({"success": False, "error": "Old track not found"}), 404 return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 except Exception as e: logger.error(f"Error replacing track in server playlist: {e}", exc_info=True) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/server/playlist//add-track', methods=['POST']) def server_playlist_add_track(playlist_id): """Add a track to a server playlist at a specific position.""" try: data = request.get_json() track_id = data.get('track_id') playlist_name = data.get('playlist_name', '') position = data.get('position') # 0-based index; None = append if not track_id: return jsonify({"success": False, "error": "track_id required"}), 400 if not playlist_name: return jsonify({"success": False, "error": "playlist_name required"}), 400 active_server = config_manager.get_active_media_server() if active_server == 'plex' and plex_client: try: raw_playlist = plex_client.server.playlist(playlist_name) except Exception: return jsonify({"success": False, "error": "Playlist not found"}), 404 new_item = plex_client.server.fetchItem(int(track_id)) if not new_item: return jsonify({"success": False, "error": "Track not found on server"}), 404 logger.info(f"[ServerPlaylist] Adding track: '{new_item.title}' (ratingKey={new_item.ratingKey}) to playlist '{playlist_name}'") raw_playlist.addItems([new_item]) # Move to correct position if specified (addItems always appends to end) if position is not None: try: raw_playlist.reload() items = list(raw_playlist.items()) pos = max(0, min(int(position), len(items) - 1)) if pos == 0: raw_playlist.moveItem(items[-1]) # Move to first position elif pos < len(items) - 1: raw_playlist.moveItem(items[-1], after=items[pos - 1]) # else: already at end, no move needed logger.info(f"[ServerPlaylist] Moved track to position {pos}") except Exception as move_err: logger.warning(f"[ServerPlaylist] Could not reposition track: {move_err}") new_id = str(raw_playlist.ratingKey) logger.info(f"[ServerPlaylist] Added track to playlist, playlist ID: {new_id}") return jsonify({"success": True, "message": "Track added", "new_playlist_id": new_id}) elif active_server == 'jellyfin' and jellyfin_client: current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] jellyfin_client.update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track added"}) elif active_server == 'navidrome' and navidrome_client: current_tracks = navidrome_client.get_playlist_tracks(playlist_id) or [] track_ids = [str(t.ratingKey) for t in current_tracks] pos = max(0, min(int(position), len(track_ids))) if position is not None else len(track_ids) track_ids.insert(pos, track_id) new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in track_ids] navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track added"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 except Exception as e: logger.error(f"Error adding track to server playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/server/playlist//remove-track', methods=['POST']) def server_playlist_remove_track(playlist_id): """Remove a track from a server playlist by its server track ID.""" try: data = request.get_json() remove_track_id = data.get('track_id') playlist_name = data.get('playlist_name', '') if not remove_track_id: return jsonify({"success": False, "error": "track_id required"}), 400 if not playlist_name: return jsonify({"success": False, "error": "playlist_name required"}), 400 active_server = config_manager.get_active_media_server() if active_server == 'plex' and plex_client: try: raw_playlist = plex_client.server.playlist(playlist_name) except Exception: return jsonify({"success": False, "error": "Playlist not found"}), 404 # Rebuild without the target track current_items = list(raw_playlist.items()) new_items = [item for item in current_items if str(item.ratingKey) != str(remove_track_id)] if len(new_items) == len(current_items): return jsonify({"success": False, "error": "Track not found in playlist"}), 404 raw_playlist.delete() if new_items: from plexapi.playlist import Playlist new_pl = Playlist.create(plex_client.server, playlist_name, items=new_items) return jsonify({"success": True, "message": "Track removed", "new_playlist_id": str(new_pl.ratingKey)}) return jsonify({"success": True, "message": "Track removed (playlist now empty)"}) elif active_server == 'jellyfin' and jellyfin_client: current_tracks = jellyfin_client.get_playlist_tracks(playlist_id) or [] new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] if len(new_ids) == len(current_tracks): return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] jellyfin_client.update_playlist(playlist_name, new_track_objs) return jsonify({"success": True, "message": "Track removed"}) elif active_server == 'navidrome' and navidrome_client: current_tracks = navidrome_client.get_playlist_tracks(playlist_id) or [] new_ids = [str(t.ratingKey) for t in current_tracks if str(t.ratingKey) != str(remove_track_id)] if len(new_ids) == len(current_tracks): return jsonify({"success": False, "error": "Track not found in playlist"}), 404 new_track_objs = [type('T', (), {'ratingKey': tid, 'title': ''})() for tid in new_ids] navidrome_client.create_playlist(playlist_name, new_track_objs, playlist_id=playlist_id) return jsonify({"success": True, "message": "Track removed"}) return jsonify({"success": False, "error": f"Unsupported server: {active_server}"}), 400 except Exception as e: logger.error(f"Error removing track from server playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/search-tracks', methods=['GET']) def library_search_tracks(): """Search SoulSync's local database for tracks (for manual match correction).""" try: query = request.args.get('q', '').strip() limit = int(request.args.get('limit', 10)) if not query: return jsonify({"success": True, "tracks": []}) active_server = config_manager.get_active_media_server() database = get_database() # Build thumb URL resolver for this server _art_prefix = '' _art_suffix = '' if active_server == 'plex' and plex_client and plex_client.server: _ab = getattr(plex_client.server, '_baseurl', '') or '' _at = getattr(plex_client.server, '_token', '') or '' if not _ab: _pc = config_manager.get_plex_config() _ab = (_pc.get('base_url', '') or '').rstrip('/') _at = _at or _pc.get('token', '') _art_prefix = _ab _art_suffix = f"?X-Plex-Token={_at}" if _at else '' def _resolve_search_thumb(url): if not url: return '' if url.startswith('http'): return url if url.startswith('/') and _art_prefix: return f"{_art_prefix}{url}{_art_suffix}" return url results = database.search_tracks(title=query, artist='', limit=limit, server_source=active_server) tracks = [] for t in results: tracks.append({ 'id': t.id, 'title': t.title, 'artist_name': t.artist_name, 'album_title': getattr(t, 'album_title', ''), 'album_thumb_url': _resolve_search_thumb(getattr(t, 'album_thumb_url', '')), 'file_path': getattr(t, 'file_path', ''), 'bitrate': getattr(t, 'bitrate', 0), 'duration': getattr(t, 'duration', 0), 'server_source': getattr(t, 'server_source', ''), }) return jsonify({"success": True, "tracks": tracks}) except Exception as e: logger.error(f"Error searching library tracks: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/playlists//start-missing-process', methods=['POST']) def start_missing_tracks_process(playlist_id): """ A single, robust endpoint to kick off the entire missing tracks workflow. It creates a batch and starts the master worker in the background. """ data = request.get_json() tracks = data.get('tracks', []) playlist_name = data.get('playlist_name', 'Unknown Playlist') force_download_all = data.get('force_download_all', False) playlist_folder_mode = data.get('playlist_folder_mode', False) wing_it = data.get('wing_it', False) # Get album/artist context for artist album downloads is_album_download = data.get('is_album_download', False) album_context = data.get('album_context', None) artist_context = data.get('artist_context', None) if not tracks: return jsonify({"success": False, "error": "No tracks provided"}), 400 # Log album context if provided if is_album_download and album_context and artist_context: logger.info(f"[Artist Album] Received album context: '{album_context.get('name')}' by '{artist_context.get('name')}' ({album_context.get('album_type', 'album')})") logger.info(f" Release: {album_context.get('release_date', 'Unknown')}, Tracks: {album_context.get('total_tracks', len(tracks))}") # Log playlist folder mode if enabled if playlist_folder_mode: logger.info(f"[Playlist Folder] Enabled for playlist: '{playlist_name}'") # Limit concurrent analysis processes to prevent resource exhaustion with tasks_lock: active_analysis_count = sum(1 for batch in download_batches.values() if batch.get('phase') == 'analysis') if active_analysis_count >= 3: # Allow max 3 concurrent analysis processes return jsonify({ "success": False, "error": "Too many analysis processes running. Please wait for one to complete." }), 429 batch_id = str(uuid.uuid4()) with tasks_lock: download_batches[batch_id] = { 'phase': 'analysis', 'playlist_id': playlist_id, 'playlist_name': playlist_name, 'queue': [], 'active_count': 0, 'max_concurrent': _get_batch_max_concurrent(is_album=is_album_download), # Track state management (replicating sync.py) 'permanently_failed_tracks': [], 'cancelled_tracks': set(), 'queue_index': 0, 'analysis_total': len(tracks), # Profile context for failed track wishlist re-adds 'profile_id': get_current_profile_id(), 'analysis_processed': 0, 'analysis_results': [], 'force_download_all': force_download_all, # Pass the force flag to the batch 'playlist_folder_mode': playlist_folder_mode, # Organize downloads by playlist folder # Album context for artist album downloads (explicit folder structure) 'is_album_download': is_album_download, 'album_context': album_context, 'artist_context': artist_context, 'wing_it': wing_it, } # Record sync history — derive source_page from context if playlist_id == 'wishlist': _source_page = 'wishlist' elif is_album_download: _source_page = 'album' elif playlist_id.startswith('youtube_'): _source_page = 'sync' else: _source_page = 'sync' _record_sync_history_start(batch_id, playlist_id, playlist_name, tracks, is_album_download, album_context, artist_context, playlist_folder_mode, source_page=_source_page) # Link YouTube playlist to download process if this is a YouTube playlist if playlist_id.startswith('youtube_'): url_hash = playlist_id.replace('youtube_', '') if url_hash in youtube_playlist_states: youtube_playlist_states[url_hash]['download_process_id'] = batch_id youtube_playlist_states[url_hash]['phase'] = 'downloading' youtube_playlist_states[url_hash]['converted_spotify_playlist_id'] = playlist_id logger.info(f"Linked YouTube playlist {url_hash} to download process {batch_id} (converted ID: {playlist_id})") # Link Tidal playlist to download process if this is a Tidal playlist if playlist_id.startswith('tidal_'): tidal_playlist_id = playlist_id.replace('tidal_', '') if tidal_playlist_id in tidal_discovery_states: tidal_discovery_states[tidal_playlist_id]['download_process_id'] = batch_id tidal_discovery_states[tidal_playlist_id]['phase'] = 'downloading' tidal_discovery_states[tidal_playlist_id]['converted_spotify_playlist_id'] = playlist_id logger.info(f"Linked Tidal playlist {tidal_playlist_id} to download process {batch_id} (converted ID: {playlist_id})") # Link Spotify Public playlist to download process if this is a Spotify Public playlist if playlist_id.startswith('spotify_public_'): sp_url_hash = playlist_id.replace('spotify_public_', '') if sp_url_hash in spotify_public_discovery_states: spotify_public_discovery_states[sp_url_hash]['download_process_id'] = batch_id spotify_public_discovery_states[sp_url_hash]['phase'] = 'downloading' spotify_public_discovery_states[sp_url_hash]['converted_spotify_playlist_id'] = playlist_id logger.info(f"Linked Spotify Public playlist {sp_url_hash} to download process {batch_id} (converted ID: {playlist_id})") # Link Deezer playlist to download process if this is a Deezer playlist if playlist_id.startswith('deezer_'): deezer_playlist_id = playlist_id.replace('deezer_', '') if deezer_playlist_id in deezer_discovery_states: deezer_discovery_states[deezer_playlist_id]['download_process_id'] = batch_id deezer_discovery_states[deezer_playlist_id]['phase'] = 'downloading' deezer_discovery_states[deezer_playlist_id]['converted_spotify_playlist_id'] = playlist_id logger.info(f"Linked Deezer playlist {deezer_playlist_id} to download process {batch_id} (converted ID: {playlist_id})") # Stamp original index to keep task indices aligned with frontend row order for i, track in enumerate(tracks): if '_original_index' not in track: track['_original_index'] = i missing_download_executor.submit(_run_full_missing_tracks_process, batch_id, playlist_id, tracks) return jsonify({ "success": True, "batch_id": batch_id }) @app.route('/api/tracks/download_missing', methods=['POST']) def start_missing_downloads(): """Legacy endpoint - redirect to new playlist-based endpoint""" dl_err = check_download_permission() if dl_err: return dl_err data = request.get_json() missing_tracks = data.get('missing_tracks', []) if not missing_tracks: return jsonify({"success": False, "error": "No missing tracks provided"}), 400 # Use a default playlist_id for legacy compatibility playlist_id = "legacy_modal" # Call the new endpoint logic directly try: batch_id = str(uuid.uuid4()) # Create task queue for this batch task_queue = [] with tasks_lock: # Initialize batch management download_batches[batch_id] = { 'queue': [], 'active_count': 0, 'max_concurrent': _get_max_concurrent(), 'queue_index': 0, # Track state management (replicating sync.py) 'permanently_failed_tracks': [], 'cancelled_tracks': set(), # Profile context for failed track wishlist re-adds 'profile_id': get_current_profile_id() } for track_index, track_data in enumerate(missing_tracks): task_id = str(uuid.uuid4()) download_tasks[task_id] = { 'status': 'pending', 'track_info': track_data, 'playlist_id': playlist_id, 'batch_id': batch_id, 'track_index': track_index, 'download_id': None, 'username': None } # Add to batch queue instead of submitting immediately download_batches[batch_id]['queue'].append(task_id) # Start the first batch of downloads (up to 3) _start_next_batch_of_downloads(batch_id) return jsonify({"success": True, "batch_id": batch_id, "message": f"Queued {len(missing_tracks)} downloads for processing."}) except Exception as e: logger.error(f"Error starting missing downloads: {e}") return jsonify({"success": False, "error": str(e)}), 500 # =============================== # == SYNC PAGE API == # =============================== def _load_sync_status_file(): """Load sync statuses from database.""" try: database = get_database() raw = database.get_preference('sync_statuses') if raw: data = json.loads(raw) return data return {} except Exception as e: logger.error(f"Error loading sync status: {e}") return {} def _save_sync_status_file(sync_statuses): """Save sync statuses to database.""" try: database = get_database() database.set_preference('sync_statuses', json.dumps(sync_statuses)) except Exception as e: logger.error(f"Error saving sync status: {e}") def _update_and_save_sync_status(playlist_id, playlist_name, playlist_owner, snapshot_id, **kwargs): """Updates the sync status for a given playlist and saves to file (same logic as GUI).""" try: # Load existing sync statuses sync_statuses = _load_sync_status_file() # Update this playlist's sync status from datetime import datetime now = datetime.now() status = { 'name': playlist_name, 'owner': playlist_owner, 'snapshot_id': snapshot_id, 'last_synced': now.isoformat() } # Store match counts and track hash for smart-skip on scheduled syncs for key in ('matched_tracks', 'total_tracks', 'discovered_tracks', 'tracks_hash'): if key in kwargs: status[key] = kwargs[key] sync_statuses[playlist_id] = status # Save to file _save_sync_status_file(sync_statuses) logger.info(f"Updated sync status for playlist '{playlist_name}' (ID: {playlist_id})") except Exception as e: logger.error(f"Error updating sync status for {playlist_id}: {e}") @app.route('/api/spotify/playlists', methods=['GET']) def get_spotify_playlists(): """Fetches all user playlists from Spotify and enriches them with local sync status.""" client = get_spotify_client_for_profile() if not client or not client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: playlists = client.get_user_playlists_metadata_only() sync_statuses = _load_sync_status_file() playlist_data = [] # Add regular playlists first for p in playlists: status_info = sync_statuses.get(p.id, {}) sync_status = "Never Synced" # Handle snapshot_id safely - may not exist in core Playlist class playlist_snapshot = getattr(p, 'snapshot_id', '') if 'last_synced' in status_info: stored_snapshot = status_info.get('snapshot_id') last_sync_time = datetime.fromisoformat(status_info['last_synced']).strftime('%b %d, %H:%M') if playlist_snapshot != stored_snapshot: sync_status = f"Last Sync: {last_sync_time}" logger.info( "Playlist sync status: name=%s id=%s snapshot=%r stored_snapshot=%r result=Needs Sync display=%s", p.name, p.id, playlist_snapshot, stored_snapshot, sync_status, ) else: sync_status = f"Synced: {last_sync_time}" logger.info( "Playlist sync status: name=%s id=%s snapshot=%r stored_snapshot=%r result=Synced display=%s", p.name, p.id, playlist_snapshot, stored_snapshot, sync_status, ) else: logger.warning( "Playlist sync status: name=%s id=%s snapshot=%r result=Never Synced display=%s", p.name, p.id, playlist_snapshot, sync_status, ) playlist_data.append({ "id": p.id, "name": p.name, "owner": p.owner, "track_count": p.total_tracks, "image_url": getattr(p, 'image_url', None), "sync_status": sync_status, "snapshot_id": playlist_snapshot }) # Add virtual "Liked Songs" playlist at the END (just count, no full fetch) try: liked_songs_count = spotify_client.get_saved_tracks_count() if liked_songs_count > 0: liked_songs_id = "spotify:liked-songs" status_info = sync_statuses.get(liked_songs_id, {}) sync_status = "Never Synced" if 'last_synced' in status_info: last_sync_time = datetime.fromisoformat(status_info['last_synced']).strftime('%b %d, %H:%M') sync_status = f"Synced: {last_sync_time}" # Get user info for owner name user_info = spotify_client.get_user_info() owner_name = user_info.get('display_name', 'You') if user_info else 'You' # Add Liked Songs as LAST playlist playlist_data.append({ "id": liked_songs_id, "name": "Liked Songs", "owner": owner_name, "track_count": liked_songs_count, "image_url": None, # Spotify doesn't provide image for Liked Songs "sync_status": sync_status, "snapshot_id": "" # Liked Songs doesn't have a snapshot_id }) logger.info(f"Added virtual 'Liked Songs' playlist with {liked_songs_count} tracks (count only)") except Exception as liked_error: logger.error(f"Failed to add Liked Songs playlist: {liked_error}") # Don't fail the entire request if Liked Songs fails return jsonify(playlist_data) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/spotify/playlist/', methods=['GET']) def get_playlist_tracks(playlist_id): """Fetches full track details for a specific playlist.""" if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: # Handle special "Liked Songs" virtual playlist if playlist_id == "spotify:liked-songs": user_info = spotify_client.get_user_info() owner_name = user_info.get('display_name', 'You') if user_info else 'You' # Fetch raw saved tracks with full album data tracks = [] limit = 50 offset = 0 while True: if _spotify_rate_limited(): return jsonify({"error": "Spotify is currently rate limited. Please try again later."}), 429 from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='current_user_saved_tracks') results = spotify_client.sp.current_user_saved_tracks(limit=limit, offset=offset) if not results or 'items' not in results: break for item in results['items']: if item['track'] and item['track']['id']: track_data = item['track'] tracks.append({ 'id': track_data['id'], 'name': track_data['name'], 'artists': track_data['artists'], # Full artist objects (matches Download Missing Tracks behavior) 'album': track_data['album'], # Full album object 'duration_ms': track_data['duration_ms'], 'popularity': track_data.get('popularity', 0), 'spotify_track_id': track_data['id'] }) if len(results['items']) < limit or not results.get('next'): break offset += limit # Create virtual playlist dict for Liked Songs playlist_dict = { 'id': 'spotify:liked-songs', 'name': 'Liked Songs', 'description': 'Your saved tracks on Spotify', 'owner': owner_name, 'public': False, 'collaborative': False, 'track_count': len(tracks), 'image_url': None, 'snapshot_id': '', 'tracks': tracks } return jsonify(playlist_dict) # Handle regular playlists if _spotify_rate_limited(): return jsonify({"error": "Spotify is currently rate limited. Please try again later."}), 429 # Fetch raw playlist data to preserve full album objects from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='playlist') playlist_data = spotify_client.sp.playlist(playlist_id) # Fetch all tracks with full album data tracks = [] try: results = spotify_client._get_playlist_items_page(playlist_id, limit=100) except Exception as items_err: # 403 on followed playlists — try the public embed scraper as fallback logger.warning(f"Playlist items fetch failed ({items_err}), trying public embed scraper") try: from core.spotify_public_scraper import scrape_spotify_embed embed_data = scrape_spotify_embed('playlist', playlist_id) if embed_data and not embed_data.get('error') and embed_data.get('tracks'): for t in embed_data['tracks']: artists = t.get('artists', []) tracks.append({ 'id': t.get('id', ''), 'name': t.get('name', ''), 'artists': artists if artists else [{'name': 'Unknown'}], 'album': {'name': '', 'images': []}, 'duration_ms': t.get('duration_ms', 0), 'popularity': 0, 'spotify_track_id': t.get('id', '') }) playlist_dict = { 'id': playlist_data['id'], 'name': playlist_data['name'], 'description': playlist_data.get('description', ''), 'owner': playlist_data['owner']['display_name'], 'public': playlist_data.get('public', False), 'collaborative': playlist_data.get('collaborative', False), 'track_count': len(tracks), 'image_url': playlist_data['images'][0]['url'] if playlist_data.get('images') else None, 'snapshot_id': playlist_data.get('snapshot_id', ''), 'tracks': tracks } return jsonify(playlist_dict) except Exception as scrape_err: logger.warning(f"Public embed scraper also failed: {scrape_err}") raise items_err # Re-raise original error if both failed while results: for item in results['items']: # Handle both old API ('track') and new Feb 2026 API ('item') field names track_data = item.get('track') or item.get('item') if track_data and track_data.get('id'): tracks.append({ 'id': track_data['id'], 'name': track_data['name'], 'artists': track_data['artists'], # Full artist objects (matches Download Missing Tracks behavior) 'album': track_data['album'], # Full album object with album_type, total_tracks, etc. 'duration_ms': track_data['duration_ms'], 'popularity': track_data.get('popularity', 0), 'spotify_track_id': track_data['id'] # Also include as spotify_track_id for consistency }) if results.get('next'): api_call_tracker.record_call('spotify', endpoint='playlist_tracks_page') results = spotify_client.sp.next(results) else: results = None # Convert playlist to dict playlist_dict = { 'id': playlist_data['id'], 'name': playlist_data['name'], 'description': playlist_data.get('description', ''), 'owner': playlist_data['owner']['display_name'], 'public': playlist_data.get('public', False), 'collaborative': playlist_data.get('collaborative', False), 'track_count': len(tracks), 'image_url': playlist_data['images'][0]['url'] if playlist_data.get('images') else None, 'snapshot_id': playlist_data.get('snapshot_id', ''), 'tracks': tracks } return jsonify(playlist_dict) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/spotify/album/', methods=['GET']) def get_spotify_album_tracks(album_id): """Fetches full track details for a specific album.""" use_hydrabase = _is_hydrabase_active() # Try Hydrabase first when active — look up by album soul_id if use_hydrabase: album_name = request.args.get('name', '') album_artist = request.args.get('artist', '') try: hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) if hydra_tracks: track_items = [] for t in hydra_tracks: artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] track_items.append({ 'name': t.name, 'track_number': t.track_number or 0, 'disc_number': t.disc_number or 1, 'duration_ms': t.duration_ms, 'id': t.id, 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], 'uri': '' }) return jsonify({ 'id': album_id, 'name': album_name or hydra_tracks[0].album or '', 'artists': [{'name': album_artist}] if album_artist else [], 'release_date': '', 'total_tracks': len(track_items), 'album_type': 'album', 'images': [], 'tracks': track_items }) except Exception as e: logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to Spotify: {e}") # Source override: when user clicked from a specific search tab source_override = request.args.get('source', '') if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: # Use explicit source client when overridden (prevents numeric ID misrouting) client = spotify_client if source_override == 'itunes': client = _get_itunes_client() elif source_override == 'hydrabase': # Hydrabase IDs originate from whichever plugin the peer runs. # 'plugin' param is authoritative; fall back to ID format detection. plugin = request.args.get('plugin', '').lower() if plugin == 'itunes' or (not plugin and album_id.isdigit()): client = _get_itunes_client() elif plugin == 'deezer': client = _get_deezer_client() # else: spotify (default) elif source_override == 'deezer': client = _get_deezer_client() elif source_override == 'discogs': client = _get_discogs_client() elif source_override == 'musicbrainz': try: from core.musicbrainz_search import MusicBrainzSearchClient mb_search = MusicBrainzSearchClient() album_data = mb_search.get_album(album_id) if not album_data: return jsonify({"error": "Album not found on MusicBrainz"}), 404 return jsonify(album_data) except Exception as e: logger.error(f"MusicBrainz album detail failed: {e}") return jsonify({"error": str(e)}), 500 album_data = client.get_album(album_id) if not album_data: return jsonify({"error": "Album not found"}), 404 # Extract tracks from album data (Spotify format) tracks = album_data.get('tracks', {}).get('items', []) # If no tracks in album data (iTunes format), fetch them separately if not tracks: tracks_data = client.get_album_tracks(album_id) if tracks_data and 'items' in tracks_data: tracks = tracks_data['items'] # Format response album_dict = { 'id': album_data['id'], 'name': album_data['name'], 'artists': album_data.get('artists', []), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', 0), 'album_type': album_data.get('album_type', 'album'), 'images': album_data.get('images', []), 'tracks': tracks } return jsonify(album_dict) except Exception as e: logger.error(f"Error fetching album tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify/track/', methods=['GET']) def get_spotify_track(track_id): """Fetches full track details including album data for a specific track.""" # Try Hydrabase first when active and track name provided if _is_hydrabase_active(): track_name = request.args.get('name', '') track_artist = request.args.get('artist', '') if track_name: try: query = f"{track_artist} {track_name}".strip() if track_artist else track_name hydra_tracks = hydrabase_client.search_tracks(query, limit=1) if hydra_tracks: t = hydra_tracks[0] artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] return jsonify({ 'id': t.id, 'name': t.name, 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], 'album': {'name': t.album, 'images': [{'url': t.image_url}] if t.image_url else []}, 'duration_ms': t.duration_ms, 'preview_url': t.preview_url, 'external_urls': t.external_urls or {}, 'popularity': t.popularity, }) except Exception as e: logger.warning(f"Hydrabase track lookup failed for '{track_name}', falling back to Spotify: {e}") if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: track_data = spotify_client.get_track_details(track_id) if not track_data: return jsonify({"error": "Track not found"}), 404 return jsonify(track_data) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/spotify/search', methods=['GET']) def search_spotify(): """Generic Spotify search endpoint - supports tracks, albums, artists""" use_hydrabase = _is_hydrabase_active() if not use_hydrabase: if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: query = request.args.get('q', '').strip() search_type = request.args.get('type', 'track').strip() limit = int(request.args.get('limit', 20)) if not query: return jsonify({"error": "Query parameter 'q' is required"}), 400 if use_hydrabase: tracks = hydrabase_client.search_tracks(query, limit=limit) else: # Mirror to Hydrabase P2P network if hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, search_type) tracks = spotify_client.search_tracks(query, limit=limit) tracks_items = [{ 'id': t.id, 'name': t.name, 'artists': t.artists if isinstance(t.artists, list) else [t.artists], 'album': t.album, 'duration_ms': t.duration_ms, 'uri': f"spotify:track:{t.id}" } for t in tracks] return jsonify({'tracks': {'items': tracks_items}}) except Exception as e: logger.error(f"Error searching Spotify: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify/search_tracks', methods=['GET']) def search_spotify_tracks(): """Search for tracks on Spotify - used by discovery fix modal""" use_hydrabase = _is_hydrabase_active() if not use_hydrabase: if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"error": "Spotify not authenticated."}), 401 try: # Support field-specific search params (track, artist) or legacy combined query track_q = request.args.get('track', '').strip() artist_q = request.args.get('artist', '').strip() legacy_query = request.args.get('query', '').strip() limit = int(request.args.get('limit', 20)) # Build Spotify field-filtered query if track_q or artist_q: parts = [] if track_q: parts.append(f'track:{track_q}') if artist_q: parts.append(f'artist:{artist_q}') query = ' '.join(parts) elif legacy_query: query = legacy_query else: return jsonify({"error": "Query parameter is required"}), 400 if use_hydrabase: tracks = hydrabase_client.search_tracks(query, limit=limit) else: if hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, 'tracks') tracks = spotify_client.search_tracks(query, limit=limit) tracks_dict = [{ 'id': t.id, 'name': t.name, 'artists': t.artists, 'album': t.album, 'duration_ms': t.duration_ms, 'image_url': getattr(t, 'image_url', None), } for t in tracks] return jsonify({'tracks': tracks_dict}) except Exception as e: logger.error(f"Error searching Spotify tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/itunes/search_tracks', methods=['GET']) def search_itunes_tracks(): """Search for tracks on iTunes - used by discovery fix modal when iTunes is the source""" try: # Support field-specific search params or legacy combined query track_q = request.args.get('track', '').strip() artist_q = request.args.get('artist', '').strip() legacy_query = request.args.get('query', '').strip() limit = int(request.args.get('limit', 20)) if track_q or artist_q: parts = [] if track_q: parts.append(track_q) if artist_q: parts.append(artist_q) query = ' '.join(parts) elif legacy_query: query = legacy_query else: return jsonify({"error": "Query parameter is required"}), 400 use_hydrabase = _is_hydrabase_active() if use_hydrabase: tracks = hydrabase_client.search_tracks(query, limit=limit) source = 'hydrabase' else: if hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, 'tracks') fallback_client = _get_metadata_fallback_client() tracks = fallback_client.search_tracks(query, limit=limit) source = _get_metadata_fallback_source() tracks_dict = [{ 'id': t.id, 'name': t.name, 'artists': t.artists, 'album': t.album, 'duration_ms': t.duration_ms, 'image_url': t.image_url, 'source': source } for t in tracks] return jsonify({'tracks': tracks_dict}) except Exception as e: logger.error(f"Error searching iTunes tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/search_tracks', methods=['GET']) def search_deezer_tracks(): """Search for tracks on Deezer - used by discovery fix modal when Deezer is the source""" try: track_q = request.args.get('track', '').strip() artist_q = request.args.get('artist', '').strip() legacy_query = request.args.get('query', '').strip() limit = int(request.args.get('limit', 20)) if track_q or artist_q: parts = [] if track_q: parts.append(track_q) if artist_q: parts.append(artist_q) query = ' '.join(parts) elif legacy_query: query = legacy_query else: return jsonify({"error": "Query parameter is required"}), 400 from core.deezer_client import DeezerClient client = _get_deezer_client() tracks = client.search_tracks(query, limit=limit) tracks_dict = [{ 'id': t.id, 'name': t.name, 'artists': t.artists, 'album': t.album, 'duration_ms': t.duration_ms, 'image_url': getattr(t, 'image_url', None), 'source': 'deezer' } for t in tracks] return jsonify({'tracks': tracks_dict}) except Exception as e: logger.error(f"Error searching Deezer tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/itunes/album/', methods=['GET']) def get_itunes_album_tracks(album_id): """Fetches full track details for a specific iTunes album.""" try: # Try Hydrabase first when active — look up by album soul_id if _is_hydrabase_active(): album_name = request.args.get('name', '') album_artist = request.args.get('artist', '') try: hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) if hydra_tracks: track_items = [] for t in hydra_tracks: artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] track_items.append({ 'name': t.name, 'track_number': t.track_number or 0, 'disc_number': t.disc_number or 1, 'duration_ms': t.duration_ms, 'id': t.id, 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], 'uri': '' }) return jsonify({ 'id': album_id, 'name': album_name or hydra_tracks[0].album or '', 'artists': [{'name': album_artist}] if album_artist else [], 'release_date': '', 'total_tracks': len(track_items), 'album_type': 'album', 'images': [], 'tracks': track_items, 'source': 'hydrabase' }) except Exception as e: logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to iTunes: {e}") fallback_client = _get_metadata_fallback_client() album_data = fallback_client.get_album(album_id) if not album_data: return jsonify({"error": "Album not found"}), 404 # Get tracks for this album tracks_data = fallback_client.get_album_tracks(album_id) tracks = tracks_data.get('items', []) if tracks_data else [] # Format response to match Spotify structure for frontend compatibility album_dict = { 'id': album_data.get('id', album_id), 'name': album_data.get('name', 'Unknown Album'), 'artists': album_data.get('artists', []), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', len(tracks)), 'album_type': album_data.get('album_type', 'album'), 'images': album_data.get('images', []), 'tracks': tracks, 'source': _get_metadata_fallback_source() } return jsonify(album_dict) except Exception as e: logger.error(f"Error fetching album tracks: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discover/album//', methods=['GET']) def get_discover_album(source, album_id): """ Source-agnostic album endpoint for discover page. Fetches album from the appropriate source (spotify, itunes, or hydrabase when active). """ try: # Try Hydrabase first when active — look up by album soul_id if _is_hydrabase_active(): album_name = request.args.get('name', '') album_artist = request.args.get('artist', '') try: hydra_tracks = hydrabase_client.get_album_tracks(album_id, limit=50) if hydra_tracks: track_items = [] for t in hydra_tracks: artist_list = t.artists if isinstance(t.artists, list) else [t.artists] if t.artists else [] track_items.append({ 'name': t.name, 'track_number': t.track_number or 0, 'disc_number': t.disc_number or 1, 'duration_ms': t.duration_ms, 'id': t.id, 'artists': [{'name': a} if isinstance(a, str) else a for a in artist_list], 'uri': '' }) return jsonify({ 'id': album_id, 'name': album_name or hydra_tracks[0].album or '', 'artists': [{'name': album_artist}] if album_artist else [], 'release_date': '', 'total_tracks': len(track_items), 'album_type': 'album', 'images': [], 'tracks': track_items, 'source': 'hydrabase' }) except Exception as e: logger.warning(f"Hydrabase album_tracks failed for '{album_id}', falling back to {source}: {e}") if source == 'spotify': album_data = spotify_client.get_album(album_id) if spotify_client and spotify_client.is_authenticated() else None if album_data: tracks = album_data.get('tracks', {}).get('items', []) if not tracks: tracks_data = spotify_client.get_album_tracks(album_id) if tracks_data and 'items' in tracks_data: tracks = tracks_data['items'] return jsonify({ 'id': album_data['id'], 'name': album_data['name'], 'artists': album_data.get('artists', []), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', 0), 'album_type': album_data.get('album_type', 'album'), 'images': album_data.get('images', []), 'tracks': tracks, 'source': 'spotify' }) # Spotify failed (not authenticated, album removed, rate limited) — try fallback album_name = request.args.get('name', '') album_artist = request.args.get('artist', '') fallback = _get_metadata_fallback_client() if fallback and (album_name or album_artist): clean_name = album_name.replace(' - Single', '').replace(' - EP', '').replace(' (Single)', '').strip() search_query = f"{album_artist} {clean_name}" if album_artist else clean_name try: results = fallback.search_albums(search_query, limit=3) for r in (results or []): tracks_data = fallback.get_album_tracks(str(r.id)) tracks = tracks_data.get('items', []) if tracks_data else [] if tracks: return jsonify({ 'id': str(r.id), 'name': r.name, 'artists': [{'name': getattr(r, 'artist', album_artist) or album_artist}], 'release_date': getattr(r, 'release_date', '') or '', 'total_tracks': getattr(r, 'total_tracks', len(tracks)), 'album_type': getattr(r, 'album_type', 'album') or 'album', 'images': [{'url': r.image_url}] if getattr(r, 'image_url', None) else [], 'tracks': tracks, 'source': _get_metadata_fallback_source(), }) except Exception as e: logger.debug(f"Fallback album resolve failed: {e}") return jsonify({"error": "Album not found"}), 404 elif source in ('itunes', 'deezer'): # Use the source-specific client, not just the active fallback if source == 'deezer': fallback_client = _get_deezer_client() fallback_source = 'deezer' else: fallback_client = _get_itunes_client() fallback_source = 'itunes' album_data = fallback_client.get_album(album_id) # If ID doesn't resolve (cross-source ID), search by name+artist if not album_data: album_name = request.args.get('name', '') album_artist = request.args.get('artist', '') if album_name or album_artist: clean_name = album_name.replace(' - Single', '').replace(' - EP', '').replace(' (Single)', '').strip() search_query = f"{album_artist} {clean_name}" if album_artist else clean_name try: results = fallback_client.search_albums(search_query, limit=3) for r in (results or []): tracks_data = fallback_client.get_album_tracks(str(r.id)) tracks = tracks_data.get('items', []) if tracks_data else [] if tracks: return jsonify({ 'id': str(r.id), 'name': r.name, 'artists': [{'name': getattr(r, 'artist', album_artist) or album_artist}], 'release_date': getattr(r, 'release_date', '') or '', 'total_tracks': getattr(r, 'total_tracks', len(tracks)), 'album_type': getattr(r, 'album_type', 'album') or 'album', 'images': [{'url': r.image_url}] if getattr(r, 'image_url', None) else [], 'tracks': tracks, 'source': fallback_source, }) except Exception as e: logger.debug(f"Fallback album name search failed: {e}") if not album_data: return jsonify({"error": "Album not found"}), 404 tracks_data = fallback_client.get_album_tracks(album_id) tracks = tracks_data.get('items', []) if tracks_data else [] return jsonify({ 'id': album_data.get('id', album_id), 'name': album_data.get('name', 'Unknown Album'), 'artists': album_data.get('artists', []), 'release_date': album_data.get('release_date', ''), 'total_tracks': album_data.get('total_tracks', len(tracks)), 'album_type': album_data.get('album_type', 'album'), 'images': album_data.get('images', []), 'tracks': tracks, 'source': fallback_source, }) else: return jsonify({"error": f"Unknown source: {source}"}), 400 except Exception as e: logger.error(f"Error fetching discover album: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # HIFI DOWNLOAD ENDPOINTS # =================================================================== @app.route('/api/hifi/status', methods=['GET']) def hifi_status(): """Check if HiFi API instances are reachable.""" try: hifi = soulseek_client.hifi available = hifi.is_available() version = hifi.get_version() if available else None return jsonify({ "available": available, "version": version, "instance": hifi._get_instance(), }) except Exception as e: return jsonify({"available": False, "error": str(e)}) @app.route('/api/hifi/instances', methods=['GET']) def hifi_instances(): """Check availability of all HiFi API instances.""" import requests as req try: hifi = soulseek_client.hifi instances = list(hifi._instances) results = [] for url in instances: entry = {'url': url, 'status': 'unknown', 'version': None, 'can_search': False, 'can_download': False} try: # Check root for version r = req.get(f'{url}/', timeout=5, headers={'Accept': 'application/json'}) if r.ok: data = r.json() entry['version'] = data.get('version') entry['status'] = 'online' # Check search sr = req.get(f'{url}/search', params={'s': 'test', 'limit': 1}, timeout=5) entry['can_search'] = sr.ok # Check track (download capability) tr = req.get(f'{url}/track', params={'id': '1550546', 'quality': 'LOSSLESS'}, timeout=5) entry['can_download'] = tr.ok if not tr.ok: entry['download_error'] = f'HTTP {tr.status_code}' else: entry['status'] = f'error (HTTP {r.status_code})' except req.exceptions.SSLError: entry['status'] = 'ssl_error' except req.exceptions.ConnectTimeout: entry['status'] = 'timeout' except req.exceptions.ConnectionError: entry['status'] = 'offline' except Exception as e: entry['status'] = f'error ({type(e).__name__})' results.append(entry) return jsonify({'instances': results, 'active': hifi._get_instance()}) except Exception as e: return jsonify({'error': str(e)}), 500 # =================================================================== # DEEZER DOWNLOAD ENDPOINTS # =================================================================== @app.route('/api/deezer-download/test', methods=['POST']) def deezer_download_test(): """Test Deezer ARL token authentication.""" try: data = request.get_json() or {} arl = data.get('arl', '') if not arl: return jsonify({'success': False, 'error': 'No ARL token provided'}) import requests as req import threading session = req.Session() session.headers.update({ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 ' '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Accept-Language': 'en-US,en;q=0.9', }) session.cookies.set('arl', arl) resp = session.post( 'https://www.deezer.com/ajax/gw-light.php', params={'method': 'deezer.getUserData', 'api_version': '1.0', 'api_token': 'null'}, json={}, timeout=15 ) logger.debug(f"Deezer test raw response status={resp.status_code}, body_preview={resp.text[:500]}") resp.raise_for_status() result = resp.json().get('results', {}) user = result.get('USER', {}) user_id = user.get('USER_ID', 0) logger.info(f"Deezer test: USER_ID={user_id}, keys={list(result.keys())}, user_keys={list(user.keys()) if user else 'none'}") if not user_id or user_id == 0: # Log more detail for debugging error_info = result.get('error', result.get('ERROR', '')) logger.warning(f"Deezer ARL test failed — USER_ID={user_id}, error={error_info}, response_keys={list(result.keys())}") return jsonify({'success': False, 'error': f'Invalid ARL token — Deezer returned no user (USER_ID={user_id})'}) user_name = user.get('BLOG_NAME', 'Unknown') options = user.get('OPTIONS', {}) can_lossless = options.get('web_lossless', False) can_hq = options.get('web_hq', False) tier = 'HiFi' if can_lossless else ('Premium' if can_hq else 'Free') return jsonify({'success': True, 'user': user_name, 'tier': tier}) except Exception as e: logger.error(f"Deezer download test failed: {e}") return jsonify({'success': False, 'error': str(e)}) @app.route('/api/deezer-download/test-search', methods=['GET']) def deezer_download_test_search(): """Test Deezer download search (temporary testing endpoint).""" try: query = request.args.get('q', '') if not query: return jsonify({'success': False, 'error': 'No query provided'}) arl = config_manager.get('deezer_download.arl', '') if not arl: return jsonify({'success': False, 'error': 'No ARL configured'}) from core.deezer_download_client import DeezerDownloadClient client = DeezerDownloadClient() if not client.is_authenticated(): client.reconnect(arl) if not client.is_authenticated(): return jsonify({'success': False, 'error': 'Authentication failed'}) tracks, albums = client._search_sync(query) results = [] for t in tracks[:10]: results.append({ 'title': t.title, 'artist': t.artist, 'album': t.album, 'quality': t.quality, 'bitrate': t.bitrate, 'duration_ms': t.duration, 'size': t.size, 'filename': t.filename, }) return jsonify({'success': True, 'count': len(tracks), 'results': results}) except Exception as e: logger.error(f"Deezer search test failed: {e}") return jsonify({'success': False, 'error': str(e)}) @app.route('/api/deezer-download/test-download', methods=['POST']) def deezer_download_test_download(): """Test Deezer download of a single track (temporary testing endpoint).""" try: data = request.get_json() or {} filename = data.get('filename', '') if not filename: return jsonify({'success': False, 'error': 'No filename provided (use track_id||display_name format)'}) arl = config_manager.get('deezer_download.arl', '') if not arl: return jsonify({'success': False, 'error': 'No ARL configured'}) from core.deezer_download_client import DeezerDownloadClient client = DeezerDownloadClient() if not client.is_authenticated(): client.reconnect(arl) if not client.is_authenticated(): return jsonify({'success': False, 'error': 'Authentication failed'}) download_id = run_async(client.download('deezer_dl', filename)) if not download_id: return jsonify({'success': False, 'error': 'Download failed to start'}) return jsonify({'success': True, 'download_id': download_id, 'message': 'Download started — check logs'}) except Exception as e: logger.error(f"Deezer download test failed: {e}") return jsonify({'success': False, 'error': str(e)}) # =================================================================== # TIDAL DOWNLOAD AUTH ENDPOINTS # =================================================================== def _get_tidal_download_client(): """Get Tidal download client from the orchestrator, with helpful error if unavailable.""" if not soulseek_client: raise RuntimeError("Download orchestrator not initialized — check startup logs for errors") if not hasattr(soulseek_client, 'tidal') or not soulseek_client.tidal: raise RuntimeError("Tidal download client not available — ensure tidalapi is installed") return soulseek_client.tidal @app.route('/api/tidal/download/auth/start', methods=['POST']) def tidal_download_auth_start(): """Start Tidal device-code OAuth flow for download client.""" try: tidal_dl = _get_tidal_download_client() result = tidal_dl.start_device_auth() if result: return jsonify({"success": True, **result}) else: return jsonify({"error": "Failed to start Tidal auth. Is tidalapi installed?"}), 500 except Exception as e: return jsonify({"error": f"Failed to start Tidal auth: {e}"}), 500 @app.route('/api/tidal/download/auth/check', methods=['GET']) def tidal_download_auth_check(): """Check status of Tidal device-code OAuth flow.""" try: tidal_dl = _get_tidal_download_client() result = tidal_dl.check_device_auth() return jsonify(result) except Exception as e: return jsonify({"status": "error", "message": str(e)}), 500 @app.route('/api/tidal/download/auth/status', methods=['GET']) def tidal_download_auth_status(): """Check if Tidal download client is authenticated.""" try: tidal_dl = _get_tidal_download_client() authenticated = tidal_dl.is_authenticated() return jsonify({"authenticated": authenticated}) except Exception as e: return jsonify({"authenticated": False, "error": str(e)}) # =================================================================== # QOBUZ AUTH ENDPOINTS # =================================================================== @app.route('/api/qobuz/auth/login', methods=['POST']) def qobuz_auth_login(): """Login to Qobuz with email/password.""" try: data = request.get_json() email = data.get('email', '').strip() password = data.get('password', '').strip() if not email or not password: return jsonify({"success": False, "error": "Email and password required"}), 400 qobuz = soulseek_client.qobuz result = qobuz.login(email, password) if result['status'] == 'success': return jsonify({"success": True, **result}) else: return jsonify({"success": False, "error": result.get('message', 'Login failed')}), 400 except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/qobuz/auth/token', methods=['POST']) def qobuz_auth_token(): """Login to Qobuz with a pasted user_auth_token (bypasses CAPTCHA).""" try: data = request.get_json() token = data.get('token', '').strip() if not token: return jsonify({"success": False, "error": "Auth token required"}), 400 qobuz = soulseek_client.qobuz result = qobuz.login_with_token(token) if result['status'] == 'success': return jsonify({"success": True, **result}) else: return jsonify({"success": False, "error": result.get('message', 'Token login failed')}), 400 except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/qobuz/auth/status', methods=['GET']) def qobuz_auth_status(): """Check if Qobuz client is authenticated.""" try: qobuz = soulseek_client.qobuz authenticated = qobuz.is_authenticated() user_info = {} if authenticated and qobuz.user_info: user_info = { 'display_name': qobuz.user_info.get('display_name', ''), 'subscription': qobuz.user_info.get('credential', {}).get('label', 'Unknown'), } return jsonify({"authenticated": authenticated, "user": user_info}) except Exception as e: return jsonify({"authenticated": False, "error": str(e)}) @app.route('/api/qobuz/auth/logout', methods=['POST']) def qobuz_auth_logout(): """Logout from Qobuz.""" try: soulseek_client.qobuz.logout() return jsonify({"success": True}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # =================================================================== # TIDAL PLAYLIST API ENDPOINTS # =================================================================== @app.route('/api/tidal/playlists', methods=['GET']) def get_tidal_playlists(): """Fetches all user playlists from Tidal with full track data (like sync.py).""" if not tidal_client or not tidal_client.is_authenticated(): return jsonify({"error": "Tidal not authenticated."}), 401 try: # Use same method as sync.py - this already includes all track data playlists = tidal_client.get_user_playlists_metadata_only() playlist_data = [] for p in playlists: # Get track count from metadata (set during listing) or actual tracks track_count = getattr(p, 'track_count', 0) or (len(p.tracks) if hasattr(p, 'tracks') and p.tracks else 0) playlist_dict = { "id": p.id, "name": p.name, "owner": getattr(p, 'owner', 'Unknown'), "track_count": track_count, "image_url": getattr(p, 'image_url', None), "description": getattr(p, 'description', ''), "tracks": [] # Add tracks data like sync.py } # Include full track data if available (like sync.py has) if hasattr(p, 'tracks') and p.tracks: playlist_dict['tracks'] = [{ 'id': t.id, 'name': t.name, 'artists': t.artists or [], 'album': getattr(t, 'album', 'Unknown Album'), 'duration_ms': getattr(t, 'duration_ms', 0), 'track_number': getattr(t, 'track_number', 0) } for t in p.tracks] playlist_data.append(playlist_dict) logger.info(f"Loaded {len(playlist_data)} Tidal playlists with track data") return jsonify(playlist_data) except Exception as e: return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/playlist/', methods=['GET']) def get_tidal_playlist_tracks(playlist_id): """Fetches full track details for a specific Tidal playlist (matches sync.py pattern).""" if not tidal_client or not tidal_client.is_authenticated(): return jsonify({"error": "Tidal not authenticated."}), 401 try: logger.info(f"Getting full Tidal playlist with tracks for: {playlist_id}") # Fetch this single playlist directly — no need to re-fetch all playlists full_playlist = tidal_client.get_playlist(playlist_id) if not full_playlist: return jsonify({"error": "Playlist not found or unable to access. This may be due to privacy settings or Tidal API restrictions."}), 404 if not full_playlist.tracks: return jsonify({"error": "This playlist appears to have no tracks or they cannot be accessed"}), 403 logger.info(f"Loaded {len(full_playlist.tracks)} tracks from Tidal playlist: {full_playlist.name}") # Convert playlist to dict (matches sync.py structure) playlist_dict = { 'id': full_playlist.id, 'name': full_playlist.name, 'description': getattr(full_playlist, 'description', ''), 'owner': getattr(full_playlist, 'owner', 'Unknown'), 'track_count': len(full_playlist.tracks), 'image_url': getattr(full_playlist, 'image_url', None), 'tracks': [] } # Convert tracks to dict format (for discovery modal) playlist_dict['tracks'] = [{ 'id': t.id, 'name': t.name, 'artists': t.artists or [], 'album': getattr(t, 'album', 'Unknown Album'), 'duration_ms': getattr(t, 'duration_ms', 0), 'track_number': getattr(t, 'track_number', 0) } for t in full_playlist.tracks] return jsonify(playlist_dict) except Exception as e: logger.error(f"Error getting Tidal playlist tracks: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # TIDAL DISCOVERY API ENDPOINTS # =================================================================== # Global state for Tidal playlist discovery management tidal_discovery_states = {} # Key: playlist_id, Value: discovery state tidal_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="tidal_discovery") @app.route('/api/tidal/discovery/start/', methods=['POST']) def start_tidal_discovery(playlist_id): """Start Spotify discovery process for a Tidal playlist""" try: # Get playlist data from Tidal if not tidal_client or not tidal_client.is_authenticated(): return jsonify({"error": "Tidal not authenticated."}), 401 # Fetch this single playlist directly — no need to re-fetch all playlists target_playlist = tidal_client.get_playlist(playlist_id) if not target_playlist: return jsonify({"error": "Tidal playlist not found"}), 404 if not target_playlist.tracks: return jsonify({"error": "Playlist has no tracks"}), 400 # Initialize discovery state if it doesn't exist, or update existing state if playlist_id in tidal_discovery_states: existing_state = tidal_discovery_states[playlist_id] if existing_state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 # Update existing state for discovery existing_state['phase'] = 'discovering' existing_state['status'] = 'discovering' existing_state['last_accessed'] = time.time() state = existing_state else: # Create new state for first-time discovery state = { 'playlist': target_playlist, 'phase': 'discovering', # fresh -> discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete 'status': 'discovering', 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(target_playlist.tracks), 'discovery_results': [], 'sync_playlist_id': None, 'converted_spotify_playlist_id': None, 'download_process_id': None, # Track associated download missing tracks process 'created_at': time.time(), 'last_accessed': time.time(), 'discovery_future': None, 'sync_progress': {} } tidal_discovery_states[playlist_id] = state # Add activity for discovery start add_activity_item("", "Tidal Discovery Started", f"'{target_playlist.name}' - {len(target_playlist.tracks)} tracks", "Now") # Start discovery worker (capture profile ID while we have Flask context) state['_profile_id'] = get_current_profile_id() future = tidal_discovery_executor.submit(_run_tidal_discovery_worker, playlist_id) state['discovery_future'] = future logger.info(f"Started Spotify discovery for Tidal playlist: {target_playlist.name}") return jsonify({"success": True, "message": "Discovery started"}) except Exception as e: logger.error(f"Error starting Tidal discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/discovery/status/', methods=['GET']) def get_tidal_discovery_status(playlist_id): """Get real-time discovery status for a Tidal playlist""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal discovery not found"}), 404 state = tidal_discovery_states[playlist_id] state['last_accessed'] = time.time() # Update access time response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting Tidal discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/discovery/update_match', methods=['POST']) def update_tidal_discovery_match(): """Update a Tidal discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # playlist_id track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state state = tidal_discovery_states.get(identifier) if not state: return jsonify({'error': 'Discovery state not found'}), 404 if track_index >= len(state['discovery_results']): return jsonify({'error': 'Invalid track index'}), 400 # Update the result result = state['discovery_results'][track_index] old_status = result.get('status') # Update with user-selected track result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = spotify_track['name'] result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) result['spotify_album'] = spotify_track['album'] result['spotify_id'] = spotify_track['id'] # Format duration (Tidal doesn't show duration in table, but store it anyway) duration_ms = spotify_track.get('duration_ms', 0) if duration_ms: minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 result['duration'] = f"{minutes}:{seconds:02d}" else: result['duration'] = '0:00' # IMPORTANT: Also set spotify_data for sync/download compatibility. # Manual match from the fix modal — build a rich spotify_data (album # as dict with image info) matching the normal discovery shape, and # explicitly clear any prior wing-it flag since the user picked a # real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) result['wing_it_fallback'] = False result['manual_match'] = True # Flag for tracking # Update match count if status changed from not found/error if old_status != 'found' and old_status != 'Found': state['spotify_matches'] = state.get('spotify_matches', 0) + 1 logger.info(f"Manual match updated: tidal - {identifier} - track {track_index}") logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") # Save manual fix to discovery cache so it appears in discovery pool try: original_track = result.get('tidal_track', {}) original_name = original_track.get('name', spotify_track['name']) original_artist = '' original_artists = original_track.get('artists', []) if original_artists: original_artist = original_artists[0] if isinstance(original_artists[0], str) else original_artists[0].get('name', '') cache_key = _get_discovery_cache_key(original_name, original_artist) # Normalize artists to plain strings for cache consistency artists_list = spotify_track['artists'] if isinstance(artists_list, list): artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] # Preserve cover image info so the download pipeline can find # artwork when this cached match is used later. The fix modal # sends image_url at the top level; search results often return # album as a bare string, which previously dropped the artwork. image_url = spotify_track.get('image_url') or '' album_raw = spotify_track.get('album', '') if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] matched_data = { 'id': spotify_track['id'], 'name': spotify_track['name'], 'artists': artists_list, 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, original_name, original_artist ) logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") except Exception as cache_err: logger.error(f"Error saving manual fix to discovery cache: {cache_err}") return jsonify({'success': True, 'result': result}) except Exception as e: logger.error(f"Error updating Tidal discovery match: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/tidal/playlists/states', methods=['GET']) def get_tidal_playlist_states(): """Get all stored Tidal playlist discovery states for frontend hydration (similar to YouTube playlists)""" try: states = [] current_time = time.time() for playlist_id, state in tidal_discovery_states.items(): # Update access time when requested state['last_accessed'] = current_time # Return essential data for card state recreation state_info = { 'playlist_id': playlist_id, 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'last_accessed': state['last_accessed'] } states.append(state_info) logger.info(f"Returning {len(states)} stored Tidal playlist states for hydration") return jsonify({"states": states}) except Exception as e: logger.error(f"Error getting Tidal playlist states: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/state/', methods=['GET']) def get_tidal_playlist_state(playlist_id): """Get specific Tidal playlist state (detailed version matching YouTube's state endpoint)""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] state['last_accessed'] = time.time() # Return full state information (including results for modal hydration) response = { 'playlist_id': playlist_id, 'playlist': state['playlist'].__dict__ if hasattr(state['playlist'], '__dict__') else state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'sync_playlist_id': state.get('sync_playlist_id'), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'sync_progress': state.get('sync_progress', {}), 'last_accessed': state['last_accessed'] } return jsonify(response) except Exception as e: logger.error(f"Error getting Tidal playlist state: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/reset/', methods=['POST']) def reset_tidal_playlist(playlist_id): """Reset Tidal playlist to fresh phase (clear discovery/sync data)""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Reset state to fresh (preserve original playlist data) state['phase'] = 'fresh' state['status'] = 'fresh' state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['sync_playlist_id'] = None state['converted_spotify_playlist_id'] = None state['download_process_id'] = None state['sync_progress'] = {} state['discovery_future'] = None state['last_accessed'] = time.time() logger.info(f"Reset Tidal playlist to fresh: {playlist_id}") return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) except Exception as e: logger.error(f"Error resetting Tidal playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/delete/', methods=['POST']) def delete_tidal_playlist(playlist_id): """Delete Tidal playlist state completely""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Remove from state dictionary del tidal_discovery_states[playlist_id] logger.info(f"Deleted Tidal playlist state: {playlist_id}") return jsonify({"success": True, "message": "Playlist deleted"}) except Exception as e: logger.error(f"Error deleting Tidal playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/update_phase/', methods=['POST']) def update_tidal_playlist_phase(playlist_id): """Update Tidal playlist phase (used when modal closes to reset from download_complete to discovered)""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 data = request.get_json() if not data or 'phase' not in data: return jsonify({"error": "Phase not provided"}), 400 new_phase = data['phase'] valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] if new_phase not in valid_phases: return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 state = tidal_discovery_states[playlist_id] old_phase = state.get('phase', 'unknown') state['phase'] = new_phase state['last_accessed'] = time.time() logger.info(f"Updated Tidal playlist {playlist_id} phase: {old_phase} → {new_phase}") return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) except Exception as e: logger.error(f"Error updating Tidal playlist phase: {e}") return jsonify({"error": str(e)}), 500 _playlist_discovery_cancelled = set() # Set of automation_ids that have been cancelled def _pause_enrichment_workers(label='discovery'): """Pause enrichment workers during discovery to reduce API contention. Returns dict of {name: was_running} for resume.""" was_running = {} workers = { 'Spotify': spotify_enrichment_worker, 'iTunes': itunes_enrichment_worker, 'Tidal': tidal_enrichment_worker if 'tidal_enrichment_worker' in globals() else None, 'Qobuz': qobuz_enrichment_worker if 'qobuz_enrichment_worker' in globals() else None, } for name, worker in workers.items(): try: if worker and not worker.paused: worker.pause() was_running[name] = True logger.warning(f"Paused {name} enrichment worker during {label}") except Exception: pass return was_running def _resume_enrichment_workers(was_running, label='discovery'): """Resume enrichment workers that were paused by _pause_enrichment_workers.""" workers = { 'Spotify': spotify_enrichment_worker, 'iTunes': itunes_enrichment_worker, 'Tidal': tidal_enrichment_worker if 'tidal_enrichment_worker' in globals() else None, 'Qobuz': qobuz_enrichment_worker if 'qobuz_enrichment_worker' in globals() else None, } for name, worker in workers.items(): try: if was_running.get(name) and worker: worker.resume() logger.info(f"Resumed {name} enrichment worker after {label}") except Exception: pass def _sync_discovery_results_to_mirrored(source_type, source_playlist_id, discovery_results, discovery_source, profile_id=1): """Write discovery results back to the mirrored playlist's extra_data. Called after Tidal/Deezer/Beatport discovery completes. Matches by source_track_id first, then by track position (index).""" try: db = get_database() playlists = db.get_mirrored_playlists(profile_id=profile_id) mirrored_pl = None for pl in playlists: if pl.get('source') == source_type and str(pl.get('source_playlist_id')) == str(source_playlist_id): mirrored_pl = pl break if not mirrored_pl: logger.warning(f"[Discovery Sync] No mirrored playlist found for {source_type}:{source_playlist_id} (profile {profile_id})") return logger.info(f"[Discovery Sync] Found mirrored playlist '{mirrored_pl.get('name')}' (DB id={mirrored_pl['id']}) for {source_type}:{source_playlist_id}") mirrored_tracks = db.get_mirrored_playlist_tracks(mirrored_pl['id']) if not mirrored_tracks: return # Build lookup maps: source_track_id → db_id AND position → db_id source_id_to_db_id = {} position_to_db_id = {} for mt in mirrored_tracks: sid = mt.get('source_track_id', '') if sid: source_id_to_db_id[str(sid)] = mt['id'] pos = mt.get('position') if pos is not None: position_to_db_id[pos] = mt['id'] updated = 0 for result in discovery_results: if result.get('status') not in ('found', 'Found', 'Wing It'): continue match_data = result.get('match_data') or result.get('spotify_data') if not match_data: continue confidence = result.get('confidence', 0.85) # Try to find the mirrored track DB ID db_track_id = None # Method 1: match by source track ID source_track = result.get('tidal_track') or result.get('source_track') or {} source_tid = str(source_track.get('id', '')) if source_track else '' if source_tid and source_tid in source_id_to_db_id: db_track_id = source_id_to_db_id[source_tid] # Method 2: match by position/index if not db_track_id: idx = result.get('index') if idx is not None and idx in position_to_db_id: db_track_id = position_to_db_id[idx] if not db_track_id: continue extra_data = { 'discovered': True, 'provider': discovery_source, 'confidence': confidence, 'matched_data': match_data, } if result.get('wing_it_fallback'): extra_data['wing_it_fallback'] = True extra_data['provider'] = 'wing_it_fallback' db.update_mirrored_track_extra_data(db_track_id, extra_data) updated += 1 if updated > 0: logger.info(f"Synced {updated} discovery results back to mirrored playlist '{mirrored_pl.get('name', '')}'") except Exception as e: import traceback logger.error(f"Failed to sync discovery results to mirrored playlist: {e}") traceback.print_exc() def _run_playlist_discovery_worker(playlists, automation_id=None): """Background worker that discovers Spotify/iTunes metadata for undiscovered mirrored playlist tracks. Stores results in extra_data for use by sync.""" _ew_state = {} try: _ew_state = _pause_enrichment_workers('mirrored playlist discovery') discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() itunes_client_instance = None if not use_spotify: try: itunes_client_instance = _get_metadata_fallback_client() except Exception: logger.warning(f"Neither Spotify nor {_get_metadata_fallback_source()} available for discovery") _update_automation_progress(automation_id, status='error', progress=100, phase='Error', log_line=f'Neither Spotify nor {_get_metadata_fallback_source()} available', log_type='error') return total_discovered = 0 total_failed = 0 total_skipped = 0 total_tracks = 0 last_playlist_name = '' # Pre-compute grand total for progress tracking grand_total = 0 db_init = get_database() for pl in playlists: t = db_init.get_mirrored_playlist_tracks(pl['id']) if t: grand_total += len(t) _update_automation_progress(automation_id, total=grand_total) for pl in playlists: pl_id = pl['id'] pl_name = pl.get('name', '') last_playlist_name = pl_name source = pl.get('source', '') db = get_database() tracks = db.get_mirrored_playlist_tracks(pl_id) if not tracks: continue logger.info(f"Starting discovery for playlist '{pl_name}' ({len(tracks)} tracks, using {discovery_source.upper()})") _update_automation_progress(automation_id, phase=f'Discovering: "{pl_name}"', log_line=f'Playlist "{pl_name}" — {len(tracks)} tracks ({discovery_source.upper()})', log_type='info') # Fast pre-scan: separate already-discovered from undiscovered undiscovered_tracks = [] pl_skipped = 0 for track in tracks: existing_extra = {} if track.get('extra_data'): try: existing_extra = json.loads(track['extra_data']) if isinstance(track['extra_data'], str) else track['extra_data'] except (json.JSONDecodeError, TypeError): pass if existing_extra.get('discovered'): if existing_extra.get('wing_it_fallback'): # Wing It stub — always re-attempt to find a real match undiscovered_tracks.append(track) else: # Check if matched_data is complete — old discoveries may be missing # track_number/release_date due to the Track dataclass stripping them. # Re-discover these so the enriched pipeline fills in the gaps. md = existing_extra.get('matched_data', {}) album = md.get('album', {}) has_track_num = md.get('track_number') has_release = album.get('release_date') if isinstance(album, dict) else None has_album_id = album.get('id') if isinstance(album, dict) else None if has_track_num and (has_release or has_album_id): pl_skipped += 1 total_skipped += 1 else: # Incomplete discovery — re-discover to get full metadata undiscovered_tracks.append(track) elif existing_extra.get('unmatched_by_user'): # User explicitly removed this match — respect their choice pl_skipped += 1 total_skipped += 1 else: undiscovered_tracks.append(track) if pl_skipped > 0: _update_automation_progress(automation_id, log_line=f'{pl_skipped} tracks already discovered — skipped', log_type='skip') if not undiscovered_tracks: _update_automation_progress(automation_id, progress=((total_skipped + total_discovered + total_failed) / max(1, grand_total)) * 100, log_line=f'All {len(tracks)} tracks already discovered', log_type='skip') continue _update_automation_progress(automation_id, log_line=f'{len(undiscovered_tracks)} tracks to discover', log_type='info') for i, track in enumerate(undiscovered_tracks): # Check for cancellation if automation_id and automation_id in _playlist_discovery_cancelled: _playlist_discovery_cancelled.discard(automation_id) logger.warning(f"Playlist discovery cancelled (automation {automation_id})") _update_automation_progress(automation_id, status='finished', progress=100, phase='Discovery cancelled', log_line=f'Cancelled: {total_discovered} discovered, {total_failed} failed', log_type='info') return total_tracks += 1 track_id = track['id'] track_name = track.get('track_name', '') artist_name = track.get('artist_name', '') duration_ms = track.get('duration_ms', 0) # Step 1: Check discovery cache cache_key = _get_discovery_cache_key(track_name, artist_name) try: cached_match = db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(artist_name, cached_match): extra_data = { 'discovered': True, 'provider': discovery_source, 'confidence': cached_match.get('confidence', 0.85), 'matched_data': cached_match, } db.update_mirrored_track_extra_data(track_id, extra_data) total_discovered += 1 logger.info(f"CACHE [{i+1}/{len(undiscovered_tracks)}]: {track_name} → {cached_match.get('name', '?')}") _update_automation_progress(automation_id, progress=((total_skipped + total_discovered + total_failed) / max(1, grand_total)) * 100, current_item=track_name, log_line=f'{track_name} → {cached_match.get("name", "?")} (cache)', log_type='success') continue except Exception: pass # Step 2: Generate search queries try: temp_track = type('TempTrack', (), { 'name': track_name, 'artists': [artist_name], 'album': None })() search_queries = matching_engine.generate_download_queries(temp_track) except Exception: search_queries = [f"{artist_name} {track_name}", track_name] # Step 3: Search and score best_match = None best_confidence = 0.0 min_confidence = 0.7 for search_query in search_queries: try: if use_spotify: results = spotify_client.search_tracks(search_query, limit=10) else: results = itunes_client_instance.search_tracks(search_query, limit=10) if not results: continue match, confidence, _ = _discovery_score_candidates( track_name, artist_name, duration_ms, results ) if match and confidence > best_confidence: best_confidence = confidence best_match = match if best_confidence >= 0.9: break except Exception: continue # Extended search fallback if not best_match or best_confidence < min_confidence: try: query = f"{artist_name} {track_name}" if use_spotify: extended = spotify_client.search_tracks(query, limit=50) else: extended = itunes_client_instance.search_tracks(query, limit=50) if extended: match, confidence, _ = _discovery_score_candidates( track_name, artist_name, duration_ms, extended ) if match and confidence > best_confidence: best_confidence = confidence best_match = match except Exception: pass # Step 4: Store results if best_match and best_confidence >= min_confidence: match_artists = best_match.artists if hasattr(best_match, 'artists') else [] match_image = getattr(best_match, 'image_url', None) album_name = best_match.album if hasattr(best_match, 'album') else '' album_obj = {'name': album_name, 'release_date': getattr(best_match, 'release_date', '') or ''} if match_image: album_obj['images'] = [{'url': match_image, 'height': 600, 'width': 600}] # Enrich album data from metadata cache — search_tracks() caches the # raw API response which has full album info (id, images, total_tracks) # that the Track dataclass strips to just a name string track_number = None disc_number = None if hasattr(best_match, 'id') and best_match.id: try: cache = get_metadata_cache() _raw = cache.get_entity(discovery_source if not use_spotify else 'spotify', 'track', best_match.id) if _raw and isinstance(_raw.get('album'), dict): _raw_album = _raw['album'] if _raw_album.get('id'): album_obj['id'] = _raw_album['id'] if _raw_album.get('images') and not album_obj.get('images'): album_obj['images'] = _raw_album['images'] if _raw_album.get('total_tracks'): album_obj['total_tracks'] = _raw_album['total_tracks'] if _raw_album.get('album_type'): album_obj['album_type'] = _raw_album['album_type'] if _raw_album.get('release_date') and not album_obj.get('release_date'): album_obj['release_date'] = _raw_album['release_date'] if _raw_album.get('artists'): album_obj['artists'] = _raw_album['artists'] if _raw: track_number = _raw.get('track_number') disc_number = _raw.get('disc_number') except Exception: pass matched_data = { 'id': best_match.id if hasattr(best_match, 'id') else '', 'name': best_match.name if hasattr(best_match, 'name') else '', 'artists': [{'name': a} if isinstance(a, str) else a for a in match_artists], 'album': album_obj, 'duration_ms': best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0, 'image_url': match_image, 'source': discovery_source, } if track_number: matched_data['track_number'] = track_number if disc_number: matched_data['disc_number'] = disc_number extra_data = { 'discovered': True, 'provider': discovery_source, 'confidence': best_confidence, 'matched_data': matched_data, } db.update_mirrored_track_extra_data(track_id, extra_data) total_discovered += 1 # Save to discovery cache try: db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, best_confidence, matched_data, track_name, artist_name ) except Exception: pass logger.info(f"[{i+1}/{len(undiscovered_tracks)}] {track_name} → {matched_data['name']} ({best_confidence:.2f})") _update_automation_progress(automation_id, progress=((total_skipped + total_discovered + total_failed) / max(1, grand_total)) * 100, processed=total_discovered + total_failed, current_item=f'{track_name} - {artist_name}', log_line=f'{track_name} → {matched_data["name"]} ({best_confidence:.2f})', log_type='success') else: # Auto Wing It fallback — mark as discovered with stub metadata stub = _build_discovery_wing_it_stub(track_name, artist_name, duration_ms) extra_data = { 'discovered': True, 'provider': 'wing_it_fallback', 'confidence': 0, 'wing_it_fallback': True, 'matched_data': stub, } db.update_mirrored_track_extra_data(track_id, extra_data) total_discovered += 1 logger.info(f"[{i+1}/{len(undiscovered_tracks)}] Wing It: {track_name} by {artist_name}") _update_automation_progress(automation_id, progress=((total_skipped + total_discovered + total_failed) / max(1, grand_total)) * 100, processed=total_discovered + total_failed, current_item=f'{track_name} - {artist_name}', log_line=f'{track_name} by {artist_name} → wing it (no API match)', log_type='info') time.sleep(0.15) # Emit completion event only if new tracks were actually discovered # (no point triggering downstream sync if nothing changed) try: if automation_engine and total_discovered > 0: _disc_pl_id = str(playlists[0]['id']) if len(playlists) == 1 else '' automation_engine.emit('discovery_completed', { 'playlist_name': last_playlist_name if len(playlists) == 1 else f'{len(playlists)} playlists', 'playlist_id': _disc_pl_id, 'total_tracks': str(total_tracks), 'discovered_count': str(total_discovered), 'failed_count': str(total_failed), 'skipped_count': str(total_skipped), }) except Exception: pass logger.error(f"Playlist discovery complete: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped") _update_automation_progress(automation_id, status='finished', progress=100, phase='Discovery complete', log_line=f'Done: {total_discovered} discovered, {total_failed} failed, {total_skipped} skipped', log_type='success') except Exception as e: logger.error(f"Error in playlist discovery worker: {e}") import traceback traceback.print_exc() _update_automation_progress(automation_id, status='error', progress=100, phase='Error', log_line=f'Error: {str(e)}', log_type='error') finally: _resume_enrichment_workers(_ew_state, 'mirrored playlist discovery') def _extract_artist_name(artist): """Extract artist name string from either a string or dict ({"name": "..."}) format.""" if isinstance(artist, dict): return artist.get('name', '') return artist or '' def _extract_artist_names(artists): """Extract a list of artist name strings from a list that may contain dicts or strings.""" return [_extract_artist_name(a) for a in (artists or [])] def _join_artist_names(artists): """Join artist names from a list that may contain dicts or strings.""" return ', '.join(_extract_artist_names(artists)) def _get_discovery_cache_key(title, artist): """Normalize title/artist for discovery cache lookup using matching_engine.""" norm_title = matching_engine.clean_title(title) norm_artist = matching_engine.clean_artist(_extract_artist_name(artist)) return (norm_title, norm_artist) def _validate_discovery_cache_artist(source_artist, cached_match): """Check if a cached discovery match has a valid artist. Returns False if the cached result's artist doesn't match the source artist (stale/wrong cache entry).""" min_artist_similarity = 0.5 source_artist_cleaned = matching_engine.clean_artist(source_artist) if not source_artist_cleaned: return True # No source artist to validate against cached_artists = cached_match.get('artists', []) if not cached_artists: return True # No cached artist to check best_sim = 0.0 for cand_artist in cached_artists: if not cand_artist: continue # Handle both string artists and dict artists ({"name": "..."}) if isinstance(cand_artist, dict): cand_artist = cand_artist.get('name', '') if not cand_artist: continue cand_normalized = matching_engine.normalize_string(cand_artist) if source_artist_cleaned in cand_normalized: return True cand_cleaned = matching_engine.clean_artist(cand_artist) sim = matching_engine.similarity_score(source_artist_cleaned, cand_cleaned) if sim > best_sim: best_sim = sim if best_sim < min_artist_similarity: logger.info(f"Cache artist mismatch: source='{source_artist}' vs cached='{cached_artists[0]}' (sim={best_sim:.2f}), re-searching") return False return True def _discovery_score_candidates(source_title, source_artist, source_duration_ms, search_results): """Score search results against a source track using the matching engine. Both artist AND title must independently pass minimum similarity floors. This prevents weighted scoring from allowing a perfect artist to carry a garbage title (or vice versa). If either dimension doesn't match, the candidate is rejected — no match is better than a wrong match. Args: source_title: The source track title (already cleaned for YouTube, raw for others) source_artist: The source track primary artist source_duration_ms: The source track duration in ms (0 if unknown) search_results: List of Track objects (Spotify or iTunes) from search Returns: (best_match, best_confidence, best_index) or (None, 0.0, -1) if no results """ best_match = None best_confidence = 0.0 best_index = -1 min_artist_similarity = 0.5 min_title_similarity = 0.5 source_artist_cleaned = matching_engine.clean_artist(source_artist) source_title_cleaned = matching_engine.clean_title(source_title) source_core_title = matching_engine.get_core_string(source_title) for idx, result in enumerate(search_results): try: result_artists = result.artists if hasattr(result, 'artists') and result.artists else [] result_name = result.name if hasattr(result, 'name') else '' result_duration = result.duration_ms if hasattr(result, 'duration_ms') else 0 # Artist floor — both must match, not just the weighted score best_artist_sim = 0.0 for cand_artist in result_artists: if not cand_artist: continue cand_cleaned = matching_engine.clean_artist(cand_artist) cand_normalized = matching_engine.normalize_string(cand_artist) if source_artist_cleaned and source_artist_cleaned in cand_normalized: best_artist_sim = 1.0 break sim = matching_engine.similarity_score(source_artist_cleaned, cand_cleaned) if sim > best_artist_sim: best_artist_sim = sim if best_artist_sim < min_artist_similarity: continue # Title floor — both must match, not just the weighted score cand_title_cleaned = matching_engine.clean_title(result_name) cand_core_title = matching_engine.get_core_string(result_name) # Core title exact match bypasses the floor (e.g., "edamame" == "edamame") title_passes = False if source_core_title and cand_core_title and source_core_title == cand_core_title: title_passes = True else: title_sim = matching_engine.similarity_score(source_title_cleaned, cand_title_cleaned) if title_sim >= min_title_similarity: title_passes = True if not title_passes: continue # Both floors passed — now do full scoring confidence, match_type = matching_engine.score_track_match( source_title=source_title, source_artists=[source_artist], source_duration_ms=source_duration_ms, candidate_title=result_name, candidate_artists=result_artists, candidate_duration_ms=result_duration ) if confidence > best_confidence: best_confidence = confidence best_match = result best_index = idx except Exception as e: logger.error(f"Error scoring candidate {idx}: {e}") continue return best_match, best_confidence, best_index def _run_tidal_discovery_worker(playlist_id): """Background worker for Tidal discovery process (Spotify preferred, iTunes fallback)""" _ew_state = {} try: _ew_state = _pause_enrichment_workers('Tidal discovery') state = tidal_discovery_states[playlist_id] playlist = state['playlist'] # Determine which provider to use — respect user's configured primary source discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Initialize fallback client if needed itunes_client_instance = None if not use_spotify: itunes_client_instance = _get_metadata_fallback_client() logger.info(f"Starting Tidal discovery for: {playlist.name} (using {discovery_source.upper()})") # Store discovery source in state for frontend state['discovery_source'] = discovery_source successful_discoveries = 0 for i, tidal_track in enumerate(playlist.tracks): if state.get('cancelled', False): break try: logger.info(f"[{i+1}/{len(playlist.tracks)}] Searching {discovery_source.upper()}: {tidal_track.name} by {', '.join(tidal_track.artists)}") # Check discovery cache first cache_key = _get_discovery_cache_key(tidal_track.name, tidal_track.artists[0] if tidal_track.artists else '') try: cache_db = get_database() cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(tidal_track.artists[0] if tidal_track.artists else '', cached_match): logger.debug(f"CACHE HIT [{i+1}/{len(playlist.tracks)}]: {tidal_track.name} by {', '.join(tidal_track.artists)}") result = { 'tidal_track': { 'id': tidal_track.id, 'name': tidal_track.name, 'artists': tidal_track.artists or [], 'album': getattr(tidal_track, 'album', 'Unknown Album'), 'duration_ms': getattr(tidal_track, 'duration_ms', 0), }, 'spotify_data': cached_match, 'match_data': cached_match, 'status': 'found', 'discovery_source': discovery_source } successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100) continue except Exception as cache_err: logger.error(f"Cache lookup error: {cache_err}") # Use the search function with appropriate provider track_result = _search_spotify_for_tidal_track( tidal_track, use_spotify=use_spotify, itunes_client=itunes_client_instance ) # Create result entry - use 'match_data' as generic key for both providers result = { 'tidal_track': { 'id': tidal_track.id, 'name': tidal_track.name, 'artists': tidal_track.artists or [], 'album': getattr(tidal_track, 'album', 'Unknown Album'), 'duration_ms': getattr(tidal_track, 'duration_ms', 0), }, 'spotify_data': None, # Keep for backwards compatibility 'match_data': None, # Generic field for any provider 'status': 'not_found', 'discovery_source': discovery_source } match_confidence = 0.0 if use_spotify and isinstance(track_result, tuple): # Spotify: Function returns (Track, raw_data, confidence) track_obj, raw_track_data, match_confidence = track_result album_obj = raw_track_data.get('album', {}) if raw_track_data else {} # Ensure album has a name — fall back to track_obj.album if raw_data was missing if isinstance(album_obj, dict) and not album_obj.get('name') and track_obj.album: album_obj['name'] = track_obj.album elif not album_obj and track_obj.album: album_obj = {'name': track_obj.album} # Ensure release_date is present (raw Spotify data has it, but fallback may not) if isinstance(album_obj, dict) and not album_obj.get('release_date'): album_obj['release_date'] = getattr(track_obj, 'release_date', '') or '' # Extract image URL from album data or track object _album_images = album_obj.get('images', []) if isinstance(album_obj, dict) else [] _image_url = _album_images[0].get('url', '') if _album_images else (getattr(track_obj, 'image_url', '') or '') match_data = { 'id': track_obj.id, 'name': track_obj.name, 'artists': track_obj.artists, 'album': album_obj, 'duration_ms': track_obj.duration_ms, 'external_urls': track_obj.external_urls, 'image_url': _image_url, 'source': 'spotify' } # Preserve track_number/disc_number from raw Spotify API data if raw_track_data and raw_track_data.get('track_number'): match_data['track_number'] = raw_track_data['track_number'] if raw_track_data and raw_track_data.get('disc_number'): match_data['disc_number'] = raw_track_data['disc_number'] result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'found' result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries elif not use_spotify and track_result and isinstance(track_result, dict): # Fallback: Function returns a dict with track data (includes 'confidence' key) match_confidence = track_result.pop('confidence', 0.80) match_data = track_result match_data['source'] = discovery_source # Extract image URL from album images _fb_album = match_data.get('album', {}) _fb_images = _fb_album.get('images', []) if isinstance(_fb_album, dict) else [] if _fb_images and 'image_url' not in match_data: match_data['image_url'] = _fb_images[0].get('url', '') result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'found' result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries # Save to discovery cache if match found if result['status'] == 'found' and result.get('match_data'): try: cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, match_confidence, result['match_data'], tidal_track.name, tidal_track.artists[0] if tidal_track.artists else '' ) logger.info(f"CACHE SAVED: {tidal_track.name} (confidence: {match_confidence:.3f})") except Exception as cache_err: logger.error(f"Cache save error: {cache_err}") # Auto Wing It fallback for unmatched tracks if result['status'] != 'found': tidal_t = result.get('tidal_track', {}) stub = _build_discovery_wing_it_stub( tidal_t.get('name', ''), ', '.join(tidal_t.get('artists', [])), tidal_t.get('duration_ms', 0) ) result['status'] = 'found' result['status_class'] = 'wing-it' result['spotify_data'] = stub result['match_data'] = stub result['wing_it_fallback'] = True result['confidence'] = 0 successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['wing_it_count'] = state.get('wing_it_count', 0) + 1 state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100) # Add delay between requests time.sleep(0.1) except Exception as e: logger.error(f"Error processing track {i+1}: {e}") # Add error result result = { 'tidal_track': { 'name': tidal_track.name, 'artists': tidal_track.artists or [], }, 'spotify_data': None, 'match_data': None, 'status': 'error', 'error': str(e), 'discovery_source': discovery_source } state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(playlist.tracks)) * 100) # Mark as complete state['phase'] = 'discovered' state['status'] = 'discovered' state['discovery_progress'] = 100 # Add activity for discovery completion source_label = discovery_source.upper() add_activity_item("", f"Tidal Discovery Complete ({source_label})", f"'{playlist.name}' - {successful_discoveries}/{len(playlist.tracks)} tracks found", "Now") logger.info(f"Tidal discovery complete ({source_label}): {successful_discoveries}/{len(playlist.tracks)} tracks found") # Sync discovery results back to mirrored playlist _sync_discovery_results_to_mirrored('tidal', playlist_id, state.get('discovery_results', []), discovery_source, profile_id=state.get('_profile_id', 1)) except Exception as e: logger.error(f"Error in Tidal discovery worker: {e}") state['phase'] = 'error' state['status'] = f'error: {str(e)}' finally: _resume_enrichment_workers(_ew_state, 'Tidal discovery') def _search_spotify_for_tidal_track(tidal_track, use_spotify=True, itunes_client=None): """Search Spotify/fallback for a Tidal track using matching_engine for better accuracy Args: tidal_track: The Tidal track to search for use_spotify: If True, use Spotify; if False, use fallback source itunes_client: Fallback client instance (required when use_spotify=False) Returns: For Spotify: (Track, raw_data, confidence) tuple or None For fallback: dict with track data (includes 'confidence' key) or None """ if use_spotify: if not spotify_client or not spotify_client.is_authenticated(): return None else: if not itunes_client: return None try: # Get track info track_name = tidal_track.name artists = tidal_track.artists or [] if not artists: return None artist_name = artists[0] # Use primary artist source_duration = getattr(tidal_track, 'duration_ms', 0) or 0 source_name = "Spotify" if use_spotify else _get_metadata_fallback_source().capitalize() logger.info(f"Tidal track: '{artist_name}' - '{track_name}' (searching {source_name})") # Use matching engine to generate search queries (with fallback) try: temp_track = type('TempTrack', (), { 'name': track_name, 'artists': [artist_name], 'album': None })() search_queries = matching_engine.generate_download_queries(temp_track) logger.info(f"Generated {len(search_queries)} search queries for Tidal track") except Exception as e: logger.error(f"Matching engine failed for Tidal, falling back to basic queries: {e}") if use_spotify: search_queries = [ f'track:"{track_name}" artist:"{artist_name}"', f'"{track_name}" "{artist_name}"', f'{track_name} {artist_name}' ] else: search_queries = [ f'{artist_name} {track_name}', f'{track_name} {artist_name}', track_name ] best_match = None best_match_raw = None best_confidence = 0.0 min_confidence = 0.9 for query_idx, search_query in enumerate(search_queries): try: logger.debug(f"Tidal query {query_idx + 1}/{len(search_queries)}: {search_query} ({source_name})") if use_spotify and not _spotify_rate_limited(): results = spotify_client.search_tracks(search_query, limit=10) if not results: continue else: results = itunes_client.search_tracks(search_query, limit=10) if not results: continue # Score all results using the matching engine match, confidence, match_idx = _discovery_score_candidates( track_name, artist_name, source_duration, results ) if match and confidence > best_confidence and confidence >= min_confidence: best_confidence = confidence best_match = match if use_spotify and match.id: _cache = get_metadata_cache() best_match_raw = _cache.get_entity('spotify', 'track', match.id) else: best_match_raw = None logger.info(f"New best Tidal match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") if best_confidence >= 0.9: logger.info(f"High confidence Tidal match found ({best_confidence:.3f}), stopping search") break except Exception as e: logger.debug(f"Error in Tidal {source_name} search for query '{search_query}': {e}") continue # Strategy 4: Extended search with higher limit (last resort) if not best_match: logger.info("Tidal Strategy 4: Extended search with limit=50") query = f"{artist_name} {track_name}" if use_spotify: extended_results = spotify_client.search_tracks(query, limit=50) else: extended_results = itunes_client.search_tracks(query, limit=50) if extended_results: match, confidence, match_idx = _discovery_score_candidates( track_name, artist_name, source_duration, extended_results ) if match and confidence >= min_confidence: best_match = match best_confidence = confidence logger.info(f"Strategy 4 Tidal match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") if best_match: if use_spotify: logger.info(f"Final Tidal Spotify match: {best_match.artists[0]} - {best_match.name} (confidence: {best_confidence:.3f})") return (best_match, best_match_raw, best_confidence) else: result_artists = best_match.artists if hasattr(best_match, 'artists') else [] result_artist = result_artists[0] if result_artists else 'Unknown' result_name = best_match.name if hasattr(best_match, 'name') else 'Unknown' logger.info(f"Final Tidal {source_name} match: {result_artist} - {result_name} (confidence: {best_confidence:.3f})") album_name = best_match.album if hasattr(best_match, 'album') else 'Unknown Album' image_url = best_match.image_url if hasattr(best_match, 'image_url') else '' track_id = best_match.id if hasattr(best_match, 'id') else '' duration_ms = best_match.duration_ms if hasattr(best_match, 'duration_ms') else 0 # Fetch full track details to get album ID, track_number, etc. # The Track dataclass strips this data — the API has it album_obj = { 'name': album_name, 'album_type': 'album', 'release_date': getattr(best_match, 'release_date', '') or '', 'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else [] } track_number = None disc_number = None if track_id: try: detailed = itunes_client.get_track_details(track_id) if detailed and isinstance(detailed.get('album'), dict): dt_album = detailed['album'] if dt_album.get('id'): album_obj['id'] = dt_album['id'] if dt_album.get('total_tracks'): album_obj['total_tracks'] = dt_album['total_tracks'] if dt_album.get('release_date') and not album_obj.get('release_date'): album_obj['release_date'] = dt_album['release_date'] if dt_album.get('album_type'): album_obj['album_type'] = dt_album['album_type'] if dt_album.get('images') and not album_obj.get('images'): album_obj['images'] = dt_album['images'] if dt_album.get('artists'): album_obj['artists'] = dt_album['artists'] if detailed: track_number = detailed.get('track_number') disc_number = detailed.get('disc_number') logger.info(f"[Discovery Enrich] {result_name}: track_number={track_number}, disc={disc_number}") else: logger.info(f"[Discovery Enrich] get_track_details returned None for ID {track_id} ({result_name})") except Exception as _enrich_err: logger.error(f"[Discovery Enrich] Failed for {result_name} (ID {track_id}): {_enrich_err}") result_data = { 'id': track_id, 'name': result_name, 'artists': [result_artist], 'album': album_obj, 'duration_ms': duration_ms, 'source': _get_metadata_fallback_source(), 'confidence': best_confidence } if track_number: result_data['track_number'] = track_number if disc_number: result_data['disc_number'] = disc_number return result_data else: logger.warning(f"No suitable Tidal match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})") return None except Exception as e: logger.error(f"Error searching Spotify for Tidal track: {e}") return None def convert_tidal_results_to_spotify_tracks(discovery_results): """Convert Tidal discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] # Create track object matching the expected format track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': spotify_data['artists'], 'album': spotify_data['album'], 'duration_ms': spotify_data.get('duration_ms', 0) } if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': # Build from individual fields (automatic discovery format) track = { 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': result.get('spotify_album', 'Unknown Album'), 'duration_ms': 0 } spotify_tracks.append(track) logger.info(f"Converted {len(spotify_tracks)} Tidal matches to Spotify tracks for sync") return spotify_tracks # =================================================================== # TIDAL SYNC API ENDPOINTS # =================================================================== @app.route('/api/tidal/sync/start/', methods=['POST']) def start_tidal_sync(playlist_id): """Start sync process for a Tidal playlist using discovered Spotify tracks""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] state['last_accessed'] = time.time() # Update access time if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: return jsonify({"error": "Tidal playlist not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_tidal_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"tidal_{playlist_id}" playlist_name = state['playlist'].name # Tidal playlist object has .name attribute # Add activity for sync start add_activity_item("", "Tidal Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") # Update Tidal state state['phase'] = 'syncing' state['sync_playlist_id'] = sync_playlist_id state['sync_progress'] = {} # Start the sync using existing sync infrastructure sync_data = { 'playlist_id': sync_playlist_id, 'playlist_name': playlist_name, 'tracks': spotify_tracks } with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Submit sync task playlist_image_url = getattr(state['playlist'], 'image_url', '') future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) active_sync_workers[sync_playlist_id] = future logger.info(f"Started Tidal sync for: {playlist_name} ({len(spotify_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting Tidal sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/sync/status/', methods=['GET']) def get_tidal_sync_status(playlist_id): """Get sync status for a Tidal playlist""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync in progress"}), 404 # Get sync status from existing sync infrastructure with sync_lock: sync_state = sync_states.get(sync_playlist_id, {}) response = { 'phase': state['phase'], 'sync_status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Update Tidal state if sync completed if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' state['sync_progress'] = sync_state.get('progress', {}) # Add activity for sync completion playlist = state.get('playlist') playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist' add_activity_item("", "Sync Complete", f"Tidal playlist '{playlist_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error playlist = state.get('playlist') playlist_name = playlist.name if playlist and hasattr(playlist, 'name') else 'Unknown Playlist' add_activity_item("", "Sync Failed", f"Tidal playlist '{playlist_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting Tidal sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/tidal/sync/cancel/', methods=['POST']) def cancel_tidal_sync(playlist_id): """Cancel sync for a Tidal playlist""" try: if playlist_id not in tidal_discovery_states: return jsonify({"error": "Tidal playlist not found"}), 404 state = tidal_discovery_states[playlist_id] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Clean up sync worker if sync_playlist_id in active_sync_workers: del active_sync_workers[sync_playlist_id] # Revert Tidal state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} return jsonify({"success": True, "message": "Tidal sync cancelled"}) except Exception as e: logger.error(f"Error cancelling Tidal sync: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # DEEZER PLAYLIST DISCOVERY API ENDPOINTS # =================================================================== # Global state for Deezer playlist discovery management deezer_discovery_states = {} # Key: playlist_id, Value: discovery state deezer_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="deezer_discovery") def _get_deezer_client(): """Get cached Deezer client.""" from core.metadata_service import get_deezer_client return get_deezer_client() def _get_itunes_client(): """Get cached iTunes client.""" from core.metadata_service import get_itunes_client return get_itunes_client() def _get_discogs_client(token=None): """Get cached Discogs client.""" from core.metadata_service import get_discogs_client return get_discogs_client(token) def _get_metadata_fallback_source(): """Get the configured primary metadata source. Returns 'spotify', 'itunes', 'deezer', 'discogs', or 'hydrabase'. NOTE: This is a thin wrapper — canonical logic lives in core.metadata_service.get_primary_source(). Kept as a local function because 70+ callers reference it by name.""" from core.metadata_service import get_primary_source return get_primary_source() def _get_metadata_fallback_client(): """Get the active metadata client based on settings. Returns a SpotifyClient, iTunesClient, DeezerClient, DiscogsClient, or HydrabaseClient instance.""" source = _get_metadata_fallback_source() if source == 'spotify': if spotify_client and spotify_client.is_spotify_authenticated(): return spotify_client # Spotify selected but not authed — fall back to deezer return _get_deezer_client() if source == 'deezer': return _get_deezer_client() if source == 'discogs': token = config_manager.get('discogs.token', '') if token: return _get_discogs_client(token) return _get_itunes_client() if source == 'hydrabase': if hydrabase_client and hydrabase_client.is_connected(): return hydrabase_client return _get_itunes_client() return _get_itunes_client() @app.route('/api/deezer/arl-status', methods=['GET']) def get_deezer_arl_status(): """Check if Deezer ARL is configured and authenticated.""" try: deezer_dl = None if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: deezer_dl = soulseek_client.deezer_dl if deezer_dl and deezer_dl.is_authenticated(): user_data = deezer_dl._user_data or {} return jsonify({ 'authenticated': True, 'user_name': user_data.get('BLOG_NAME', 'Unknown'), 'user_id': user_data.get('USER_ID'), }) return jsonify({'authenticated': False}) except Exception as e: return jsonify({'authenticated': False, 'error': str(e)}) @app.route('/api/deezer/arl-playlists', methods=['GET']) def get_deezer_arl_playlists(): """Fetch user playlists via Deezer ARL authentication (like /api/spotify/playlists).""" try: deezer_dl = None if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: deezer_dl = soulseek_client.deezer_dl if not deezer_dl or not deezer_dl.is_authenticated(): return jsonify({'error': 'Deezer ARL not authenticated. Configure your ARL token in Settings > Downloads.'}), 401 playlists = deezer_dl.get_user_playlists() # Add sync_status field to match Spotify format playlist_data = [] for p in playlists: playlist_data.append({ 'id': p['id'], 'name': p['name'], 'owner': p.get('owner', ''), 'track_count': p.get('track_count', 0), 'image_url': p.get('image_url', ''), 'sync_status': 'Never Synced', }) logger.info(f"Loaded {len(playlist_data)} Deezer user playlists via ARL") return jsonify(playlist_data) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/deezer/arl-playlist/', methods=['GET']) def get_deezer_arl_playlist_tracks(playlist_id): """Fetch full playlist with tracks via ARL (like /api/spotify/playlist/).""" try: deezer_dl = None if soulseek_client and hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl: deezer_dl = soulseek_client.deezer_dl if not deezer_dl or not deezer_dl.is_authenticated(): return jsonify({'error': 'Deezer ARL not authenticated.'}), 401 playlist = deezer_dl.get_playlist_tracks(playlist_id) if not playlist: return jsonify({'error': 'Playlist not found or unable to access.'}), 404 logger.info(f"Loaded {len(playlist.get('tracks', []))} tracks from Deezer playlist: {playlist.get('name')}") return jsonify(playlist) except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/deezer/playlist/', methods=['GET']) def get_deezer_playlist(playlist_id): """Fetch a Deezer playlist by ID or URL""" try: from core.deezer_client import DeezerClient # Parse URL if needed parsed_id = DeezerClient.parse_playlist_url(playlist_id) if not parsed_id: return jsonify({"error": "Invalid Deezer playlist ID or URL"}), 400 client = _get_deezer_client() playlist = client.get_playlist(parsed_id) if not playlist: return jsonify({"error": "Deezer playlist not found"}), 404 return jsonify(playlist) except Exception as e: logger.error(f"Error fetching Deezer playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/discovery/start/', methods=['POST']) def start_deezer_discovery(playlist_id): """Start Spotify discovery process for a Deezer playlist""" try: from core.deezer_client import DeezerClient # Parse URL if needed parsed_id = DeezerClient.parse_playlist_url(playlist_id) if parsed_id: playlist_id = parsed_id # Initialize discovery state if it doesn't exist, or update existing state if playlist_id in deezer_discovery_states: existing_state = deezer_discovery_states[playlist_id] if existing_state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 # Fetch fresh playlist data if not already stored if not existing_state.get('playlist'): client = _get_deezer_client() playlist_data = client.get_playlist(playlist_id) if not playlist_data: return jsonify({"error": "Deezer playlist not found"}), 404 existing_state['playlist'] = playlist_data # Update existing state for discovery existing_state['phase'] = 'discovering' existing_state['status'] = 'discovering' existing_state['last_accessed'] = time.time() state = existing_state else: # Fetch playlist data from Deezer client = _get_deezer_client() playlist_data = client.get_playlist(playlist_id) if not playlist_data: return jsonify({"error": "Deezer playlist not found"}), 404 if not playlist_data.get('tracks'): return jsonify({"error": "Playlist has no tracks"}), 400 # Create new state for first-time discovery state = { 'playlist': playlist_data, 'phase': 'discovering', # fresh -> discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete 'status': 'discovering', 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(playlist_data['tracks']), 'discovery_results': [], 'sync_playlist_id': None, 'converted_spotify_playlist_id': None, 'download_process_id': None, 'created_at': time.time(), 'last_accessed': time.time(), 'discovery_future': None, 'sync_progress': {} } deezer_discovery_states[playlist_id] = state # Add activity for discovery start playlist_name = state['playlist']['name'] track_count = len(state['playlist']['tracks']) add_activity_item("", "Deezer Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now") # Start discovery worker (capture profile ID while we have Flask context) deezer_discovery_states[playlist_id]['_profile_id'] = get_current_profile_id() future = deezer_discovery_executor.submit(_run_deezer_discovery_worker, playlist_id) state['discovery_future'] = future logger.info(f"Started Spotify discovery for Deezer playlist: {playlist_name}") return jsonify({"success": True, "message": "Discovery started"}) except Exception as e: logger.error(f"Error starting Deezer discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/discovery/status/', methods=['GET']) def get_deezer_discovery_status(playlist_id): """Get real-time discovery status for a Deezer playlist""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer discovery not found"}), 404 state = deezer_discovery_states[playlist_id] state['last_accessed'] = time.time() response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting Deezer discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/discovery/update_match', methods=['POST']) def update_deezer_discovery_match(): """Update a Deezer discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # playlist_id track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state state = deezer_discovery_states.get(identifier) if not state: return jsonify({'error': 'Discovery state not found'}), 404 if track_index >= len(state['discovery_results']): return jsonify({'error': 'Invalid track index'}), 400 # Update the result result = state['discovery_results'][track_index] old_status = result.get('status') # Update with user-selected track result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = spotify_track['name'] result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) result['spotify_album'] = spotify_track['album'] result['spotify_id'] = spotify_track['id'] # Format duration duration_ms = spotify_track.get('duration_ms', 0) if duration_ms: minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 result['duration'] = f"{minutes}:{seconds:02d}" else: result['duration'] = '0:00' # IMPORTANT: Also set spotify_data for sync/download compatibility. # Manual match from the fix modal — build a rich spotify_data (album # as dict with image info) matching the normal discovery shape, and # explicitly clear any prior wing-it flag since the user picked a # real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) result['wing_it_fallback'] = False result['manual_match'] = True # Update match count if status changed from not found/error if old_status != 'found' and old_status != 'Found': state['spotify_matches'] = state.get('spotify_matches', 0) + 1 logger.info(f"Manual match updated: deezer - {identifier} - track {track_index}") logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") # Save manual fix to discovery cache so it appears in discovery pool try: original_track = result.get('deezer_track', {}) original_name = original_track.get('name', spotify_track['name']) original_artists = original_track.get('artists', []) original_artist = original_artists[0] if original_artists else '' cache_key = _get_discovery_cache_key(original_name, original_artist) # Normalize artists to plain strings for cache consistency artists_list = spotify_track['artists'] if isinstance(artists_list, list): artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] # Preserve cover image info so the download pipeline can find # artwork when this cached match is used later. The fix modal # sends image_url at the top level; search results often return # album as a bare string, which previously dropped the artwork. image_url = spotify_track.get('image_url') or '' album_raw = spotify_track.get('album', '') if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] matched_data = { 'id': spotify_track['id'], 'name': spotify_track['name'], 'artists': artists_list, 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, original_name, original_artist ) logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") except Exception as cache_err: logger.error(f"Error saving manual fix to discovery cache: {cache_err}") return jsonify({'success': True, 'result': result}) except Exception as e: logger.error(f"Error updating Deezer discovery match: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/deezer/playlists/states', methods=['GET']) def get_deezer_playlist_states(): """Get all stored Deezer playlist discovery states for frontend hydration""" try: states = [] current_time = time.time() for playlist_id, state in deezer_discovery_states.items(): state['last_accessed'] = current_time state_info = { 'playlist_id': playlist_id, 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'last_accessed': state['last_accessed'] } states.append(state_info) logger.info(f"Returning {len(states)} stored Deezer playlist states for hydration") return jsonify({"states": states}) except Exception as e: logger.error(f"Error getting Deezer playlist states: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/state/', methods=['GET']) def get_deezer_playlist_state(playlist_id): """Get specific Deezer playlist state (detailed version)""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] state['last_accessed'] = time.time() # Deezer playlist is a dict, no __dict__ needed response = { 'playlist_id': playlist_id, 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'sync_playlist_id': state.get('sync_playlist_id'), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'sync_progress': state.get('sync_progress', {}), 'last_accessed': state['last_accessed'] } return jsonify(response) except Exception as e: logger.error(f"Error getting Deezer playlist state: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/reset/', methods=['POST']) def reset_deezer_playlist(playlist_id): """Reset Deezer playlist to fresh phase (clear discovery/sync data)""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Reset state to fresh (preserve original playlist data) state['phase'] = 'fresh' state['status'] = 'fresh' state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['sync_playlist_id'] = None state['converted_spotify_playlist_id'] = None state['download_process_id'] = None state['sync_progress'] = {} state['discovery_future'] = None state['last_accessed'] = time.time() logger.info(f"Reset Deezer playlist to fresh: {playlist_id}") return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) except Exception as e: logger.error(f"Error resetting Deezer playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/delete/', methods=['POST']) def delete_deezer_playlist(playlist_id): """Delete Deezer playlist state completely""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Remove from state dictionary del deezer_discovery_states[playlist_id] logger.info(f"Deleted Deezer playlist state: {playlist_id}") return jsonify({"success": True, "message": "Playlist deleted"}) except Exception as e: logger.error(f"Error deleting Deezer playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/update_phase/', methods=['POST']) def update_deezer_playlist_phase(playlist_id): """Update Deezer playlist phase (used when modal closes to reset from download_complete to discovered)""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 data = request.get_json() if not data or 'phase' not in data: return jsonify({"error": "Phase not provided"}), 400 new_phase = data['phase'] valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] if new_phase not in valid_phases: return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 state = deezer_discovery_states[playlist_id] old_phase = state.get('phase', 'unknown') state['phase'] = new_phase state['last_accessed'] = time.time() # Update download process ID if provided (for download persistence) if 'download_process_id' in data: state['download_process_id'] = data['download_process_id'] # Update converted Spotify playlist ID if provided (for download persistence) if 'converted_spotify_playlist_id' in data: state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] logger.info(f"Updated Deezer playlist {playlist_id} phase: {old_phase} → {new_phase}") return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) except Exception as e: logger.error(f"Error updating Deezer playlist phase: {e}") return jsonify({"error": str(e)}), 500 def _run_deezer_discovery_worker(playlist_id): """Background worker for Deezer discovery process (Spotify preferred, iTunes fallback)""" _ew_state = {} try: _ew_state = _pause_enrichment_workers('Deezer discovery') state = deezer_discovery_states[playlist_id] playlist = state['playlist'] # Determine which provider to use discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Initialize fallback client if needed itunes_client_instance = None if not use_spotify: itunes_client_instance = _get_metadata_fallback_client() logger.info(f"Starting Deezer discovery for: {playlist['name']} (using {discovery_source.upper()})") # Store discovery source in state for frontend state['discovery_source'] = discovery_source successful_discoveries = 0 tracks = playlist['tracks'] for i, deezer_track in enumerate(tracks): if state.get('cancelled', False): break try: track_name = deezer_track['name'] track_artists = deezer_track['artists'] track_id = deezer_track['id'] track_album = deezer_track.get('album', '') track_duration_ms = deezer_track.get('duration_ms', 0) logger.info(f"[{i+1}/{len(tracks)}] Searching {discovery_source.upper()}: {track_name} by {', '.join(track_artists)}") # Check discovery cache first cache_key = _get_discovery_cache_key(track_name, track_artists[0] if track_artists else '') try: cache_db = get_database() cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(track_artists[0] if track_artists else '', cached_match): logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {track_name} by {', '.join(track_artists)}") # Extract display-friendly artist string from cached match cached_artists = cached_match.get('artists', []) if cached_artists: cached_artist_str = ', '.join( a if isinstance(a, str) else a.get('name', '') for a in cached_artists ) else: cached_artist_str = '' cached_album = cached_match.get('album', '') if isinstance(cached_album, dict): cached_album = cached_album.get('name', '') result = { 'deezer_track': { 'id': track_id, 'name': track_name, 'artists': track_artists or [], 'album': track_album, 'duration_ms': track_duration_ms, }, 'spotify_data': cached_match, 'match_data': cached_match, 'status': 'Found', 'status_class': 'found', 'spotify_track': cached_match.get('name', ''), 'spotify_artist': cached_artist_str, 'spotify_album': cached_album, 'spotify_id': cached_match.get('id', ''), 'discovery_source': discovery_source, 'index': i } successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) continue except Exception as cache_err: logger.error(f"Cache lookup error: {cache_err}") # Create a SimpleNamespace duck-type object for _search_spotify_for_tidal_track track_ns = types.SimpleNamespace( id=track_id, name=track_name, artists=track_artists, album=track_album, duration_ms=track_duration_ms ) # Use the search function with appropriate provider track_result = _search_spotify_for_tidal_track( track_ns, use_spotify=use_spotify, itunes_client=itunes_client_instance ) # Create result entry result = { 'deezer_track': { 'id': track_id, 'name': track_name, 'artists': track_artists or [], 'album': track_album, 'duration_ms': track_duration_ms, }, 'spotify_data': None, 'match_data': None, 'status': 'Not Found', 'status_class': 'not-found', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'discovery_source': discovery_source } match_confidence = 0.0 if use_spotify and isinstance(track_result, tuple): # Spotify: Function returns (Track, raw_data, confidence) track_obj, raw_track_data, match_confidence = track_result album_obj = raw_track_data.get('album', {}) if raw_track_data else {} # Ensure album has a name — fall back to track_obj.album if raw_data was missing if isinstance(album_obj, dict) and not album_obj.get('name') and track_obj.album: album_obj['name'] = track_obj.album elif not album_obj and track_obj.album: album_obj = {'name': track_obj.album} # Ensure release_date is present (raw Spotify data has it, but fallback may not) if isinstance(album_obj, dict) and not album_obj.get('release_date'): album_obj['release_date'] = getattr(track_obj, 'release_date', '') or '' # Extract image URL from album data or track object _album_images = album_obj.get('images', []) if isinstance(album_obj, dict) else [] _image_url = _album_images[0].get('url', '') if _album_images else (getattr(track_obj, 'image_url', '') or '') match_data = { 'id': track_obj.id, 'name': track_obj.name, 'artists': track_obj.artists, 'album': album_obj, 'duration_ms': track_obj.duration_ms, 'external_urls': track_obj.external_urls, 'image_url': _image_url, 'source': 'spotify' } # Preserve track_number/disc_number from raw Spotify API data if raw_track_data and raw_track_data.get('track_number'): match_data['track_number'] = raw_track_data['track_number'] if raw_track_data and raw_track_data.get('disc_number'): match_data['disc_number'] = raw_track_data['disc_number'] result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = track_obj.name result['spotify_artist'] = ', '.join(track_obj.artists) if isinstance(track_obj.artists, list) else str(track_obj.artists) result['spotify_album'] = album_obj.get('name', '') if isinstance(album_obj, dict) else str(album_obj) result['spotify_id'] = track_obj.id result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries elif not use_spotify and track_result and isinstance(track_result, dict): # Fallback: Function returns a dict with track data (includes 'confidence' key) match_confidence = track_result.pop('confidence', 0.80) match_data = track_result match_data['source'] = discovery_source # Extract image URL from album images _fb_album = match_data.get('album', {}) _fb_images = _fb_album.get('images', []) if isinstance(_fb_album, dict) else [] if _fb_images and 'image_url' not in match_data: match_data['image_url'] = _fb_images[0].get('url', '') result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = match_data.get('name', '') itunes_artists = match_data.get('artists', []) result['spotify_artist'] = ', '.join(a if isinstance(a, str) else a.get('name', '') for a in itunes_artists) if itunes_artists else '' result['spotify_album'] = match_data.get('album', {}).get('name', '') if isinstance(match_data.get('album'), dict) else match_data.get('album', '') result['spotify_id'] = match_data.get('id', '') result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries # Save to discovery cache if match found if result['status_class'] == 'found' and result.get('match_data'): try: cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, match_confidence, result['match_data'], track_name, track_artists[0] if track_artists else '' ) logger.info(f"CACHE SAVED: {track_name} (confidence: {match_confidence:.3f})") except Exception as cache_err: logger.error(f"Cache save error: {cache_err}") # Auto Wing It fallback for unmatched tracks if result['status_class'] == 'not-found': deezer_t = result.get('deezer_track', {}) stub = _build_discovery_wing_it_stub( deezer_t.get('name', ''), ', '.join(deezer_t.get('artists', [])), deezer_t.get('duration_ms', 0) ) result['status'] = 'Wing It' result['status_class'] = 'wing-it' result['spotify_data'] = stub result['match_data'] = stub result['spotify_track'] = deezer_t.get('name', '') result['spotify_artist'] = ', '.join(deezer_t.get('artists', [])) result['wing_it_fallback'] = True result['confidence'] = 0 successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['wing_it_count'] = state.get('wing_it_count', 0) + 1 result['index'] = i state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) # Add delay between requests time.sleep(0.1) except Exception as e: logger.error(f"Error processing track {i+1}: {e}") # Add error result result = { 'deezer_track': { 'name': deezer_track.get('name', 'Unknown'), 'artists': deezer_track.get('artists', []), }, 'spotify_data': None, 'match_data': None, 'status': 'Error', 'status_class': 'error', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'error': str(e), 'discovery_source': discovery_source, 'index': i } state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) # Mark as complete state['phase'] = 'discovered' state['status'] = 'discovered' state['discovery_progress'] = 100 # Add activity for discovery completion source_label = discovery_source.upper() add_activity_item("", f"Deezer Discovery Complete ({source_label})", f"'{playlist['name']}' - {successful_discoveries}/{len(tracks)} tracks found", "Now") logger.info(f"Deezer discovery complete ({source_label}): {successful_discoveries}/{len(tracks)} tracks found") # Sync discovery results back to mirrored playlist _sync_discovery_results_to_mirrored('deezer', playlist_id, state.get('discovery_results', []), discovery_source, profile_id=state.get('_profile_id', 1)) except Exception as e: logger.error(f"Error in Deezer discovery worker: {e}") if playlist_id in deezer_discovery_states: deezer_discovery_states[playlist_id]['phase'] = 'error' deezer_discovery_states[playlist_id]['status'] = f'error: {str(e)}' finally: _resume_enrichment_workers(_ew_state, 'Deezer discovery') def convert_deezer_results_to_spotify_tracks(discovery_results): """Convert Deezer discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': spotify_data['artists'], 'album': spotify_data['album'], 'duration_ms': spotify_data.get('duration_ms', 0) } if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': track = { 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': result.get('spotify_album', 'Unknown Album'), 'duration_ms': 0 } spotify_tracks.append(track) logger.info(f"Converted {len(spotify_tracks)} Deezer matches to Spotify tracks for sync") return spotify_tracks # =================================================================== # DEEZER SYNC API ENDPOINTS # =================================================================== @app.route('/api/deezer/sync/start/', methods=['POST']) def start_deezer_sync(playlist_id): """Start sync process for a Deezer playlist using discovered Spotify tracks""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] state['last_accessed'] = time.time() if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: return jsonify({"error": "Deezer playlist not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_deezer_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"deezer_{playlist_id}" playlist_name = state['playlist']['name'] # Add activity for sync start add_activity_item("", "Deezer Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") # Update Deezer state state['phase'] = 'syncing' state['sync_playlist_id'] = sync_playlist_id state['sync_progress'] = {} # Start the sync using existing sync infrastructure sync_data = { 'playlist_id': sync_playlist_id, 'playlist_name': playlist_name, 'tracks': spotify_tracks } with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Submit sync task playlist_image_url = state['playlist'].get('image_url', '') future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) active_sync_workers[sync_playlist_id] = future logger.info(f"Started Deezer sync for: {playlist_name} ({len(spotify_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting Deezer sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/sync/status/', methods=['GET']) def get_deezer_sync_status(playlist_id): """Get sync status for a Deezer playlist""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] state['last_accessed'] = time.time() sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync in progress"}), 404 # Get sync status from existing sync infrastructure with sync_lock: sync_state = sync_states.get(sync_playlist_id, {}) response = { 'phase': state['phase'], 'sync_status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Update Deezer state if sync completed if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' state['sync_progress'] = sync_state.get('progress', {}) playlist_name = state['playlist']['name'] add_activity_item("", "Sync Complete", f"Deezer playlist '{playlist_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error playlist_name = state['playlist']['name'] add_activity_item("", "Sync Failed", f"Deezer playlist '{playlist_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting Deezer sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/deezer/sync/cancel/', methods=['POST']) def cancel_deezer_sync(playlist_id): """Cancel sync for a Deezer playlist""" try: if playlist_id not in deezer_discovery_states: return jsonify({"error": "Deezer playlist not found"}), 404 state = deezer_discovery_states[playlist_id] state['last_accessed'] = time.time() sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Clean up sync worker if sync_playlist_id in active_sync_workers: del active_sync_workers[sync_playlist_id] # Revert Deezer state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} return jsonify({"success": True, "message": "Deezer sync cancelled"}) except Exception as e: logger.error(f"Error cancelling Deezer sync: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # SPOTIFY PUBLIC PLAYLIST DISCOVERY API ENDPOINTS # =================================================================== # Global state for Spotify Public playlist management spotify_public_discovery_states = {} # Key: url_hash, Value: discovery state spotify_public_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="spotify_public_discovery") @app.route('/api/spotify/parse-public', methods=['POST']) def parse_spotify_public_endpoint(): """Parse a public Spotify playlist or album URL without API auth""" try: data = request.get_json() url = data.get('url', '').strip() if not url: return jsonify({"error": "Spotify URL is required"}), 400 from core.spotify_public_scraper import parse_spotify_url, scrape_spotify_embed parsed = parse_spotify_url(url) if not parsed: return jsonify({"error": "Invalid Spotify URL. Please use a playlist or album link from open.spotify.com"}), 400 logger.info(f"Scraping public Spotify {parsed['type']}: {parsed['id']}") result = scrape_spotify_embed(parsed['type'], parsed['id']) if 'error' in result: return jsonify(result), 400 # Convert scraped tracks to Spotify-compatible format spotify_tracks = [] for track in result['tracks']: spotify_tracks.append({ 'id': track['id'], 'name': track['name'], 'artists': track['artists'], 'album': { 'name': result['name'] if result['type'] == 'album' else '', 'images': [] }, 'duration_ms': track['duration_ms'], 'explicit': track.get('is_explicit', False), 'track_number': track.get('track_number', 0) }) url_hash = result['url_hash'] response_data = { 'id': result['id'], 'type': result['type'], 'name': result['name'], 'subtitle': result['subtitle'], 'url': result['url'], 'url_hash': url_hash, 'track_count': len(spotify_tracks), 'tracks': spotify_tracks } # Store playlist data in state for discovery (if not already there) if url_hash not in spotify_public_discovery_states: spotify_public_discovery_states[url_hash] = { 'playlist': response_data, 'phase': 'fresh', 'status': 'fresh', 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(spotify_tracks), 'discovery_results': [], 'sync_playlist_id': None, 'converted_spotify_playlist_id': None, 'download_process_id': None, 'created_at': time.time(), 'last_accessed': time.time(), 'discovery_future': None, 'sync_progress': {} } else: # Update playlist data in existing state spotify_public_discovery_states[url_hash]['playlist'] = response_data spotify_public_discovery_states[url_hash]['last_accessed'] = time.time() logger.info(f"Spotify {parsed['type']} scraped: {result['name']} ({len(spotify_tracks)} tracks)") return jsonify(response_data) except Exception as e: logger.error(f"Error parsing Spotify URL: {e}") import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/discovery/start/', methods=['POST']) def start_spotify_public_discovery(url_hash): """Start Spotify discovery process for a Spotify Public playlist""" try: # Initialize discovery state if it doesn't exist, or update existing state if url_hash in spotify_public_discovery_states: existing_state = spotify_public_discovery_states[url_hash] if existing_state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 if not existing_state.get('playlist'): return jsonify({"error": "Spotify Public playlist not found. Please parse the URL first."}), 404 # Update existing state for discovery existing_state['phase'] = 'discovering' existing_state['status'] = 'discovering' existing_state['last_accessed'] = time.time() state = existing_state else: return jsonify({"error": "Spotify Public playlist not found. Please parse the URL first."}), 404 # Add activity for discovery start playlist_name = state['playlist']['name'] track_count = len(state['playlist']['tracks']) add_activity_item("", "Spotify Link Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now") # Start discovery worker future = spotify_public_discovery_executor.submit(_run_spotify_public_discovery_worker, url_hash) state['discovery_future'] = future logger.info(f"Started Spotify discovery for Spotify Public playlist: {playlist_name}") return jsonify({"success": True, "message": "Discovery started"}) except Exception as e: logger.error(f"Error starting Spotify Public discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/discovery/status/', methods=['GET']) def get_spotify_public_discovery_status(url_hash): """Get real-time discovery status for a Spotify Public playlist""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public discovery not found"}), 404 state = spotify_public_discovery_states[url_hash] state['last_accessed'] = time.time() response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting Spotify Public discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/discovery/update_match', methods=['POST']) def update_spotify_public_discovery_match(): """Update a Spotify Public discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # url_hash track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state state = spotify_public_discovery_states.get(identifier) if not state: return jsonify({'error': 'Discovery state not found'}), 404 if track_index >= len(state['discovery_results']): return jsonify({'error': 'Invalid track index'}), 400 # Update the result result = state['discovery_results'][track_index] old_status = result.get('status') # Update with user-selected track result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = spotify_track['name'] result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) result['spotify_album'] = spotify_track['album'] result['spotify_id'] = spotify_track['id'] # Format duration duration_ms = spotify_track.get('duration_ms', 0) if duration_ms: minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 result['duration'] = f"{minutes}:{seconds:02d}" else: result['duration'] = '0:00' # IMPORTANT: Also set spotify_data for sync/download compatibility. # Manual match from the fix modal — build a rich spotify_data (album # as dict with image info) matching the normal discovery shape, and # explicitly clear any prior wing-it flag since the user picked a # real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) result['wing_it_fallback'] = False result['manual_match'] = True # Update match count if status changed from not found/error if old_status != 'found' and old_status != 'Found': state['spotify_matches'] = state.get('spotify_matches', 0) + 1 logger.info(f"Manual match updated: spotify_public - {identifier} - track {track_index}") logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") # Save manual fix to discovery cache so it appears in discovery pool try: original_track = result.get('spotify_public_track', {}) original_name = original_track.get('name', spotify_track['name']) original_artists = original_track.get('artists', []) original_artist = original_artists[0] if original_artists else '' cache_key = _get_discovery_cache_key(original_name, original_artist) # Normalize artists to plain strings for cache consistency artists_list = spotify_track['artists'] if isinstance(artists_list, list): artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] # Preserve cover image info so the download pipeline can find # artwork when this cached match is used later. The fix modal # sends image_url at the top level; search results often return # album as a bare string, which previously dropped the artwork. image_url = spotify_track.get('image_url') or '' album_raw = spotify_track.get('album', '') if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] matched_data = { 'id': spotify_track['id'], 'name': spotify_track['name'], 'artists': artists_list, 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, original_name, original_artist ) logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") except Exception as cache_err: logger.error(f"Error saving manual fix to discovery cache: {cache_err}") return jsonify({'success': True, 'result': result}) except Exception as e: logger.error(f"Error updating Spotify Public discovery match: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/spotify-public/playlists/states', methods=['GET']) def get_spotify_public_playlist_states(): """Get all stored Spotify Public playlist discovery states for frontend hydration""" try: states = [] current_time = time.time() for url_hash, state in spotify_public_discovery_states.items(): state['last_accessed'] = current_time state_info = { 'playlist_id': url_hash, 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'last_accessed': state['last_accessed'] } states.append(state_info) logger.info(f"Returning {len(states)} stored Spotify Public playlist states for hydration") return jsonify({"states": states}) except Exception as e: logger.error(f"Error getting Spotify Public playlist states: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/state/', methods=['GET']) def get_spotify_public_playlist_state(url_hash): """Get specific Spotify Public playlist state (detailed version)""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] state['last_accessed'] = time.time() response = { 'playlist_id': url_hash, 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'sync_playlist_id': state.get('sync_playlist_id'), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'sync_progress': state.get('sync_progress', {}), 'last_accessed': state['last_accessed'] } return jsonify(response) except Exception as e: logger.error(f"Error getting Spotify Public playlist state: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/reset/', methods=['POST']) def reset_spotify_public_playlist(url_hash): """Reset Spotify Public playlist to fresh phase (clear discovery/sync data)""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Reset state to fresh (preserve original playlist data) state['phase'] = 'fresh' state['status'] = 'fresh' state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['sync_playlist_id'] = None state['converted_spotify_playlist_id'] = None state['download_process_id'] = None state['sync_progress'] = {} state['discovery_future'] = None state['last_accessed'] = time.time() logger.info(f"Reset Spotify Public playlist to fresh: {url_hash}") return jsonify({"success": True, "message": "Playlist reset to fresh phase"}) except Exception as e: logger.error(f"Error resetting Spotify Public playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/delete/', methods=['POST']) def delete_spotify_public_playlist(url_hash): """Delete Spotify Public playlist state completely""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Remove from state dictionary del spotify_public_discovery_states[url_hash] logger.info(f"Deleted Spotify Public playlist state: {url_hash}") return jsonify({"success": True, "message": "Playlist deleted"}) except Exception as e: logger.error(f"Error deleting Spotify Public playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/update_phase/', methods=['POST']) def update_spotify_public_playlist_phase(url_hash): """Update Spotify Public playlist phase (used when modal closes to reset from download_complete to discovered)""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 data = request.get_json() if not data or 'phase' not in data: return jsonify({"error": "Phase not provided"}), 400 new_phase = data['phase'] valid_phases = ['fresh', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] if new_phase not in valid_phases: return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 state = spotify_public_discovery_states[url_hash] old_phase = state.get('phase', 'unknown') state['phase'] = new_phase state['last_accessed'] = time.time() # Update download process ID if provided (for download persistence) if 'download_process_id' in data: state['download_process_id'] = data['download_process_id'] # Update converted Spotify playlist ID if provided (for download persistence) if 'converted_spotify_playlist_id' in data: state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] logger.info(f"Updated Spotify Public playlist {url_hash} phase: {old_phase} → {new_phase}") return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) except Exception as e: logger.error(f"Error updating Spotify Public playlist phase: {e}") return jsonify({"error": str(e)}), 500 def _run_spotify_public_discovery_worker(url_hash): """Background worker for Spotify Public discovery process (Spotify preferred, iTunes fallback)""" _ew_state = {} try: _ew_state = _pause_enrichment_workers('Spotify Public discovery') state = spotify_public_discovery_states[url_hash] playlist = state['playlist'] # Determine which provider to use — respect user's configured primary source discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Initialize fallback client if needed itunes_client_instance = None if not use_spotify: itunes_client_instance = _get_metadata_fallback_client() logger.info(f"Starting Spotify Public discovery for: {playlist['name']} (using {discovery_source.upper()})") # Store discovery source in state for frontend state['discovery_source'] = discovery_source successful_discoveries = 0 tracks = playlist['tracks'] for i, sp_track in enumerate(tracks): if state.get('cancelled', False): break try: track_name = sp_track['name'] track_artists_raw = sp_track.get('artists', []) # Normalize artists to list of strings track_artists = [] for a in track_artists_raw: if isinstance(a, dict): track_artists.append(a.get('name', '')) else: track_artists.append(str(a)) track_id = sp_track.get('id', '') track_album = sp_track.get('album', '') if isinstance(track_album, dict): track_album_name = track_album.get('name', '') else: track_album_name = track_album or '' track_duration_ms = sp_track.get('duration_ms', 0) logger.info(f"[{i+1}/{len(tracks)}] Searching {discovery_source.upper()}: {track_name} by {', '.join(track_artists)}") # Check discovery cache first cache_key = _get_discovery_cache_key(track_name, track_artists[0] if track_artists else '') try: cache_db = get_database() cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(track_artists[0] if track_artists else '', cached_match): logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {track_name} by {', '.join(track_artists)}") # Extract display-friendly artist string from cached match cached_artists = cached_match.get('artists', []) if cached_artists: cached_artist_str = ', '.join( a if isinstance(a, str) else a.get('name', '') for a in cached_artists ) else: cached_artist_str = '' cached_album = cached_match.get('album', '') if isinstance(cached_album, dict): cached_album = cached_album.get('name', '') result = { 'spotify_public_track': { 'id': track_id, 'name': track_name, 'artists': track_artists or [], 'album': track_album_name, 'duration_ms': track_duration_ms, }, 'spotify_data': cached_match, 'match_data': cached_match, 'status': 'Found', 'status_class': 'found', 'spotify_track': cached_match.get('name', ''), 'spotify_artist': cached_artist_str, 'spotify_album': cached_album, 'spotify_id': cached_match.get('id', ''), 'discovery_source': discovery_source, 'index': i } successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) continue except Exception as cache_err: logger.error(f"Cache lookup error: {cache_err}") # Create a SimpleNamespace duck-type object for _search_spotify_for_tidal_track track_ns = types.SimpleNamespace( id=track_id, name=track_name, artists=track_artists, album=track_album_name, duration_ms=track_duration_ms ) # Use the search function with appropriate provider track_result = _search_spotify_for_tidal_track( track_ns, use_spotify=use_spotify, itunes_client=itunes_client_instance ) # Create result entry result = { 'spotify_public_track': { 'id': track_id, 'name': track_name, 'artists': track_artists or [], 'album': track_album_name, 'duration_ms': track_duration_ms, }, 'spotify_data': None, 'match_data': None, 'status': 'Not Found', 'status_class': 'not-found', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'discovery_source': discovery_source } match_confidence = 0.0 if use_spotify and isinstance(track_result, tuple): # Spotify: Function returns (Track, raw_data, confidence) track_obj, raw_track_data, match_confidence = track_result album_obj = raw_track_data.get('album', {}) if raw_track_data else {} # Ensure album has a name — fall back to track_obj.album if raw_data was missing if isinstance(album_obj, dict) and not album_obj.get('name') and track_obj.album: album_obj['name'] = track_obj.album elif not album_obj and track_obj.album: album_obj = {'name': track_obj.album} # Ensure release_date is present (raw Spotify data has it, but fallback may not) if isinstance(album_obj, dict) and not album_obj.get('release_date'): album_obj['release_date'] = getattr(track_obj, 'release_date', '') or '' # Extract image URL from album data or track object _album_images = album_obj.get('images', []) if isinstance(album_obj, dict) else [] _image_url = _album_images[0].get('url', '') if _album_images else (getattr(track_obj, 'image_url', '') or '') match_data = { 'id': track_obj.id, 'name': track_obj.name, 'artists': track_obj.artists, 'album': album_obj, 'duration_ms': track_obj.duration_ms, 'external_urls': track_obj.external_urls, 'image_url': _image_url, 'source': 'spotify' } # Preserve track_number/disc_number from raw Spotify API data if raw_track_data and raw_track_data.get('track_number'): match_data['track_number'] = raw_track_data['track_number'] if raw_track_data and raw_track_data.get('disc_number'): match_data['disc_number'] = raw_track_data['disc_number'] result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = track_obj.name result['spotify_artist'] = ', '.join(track_obj.artists) if isinstance(track_obj.artists, list) else str(track_obj.artists) result['spotify_album'] = album_obj.get('name', '') if isinstance(album_obj, dict) else str(album_obj) result['spotify_id'] = track_obj.id result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries elif not use_spotify and track_result and isinstance(track_result, dict): # Fallback: Function returns a dict with track data (includes 'confidence' key) match_confidence = track_result.pop('confidence', 0.80) match_data = track_result match_data['source'] = discovery_source # Extract image URL from album images _fb_album = match_data.get('album', {}) _fb_images = _fb_album.get('images', []) if isinstance(_fb_album, dict) else [] if _fb_images and 'image_url' not in match_data: match_data['image_url'] = _fb_images[0].get('url', '') result['spotify_data'] = match_data result['match_data'] = match_data result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = match_data.get('name', '') itunes_artists = match_data.get('artists', []) result['spotify_artist'] = ', '.join(a if isinstance(a, str) else a.get('name', '') for a in itunes_artists) if itunes_artists else '' result['spotify_album'] = match_data.get('album', {}).get('name', '') if isinstance(match_data.get('album'), dict) else match_data.get('album', '') result['spotify_id'] = match_data.get('id', '') result['confidence'] = match_confidence successful_discoveries += 1 state['spotify_matches'] = successful_discoveries # Save to discovery cache if match found if result['status_class'] == 'found' and result.get('match_data'): try: cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, match_confidence, result['match_data'], track_name, track_artists[0] if track_artists else '' ) logger.info(f"CACHE SAVED: {track_name} (confidence: {match_confidence:.3f})") except Exception as cache_err: logger.error(f"Cache save error: {cache_err}") # Auto Wing It fallback for unmatched tracks if result['status_class'] == 'not-found': sp_t = result.get('spotify_public_track', {}) stub = _build_discovery_wing_it_stub( sp_t.get('name', ''), ', '.join(sp_t.get('artists', [])), sp_t.get('duration_ms', 0) ) result['status'] = 'Wing It' result['status_class'] = 'wing-it' result['spotify_data'] = stub result['match_data'] = stub result['spotify_track'] = sp_t.get('name', '') result['spotify_artist'] = ', '.join(sp_t.get('artists', [])) result['wing_it_fallback'] = True result['confidence'] = 0 successful_discoveries += 1 state['spotify_matches'] = successful_discoveries state['wing_it_count'] = state.get('wing_it_count', 0) + 1 result['index'] = i state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) # Add delay between requests time.sleep(0.1) except Exception as e: logger.error(f"Error processing track {i+1}: {e}") # Add error result result = { 'spotify_public_track': { 'name': sp_track.get('name', 'Unknown'), 'artists': sp_track.get('artists', []), }, 'spotify_data': None, 'match_data': None, 'status': 'Error', 'status_class': 'error', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'error': str(e), 'discovery_source': discovery_source, 'index': i } state['discovery_results'].append(result) state['discovery_progress'] = int(((i + 1) / len(tracks)) * 100) # Mark as complete state['phase'] = 'discovered' state['status'] = 'discovered' state['discovery_progress'] = 100 # Add activity for discovery completion source_label = discovery_source.upper() add_activity_item("", f"Spotify Link Discovery Complete ({source_label})", f"'{playlist['name']}' - {successful_discoveries}/{len(tracks)} tracks found", "Now") logger.info(f"Spotify Public discovery complete ({source_label}): {successful_discoveries}/{len(tracks)} tracks found") except Exception as e: logger.error(f"Error in Spotify Public discovery worker: {e}") if url_hash in spotify_public_discovery_states: spotify_public_discovery_states[url_hash]['phase'] = 'error' spotify_public_discovery_states[url_hash]['status'] = f'error: {str(e)}' finally: _resume_enrichment_workers(_ew_state, 'Spotify Public discovery') def convert_spotify_public_results_to_spotify_tracks(discovery_results): """Convert Spotify Public discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': spotify_data['artists'], 'album': spotify_data['album'], 'duration_ms': spotify_data.get('duration_ms', 0) } # Preserve track_number/disc_number from discovery enrichment if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': track = { 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': result.get('spotify_album', 'Unknown Album'), 'duration_ms': 0 } spotify_tracks.append(track) logger.info(f"Converted {len(spotify_tracks)} Spotify Public matches to Spotify tracks for sync") return spotify_tracks # =================================================================== # SPOTIFY PUBLIC SYNC API ENDPOINTS # =================================================================== @app.route('/api/spotify-public/sync/start/', methods=['POST']) def start_spotify_public_sync(url_hash): """Start sync process for a Spotify Public playlist using discovered Spotify tracks""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] state['last_accessed'] = time.time() if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: return jsonify({"error": "Spotify Public playlist not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_spotify_public_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"spotify_public_{url_hash}" playlist_name = state['playlist']['name'] # Add activity for sync start add_activity_item("", "Spotify Link Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") # Update Spotify Public state state['phase'] = 'syncing' state['sync_playlist_id'] = sync_playlist_id state['sync_progress'] = {} # Start the sync using existing sync infrastructure sync_data = { 'playlist_id': sync_playlist_id, 'playlist_name': playlist_name, 'tracks': spotify_tracks } with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Submit sync task playlist_image_url = state['playlist'].get('image_url', '') future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) active_sync_workers[sync_playlist_id] = future logger.info(f"Started Spotify Public sync for: {playlist_name} ({len(spotify_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting Spotify Public sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/sync/status/', methods=['GET']) def get_spotify_public_sync_status(url_hash): """Get sync status for a Spotify Public playlist""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] state['last_accessed'] = time.time() sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync in progress"}), 404 # Get sync status from existing sync infrastructure with sync_lock: sync_state = sync_states.get(sync_playlist_id, {}) response = { 'phase': state['phase'], 'sync_status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Update Spotify Public state if sync completed if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' state['sync_progress'] = sync_state.get('progress', {}) playlist_name = state['playlist']['name'] add_activity_item("", "Sync Complete", f"Spotify Link playlist '{playlist_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error playlist_name = state['playlist']['name'] add_activity_item("", "Sync Failed", f"Spotify Link playlist '{playlist_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting Spotify Public sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/spotify-public/sync/cancel/', methods=['POST']) def cancel_spotify_public_sync(url_hash): """Cancel sync for a Spotify Public playlist""" try: if url_hash not in spotify_public_discovery_states: return jsonify({"error": "Spotify Public playlist not found"}), 404 state = spotify_public_discovery_states[url_hash] state['last_accessed'] = time.time() sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Clean up sync worker if sync_playlist_id in active_sync_workers: del active_sync_workers[sync_playlist_id] # Revert Spotify Public state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} return jsonify({"success": True, "message": "Spotify Public sync cancelled"}) except Exception as e: logger.error(f"Error cancelling Spotify Public sync: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # YOUTUBE PLAYLIST API ENDPOINTS # =================================================================== # Global state for YouTube playlist management (persistent across page reloads) youtube_playlist_states = {} # Key: url_hash, Value: persistent playlist state youtube_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="youtube_discovery") # Global state for Beatport chart management (persistent across page reloads) beatport_chart_states = {} # Key: url_hash, Value: persistent chart state beatport_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="beatport_discovery") # Global state for ListenBrainz playlist management (persistent across page reloads) listenbrainz_playlist_states = {} # Key: playlist_mbid, Value: persistent playlist state listenbrainz_discovery_executor = ThreadPoolExecutor(max_workers=3, thread_name_prefix="listenbrainz_discovery") @app.route('/api/youtube/parse', methods=['POST']) def parse_youtube_playlist_endpoint(): """Parse a YouTube playlist URL and return structured track data""" try: data = request.get_json() url = data.get('url', '').strip() if not url: return jsonify({"error": "YouTube URL is required"}), 400 # Validate URL if not ('youtube.com/playlist' in url or 'music.youtube.com/playlist' in url): return jsonify({"error": "Invalid YouTube playlist URL"}), 400 logger.info(f"Parsing YouTube playlist: {url}") # Parse the playlist using our function playlist_data = parse_youtube_playlist(url) if not playlist_data: return jsonify({"error": "Failed to parse YouTube playlist"}), 500 # Use deterministic hash for state tracking (built-in hash() is randomized per process restart) import hashlib yt_playlist_id = playlist_data.get('id', '') if yt_playlist_id and yt_playlist_id != 'unknown_id': # Use canonical URL with the stable YouTube playlist ID canonical_url = f"https://youtube.com/playlist?list={yt_playlist_id}" else: canonical_url = url url_hash = hashlib.md5(canonical_url.encode()).hexdigest()[:12] # Migrate existing mirrored playlists that used the old non-deterministic hash() # and deduplicate any copies created by the bug try: database = get_database() profile_id = get_current_profile_id() existing = database.get_mirrored_playlists(profile_id=profile_id) yt_dupes = [mp for mp in existing if mp['source'] == 'youtube' and mp['name'] == playlist_data['name']] if yt_dupes: # Keep the newest one, delete the rest keep = yt_dupes[0] # Already sorted by updated_at DESC from get_mirrored_playlists for dupe in yt_dupes[1:]: database.delete_mirrored_playlist(dupe['id']) logger.info(f"Removed duplicate YouTube mirrored playlist '{dupe['name']}' (id={dupe['id']})") # Update the kept entry's source_playlist_id to the new deterministic hash if keep['source_playlist_id'] != url_hash: with database._get_connection() as conn: cursor = conn.cursor() cursor.execute( "UPDATE mirrored_playlists SET source_playlist_id = ? WHERE id = ?", (url_hash, keep['id']) ) conn.commit() logger.info(f"Migrated YouTube mirrored playlist '{keep['name']}' source_playlist_id to deterministic hash {url_hash}") except Exception as e: logger.debug(f"YouTube mirror migration check: {e}") # Initialize persistent playlist state (similar to Spotify download_batches structure) youtube_playlist_states[url_hash] = { 'playlist': playlist_data, 'phase': 'fresh', # fresh -> discovering -> discovered -> syncing -> sync_complete -> downloading -> download_complete 'discovery_results': [], 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(playlist_data['tracks']), 'status': 'parsed', 'url': url, 'sync_playlist_id': None, 'converted_spotify_playlist_id': None, 'download_process_id': None, # Track associated download missing tracks process 'created_at': time.time(), 'last_accessed': time.time(), 'discovery_future': None, 'sync_progress': {} } playlist_data['url_hash'] = url_hash logger.info(f"YouTube playlist parsed successfully: {playlist_data['name']} ({len(playlist_data['tracks'])} tracks)") return jsonify(playlist_data) except Exception as e: logger.error(f"Error parsing YouTube playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/discovery/start/', methods=['POST']) def start_youtube_discovery(url_hash): """Start Spotify discovery process for a YouTube playlist""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Update access time if state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 # Update phase to discovering state['phase'] = 'discovering' state['status'] = 'discovering' state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['discovery_results'] = [] # Clear skip_discovery flags on all tracks (in case of prior retry) for track in state['playlist']['tracks']: track.pop('skip_discovery', None) # Add activity for discovery start playlist_name = state['playlist']['name'] track_count = len(state['playlist']['tracks']) add_activity_item("", "YouTube Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now") # Start discovery worker future = youtube_discovery_executor.submit(_run_youtube_discovery_worker, url_hash) state['discovery_future'] = future logger.info(f"Started Spotify discovery for YouTube playlist: {state['playlist']['name']}") return jsonify({"success": True, "message": "Discovery started"}) except Exception as e: logger.error(f"Error starting YouTube discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/discovery/status/', methods=['GET']) def get_youtube_discovery_status(url_hash): """Get real-time discovery status for a YouTube playlist""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Update access time response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting YouTube discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/discovery/unmatch', methods=['POST']) @app.route('/api/tidal/discovery/unmatch', methods=['POST']) @app.route('/api/deezer/discovery/unmatch', methods=['POST']) @app.route('/api/spotify-public/discovery/unmatch', methods=['POST']) @app.route('/api/beatport/discovery/unmatch', methods=['POST']) @app.route('/api/listenbrainz/discovery/unmatch', methods=['POST']) def unmatch_discovery_track(): """Remove a discovery match — sets track back to Not Found""" try: data = request.get_json() identifier = data.get('identifier') track_index = data.get('track_index') if not identifier or track_index is None: return jsonify({'success': False, 'error': 'Missing required fields'}), 400 # Find the state dict for this discovery state = (youtube_playlist_states.get(identifier) or tidal_discovery_states.get(identifier) or deezer_discovery_states.get(identifier) or spotify_public_discovery_states.get(identifier) or beatport_chart_states.get(identifier) or listenbrainz_playlist_states.get(identifier)) if not state: return jsonify({'success': False, 'error': 'Discovery state not found'}), 404 results = state.get('discovery_results', []) if track_index >= len(results): return jsonify({'success': False, 'error': 'Invalid track index'}), 400 result = results[track_index] old_status = result.get('status_class') # Clear the match result['status'] = 'Not Found' result['status_class'] = 'not-found' result['spotify_track'] = '' result['spotify_artist'] = '' result['spotify_album'] = '' result['spotify_data'] = None result['matched_data'] = None result['match_data'] = None result['confidence'] = 0 result['wing_it_fallback'] = False result['manual_match'] = False # Update match count if old_status in ('found', 'wing-it'): state['spotify_matches'] = max(0, state.get('spotify_matches', 0) - 1) if old_status == 'wing-it': state['wing_it_count'] = max(0, state.get('wing_it_count', 0) - 1) # If mirrored playlist, also clear in DB if identifier.startswith('mirrored_'): try: db = get_database() tracks = state.get('tracks', []) if track_index < len(tracks): db_track_id = tracks[track_index].get('db_track_id') if db_track_id: db.update_mirrored_track_extra_data(db_track_id, { 'discovered': False, 'discovery_attempted': True, 'provider': '', 'unmatched_by_user': True, }) except Exception as e: logger.error(f"Error clearing mirrored track match: {e}") logger.info(f"Unmatched discovery track {track_index}: {result.get('yt_track', result.get('lb_track', ''))}") return jsonify({'success': True}) except Exception as e: logger.error(f"Error unmatching discovery track: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/youtube/discovery/update_match', methods=['POST']) def update_youtube_discovery_match(): """Update a YouTube discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # url_hash track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state state = youtube_playlist_states.get(identifier) if not state: return jsonify({'error': 'Discovery state not found'}), 404 if track_index >= len(state['discovery_results']): return jsonify({'error': 'Invalid track index'}), 400 # Update the result result = state['discovery_results'][track_index] old_status = result.get('status') # Update with user-selected track result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = spotify_track['name'] result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) result['spotify_album'] = spotify_track['album'] result['spotify_id'] = spotify_track['id'] # Format duration duration_ms = spotify_track.get('duration_ms', 0) if duration_ms: minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 result['duration'] = f"{minutes}:{seconds:02d}" else: result['duration'] = '0:00' # IMPORTANT: Also set spotify_data for sync/download compatibility. # Manual match from the fix modal — build a rich spotify_data (album # as dict with image info) matching the normal discovery shape, and # explicitly clear any prior wing-it flag since the user picked a # real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) result['wing_it_fallback'] = False result['manual_match'] = True # Flag for tracking # Update match count if status changed from not found/error if old_status != 'found' and old_status != 'Found': state['spotify_matches'] = state.get('spotify_matches', 0) + 1 logger.info(f"Manual match updated: youtube - {identifier} - track {track_index}") logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") # Save manual fix to discovery cache so it appears in discovery pool try: # Get original track name from the YouTube/source track data original_track = result.get('youtube_track', result.get('tidal_track', result.get('deezer_track', {}))) original_name = original_track.get('name', spotify_track['name']) original_artists = original_track.get('artists', []) if original_artists: original_artist = original_artists[0] if isinstance(original_artists[0], str) else original_artists[0].get('name', '') else: original_artist = '' cache_key = _get_discovery_cache_key(original_name, original_artist) # Normalize artists to plain strings for cache consistency artists_list = spotify_track['artists'] if isinstance(artists_list, list): artists_list = [a if isinstance(a, str) else a.get('name', '') for a in artists_list] # Preserve cover image info so the download pipeline can find # artwork when this cached match is used later. The fix modal # sends image_url at the top level; search results often return # album as a bare string, which previously dropped the artwork. image_url = spotify_track.get('image_url') or '' album_raw = spotify_track.get('album', '') if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] matched_data = { 'id': spotify_track['id'], 'name': spotify_track['name'], 'artists': artists_list, 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, original_name, original_artist ) logger.info(f"Manual fix saved to discovery cache: {original_name} by {original_artist}") except Exception as cache_err: logger.error(f"Error saving manual fix to discovery cache: {cache_err}") # Persist manual fix to DB for mirrored playlists if identifier.startswith('mirrored_'): try: tracks = state['playlist']['tracks'] if track_index < len(tracks): db_track_id = tracks[track_index].get('db_track_id') if db_track_id: db = get_database() extra_data = { 'discovered': True, 'provider': 'spotify', 'confidence': 1.0, 'matched_data': matched_data, 'manual_match': True, } db.update_mirrored_track_extra_data(db_track_id, extra_data) result['matched_data'] = matched_data logger.info(f"Persisted manual fix to DB for track {db_track_id}") except Exception as wb_err: logger.error(f"Error persisting manual fix to DB: {wb_err}") return jsonify({'success': True, 'result': result}) except Exception as e: logger.error(f"Error updating YouTube discovery match: {e}") return jsonify({'error': str(e)}), 500 def _build_discovery_wing_it_stub(track_name, artist_name, duration_ms=0, image_url=''): """Build stub matched_data for tracks that failed metadata discovery. Used as automatic Wing It fallback so tracks still flow through the download pipeline.""" return { 'id': f"wing_it_{hash(f'{artist_name}_{track_name}') % 100000}", 'name': track_name, 'artists': [{'name': artist_name}] if isinstance(artist_name, str) else artist_name, 'album': {'name': '', 'album_type': 'single', 'images': [], 'release_date': ''}, 'duration_ms': duration_ms, 'image_url': image_url, 'source': 'wing_it_fallback', } def _build_fix_modal_spotify_data(spotify_track): """Build a rich spotify_data dict from the fix-modal POST payload so manual matches carry the same shape as normal discovery results. Key points: - album is always a dict (normal discovery has it this way; legacy fix-modal produced a bare string which broke cover art lookup downstream) - image_url is carried both at top level and inside album.images for parity with Spotify API responses - handles both legacy string albums (most search endpoints return this) and newer object albums """ if not isinstance(spotify_track, dict): spotify_track = {} image_url = spotify_track.get('image_url') or '' album_raw = spotify_track.get('album', '') if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] return { 'id': spotify_track.get('id', ''), 'name': spotify_track.get('name', ''), 'artists': spotify_track.get('artists', []), 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, } # YouTube discovery worker logic lives in core/discovery/youtube.py. from core.discovery import youtube as _discovery_youtube def _build_youtube_discovery_deps(): """Build the YoutubeDiscoveryDeps bundle from web_server.py globals on each call.""" return _discovery_youtube.YoutubeDiscoveryDeps( youtube_playlist_states=youtube_playlist_states, spotify_client=spotify_client, matching_engine=matching_engine, pause_enrichment_workers=_pause_enrichment_workers, resume_enrichment_workers=_resume_enrichment_workers, get_active_discovery_source=_get_active_discovery_source, get_metadata_fallback_client=_get_metadata_fallback_client, get_discovery_cache_key=_get_discovery_cache_key, validate_discovery_cache_artist=_validate_discovery_cache_artist, extract_artist_name=_extract_artist_name, spotify_rate_limited=_spotify_rate_limited, discovery_score_candidates=_discovery_score_candidates, get_metadata_cache=get_metadata_cache, build_discovery_wing_it_stub=_build_discovery_wing_it_stub, get_database=get_database, add_activity_item=add_activity_item, ) def _run_youtube_discovery_worker(url_hash): return _discovery_youtube.run_youtube_discovery_worker(url_hash, _build_youtube_discovery_deps()) def _run_listenbrainz_discovery_worker(state_key): """Background worker for ListenBrainz music discovery process (Spotify preferred, iTunes fallback)""" playlist_mbid = state_key.split(':', 1)[1] if ':' in state_key else state_key _ew_state = {} try: _ew_state = _pause_enrichment_workers('ListenBrainz discovery') state = listenbrainz_playlist_states[state_key] playlist = state['playlist'] tracks = playlist['tracks'] # Determine which provider to use (Spotify preferred, iTunes fallback) discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Get fallback client itunes_client = _get_metadata_fallback_client() logger.info(f"Starting {discovery_source} discovery for {len(tracks)} ListenBrainz tracks...") # Store the discovery source in state state['discovery_source'] = discovery_source # Process each track for discovery for i, track in enumerate(tracks): try: # Check for cancellation if state.get('phase') != 'discovering': logger.warning(f"ListenBrainz discovery cancelled (phase changed to '{state.get('phase')}')") return # Update progress state['discovery_progress'] = int((i / len(tracks)) * 100) # Get cleaned track data from ListenBrainz cleaned_title = track['track_name'] cleaned_artist = track['artist_name'] album_name = track.get('album_name', '') duration_ms = track.get('duration_ms', 0) logger.info(f"Searching {discovery_source} for: '{cleaned_artist}' - '{cleaned_title}'") # Check discovery cache first cache_key = _get_discovery_cache_key(cleaned_title, cleaned_artist) try: cache_db = get_database() cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(cleaned_artist, cached_match): logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {cleaned_artist} - {cleaned_title}") result = { 'index': i, 'lb_track': cleaned_title, 'lb_artist': cleaned_artist, 'status': 'Found', 'status_class': 'found', 'spotify_track': cached_match.get('name', ''), 'spotify_artist': _extract_artist_name(cached_match.get('artists', [''])[0]) if cached_match.get('artists') else '', 'spotify_album': cached_match.get('album', {}).get('name', '') if isinstance(cached_match.get('album'), dict) else cached_match.get('album', ''), 'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00', 'discovery_source': discovery_source, 'matched_data': cached_match, 'spotify_data': cached_match } state['spotify_matches'] += 1 state['discovery_results'].append(result) continue except Exception as cache_err: logger.error(f"Cache lookup error: {cache_err}") # Try multiple search strategies using matching engine matched_track = None best_confidence = 0.0 best_raw_track = None min_confidence = 0.9 source_duration = duration_ms or 0 # Strategy 1: Use matching_engine search queries try: temp_track = type('TempTrack', (), { 'name': cleaned_title, 'artists': [cleaned_artist], 'album': album_name if album_name else None })() search_queries = matching_engine.generate_download_queries(temp_track) logger.info(f"Generated {len(search_queries)} search queries for ListenBrainz track") except Exception as e: logger.error(f"Matching engine failed for ListenBrainz, falling back to basic query: {e}") search_queries = [f"{cleaned_artist} {cleaned_title}", cleaned_title] for query_idx, search_query in enumerate(search_queries): try: logger.debug(f"ListenBrainz query {query_idx + 1}/{len(search_queries)}: {search_query}") search_results = None if use_spotify and not _spotify_rate_limited(): search_results = spotify_client.search_tracks(search_query, limit=10) else: search_results = itunes_client.search_tracks(search_query, limit=10) if not search_results: continue # Score all results using the matching engine match, confidence, match_idx = _discovery_score_candidates( cleaned_title, cleaned_artist, source_duration, search_results ) if match and confidence > best_confidence and confidence >= min_confidence: best_confidence = confidence matched_track = match if use_spotify and match.id: _cache = get_metadata_cache() best_raw_track = _cache.get_entity('spotify', 'track', match.id) else: best_raw_track = None logger.info(f"New best ListenBrainz match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") if best_confidence >= 0.9: logger.info(f"High confidence ListenBrainz match found ({best_confidence:.3f}), stopping search") break except Exception as e: logger.debug(f"Error in ListenBrainz search for query '{search_query}': {e}") continue if matched_track: logger.info(f"Strategy 1 ListenBrainz match: {matched_track.artists[0]} - {matched_track.name} (confidence: {best_confidence:.3f})") # Strategy 2: Swapped search (if first failed) - score results properly if not matched_track: logger.info("ListenBrainz Strategy 2: Trying swapped search (artist/title reversed)") if use_spotify: query = f"artist:{cleaned_title} track:{cleaned_artist}" fallback_results = spotify_client.search_tracks(query, limit=5) else: query = f"{cleaned_title} {cleaned_artist}" fallback_results = itunes_client.search_tracks(query, limit=5) if fallback_results: match, confidence, _ = _discovery_score_candidates( cleaned_title, cleaned_artist, source_duration, fallback_results ) if match and confidence >= min_confidence: matched_track = match best_confidence = confidence logger.info(f"Strategy 2 ListenBrainz match (swapped): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") # Strategy 3: Album-based search (if still failed and we have album name) - score results properly if not matched_track and album_name: logger.info(f"ListenBrainz Strategy 3: Trying album-based search: '{cleaned_artist} {album_name} {cleaned_title}'") if use_spotify: query = f"artist:{cleaned_artist} album:{album_name} track:{cleaned_title}" fallback_results = spotify_client.search_tracks(query, limit=5) else: query = f"{cleaned_artist} {album_name} {cleaned_title}" fallback_results = itunes_client.search_tracks(query, limit=5) if fallback_results: match, confidence, _ = _discovery_score_candidates( cleaned_title, cleaned_artist, source_duration, fallback_results ) if match and confidence >= min_confidence: matched_track = match best_confidence = confidence logger.info(f"Strategy 3 ListenBrainz match (album): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") # Strategy 4: Extended search with higher limit (last resort) if not matched_track: logger.info("ListenBrainz Strategy 4: Extended search with limit=50") query = f"{cleaned_artist} {cleaned_title}" if use_spotify: extended_results = spotify_client.search_tracks(query, limit=50) else: extended_results = itunes_client.search_tracks(query, limit=50) if extended_results: match, confidence, _ = _discovery_score_candidates( cleaned_title, cleaned_artist, source_duration, extended_results ) if match and confidence >= min_confidence: matched_track = match best_confidence = confidence logger.info(f"Strategy 4 ListenBrainz match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") # Create result entry result = { 'index': i, 'lb_track': cleaned_title, 'lb_artist': cleaned_artist, 'status': 'Found' if matched_track else 'Not Found', 'status_class': 'found' if matched_track else 'not-found', 'spotify_track': matched_track.name if matched_track else '', 'spotify_artist': _extract_artist_name(matched_track.artists[0]) if matched_track else '', 'spotify_album': matched_track.album if matched_track else '', 'duration': f"{duration_ms // 60000}:{(duration_ms % 60000) // 1000:02d}" if duration_ms else '0:00', 'discovery_source': discovery_source, 'confidence': best_confidence } if matched_track: state['spotify_matches'] += 1 # Build album data based on provider if use_spotify and best_raw_track: album_data = best_raw_track.get('album', {}) else: album_data = { 'name': matched_track.album, 'album_type': 'album', 'release_date': getattr(matched_track, 'release_date', '') or '', 'images': [{'url': matched_track.image_url}] if hasattr(matched_track, 'image_url') and matched_track.image_url else [] } # Extract image URL for discovery pool display _yt_album_images = album_data.get('images', []) _yt_image_url = _yt_album_images[0].get('url', '') if _yt_album_images else (getattr(matched_track, 'image_url', '') or '') result['matched_data'] = { 'id': matched_track.id, 'name': matched_track.name, 'artists': matched_track.artists, 'album': album_data, 'duration_ms': matched_track.duration_ms, 'image_url': _yt_image_url, 'source': discovery_source } result['spotify_data'] = result['matched_data'] # Save to discovery cache (only high-confidence matches) if best_confidence >= 0.7: try: cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, best_confidence, result['matched_data'], cleaned_title, cleaned_artist ) logger.info(f"CACHE SAVED: {cleaned_artist} - {cleaned_title} (confidence: {best_confidence:.3f})") except Exception as cache_err: logger.error(f"Cache save error: {cache_err}") else: # Auto Wing It fallback — build stub from raw source data stub = _build_discovery_wing_it_stub(cleaned_title, cleaned_artist, duration_ms) result['status'] = 'Wing It' result['status_class'] = 'wing-it' result['spotify_track'] = cleaned_title result['spotify_artist'] = cleaned_artist result['spotify_album'] = '' result['matched_data'] = stub result['spotify_data'] = stub result['wing_it_fallback'] = True state['wing_it_count'] = state.get('wing_it_count', 0) + 1 state['discovery_results'].append(result) logger.info(f" {'' if matched_track else ''} Track {i+1}/{len(tracks)}: {result['status']}") except Exception as e: logger.error(f"Error processing track {i}: {e}") result = { 'index': i, 'lb_track': track['track_name'], 'lb_artist': track['artist_name'], 'status': 'Error', 'status_class': 'error', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'duration': '0:00' } state['discovery_results'].append(result) # Complete discovery state['phase'] = 'discovered' state['status'] = 'complete' state['discovery_progress'] = 100 playlist_name = playlist.get('name') or playlist.get('title') or 'Unknown Playlist' source_label = discovery_source.upper() add_activity_item("", f"ListenBrainz Discovery Complete ({source_label})", f"'{playlist_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") logger.info(f"ListenBrainz discovery complete ({discovery_source}): {state['spotify_matches']}/{len(tracks)} tracks matched") except Exception as e: logger.error(f"Error in ListenBrainz discovery worker: {e}") state['status'] = 'error' state['phase'] = 'fresh' finally: _resume_enrichment_workers(_ew_state, 'ListenBrainz discovery') def _calculate_similarity(str1, str2): """Calculate string similarity using simple character overlap""" if not str1 or not str2: return 0 # Convert to lowercase and remove extra spaces str1 = str1.lower().strip() str2 = str2.lower().strip() if str1 == str2: return 1.0 # Calculate character overlap set1 = set(str1.replace(' ', '')) set2 = set(str2.replace(' ', '')) if not set1 or not set2: return 0 intersection = len(set1.intersection(set2)) union = len(set1.union(set2)) return intersection / union if union > 0 else 0 @app.route('/api/youtube/sync/start/', methods=['POST']) def start_youtube_sync(url_hash): """Start sync process for a YouTube playlist using discovered Spotify tracks""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Update access time if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: return jsonify({"error": "YouTube playlist not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_youtube_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"youtube_{url_hash}" playlist_name = state['playlist']['name'] # Add activity for sync start add_activity_item("", "YouTube Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") # Update YouTube state state['phase'] = 'syncing' state['sync_playlist_id'] = sync_playlist_id state['sync_progress'] = {} # Start the sync using existing sync infrastructure sync_data = { 'playlist_id': sync_playlist_id, 'playlist_name': playlist_name, 'tracks': spotify_tracks } with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Submit sync task playlist_image_url = state['playlist'].get('image_url', '') future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id(), playlist_image_url) active_sync_workers[sync_playlist_id] = future logger.info(f"Started YouTube sync for: {playlist_name} ({len(spotify_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting YouTube sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/sync/status/', methods=['GET']) def get_youtube_sync_status(url_hash): """Get sync status for a YouTube playlist""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync in progress"}), 404 # Get sync status from existing sync infrastructure with sync_lock: sync_state = sync_states.get(sync_playlist_id, {}) response = { 'phase': state['phase'], 'sync_status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Update YouTube state if sync completed if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' state['sync_progress'] = sync_state.get('progress', {}) # Add activity for sync completion playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') add_activity_item("", "Sync Complete", f"YouTube playlist '{playlist_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') add_activity_item("", "Sync Failed", f"YouTube playlist '{playlist_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting YouTube sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/sync/cancel/', methods=['POST']) def cancel_youtube_sync(url_hash): """Cancel sync for a YouTube playlist""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Clean up sync worker if sync_playlist_id in active_sync_workers: del active_sync_workers[sync_playlist_id] # Revert YouTube state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} return jsonify({"success": True, "message": "YouTube sync cancelled"}) except Exception as e: logger.error(f"Error cancelling YouTube sync: {e}") return jsonify({"error": str(e)}), 500 # New YouTube Playlist Management Endpoints (for persistent state) @app.route('/api/youtube/playlists', methods=['GET']) def get_all_youtube_playlists(): """Get all stored YouTube playlists for frontend hydration (similar to Spotify playlists)""" try: playlists = [] current_time = time.time() for url_hash, state in youtube_playlist_states.items(): # Skip mirrored playlist entries — they have their own hydration if url_hash.startswith('mirrored_'): continue # Update access time when requested state['last_accessed'] = current_time # Return essential data for card recreation playlist_info = { 'url_hash': url_hash, 'url': state['url'], 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'created_at': state['created_at'], 'last_accessed': state['last_accessed'] } playlists.append(playlist_info) logger.info(f"Returning {len(playlists)} stored YouTube playlists for hydration") return jsonify({"playlists": playlists}) except Exception as e: logger.error(f"Error getting YouTube playlists: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/state/', methods=['GET']) def get_youtube_playlist_state(url_hash): """Get specific YouTube playlist state (detailed version of status endpoint)""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] state['last_accessed'] = time.time() # Return full state information (including results for modal hydration) response = { 'url_hash': url_hash, 'url': state['url'], 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'sync_playlist_id': state['sync_playlist_id'], 'converted_spotify_playlist_id': state['converted_spotify_playlist_id'], 'sync_progress': state['sync_progress'], 'created_at': state['created_at'], 'last_accessed': state['last_accessed'] } return jsonify(response) except Exception as e: logger.error(f"Error getting YouTube playlist state: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/reset/', methods=['POST']) def reset_youtube_playlist(url_hash): """Reset YouTube playlist to fresh phase (clear discovery/sync data)""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Reset state to fresh (preserve original playlist data) state['phase'] = 'fresh' state['status'] = 'parsed' state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['sync_playlist_id'] = None state['converted_spotify_playlist_id'] = None state['sync_progress'] = {} state['discovery_future'] = None state['last_accessed'] = time.time() logger.info(f"Reset YouTube playlist to fresh phase: {state['playlist']['name']}") return jsonify({"success": True, "message": "Playlist reset to fresh state"}) except Exception as e: logger.error(f"Error resetting YouTube playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/delete/', methods=['DELETE']) def delete_youtube_playlist(url_hash): """Remove YouTube playlist from backend storage entirely""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 state = youtube_playlist_states[url_hash] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Remove from storage playlist_name = state['playlist']['name'] del youtube_playlist_states[url_hash] logger.info(f"Deleted YouTube playlist from backend: {playlist_name}") return jsonify({"success": True, "message": f"Playlist '{playlist_name}' deleted"}) except Exception as e: logger.error(f"Error deleting YouTube playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/youtube/update_phase/', methods=['POST']) def update_youtube_playlist_phase(url_hash): """Update YouTube playlist phase (used when modal closes to reset from download_complete to discovered)""" try: if url_hash not in youtube_playlist_states: return jsonify({"error": "YouTube playlist not found"}), 404 data = request.get_json() if not data or 'phase' not in data: return jsonify({"error": "Phase not provided"}), 400 new_phase = data['phase'] valid_phases = ['fresh', 'parsed', 'discovering', 'discovered', 'syncing', 'sync_complete', 'downloading', 'download_complete'] if new_phase not in valid_phases: return jsonify({"error": f"Invalid phase. Must be one of: {', '.join(valid_phases)}"}), 400 state = youtube_playlist_states[url_hash] old_phase = state.get('phase', 'unknown') state['phase'] = new_phase state['last_accessed'] = time.time() logger.info(f"Updated YouTube playlist {url_hash} phase: {old_phase} → {new_phase}") return jsonify({"success": True, "message": f"Phase updated to {new_phase}", "old_phase": old_phase, "new_phase": new_phase}) except Exception as e: logger.error(f"Error updating YouTube playlist phase: {e}") return jsonify({"error": str(e)}), 500 def convert_youtube_results_to_spotify_tracks(discovery_results): """Convert YouTube discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] # Create track object matching the expected format track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': spotify_data['artists'], 'album': spotify_data['album'], 'duration_ms': spotify_data.get('duration_ms', 0) } if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': # Build from individual fields (automatic discovery format) track = { 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': result.get('spotify_album', 'Unknown Album'), 'duration_ms': 0 } spotify_tracks.append(track) logger.info(f"Converted {len(spotify_tracks)} YouTube matches to Spotify tracks for sync") return spotify_tracks # Add these new endpoints to the end of web_server.py # Sync background worker logic lives in core/discovery/sync.py. from core.discovery import sync as _discovery_sync def _build_sync_deps(): """Build the SyncDeps bundle from web_server.py globals on each call.""" return _discovery_sync.SyncDeps( config_manager=config_manager, sync_service=sync_service, plex_client=plex_client, jellyfin_client=jellyfin_client, automation_engine=automation_engine, run_async=run_async, record_sync_history_start=_record_sync_history_start, update_automation_progress=_update_automation_progress, update_and_save_sync_status=_update_and_save_sync_status, sync_states=sync_states, sync_lock=sync_lock, ) def _run_sync_task(playlist_id, playlist_name, tracks_json, automation_id=None, profile_id=1, playlist_image_url=''): return _discovery_sync.run_sync_task( playlist_id, playlist_name, tracks_json, automation_id, profile_id, playlist_image_url, _build_sync_deps(), ) @app.route('/api/sync/start', methods=['POST']) def start_playlist_sync(): """Starts a new sync process for a given playlist.""" request_start_time = time.time() logger.info(f"⏱️ [TIMING] Sync request received at {time.strftime('%H:%M:%S')}") data = request.get_json() playlist_id = data.get('playlist_id') playlist_name = data.get('playlist_name') tracks_json = data.get('tracks') # Pass the full track list playlist_image_url = data.get('image_url', '') if not all([playlist_id, playlist_name, tracks_json]): return jsonify({"success": False, "error": "Missing playlist_id, name, or tracks."}), 400 # Add activity for sync start add_activity_item("", "Spotify Sync Started", f"'{playlist_name}' - {len(tracks_json)} tracks", "Now") logger.info(f"Starting playlist sync for '{playlist_name}' with {len(tracks_json)} tracks") logger.debug(f"Request parsed at {time.strftime('%H:%M:%S')} (took {(time.time()-request_start_time)*1000:.1f}ms)") with sync_lock: if playlist_id in active_sync_workers and not active_sync_workers[playlist_id].done(): return jsonify({"success": False, "error": "Sync is already in progress for this playlist."}), 409 # Initial state sync_states[playlist_id] = {"status": "starting", "progress": {}} # Submit the task to the thread pool (capture profile_id while still in request context) _sync_profile_id = get_current_profile_id() thread_submit_time = time.time() future = sync_executor.submit(_run_sync_task, playlist_id, playlist_name, tracks_json, None, _sync_profile_id, playlist_image_url) active_sync_workers[playlist_id] = future thread_submit_duration = (time.time() - thread_submit_time) * 1000 logger.info(f"⏱️ [TIMING] Thread submitted at {time.strftime('%H:%M:%S')} (took {thread_submit_duration:.1f}ms)") total_request_time = (time.time() - request_start_time) * 1000 logger.info(f"⏱️ [TIMING] Request completed at {time.strftime('%H:%M:%S')} (total: {total_request_time:.1f}ms)") return jsonify({"success": True, "message": "Sync started."}) @app.route('/api/sync/status/', methods=['GET']) def get_sync_status(playlist_id): """Polls for the status of an ongoing sync.""" with sync_lock: state = sync_states.get(playlist_id) if not state: return jsonify({"status": "not_found"}), 404 # If the task is finished but the state hasn't been updated, check the future if state['status'] not in ['finished', 'error'] and playlist_id in active_sync_workers: if active_sync_workers[playlist_id].done(): # The task might have finished between polls, trigger final state update # This is handled by the _run_sync_task itself pass return jsonify(state) @app.route('/api/sync/cancel', methods=['POST']) def cancel_playlist_sync(): """Cancels an ongoing sync process.""" data = request.get_json() playlist_id = data.get('playlist_id') if not playlist_id: return jsonify({"success": False, "error": "Missing playlist_id."}), 400 with sync_lock: future = active_sync_workers.get(playlist_id) if not future or future.done(): return jsonify({"success": False, "error": "Sync not running or already complete."}), 404 # The GUI's sync_service has a cancel_sync method. We'll replicate that idea. # Since we can't easily stop the thread, we'll set a flag. # The elegant solution is to have the sync_service check for a cancellation flag. # Your `sync_service.py` already has this logic with `self._cancelled`. sync_service.cancel_sync() # We can't guarantee immediate stop, but we can update the state sync_states[playlist_id] = {"status": "cancelled"} # It's best practice to let the task finish and clean itself up. # We don't use future.cancel() as it may not work if the task is already running. return jsonify({"success": True, "message": "Sync cancellation requested."}) @app.route('/api/sync/test-database', methods=['GET']) def test_database_access(): """Test endpoint to verify database connectivity for sync operations""" try: logger.debug("Testing database access for sync operations...") # Test database initialization from database.music_database import MusicDatabase db = MusicDatabase() logger.debug(f" Database initialized: {db is not None}") # Test basic database query stats = db.get_database_info_for_server() logger.debug(f" Database stats retrieved: {stats}") # Test track existence check (like sync service does) db_track, confidence = db.check_track_exists("test track", "test artist", confidence_threshold=0.7) logger.info(f" Track existence check works: found={db_track is not None}, confidence={confidence}") # Test config manager from config.settings import config_manager active_server = config_manager.get_active_media_server() logger.info(f" Active media server: {active_server}") # Test media clients logger.info(" Media clients status:") logger.info(f" plex_client: {plex_client is not None}") if plex_client: logger.info(f" plex_client.is_connected(): {plex_client.is_connected()}") logger.info(f" jellyfin_client: {jellyfin_client is not None}") if jellyfin_client: logger.info(f" jellyfin_client.is_connected(): {jellyfin_client.is_connected()}") return jsonify({ "success": True, "message": "Database access test successful", "details": { "database_initialized": db is not None, "database_stats": stats, "active_server": active_server, "plex_connected": plex_client.is_connected() if plex_client else False, "jellyfin_connected": jellyfin_client.is_connected() if jellyfin_client else False, } }) except Exception as e: logger.error(f" Database test failed: {e}") import traceback traceback.print_exc() return jsonify({ "success": False, "error": str(e), "message": "Database access test failed" }), 500 # --- Discover Download Snapshot System --- @app.route('/api/discover_downloads/snapshot', methods=['POST']) def save_discover_download_snapshot(): """ Saves a snapshot of current discover download state for persistence across page refreshes. """ try: from datetime import datetime data = request.json if not data or 'downloads' not in data: return jsonify({'success': False, 'error': 'No download data provided'}), 400 downloads = data['downloads'] db = get_database() db.save_bubble_snapshot('discover_downloads', downloads, profile_id=get_current_profile_id()) download_count = len(downloads) logger.info(f"Saved discover download snapshot: {download_count} downloads") return jsonify({ 'success': True, 'message': f'Snapshot saved with {download_count} downloads', 'timestamp': datetime.now().isoformat() }) except Exception as e: logger.error(f"Error saving discover download snapshot: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 @app.route('/api/discover_downloads/hydrate', methods=['GET']) def hydrate_discover_downloads(): """ Loads discover downloads with live status by cross-referencing snapshots with active processes. """ try: from datetime import datetime, timedelta db = get_database() snapshot = db.get_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id()) # Load snapshot if it exists if not snapshot: return jsonify({ 'success': True, 'downloads': {}, 'message': 'No snapshots found' }) saved_downloads = snapshot['data'] snapshot_time = snapshot['timestamp'] # Clean up old snapshots (older than 48 hours) try: if snapshot_time: snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00')) cutoff = datetime.now() - timedelta(hours=48) if snapshot_dt < cutoff: logger.info(f"Cleaning up old discover download snapshot from {snapshot_time}") db.delete_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'downloads': {}, 'message': 'Old snapshot cleaned up' }) except ValueError as e: logger.error(f"Error checking discover snapshot age: {e}") # Get current active download processes for live status current_processes = {} try: with tasks_lock: for batch_id, batch_data in download_batches.items(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled']: playlist_id = batch_data.get('playlist_id') if playlist_id: current_processes[playlist_id] = { 'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing', 'batch_id': batch_id, 'phase': batch_data.get('phase') } except Exception as e: logger.error(f"Error fetching active processes for discover download hydration: {e}") # If no active processes exist, the app likely restarted - clean up snapshots if not current_processes: logger.warning("No active processes found - app likely restarted, cleaning up discover download snapshot") db.delete_bubble_snapshot('discover_downloads', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'downloads': {}, 'message': 'No active processes - returning empty downloads' }) # Update download statuses with live data hydrated_downloads = {} for playlist_id, download_data in saved_downloads.items(): # Determine current live status if playlist_id in current_processes: process_info = current_processes[playlist_id] live_status = 'in_progress' logger.info(f"Found active process for discover download {playlist_id}: {process_info['phase']}") else: # No active process - likely completed live_status = 'completed' logger.warning(f"No active process for discover download {playlist_id} - marking as completed") # Create updated download entry hydrated_downloads[playlist_id] = { 'name': download_data.get('name'), 'type': download_data.get('type'), 'status': live_status, 'virtualPlaylistId': playlist_id, 'imageUrl': download_data.get('imageUrl'), 'startTime': download_data.get('startTime', datetime.now().isoformat()) } download_count = len(hydrated_downloads) active_count = sum(1 for d in hydrated_downloads.values() if d['status'] == 'in_progress') completed_count = sum(1 for d in hydrated_downloads.values() if d['status'] == 'completed') logger.info(f"Hydrated {download_count} discover downloads: {active_count} active, {completed_count} completed") return jsonify({ 'success': True, 'downloads': hydrated_downloads, 'stats': { 'total_downloads': download_count, 'active_downloads': active_count, 'completed_downloads': completed_count } }) except Exception as e: logger.error(f"Error hydrating discover downloads: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 # --- Artist Bubble Snapshot System --- @app.route('/api/artist_bubbles/snapshot', methods=['POST']) def save_artist_bubble_snapshot(): """ Saves a snapshot of current artist bubble state for persistence across page refreshes. """ try: from datetime import datetime data = request.json if not data or 'bubbles' not in data: return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 bubbles = data['bubbles'] db = get_database() db.save_bubble_snapshot('artist_bubbles', bubbles, profile_id=get_current_profile_id()) bubble_count = len(bubbles) logger.info(f"Saved artist bubble snapshot: {bubble_count} artists") return jsonify({ 'success': True, 'message': f'Snapshot saved with {bubble_count} artist bubbles', 'timestamp': datetime.now().isoformat() }) except Exception as e: logger.error(f"Error saving artist bubble snapshot: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 @app.route('/api/artist_bubbles/hydrate', methods=['GET']) def hydrate_artist_bubbles(): """ Loads artist bubbles with live status by cross-referencing snapshots with active processes. """ try: from datetime import datetime, timedelta db = get_database() snapshot = db.get_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id()) # Load snapshot if it exists if not snapshot: return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No snapshots found' }) saved_bubbles = snapshot['data'] snapshot_time = snapshot['timestamp'] # Clean up old snapshots (older than 48 hours) try: if snapshot_time: snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00')) cutoff = datetime.now() - timedelta(hours=48) if snapshot_dt < cutoff: logger.info(f"Cleaning up old snapshot from {snapshot_time}") db.delete_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'Old snapshot cleaned up' }) except ValueError as e: logger.error(f"Error checking snapshot age: {e}") # Get current active download processes for live status current_processes = {} try: with tasks_lock: for batch_id, batch_data in download_batches.items(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled']: playlist_id = batch_data.get('playlist_id') if playlist_id: current_processes[playlist_id] = { 'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing', 'batch_id': batch_id, 'phase': batch_data.get('phase') } except Exception as e: logger.error(f"Error fetching active processes for hydration: {e}") # If no active processes exist, the app likely restarted - clean up snapshots if not current_processes: logger.warning("No active processes found - app likely restarted, cleaning up snapshot") db.delete_bubble_snapshot('artist_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No active processes - returning empty bubbles' }) # Update bubble statuses with live data hydrated_bubbles = {} for artist_id, bubble_data in saved_bubbles.items(): hydrated_bubble = { 'artist': bubble_data['artist'], 'downloads': [], 'hasCompletedDownloads': False } for download in bubble_data.get('downloads', []): virtual_playlist_id = download['virtualPlaylistId'] # Determine current live status if virtual_playlist_id in current_processes: process_info = current_processes[virtual_playlist_id] live_status = 'in_progress' logger.info(f"Found active process for {download['album']['name']}: {process_info['phase']}") else: # No active process - likely completed live_status = 'view_results' logger.warning(f"No active process for {download['album']['name']} - marking as completed") # Create updated download entry updated_download = { 'virtualPlaylistId': virtual_playlist_id, 'album': download['album'], 'albumType': download.get('albumType', 'album'), 'status': live_status, 'startTime': download.get('startTime', datetime.now().isoformat()) } hydrated_bubble['downloads'].append(updated_download) # Update hasCompletedDownloads flag if live_status == 'view_results': hydrated_bubble['hasCompletedDownloads'] = True # Only include artists that still have downloads if hydrated_bubble['downloads']: hydrated_bubbles[artist_id] = hydrated_bubble bubble_count = len(hydrated_bubbles) active_count = sum(1 for bubble in hydrated_bubbles.values() for download in bubble['downloads'] if download['status'] == 'in_progress') completed_count = sum(1 for bubble in hydrated_bubbles.values() for download in bubble['downloads'] if download['status'] == 'view_results') logger.info(f"Hydrated {bubble_count} artist bubbles: {active_count} active, {completed_count} completed") return jsonify({ 'success': True, 'bubbles': hydrated_bubbles, 'stats': { 'total_artists': bubble_count, 'active_downloads': active_count, 'completed_downloads': completed_count, 'snapshot_time': snapshot_time } }) except Exception as e: logger.error(f"Error hydrating artist bubbles: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 # --- Search Bubble Snapshot System --- @app.route('/api/search_bubbles/snapshot', methods=['POST']) def save_search_bubble_snapshot(): """ Saves a snapshot of current search bubble state for persistence across page refreshes. """ try: from datetime import datetime data = request.json if not data or 'bubbles' not in data: return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 bubbles = data['bubbles'] db = get_database() db.save_bubble_snapshot('search_bubbles', bubbles, profile_id=get_current_profile_id()) bubble_count = len(bubbles) logger.info(f"Saved search bubble snapshot: {bubble_count} albums/tracks") return jsonify({ 'success': True, 'message': f'Snapshot saved with {bubble_count} search bubbles', 'timestamp': datetime.now().isoformat() }) except Exception as e: logger.error(f"Error saving search bubble snapshot: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 @app.route('/api/search_bubbles/hydrate', methods=['GET']) def hydrate_search_bubbles(): """ Loads search bubbles with live status by cross-referencing snapshots with active processes. """ try: from datetime import datetime, timedelta db = get_database() snapshot = db.get_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id()) # Load snapshot if it exists if not snapshot: return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No snapshots found' }) saved_bubbles = snapshot['data'] snapshot_time = snapshot['timestamp'] # Clean up old snapshots (older than 48 hours) try: if snapshot_time: snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00')) cutoff = datetime.now() - timedelta(hours=48) if snapshot_dt < cutoff: logger.info(f"Cleaning up old search snapshot from {snapshot_time}") db.delete_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'Old snapshot cleaned up' }) except ValueError as e: logger.error(f"Error checking snapshot age: {e}") # Get current active download processes for live status current_processes = {} try: with tasks_lock: for batch_id, batch_data in download_batches.items(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled']: playlist_id = batch_data.get('playlist_id') if playlist_id: current_processes[playlist_id] = { 'status': 'in_progress' if batch_data.get('phase') == 'downloading' else 'analyzing', 'batch_id': batch_id, 'phase': batch_data.get('phase') } except Exception as e: logger.error(f"Error fetching active processes for hydration: {e}") # If no active processes exist, the app likely restarted - clean up snapshots if not current_processes: logger.warning("No active processes found - app likely restarted, cleaning up search snapshot") db.delete_bubble_snapshot('search_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No active processes - returning empty bubbles' }) # Update bubble statuses with live data (artist-grouped structure) hydrated_bubbles = {} for artist_name, bubble_data in saved_bubbles.items(): hydrated_bubble = { 'artist': bubble_data['artist'], 'downloads': [] } for download in bubble_data.get('downloads', []): virtual_playlist_id = download['virtualPlaylistId'] # Determine current live status if virtual_playlist_id in current_processes: process_info = current_processes[virtual_playlist_id] live_status = 'in_progress' logger.info(f"Found active process for {download['item']['name']}: {process_info['phase']}") else: # No active process - likely completed live_status = 'view_results' logger.warning(f"No active process for {download['item']['name']} - marking as completed") # Create updated download entry updated_download = { 'virtualPlaylistId': virtual_playlist_id, 'item': download['item'], 'type': download.get('type', 'album'), 'status': live_status, 'startTime': download.get('startTime', datetime.now().isoformat()) } hydrated_bubble['downloads'].append(updated_download) # Only include artists that still have downloads if hydrated_bubble['downloads']: hydrated_bubbles[artist_name] = hydrated_bubble bubble_count = len(hydrated_bubbles) active_count = sum(1 for bubble in hydrated_bubbles.values() for download in bubble['downloads'] if download['status'] == 'in_progress') completed_count = sum(1 for bubble in hydrated_bubbles.values() for download in bubble['downloads'] if download['status'] == 'view_results') logger.info(f"Hydrated {bubble_count} search bubbles (artists): {active_count} active, {completed_count} completed") return jsonify({ 'success': True, 'bubbles': hydrated_bubbles, 'stats': { 'total_items': bubble_count, 'active_downloads': active_count, 'completed_downloads': completed_count, 'snapshot_time': snapshot_time } }) except Exception as e: logger.error(f"Error hydrating search bubbles: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 @app.route('/api/beatport_bubbles/snapshot', methods=['POST']) def save_beatport_bubble_snapshot(): """Saves a snapshot of current Beatport download bubble state for persistence.""" try: from datetime import datetime data = request.json if not data or 'bubbles' not in data: return jsonify({'success': False, 'error': 'No bubble data provided'}), 400 bubbles = data['bubbles'] db = get_database() db.save_bubble_snapshot('beatport_bubbles', bubbles, profile_id=get_current_profile_id()) bubble_count = len(bubbles) logger.info(f"Saved Beatport bubble snapshot: {bubble_count} charts") return jsonify({ 'success': True, 'message': f'Snapshot saved with {bubble_count} Beatport bubbles', 'timestamp': datetime.now().isoformat() }) except Exception as e: logger.error(f"Error saving Beatport bubble snapshot: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 @app.route('/api/beatport_bubbles/hydrate', methods=['GET']) def hydrate_beatport_bubbles(): """Loads Beatport download bubbles with live status from snapshot.""" try: from datetime import datetime, timedelta db = get_database() snapshot = db.get_bubble_snapshot('beatport_bubbles', profile_id=get_current_profile_id()) if not snapshot: return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No snapshots found' }) saved_bubbles = snapshot['data'] snapshot_time = snapshot['timestamp'] # Clean up old snapshots (older than 48 hours) try: if snapshot_time: snapshot_dt = datetime.fromisoformat(snapshot_time.replace('Z', '+00:00')) cutoff = datetime.now() - timedelta(hours=48) if snapshot_dt < cutoff: logger.info(f"Cleaning up old Beatport snapshot from {snapshot_time}") db.delete_bubble_snapshot('beatport_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'Old snapshot cleaned up' }) except ValueError as e: logger.error(f"Error checking Beatport snapshot age: {e}") # Get current active download processes for live status current_processes = {} try: with tasks_lock: for batch_id, batch_data in download_batches.items(): if batch_data.get('phase') not in ['complete', 'error', 'cancelled']: playlist_id = batch_data.get('playlist_id') if playlist_id: current_processes[playlist_id] = { 'status': 'in_progress', 'batch_id': batch_id, 'phase': batch_data.get('phase') } except Exception as e: logger.error(f"Error fetching active processes for Beatport hydration: {e}") # If no active processes exist, app likely restarted — clean up if not current_processes: logger.warning("No active processes found - cleaning up Beatport snapshot") db.delete_bubble_snapshot('beatport_bubbles', profile_id=get_current_profile_id()) return jsonify({ 'success': True, 'bubbles': {}, 'message': 'No active processes - returning empty bubbles' }) # Update bubble statuses with live data hydrated_bubbles = {} for chart_key, bubble_data in saved_bubbles.items(): hydrated_bubble = { 'chart': bubble_data['chart'], 'downloads': [] } for download in bubble_data.get('downloads', []): virtual_playlist_id = download['virtualPlaylistId'] if virtual_playlist_id in current_processes: live_status = 'in_progress' else: live_status = 'view_results' hydrated_bubble['downloads'].append({ 'virtualPlaylistId': virtual_playlist_id, 'status': live_status, 'startTime': download.get('startTime', datetime.now().isoformat()) }) if hydrated_bubble['downloads']: hydrated_bubbles[chart_key] = hydrated_bubble bubble_count = len(hydrated_bubbles) active_count = sum(1 for b in hydrated_bubbles.values() for d in b['downloads'] if d['status'] == 'in_progress') completed_count = sum(1 for b in hydrated_bubbles.values() for d in b['downloads'] if d['status'] == 'view_results') logger.info(f"Hydrated {bubble_count} Beatport bubbles: {active_count} active, {completed_count} completed") return jsonify({ 'success': True, 'bubbles': hydrated_bubbles, 'stats': { 'total_charts': bubble_count, 'active_downloads': active_count, 'completed_downloads': completed_count } }) except Exception as e: logger.error(f"Error hydrating Beatport bubbles: {e}") import traceback traceback.print_exc() return jsonify({ 'success': False, 'error': str(e) }), 500 # --- Profile API Endpoints --- @app.route('/api/profiles', methods=['GET']) def list_profiles(): """List all profiles""" try: database = get_database() profiles = database.get_all_profiles() return jsonify({'success': True, 'profiles': profiles}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles', methods=['POST']) def create_profile(): """Create a new profile (admin only)""" try: # Check that requester is admin database = get_database() current = database.get_profile(get_current_profile_id()) if current and not current['is_admin']: return jsonify({'success': False, 'error': 'Admin only'}), 403 data = request.json or {} name = data.get('name', '').strip() if not name: return jsonify({'success': False, 'error': 'Name is required'}), 400 avatar_color = data.get('avatar_color', '#6366f1') avatar_url = data.get('avatar_url') or None pin = data.get('pin') pin_hash = None if pin: from werkzeug.security import generate_password_hash pin_hash = generate_password_hash(pin, method='pbkdf2:sha256') # Profile settings: home_page, allowed_pages, can_download home_page = data.get('home_page') or None allowed_pages = data.get('allowed_pages') # list or None can_download = data.get('can_download', True) # Validate page IDs if home_page and home_page not in VALID_PAGE_IDS: home_page = None if allowed_pages is not None: allowed_pages = [p for p in allowed_pages if p in VALID_PAGE_IDS] # Non-admin should never have 'settings' in allowed_pages if 'settings' in allowed_pages: allowed_pages.remove('settings') # If home_page not in allowed list, reset to first allowed or 'discover' if home_page and home_page not in allowed_pages: home_page = allowed_pages[0] if allowed_pages else None profile_id = database.create_profile( name, avatar_color, pin_hash, is_admin=False, avatar_url=avatar_url, home_page=home_page, allowed_pages=allowed_pages, can_download=bool(can_download) ) if profile_id is None: return jsonify({'success': False, 'error': 'Profile name already exists'}), 409 return jsonify({'success': True, 'profile_id': profile_id}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/', methods=['PUT']) def update_profile(profile_id): """Update a profile (admin or self)""" try: database = get_database() current_pid = get_current_profile_id() current = database.get_profile(current_pid) if not current: return jsonify({'success': False, 'error': 'Current profile not found'}), 404 # Only admin or self can update if not current['is_admin'] and current_pid != profile_id: return jsonify({'success': False, 'error': 'Unauthorized'}), 403 data = request.json or {} kwargs = {} if 'name' in data: name = data['name'].strip() if not name: return jsonify({'success': False, 'error': 'Name cannot be empty'}), 400 kwargs['name'] = name if 'avatar_color' in data: kwargs['avatar_color'] = data['avatar_color'] if 'avatar_url' in data: kwargs['avatar_url'] = data['avatar_url'] or None if 'is_admin' in data and current['is_admin']: # Prevent demoting the last admin if not data['is_admin']: all_profiles = database.get_all_profiles() admin_count = sum(1 for p in all_profiles if p['is_admin']) target = database.get_profile(profile_id) if target and target['is_admin'] and admin_count <= 1: return jsonify({'success': False, 'error': 'Cannot remove the last admin'}), 400 kwargs['is_admin'] = int(data['is_admin']) # Home page — any user can change their own, admin can change anyone's if 'home_page' in data: hp = data['home_page'] or None if hp and hp not in VALID_PAGE_IDS: hp = None # Non-admin self-edit: validate home_page is in their allowed pages if not current['is_admin'] and current_pid == profile_id: target = database.get_profile(profile_id) ap = target.get('allowed_pages') if target else None if ap is not None and hp and hp not in ap: return jsonify({'success': False, 'error': 'Page not permitted'}), 400 kwargs['home_page'] = hp # Allowed pages & can_download — admin only if current['is_admin']: if 'allowed_pages' in data: ap = data['allowed_pages'] if ap is not None: ap = [p for p in ap if p in VALID_PAGE_IDS] # Non-admin target should never have 'settings' target = database.get_profile(profile_id) if target and not target.get('is_admin'): ap = [p for p in ap if p != 'settings'] # If current home_page not in new allowed list, reset it current_hp = kwargs.get('home_page') or (target.get('home_page') if target else None) if current_hp and current_hp not in ap: kwargs['home_page'] = ap[0] if ap else None kwargs['allowed_pages'] = ap if 'can_download' in data: kwargs['can_download'] = int(bool(data['can_download'])) success = database.update_profile(profile_id, **kwargs) return jsonify({'success': success}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/', methods=['DELETE']) def delete_profile(profile_id): """Delete a profile (admin only, can't delete self)""" try: database = get_database() current_pid = get_current_profile_id() current = database.get_profile(current_pid) if not current or not current['is_admin']: return jsonify({'success': False, 'error': 'Admin only'}), 403 if current_pid == profile_id: return jsonify({'success': False, 'error': 'Cannot delete your own profile'}), 400 target = database.get_profile(profile_id) if not target: return jsonify({'success': False, 'error': 'Profile not found'}), 404 success = database.delete_profile(profile_id) return jsonify({'success': success}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/select', methods=['POST']) def select_profile(): """Select a profile (validates PIN if set)""" try: data = request.json or {} try: profile_id = int(data.get('profile_id', 0)) except (TypeError, ValueError): return jsonify({'success': False, 'error': 'Invalid profile_id'}), 400 pin = data.get('pin', '') if not profile_id: return jsonify({'success': False, 'error': 'profile_id required'}), 400 database = get_database() profile = database.get_profile(profile_id) if not profile: return jsonify({'success': False, 'error': 'Profile not found'}), 404 # Only enforce PIN when multiple profiles exist (PIN protects against profile switching) all_profiles = database.get_all_profiles() if len(all_profiles) > 1 and profile['has_pin']: if not pin: return jsonify({'success': False, 'error': 'PIN required', 'pin_required': True}), 401 if not database.verify_profile_pin(profile_id, pin): return jsonify({'success': False, 'error': 'Invalid PIN'}), 401 session['profile_id'] = profile_id # If PIN was just validated, also mark launch PIN as verified # so the subsequent page reload doesn't ask again if pin: session['launch_pin_verified'] = True return jsonify({'success': True, 'profile': profile}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/current', methods=['GET']) def get_current_profile(): """Get the currently selected profile from session""" try: pid = session.get('profile_id') if not pid: return jsonify({'success': False, 'error': 'No profile selected'}), 200 database = get_database() profile = database.get_profile(pid) if not profile: session.pop('profile_id', None) return jsonify({'success': False, 'error': 'Profile not found'}), 200 # Check if launch PIN is required require_pin = config_manager.get('security.require_pin_on_launch', False) if config_manager else False # Check if PIN was verified this page load, then consume the flag pin_verified = session.pop('launch_pin_verified', False) return jsonify({ 'success': True, 'profile': profile, 'launch_pin_required': bool(require_pin) and not pin_verified, }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/verify-launch-pin', methods=['POST']) def verify_launch_pin(): """Verify PIN for launch lock screen""" try: data = request.json or {} pin = data.get('pin', '') if not pin: return jsonify({'success': False, 'error': 'PIN required'}), 401 database = get_database() # Validate against admin profile (ID 1) if not database.verify_profile_pin(1, pin): return jsonify({'success': False, 'error': 'Invalid PIN'}), 401 session['launch_pin_verified'] = True return jsonify({'success': True}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/reset-pin-via-credential', methods=['POST']) def reset_pin_via_credential(): """Reset admin PIN by verifying a known API credential""" try: data = request.json or {} credential = (data.get('credential') or '').strip() if not credential or len(credential) < 4: return jsonify({'success': False, 'error': 'Enter a valid credential'}), 400 # Check credential against all stored API secrets/tokens checks = [ ('Spotify Client Secret', config_manager.get('spotify.client_secret', '')), ('Tidal Client Secret', config_manager.get('tidal.client_secret', '')), ('Plex Token', config_manager.get('plex.token', '')), ('Jellyfin API Key', config_manager.get('jellyfin.api_key', '')), ('Navidrome Password', config_manager.get('navidrome.password', '')), ('ListenBrainz Token', config_manager.get('listenbrainz.token', '')), ('AcoustID API Key', config_manager.get('acoustid.api_key', '')), ('Last.fm API Secret', config_manager.get('lastfm.api_secret', '')), ('Genius Access Token', config_manager.get('genius.access_token', '')), ] matched = False for _name, stored in checks: if stored and credential == stored: matched = True break if not matched: return jsonify({'success': False, 'error': 'Credential does not match any configured service'}), 401 # Credential verified — clear PIN for the requested profile (default: admin) database = get_database() target_profile = data.get('profile_id', 1) database.update_profile(target_profile, pin_hash=None) # If clearing admin PIN, also disable launch lock if target_profile == 1: config_manager.set('security.require_pin_on_launch', False) session['launch_pin_verified'] = True return jsonify({'success': True, 'message': 'PIN cleared. You can set a new PIN in Settings.'}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/logout', methods=['POST']) def logout_profile(): """Clear session — back to profile picker""" session.pop('profile_id', None) return jsonify({'success': True}) @app.route('/api/profiles//set-pin', methods=['POST']) def set_profile_pin(profile_id): """Set or change PIN for a profile (admin or self)""" try: database = get_database() current_pid = get_current_profile_id() current = database.get_profile(current_pid) if not current or (not current['is_admin'] and current_pid != profile_id): return jsonify({'success': False, 'error': 'Unauthorized'}), 403 data = request.json or {} pin = data.get('pin', '') if pin: from werkzeug.security import generate_password_hash pin_hash = generate_password_hash(pin, method='pbkdf2:sha256') else: pin_hash = None # Remove PIN success = database.update_profile(profile_id, pin_hash=pin_hash) return jsonify({'success': success}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 # --- Per-Profile ListenBrainz Settings --- def _get_lb_credentials_for_profile(profile_id=None): """Get LB token + base_url for profile, falling back to global config.""" if profile_id is None: profile_id = get_current_profile_id() db = get_database() settings = db.get_profile_listenbrainz(profile_id) if settings and settings.get('token'): return settings['token'], settings.get('base_url', ''), settings.get('username', ''), 'profile' # Fallback to global config return (config_manager.get('listenbrainz.token', ''), config_manager.get('listenbrainz.base_url', ''), None, 'global') def _validate_lb_token(token, base_url=''): """Validate a ListenBrainz token and return (success, username_or_error)""" try: custom_base = (base_url or '').rstrip('/') if custom_base: if not custom_base.endswith('/1'): custom_base += '/1' lb_api_base = custom_base else: lb_api_base = "https://api.listenbrainz.org/1" url = f"{lb_api_base}/validate-token" headers = {'Authorization': f'Token {token}'} response = requests.get(url, headers=headers, timeout=5) if response.status_code == 200: data = response.json() if data.get('valid'): return True, data.get('user_name', 'Unknown') return False, "Invalid ListenBrainz token." elif response.status_code == 401: return False, "Invalid ListenBrainz token (unauthorized)." else: return False, f"Could not connect to ListenBrainz (HTTP {response.status_code})" except Exception as e: return False, f"ListenBrainz connection error: {str(e)}" @app.route('/api/profiles/me/listenbrainz', methods=['GET']) def get_profile_listenbrainz(): """Get current profile's ListenBrainz connection status""" try: profile_id = get_current_profile_id() token, base_url, username, source = _get_lb_credentials_for_profile(profile_id) connected = bool(token) return jsonify({ 'success': True, 'connected': connected, 'username': username if connected else None, 'base_url': base_url or '', 'source': source }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/listenbrainz', methods=['POST']) def save_profile_listenbrainz(): """Save ListenBrainz credentials for current profile""" try: data = request.json or {} token = data.get('token', '').strip() base_url = data.get('base_url', '').strip() if not token: return jsonify({'success': False, 'error': 'Token is required'}), 400 # Validate token first valid, result = _validate_lb_token(token, base_url) if not valid: return jsonify({'success': False, 'error': result}), 400 username = result profile_id = get_current_profile_id() db = get_database() success = db.set_profile_listenbrainz(profile_id, token, base_url, username) if success: return jsonify({'success': True, 'username': username}) return jsonify({'success': False, 'error': 'Failed to save credentials'}), 500 except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/listenbrainz', methods=['DELETE']) def delete_profile_listenbrainz(): """Clear ListenBrainz credentials for current profile""" try: profile_id = get_current_profile_id() db = get_database() db.clear_profile_listenbrainz(profile_id) return jsonify({'success': True}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/listenbrainz/test', methods=['POST']) def test_profile_listenbrainz(): """Test a ListenBrainz token without saving""" try: data = request.json or {} token = data.get('token', '').strip() base_url = data.get('base_url', '').strip() if not token: return jsonify({'success': False, 'error': 'Token is required'}), 400 valid, result = _validate_lb_token(token, base_url) if valid: return jsonify({'success': True, 'username': result}) return jsonify({'success': False, 'error': result}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 # --- Per-Profile Service Credentials API --- @app.route('/api/profiles/me/spotify', methods=['GET']) def get_profile_spotify_creds(): """Get current profile's Spotify credentials (if set)""" try: profile_id = get_current_profile_id() db = get_database() creds = db.get_profile_spotify(profile_id) return jsonify({ 'success': True, 'has_credentials': bool(creds), 'client_id': creds.get('client_id', '') if creds else '', 'redirect_uri': creds.get('redirect_uri', '') if creds else '', # Never return client_secret or tokens to frontend }) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/spotify', methods=['POST']) def save_profile_spotify_creds(): """Save Spotify API credentials for current profile""" try: data = request.json or {} client_id = data.get('client_id', '').strip() client_secret = data.get('client_secret', '').strip() redirect_uri = data.get('redirect_uri', '').strip() if not client_id or not client_secret: return jsonify({'success': False, 'error': 'Client ID and Secret are required'}), 400 profile_id = get_current_profile_id() db = get_database() success = db.set_profile_spotify(profile_id, client_id, client_secret, redirect_uri) if success: return jsonify({'success': True}) return jsonify({'success': False, 'error': 'Failed to save credentials'}), 500 except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/spotify', methods=['DELETE']) def delete_profile_spotify_creds(): """Clear Spotify credentials for current profile (revert to global)""" try: profile_id = get_current_profile_id() db = get_database() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" UPDATE profiles SET spotify_client_id = NULL, spotify_client_secret = NULL, spotify_redirect_uri = NULL, spotify_access_token = NULL, spotify_refresh_token = NULL, updated_at = CURRENT_TIMESTAMP WHERE id = ? """, (profile_id,)) conn.commit() return jsonify({'success': True}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/server-library', methods=['GET']) def get_profile_server_library(): """Get current profile's media server library selection""" try: profile_id = get_current_profile_id() db = get_database() libs = db.get_profile_server_library(profile_id) return jsonify({'success': True, **libs}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/profiles/me/server-library', methods=['POST']) def save_profile_server_library(): """Save media server library/user selection for current profile""" try: data = request.json or {} server_type = data.get('server_type', '') library_id = data.get('library_id') user_id = data.get('user_id') if server_type not in ('plex', 'jellyfin', 'navidrome'): return jsonify({'success': False, 'error': 'Invalid server type'}), 400 profile_id = get_current_profile_id() db = get_database() success = db.set_profile_server_library(profile_id, server_type, library_id, user_id) if success: return jsonify({'success': True}) return jsonify({'success': False, 'error': 'Failed to save library selection'}), 500 except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 # --- Watchlist API Endpoints --- @app.route('/api/watchlist/count', methods=['GET']) def get_watchlist_count(): """Get the number of artists in the watchlist""" try: database = get_database() count = database.get_watchlist_count(profile_id=get_current_profile_id()) # Calculate time until next auto-scanning next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('scan_watchlist') if automation_engine else 0 return jsonify({ "success": True, "count": count, "next_run_in_seconds": next_run_in_seconds }) except Exception as e: logger.error(f"Error getting watchlist count: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/artists', methods=['GET']) def get_watchlist_artists(): """Get all artists in the watchlist with cached images""" try: database = get_database() watchlist_artists = database.get_watchlist_artists(profile_id=get_current_profile_id()) # Convert to JSON serializable format (images are cached from watchlist scans) artists_data = [] for artist in watchlist_artists: artists_data.append({ "id": artist.id, "spotify_artist_id": artist.spotify_artist_id, "artist_name": artist.artist_name, "date_added": artist.date_added.isoformat() if artist.date_added else None, "last_scan_timestamp": artist.last_scan_timestamp.isoformat() if artist.last_scan_timestamp else None, "created_at": artist.created_at.isoformat() if artist.created_at else None, "updated_at": artist.updated_at.isoformat() if artist.updated_at else None, "image_url": artist.image_url, # Cached during watchlist scans "itunes_artist_id": artist.itunes_artist_id, # For iTunes-only artists "deezer_artist_id": getattr(artist, 'deezer_artist_id', None), "discogs_artist_id": getattr(artist, 'discogs_artist_id', None), "include_albums": artist.include_albums, "include_eps": artist.include_eps, "include_singles": artist.include_singles, "include_live": artist.include_live, "include_remixes": artist.include_remixes, "include_acoustic": artist.include_acoustic, "include_compilations": artist.include_compilations, }) return jsonify({"success": True, "artists": artists_data}) except Exception as e: logger.error(f"Error getting watchlist artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/add', methods=['POST']) def add_to_watchlist(): """Add an artist to the watchlist""" try: data = request.get_json() artist_id = data.get('artist_id') artist_name = data.get('artist_name') if not artist_id or not artist_name: return jsonify({"success": False, "error": "Missing artist_id or artist_name"}), 400 database = get_database() # Detect source from ID — check if it's a library DB ID first is_numeric_id = artist_id.isdigit() source = None if is_numeric_id: # Could be a library DB ID, iTunes ID, Deezer ID, or Discogs ID # Check if this is a library DB artist and use their actual source IDs try: conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT spotify_artist_id, itunes_artist_id, deezer_id, discogs_id FROM artists WHERE id = ? LIMIT 1 """, (artist_id,)) row = cursor.fetchone() conn.close() if row: # Library artist — use the best available source ID fallback = _get_metadata_fallback_source() if fallback == 'discogs' and row['discogs_id']: artist_id = row['discogs_id'] source = 'discogs' elif fallback == 'deezer' and row['deezer_id']: artist_id = row['deezer_id'] source = 'deezer' elif row['spotify_artist_id']: artist_id = row['spotify_artist_id'] source = 'spotify' elif row['itunes_artist_id']: artist_id = row['itunes_artist_id'] source = 'itunes' elif row['deezer_id']: artist_id = row['deezer_id'] source = 'deezer' elif row['discogs_id']: artist_id = row['discogs_id'] source = 'discogs' except Exception: pass if not source: fallback_source = _get_metadata_fallback_source() source = fallback_source if is_numeric_id else 'spotify' success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id(), source=source) if success: # Fetch and cache artist image immediately try: if is_numeric_id: # For numeric IDs, fetch image from the configured fallback source try: if source == 'discogs': # Discogs: fetch artist image from API dc = _get_discogs_client() dc_data = dc.get_artist(artist_id) if dc_data: image_url = dc_data.get('image_url') logger.info(f"Discogs artist image: {image_url[:60] if image_url else 'None'}") elif source == 'deezer' or fallback_source == 'deezer': # Deezer: fetch artist image directly from API dz_resp = requests.get(f'https://api.deezer.com/artist/{artist_id}', timeout=5) if dz_resp.ok: dz_data = dz_resp.json() image_url = dz_data.get('picture_xl') or dz_data.get('picture_big') or dz_data.get('picture_medium') logger.info(f"Deezer artist image: {image_url[:60] if image_url else 'None'}") else: # iTunes: look up album entity for artwork itunes_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=5" logger.info(f"Fetching iTunes artist image: {itunes_url}") resp = requests.get(itunes_url, timeout=5) image_url = None if resp.status_code == 200: resp_data = resp.json() results = resp_data.get('results', []) # Iterate results to find one with artwork for res in results: if 'artworkUrl100' in res: image_url = res['artworkUrl100'].replace('100x100', '600x600') break if image_url: database.update_watchlist_artist_image(artist_id, image_url) logger.warning(f"Cached {fallback_source} artist image for {artist_name}") else: logger.warning(f"No artwork found for {fallback_source} artist {artist_name}") except Exception as fb_error: logger.error(f"Error fetching {fallback_source} artwork: {fb_error}") elif spotify_client and spotify_client.is_authenticated(): # For Spotify artists, fetch from Spotify API artist_data = spotify_client.get_artist(artist_id) if artist_data and 'images' in artist_data and artist_data['images']: # Get medium-sized image (usually the second one, or first if only one) image_url = None if len(artist_data['images']) > 1: image_url = artist_data['images'][1]['url'] else: image_url = artist_data['images'][0]['url'] # Update in database if image_url: database.update_watchlist_artist_image(artist_id, image_url) logger.info(f"Cached artist image for {artist_name}") else: logger.warning(f"No image URL found for {artist_name}") else: logger.warning(f"No images in Spotify data for {artist_name}") else: logger.info("Spotify client not available for fetching artist image") except Exception as img_error: # Don't fail the add operation if image fetch fails logger.error(f"Could not fetch artist image for {artist_name}: {img_error}") # Push updated count to this profile's WebSocket room immediately try: pid = get_current_profile_id() socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}') except Exception: pass try: if automation_engine: automation_engine.emit('watchlist_artist_added', { 'artist': artist_name, 'artist_id': str(artist_id), }) except Exception: pass _artmap_cache_invalidate(get_current_profile_id()) return jsonify({"success": True, "message": f"Added {artist_name} to watchlist"}) else: return jsonify({"success": False, "error": "Failed to add artist to watchlist"}), 500 except Exception as e: logger.error(f"Error adding to watchlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/remove', methods=['POST']) def remove_from_watchlist(): """Remove an artist from the watchlist""" try: data = request.get_json() artist_id = data.get('artist_id') if not artist_id: return jsonify({"success": False, "error": "Missing artist_id"}), 400 database = get_database() success = database.remove_artist_from_watchlist(artist_id, profile_id=get_current_profile_id()) if success: # Push updated count to this profile's WebSocket room immediately try: pid = get_current_profile_id() socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}') except Exception: pass try: if automation_engine: automation_engine.emit('watchlist_artist_removed', { 'artist': data.get('artist_name', str(artist_id)), 'artist_id': str(artist_id), }) except Exception: pass _artmap_cache_invalidate(get_current_profile_id()) return jsonify({"success": True, "message": "Removed artist from watchlist"}) else: return jsonify({"success": False, "error": "Failed to remove artist from watchlist"}), 500 except Exception as e: logger.error(f"Error removing from watchlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/add-batch', methods=['POST']) def add_batch_to_watchlist(): """Add multiple artists to the watchlist at once""" try: data = request.get_json() artists = data.get('artists', []) if not artists or not isinstance(artists, list): return jsonify({"success": False, "error": "Missing or invalid artists list"}), 400 database = get_database() added = 0 skipped = 0 for artist in artists: artist_id = artist.get('artist_id') artist_name = artist.get('artist_name') if not artist_id or not artist_name: continue # Check if already watched (by ID or name) if database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id(), artist_name=artist_name): skipped += 1 continue is_numeric = artist_id.isdigit() fb_source = _get_metadata_fallback_source() src = fb_source if is_numeric else 'spotify' success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id(), source=src) if success: added += 1 # Cache artist image try: is_numeric_id = artist_id.isdigit() if is_numeric_id: fb_source = _get_metadata_fallback_source() if fb_source == 'deezer': fb_client = _get_metadata_fallback_client() fb_artist = fb_client.get_artist(artist_id) if fb_artist and fb_artist.get('images'): image_url = fb_artist['images'][0].get('url') if image_url: database.update_watchlist_artist_image(artist_id, image_url) else: itunes_url = f"https://itunes.apple.com/lookup?id={artist_id}&entity=album&limit=5" resp = requests.get(itunes_url, timeout=5) if resp.status_code == 200: results = resp.json().get('results', []) for res in results: if 'artworkUrl100' in res: image_url = res['artworkUrl100'].replace('100x100', '600x600') database.update_watchlist_artist_image(artist_id, image_url) break elif spotify_client and spotify_client.is_authenticated(): artist_data = spotify_client.get_artist(artist_id) if artist_data and 'images' in artist_data and artist_data['images']: image_url = artist_data['images'][1]['url'] if len(artist_data['images']) > 1 else artist_data['images'][0]['url'] if image_url: database.update_watchlist_artist_image(artist_id, image_url) except Exception as img_error: logger.error(f"Could not fetch artist image for {artist_name}: {img_error}") return jsonify({ "success": True, "added": added, "skipped": skipped, "message": f"Added {added} artist{'s' if added != 1 else ''} to watchlist ({skipped} already watched)" }) except Exception as e: logger.error(f"Error batch adding to watchlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/library/watchlist-all-unwatched', methods=['POST']) def watchlist_all_unwatched_library_artists(): """Add all unwatched library artists (that have valid external IDs) to the watchlist""" try: database = get_database() active_source = _get_active_discovery_source() # Fetch all unwatched artists in pages (SQLite variable limit safe) unwatched_artists = [] page = 1 page_size = 400 while True: result = database.get_library_artists( search_query='', letter='all', page=page, limit=page_size, watchlist_filter='unwatched', profile_id=get_current_profile_id() ) unwatched_artists.extend(result.get('artists', [])) if not result.get('pagination', {}).get('has_next', False): break page += 1 added = 0 skipped_no_id = 0 skipped_already = 0 for artist in unwatched_artists: # Use only the active source's ID — matches frontend modal filtering artist_id = None if active_source == 'spotify': artist_id = artist.get('spotify_artist_id') elif active_source == 'itunes': artist_id = artist.get('itunes_artist_id') elif active_source == 'deezer': artist_id = artist.get('deezer_id') if not artist_id: skipped_no_id += 1 continue artist_name = artist.get('name', '') if not artist_name: continue # Check if already watched (shouldn't be since we filtered, but safety check) if database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id(), artist_name=artist_name): skipped_already += 1 continue src = active_source if active_source in ('spotify', 'itunes', 'deezer') else _get_metadata_fallback_source() success = database.add_artist_to_watchlist(artist_id, artist_name, profile_id=get_current_profile_id(), source=src) if success: added += 1 # Use library thumb_url if available (no HTTP calls needed) if artist.get('image_url'): try: database.update_watchlist_artist_image(artist_id, artist['image_url']) except Exception: pass total_unwatched = len(unwatched_artists) message_parts = [f"Added {added} artist{'s' if added != 1 else ''} to watchlist"] if skipped_no_id > 0: message_parts.append(f"{skipped_no_id} skipped (no matching ID yet)") return jsonify({ "success": True, "added": added, "skipped_no_id": skipped_no_id, "skipped_already": skipped_already, "total_unwatched": total_unwatched, "message": " — ".join(message_parts) }) except Exception as e: logger.error(f"Error bulk watchlisting library artists: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/remove-batch', methods=['POST']) def remove_batch_from_watchlist(): """Remove multiple artists from the watchlist""" try: data = request.get_json() artist_ids = data.get('artist_ids', []) if not artist_ids or not isinstance(artist_ids, list): return jsonify({"success": False, "error": "Missing or invalid artist_ids"}), 400 database = get_database() removed = 0 for artist_id in artist_ids: if database.remove_artist_from_watchlist(artist_id, profile_id=get_current_profile_id()): removed += 1 return jsonify({ "success": True, "removed": removed, "message": f"Removed {removed} artist{'s' if removed != 1 else ''} from watchlist" }) except Exception as e: logger.error(f"Error batch removing from watchlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/check', methods=['POST']) def check_watchlist_status(): """Check if an artist is in the watchlist""" try: data = request.get_json() artist_id = data.get('artist_id') if not artist_id: return jsonify({"success": False, "error": "Missing artist_id"}), 400 database = get_database() is_watching = database.is_artist_in_watchlist(artist_id, profile_id=get_current_profile_id()) return jsonify({"success": True, "is_watching": is_watching}) except Exception as e: logger.error(f"Error checking watchlist status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/check-batch', methods=['POST']) def check_watchlist_status_batch(): """Check watchlist status for multiple artists in one request""" try: data = request.get_json() artist_ids = data.get('artist_ids', []) if not artist_ids: return jsonify({"success": False, "error": "Missing artist_ids"}), 400 database = get_database() pid = get_current_profile_id() results = {} for aid in artist_ids: results[aid] = database.is_artist_in_watchlist(aid, profile_id=pid) return jsonify({"success": True, "results": results}) except Exception as e: logger.error(f"Error batch checking watchlist status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/scan', methods=['POST']) def start_watchlist_scan(): """Start a watchlist scan for new releases""" try: # Check if MetadataService can provide a working client (Spotify OR fallback) from core.metadata_service import MetadataService metadata_service = MetadataService() # Get active provider - will be spotify or the configured fallback active_provider = metadata_service.get_active_provider() provider_info = metadata_service.get_provider_info() # Verify we have at least one working provider if not provider_info['spotify_authenticated'] and not provider_info['itunes_available']: fallback_name = provider_info.get('fallback_source', 'iTunes').capitalize() return jsonify({ "success": False, "error": f"No music provider available. Please authenticate Spotify or ensure {fallback_name} is accessible." }), 400 logger.info(f"Starting watchlist scan with {active_provider} provider") # Check if watchlist is already scanning if is_watchlist_actually_scanning(): return jsonify({"success": False, "error": "Watchlist scan is already in progress."}), 409 # Start the scan in a background thread scan_profile_id = get_current_profile_id() def run_scan(): _ew_state = {} try: global watchlist_scan_state, watchlist_auto_scanning, watchlist_auto_scanning_timestamp from core.watchlist_scanner import WatchlistScanner from database.music_database import get_database # Set flag and timestamp for manual scan import time with watchlist_timer_lock: watchlist_auto_scanning = True watchlist_auto_scanning_timestamp = time.time() logger.info(f"[Manual Watchlist Scan] Flag set at timestamp {watchlist_auto_scanning_timestamp}") # Get list of artists to scan (for the current profile) database = get_database() watchlist_artists = database.get_watchlist_artists(profile_id=scan_profile_id) if not watchlist_artists: watchlist_scan_state['status'] = 'completed' watchlist_scan_state['summary'] = { 'total_artists': 0, 'successful_scans': 0, 'new_tracks_found': 0, 'tracks_added_to_wishlist': 0 } # Reset flag with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 return # Initialize scanner with MetadataService for cross-provider support scanner = WatchlistScanner(metadata_service=metadata_service) # PROACTIVE ID BACKFILLING (cross-provider support) # Before scanning, ensure all artists have IDs for ALL available sources providers_to_backfill = ['itunes', 'deezer'] if spotify_client and spotify_client.is_spotify_authenticated(): providers_to_backfill.append('spotify') try: if config_manager.get('discogs.token', ''): providers_to_backfill.append('discogs') except Exception: pass for _bf_provider in providers_to_backfill: try: logger.debug(f"Checking for missing {_bf_provider} IDs in watchlist...") scanner._backfill_missing_ids(watchlist_artists, _bf_provider) except Exception as backfill_error: logger.error(f"Error during {_bf_provider} ID backfilling: {backfill_error}") # Continue with next provider try: filled = scanner.backfill_watchlist_artist_images(scan_profile_id) if filled: logger.info(f"Backfilled {filled} watchlist artist images") except Exception as img_err: logger.error(f"Image backfill error: {img_err}") # Initialize detailed progress tracking watchlist_scan_state.update({ 'total_artists': len(watchlist_artists), 'current_artist_index': 0, 'current_artist_name': '', 'current_artist_image_url': '', 'current_phase': 'starting', 'albums_to_check': 0, 'albums_checked': 0, 'current_album': '', 'current_album_image_url': '', 'current_track_name': '', 'tracks_found_this_scan': 0, 'tracks_added_this_scan': 0, 'recent_wishlist_additions': [] }) scan_results = [] # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('watchlist scan') scan_results = scanner.scan_watchlist_profile( scan_profile_id, watchlist_artists=watchlist_artists, scan_state=watchlist_scan_state, cancel_check=lambda: watchlist_scan_state.get('cancel_requested', False), ) # Store final results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) if not was_cancelled: _artmap_cache_invalidate(scan_profile_id) successful_scans = [r for r in scan_results if r.success] total_new_tracks = sum(r.new_tracks_found for r in successful_scans) total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) watchlist_scan_state['status'] = 'completed' watchlist_scan_state['results'] = scan_results watchlist_scan_state['completed_at'] = datetime.now() watchlist_scan_state['current_phase'] = 'completed' watchlist_scan_state['summary'] = { 'total_artists': len(scan_results), 'successful_scans': len(successful_scans), 'new_tracks_found': total_new_tracks, 'tracks_added_to_wishlist': total_added_to_wishlist } logger.info(f"Watchlist scan completed: {len(successful_scans)}/{len(scan_results)} artists scanned successfully") logger.info(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist") else: logger.warning("Watchlist scan cancelled — skipping post-scan steps") # Post-scan steps — skip if cancelled if not was_cancelled: # Populate discovery pool from similar artists logger.info("Starting discovery pool population...") watchlist_scan_state['current_phase'] = 'populating_discovery_pool' try: scanner.populate_discovery_pool(profile_id=scan_profile_id) logger.info("Discovery pool population complete") except Exception as discovery_error: logger.error(f"Error populating discovery pool: {discovery_error}") import traceback traceback.print_exc() # Update ListenBrainz playlists cache logger.info("Starting ListenBrainz playlists update...") watchlist_scan_state['current_phase'] = 'updating_listenbrainz' try: from core.listenbrainz_manager import ListenBrainzManager db = get_database() db_path = str(db.database_path) # Update for all profiles with LB tokens lb_profiles = db.get_profiles_with_listenbrainz() if lb_profiles: for lb_prof in lb_profiles: lb_manager = ListenBrainzManager(db_path, profile_id=lb_prof['id'], token=lb_prof['token'], base_url=lb_prof['base_url']) lb_result = lb_manager.update_all_playlists() if lb_result.get('success'): logger.info(f"ListenBrainz update complete for profile {lb_prof['id']}: {lb_result.get('summary', {})}") else: # Fallback: use global config token lb_manager = ListenBrainzManager(db_path) lb_result = lb_manager.update_all_playlists() if lb_result.get('success'): logger.info(f"ListenBrainz update complete (global): {lb_result.get('summary', {})}") elif lb_result.get('error'): logger.error(f"ListenBrainz update skipped: {lb_result.get('error')}") except Exception as lb_error: logger.error(f"Error updating ListenBrainz: {lb_error}") import traceback traceback.print_exc() # Update current seasonal playlist (weekly refresh) logger.info("Starting seasonal content update...") watchlist_scan_state['current_phase'] = 'updating_seasonal' try: from core.seasonal_discovery import get_seasonal_discovery_service seasonal_service = get_seasonal_discovery_service(spotify_client, database) # Only update the current active season current_season = seasonal_service.get_current_season() if current_season: if seasonal_service.should_populate_seasonal_content(current_season, days_threshold=7): logger.info(f"Updating {current_season} seasonal content...") seasonal_service.populate_seasonal_content(current_season) seasonal_service.curate_seasonal_playlist(current_season) logger.info(f"{current_season.capitalize()} seasonal content updated") else: logger.info(f"{current_season.capitalize()} seasonal content recently updated, skipping") else: logger.warning("ℹ️ No active season at this time") except Exception as seasonal_error: logger.error(f"Error updating seasonal content: {seasonal_error}") import traceback traceback.print_exc() # Generate Last.fm radio playlists (weekly refresh) logger.info("Starting Last.fm radio generation...") watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' try: scanner._generate_lastfm_radio_playlists() logger.info("Last.fm radio generation complete") except Exception as lastfm_error: logger.error(f"Error generating Last.fm radio playlists: {lastfm_error}") # Sync Spotify library cache logger.info("Syncing Spotify library cache...") watchlist_scan_state['current_phase'] = 'syncing_spotify_library' try: scanner.sync_spotify_library_cache(profile_id=scan_profile_id) logger.info("Spotify library cache sync complete") except Exception as lib_error: logger.error(f"Error syncing Spotify library: {lib_error}") except Exception as e: logger.error(f"Error during watchlist scan: {e}") watchlist_scan_state['status'] = 'error' watchlist_scan_state['error'] = str(e) finally: # Resume enrichment workers if we paused them _resume_enrichment_workers(_ew_state, 'watchlist scan') # Clear one-time rescan cutoff after full scan cycle try: scanner._clear_rescan_cutoff() except Exception: pass # Always reset flag when scan completes (success or error) with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 logger.info("[Manual Watchlist Scan] Flag reset - scan complete") # Initialize scan state global watchlist_scan_state watchlist_scan_state = { 'status': 'scanning', 'started_at': datetime.now(), 'results': [], 'summary': {}, 'error': None, 'cancel_requested': False } # Start scan in background thread = threading.Thread(target=run_scan) thread.daemon = True thread.start() return jsonify({"success": True, "message": "Watchlist scan started"}) except Exception as e: logger.error(f"Error starting watchlist scan: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/scan/status', methods=['GET']) def get_watchlist_scan_status(): """Get the current status of watchlist scanning""" try: global watchlist_scan_state if 'watchlist_scan_state' not in globals(): return jsonify({ "success": True, "status": "idle", "summary": {} }) # Convert datetime objects to ISO format for JSON serialization state = watchlist_scan_state.copy() if 'started_at' in state and state['started_at']: state['started_at'] = state['started_at'].isoformat() if 'completed_at' in state and state['completed_at']: state['completed_at'] = state['completed_at'].isoformat() # Remove results array - it contains ScanResult objects that aren't JSON serializable # The summary already contains the aggregate data we need if 'results' in state: del state['results'] return jsonify({"success": True, **state}) except Exception as e: logger.error(f"Error getting watchlist scan status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/scan/cancel', methods=['POST']) def cancel_watchlist_scan(): """Cancel a running watchlist scan""" try: global watchlist_scan_state if watchlist_scan_state.get('status') != 'scanning': return jsonify({"success": False, "error": "No scan is currently running"}), 400 watchlist_scan_state['cancel_requested'] = True logger.info("[Watchlist Scan] Cancel requested by user") return jsonify({"success": True, "message": "Cancel request sent"}) except Exception as e: logger.error(f"Error cancelling watchlist scan: {e}") return jsonify({"success": False, "error": str(e)}), 500 # Similar Artists Update State similar_artists_update_state = { 'status': 'idle', # idle, running, completed, error 'artists_processed': 0, 'total_artists': 0, 'current_artist': None, 'error': None } similar_artists_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="SimilarArtistsUpdate") @app.route('/api/watchlist/update-similar-artists', methods=['POST']) def update_similar_artists_endpoint(): """Update similar artists for all watchlist artists (for discovery feature)""" try: global similar_artists_update_state if similar_artists_update_state['status'] == 'running': return jsonify({"success": False, "error": "Similar artists update already in progress"}), 409 if not spotify_client or not spotify_client.is_authenticated(): return jsonify({"success": False, "error": "Spotify client not available"}), 400 # Reset state similar_artists_update_state = { 'status': 'running', 'artists_processed': 0, 'total_artists': 0, 'current_artist': None, 'error': None } # Start update in background similar_artists_executor.submit(_update_similar_artists_worker) return jsonify({"success": True, "message": "Similar artists update started"}) except Exception as e: logger.error(f"Error starting similar artists update: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/similar-artists-status', methods=['GET']) def get_similar_artists_update_status(): """Get status of similar artists update""" try: global similar_artists_update_state return jsonify({"success": True, **similar_artists_update_state}) except Exception as e: logger.error(f"Error getting similar artists status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/artist//config', methods=['GET', 'POST']) def watchlist_artist_config(artist_id): """Get or update watchlist artist configuration""" try: from database.music_database import get_database database = get_database() if request.method == 'GET': # Get current config from database conn = sqlite3.connect(str(database.database_path)) cursor = conn.cursor() cursor.execute(""" SELECT include_albums, include_eps, include_singles, include_live, include_remixes, include_acoustic, include_compilations, artist_name, image_url, spotify_artist_id, itunes_artist_id, last_scan_timestamp, date_added, include_instrumentals, deezer_artist_id, lookback_days, discogs_artist_id, preferred_metadata_source FROM watchlist_artists WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? """, (artist_id, artist_id, artist_id, artist_id)) result = cursor.fetchone() conn.close() if not result: return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404 # Determine if this is an iTunes or Spotify artist is_itunes_artist = artist_id.isdigit() spotify_id = result[9] # spotify_artist_id from query itunes_id = result[10] # itunes_artist_id from query deezer_id = result[14] # deezer_artist_id from query discogs_id = result[16] # discogs_artist_id from query # Get artist info from Spotify (only for Spotify artists) artist_info = None if not is_itunes_artist and spotify_client and spotify_client.is_authenticated() and spotify_id and not _spotify_rate_limited(): try: from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='artist') artist_data = spotify_client.sp.artist(spotify_id) if artist_data: artist_info = { 'id': artist_data['id'], 'name': artist_data['name'], 'image_url': artist_data['images'][0]['url'] if artist_data.get('images') else None, 'followers': artist_data.get('followers', {}).get('total', 0), 'popularity': artist_data.get('popularity', 0), 'genres': artist_data.get('genres', []) } except Exception as e: logger.error(f"Could not fetch artist info from Spotify: {e}") # Fallback to database info if Spotify fetch failed if not artist_info: artist_info = { 'id': artist_id, 'name': result[7], # artist_name 'image_url': result[8], # image_url 'followers': 0, 'popularity': 0, 'genres': [] } # Enrich with library artist data (banner, bio, style, mood, label) try: conn2 = sqlite3.connect(str(database.database_path)) cur2 = conn2.cursor() # The library `artists` table uses `deezer_id` / `discogs_id` for # those columns; only the `watchlist_artists` table uses the # `_artist_id` suffix for them. Mixing them was producing a # 'no such column' on every watchlist-config GET. cur2.execute(""" SELECT banner_url, summary, style, mood, label, genres FROM artists WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_id = ? OR discogs_id = ? LIMIT 1 """, (artist_id, artist_id, artist_id, artist_id)) lib_row = cur2.fetchone() if lib_row: artist_info['banner_url'] = lib_row[0] artist_info['summary'] = lib_row[1] artist_info['style'] = lib_row[2] artist_info['mood'] = lib_row[3] artist_info['label'] = lib_row[4] # Backfill genres from library if Spotify didn't provide any if not artist_info.get('genres') and lib_row[5]: try: artist_info['genres'] = json.loads(lib_row[5]) except (json.JSONDecodeError, TypeError): pass # Get recent releases for this watchlist artist cur2.execute(""" SELECT rr.album_name, rr.release_date, rr.album_cover_url, rr.track_count FROM recent_releases rr JOIN watchlist_artists wa ON rr.watchlist_artist_id = wa.id WHERE wa.spotify_artist_id = ? OR wa.itunes_artist_id = ? OR wa.deezer_artist_id = ? ORDER BY rr.release_date DESC LIMIT 6 """, (artist_id, artist_id, artist_id)) releases = [ { 'album_name': r[0], 'release_date': r[1], 'album_cover_url': r[2], 'track_count': r[3], } for r in cur2.fetchall() ] conn2.close() except Exception as e: logger.error(f"Could not enrich artist from library: {e}") releases = [] config = { 'include_albums': bool(result[0]), # Convert INTEGER to boolean 'include_eps': bool(result[1]), 'include_singles': bool(result[2]), 'include_live': bool(result[3]), 'include_remixes': bool(result[4]), 'include_acoustic': bool(result[5]), 'include_compilations': bool(result[6]), 'include_instrumentals': bool(result[13]) if result[13] is not None else False, 'last_scan_timestamp': result[11], 'date_added': result[12], 'lookback_days': result[15] if len(result) > 15 else None, 'preferred_metadata_source': result[17] if len(result) > 17 else None, } from core.metadata_service import get_primary_source return jsonify({ "success": True, "config": config, "artist": artist_info, "recent_releases": releases, "spotify_artist_id": spotify_id, "itunes_artist_id": itunes_id, "deezer_artist_id": deezer_id, "discogs_artist_id": discogs_id, "watchlist_name": result[7], # Original stored watchlist artist name "global_metadata_source": get_primary_source(), }) else: # POST data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 include_albums = data.get('include_albums', True) include_eps = data.get('include_eps', True) include_singles = data.get('include_singles', True) include_live = data.get('include_live', False) include_remixes = data.get('include_remixes', False) include_acoustic = data.get('include_acoustic', False) include_compilations = data.get('include_compilations', False) include_instrumentals = data.get('include_instrumentals', False) lookback_days = data.get('lookback_days', None) # None = use global setting # Validate lookback_days if provided if lookback_days is not None: lookback_days = int(lookback_days) if lookback_days != '' else None preferred_metadata_source = data.get('preferred_metadata_source', None) # Validate — only accept known sources, empty string means clear override if preferred_metadata_source == '' or preferred_metadata_source not in ('spotify', 'deezer', 'itunes', 'discogs'): preferred_metadata_source = None # Validate at least one release type is selected if not (include_albums or include_eps or include_singles): return jsonify({"success": False, "error": "At least one release type must be selected"}), 400 # Update database conn = sqlite3.connect(str(database.database_path)) cursor = conn.cursor() # Check if lookback_days changed — if so, clear last_scan_timestamp to force rescan cursor.execute(""" SELECT lookback_days FROM watchlist_artists WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? """, (artist_id, artist_id, artist_id, artist_id)) old_row = cursor.fetchone() old_lookback = old_row[0] if old_row else None lookback_changed = old_lookback != lookback_days cursor.execute(""" UPDATE watchlist_artists SET include_albums = ?, include_eps = ?, include_singles = ?, include_live = ?, include_remixes = ?, include_acoustic = ?, include_compilations = ?, include_instrumentals = ?, lookback_days = ?, preferred_metadata_source = ?, last_scan_timestamp = CASE WHEN ? THEN NULL ELSE last_scan_timestamp END, updated_at = CURRENT_TIMESTAMP WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? """, (int(include_albums), int(include_eps), int(include_singles), int(include_live), int(include_remixes), int(include_acoustic), int(include_compilations), int(include_instrumentals), lookback_days, preferred_metadata_source, lookback_changed, artist_id, artist_id, artist_id, artist_id)) conn.commit() if cursor.rowcount == 0: conn.close() return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404 conn.close() logger.info(f"Updated watchlist config for artist {artist_id}: albums={include_albums}, eps={include_eps}, singles={include_singles}, live={include_live}, remixes={include_remixes}, acoustic={include_acoustic}, compilations={include_compilations}, instrumentals={include_instrumentals}") return jsonify({ "success": True, "message": "Artist configuration updated successfully", "config": { 'include_albums': include_albums, 'include_eps': include_eps, 'include_singles': include_singles, 'include_live': include_live, 'include_remixes': include_remixes, 'include_acoustic': include_acoustic, 'include_compilations': include_compilations, 'include_instrumentals': include_instrumentals, } }) except Exception as e: logger.error(f"Error in watchlist artist config: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/artist//link-provider', methods=['POST']) def watchlist_artist_link_provider(artist_id): """Manually link a watchlist artist to a different Spotify/iTunes artist.""" try: from database.music_database import get_database database = get_database() data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 new_provider_id = data.get('provider_id', '').strip() provider = data.get('provider', '').strip() valid_providers = ('spotify', 'itunes', 'deezer', 'discogs') if provider not in valid_providers: return jsonify({"success": False, "error": f"Invalid provider. Must be one of: {', '.join(valid_providers)}"}), 400 # Empty provider_id = clear the match for this source is_clear = not new_provider_id conn = sqlite3.connect(str(database.database_path)) cursor = conn.cursor() # Find the watchlist artist row cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id FROM watchlist_artists WHERE spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_artist_id = ? OR discogs_artist_id = ? """, (artist_id, artist_id, artist_id, artist_id)) row = cursor.fetchone() if not row: conn.close() return jsonify({"success": False, "error": "Artist not found in watchlist"}), 404 watchlist_row_id = row[0] artist_name = row[1] # Check for duplicate — another watchlist artist already has this provider ID col_map = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'} col = col_map[provider] if not is_clear: cursor.execute(f"SELECT id, artist_name FROM watchlist_artists WHERE {col} = ? AND id != ?", (new_provider_id, watchlist_row_id)) duplicate = cursor.fetchone() if duplicate: conn.close() return jsonify({"success": False, "error": f"Another watchlist artist ('{duplicate[1]}') already has this {provider} ID"}), 409 # Set to new ID or NULL (clear) update_val = new_provider_id if not is_clear else None cursor.execute(f"UPDATE watchlist_artists SET {col} = ?, updated_at = CURRENT_TIMESTAMP WHERE id = ?", (update_val, watchlist_row_id)) conn.commit() conn.close() action = 'Cleared' if is_clear else 'Linked' logger.info(f"{action} watchlist artist '{artist_name}' {provider} ID: {new_provider_id or 'NULL'}") return jsonify({ "success": True, "message": f"Linked to {provider} artist successfully", "new_provider_id": new_provider_id }) except Exception as e: logger.error(f"Error linking watchlist artist provider: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/watchlist/global-config', methods=['GET', 'POST']) def watchlist_global_config(): """Get or update global watchlist configuration (overrides per-artist settings)""" try: if request.method == 'GET': config = { 'global_override_enabled': config_manager.get('watchlist.global_override_enabled', False), 'include_albums': config_manager.get('watchlist.global_include_albums', True), 'include_eps': config_manager.get('watchlist.global_include_eps', True), 'include_singles': config_manager.get('watchlist.global_include_singles', True), 'include_live': config_manager.get('watchlist.global_include_live', False), 'include_remixes': config_manager.get('watchlist.global_include_remixes', False), 'include_acoustic': config_manager.get('watchlist.global_include_acoustic', False), 'include_compilations': config_manager.get('watchlist.global_include_compilations', False), 'include_instrumentals': config_manager.get('watchlist.global_include_instrumentals', False), 'exclude_terms': config_manager.get('watchlist.exclude_terms', ''), } return jsonify({"success": True, "config": config}) else: # POST data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 global_override_enabled = data.get('global_override_enabled', False) include_albums = data.get('include_albums', True) include_eps = data.get('include_eps', True) include_singles = data.get('include_singles', True) include_live = data.get('include_live', False) include_remixes = data.get('include_remixes', False) include_acoustic = data.get('include_acoustic', False) include_compilations = data.get('include_compilations', False) include_instrumentals = data.get('include_instrumentals', False) exclude_terms = data.get('exclude_terms', '') # When override is enabled, validate at least one release type if global_override_enabled and not (include_albums or include_eps or include_singles): return jsonify({"success": False, "error": "At least one release type must be selected"}), 400 config_manager.set('watchlist.global_override_enabled', global_override_enabled) config_manager.set('watchlist.global_include_albums', include_albums) config_manager.set('watchlist.global_include_eps', include_eps) config_manager.set('watchlist.global_include_singles', include_singles) config_manager.set('watchlist.global_include_live', include_live) config_manager.set('watchlist.global_include_remixes', include_remixes) config_manager.set('watchlist.global_include_acoustic', include_acoustic) config_manager.set('watchlist.global_include_compilations', include_compilations) config_manager.set('watchlist.global_include_instrumentals', include_instrumentals) config_manager.set('watchlist.exclude_terms', exclude_terms) logger.info(f"Updated global watchlist config: override={global_override_enabled}, " f"albums={include_albums}, eps={include_eps}, singles={include_singles}, " f"live={include_live}, remixes={include_remixes}, acoustic={include_acoustic}, " f"compilations={include_compilations}, instrumentals={include_instrumentals}, " f"exclude_terms='{exclude_terms}'") return jsonify({ "success": True, "message": "Global watchlist configuration updated", "config": { 'global_override_enabled': global_override_enabled, 'include_albums': include_albums, 'include_eps': include_eps, 'include_singles': include_singles, 'include_live': include_live, 'include_remixes': include_remixes, 'include_acoustic': include_acoustic, 'include_compilations': include_compilations, 'include_instrumentals': include_instrumentals, 'exclude_terms': exclude_terms, } }) except Exception as e: logger.error(f"Error in watchlist global config: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 def _update_similar_artists_worker(): """Background worker to update similar artists for all watchlist artists""" global similar_artists_update_state try: from core.watchlist_scanner import get_watchlist_scanner from database.music_database import get_database import time logger.info("[Similar Artists] Starting similar artists update...") database = get_database() all_profiles = database.get_all_profiles() # Build per-profile artist lists and deduplicate for API calls # artist_key -> (artist_obj, [profile_ids]) artist_profiles = {} for p in all_profiles: for artist in database.get_watchlist_artists(profile_id=p['id']): key = (artist.spotify_artist_id or '', artist.itunes_artist_id or '', artist.artist_name.lower()) if key not in artist_profiles: artist_profiles[key] = (artist, []) artist_profiles[key][1].append(p['id']) if not artist_profiles: similar_artists_update_state['status'] = 'completed' logger.warning("[Similar Artists] No watchlist artists to process") return similar_artists_update_state['total_artists'] = len(artist_profiles) logger.info(f"[Similar Artists] Processing {len(artist_profiles)} unique watchlist artists across {len(all_profiles)} profiles") scanner = get_watchlist_scanner(spotify_client) for idx, (_key, (artist, profile_ids)) in enumerate(artist_profiles.items(), 1): try: similar_artists_update_state['artists_processed'] = idx similar_artists_update_state['current_artist'] = artist.artist_name logger.info(f"[{idx}/{len(artist_profiles)}] Updating similar artists for {artist.artist_name} (profiles: {profile_ids})") # Update similar artists for each profile that watches this artist for pid in profile_ids: scanner.update_similar_artists(artist, limit=10, profile_id=pid) # Rate limiting if idx < len(artist_profiles): time.sleep(2.0) # 2 seconds between artists except Exception as artist_error: logger.error(f"[Similar Artists] Error processing {artist.artist_name}: {artist_error}") continue # Update complete similar_artists_update_state['status'] = 'completed' similar_artists_update_state['current_artist'] = None logger.info(f"[Similar Artists] Update complete! Processed {len(artist_profiles)} artists") except Exception as e: logger.error(f"[Similar Artists] Critical error: {e}") import traceback traceback.print_exc() similar_artists_update_state['status'] = 'error' similar_artists_update_state['error'] = str(e) # --- Watchlist Auto-Scanning System --- watchlist_scan_state = { 'status': 'idle', 'results': [], 'summary': {}, 'error': None } def _process_watchlist_scan_automatically(automation_id=None, profile_id=None): """Main automatic scanning logic that runs in background thread. Args: automation_id: ID of the automation triggering this scan profile_id: If provided, only scan this profile's watchlist (manual trigger). If None, scan all profiles (scheduled automation). """ global watchlist_auto_scanning, watchlist_auto_scanning_timestamp, watchlist_scan_state scope_label = f"profile {profile_id}" if profile_id else "all profiles" logger.info(f"[Auto-Watchlist] Timer triggered - starting automatic watchlist scan ({scope_label})...") _ew_state = {} try: # CRITICAL FIX: Use smart stuck detection BEFORE acquiring lock # This prevents deadlock and handles stuck flags (2-hour timeout) if is_watchlist_actually_scanning(): logger.info("[Auto-Watchlist] Already scanning (verified with stuck detection), skipping.") return with watchlist_timer_lock: # Re-check inside lock to handle race conditions if watchlist_auto_scanning: logger.info("[Auto-Watchlist] Already scanning (race condition check), skipping.") return # Set flag and timestamp import time watchlist_auto_scanning = True watchlist_auto_scanning_timestamp = time.time() logger.info(f"[Auto-Watchlist] Flag set at timestamp {watchlist_auto_scanning_timestamp}") # Use app context for database operations with app.app_context(): from core.watchlist_scanner import get_watchlist_scanner from database.music_database import get_database database = get_database() # Determine which profiles to scan if profile_id: # Manual trigger — scan only the triggering profile scan_profiles = [{'id': profile_id}] else: # Scheduled automation — scan all profiles scan_profiles = database.get_all_profiles() watchlist_count = sum(database.get_watchlist_count(profile_id=p['id']) for p in scan_profiles) profile_label = f"profile {profile_id}" if profile_id else f"{len(scan_profiles)} profiles" logger.info(f"[Auto-Watchlist] Watchlist count check: {watchlist_count} artists found ({profile_label})") if watchlist_count == 0: logger.warning("ℹ️ [Auto-Watchlist] No artists in watchlist for auto-scanning.") with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 return if not spotify_client or not spotify_client.is_authenticated(): logger.info("ℹ️ [Auto-Watchlist] Spotify client not available or not authenticated.") with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 return logger.info(f"[Auto-Watchlist] Found {watchlist_count} artists in watchlist, starting automatic scan...") _update_automation_progress(automation_id, progress=5, phase='Loading watchlist', log_line=f'{watchlist_count} artists ({profile_label})', log_type='info') # Get list of artists to scan watchlist_artists = [] for p in scan_profiles: watchlist_artists.extend(database.get_watchlist_artists(profile_id=p['id'])) scanner = get_watchlist_scanner(spotify_client) all_profiles = scan_profiles # Used later for discovery pool population for p in scan_profiles: try: filled = scanner.backfill_watchlist_artist_images(p['id']) if filled: logger.info(f"Backfilled {filled} watchlist artist images for profile {p['id']}") except Exception as img_err: logger.error(f"Image backfill error for profile {p['id']}: {img_err}") # Initialize detailed progress tracking (same as manual scan) watchlist_scan_state = { 'status': 'scanning', 'started_at': datetime.now(), 'total_artists': len(watchlist_artists), 'current_artist_index': 0, 'current_artist_name': '', 'current_artist_image_url': '', 'current_phase': 'starting', 'albums_to_check': 0, 'albums_checked': 0, 'current_album': '', 'current_album_image_url': '', 'current_track_name': '', 'tracks_found_this_scan': 0, 'tracks_added_this_scan': 0, 'recent_wishlist_additions': [], 'results': [], 'summary': {}, 'error': None, 'cancel_requested': False } scan_results = [] # Pause enrichment workers during scan to reduce API contention _ew_state = _pause_enrichment_workers('auto-watchlist scan') def _scan_progress(event_type, payload): if event_type == 'scan_started': _update_automation_progress( automation_id, progress=5, phase='Loading watchlist', log_line=f"{len(watchlist_artists)} artists ({profile_label})", log_type='info', ) elif event_type == 'artist_started': total = max(1, payload.get('total_artists', len(watchlist_artists))) idx = payload.get('artist_index', 1) artist_name = payload.get('artist_name', '') pct = 5 + ((idx - 1) / total) * 90 _update_automation_progress( automation_id, progress=pct, phase=f'Scanning: {artist_name} ({idx}/{total})', current_item=artist_name, processed=idx - 1, total=total, ) elif event_type == 'artist_completed': artist_name = payload.get('artist_name', '') new_tracks = payload.get('new_tracks_found', 0) added = payload.get('tracks_added_to_wishlist', 0) if new_tracks > 0: _update_automation_progress( automation_id, log_line=f'{artist_name} — {new_tracks} new, {added} added', log_type='success', ) else: _update_automation_progress( automation_id, log_line=f'{artist_name} — no new tracks', log_type='skip', ) elif event_type == 'artist_error': artist_name = payload.get('artist_name', '') error_message = payload.get('error_message', 'error') _update_automation_progress( automation_id, log_line=f'{artist_name} — error: {error_message[:60]}', log_type='error', ) elif event_type == 'cancelled': _update_automation_progress( automation_id, progress=100, phase='Cancelled by user', log_line='Scan cancelled by user', log_type='warning', ) elif event_type == 'scan_completed': _update_automation_progress( automation_id, progress=95, phase='Scan complete', log_line=( f"Scanned {payload.get('successful_scans', 0)} artists — " f"{payload.get('new_tracks_found', 0)} new tracks, " f"{payload.get('tracks_added_to_wishlist', 0)} added to wishlist" ), log_type='success' if payload.get('new_tracks_found', 0) > 0 else 'info', ) scan_results = scanner.scan_watchlist_artists( watchlist_artists, scan_state=watchlist_scan_state, progress_callback=_scan_progress, cancel_check=lambda: watchlist_scan_state.get('cancel_requested'), ) # Update state with results (skip if cancelled — already set by cancel handler) was_cancelled = watchlist_scan_state.get('cancel_requested', False) if not was_cancelled: successful_scans = [r for r in scan_results if r.success] total_new_tracks = sum(r.new_tracks_found for r in successful_scans) total_added_to_wishlist = sum(r.tracks_added_to_wishlist for r in successful_scans) watchlist_scan_state['status'] = 'completed' watchlist_scan_state['results'] = scan_results watchlist_scan_state['completed_at'] = datetime.now() watchlist_scan_state['summary'] = { 'total_artists': len(scan_results), 'successful_scans': len(successful_scans), 'new_tracks_found': total_new_tracks, 'tracks_added_to_wishlist': total_added_to_wishlist } logger.info(f"Automatic watchlist scan completed: {len(successful_scans)}/{len(scan_results)} artists scanned successfully") logger.info(f"Found {total_new_tracks} new tracks, added {total_added_to_wishlist} to wishlist") _update_automation_progress(automation_id, progress=95, phase='Scan complete', log_line=f'Scanned {len(successful_scans)} artists — {total_new_tracks} new tracks, {total_added_to_wishlist} added to wishlist', log_type='success' if total_new_tracks > 0 else 'info') else: total_new_tracks = watchlist_scan_state.get('summary', {}).get('new_tracks_found', 0) total_added_to_wishlist = watchlist_scan_state.get('summary', {}).get('tracks_added_to_wishlist', 0) logger.warning("Automatic watchlist scan cancelled — skipping post-scan steps") # Post-scan steps — skip if cancelled if not was_cancelled: # Populate discovery pool from similar artists (per-profile) logger.info("Starting discovery pool population...") watchlist_scan_state['current_phase'] = 'populating_discovery_pool' _update_automation_progress(automation_id, progress=96, phase='Populating discovery pool', log_line='Building discovery pool from similar artists...', log_type='info') try: def _discovery_progress(event_type, message): if event_type == 'artist': _update_automation_progress(automation_id, phase=f'Discovery pool: {message}', log_line=message, log_type='info', current_item=message) elif event_type == 'phase': _update_automation_progress(automation_id, phase=message, log_line=message, log_type='info') elif event_type == 'success': _update_automation_progress(automation_id, log_line=message, log_type='success') elif event_type == 'skip': _update_automation_progress(automation_id, log_line=message, log_type='info') for p in all_profiles: scanner.populate_discovery_pool(profile_id=p['id'], progress_callback=_discovery_progress) logger.info("Discovery pool population complete") except Exception as discovery_error: logger.error(f"Error populating discovery pool: {discovery_error}") import traceback traceback.print_exc() _update_automation_progress(automation_id, log_line=f'Discovery pool error: {discovery_error}', log_type='error') # Update ListenBrainz playlists cache logger.info("Starting ListenBrainz playlists update...") watchlist_scan_state['current_phase'] = 'updating_listenbrainz' _update_automation_progress(automation_id, progress=97, phase='Updating ListenBrainz', log_line='Fetching ListenBrainz playlists...', log_type='info') try: from core.listenbrainz_manager import ListenBrainzManager db = get_database() db_path = str(db.database_path) lb_profiles = db.get_profiles_with_listenbrainz() if lb_profiles: for lb_prof in lb_profiles: lb_manager = ListenBrainzManager(db_path, profile_id=lb_prof['id'], token=lb_prof['token'], base_url=lb_prof['base_url']) lb_result = lb_manager.update_all_playlists() if lb_result.get('success'): summary = lb_result.get('summary', {}) logger.info(f"ListenBrainz update complete for profile {lb_prof['id']}: {summary}") _update_automation_progress(automation_id, log_line=f'ListenBrainz (profile {lb_prof["id"]}): playlists updated', log_type='success') else: lb_manager = ListenBrainzManager(db_path) lb_result = lb_manager.update_all_playlists() if lb_result.get('success'): summary = lb_result.get('summary', {}) logger.info(f"ListenBrainz update complete (global): {summary}") _update_automation_progress(automation_id, log_line='ListenBrainz: playlists updated', log_type='success') else: logger.error(f"ListenBrainz update had issues: {lb_result.get('error', 'Unknown error')}") _update_automation_progress(automation_id, log_line=f'ListenBrainz: {lb_result.get("error", "Unknown error")}', log_type='error') except Exception as lb_error: logger.error(f"Error updating ListenBrainz: {lb_error}") import traceback traceback.print_exc() _update_automation_progress(automation_id, log_line=f'ListenBrainz error: {lb_error}', log_type='error') # Update current seasonal playlist (weekly refresh) logger.info("Starting seasonal content update...") watchlist_scan_state['current_phase'] = 'updating_seasonal' _update_automation_progress(automation_id, progress=98, phase='Updating seasonal content', log_line='Checking seasonal playlists...', log_type='info') try: from core.seasonal_discovery import get_seasonal_discovery_service seasonal_service = get_seasonal_discovery_service(spotify_client, database) # Only update the current active season current_season = seasonal_service.get_current_season() if current_season: if seasonal_service.should_populate_seasonal_content(current_season, days_threshold=7): logger.info(f"Updating {current_season} seasonal content...") _update_automation_progress(automation_id, log_line=f'Updating {current_season} seasonal content...', log_type='info') seasonal_service.populate_seasonal_content(current_season) seasonal_service.curate_seasonal_playlist(current_season) logger.info(f"{current_season.capitalize()} seasonal content updated") _update_automation_progress(automation_id, log_line=f'{current_season.capitalize()} seasonal content updated', log_type='success') else: logger.info(f"{current_season.capitalize()} seasonal content recently updated, skipping") _update_automation_progress(automation_id, log_line=f'{current_season.capitalize()} seasonal content up to date', log_type='info') else: logger.warning("ℹ️ No active season at this time") _update_automation_progress(automation_id, log_line='No active season', log_type='info') except Exception as seasonal_error: logger.error(f"Error updating seasonal content: {seasonal_error}") import traceback traceback.print_exc() _update_automation_progress(automation_id, log_line=f'Seasonal error: {seasonal_error}', log_type='error') # Generate Last.fm radio playlists (weekly refresh) logger.info("Starting Last.fm radio generation...") watchlist_scan_state['current_phase'] = 'generating_lastfm_radio' _update_automation_progress(automation_id, progress=99, phase='Generating Last.fm radio', log_line='Building Last.fm radio playlists...', log_type='info') try: scanner._generate_lastfm_radio_playlists() logger.info("Last.fm radio generation complete") _update_automation_progress(automation_id, log_line='Last.fm radio playlists updated', log_type='success') except Exception as lastfm_error: logger.error(f"Error generating Last.fm radio playlists: {lastfm_error}") _update_automation_progress(automation_id, log_line=f'Last.fm radio error: {lastfm_error}', log_type='error') # Sync Spotify library cache logger.info("Syncing Spotify library cache...") try: for p in all_profiles: scanner.sync_spotify_library_cache(profile_id=p['id']) logger.info("Spotify library cache sync complete") _update_automation_progress(automation_id, log_line='Spotify library cache synced', log_type='info') except Exception as lib_error: logger.error(f"Error syncing Spotify library: {lib_error}") _update_automation_progress(automation_id, log_line=f'Library cache error: {lib_error}', log_type='error') # Add activity for watchlist scan completion if total_added_to_wishlist > 0: add_activity_item("", "Watchlist Scan Complete", f"{total_added_to_wishlist} new tracks added to wishlist", "Now") try: if automation_engine: automation_engine.emit('watchlist_scan_completed', { 'artists_scanned': str(len(scan_results)), 'new_tracks_found': str(total_new_tracks), 'tracks_added': str(total_added_to_wishlist), }) except Exception: pass except Exception as e: logger.error(f"Error in automatic watchlist scan: {e}") import traceback traceback.print_exc() _update_automation_progress(automation_id, log_line=f'Error: {str(e)}', log_type='error') watchlist_scan_state['status'] = 'error' watchlist_scan_state['error'] = str(e) raise # re-raise so automation wrapper returns error status finally: # Resume enrichment workers if we paused them _resume_enrichment_workers(_ew_state, 'auto-watchlist scan') # Clear one-time rescan cutoff after full scan cycle try: scanner._clear_rescan_cutoff() except Exception: pass # Always reset flag with watchlist_timer_lock: watchlist_auto_scanning = False watchlist_auto_scanning_timestamp = 0 logger.info("Automatic watchlist scanning complete") # --- Metadata Updater System --- from concurrent.futures import ThreadPoolExecutor # Global state for metadata update process metadata_update_state = { 'status': 'idle', 'current_artist': '', 'processed': 0, 'total': 0, 'percentage': 0.0, 'successful': 0, 'failed': 0, 'started_at': None, 'completed_at': None, 'error': None, 'refresh_interval_days': 30 } metadata_update_worker = None metadata_update_runtime_worker = None metadata_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="metadata_update") # =============================== # == DISCOVER PAGE ENDPOINTS == # =============================== def _get_active_discovery_source(): """ Determine which music source is active for discovery. Returns the user's configured primary metadata source. If the selected source requires auth and isn't available, falls back. NOTE: Thin wrapper — canonical logic lives in core.metadata_service.get_primary_source(). """ from core.metadata_service import get_primary_source return get_primary_source() @app.route('/api/discover/hero', methods=['GET']) def get_discover_hero(): """Get featured similar artists for hero slideshow""" try: database = get_database() # Determine active source active_source = _get_active_discovery_source() logger.info(f"Discover hero using source: {active_source}") # Import fallback client for non-Spotify lookups itunes_client = _get_metadata_fallback_client() # Get top similar artists (excluding watchlist, cycled by last_featured) # Fetch more than needed since strict source filtering may drop many pid = get_current_profile_id() logger.info(f"[Discover Hero] Profile ID: {pid}, Active source: {active_source}") similar_artists = database.get_top_similar_artists(limit=200, profile_id=pid, require_source=active_source) # FALLBACK: If no similar artists exist, use watchlist artists for Hero section if not similar_artists: logger.warning("[Discover Hero] No similar artists found, falling back to watchlist artists") watchlist_artists = database.get_watchlist_artists(profile_id=pid) if not watchlist_artists: return jsonify({"success": True, "artists": [], "source": active_source}) # Convert watchlist artists to hero format import random shuffled_watchlist = list(watchlist_artists) random.shuffle(shuffled_watchlist) hero_artists = [] for artist in shuffled_watchlist[:10]: if active_source == 'spotify': artist_id = artist.spotify_artist_id elif active_source == 'deezer': artist_id = getattr(artist, 'deezer_artist_id', None) or artist.itunes_artist_id else: artist_id = artist.itunes_artist_id if not artist_id: continue artist_data = { "spotify_artist_id": artist.spotify_artist_id, "itunes_artist_id": artist.itunes_artist_id, "artist_id": artist_id, "artist_name": artist.artist_name, "occurrence_count": 1, "similarity_rank": 1, "source": active_source, "is_watchlist": True } # Use cached image from watchlist — no API call needed if hasattr(artist, 'image_url') and artist.image_url: artist_data['image_url'] = artist.image_url hero_artists.append(artist_data) logger.warning(f"[Discover Hero] Returning {len(hero_artists)} watchlist artists as fallback") return jsonify({"success": True, "artists": hero_artists, "source": active_source, "fallback": "watchlist"}) # Artists are already filtered by source in SQL — no post-filter needed valid_artists = list(similar_artists) # FALLBACK: If no valid artists for fallback source, try to resolve IDs on-the-fly if active_source in ('itunes', 'deezer') and not valid_artists: logger.warning(f"[{active_source} Fallback] No artists with {active_source} IDs found, attempting on-the-fly resolution for {len(similar_artists)} artists") resolved_count = 0 for artist in similar_artists: existing_id = getattr(artist, f'similar_artist_{active_source}_id', None) or (artist.similar_artist_itunes_id if active_source == 'itunes' else None) if existing_id: valid_artists.append(artist) continue # Try to resolve ID by name try: search_results = itunes_client.search_artists(artist.similar_artist_name, limit=1) if search_results and len(search_results) > 0: resolved_id = search_results[0].id # Cache the resolved ID for future use if active_source == 'deezer': database.update_similar_artist_deezer_id(artist.id, resolved_id) artist.similar_artist_deezer_id = resolved_id else: database.update_similar_artist_itunes_id(artist.id, resolved_id) artist.similar_artist_itunes_id = resolved_id valid_artists.append(artist) resolved_count += 1 logger.info(f" [Resolved] {artist.similar_artist_name} -> {active_source} ID: {resolved_id}") except Exception as resolve_err: logger.error(f" [Failed] Could not resolve {active_source} ID for {artist.similar_artist_name}: {resolve_err}") # Stop after 10 successful resolutions to avoid rate limiting if len(valid_artists) >= 10: break logger.warning(f"[{active_source} Fallback] Resolved {resolved_count} artists with IDs") logger.info(f"[Discover Hero] Found {len(valid_artists)} valid artists for source: {active_source}") # Filter out blacklisted artists blacklisted = database.get_discovery_blacklist_names() if blacklisted: valid_artists = [a for a in valid_artists if a.similar_artist_name.lower() not in blacklisted] # Take top 10 (already ordered by least-recently-featured, then quality) similar_artists = valid_artists[:10] # Convert to JSON format — use cached metadata, only fetch from API if missing hero_artists = [] for artist in similar_artists: # Use the ID for the active source, falling back to the other if needed if active_source == 'spotify': artist_id = artist.similar_artist_spotify_id or artist.similar_artist_itunes_id elif active_source == 'deezer': artist_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id or artist.similar_artist_spotify_id else: artist_id = artist.similar_artist_itunes_id or artist.similar_artist_spotify_id artist_data = { "spotify_artist_id": artist.similar_artist_spotify_id, "itunes_artist_id": artist.similar_artist_itunes_id, "artist_id": artist_id, "artist_name": artist.similar_artist_name, "occurrence_count": artist.occurrence_count, "similarity_rank": artist.similarity_rank, "source": active_source } # Use cached metadata if available if artist.image_url: artist_data['image_url'] = artist.image_url artist_data['genres'] = artist.genres or [] artist_data['popularity'] = artist.popularity or 0 else: # No cached metadata — fetch from API and cache for next time try: if active_source == 'spotify' and artist.similar_artist_spotify_id: if spotify_client and spotify_client.is_authenticated(): sp_artist = spotify_client.get_artist(artist.similar_artist_spotify_id) if sp_artist and sp_artist.get('images'): artist_data['artist_name'] = sp_artist.get('name', artist.similar_artist_name) artist_data['image_url'] = sp_artist['images'][0]['url'] if sp_artist['images'] else None artist_data['genres'] = sp_artist.get('genres', []) artist_data['popularity'] = sp_artist.get('popularity', 0) # Cache it database.update_similar_artist_metadata( artist.id, artist_data.get('image_url'), artist_data.get('genres'), artist_data.get('popularity') ) elif active_source in ('itunes', 'deezer'): fb_artist_id = getattr(artist, 'similar_artist_deezer_id', None) if active_source == 'deezer' else None fb_artist_id = fb_artist_id or artist.similar_artist_itunes_id if fb_artist_id: fb_artist_data = itunes_client.get_artist(fb_artist_id) if fb_artist_data: artist_data['artist_name'] = fb_artist_data.get('name', artist.similar_artist_name) artist_data['image_url'] = fb_artist_data.get('images', [{}])[0].get('url') if fb_artist_data.get('images') else None artist_data['genres'] = fb_artist_data.get('genres', []) artist_data['popularity'] = fb_artist_data.get('popularity', 0) # Cache it database.update_similar_artist_metadata( artist.id, artist_data.get('image_url'), artist_data.get('genres'), artist_data.get('popularity') ) except Exception as img_err: logger.error(f"Could not fetch artist image: {img_err}") hero_artists.append(artist_data) # Mark these artists as featured so they cycle to the back of the queue featured_names = [a["artist_name"] for a in hero_artists] database.mark_artists_featured(featured_names) return jsonify({"success": True, "artists": hero_artists, "source": active_source}) except Exception as e: logger.error(f"Error getting discover hero: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/similar-artists', methods=['GET']) def get_discover_similar_artists(): """Get all recommended similar artists (basic data, no enrichment for speed)""" try: database = get_database() active_source = _get_active_discovery_source() similar_artists = database.get_top_similar_artists(limit=200, profile_id=get_current_profile_id(), require_source=active_source) if not similar_artists: return jsonify({"success": True, "artists": [], "source": active_source, "count": 0}) # Artists already filtered by source in SQL result_artists = [] for artist in similar_artists: if active_source == 'spotify': artist_id = artist.similar_artist_spotify_id elif active_source == 'deezer': artist_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id else: artist_id = artist.similar_artist_itunes_id artist_data = { "artist_id": artist_id, "spotify_artist_id": artist.similar_artist_spotify_id, "itunes_artist_id": artist.similar_artist_itunes_id, "artist_name": artist.similar_artist_name, "occurrence_count": artist.occurrence_count, "similarity_rank": artist.similarity_rank, "source": active_source, } # Include cached metadata if available if artist.image_url: artist_data["image_url"] = artist.image_url if artist.genres: artist_data["genres"] = artist.genres[:3] if artist.popularity: artist_data["popularity"] = artist.popularity result_artists.append(artist_data) logger.info(f"[Similar Artists] {len(similar_artists)} from DB, {len(result_artists)} valid for {active_source}") return jsonify({ "success": True, "artists": result_artists, "source": active_source, "count": len(result_artists) }) except Exception as e: logger.error(f"Error getting similar artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/similar-artists/enrich', methods=['POST']) def enrich_similar_artists(): """Enrich a batch of artist IDs with images/genres from Spotify or iTunes. Uses cached metadata from DB when available, only makes API calls for uncached artists, and saves new results back to DB for future use.""" try: data = request.get_json() artist_ids = data.get('artist_ids', []) source = data.get('source', 'spotify') if not artist_ids: return jsonify({"success": True, "artists": {}}) database = get_database() enriched = {} uncached_ids = [] # Check DB cache first — get all similar artists and index by external ID cached_artists = database.get_top_similar_artists(limit=500, profile_id=get_current_profile_id()) cache_map = {} for artist in cached_artists: if source == 'spotify': ext_id = artist.similar_artist_spotify_id elif source == 'deezer': ext_id = getattr(artist, 'similar_artist_deezer_id', None) or artist.similar_artist_itunes_id else: ext_id = artist.similar_artist_itunes_id if ext_id and ext_id not in cache_map: cache_map[ext_id] = artist for aid in artist_ids[:50]: cached = cache_map.get(aid) if cached and cached.image_url: # Use cached metadata enriched[aid] = { "artist_name": cached.similar_artist_name, "image_url": cached.image_url, "genres": cached.genres[:3] if cached.genres else [], "popularity": cached.popularity or 0 } else: uncached_ids.append(aid) # Only make API calls for uncached artists if uncached_ids: if source == 'spotify' and spotify_client and spotify_client.is_authenticated() and not _spotify_rate_limited(): try: from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='artists_batch') batch_result = spotify_client.sp.artists(uncached_ids[:50]) if batch_result and 'artists' in batch_result: for sp_artist in batch_result['artists']: if sp_artist: img_url = sp_artist['images'][0].get('url') if sp_artist.get('images') else None genres = sp_artist.get('genres', [])[:3] pop = sp_artist.get('popularity', 0) enriched[sp_artist['id']] = { "artist_name": sp_artist.get('name'), "image_url": img_url, "genres": genres, "popularity": pop } # Cache to DB for future use database.update_similar_artist_metadata_by_external_id( sp_artist['id'], 'spotify', image_url=img_url, genres=genres, popularity=pop ) except Exception as e: from core.spotify_client import _detect_and_set_rate_limit _detect_and_set_rate_limit(e, 'enrich_similar_artists') logger.error(f"Error enriching Spotify batch: {e}") else: fallback_client = _get_metadata_fallback_client() fallback_source = _get_metadata_fallback_source() for aid in uncached_ids[:50]: try: fb_artist = fallback_client.get_artist(aid) if fb_artist: img_url = fb_artist.get('images', [{}])[0].get('url') if fb_artist.get('images') else None genres = fb_artist.get('genres', [])[:3] enriched[aid] = { "artist_name": fb_artist.get('name'), "image_url": img_url, "genres": genres, "popularity": 0 } # Cache to DB for future use database.update_similar_artist_metadata_by_external_id( aid, fallback_source, image_url=img_url, genres=genres, popularity=0 ) except Exception: pass cached_count = len(enriched) - len([aid for aid in uncached_ids if aid in enriched]) api_count = len([aid for aid in uncached_ids if aid in enriched]) if uncached_ids: logger.warning(f"[Enrich] {cached_count} from cache, {api_count} from API ({len(uncached_ids) - api_count} missed)") return jsonify({"success": True, "artists": enriched}) except Exception as e: logger.error(f"Error enriching similar artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/spotify-library', methods=['GET']) def get_spotify_library(): """Get cached Spotify library albums with ownership status. Only available when Spotify is authenticated.""" try: # Skip entirely if Spotify is not the active source if not spotify_client or not spotify_client.is_spotify_authenticated(): return jsonify({ "success": True, "albums": [], "total": 0, "offset": 0, "limit": 0, "stats": {"total": 0, "owned": 0, "missing": 0} }) database = get_database() profile_id = get_current_profile_id() offset = request.args.get('offset', 0, type=int) limit = request.args.get('limit', 48, type=int) search = request.args.get('search', '', type=str) status_filter = request.args.get('status', 'all', type=str) sort = request.args.get('sort', 'date_saved', type=str) sort_dir = request.args.get('sort_dir', 'desc', type=str) # Fetch all matching albums (ownership requires post-query computation) all_albums, total = database.get_spotify_library_albums( offset=0, limit=10000, search=search, sort=sort, sort_dir=sort_dir, profile_id=profile_id ) if not all_albums: return jsonify({ "success": True, "albums": [], "total": 0, "offset": offset, "limit": limit, "stats": {"total": 0, "owned": 0, "missing": 0} }) # Cross-reference with local library for ownership status library_spotify_ids = database.get_library_spotify_album_ids(profile_id) library_album_names = database.get_library_album_names() owned_count = 0 for album in all_albums: # Check by Spotify album ID first, then fuzzy match by name if album['spotify_album_id'] in library_spotify_ids: album['in_library'] = True elif (album['artist_name'].lower(), album['album_name'].lower()) in library_album_names: album['in_library'] = True else: album['in_library'] = False if album['in_library']: owned_count += 1 # Apply status filter then paginate if status_filter == 'missing': filtered = [a for a in all_albums if not a['in_library']] elif status_filter == 'owned': filtered = [a for a in all_albums if a['in_library']] else: filtered = all_albums filtered_total = len(filtered) albums = filtered[offset:offset + limit] stats = { 'total': total, 'owned': owned_count, 'missing': total - owned_count, } return jsonify({ "success": True, "albums": albums, "total": filtered_total, "offset": offset, "limit": limit, "stats": stats, }) except Exception as e: logger.error(f"Error getting Spotify library: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/spotify-library/refresh', methods=['POST']) def refresh_spotify_library(): """Manually trigger a re-sync of the Spotify library cache""" try: def _run_sync(): try: from core.watchlist_scanner import get_watchlist_scanner scanner = get_watchlist_scanner(spotify_client) if scanner: # Force full sync by clearing last_sync timestamp database = get_database() database.set_metadata('spotify_library_last_sync', '') database.set_metadata('spotify_library_last_full_sync', '') scanner.sync_spotify_library_cache(profile_id=get_current_profile_id()) logger.info("Manual Spotify library refresh complete") except Exception as e: logger.error(f"Error in manual Spotify library refresh: {e}") import threading thread = threading.Thread(target=_run_sync, daemon=True) thread.start() return jsonify({"success": True, "message": "Spotify library refresh started"}) except Exception as e: logger.error(f"Error starting Spotify library refresh: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/recent-releases', methods=['GET']) def get_discover_recent_releases(): """Get cached recent albums from watchlist and similar artists""" try: database = get_database() # Determine active source active_source = _get_active_discovery_source() # Get cached recent albums filtered by source (max 20) albums = database.get_discovery_recent_albums(limit=20, source=active_source, profile_id=get_current_profile_id()) # Backfill missing cover art from metadata source for album in albums: if not album.get('album_cover_url'): cover = None album_id = album.get('album_deezer_id') or album.get('album_itunes_id') or album.get('album_spotify_id') try: # Try direct ID lookup first if album_id: fallback = _get_metadata_fallback_client() if fallback: album_data = fallback.get_album(str(album_id)) if album_data: imgs = album_data.get('images', []) cover = album_data.get('image_url') or (imgs[0].get('url') if imgs else None) # Fallback: search by name if not cover and album.get('album_name') and album.get('artist_name'): fallback = _get_metadata_fallback_client() if fallback: results = fallback.search_albums(f"{album['artist_name']} {album['album_name']}", limit=1) if results and hasattr(results[0], 'image_url') and results[0].image_url: cover = results[0].image_url album_id = str(results[0].id) if cover: album['album_cover_url'] = cover if album_id: try: database.update_discovery_recent_album_cover(album_id, cover) except Exception: pass except Exception: pass # Filter out blacklisted artists blacklisted = database.get_discovery_blacklist_names() if blacklisted: albums = [a for a in albums if a.get('artist_name', '').lower() not in blacklisted] return jsonify({"success": True, "albums": albums, "source": active_source}) except Exception as e: logger.error(f"Error getting recent releases: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/release-radar', methods=['GET']) def get_discover_release_radar(): """Get release radar playlist - curated selection that stays consistent until next update""" try: database = get_database() # Determine active source - release radar works with any source now active_source = _get_active_discovery_source() # Try source-specific playlist first, then fall back to generic pid = get_current_profile_id() curated_track_ids = database.get_curated_playlist(f'release_radar_{active_source}', profile_id=pid) if not curated_track_ids: curated_track_ids = database.get_curated_playlist('release_radar', profile_id=pid) if curated_track_ids: # Use curated selection - fetch track data from discovery pool filtered by source discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source, profile_id=pid) # Build lookup dict with source-appropriate IDs tracks_by_id = {} for track in discovery_tracks: if active_source == 'spotify' and track.spotify_track_id: tracks_by_id[track.spotify_track_id] = track elif active_source == 'deezer' and getattr(track, 'deezer_track_id', None): tracks_by_id[track.deezer_track_id] = track elif active_source == 'itunes' and track.itunes_track_id: tracks_by_id[track.itunes_track_id] = track selected_tracks = [] for track_id in curated_track_ids: if track_id in tracks_by_id: track = tracks_by_id[track_id] # Parse track_data_json if it's a string track_data = track.track_data_json if isinstance(track_data, str): try: track_data = json.loads(track_data) except: track_data = None selected_tracks.append({ "track_id": track.spotify_track_id or getattr(track, 'deezer_track_id', None) or track.itunes_track_id, "spotify_track_id": track.spotify_track_id, "itunes_track_id": track.itunes_track_id, "deezer_track_id": getattr(track, 'deezer_track_id', None), "track_name": track.track_name, "artist_name": track.artist_name, "album_name": track.album_name, "album_cover_url": track.album_cover_url, "duration_ms": track.duration_ms, "track_data_json": track_data, "source": track.source }) return jsonify({"success": True, "tracks": selected_tracks, "source": active_source}) # Fallback: no curated playlist exists (shouldn't happen after first scan) return jsonify({"success": True, "tracks": [], "source": active_source}) except Exception as e: logger.error(f"Error getting release radar: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/because-you-listen-to', methods=['GET']) def get_discover_because_you_listen_to(): """Get 'Because You Listen To' sections — personalized by top played artists.""" try: database = get_database() active_source = _get_active_discovery_source() pid = get_current_profile_id() # Fetch pool tracks once for all sections pool_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source, profile_id=pid) tracks_by_id = {} for t in pool_tracks: if active_source == 'spotify' and t.spotify_track_id: tracks_by_id[t.spotify_track_id] = t elif active_source == 'itunes' and t.itunes_track_id: tracks_by_id[t.itunes_track_id] = t elif active_source == 'deezer' and getattr(t, 'deezer_track_id', None): tracks_by_id[t.deezer_track_id] = t sections = [] for i in range(3): artist_name = database.get_metadata(f'bylt_artist_{i}') if not artist_name: continue track_ids = database.get_curated_playlist(f'because_you_listen_to_{i}', profile_id=pid) if not track_ids: continue tracks = [] for tid in track_ids: t = tracks_by_id.get(tid) if t: tracks.append({ 'id': tid, 'name': t.track_name, 'artist': t.artist_name, 'album': t.album_name, 'image_url': t.album_cover_url, 'duration_ms': t.duration_ms, 'popularity': t.popularity, }) if tracks: # Get artist image artist_image = None try: conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT thumb_url FROM artists WHERE LOWER(name) = LOWER(?) LIMIT 1", (artist_name,)) row = cursor.fetchone() if row and row[0]: artist_image = fix_artist_image_url(row[0]) conn.close() except Exception: pass sections.append({ 'artist_name': artist_name, 'artist_image': artist_image, 'tracks': tracks, }) return jsonify({'success': True, 'sections': sections}) except Exception as e: logger.error(f"Error getting BYLT: {e}") return jsonify({'success': True, 'sections': []}) @app.route('/api/discover/undiscovered-albums', methods=['GET']) def get_discover_undiscovered_albums(): """Albums by artists you listen to that aren't in your library — from cache.""" try: database = get_database() cache = get_metadata_cache() active_source = _get_active_discovery_source() # Get top played artists top = database.get_top_artists('all', 25) artist_names = [a['name'] for a in top if a.get('name')] if not artist_names: return jsonify({'success': True, 'albums': []}) # Build library album keys for exclusion with database._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" SELECT LOWER(al.title), LOWER(ar.name) FROM albums al JOIN artists ar ON ar.id = al.artist_id """) library_keys = {(r[0].strip(), r[1].strip()) for r in cursor.fetchall()} albums = cache.get_undiscovered_albums(artist_names, library_keys, source=active_source, limit=20) return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Undiscovered albums endpoint error: {e}") return jsonify({'success': True, 'albums': []}) @app.route('/api/discover/genre-new-releases', methods=['GET']) def get_discover_genre_new_releases(): """Recent releases matching your top genres — from cache.""" try: database = get_database() cache = get_metadata_cache() genres = database.get_genre_breakdown('all') genre_names = [g['genre'] for g in (genres or [])[:10] if g.get('genre')] if not genre_names: return jsonify({'success': True, 'albums': []}) allowed = _get_genre_allowed_sources() albums = cache.get_genre_new_releases(genre_names, sources=allowed, limit=20) return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Genre new releases endpoint error: {e}") return jsonify({'success': True, 'albums': []}) @app.route('/api/discover/label-explorer', methods=['GET']) def get_discover_label_explorer(): """Popular albums from labels in your library — from cache.""" try: database = get_database() cache = get_metadata_cache() with database._get_connection() as conn: cursor = conn.cursor() cursor.execute(""" SELECT DISTINCT label FROM albums WHERE label IS NOT NULL AND label != '' LIMIT 30 """) labels = {r[0] for r in cursor.fetchall()} active_source = _get_active_discovery_source() if not labels: return jsonify({'success': True, 'albums': [], 'labels': []}) albums = cache.get_label_explorer(labels, source=active_source, limit=20) return jsonify({'success': True, 'albums': albums, 'labels': sorted(labels)}) except Exception as e: logger.error(f"Label explorer endpoint error: {e}") return jsonify({'success': True, 'albums': [], 'labels': []}) @app.route('/api/discover/deep-cuts', methods=['GET']) def get_discover_deep_cuts(): """Low-popularity tracks from artists you listen to — from cache.""" try: database = get_database() cache = get_metadata_cache() top = database.get_top_artists('all', 15) artist_names = [a['name'] for a in top if a.get('name')] active_source = _get_active_discovery_source() if not artist_names: return jsonify({'success': True, 'tracks': []}) tracks = cache.get_deep_cuts(artist_names, source=active_source, popularity_cap=30, limit=20) return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Deep cuts endpoint error: {e}") return jsonify({'success': True, 'tracks': []}) def _get_genre_allowed_sources(): """Get allowed metadata sources for genre features. Spotify authed → ['spotify', 'itunes', 'deezer'] Not authed → ['itunes', 'deezer']""" sources = ['itunes', 'deezer'] if spotify_client and spotify_client.is_spotify_authenticated(): sources.append('spotify') return sources @app.route('/api/discover/genre-explorer', methods=['GET']) def get_discover_genre_explorer(): """Genre landscape from cached artists — highlights unexplored genres.""" try: database = get_database() cache = get_metadata_cache() genres = database.get_genre_breakdown('all') user_genres = {g['genre'] for g in (genres or []) if g.get('genre')} allowed = _get_genre_allowed_sources() data = cache.get_genre_explorer(user_genres, sources=allowed) return jsonify({'success': True, 'genres': data}) except Exception as e: logger.error(f"Genre explorer endpoint error: {e}") return jsonify({'success': True, 'genres': []}) @app.route('/api/discover/genre-deep-dive', methods=['GET']) def get_discover_genre_deep_dive(): """Get artists + albums for a genre — from cache.""" try: genre = request.args.get('genre', '').strip() if not genre: return jsonify({'success': False, 'error': 'genre required'}), 400 cache = get_metadata_cache() allowed = _get_genre_allowed_sources() data = cache.get_genre_deep_dive(genre, sources=allowed) return jsonify({'success': True, **data}) except Exception as e: logger.error(f"Genre albums endpoint error: {e}") return jsonify({'success': True, 'albums': []}) @app.route('/api/discover/resolve-cache-album', methods=['GET']) def resolve_cache_album(): """Look up a real album entity in the cache by name+artist (avoids playlist ID confusion).""" try: name = request.args.get('name', '').strip() artist = request.args.get('artist', '').strip() if not name or not artist: return jsonify({'success': False, 'error': 'name and artist required'}), 400 active_source = _get_active_discovery_source() database = get_database() with database._get_connection() as conn: cursor = conn.cursor() # Strategy 1: exact match, prefer active source cursor.execute(""" SELECT entity_id, source FROM metadata_cache_entities WHERE entity_type = 'album' AND name COLLATE NOCASE = ? COLLATE NOCASE AND artist_name COLLATE NOCASE = ? COLLATE NOCASE ORDER BY CASE WHEN source = ? THEN 0 ELSE 1 END LIMIT 1 """, (name, artist, active_source)) row = cursor.fetchone() if row: return jsonify({'success': True, 'entity_id': row['entity_id'], 'source': row['source']}) # Strategy 2: partial match (handles "Album - Single" vs "Album" naming) cursor.execute(""" SELECT entity_id, source FROM metadata_cache_entities WHERE entity_type = 'album' AND name COLLATE NOCASE LIKE ? COLLATE NOCASE AND artist_name COLLATE NOCASE LIKE ? COLLATE NOCASE ORDER BY CASE WHEN source = ? THEN 0 ELSE 1 END LIMIT 1 """, (f'%{name}%', f'%{artist}%', active_source)) row = cursor.fetchone() if row: return jsonify({'success': True, 'entity_id': row['entity_id'], 'source': row['source']}) # Strategy 3: not in cache — try searching the fallback client directly fallback = _get_metadata_fallback_client() if fallback: try: results = fallback.search_albums(f"{artist} {name}", limit=3) if results: r = results[0] return jsonify({'success': True, 'entity_id': str(r.id), 'source': _get_metadata_fallback_source()}) except Exception: pass return jsonify({'success': False, 'error': 'Album not found in cache'}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/discover/weekly', methods=['GET']) def get_discover_weekly(): """Get discovery weekly playlist - curated selection that stays consistent until next update""" try: database = get_database() # Determine active source active_source = _get_active_discovery_source() # Try source-specific playlist first, then fall back to generic pid = get_current_profile_id() curated_track_ids = database.get_curated_playlist(f'discovery_weekly_{active_source}', profile_id=pid) if not curated_track_ids: curated_track_ids = database.get_curated_playlist('discovery_weekly', profile_id=pid) if curated_track_ids: # Use curated selection - fetch track data from discovery pool filtered by source discovery_tracks = database.get_discovery_pool_tracks(limit=5000, new_releases_only=False, source=active_source, profile_id=pid) # Build lookup dict with source-appropriate IDs tracks_by_id = {} for track in discovery_tracks: if active_source == 'spotify' and track.spotify_track_id: tracks_by_id[track.spotify_track_id] = track elif active_source == 'deezer' and getattr(track, 'deezer_track_id', None): tracks_by_id[track.deezer_track_id] = track elif active_source == 'itunes' and track.itunes_track_id: tracks_by_id[track.itunes_track_id] = track selected_tracks = [] for track_id in curated_track_ids: if track_id in tracks_by_id: track = tracks_by_id[track_id] # Parse track_data_json if it's a string track_data = track.track_data_json if isinstance(track_data, str): try: track_data = json.loads(track_data) except: track_data = None selected_tracks.append({ "track_id": track.spotify_track_id or getattr(track, 'deezer_track_id', None) or track.itunes_track_id, "spotify_track_id": track.spotify_track_id, "itunes_track_id": track.itunes_track_id, "deezer_track_id": getattr(track, 'deezer_track_id', None), "track_name": track.track_name, "artist_name": track.artist_name, "album_name": track.album_name, "album_cover_url": track.album_cover_url, "duration_ms": track.duration_ms, "track_data_json": track_data, "source": track.source }) return jsonify({"success": True, "tracks": selected_tracks, "source": active_source}) # Fallback: no curated playlist exists (shouldn't happen after first scan) return jsonify({"success": True, "tracks": [], "source": active_source}) except Exception as e: logger.error(f"Error getting discovery weekly: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/refresh', methods=['POST']) def refresh_discover_data(): """ Force refresh discover page data (recent albums cache and curated playlists). Useful for initial setup or when data appears stale. """ try: from core.watchlist_scanner import WatchlistScanner database = get_database() scanner = WatchlistScanner(spotify_client, database) logger.info("[Discover Refresh] Starting forced refresh of discover data...") refresh_pid = get_current_profile_id() # Cache recent albums from watchlist and similar artists logger.info("[Discover Refresh] Caching recent albums...") scanner.cache_discovery_recent_albums(profile_id=refresh_pid) # Curate playlists logger.info("[Discover Refresh] Curating discovery playlists...") scanner.curate_discovery_playlists(profile_id=refresh_pid) # Get counts for response active_source = _get_active_discovery_source() pid = get_current_profile_id() recent_albums = database.get_discovery_recent_albums(limit=100, source=active_source, profile_id=pid) release_radar = database.get_curated_playlist(f'release_radar_{active_source}', profile_id=pid) or [] discovery_weekly = database.get_curated_playlist(f'discovery_weekly_{active_source}', profile_id=pid) or [] logger.info(f"[Discover Refresh] Complete! Recent albums: {len(recent_albums)}, Release Radar: {len(release_radar)} tracks, Discovery Weekly: {len(discovery_weekly)} tracks") return jsonify({ "success": True, "message": "Discover data refreshed", "source": active_source, "recent_albums_count": len(recent_albums), "release_radar_tracks": len(release_radar), "discovery_weekly_tracks": len(discovery_weekly) }) except Exception as e: logger.error(f"Error refreshing discover data: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/diagnose', methods=['GET']) def diagnose_discover_data(): """ Diagnostic endpoint to check the state of discover data. Returns counts of similar artists, discovery pool, recent albums, etc. """ try: database = get_database() active_source = _get_active_discovery_source() pid = get_current_profile_id() with database._get_connection() as conn: cursor = conn.cursor() # Similar artists stats cursor.execute("SELECT COUNT(*) as total FROM similar_artists WHERE profile_id = ?", (pid,)) total_similar = cursor.fetchone()['total'] cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_itunes_id IS NOT NULL AND profile_id = ?", (pid,)) similar_with_itunes = cursor.fetchone()['count'] cursor.execute("SELECT COUNT(*) as count FROM similar_artists WHERE similar_artist_spotify_id IS NOT NULL AND profile_id = ?", (pid,)) similar_with_spotify = cursor.fetchone()['count'] # Discovery pool stats cursor.execute("SELECT source, COUNT(*) as count FROM discovery_pool WHERE profile_id = ? GROUP BY source", (pid,)) pool_by_source = {row['source']: row['count'] for row in cursor.fetchall()} # Recent albums stats cursor.execute("SELECT source, COUNT(*) as count FROM discovery_recent_albums WHERE profile_id = ? GROUP BY source", (pid,)) albums_by_source = {row['source']: row['count'] for row in cursor.fetchall()} # Curated playlists cursor.execute("SELECT playlist_type, track_ids_json FROM discovery_curated_playlists WHERE profile_id = ?", (pid,)) playlists = {} for row in cursor.fetchall(): import json track_ids = json.loads(row['track_ids_json']) if row['track_ids_json'] else [] playlists[row['playlist_type']] = len(track_ids) # Watchlist artists cursor.execute("SELECT COUNT(*) as total FROM watchlist_artists WHERE profile_id = ?", (pid,)) total_watchlist = cursor.fetchone()['total'] cursor.execute("SELECT COUNT(*) as count FROM watchlist_artists WHERE itunes_artist_id IS NOT NULL AND profile_id = ?", (pid,)) watchlist_with_itunes = cursor.fetchone()['count'] return jsonify({ "success": True, "active_source": active_source, "similar_artists": { "total": total_similar, "with_itunes_id": similar_with_itunes, "with_spotify_id": similar_with_spotify }, "discovery_pool": pool_by_source, "recent_albums": albums_by_source, "curated_playlists": playlists, "watchlist_artists": { "total": total_watchlist, "with_itunes_id": watchlist_with_itunes } }) except Exception as e: logger.error(f"Error diagnosing discover data: {e}") return jsonify({"success": False, "error": str(e)}), 500 # ======================================== # SEASONAL DISCOVERY ENDPOINTS # ======================================== @app.route('/api/discover/seasonal/current', methods=['GET']) def get_current_seasonal_content(): """Auto-detect and return current season's content""" try: from core.seasonal_discovery import get_seasonal_discovery_service database = get_database() seasonal_service = get_seasonal_discovery_service(spotify_client, database) # Get current season current_season = seasonal_service.get_current_season() if not current_season: return jsonify({"success": True, "season": None, "albums": [], "playlist_available": False}) # Get seasonal config from core.seasonal_discovery import SEASONAL_CONFIG config = SEASONAL_CONFIG[current_season] # Get albums for active source (increased limit for more variety) active_source = _get_active_discovery_source() albums = seasonal_service.get_seasonal_albums(current_season, limit=40, source=active_source) # Check if playlist is curated for active source playlist_track_ids = seasonal_service.get_curated_seasonal_playlist(current_season, source=active_source) return jsonify({ "success": True, "season": current_season, "name": config['name'], "description": config['description'], "icon": config['icon'], "albums": albums, "playlist_available": len(playlist_track_ids) > 0 }) except Exception as e: logger.error(f"Error getting current seasonal content: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/seasonal//albums', methods=['GET']) def get_seasonal_albums(season_key): """Get albums for a specific season""" try: from core.seasonal_discovery import get_seasonal_discovery_service, SEASONAL_CONFIG if season_key not in SEASONAL_CONFIG: return jsonify({"success": False, "error": "Invalid season"}), 400 database = get_database() seasonal_service = get_seasonal_discovery_service(spotify_client, database) active_source = _get_active_discovery_source() albums = seasonal_service.get_seasonal_albums(season_key, limit=40, source=active_source) config = SEASONAL_CONFIG[season_key] return jsonify({ "success": True, "season": season_key, "name": config['name'], "description": config['description'], "icon": config['icon'], "albums": albums }) except Exception as e: logger.error(f"Error getting seasonal albums: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/seasonal//playlist', methods=['GET']) def get_seasonal_playlist(season_key): """Get curated playlist for a specific season""" try: from core.seasonal_discovery import get_seasonal_discovery_service, SEASONAL_CONFIG if season_key not in SEASONAL_CONFIG: return jsonify({"success": False, "error": "Invalid season"}), 400 database = get_database() seasonal_service = get_seasonal_discovery_service(spotify_client, database) # Get curated track IDs for active source active_source = _get_active_discovery_source() track_ids = seasonal_service.get_curated_seasonal_playlist(season_key, source=active_source) if not track_ids: return jsonify({"success": True, "tracks": []}) # Use source-appropriate ID column for lookups track_id_col = 'spotify_track_id' if active_source == 'spotify' else 'itunes_track_id' # Fetch track details from seasonal tracks or discovery pool (filtered by source) tracks = [] with database._get_connection() as conn: cursor = conn.cursor() for track_id in track_ids: # Try seasonal_tracks first (filtered by source) cursor.execute(""" SELECT spotify_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, track_data_json FROM seasonal_tracks WHERE spotify_track_id = ? AND source = ? """, (track_id, active_source)) result = cursor.fetchone() if result: track_dict = dict(result) # Parse track_data_json if available if track_dict.get('track_data_json'): try: import json track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) except: pass tracks.append(track_dict) else: # Try discovery_pool as fallback (filtered by source) cursor.execute(f""" SELECT {track_id_col} as spotify_track_id, track_name, artist_name, album_name, album_cover_url, duration_ms, popularity, track_data_json FROM discovery_pool WHERE {track_id_col} = ? AND source = ? """, (track_id, active_source)) result = cursor.fetchone() if result: track_dict = dict(result) # Parse track_data_json if available if track_dict.get('track_data_json'): try: import json track_dict['track_data_json'] = json.loads(track_dict['track_data_json']) except: pass tracks.append(track_dict) config = SEASONAL_CONFIG[season_key] return jsonify({ "success": True, "season": season_key, "name": config['name'], "description": config['description'], "icon": config['icon'], "tracks": tracks }) except Exception as e: logger.error(f"Error getting seasonal playlist: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/seasonal/refresh', methods=['POST']) def refresh_seasonal_content(): """Manually trigger seasonal content refresh (admin function)""" try: from core.seasonal_discovery import get_seasonal_discovery_service database = get_database() seasonal_service = get_seasonal_discovery_service(spotify_client, database) # Force populate current season in background thread (bypass 7-day threshold) import threading def populate_all(): try: current_season = seasonal_service.get_current_season() if current_season: logger.info(f"Force-refreshing seasonal content for: {current_season}") seasonal_service.populate_seasonal_content(current_season) seasonal_service.curate_seasonal_playlist(current_season) logger.info(f"Seasonal content refreshed for: {current_season}") else: logger.warning("ℹ️ No active season to refresh") except Exception as e: logger.error(f"Error in background seasonal population: {e}") thread = threading.Thread(target=populate_all, daemon=True) thread.start() return jsonify({"success": True, "message": "Seasonal content refresh started"}) except Exception as e: logger.error(f"Error refreshing seasonal content: {e}") return jsonify({"success": False, "error": str(e)}), 500 # ======================================== # PERSONALIZED PLAYLISTS ENDPOINTS # ======================================== @app.route('/api/discover/personalized/recently-added', methods=['GET']) def get_recently_added_playlist(): """Get recently added tracks from library""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_recently_added(limit=50) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting recently added playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/top-tracks', methods=['GET']) def get_top_tracks_playlist(): """Get user's all-time top tracks""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_top_tracks(limit=50) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting top tracks playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/forgotten-favorites', methods=['GET']) def get_forgotten_favorites_playlist(): """Get forgotten favorites - tracks you loved but haven't played recently""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_forgotten_favorites(limit=50) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting forgotten favorites playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/decade/', methods=['GET']) def get_decade_playlist(decade): """Get tracks from a specific decade""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_decade_playlist(decade, limit=100) return jsonify({ "success": True, "decade": decade, "tracks": tracks }) except Exception as e: logger.error(f"Error getting decade playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/popular-picks', methods=['GET']) def get_popular_picks_playlist(): """Get high popularity tracks from discovery pool""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_popular_picks(limit=50) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting popular picks playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/hidden-gems', methods=['GET']) def get_hidden_gems_playlist(): """Get hidden gems (low popularity) from discovery pool""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_hidden_gems(limit=50) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting hidden gems playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/daily-mixes', methods=['GET']) def get_daily_mixes(): """Get all Daily Mix playlists (hybrid library + discovery)""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) mixes = service.get_all_daily_mixes(max_mixes=4) return jsonify({ "success": True, "mixes": mixes }) except Exception as e: logger.error(f"Error getting daily mixes: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/discovery-shuffle', methods=['GET']) def get_discovery_shuffle(): """Get Discovery Shuffle playlist - random tracks from discovery pool""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) limit = int(request.args.get('limit', 50)) tracks = service.get_discovery_shuffle(limit=limit) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting discovery shuffle playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/personalized/familiar-favorites', methods=['GET']) def get_familiar_favorites(): """Get Familiar Favorites playlist - reliable go-to tracks""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) limit = int(request.args.get('limit', 50)) tracks = service.get_familiar_favorites(limit=limit) return jsonify({ "success": True, "tracks": tracks }) except Exception as e: logger.error(f"Error getting familiar favorites playlist: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-blacklist', methods=['GET']) def get_discovery_artist_blacklist(): """Get all blacklisted discovery artists.""" try: database = get_database() entries = database.get_discovery_blacklist() return jsonify({"success": True, "entries": entries}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-blacklist', methods=['POST']) def add_discovery_artist_blacklist(): """Block an artist from appearing in discovery results.""" try: data = request.get_json() or {} artist_name = data.get('artist_name', '').strip() if not artist_name: return jsonify({"success": False, "error": "artist_name is required"}), 400 database = get_database() success = database.add_to_discovery_blacklist( artist_name=artist_name, spotify_id=data.get('spotify_artist_id'), itunes_id=data.get('itunes_artist_id'), deezer_id=data.get('deezer_artist_id'), ) if success: logger.info(f"Blocked artist from discovery: {artist_name}") return jsonify({"success": success}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-blacklist/', methods=['DELETE']) def remove_discovery_artist_blacklist(blacklist_id): """Unblock an artist from discovery.""" try: database = get_database() success = database.remove_from_discovery_blacklist(blacklist_id) return jsonify({"success": success}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 # ── Your Artists (Liked Artists Pool) ── @app.route('/api/discover/your-artists', methods=['GET']) def get_your_artists(): """Get liked artists for the Discover carousel (20 random matched on active source).""" try: database = get_database() profile_id = get_current_profile_id() # Determine active source column — only show artists with THIS source's ID active_source = 'spotify' if spotify_client and spotify_client.is_spotify_authenticated(): active_source = 'spotify' else: fb = _get_metadata_fallback_source() if fb: active_source = fb active_col = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'}.get(active_source, 'spotify_artist_id') # Check if refresh needed (>24h stale or empty) last_fetch = database.get_liked_artists_last_fetch(profile_id) stale = True if last_fetch: from datetime import datetime, timedelta try: if isinstance(last_fetch, str): last_dt = datetime.fromisoformat(last_fetch.replace('Z', '+00:00')) else: last_dt = last_fetch stale = (datetime.now() - last_dt.replace(tzinfo=None)) > timedelta(hours=24) except Exception: stale = True if stale: _trigger_your_artists_refresh(profile_id) database.sync_liked_artists_watchlist_flags(profile_id) # Only return artists matched to the active source result = database.get_liked_artists( profile_id=profile_id, limit=20, random=True, matched_only=True, require_source_id=active_col ) result['stale'] = stale result['success'] = True result['active_source'] = active_source return jsonify(result) except Exception as e: logger.error(f"Error getting your artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/your-artists/all', methods=['GET']) def get_your_artists_all(): """Get all liked artists for the View All modal (paginated).""" try: database = get_database() profile_id = get_current_profile_id() page = int(request.args.get('page', 1)) per_page = int(request.args.get('per_page', 50)) search = request.args.get('search', '').strip() source_filter = request.args.get('source', '').strip() sort = request.args.get('sort', 'name') # Same active source filtering as carousel active_source = 'spotify' if spotify_client and spotify_client.is_spotify_authenticated(): active_source = 'spotify' else: fb = _get_metadata_fallback_source() if fb: active_source = fb active_col = {'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id'}.get(active_source, 'spotify_artist_id') database.sync_liked_artists_watchlist_flags(profile_id) result = database.get_liked_artists( profile_id=profile_id, matched_only=True, page=page, per_page=per_page, search=search, source_filter=source_filter or None, sort=sort, require_source_id=active_col ) result['success'] = True result['active_source'] = active_source return jsonify(result) except Exception as e: logger.error(f"Error getting all your artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/your-artists/refresh', methods=['POST']) def refresh_your_artists(): """Force-trigger a fetch + match cycle for liked artists. ?clear=true wipes pool first.""" try: profile_id = get_current_profile_id() if request.args.get('clear', '').lower() == 'true': database = get_database() cleared = database.clear_liked_artists(profile_id) logger.info(f"[Your Artists] Cleared {cleared} entries before refresh") _trigger_your_artists_refresh(profile_id) return jsonify({"success": True, "message": "Refresh started"}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/your-artists/sources', methods=['GET']) def get_your_artists_sources(): """Return current source config + which services are connected.""" try: enabled_raw = config_manager.get('discover.your_artists_sources', 'spotify,tidal,lastfm,deezer') enabled = [s.strip() for s in enabled_raw.split(',') if s.strip()] connected = [] # Spotify if spotify_client and spotify_client.is_spotify_authenticated(): connected.append('spotify') # Tidal try: if tidal_client and hasattr(tidal_client, '_ensure_valid_token') and tidal_client._ensure_valid_token(): connected.append('tidal') except Exception: pass # Last.fm if config_manager.get('lastfm.api_key', '') and config_manager.get('lastfm.session_key', ''): connected.append('lastfm') # Deezer — OAuth token OR ARL token both count as connected try: deezer_cl = _get_deezer_client() deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated() deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl and soulseek_client.deezer_dl.is_authenticated()) if deezer_oauth or deezer_arl: connected.append('deezer') except Exception: pass return jsonify({"success": True, "enabled": enabled, "connected": connected}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 _your_artists_refresh_lock = threading.Lock() _your_artists_refreshing = False def _trigger_your_artists_refresh(profile_id: int): """Start background fetch + match if not already running.""" global _your_artists_refreshing if _your_artists_refreshing: return with _your_artists_refresh_lock: if _your_artists_refreshing: return _your_artists_refreshing = True def _run(): global _your_artists_refreshing try: _fetch_and_match_liked_artists(profile_id) except Exception as e: logger.error(f"Your artists refresh failed: {e}") import traceback traceback.print_exc() finally: _your_artists_refreshing = False threading.Thread(target=_run, daemon=True, name="YourArtistsRefresh").start() def _fetch_and_match_liked_artists(profile_id: int): """Background worker: fetch from services, deduplicate, match to active source.""" database = get_database() fetched = 0 enabled_raw = config_manager.get('discover.your_artists_sources', 'spotify,tidal,lastfm,deezer') enabled_sources = {s.strip() for s in enabled_raw.split(',') if s.strip()} # 1. Fetch from Spotify (followed artists) try: if 'spotify' not in enabled_sources: logger.warning("[Your Artists] Spotify skipped (disabled in sources config)") elif spotify_client and spotify_client.is_spotify_authenticated(): logger.info("[Your Artists] Fetching followed artists from Spotify...") artists = spotify_client.get_followed_artists() for a in artists: database.upsert_liked_artist( artist_name=a['name'], source_service='spotify', source_id=a['spotify_id'], source_id_type='spotify', image_url=a.get('image_url'), genres=a.get('genres'), profile_id=profile_id ) fetched += len(artists) logger.info(f"[Your Artists] Fetched {len(artists)} from Spotify") except Exception as e: logger.error(f"[Your Artists] Spotify fetch error: {e}") # 2. Fetch from Tidal (favorite artists) try: if 'tidal' not in enabled_sources: logger.warning("[Your Artists] Tidal skipped (disabled in sources config)") elif tidal_client and hasattr(tidal_client, 'get_favorite_artists'): tidal_auth = tidal_client._ensure_valid_token() if hasattr(tidal_client, '_ensure_valid_token') else False if tidal_auth: logger.info("[Your Artists] Fetching favorite artists from Tidal...") artists = tidal_client.get_favorite_artists(limit=200) for a in artists: database.upsert_liked_artist( artist_name=a['name'], source_service='tidal', image_url=a.get('image_url'), profile_id=profile_id ) fetched += len(artists) logger.info(f"[Your Artists] Fetched {len(artists)} from Tidal") except Exception as e: logger.error(f"[Your Artists] Tidal fetch error: {e}") # 3. Fetch from Last.fm (top artists) try: if 'lastfm' not in enabled_sources: logger.warning("[Your Artists] Last.fm skipped (disabled in sources config)") else: lastfm_key = config_manager.get('lastfm.api_key', '') lastfm_secret = config_manager.get('lastfm.api_secret', '') lastfm_session = config_manager.get('lastfm.session_key', '') logger.info(f"[Your Artists] Last.fm credentials: key={'yes' if lastfm_key else 'NO'}, secret={'yes' if lastfm_secret else 'NO'}, session={'yes' if lastfm_session else 'NO'}") if lastfm_key and lastfm_secret and lastfm_session: from core.lastfm_client import LastFMClient lfm = LastFMClient(api_key=lastfm_key, api_secret=lastfm_secret, session_key=lastfm_session) username = lfm.get_authenticated_username() logger.info(f"[Your Artists] Last.fm username resolved: {username or 'NONE'}") if username: logger.info(f"[Your Artists] Fetching top artists from Last.fm ({username})...") artists = lfm.get_user_top_artists(username, period='overall', limit=200) for a in artists: database.upsert_liked_artist( artist_name=a['name'], source_service='lastfm', image_url=a.get('image_url'), profile_id=profile_id ) fetched += len(artists) logger.info(f"[Your Artists] Fetched {len(artists)} from Last.fm") except Exception as e: logger.error(f"[Your Artists] Last.fm fetch error: {e}") # 4. Fetch from Deezer (favorite artists — OAuth or ARL) try: if 'deezer' not in enabled_sources: logger.warning("[Your Artists] Deezer skipped (disabled in sources config)") else: deezer_cl = _get_deezer_client() artists = [] if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated(): logger.info("[Your Artists] Fetching favorite artists from Deezer (OAuth)...") artists = deezer_cl.get_user_favorite_artists(limit=200) elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl and soulseek_client.deezer_dl.is_authenticated()): logger.info("[Your Artists] Fetching favorite artists from Deezer (ARL)...") artists = soulseek_client.deezer_dl.get_user_favorite_artists(limit=200) for a in artists: database.upsert_liked_artist( artist_name=a['name'], source_service='deezer', source_id=a.get('deezer_id'), source_id_type='deezer', image_url=a.get('image_url'), profile_id=profile_id ) fetched += len(artists) if artists: logger.info(f"[Your Artists] Fetched {len(artists)} from Deezer") except Exception as e: logger.error(f"[Your Artists] Deezer fetch error: {e}") logger.info(f"[Your Artists] Total fetched: {fetched}") # 5. Match pending artists to active source _match_liked_artists_to_all_sources(database, profile_id) def _match_liked_artists_to_all_sources(database, profile_id: int): """Match pending liked artists to ALL metadata sources (Spotify, iTunes, Deezer, Discogs). Uses the same matching pattern as the watchlist scanner: DB-first, then API search with fuzzy name matching. Stores all resolved IDs so source switching works instantly.""" pending = database.get_liked_artists_pending_match(profile_id, limit=200) if not pending: return # Source → column mapping source_cols = { 'spotify': 'spotify_artist_id', 'itunes': 'itunes_artist_id', 'deezer': 'deezer_artist_id', 'discogs': 'discogs_artist_id', } id_cols = list(source_cols.values()) # Reject known placeholder images and local server paths _placeholder_hashes = {'2a96cbd8b46e442fc41c2b86b821562f'} def _valid_image(url): if not url or not url.strip(): return None if any(ph in url for ph in _placeholder_hashes): return None # Reject local media server paths (Plex/Jellyfin) — not loadable in browser if url.startswith('/') or url.startswith('\\'): return None if not url.startswith('http'): return None return url # Build search clients for each source from core.deezer_client import DeezerClient search_clients = {} if spotify_client and spotify_client.is_spotify_authenticated(): search_clients['spotify'] = spotify_client try: search_clients['itunes'] = _get_itunes_client() except Exception: pass try: search_clients['deezer'] = _get_deezer_client() except Exception: pass try: dc = _get_discogs_client() # Only use Discogs if token is configured from config.settings import config_manager as _cm if _cm.get('discogs.token', ''): search_clients['discogs'] = dc except Exception: pass # Reuse watchlist scanner's fuzzy matching logic from core.watchlist_scanner import WatchlistScanner _normalize = WatchlistScanner._normalize_artist_name def _best_match(results, artist_name): """Pick best match from search results using name similarity (same as watchlist scanner).""" if not results: return None # Exact normalized match for r in results: if _normalize(r.name) == _normalize(artist_name): return r # Fuzzy scoring best = None best_sim = 0 for r in results: # Simple normalized comparison n1 = _normalize(artist_name) n2 = _normalize(r.name) if n1 == n2: return r # Levenshtein-style similarity max_len = max(len(n1), len(n2)) if max_len == 0: continue distance = sum(1 for a, b in zip(n1, n2, strict=False) if a != b) + abs(len(n1) - len(n2)) sim = (max_len - distance) / max_len if sim > best_sim: best_sim = sim best = r if best and best_sim >= 0.85: return best return None api_calls = 0 matched = 0 for entry in pending: name = entry['artist_name'] pool_id = entry['id'] harvested_ids = {} best_image = None # Pre-load existing IDs from the entry itself for col in id_cols: if entry.get(col): harvested_ids[col] = entry[col] # --- DB STRATEGIES (free, no API calls) --- # 1. Library artists table try: conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT * FROM artists WHERE name = ? COLLATE NOCASE LIMIT 1", (name,)) row = cursor.fetchone() if row: r = dict(row) for col in id_cols: if r.get(col) and col not in harvested_ids: harvested_ids[col] = str(r[col]) if _valid_image(r.get('thumb_url')): best_image = r['thumb_url'] except Exception: pass # 2. Watchlist artists try: conn = database._get_connection() cursor = conn.cursor() cursor.execute( "SELECT * FROM watchlist_artists WHERE artist_name = ? COLLATE NOCASE AND profile_id = ? LIMIT 1", (name, profile_id) ) row = cursor.fetchone() if row: wl = dict(row) for col in id_cols: if wl.get(col) and col not in harvested_ids: harvested_ids[col] = str(wl[col]) if _valid_image(wl.get('image_url')) and not best_image: best_image = wl['image_url'] except Exception: pass # 3. Metadata cache (all sources) try: conn = database._get_connection() cursor = conn.cursor() cursor.execute( "SELECT entity_id, source, image_url FROM metadata_cache_entities WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE", (name,) ) for row in cursor.fetchall(): col = source_cols.get(row['source']) if col and col not in harvested_ids: harvested_ids[col] = row['entity_id'] if _valid_image(row['image_url']) and not best_image: best_image = row['image_url'] except Exception: pass # --- API STRATEGIES (search each missing source) --- # Same pattern as watchlist scanner's _backfill_missing_ids for source, col in source_cols.items(): if col in harvested_ids: continue # Already have this source's ID client = search_clients.get(source) if not client: continue if api_calls >= 200: # Hard cap per refresh cycle break try: results = client.search_artists(name, limit=5) best = _best_match(results, name) if best: harvested_ids[col] = best.id if hasattr(best, 'image_url') and _valid_image(best.image_url) and not best_image: best_image = best.image_url api_calls += 1 time.sleep(0.4) # Rate limit breathing room except Exception as e: logger.debug(f"[Your Artists] {source} search failed for '{name}': {e}") api_calls += 1 # Save all harvested IDs if harvested_ids: # Determine best active source/ID — prefer Spotify, then iTunes, Deezer, Discogs resolved_source = None resolved_id = None for src in ('spotify', 'itunes', 'deezer', 'discogs'): col = source_cols[src] if col in harvested_ids: resolved_source = src resolved_id = harvested_ids[col] break database.update_liked_artist_match( pool_id, active_source=resolved_source, active_source_id=resolved_id, image_url=best_image, all_ids=harvested_ids ) matched += 1 database.sync_liked_artists_watchlist_flags(profile_id) logger.info(f"[Your Artists] Matched {matched}/{len(pending)} artists to {len(search_clients)} sources ({api_calls} API calls)") # Image backfill: fetch images for matched artists that have IDs but no image _backfill_liked_artist_images(database, profile_id, search_clients) def _backfill_liked_artist_images(database, profile_id: int, search_clients: dict): """Fetch images for matched artists missing artwork using their stored source IDs.""" try: conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id, deezer_artist_id FROM liked_artists_pool WHERE profile_id = ? AND match_status = 'matched' AND (image_url IS NULL OR image_url = '' OR image_url LIKE '%2a96cbd8b46e442fc41c2b86b821562f%' OR image_url NOT LIKE 'http%') LIMIT 100 """, (profile_id,)) rows = cursor.fetchall() if not rows: return logger.info(f"[Your Artists] Backfilling images for {len(rows)} artists...") filled = 0 for row in rows: r = dict(row) image_url = None # Try Spotify artist lookup (has best images) if r.get('spotify_artist_id') and 'spotify' in search_clients: try: sp = search_clients['spotify'] if hasattr(sp, 'sp') and sp.sp: artist_data = sp.sp.artist(r['spotify_artist_id']) if artist_data and artist_data.get('images'): image_url = artist_data['images'][0]['url'] except Exception: pass # Try Deezer (direct image URL from ID) if not image_url and r.get('deezer_artist_id'): image_url = f"https://api.deezer.com/artist/{r['deezer_artist_id']}/image?size=big" if image_url: try: cursor2 = conn.cursor() cursor2.execute( "UPDATE liked_artists_pool SET image_url = ? WHERE id = ?", (image_url, r['id']) ) filled += 1 except Exception: pass time.sleep(0.3) conn.commit() if filled: logger.info(f"[Your Artists] Backfilled {filled}/{len(rows)} artist images") except Exception as e: logger.debug(f"[Your Artists] Image backfill error: {e}") # ── Your Albums (Liked Albums Pool) ── @app.route('/api/discover/your-albums', methods=['GET']) def get_your_albums(): """Get liked albums with library ownership status, paginated.""" try: database = get_database() profile_id = get_current_profile_id() page = request.args.get('page', 1, type=int) per_page = request.args.get('per_page', 48, type=int) search = request.args.get('search', '', type=str).strip() status_filter = request.args.get('status', 'all', type=str) source_filter = request.args.get('source', '', type=str).strip() sort = request.args.get('sort', 'artist_name', type=str) # Auto-trigger refresh if stale (>24h or empty) last_fetch = database.get_liked_albums_last_fetch(profile_id) stale = True if last_fetch: from datetime import datetime, timedelta try: if isinstance(last_fetch, str): last_dt = datetime.fromisoformat(last_fetch.replace('Z', '+00:00')) else: last_dt = last_fetch stale = (datetime.now() - last_dt.replace(tzinfo=None)) > timedelta(hours=24) except Exception: stale = True if stale: _trigger_your_albums_refresh(profile_id) # Fetch all (ownership check requires full set) all_result = database.get_liked_albums( profile_id=profile_id, page=1, per_page=100000, search=search, source_filter=source_filter or None, sort=sort ) all_albums = all_result['albums'] if not all_albums: return jsonify({ "success": True, "albums": [], "total": 0, "page": page, "per_page": per_page, "stale": stale, "stats": {"total": 0, "owned": 0, "missing": 0} }) # Ownership check — same strategy as Spotify library endpoint library_spotify_ids = database.get_library_spotify_album_ids(profile_id) library_album_names = database.get_library_album_names() owned_count = 0 for album in all_albums: if album.get('spotify_album_id') and album['spotify_album_id'] in library_spotify_ids: album['in_library'] = True elif (album['artist_name'].lower(), album['album_name'].lower()) in library_album_names: album['in_library'] = True else: album['in_library'] = False if album['in_library']: owned_count += 1 # Apply status filter if status_filter == 'missing': filtered = [a for a in all_albums if not a['in_library']] elif status_filter == 'owned': filtered = [a for a in all_albums if a['in_library']] else: filtered = all_albums filtered_total = len(filtered) offset = (page - 1) * per_page albums = filtered[offset:offset + per_page] stats = { 'total': all_result['total'], 'owned': owned_count, 'missing': all_result['total'] - owned_count, } return jsonify({ "success": True, "albums": albums, "total": filtered_total, "page": page, "per_page": per_page, "stale": stale, "stats": stats, }) except Exception as e: logger.error(f"Error getting your albums: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/your-albums/refresh', methods=['POST']) def refresh_your_albums(): """Force-trigger a fetch cycle for liked albums. ?clear=true wipes pool first.""" try: profile_id = get_current_profile_id() if request.args.get('clear', '').lower() == 'true': database = get_database() cleared = database.clear_liked_albums(profile_id) logger.info(f"[Your Albums] Cleared {cleared} entries before refresh") _trigger_your_albums_refresh(profile_id) return jsonify({"success": True, "message": "Refresh started"}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/your-albums/sources', methods=['GET']) def get_your_albums_sources(): """Return current source config + which services are connected (albums).""" try: enabled_raw = config_manager.get('discover.your_albums_sources', 'spotify,tidal,deezer') enabled = [s.strip() for s in enabled_raw.split(',') if s.strip()] connected = [] if spotify_client and spotify_client.is_spotify_authenticated(): connected.append('spotify') try: if tidal_client and hasattr(tidal_client, '_ensure_valid_token') and tidal_client._ensure_valid_token(): connected.append('tidal') except Exception: pass try: deezer_cl = _get_deezer_client() deezer_oauth = deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated() deezer_arl = (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl and soulseek_client.deezer_dl.is_authenticated()) if deezer_oauth or deezer_arl: connected.append('deezer') except Exception: pass return jsonify({"success": True, "enabled": enabled, "connected": connected}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 _your_albums_refresh_lock = threading.Lock() _your_albums_refreshing = False def _trigger_your_albums_refresh(profile_id: int): """Start background album fetch if not already running.""" global _your_albums_refreshing if _your_albums_refreshing: return with _your_albums_refresh_lock: if _your_albums_refreshing: return _your_albums_refreshing = True def _run(): global _your_albums_refreshing try: _fetch_liked_albums(profile_id) except Exception as e: logger.error(f"Your albums refresh failed: {e}") import traceback traceback.print_exc() finally: _your_albums_refreshing = False threading.Thread(target=_run, daemon=True, name="YourAlbumsRefresh").start() def _fetch_liked_albums(profile_id: int): """Background worker: fetch liked/saved albums from all connected services.""" database = get_database() fetched = 0 enabled_raw = config_manager.get('discover.your_albums_sources', 'spotify,tidal,deezer') enabled_sources = {s.strip() for s in enabled_raw.split(',') if s.strip()} # 1. Fetch from Spotify (saved albums) try: if 'spotify' not in enabled_sources: logger.warning("[Your Albums] Spotify skipped (disabled in sources config)") elif spotify_client and spotify_client.is_spotify_authenticated(): logger.info("[Your Albums] Fetching saved albums from Spotify...") albums = spotify_client.get_saved_albums() for a in albums: database.upsert_liked_album( album_name=a['album_name'], artist_name=a['artist_name'], source_service='spotify', source_id=a['spotify_album_id'], source_id_type='spotify', image_url=a.get('image_url'), release_date=a.get('release_date'), total_tracks=a.get('total_tracks', 0), profile_id=profile_id ) fetched += len(albums) logger.info(f"[Your Albums] Fetched {len(albums)} from Spotify") except Exception as e: logger.error(f"[Your Albums] Spotify fetch error: {e}") # 2. Fetch from Tidal (favorite albums) try: if 'tidal' not in enabled_sources: logger.warning("[Your Albums] Tidal skipped (disabled in sources config)") elif tidal_client and hasattr(tidal_client, 'get_favorite_albums'): tidal_auth = tidal_client._ensure_valid_token() if hasattr(tidal_client, '_ensure_valid_token') else False if tidal_auth: logger.info("[Your Albums] Fetching favorite albums from Tidal...") albums = tidal_client.get_favorite_albums(limit=500) for a in albums: database.upsert_liked_album( album_name=a['album_name'], artist_name=a['artist_name'], source_service='tidal', source_id=a.get('tidal_id'), source_id_type='tidal', image_url=a.get('image_url'), release_date=a.get('release_date'), total_tracks=a.get('total_tracks', 0), profile_id=profile_id ) fetched += len(albums) logger.info(f"[Your Albums] Fetched {len(albums)} from Tidal") except Exception as e: logger.error(f"[Your Albums] Tidal fetch error: {e}") # 3. Fetch from Deezer (favorite albums — OAuth or ARL) try: if 'deezer' not in enabled_sources: logger.warning("[Your Albums] Deezer skipped (disabled in sources config)") else: deezer_cl = _get_deezer_client() albums = [] if deezer_cl and hasattr(deezer_cl, 'is_user_authenticated') and deezer_cl.is_user_authenticated(): logger.info("[Your Albums] Fetching favorite albums from Deezer (OAuth)...") albums = deezer_cl.get_user_favorite_albums(limit=500) elif (hasattr(soulseek_client, 'deezer_dl') and soulseek_client.deezer_dl and soulseek_client.deezer_dl.is_authenticated()): logger.info("[Your Albums] Fetching favorite albums from Deezer (ARL)...") albums = soulseek_client.deezer_dl.get_user_favorite_albums(limit=500) for a in albums: database.upsert_liked_album( album_name=a['album_name'], artist_name=a['artist_name'], source_service='deezer', source_id=a.get('deezer_id'), source_id_type='deezer', image_url=a.get('image_url'), release_date=a.get('release_date'), total_tracks=a.get('total_tracks', 0), profile_id=profile_id ) fetched += len(albums) if albums: logger.info(f"[Your Albums] Fetched {len(albums)} from Deezer") except Exception as e: logger.error(f"[Your Albums] Deezer fetch error: {e}") logger.info(f"[Your Albums] Total fetched: {fetched}") @app.route('/api/discover/your-artists/info/', methods=['GET']) def get_your_artist_info(artist_id): """Get artist info for the Your Artists info modal. Checks library, cache, then API.""" try: artist_name = request.args.get('name', '') result = {'name': artist_name, 'success': True} # 1. Try library DB (has enrichment data) try: database = get_database() conn = database._get_connection() cursor = conn.cursor() # Check by various ID columns cursor.execute(""" SELECT * FROM artists WHERE id = ? OR spotify_artist_id = ? OR itunes_artist_id = ? OR deezer_id = ? OR discogs_id = ? LIMIT 1 """, (artist_id, artist_id, artist_id, artist_id, artist_id)) row = cursor.fetchone() if row: r = dict(row) result.update({ 'name': r.get('name', artist_name), 'genres': json.loads(r['genres']) if r.get('genres') else [], 'summary': r.get('summary', ''), 'image_url': r.get('thumb_url', ''), 'spotify_artist_id': r.get('spotify_artist_id'), 'musicbrainz_id': r.get('musicbrainz_id'), 'deezer_id': r.get('deezer_id'), 'itunes_artist_id': r.get('itunes_artist_id'), 'discogs_id': r.get('discogs_id'), 'lastfm_url': r.get('lastfm_url'), 'tidal_id': r.get('tidal_id'), 'lastfm_listeners': r.get('lastfm_listeners', 0), 'lastfm_playcount': r.get('lastfm_playcount', 0), }) return jsonify(result) except Exception: pass # 2. Try metadata cache try: conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT raw_json, image_url FROM metadata_cache_entities WHERE entity_type = 'artist' AND entity_id = ? LIMIT 1 """, (artist_id,)) row = cursor.fetchone() if row and row['raw_json']: cached = json.loads(row['raw_json']) result.update({ 'name': cached.get('name', artist_name), 'genres': cached.get('genres', []), 'image_url': row['image_url'] or cached.get('image_url', ''), 'popularity': cached.get('popularity', 0), 'followers': cached.get('followers', {}).get('total', 0) if isinstance(cached.get('followers'), dict) else cached.get('followers', 0), }) return jsonify(result) except Exception: pass # 3. Try Spotify API directly (genres, image, followers) try: if spotify_client and spotify_client.is_spotify_authenticated() and not artist_id.isdigit(): from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='artist') artist_data = spotify_client.sp.artist(artist_id) if artist_data: result.update({ 'name': artist_data.get('name', artist_name), 'genres': artist_data.get('genres', []), 'image_url': artist_data['images'][0]['url'] if artist_data.get('images') else '', 'spotify_artist_id': artist_data.get('id'), 'popularity': artist_data.get('popularity', 0), 'followers': artist_data.get('followers', {}).get('total', 0), }) except Exception as e: logger.debug(f"Spotify artist lookup failed for {artist_id}: {e}") # 4. Last.fm: bio, listeners, playcount (skip if name is too short/generic) try: _lfm_name = result.get('name') or artist_name if _lfm_name and len(_lfm_name) > 1 and lastfm_worker and lastfm_worker.client: lfm_info = lastfm_worker.client.get_artist_info(_lfm_name) if lfm_info: bio = lfm_info.get('bio', {}) if isinstance(bio, dict): summary = bio.get('summary', '') else: summary = str(bio) if bio else '' if summary and not result.get('summary'): result['summary'] = summary stats = lfm_info.get('stats', {}) if stats: result['lastfm_listeners'] = int(stats.get('listeners', 0)) result['lastfm_playcount'] = int(stats.get('playcount', 0)) if not result.get('genres'): tags = lfm_info.get('tags', {}).get('tag', []) if tags: result['genres'] = [t.get('name', '') for t in tags[:5] if isinstance(t, dict)] lfm_url = lfm_info.get('url') if lfm_url: result['lastfm_url'] = lfm_url except Exception as e: logger.debug(f"Last.fm artist info failed for {artist_name}: {e}") # 5. Return combined info return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/image-proxy', methods=['GET']) def image_proxy(): """Proxy external images to avoid CORS issues for canvas rendering.""" url = request.args.get('url', '') if not url or not url.startswith('http'): return '', 400 host = urlparse(url).hostname or '' allowed_hosts = [ 'i.scdn.co', 'mosaic.scdn.co', # Spotify 'lastfm.freetls.fastly.net', 'lastfm-img2.akamaized.net', # Last.fm 'e-cdns-images.dzcdn.net', 'cdns-images.dzcdn.net', 'api.deezer.com', # Deezer 'is1-ssl.mzstatic.com', 'is2-ssl.mzstatic.com', 'is3-ssl.mzstatic.com', 'is4-ssl.mzstatic.com', 'is5-ssl.mzstatic.com', # iTunes/Apple 'img.discogs.com', 'i.discogs.com', # Discogs 'localhost', '127.0.0.1', 'host.docker.internal', # Local/Docker media servers ] if not any(host == h or host.endswith('.' + h) for h in allowed_hosts) and not _is_internal_image_host(url): return '', 403 try: resp = requests.get(url, timeout=10, stream=True, headers={ 'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36', 'Referer': 'https://www.deezer.com/', }) if resp.status_code != 200: return '', resp.status_code content_type = resp.headers.get('Content-Type', 'image/jpeg') if not content_type.startswith('image/'): return '', 400 return Response( resp.content, content_type=content_type, headers={ 'Cache-Control': 'public, max-age=86400', 'Access-Control-Allow-Origin': '*', } ) except Exception: return '', 502 # Artist Map data cache — avoids re-querying 4+ tables on every request # Keys: 'watchlist_{profile}', 'genres_{profile}', 'genre_list' # Values: {'data': , 'ts': } _artist_map_cache = {} _ARTIST_MAP_CACHE_TTL = 300 # 5 minutes def _artmap_cache_get(key): """Get cached artist map data if still fresh.""" entry = _artist_map_cache.get(key) if entry and (time.time() - entry['ts']) < _ARTIST_MAP_CACHE_TTL: return entry['data'] return None def _artmap_cache_set(key, data): """Store artist map data in cache.""" _artist_map_cache[key] = {'data': data, 'ts': time.time()} def _artmap_cache_invalidate(profile_id=None): """Invalidate artist map cache (call after watchlist changes, scans, etc.).""" if profile_id: _artist_map_cache.pop(f'watchlist_{profile_id}', None) _artist_map_cache.pop(f'genres_{profile_id}', None) _artist_map_cache.pop('genre_list', None) @app.route('/api/discover/artist-map', methods=['GET']) def get_artist_map_data(): """Get watchlist artists + their similar artists for the force-directed artist map.""" try: database = get_database() profile_id = get_current_profile_id() cached = _artmap_cache_get(f'watchlist_{profile_id}') if cached: return jsonify(cached) # Get all watchlist artists conn = database._get_connection() cursor = conn.cursor() cursor.execute(""" SELECT id, artist_name, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id, image_url FROM watchlist_artists WHERE profile_id = ? """, (profile_id,)) watchlist_rows = cursor.fetchall() nodes = [] # {id, name, image_url, type: 'watchlist'|'similar', genres, size} edges = [] # {source, target, weight} seen_names = {} # normalized_name → node index def _norm(name): return (name or '').lower().strip() # Add watchlist artists as anchor nodes for wa in watchlist_rows: w = dict(wa) norm = _norm(w['artist_name']) if norm in seen_names: continue idx = len(nodes) seen_names[norm] = idx # Get image — prefer HTTP URLs img = w.get('image_url', '') or '' if img and not img.startswith('http'): img = '' nodes.append({ 'id': idx, 'name': w['artist_name'], 'image_url': img, 'type': 'watchlist', 'genres': [], 'spotify_id': w.get('spotify_artist_id') or '', 'itunes_id': w.get('itunes_artist_id') or '', 'deezer_id': w.get('deezer_artist_id') or '', 'discogs_id': w.get('discogs_artist_id') or '', 'source_db_id': str(w['id']), }) # Get all similar artists for all watchlist artists watchlist_ids = [dict(wa)['spotify_artist_id'] or dict(wa)['itunes_artist_id'] or str(dict(wa)['id']) for wa in watchlist_rows] if watchlist_ids: placeholders = ','.join(['?'] * len(watchlist_ids)) cursor.execute(f""" SELECT source_artist_id, similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, similarity_rank, occurrence_count, image_url, genres, popularity FROM similar_artists WHERE profile_id = ? AND source_artist_id IN ({placeholders}) ORDER BY similarity_rank ASC """, [profile_id] + watchlist_ids) for row in cursor.fetchall(): r = dict(row) sim_norm = _norm(r['similar_artist_name']) # Find or create similar artist node if sim_norm not in seen_names: idx = len(nodes) seen_names[sim_norm] = idx img = r.get('image_url', '') or '' if img and not img.startswith('http'): img = '' genres = [] if r.get('genres'): try: genres = json.loads(r['genres']) except Exception: pass nodes.append({ 'id': idx, 'name': r['similar_artist_name'], 'image_url': img, 'type': 'similar', 'genres': genres, 'spotify_id': r.get('similar_artist_spotify_id') or '', 'itunes_id': r.get('similar_artist_itunes_id') or '', 'deezer_id': r.get('similar_artist_deezer_id') or '', 'rank': r.get('similarity_rank', 5), 'occurrence': r.get('occurrence_count', 1), 'popularity': r.get('popularity', 0), }) sim_idx = seen_names[sim_norm] # Find the watchlist node that sourced this similar artist source_norm = None for wa in watchlist_rows: w = dict(wa) sid = w.get('spotify_artist_id') or w.get('itunes_artist_id') or str(w['id']) if sid == r['source_artist_id']: source_norm = _norm(w['artist_name']) break if source_norm and source_norm in seen_names: source_idx = seen_names[source_norm] # Weight: inverse of rank (rank 1 = strongest connection) weight = max(1, 11 - (r.get('similarity_rank', 5))) edges.append({ 'source': source_idx, 'target': sim_idx, 'weight': weight, }) # Also check if any similar artists ARE watchlist artists (cross-links) # These create extra connections between watchlist nodes for i, node in enumerate(nodes): if node['type'] == 'similar': # Check if this similar artist is also a watchlist artist for j, wnode in enumerate(nodes): if wnode['type'] == 'watchlist' and i != j: if _norm(node['name']) == _norm(wnode['name']): # Merge: upgrade the similar node to watchlist node['type'] = 'watchlist' break # ── Backfill from metadata cache: batch-lookup all node names across all sources ── # Single query to get ALL cached artist entries matching ANY node name try: all_names = list(set(_norm(n['name']) for n in nodes if n.get('name'))) if all_names: # Build case-insensitive IN clause via temp matching # Lightweight query — no raw_json (can be huge) cursor.execute(""" SELECT entity_id, source, name, image_url, genres, popularity FROM metadata_cache_entities WHERE entity_type = 'artist' """) cache_rows = cursor.fetchall() # Index cache by normalized name → {source: {id, image_url, genres}} cache_by_name = {} for cr in cache_rows: cn = _norm(cr['name'] or '') if cn not in cache_by_name: cache_by_name[cn] = {} source = cr['source'] genres = [] if cr['genres']: try: genres = json.loads(cr['genres']) if isinstance(cr['genres'], str) else [] except Exception: pass cache_by_name[cn][source] = { 'id': cr['entity_id'], 'image_url': cr['image_url'] or '', 'genres': genres, } # Apply cache data to nodes source_id_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} for n in nodes: nn = _norm(n['name']) cached = cache_by_name.get(nn) if not cached: continue for source, field in source_id_map.items(): if not n.get(field) and source in cached: n[field] = cached[source]['id'] # Backfill image if missing or local path if not n.get('image_url') or not n['image_url'].startswith('http'): for source in ('spotify', 'deezer', 'itunes'): if source in cached and cached[source].get('image_url', '').startswith('http'): n['image_url'] = cached[source]['image_url'] break # Backfill genres if missing if not n.get('genres') or len(n.get('genres', [])) == 0: for source in ('spotify', 'deezer', 'itunes', 'discogs'): if source in cached and cached[source].get('genres'): n['genres'] = cached[source]['genres'][:5] break # Deezer direct URL fallback for n in nodes: if not n.get('image_url') or not n['image_url'].startswith('http'): if n.get('deezer_id'): n['image_url'] = f"https://api.deezer.com/artist/{n['deezer_id']}/image?size=big" # Album art fallback (iTunes artists have no artist images) _album_art = {} try: cursor.execute(""" SELECT artist_name, image_url FROM metadata_cache_entities WHERE entity_type = 'album' AND image_url LIKE 'http%' AND artist_name IS NOT NULL AND artist_name != '' """) for r in cursor.fetchall(): an = _norm(r['artist_name']) if an and an not in _album_art: _album_art[an] = r['image_url'] except Exception: pass for n in nodes: if not n.get('image_url') or not n['image_url'].startswith('http'): nn = _norm(n['name']) if nn in _album_art: n['image_url'] = _album_art[nn] except Exception as cache_err: logger.debug(f"Artist map cache backfill error: {cache_err}") result = { 'success': True, 'nodes': nodes, 'edges': edges, 'watchlist_count': sum(1 for n in nodes if n['type'] == 'watchlist'), 'similar_count': sum(1 for n in nodes if n['type'] == 'similar'), } _artmap_cache_set(f'watchlist_{profile_id}', result) return jsonify(result) except Exception as e: logger.error(f"Error getting artist map data: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-map/genre-list', methods=['GET']) def get_artist_map_genre_list(): """Lightweight endpoint — just genre names + counts for the picker. No node data.""" try: cached = _artmap_cache_get('genre_list') if cached: return jsonify(cached) database = get_database() conn = database._get_connection() cursor = conn.cursor() # Fast query: just count artists per genre from cache genre_counts = {} cursor.execute(""" SELECT genres FROM metadata_cache_entities WHERE entity_type = 'artist' AND genres IS NOT NULL AND genres != '' AND genres != '[]' """) for r in cursor.fetchall(): try: for g in json.loads(r['genres']): if g and isinstance(g, str): gl = g.lower().strip() genre_counts[gl] = genre_counts.get(gl, 0) + 1 except Exception: pass # Sort by count descending sorted_genres = sorted(genre_counts.items(), key=lambda x: -x[1]) result = { 'success': True, 'genres': [{'name': g, 'count': c} for g, c in sorted_genres], 'total': len(sorted_genres) } _artmap_cache_set('genre_list', result) return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-map/genres', methods=['GET']) def get_artist_map_genres(): """Get ALL artists from every data source, grouped by genre for the genre map.""" try: database = get_database() profile_id = get_current_profile_id() cached = _artmap_cache_get(f'genres_{profile_id}') if cached: return jsonify(cached) conn = database._get_connection() cursor = conn.cursor() artists_by_name = {} # normalized_name → {name, image, genres[], sources, ids} def _norm(n): return (n or '').lower().strip() def _add(name, image_url=None, genres=None, spotify_id=None, itunes_id=None, deezer_id=None, discogs_id=None, source='unknown', popularity=0): n = _norm(name) if not n or len(n) < 2: return if n not in artists_by_name: artists_by_name[n] = { 'name': name, 'image_url': '', 'genres': set(), 'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': '', 'sources': set(), 'popularity': 0 } a = artists_by_name[n] if image_url and image_url.startswith('http') and not a['image_url']: a['image_url'] = image_url if genres: for g in (genres if isinstance(genres, list) else []): if g and isinstance(g, str): a['genres'].add(g.lower().strip()) if spotify_id and not a['spotify_id']: a['spotify_id'] = str(spotify_id) if itunes_id and not a['itunes_id']: a['itunes_id'] = str(itunes_id) if deezer_id and not a['deezer_id']: a['deezer_id'] = str(deezer_id) if discogs_id and not a['discogs_id']: a['discogs_id'] = str(discogs_id) if popularity > a['popularity']: a['popularity'] = popularity a['sources'].add(source) # 1. Metadata cache — biggest source cursor.execute(""" SELECT name, entity_id, source, image_url, genres, popularity FROM metadata_cache_entities WHERE entity_type = 'artist' """) for r in cursor.fetchall(): genres = [] if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} kwargs = {src_map.get(r['source'], 'spotify_id'): r['entity_id']} _add(r['name'], image_url=r['image_url'], genres=genres, source='cache', popularity=r['popularity'] or 0, **kwargs) # 2. Similar artists cursor.execute(""" SELECT similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, image_url, genres, popularity FROM similar_artists WHERE profile_id = ? """, (profile_id,)) for r in cursor.fetchall(): genres = [] if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass _add(r['similar_artist_name'], image_url=r['image_url'], genres=genres, spotify_id=r['similar_artist_spotify_id'], itunes_id=r['similar_artist_itunes_id'], deezer_id=r['similar_artist_deezer_id'], source='similar', popularity=r['popularity'] or 0) # 3. Watchlist artists cursor.execute(""" SELECT artist_name, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id, image_url FROM watchlist_artists WHERE profile_id = ? """, (profile_id,)) for r in cursor.fetchall(): _add(r['artist_name'], image_url=r['image_url'], spotify_id=r['spotify_artist_id'], itunes_id=r['itunes_artist_id'], deezer_id=r['deezer_artist_id'], discogs_id=r['discogs_artist_id'], source='watchlist') # 4. Library artists cursor.execute("SELECT name, thumb_url, genres FROM artists") for r in cursor.fetchall(): genres = [] if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass img = r['thumb_url'] if r['thumb_url'] and r['thumb_url'].startswith('http') else None _add(r['name'], image_url=img, genres=genres, source='library') # Filter: only include artists that have at least one genre genre_artists = {k: v for k, v in artists_by_name.items() if v['genres']} # Build genre → artists map genre_map = {} # genre_name → [artist_keys] for key, a in genre_artists.items(): for g in a['genres']: if g not in genre_map: genre_map[g] = [] genre_map[g].append(key) # Sort genres by artist count, take top genres sorted_genres = sorted(genre_map.items(), key=lambda x: -len(x[1])) # Build nodes nodes = [] node_idx = {} for key, a in genre_artists.items(): idx = len(nodes) node_idx[key] = idx nodes.append({ 'id': idx, 'name': a['name'], 'image_url': a['image_url'], 'genres': list(a['genres'])[:5], 'spotify_id': a['spotify_id'], 'itunes_id': a['itunes_id'], 'deezer_id': a['deezer_id'], 'discogs_id': a['discogs_id'], 'popularity': a['popularity'], 'type': 'watchlist' if 'watchlist' in a['sources'] else 'similar', }) # Build genre clusters — allow artists in multiple genres top_genres = sorted_genres[:40] # Sort genres by co-occurrence so related genres are adjacent in the list. # This makes the spiral layout place related genres near each other. if len(top_genres) > 2: genre_sets = {g: set(keys) for g, keys in top_genres} ordered = [top_genres[0][0]] # Start with biggest genre remaining = {g for g, _ in top_genres[1:]} while remaining: last = ordered[-1] last_set = genre_sets.get(last, set()) # Find most similar remaining genre (highest artist overlap) best = None best_overlap = -1 for g in remaining: overlap = len(last_set & genre_sets.get(g, set())) if overlap > best_overlap: best_overlap = overlap best = g ordered.append(best) remaining.remove(best) # Rebuild top_genres in the ordered sequence genre_dict = dict(top_genres) top_genres = [(g, genre_dict[g]) for g in ordered if g in genre_dict] genres_out = [] for genre, artist_keys in top_genres: genres_out.append({ 'name': genre, 'count': len(artist_keys), 'artist_ids': [node_idx[k] for k in artist_keys if k in node_idx], }) # Image cleanup + multi-source fallback # Build two lookups: name→image_url AND name→deezer_entity_id _img_cache = {} _deezer_id_cache = {} _album_art_cache = {} # artist_name → album image (iTunes fallback) try: # Artist images + Deezer IDs cursor.execute(""" SELECT name, entity_id, source, image_url FROM metadata_cache_entities WHERE entity_type = 'artist' AND ((image_url IS NOT NULL AND image_url != '' AND image_url LIKE 'http%') OR source = 'deezer') """) for r in cursor.fetchall(): nn = (r['name'] or '').lower().strip() if not nn: continue if r['image_url'] and r['image_url'].startswith('http') and nn not in _img_cache: _img_cache[nn] = r['image_url'] if r['source'] == 'deezer' and r['entity_id'] and nn not in _deezer_id_cache: _deezer_id_cache[nn] = r['entity_id'] # Album art by artist name (for iTunes artists with no artist image) cursor.execute(""" SELECT artist_name, image_url FROM metadata_cache_entities WHERE entity_type = 'album' AND image_url IS NOT NULL AND image_url != '' AND image_url LIKE 'http%' AND artist_name IS NOT NULL AND artist_name != '' """) for r in cursor.fetchall(): nn = (r['artist_name'] or '').lower().strip() if nn and nn not in _album_art_cache: _album_art_cache[nn] = r['image_url'] except Exception: pass for n in nodes: img = n.get('image_url', '') if img in ('None', 'null', '') or (img and not img.startswith('http')): n['image_url'] = '' nn = n['name'].lower().strip() if not n['image_url']: # Try cache image by name n['image_url'] = _img_cache.get(nn, '') if not n['image_url'] and n.get('deezer_id'): n['image_url'] = f"https://api.deezer.com/artist/{n['deezer_id']}/image?size=big" if not n['image_url']: # Try Deezer ID from cache by name did = _deezer_id_cache.get(nn) if did: n['deezer_id'] = did n['image_url'] = f"https://api.deezer.com/artist/{did}/image?size=big" if not n['image_url']: # Try album art by artist name (iTunes artists have no artist images) n['image_url'] = _album_art_cache.get(nn, '') _img_count = sum(1 for n in nodes if n.get('image_url')) _deezer_count = sum(1 for n in nodes if n.get('image_url', '').startswith('https://api.deezer')) _none_count = sum(1 for n in nodes if not n.get('image_url')) logger.info(f"[Genre Map] {len(nodes)} artists, {len(sorted_genres)} genres") logger.warning(f"[Genre Map] Images: {_img_count} have URLs, {_deezer_count} Deezer fallback, {_none_count} missing") if _none_count > 0: samples = [n['name'] for n in nodes if not n.get('image_url')][:5] logger.warning(f"[Genre Map] Missing image samples: {samples}") result = { 'success': True, 'nodes': nodes, 'genres': genres_out, 'total_artists': len(nodes), 'total_genres': len(sorted_genres), } _artmap_cache_set(f'genres_{profile_id}', result) return jsonify(result) except Exception as e: logger.error(f"Error getting genre map data: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/artist-map/explore', methods=['GET']) def get_artist_map_explore(): """Build an exploration map outward from a single artist.""" try: artist_name = request.args.get('name', '').strip() artist_id = request.args.get('id', '').strip() if not artist_name and not artist_id: return jsonify({"success": False, "error": "Provide artist name or id"}), 400 database = get_database() profile_id = get_current_profile_id() conn = database._get_connection() cursor = conn.cursor() def _norm(n): return (n or '').lower().strip() nodes = [] edges = [] seen = {} # norm_name → node index # Find the center artist center_name = artist_name center_image = '' center_ids = {'spotify_id': '', 'itunes_id': '', 'deezer_id': '', 'discogs_id': ''} center_genres = [] # Search metadata cache for the center artist if artist_id: cursor.execute(""" SELECT name, entity_id, source, image_url, genres FROM metadata_cache_entities WHERE entity_type = 'artist' AND entity_id = ? LIMIT 1 """, (artist_id,)) else: cursor.execute(""" SELECT name, entity_id, source, image_url, genres FROM metadata_cache_entities WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE LIMIT 1 """, (artist_name,)) row = cursor.fetchone() artist_found = False if row: artist_found = True center_name = row['name'] if row['image_url'] and row['image_url'].startswith('http'): center_image = row['image_url'] src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} k = src_map.get(row['source'], 'spotify_id') center_ids[k] = row['entity_id'] if row['genres']: try: center_genres = json.loads(row['genres']) if isinstance(row['genres'], str) else [] except Exception: pass # Check watchlist + library if not in cache if not artist_found and not artist_id: cursor.execute("SELECT artist_name, image_url, spotify_artist_id, itunes_artist_id, deezer_artist_id, discogs_artist_id FROM watchlist_artists WHERE artist_name = ? COLLATE NOCASE LIMIT 1", (artist_name,)) wr = cursor.fetchone() if wr: artist_found = True center_name = wr['artist_name'] if wr['image_url'] and str(wr['image_url']).startswith('http'): center_image = wr['image_url'] for k, col in [('spotify_id', 'spotify_artist_id'), ('itunes_id', 'itunes_artist_id'), ('deezer_id', 'deezer_artist_id'), ('discogs_id', 'discogs_artist_id')]: if wr[col]: center_ids[k] = str(wr[col]) else: cursor.execute("SELECT name, thumb_url FROM artists WHERE name = ? COLLATE NOCASE LIMIT 1", (artist_name,)) lr = cursor.fetchone() if lr: artist_found = True center_name = lr['name'] if lr['thumb_url'] and str(lr['thumb_url']).startswith('http'): center_image = lr['thumb_url'] # If not found locally, validate via metadata API search if not artist_found and not artist_id: try: api_match = None if spotify_client and spotify_client.is_spotify_authenticated(): results = spotify_client.search_artists(artist_name, limit=1) if results and len(results) > 0: sa = results[0] if sa.name.lower().strip() == artist_name.lower().strip() or \ artist_name.lower().strip() in sa.name.lower().strip(): api_match = sa center_name = sa.name center_ids['spotify_id'] = sa.id center_image = sa.image_url if hasattr(sa, 'image_url') else '' center_genres = sa.genres if hasattr(sa, 'genres') else [] artist_found = True if not artist_found: ic = _get_itunes_client() results = ic.search_artists(artist_name, limit=1) if results and len(results) > 0: ia = results[0] if ia.name.lower().strip() == artist_name.lower().strip() or \ artist_name.lower().strip() in ia.name.lower().strip(): center_name = ia.name center_ids['itunes_id'] = str(ia.id) center_image = ia.image_url if hasattr(ia, 'image_url') else '' artist_found = True except Exception as e: logger.debug(f"[Artist Explorer] API validation failed for '{artist_name}': {e}") if not artist_found: return jsonify({"success": False, "error": f"Artist '{artist_name}' not found"}), 404 # Also check cache for other source IDs cursor.execute(""" SELECT entity_id, source, image_url, genres FROM metadata_cache_entities WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE """, (center_name,)) for r in cursor.fetchall(): src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} k = src_map.get(r['source'], 'spotify_id') if not center_ids.get(k): center_ids[k] = r['entity_id'] if r['image_url'] and r['image_url'].startswith('http') and not center_image: center_image = r['image_url'] if r['genres'] and not center_genres: try: center_genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass # Add center node center_idx = 0 seen[_norm(center_name)] = center_idx nodes.append({ 'id': 0, 'name': center_name, 'image_url': center_image, 'type': 'center', 'genres': center_genres[:5], **center_ids, 'ring': 0 }) # Ring 1: Direct similar artists from similar_artists table # Search by all known IDs id_values = [v for v in center_ids.values() if v] ring1_artists = [] if id_values: placeholders = ','.join(['?'] * len(id_values)) cursor.execute(f""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? ORDER BY similarity_rank ASC """, id_values + [profile_id]) ring1_artists = cursor.fetchall() # Also search by name (the center artist might be a watchlist source) cursor.execute(""" SELECT DISTINCT sa.similar_artist_name, sa.similar_artist_spotify_id, sa.similar_artist_itunes_id, sa.similar_artist_deezer_id, sa.image_url, sa.genres, sa.popularity, sa.similarity_rank FROM similar_artists sa JOIN watchlist_artists wa ON sa.source_artist_id = COALESCE(wa.spotify_artist_id, wa.itunes_artist_id, CAST(wa.id AS TEXT)) WHERE wa.artist_name = ? COLLATE NOCASE AND sa.profile_id = ? ORDER BY sa.similarity_rank ASC """, (center_name, profile_id)) ring1_artists.extend(cursor.fetchall()) # If no similar artists in DB, fetch from MusicMap on-the-fly if not ring1_artists: try: logger.debug(f"[Artist Explorer] No stored similar artists for '{center_name}', fetching from MusicMap...") from core.watchlist_scanner import WatchlistScanner scanner = WatchlistScanner(spotify_client=spotify_client) if spotify_client else None if scanner: similar = scanner._fetch_similar_artists_from_musicmap(center_name, limit=15) if similar: source_artist_id = center_ids.get('spotify_id') or center_ids.get('itunes_id') or center_name # Store in DB for future use for rank, sa in enumerate(similar, 1): try: database.add_or_update_similar_artist( source_artist_id=source_artist_id, similar_artist_name=sa['name'], similar_artist_spotify_id=sa.get('spotify_id'), similar_artist_itunes_id=sa.get('itunes_id'), similarity_rank=rank, profile_id=profile_id, image_url=sa.get('image_url'), genres=sa.get('genres'), popularity=sa.get('popularity', 0), similar_artist_deezer_id=sa.get('deezer_id') ) except Exception: pass # Re-query from DB to get consistent format if id_values: placeholders = ','.join(['?'] * len(id_values)) cursor.execute(f""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? ORDER BY similarity_rank ASC """, id_values + [profile_id]) ring1_artists = cursor.fetchall() if not ring1_artists: # Fallback: query by name-based source ID cursor.execute(""" SELECT DISTINCT similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id = ? AND profile_id = ? ORDER BY similarity_rank ASC """, (source_artist_id, profile_id)) ring1_artists = cursor.fetchall() logger.debug(f"[Artist Explorer] Fetched {len(ring1_artists)} similar artists from MusicMap for '{center_name}'") _artmap_cache_invalidate(profile_id) # New similar artists added except Exception as e: logger.debug(f"[Artist Explorer] MusicMap fetch failed for '{center_name}': {e}") # Deduplicate ring 1 for r in ring1_artists: nn = _norm(r['similar_artist_name']) if nn in seen: continue idx = len(nodes) seen[nn] = idx genres = [] if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass img = r['image_url'] if r['image_url'] and r['image_url'].startswith('http') else '' nodes.append({ 'id': idx, 'name': r['similar_artist_name'], 'image_url': img, 'type': 'ring1', 'genres': genres[:5], 'spotify_id': r['similar_artist_spotify_id'] or '', 'itunes_id': r['similar_artist_itunes_id'] or '', 'deezer_id': r['similar_artist_deezer_id'] or '', 'discogs_id': '', 'popularity': r['popularity'] or 0, 'rank': r['similarity_rank'] or 5, 'ring': 1, }) weight = max(1, 11 - (r['similarity_rank'] or 5)) edges.append({'source': center_idx, 'target': idx, 'weight': weight}) # Ring 2: Similar artists of ring 1 artists (from similar_artists table) ring1_ids = [] for n in nodes[1:]: # skip center for sid in [n.get('spotify_id'), n.get('itunes_id')]: if sid: ring1_ids.append(sid) if ring1_ids: placeholders = ','.join(['?'] * len(ring1_ids)) cursor.execute(f""" SELECT DISTINCT source_artist_id, similar_artist_name, similar_artist_spotify_id, similar_artist_itunes_id, similar_artist_deezer_id, image_url, genres, popularity, similarity_rank FROM similar_artists WHERE source_artist_id IN ({placeholders}) AND profile_id = ? ORDER BY similarity_rank ASC """, ring1_ids + [profile_id]) for r in cursor.fetchall(): nn = _norm(r['similar_artist_name']) if nn in seen: # Create edge to existing node if not center existing_idx = seen[nn] # Find the ring1 node that sourced this source_norm = None for n in nodes[1:]: for sid in [n.get('spotify_id'), n.get('itunes_id')]: if sid == r['source_artist_id']: source_norm = _norm(n['name']) break if source_norm: break if source_norm and source_norm in seen and existing_idx != seen[source_norm]: edges.append({'source': seen[source_norm], 'target': existing_idx, 'weight': 3}) continue idx = len(nodes) if idx >= 500: # Cap at 500 nodes for performance break seen[nn] = idx genres = [] if r['genres']: try: genres = json.loads(r['genres']) if isinstance(r['genres'], str) else [] except Exception: pass img = r['image_url'] if r['image_url'] and r['image_url'].startswith('http') else '' nodes.append({ 'id': idx, 'name': r['similar_artist_name'], 'image_url': img, 'type': 'ring2', 'genres': genres[:5], 'spotify_id': r['similar_artist_spotify_id'] or '', 'itunes_id': r['similar_artist_itunes_id'] or '', 'deezer_id': r['similar_artist_deezer_id'] or '', 'discogs_id': '', 'popularity': r['popularity'] or 0, 'rank': r['similarity_rank'] or 5, 'ring': 2, }) # Find the ring1 source for n in nodes[1:]: for sid in [n.get('spotify_id'), n.get('itunes_id')]: if sid == r['source_artist_id']: edges.append({'source': n['id'], 'target': idx, 'weight': max(1, 11 - (r['similarity_rank'] or 5))}) break # Backfill images/genres from ALL cache sources + Deezer fallback for n in nodes: # Clean up string "None" stored as image URL if n['image_url'] in ('None', 'null', ''): n['image_url'] = '' if n['image_url'] and n['genres']: continue # Check all cache entries for this artist (multiple sources) cursor.execute(""" SELECT entity_id, source, image_url, genres FROM metadata_cache_entities WHERE entity_type = 'artist' AND name = ? COLLATE NOCASE """, (n['name'],)) for cr in cursor.fetchall(): if not n['image_url'] and cr['image_url'] and cr['image_url'].startswith('http'): n['image_url'] = cr['image_url'] if not n['genres'] and cr['genres']: try: n['genres'] = json.loads(cr['genres'])[:5] if isinstance(cr['genres'], str) else [] except Exception: pass # Harvest missing IDs from cache src_map = {'spotify': 'spotify_id', 'itunes': 'itunes_id', 'deezer': 'deezer_id', 'discogs': 'discogs_id'} k = src_map.get(cr['source']) if k and not n.get(k): n[k] = cr['entity_id'] # Deezer image fallback — construct URL directly from ID if not n['image_url'] and n.get('deezer_id'): n['image_url'] = f"https://api.deezer.com/artist/{n['deezer_id']}/image?size=big" # Spotify image fallback — try API if authenticated if not n['image_url'] and n.get('spotify_id'): try: if spotify_client and spotify_client.is_spotify_authenticated(): from core.api_call_tracker import api_call_tracker api_call_tracker.record_call('spotify', endpoint='artist') artist_data = spotify_client.sp.artist(n['spotify_id']) if artist_data and artist_data.get('images'): n['image_url'] = artist_data['images'][0]['url'] if not n['genres'] and artist_data.get('genres'): n['genres'] = artist_data['genres'][:5] except Exception: pass # Album art fallback (iTunes artists have no artist images) if not n['image_url']: cursor.execute(""" SELECT image_url FROM metadata_cache_entities WHERE entity_type = 'album' AND image_url LIKE 'http%' AND artist_name = ? COLLATE NOCASE LIMIT 1 """, (n['name'],)) alb = cursor.fetchone() if alb: n['image_url'] = alb['image_url'] logger.info(f"[Artist Explorer] Center: {center_name}, Ring 1: {sum(1 for n in nodes if n.get('ring')==1)}, Ring 2: {sum(1 for n in nodes if n.get('ring')==2)}, Edges: {len(edges)}") return jsonify({ 'success': True, 'nodes': nodes, 'edges': edges, 'center': center_name, }) except Exception as e: logger.error(f"Error getting artist explorer data: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/build-playlist/search-artists', methods=['GET']) def search_artists_for_playlist(): """Search for artists to use as seeds for custom playlist building""" try: query = request.args.get('query', '').strip() if not query: return jsonify({"success": False, "error": "Query required"}), 400 artists = [] if _is_hydrabase_active(): artist_objs = hydrabase_client.search_artists(query, limit=10) for artist in artist_objs: artists.append({ 'id': artist.id, 'name': artist.name, 'image_url': artist.image_url }) else: if hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, 'artists') # Try Spotify first, fall back to iTunes if spotify_client.sp and not _spotify_rate_limited(): try: artist_results = spotify_client.search_artists(query, limit=10) for artist in artist_results: artists.append({ 'id': artist.id, 'name': artist.name, 'image_url': artist.image_url }) except Exception as e: logger.warning(f"Spotify artist search failed, falling back to iTunes: {e}") if not artists: fallback = _get_metadata_fallback_client() artist_objs = fallback.search_artists(query, limit=10) for artist in artist_objs: # Fallback artist search may not return images — grab from album art image = artist.image_url if not image: image = fallback._get_artist_image_from_albums(artist.id) artists.append({ 'id': artist.id, 'name': artist.name, 'image_url': image }) if artists: # Re-rank: boost exact name matches to the top query_lower = query.lower().strip() artists.sort(key=lambda a: (0 if a['name'].lower().strip() == query_lower else 1)) return jsonify({ "success": True, "artists": artists }) except Exception as e: logger.error(f"Error searching for artists: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/build-playlist/generate', methods=['POST']) def generate_custom_playlist(): """Generate custom playlist from seed artists""" try: from core.personalized_playlists import get_personalized_playlists_service data = request.get_json() seed_artist_ids = data.get('seed_artist_ids', []) if not seed_artist_ids or len(seed_artist_ids) < 1 or len(seed_artist_ids) > 5: return jsonify({ "success": False, "error": "Please provide between 1 and 5 seed artists" }), 400 database = get_database() service = get_personalized_playlists_service(database, spotify_client) playlist_size = int(data.get('playlist_size', 50)) result = service.build_custom_playlist(seed_artist_ids, playlist_size=playlist_size) if result.get('error') and not result.get('tracks'): return jsonify({"success": False, "error": result['error']}), 400 return jsonify({ "success": True, "playlist": result }) except Exception as e: logger.error(f"Error generating custom playlist: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/decades/available', methods=['GET']) def get_available_decades(): """Get list of decades that have content in discovery pool""" try: database = get_database() with database._get_connection() as conn: cursor = conn.cursor() # Get distinct decades from discovery pool cursor.execute(""" SELECT DISTINCT (CAST(SUBSTR(release_date, 1, 4) AS INTEGER) / 10) * 10 as decade, COUNT(*) as track_count FROM discovery_pool WHERE release_date IS NOT NULL AND CAST(SUBSTR(release_date, 1, 4) AS INTEGER) >= 1950 AND CAST(SUBSTR(release_date, 1, 4) AS INTEGER) <= 2029 GROUP BY decade HAVING track_count >= 10 ORDER BY decade ASC """) rows = cursor.fetchall() decades = [] for row in rows: decades.append({ 'year': row[0], 'track_count': row[1] }) return jsonify({ "success": True, "decades": decades }) except Exception as e: logger.error(f"Error getting available decades: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/decade/', methods=['GET']) def get_discover_decade_playlist(decade): """Get tracks from a specific decade for discovery page""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_decade_playlist(decade, limit=50) if not tracks: return jsonify({ "success": True, "tracks": [], "decade": decade, "message": f"No tracks found for the {decade}s" }), 200 # Convert to Spotify format for modal compatibility spotify_tracks = [] for track in tracks: spotify_tracks.append({ 'id': track.get('spotify_track_id', track.get('id')), 'name': track.get('track_name', track.get('name')), 'artists': [track.get('artist_name', 'Unknown')], 'album': { 'name': track.get('album_name', 'Unknown'), 'images': [{'url': track.get('album_cover_url')}] if track.get('album_cover_url') else [] }, 'duration_ms': track.get('duration_ms', 0) }) return jsonify({ "success": True, "tracks": spotify_tracks, "decade": decade }) except Exception as e: logger.error(f"Error getting decade playlist: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/genres/available', methods=['GET']) def get_available_genres(): """Get list of genres that have content in discovery pool""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) genres = service.get_available_genres() return jsonify({ "success": True, "genres": genres }) except Exception as e: logger.error(f"Error getting available genres: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/genre/', methods=['GET']) def get_discover_genre_playlist(genre_name): """Get tracks from a specific genre for discovery page""" try: from core.personalized_playlists import get_personalized_playlists_service database = get_database() service = get_personalized_playlists_service(database, spotify_client) tracks = service.get_genre_playlist(genre_name, limit=50) if not tracks: return jsonify({ "success": True, "tracks": [], "genre": genre_name, "message": f"No tracks found for {genre_name}" }), 200 # Convert to Spotify format for modal compatibility spotify_tracks = [] for track in tracks: spotify_tracks.append({ 'id': track.get('spotify_track_id', track.get('id')), 'name': track.get('track_name', track.get('name')), 'artists': [track.get('artist_name', 'Unknown')], 'album': { 'name': track.get('album_name', 'Unknown'), 'images': [{'url': track.get('album_cover_url')}] if track.get('album_cover_url') else [] }, 'duration_ms': track.get('duration_ms', 0) }) return jsonify({ "success": True, "tracks": spotify_tracks, "genre": genre_name }) except Exception as e: logger.error(f"Error getting genre playlist: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # =============================== # LISTENBRAINZ DISCOVER ENDPOINTS # =============================== def _get_profile_lb_manager(): """Create a profile-aware ListenBrainzManager for the current user. Always uses the actual profile_id so each profile has its own playlist cache.""" from core.listenbrainz_manager import ListenBrainzManager profile_id = get_current_profile_id() token, base_url, username, source = _get_lb_credentials_for_profile(profile_id) return ListenBrainzManager(str(get_database().database_path), profile_id=profile_id, token=token, base_url=base_url), username, source def _get_lb_discover_playlists(playlist_type): """Shared logic for the 3 LB discover endpoints""" lb_manager, username, source = _get_profile_lb_manager() # Check if cache is empty - if so, populate it on first load if not lb_manager.has_cached_playlists(): if not lb_manager.client.is_authenticated(): return jsonify({ "success": False, "error": "Not authenticated", "playlists": [], "count": 0, "username": None }) logger.warning(f"Cache empty for profile {lb_manager.profile_id}, populating ListenBrainz playlists...") lb_manager.update_all_playlists() playlists = lb_manager.get_cached_playlists(playlist_type) formatted_playlists = [] for playlist in playlists: formatted_playlists.append({ "playlist": { "identifier": f"https://listenbrainz.org/playlist/{playlist['playlist_mbid']}", "title": playlist['title'], "creator": playlist['creator'], "annotation": playlist.get('annotation', {}), "track": [] } }) return jsonify({ "success": True, "playlists": formatted_playlists, "count": len(formatted_playlists), "username": username, "source": source }) @app.route('/api/discover/listenbrainz/created-for', methods=['GET']) def get_listenbrainz_created_for(): """Get playlists created for the user by ListenBrainz (from cache)""" try: return _get_lb_discover_playlists('created_for') except Exception as e: logger.error(f"Error getting cached ListenBrainz created-for playlists: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/listenbrainz/user-playlists', methods=['GET']) def get_listenbrainz_user_playlists(): """Get user's own ListenBrainz playlists (from cache)""" try: return _get_lb_discover_playlists('user') except Exception as e: logger.error(f"Error getting cached ListenBrainz user playlists: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/listenbrainz/collaborative', methods=['GET']) def get_listenbrainz_collaborative(): """Get collaborative ListenBrainz playlists (from cache)""" try: return _get_lb_discover_playlists('collaborative') except Exception as e: logger.error(f"Error getting cached ListenBrainz collaborative playlists: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/listenbrainz/playlist/', methods=['GET']) def get_listenbrainz_playlist_tracks(playlist_mbid): """Get tracks from a specific ListenBrainz playlist (from cache, with on-demand refresh)""" try: lb_manager, username, source = _get_profile_lb_manager() tracks = lb_manager.get_cached_tracks(playlist_mbid) if not tracks: # Cache miss or stale entry with no tracks — try fetching from LB API if lb_manager.client.is_authenticated(): logger.debug(f"Cache miss for playlist {playlist_mbid}, fetching from ListenBrainz...") # Remove stale playlist row (if any) so _update_playlist doesn't # skip due to matching track_count with 0 actual tracks existing_type = lb_manager.get_playlist_type(playlist_mbid) or 'created_for' lb_manager.delete_cached_playlist(playlist_mbid) full_playlist = lb_manager.client.get_playlist_details(playlist_mbid) if full_playlist: lb_manager._update_playlist(full_playlist, existing_type) tracks = lb_manager.get_cached_tracks(playlist_mbid) if not tracks: return jsonify({ "success": False, "error": "Playlist not found in cache" }), 404 return jsonify({ "success": True, "tracks": tracks, "track_count": len(tracks) }) except Exception as e: logger.error(f"Error getting cached ListenBrainz playlist tracks: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # Manual refresh endpoint for ListenBrainz @app.route('/api/discover/listenbrainz/refresh', methods=['POST']) def refresh_listenbrainz(): """Manually refresh ListenBrainz playlists cache""" try: lb_manager, username, source = _get_profile_lb_manager() result = lb_manager.update_all_playlists() return jsonify(result) except Exception as e: logger.error(f"Error refreshing ListenBrainz: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # ======================================== # LAST.FM TRACK RADIO # ======================================== @app.route('/api/lastfm/configured', methods=['GET']) def lastfm_configured(): """Return whether a Last.fm API key is configured (used to gate the Radio section).""" lf = lastfm_worker.client if lastfm_worker else None return jsonify({"configured": bool(lf and lf.api_key)}) @app.route('/api/lastfm/search/tracks', methods=['GET']) def lastfm_search_tracks(): """Search Last.fm for tracks matching a query string. Query params: q: search query (track name, artist name, or both) Returns: JSON list of {name, artist, mbid, listeners} """ try: q = request.args.get('q', '').strip() if not q or len(q) < 2: return jsonify({"success": False, "error": "Query too short", "results": []}), 400 lf = lastfm_worker.client if lastfm_worker else None if not lf or not lf.api_key: return jsonify({"success": False, "error": "Last.fm not configured", "results": []}), 400 # Use raw API call to get multiple results (search_track only returns best match) data = lf._make_request('track.search', {'track': q, 'limit': 8}) if not data: return jsonify({"success": True, "results": []}) raw = data.get('results', {}).get('trackmatches', {}).get('track', []) if not isinstance(raw, list): raw = [raw] if raw else [] results = [] for t in raw: # Last.fm image array: [{#text: url, size: small/medium/large/extralarge}] image_url = lf.get_best_image(t.get('image', [])) results.append({ 'name': t.get('name', ''), 'artist': t.get('artist', ''), 'mbid': t.get('mbid', ''), 'listeners': int(t.get('listeners', 0)), 'image_url': image_url or '', }) return jsonify({"success": True, "results": results}) except Exception as e: logger.error(f"Error searching Last.fm tracks: {e}") return jsonify({"success": False, "error": str(e), "results": []}), 500 @app.route('/api/lastfm/radio/generate', methods=['POST']) def lastfm_radio_generate(): """Generate a Last.fm Radio playlist from a seed track. Body JSON: track_name: seed track title artist_name: seed artist name Creates/updates a 'lastfm_radio' playlist in the DB and adds it to listenbrainz_playlist_states in 'fresh' phase, ready for discovery. Returns: {success, playlist_mbid, title, track_count} """ try: data = request.get_json() or {} track_name = (data.get('track_name') or '').strip() artist_name = (data.get('artist_name') or '').strip() if not track_name or not artist_name: return jsonify({"success": False, "error": "track_name and artist_name are required"}), 400 lf = lastfm_worker.client if lastfm_worker else None if not lf or not lf.api_key: return jsonify({"success": False, "error": "Last.fm not configured"}), 400 # Fetch similar tracks from Last.fm similar = lf.get_similar_tracks(artist_name, track_name, limit=25) if not similar: return jsonify({"success": False, "error": "No similar tracks found on Last.fm"}), 404 # Persist to DB via manager lb_manager, _username, _source = _get_profile_lb_manager() playlist_mbid = lb_manager.save_lastfm_radio_playlist(track_name, artist_name, similar) title = f"Last.fm Radio: {track_name} by {artist_name}" # Build playlist dict that mirrors the LB playlist format expected by the discovery pipeline playlist_data = { 'identifier': f"lastfm_radio/{playlist_mbid}", 'name': title, 'title': title, 'creator': 'Last.fm', 'tracks': [ { 'track_name': t['name'], 'artist_name': t['artist'], 'album_name': '', 'duration_ms': 0, } for t in similar ], } # Upsert into in-memory state (fresh phase — not yet discovered) state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: listenbrainz_playlist_states[state_key] = { 'playlist_mbid': playlist_mbid, 'playlist': playlist_data, 'phase': 'fresh', 'status': 'fresh', 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(similar), 'discovery_results': [], 'created_at': time.time(), 'last_accessed': time.time(), } else: # Refresh existing state (new seed data) but preserve phase if already discovered state = listenbrainz_playlist_states[state_key] if state['phase'] not in ('discovering',): state['playlist'] = playlist_data state['spotify_total'] = len(similar) state['last_accessed'] = time.time() logger.info(f"Last.fm Radio generated: '{title}' ({len(similar)} tracks) → {playlist_mbid}") return jsonify({ "success": True, "playlist_mbid": playlist_mbid, "title": title, "track_count": len(similar), }) except Exception as e: logger.error(f"Error generating Last.fm radio: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/listenbrainz/lastfm-radio', methods=['GET']) def get_listenbrainz_lastfm_radio(): """Get cached Last.fm Radio playlists (from DB cache). Does NOT require ListenBrainz authentication — Last.fm Radio playlists are generated independently of the LB account. """ try: lb_manager, username, source = _get_profile_lb_manager() playlists = lb_manager.get_cached_playlists('lastfm_radio') formatted = [ { "playlist": { "identifier": f"https://listenbrainz.org/playlist/{p['playlist_mbid']}", "title": p['title'], "creator": p['creator'], "annotation": p.get('annotation', {}), "track": [], } } for p in playlists ] return jsonify({"success": True, "playlists": formatted, "count": len(formatted), "username": username, "source": source}) except Exception as e: logger.error(f"Error getting Last.fm radio playlists: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 # ======================================== # LISTENBRAINZ PLAYLIST MANAGEMENT (Discovery System) # ======================================== def _lb_state_key(playlist_mbid, profile_id=None): """Build profile-scoped key for listenbrainz_playlist_states""" if profile_id is None: profile_id = get_current_profile_id() return f"{profile_id}:{playlist_mbid}" @app.route('/api/listenbrainz/playlists', methods=['GET']) def get_all_listenbrainz_playlists(): """Get all stored ListenBrainz playlists for frontend hydration (scoped to current profile)""" try: playlists = [] current_time = time.time() profile_id = get_current_profile_id() prefix = f"{profile_id}:" for state_key, state in listenbrainz_playlist_states.items(): if not state_key.startswith(prefix): continue # Update access time when requested state['last_accessed'] = current_time playlist_mbid = state_key[len(prefix):] # Return essential data for card recreation playlist_info = { 'playlist_mbid': playlist_mbid, 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'created_at': state['created_at'], 'last_accessed': state['last_accessed'] } playlists.append(playlist_info) logger.info(f"Returning {len(playlists)} stored ListenBrainz playlists for profile {profile_id}") return jsonify({"playlists": playlists}) except Exception as e: logger.error(f"Error getting ListenBrainz playlists: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/state/', methods=['GET']) def get_listenbrainz_playlist_state(playlist_mbid): """Get specific ListenBrainz playlist state (detailed version)""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] state['last_accessed'] = time.time() # Return full state information (including results for modal hydration) response = { 'playlist_mbid': playlist_mbid, 'playlist': state['playlist'], 'phase': state['phase'], 'status': state['status'], 'discovery_progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'discovery_results': state['discovery_results'], 'sync_playlist_id': state.get('sync_playlist_id'), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'sync_progress': state.get('sync_progress', {}), 'created_at': state['created_at'], 'last_accessed': state['last_accessed'] } return jsonify(response) except Exception as e: logger.error(f"Error getting ListenBrainz playlist state: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/reset/', methods=['POST']) def reset_listenbrainz_playlist(playlist_mbid): """Reset ListenBrainz playlist to fresh phase (clear discovery/sync data)""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Reset state to fresh (preserve original playlist data) state['phase'] = 'fresh' state['status'] = 'cached' state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['sync_playlist_id'] = None state['converted_spotify_playlist_id'] = None state['sync_progress'] = {} state['discovery_future'] = None state['last_accessed'] = time.time() logger.info(f"Reset ListenBrainz playlist to fresh: {state['playlist']['title']}") return jsonify({"success": True, "phase": "fresh"}) except Exception as e: logger.error(f"Error resetting ListenBrainz playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/remove/', methods=['POST']) def remove_listenbrainz_playlist(playlist_mbid): """Remove ListenBrainz playlist from state (doesn't affect cache)""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] # Stop any active discovery if 'discovery_future' in state and state['discovery_future']: state['discovery_future'].cancel() # Remove from state del listenbrainz_playlist_states[state_key] logger.info(f"Removed ListenBrainz playlist from state: {playlist_mbid}") return jsonify({"success": True}) except Exception as e: logger.error(f"Error removing ListenBrainz playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/discovery/start/', methods=['POST']) def start_listenbrainz_discovery(playlist_mbid): """Initialize and start Spotify discovery process for a ListenBrainz playlist""" try: data = request.get_json() playlist_data = data.get('playlist') if not playlist_data: return jsonify({"error": "Playlist data required"}), 400 # Create or update state state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: # Initialize new state listenbrainz_playlist_states[state_key] = { 'playlist_mbid': playlist_mbid, 'playlist': playlist_data, 'phase': 'discovering', 'status': 'discovering', 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(playlist_data.get('tracks', [])), 'discovery_results': [], 'created_at': time.time(), 'last_accessed': time.time() } logger.info(f"Created new ListenBrainz playlist state: {playlist_data.get('name', 'Unknown')}") else: # State already exists, update it state = listenbrainz_playlist_states[state_key] if state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 # Reset for new discovery state['phase'] = 'discovering' state['status'] = 'discovering' state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['discovery_results'] = [] state['last_accessed'] = time.time() state = listenbrainz_playlist_states[state_key] # Add activity for discovery start playlist_name = playlist_data.get('name', 'Unknown Playlist') track_count = len(playlist_data.get('tracks', [])) add_activity_item("", "ListenBrainz Discovery Started", f"'{playlist_name}' - {track_count} tracks", "Now") # Start discovery worker (pass state_key for profile-scoped state access) future = listenbrainz_discovery_executor.submit(_run_listenbrainz_discovery_worker, state_key) state['discovery_future'] = future logger.info(f"Started Spotify discovery for ListenBrainz playlist: {playlist_name}") return jsonify({"success": True, "message": "Discovery started"}) except Exception as e: logger.error(f"Error starting ListenBrainz discovery: {e}") import traceback traceback.print_exc() return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/discovery/status/', methods=['GET']) def get_listenbrainz_discovery_status(playlist_mbid): """Get real-time discovery status for a ListenBrainz playlist""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] state['last_accessed'] = time.time() response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting ListenBrainz discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/update-phase/', methods=['POST']) def update_listenbrainz_phase(playlist_mbid): """Update ListenBrainz playlist phase (for phase transitions and persistence)""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 data = request.get_json() or {} new_phase = data.get('phase') if not new_phase: return jsonify({"error": "Phase is required"}), 400 state = listenbrainz_playlist_states[state_key] state['phase'] = new_phase state['last_accessed'] = time.time() # Update download process ID if provided (for download persistence) if 'download_process_id' in data: state['download_process_id'] = data['download_process_id'] logger.info(f"Updated ListenBrainz download_process_id: {data['download_process_id']}") # Update converted Spotify playlist ID if provided (for download persistence) if 'converted_spotify_playlist_id' in data: state['converted_spotify_playlist_id'] = data['converted_spotify_playlist_id'] logger.info(f"Updated ListenBrainz converted_spotify_playlist_id: {data['converted_spotify_playlist_id']}") logger.info(f"Updated ListenBrainz playlist {playlist_mbid} phase to: {new_phase}") return jsonify({ "success": True, "phase": new_phase }) except Exception as e: logger.error(f"Error updating ListenBrainz playlist phase: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/discovery/update_match', methods=['POST']) def update_listenbrainz_discovery_match(): """Update a ListenBrainz discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # playlist_mbid track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state (identifier is playlist_mbid) state = listenbrainz_playlist_states.get(_lb_state_key(identifier)) if not state: return jsonify({'error': 'Discovery state not found'}), 404 # Update the discovery result if track_index < len(state['discovery_results']): result = state['discovery_results'][track_index] # Was previously not found, now found if result['status_class'] == 'not-found' and spotify_track: state['spotify_matches'] += 1 # Was previously found, now not found elif result['status_class'] == 'found' and not spotify_track: state['spotify_matches'] -= 1 # Update result result['status'] = 'Found' if spotify_track else 'Not Found' result['status_class'] = 'found' if spotify_track else 'not-found' result['spotify_track'] = spotify_track.get('name', '') if spotify_track else '' # Join all artists (matching YouTube/Tidal/Beatport format) artists = spotify_track.get('artists', []) if spotify_track else [] result['spotify_artist'] = _join_artist_names(artists) if isinstance(artists, list) else _extract_artist_name(artists) # Album comes as a string from the frontend fix modal album = spotify_track.get('album', '') if spotify_track else '' result['spotify_album'] = album if isinstance(album, str) else album.get('name', '') if isinstance(album, dict) else '' result['spotify_id'] = spotify_track.get('id', '') if spotify_track else '' if spotify_track: # Store spotify_data in the same format as other platforms. # Manual match from the fix modal — build a rich spotify_data # (album as dict with image info) matching the normal discovery # shape, and explicitly clear any prior wing-it flag since the # user picked a real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) else: result['spotify_data'] = None result['wing_it_fallback'] = False result['manual_match'] = True logger.info(f"Updated ListenBrainz match for track {track_index}: {result['status']}") return jsonify({'success': True}) else: return jsonify({'error': 'Invalid track index'}), 400 except Exception as e: logger.error(f"Error updating ListenBrainz discovery match: {e}") import traceback traceback.print_exc() return jsonify({'error': str(e)}), 500 def convert_listenbrainz_results_to_spotify_tracks(discovery_results): """Convert ListenBrainz discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] # Create track object matching the expected format track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': spotify_data['artists'], 'album': spotify_data['album'], 'duration_ms': spotify_data.get('duration_ms', 0) } if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': # Build from individual fields (automatic discovery format) track = { 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': result.get('spotify_album', 'Unknown Album'), 'duration_ms': 0 } spotify_tracks.append(track) logger.info(f"Converted {len(spotify_tracks)} ListenBrainz matches to Spotify tracks for sync") return spotify_tracks @app.route('/api/wing-it/sync', methods=['POST']) def wing_it_sync(): """Sync a playlist to the media server using raw track names — no metadata discovery.""" try: data = request.get_json() tracks_raw = data.get('tracks', []) playlist_name = data.get('playlist_name', 'Wing It Playlist') if not tracks_raw: return jsonify({"error": "No tracks provided"}), 400 # Convert raw tracks to dicts — _run_sync_task expects dicts with .get() sync_tracks = [] for t in tracks_raw: artist_name = '' if isinstance(t.get('artists'), list) and t['artists']: a = t['artists'][0] artist_name = a.get('name', str(a)) if isinstance(a, dict) else str(a) elif t.get('artist_name'): artist_name = t['artist_name'] album_name = '' if isinstance(t.get('album'), dict): album_name = t['album'].get('name', '') elif isinstance(t.get('album'), str): album_name = t['album'] elif t.get('album_name'): album_name = t['album_name'] sync_tracks.append({ 'id': t.get('id', f"wing_it_{len(sync_tracks)}"), 'name': t.get('name', t.get('track_name', 'Unknown')), 'artists': [{'name': artist_name}] if artist_name else [{'name': 'Unknown'}], 'album': album_name, 'duration_ms': t.get('duration_ms', 0), }) if not sync_tracks: return jsonify({"error": "No valid tracks to sync"}), 400 sync_playlist_id = f"wing_it_sync_{int(time.time())}" add_activity_item("", "Wing It Sync Started", f"'{playlist_name}' — {len(sync_tracks)} tracks", "Now") with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Pass wing_it flag via sync state so _run_sync_task can skip wishlist with sync_lock: sync_states[sync_playlist_id]['wing_it'] = True future = sync_executor.submit(_run_sync_task, sync_playlist_id, playlist_name, sync_tracks, None, get_current_profile_id()) active_sync_workers[sync_playlist_id] = future logger.info(f"[Wing It] Started sync for: {playlist_name} ({len(sync_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error in Wing It sync: {e}", exc_info=True) return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/sync/start/', methods=['POST']) def start_listenbrainz_sync(playlist_mbid): """Start sync process for a ListenBrainz playlist using discovered Spotify tracks""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] state['last_accessed'] = time.time() # Update access time if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: return jsonify({"error": "ListenBrainz playlist not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_listenbrainz_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"listenbrainz_{playlist_mbid}" playlist_name = state['playlist']['name'] # Add activity for sync start add_activity_item("", "ListenBrainz Sync Started", f"'{playlist_name}' - {len(spotify_tracks)} tracks", "Now") # Update ListenBrainz state state['phase'] = 'syncing' state['sync_playlist_id'] = sync_playlist_id state['sync_progress'] = {} # Start the sync using existing sync infrastructure sync_data = { 'playlist_id': sync_playlist_id, 'playlist_name': playlist_name, 'tracks': spotify_tracks } with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Submit sync task future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['playlist_name'], spotify_tracks, None, get_current_profile_id()) active_sync_workers[sync_playlist_id] = future logger.info(f"Started ListenBrainz sync for: {playlist_name} ({len(spotify_tracks)} tracks)") return jsonify({"success": True, "sync_playlist_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting ListenBrainz sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/sync/status/', methods=['GET']) def get_listenbrainz_sync_status(playlist_mbid): """Get sync status for a ListenBrainz playlist""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync in progress"}), 404 # Get sync status from existing sync infrastructure with sync_lock: sync_state = sync_states.get(sync_playlist_id, {}) response = { 'phase': state['phase'], 'sync_status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Update ListenBrainz state if sync completed if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' state['sync_progress'] = sync_state.get('progress', {}) # Add activity for sync completion playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') add_activity_item("", "Sync Complete", f"ListenBrainz playlist '{playlist_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error playlist_name = state.get('playlist', {}).get('name', 'Unknown Playlist') add_activity_item("", "Sync Failed", f"ListenBrainz playlist '{playlist_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting ListenBrainz sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/listenbrainz/sync/cancel/', methods=['POST']) def cancel_listenbrainz_sync(playlist_mbid): """Cancel sync for a ListenBrainz playlist""" try: state_key = _lb_state_key(playlist_mbid) if state_key not in listenbrainz_playlist_states: return jsonify({"error": "ListenBrainz playlist not found"}), 404 state = listenbrainz_playlist_states[state_key] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Clean up sync worker if sync_playlist_id in active_sync_workers: del active_sync_workers[sync_playlist_id] # Revert ListenBrainz state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} return jsonify({"success": True, "message": "ListenBrainz sync cancelled"}) except Exception as e: logger.error(f"Error cancelling ListenBrainz sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/metadata/start', methods=['POST']) def start_metadata_update(): """Start the metadata update process - EXACT copy of dashboard.py logic""" global metadata_update_worker, metadata_update_runtime_worker, metadata_update_state try: # Check if already running if metadata_update_state['status'] == 'running': return jsonify({"success": False, "error": "Metadata update already running"}), 400 # Get refresh interval from request data = request.get_json() or {} refresh_interval_days = data.get('refresh_interval_days', 30) # Check active server and client availability - EXACTLY like dashboard.py active_server = config_manager.get_active_media_server() # Get appropriate media client - Support all three servers if active_server == "jellyfin": media_client = jellyfin_client if not media_client: add_activity_item("", "Metadata Update", "Jellyfin client not available", "Now") return jsonify({"success": False, "error": "Jellyfin client not available"}), 400 elif active_server == "navidrome": media_client = navidrome_client if not media_client: add_activity_item("", "Metadata Update", "Navidrome client not available", "Now") return jsonify({"success": False, "error": "Navidrome client not available"}), 400 else: # plex media_client = plex_client if not media_client: add_activity_item("", "Metadata Update", "Plex client not available", "Now") return jsonify({"success": False, "error": "Plex client not available"}), 400 logger.debug("Plex connection details: active_server=%s client=%s", active_server, media_client) if hasattr(media_client, 'server') and media_client.server: logger.debug( "Plex server details: url=%s name=%s", getattr(media_client.server, '_baseurl', 'NO_URL'), getattr(media_client.server, 'friendlyName', 'NO_NAME'), ) # Check available libraries try: sections = media_client.server.library.sections() logger.debug("Available Plex libraries: %s", [(s.title, s.type) for s in sections]) except Exception as e: logger.debug("Error getting Plex libraries: %s", e) else: logger.debug("Plex server is NOT connected!") # Check Spotify client - EXACTLY like dashboard.py if not spotify_client: add_activity_item("", "Metadata Update", "Spotify client not available", "Now") return jsonify({"success": False, "error": "Spotify client not available"}), 400 # Reset state metadata_update_state.update({ 'status': 'running', 'current_artist': 'Loading artists...', 'processed': 0, 'total': 0, 'percentage': 0.0, 'successful': 0, 'failed': 0, 'started_at': datetime.now(), 'completed_at': None, 'error': None, 'refresh_interval_days': refresh_interval_days }) # Start the metadata update worker - EXACTLY like dashboard.py def run_metadata_update(): global metadata_update_runtime_worker try: metadata_worker = WebMetadataUpdateWorker( None, # Artists will be loaded in the worker thread - EXACTLY like dashboard.py media_client, spotify_client, active_server, refresh_interval_days ) metadata_update_runtime_worker = metadata_worker metadata_worker.run() except Exception as e: logger.error(f"Error in metadata update worker: {e}") metadata_update_state['status'] = 'error' metadata_update_state['error'] = str(e) add_activity_item("", "Metadata Error", str(e), "Now") finally: metadata_update_runtime_worker = None metadata_update_worker = metadata_update_executor.submit(run_metadata_update) add_activity_item("", "Metadata Update", "Loading artists from library...", "Now") return jsonify({"success": True}) except Exception as e: logger.error(f"Error starting metadata update: {e}") metadata_update_state['status'] = 'error' metadata_update_state['error'] = str(e) return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata/stop', methods=['POST']) def stop_metadata_update(): """Stop the metadata update process""" global metadata_update_state try: if metadata_update_state['status'] == 'running': metadata_update_state['status'] = 'stopping' metadata_update_state['current_artist'] = 'Stopping...' add_activity_item("", "Metadata Update", "Stopping metadata update process", "Now") return jsonify({"success": True}) except Exception as e: logger.error(f"Error stopping metadata update: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata/status', methods=['GET']) def get_metadata_update_status(): """Get current metadata update status""" try: # Return a copy of the state with datetime serialization state_copy = metadata_update_state.copy() # Convert datetime objects to ISO format for JSON serialization if state_copy.get('started_at'): state_copy['started_at'] = state_copy['started_at'].isoformat() if state_copy.get('completed_at'): state_copy['completed_at'] = state_copy['completed_at'].isoformat() return jsonify({"success": True, "status": state_copy}) except Exception as e: logger.error(f"Error getting metadata update status: {e}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/active-media-server', methods=['GET']) def get_active_media_server(): """Get the currently active media server""" try: active_server = config_manager.get_active_media_server() return jsonify({"success": True, "active_server": active_server}) except Exception as e: logger.error(f"Error getting active media server: {e}") return jsonify({"success": False, "error": str(e)}), 500 # ================================= # # BEATPORT API ENDPOINTS # # ================================= # @app.route('/api/beatport/genres', methods=['GET']) def get_beatport_genres(): """Get current Beatport genres with images dynamically scraped from homepage""" try: logger.info("API request for Beatport genres") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters include_images = request.args.get('include_images', 'false').lower() == 'true' # Discover genres dynamically if include_images: logger.info("Including genre images in response (slower)") genres = scraper.discover_genres_with_images(include_images=True) else: logger.info("Returning genres without images (faster)") genres = scraper.discover_genres_from_homepage() logger.info(f"Successfully discovered {len(genres)} Beatport genres") return jsonify({ "success": True, "genres": genres, "count": len(genres), "includes_images": include_images }) except Exception as e: logger.error(f"Error fetching Beatport genres: {e}") return jsonify({ "success": False, "error": str(e), "genres": [], "count": 0 }), 500 @app.route('/api/beatport/genre///tracks', methods=['GET']) def get_beatport_genre_tracks(genre_slug, genre_id): """Get tracks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre tracks (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '100')) enrich = request.args.get('enrich', 'true').lower() != 'false' # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape tracks for this genre tracks = scraper.scrape_genre_charts(genre, limit=limit, enrich=enrich) logger.info(f"Successfully scraped {len(tracks)} tracks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching tracks for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/chart/extract', methods=['POST']) def extract_beatport_chart_tracks(): """Extract tracks from a specific Beatport chart URL""" try: data = request.get_json() chart_url = data.get('chart_url') chart_name = data.get('chart_name', 'Unknown Chart') limit = int(data.get('limit', 100)) if not chart_url: return jsonify({ "success": False, "error": "chart_url is required", "tracks": [], "count": 0 }), 400 enrich = data.get('enrich', True) logger.info(f"API request to extract tracks from chart: {chart_name}") logger.info(f"Chart URL: {chart_url}") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() if enrich: # Full extraction + enrichment (legacy synchronous path) tracks = scraper.extract_tracks_from_chart(chart_url, chart_name, limit) else: # Extract raw track list only (no per-track enrichment) soup = scraper.get_page(chart_url) tracks = [] if soup: tracks = scraper.extract_tracks_from_chart_table(soup, chart_name, limit) if len(tracks) < 10: general_tracks = scraper.extract_tracks_from_page(soup, f"New Chart: {chart_name}", limit) if len(general_tracks) > len(tracks): tracks = general_tracks if len(tracks) < 10: table_tracks = scraper.extract_tracks_from_table_format(soup, chart_name, limit) if len(table_tracks) > len(tracks): tracks = table_tracks logger.info(f"Successfully extracted {len(tracks)} tracks from chart: {chart_name}") return jsonify({ "success": True, "tracks": tracks, "chart_name": chart_name, "chart_url": chart_url, "count": len(tracks) }) except Exception as e: logger.error(f"Error extracting tracks from chart: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///top-10', methods=['GET']) def get_beatport_genre_top_10(genre_slug, genre_id): """Get top 10 tracks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre top 10 tracks (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape top 10 tracks for this genre tracks = scraper.scrape_genre_top_10(genre) logger.info(f"Successfully scraped {len(tracks)} top 10 tracks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching top 10 tracks for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///releases-top-10', methods=['GET']) def get_beatport_genre_releases_top_10(genre_slug, genre_id): """Get top 10 releases for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre top 10 releases (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape top 10 releases for this genre releases = scraper.scrape_genre_releases(genre, limit=10) logger.info(f"Successfully scraped {len(releases)} top 10 releases for {genre_slug}") return jsonify({ "success": True, "tracks": releases, "genre": genre, "count": len(releases) }) except Exception as e: logger.error(f"Error fetching top 10 releases for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///releases-top-100', methods=['GET']) def get_beatport_genre_releases_top_100(genre_slug, genre_id): """Get top 100 releases for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre top 100 releases (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '100')) # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape top releases for this genre releases = scraper.scrape_genre_releases(genre, limit=limit) logger.info(f"Successfully scraped {len(releases)} top 100 releases for {genre_slug}") return jsonify({ "success": True, "tracks": releases, "genre": genre, "count": len(releases) }) except Exception as e: logger.error(f"Error fetching top 100 releases for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///staff-picks', methods=['GET']) def get_beatport_genre_staff_picks(genre_slug, genre_id): """Get staff picks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre staff picks (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '50')) # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape staff picks for this genre tracks = scraper.scrape_genre_staff_picks(genre, limit=limit) logger.info(f"Successfully scraped {len(tracks)} staff picks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching staff picks for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///hype-top-10', methods=['GET']) def get_beatport_genre_hype_top_10(genre_slug, genre_id): """Get hype top 10 tracks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre hype top 10 (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape hype top 10 for this genre tracks = scraper.scrape_genre_hype_top_10(genre) logger.info(f"Successfully scraped {len(tracks)} hype top 10 tracks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching hype top 10 for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///hype-top-100', methods=['GET']) def get_beatport_genre_hype_top_100(genre_slug, genre_id): """Get hype top 100 tracks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre hype top 100 (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape hype top 100 for this genre tracks = scraper.scrape_genre_hype_charts(genre, limit=100) logger.info(f"Successfully scraped {len(tracks)} hype top 100 tracks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching hype top 100 for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///hype-picks', methods=['GET']) def get_beatport_genre_hype_picks(genre_slug, genre_id): """Get hype picks for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre hype picks (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '50')) # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape hype picks for this genre tracks = scraper.scrape_genre_hype_picks(genre, limit=limit) logger.info(f"Successfully scraped {len(tracks)} hype picks for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching hype picks for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///latest-releases', methods=['GET']) def get_beatport_genre_latest_releases(genre_slug, genre_id): """Get latest releases for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre latest releases (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '50')) # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape latest releases for this genre tracks = scraper.scrape_genre_latest_releases(genre, limit=limit) logger.info(f"Successfully scraped {len(tracks)} latest releases for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching latest releases for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///new-charts', methods=['GET']) def get_beatport_genre_new_charts(genre_slug, genre_id): """Get new charts for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre new charts (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '50')) # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Scrape new charts for this genre tracks = scraper.scrape_genre_new_charts(genre, limit=limit) logger.info(f"Successfully scraped {len(tracks)} new charts for {genre_slug}") return jsonify({ "success": True, "tracks": tracks, "genre": genre, "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching new charts for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre///hero', methods=['GET']) def get_beatport_genre_hero(genre_slug, genre_id): """Get hero slider data for a specific Beatport genre with 1-hour caching""" try: logger.info(f"API request for {genre_slug} genre hero slider (ID: {genre_id})") # Check cache first (1-hour TTL like other genre data) cache_key = f"hero_{genre_slug}_{genre_id}" cached_data = get_cached_beatport_data('genre', cache_key, genre_slug) if cached_data: logger.info(f"Returning cached hero data for {genre_slug}") return jsonify({ "success": True, "releases": cached_data, "count": len(cached_data), "genre_slug": genre_slug, "genre_id": genre_id, "cached": True, "cache_timestamp": time.time() }) # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Scrape hero slider data hero_releases = scraper.scrape_genre_hero_slider(genre_slug, genre_id) if hero_releases: # Cache the data (1-hour TTL) set_cached_beatport_data('genre', cache_key, hero_releases, genre_slug) logger.info(f"Successfully scraped and cached {len(hero_releases)} hero releases for {genre_slug}") return jsonify({ "success": True, "releases": hero_releases, "count": len(hero_releases), "genre_slug": genre_slug, "genre_id": genre_id, "cached": False, "scrape_timestamp": time.time() }) else: logger.info(f"No hero releases found for {genre_slug}") return jsonify({ "success": False, "releases": [], "count": 0, "genre_slug": genre_slug, "genre_id": genre_id, "message": "No hero releases found" }) except Exception as e: logger.error(f"Error fetching hero data for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "releases": [], "count": 0, "genre_slug": genre_slug, "genre_id": genre_id }), 500 @app.route('/api/beatport/genre///top-10-lists', methods=['GET']) def get_beatport_genre_top10_lists(genre_slug, genre_id): """Get Top 10 lists (Beatport + Hype) for a specific genre with 1-hour caching""" try: logger.info(f"API request for {genre_slug} Top 10 lists (ID: {genre_id})") # Check cache first (1-hour TTL) cached_data = get_cached_beatport_data('genre', 'top_10_lists', genre_slug) if cached_data: logger.info(f"Returning cached Top 10 lists for {genre_slug}") cached_data['success'] = True cached_data['cached'] = True return jsonify(cached_data) # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Scrape Top 10 lists from genre page top10_data = scraper.scrape_genre_top10_tracks(genre_slug, genre_id) if not top10_data['beatport_top10'] and not top10_data['hype_top10']: return jsonify({ "success": False, "error": "No Top 10 tracks found for this genre", "beatport_top10": [], "hype_top10": [], "beatport_count": 0, "hype_count": 0, "has_hype_section": False, "genre_slug": genre_slug, "genre_id": genre_id, "cached": False }) # Prepare response data response_data = { "beatport_top10": top10_data['beatport_top10'], "hype_top10": top10_data['hype_top10'], "beatport_count": len(top10_data['beatport_top10']), "hype_count": len(top10_data['hype_top10']), "has_hype_section": top10_data['has_hype_section'], "total_tracks": top10_data['total_tracks'], "genre_slug": genre_slug, "genre_id": genre_id, "cached": False, "cache_ttl": 3600 # 1 hour } # Cache the data (1-hour TTL) set_cached_beatport_data('genre', 'top_10_lists', response_data, genre_slug) logger.info(f"Successfully fetched {response_data['beatport_count']} Beatport + {response_data['hype_count']} Hype Top 10 tracks for {genre_slug}") response_data['success'] = True return jsonify(response_data) except Exception as e: logger.error(f"Error fetching Top 10 lists for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "beatport_top10": [], "hype_top10": [], "beatport_count": 0, "hype_count": 0, "has_hype_section": False, "genre_slug": genre_slug, "genre_id": genre_id, "cached": False }), 500 @app.route('/api/beatport/genre///top-10-releases', methods=['GET']) def get_beatport_genre_top10_releases(genre_slug, genre_id): """Get Top 10 releases for a specific genre using .partial-artwork elements with 1-hour caching""" try: logger.info(f"API request for {genre_slug} Top 10 releases (ID: {genre_id})") # Check cache first (1-hour TTL) cached_data = get_cached_beatport_data('genre', 'top_10_releases', genre_slug) if cached_data: logger.info(f"Returning cached Top 10 releases for {genre_slug}") cached_data['success'] = True cached_data['cached'] = True return jsonify(cached_data) # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Scrape Top 10 releases from genre page releases = scraper.scrape_genre_top10_releases(genre_slug, genre_id) if not releases: return jsonify({ "success": False, "error": "No Top 10 releases found for this genre", "releases": [], "releases_count": 0, "genre_slug": genre_slug, "genre_id": genre_id, "cached": False }) # Prepare response data response_data = { "releases": releases, "releases_count": len(releases), "genre_slug": genre_slug, "genre_id": genre_id, "cached": False, "cache_ttl": 3600 # 1 hour } # Cache the data (1-hour TTL) set_cached_beatport_data('genre', 'top_10_releases', response_data, genre_slug) logger.info(f"Successfully fetched {response_data['releases_count']} Top 10 releases for {genre_slug}") response_data['success'] = True return jsonify(response_data) except Exception as e: logger.error(f"Error fetching Top 10 releases for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "releases": [], "releases_count": 0, "genre_slug": genre_slug, "genre_id": genre_id, "cached": False }), 500 @app.route('/api/beatport/genre///sections', methods=['GET']) def get_beatport_genre_sections(genre_slug, genre_id): """Discover all available sections for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre sections discovery (ID: {genre_id})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Create genre dict for scraper genre = { 'name': genre_slug.replace('-', ' ').title(), 'slug': genre_slug, 'id': genre_id } # Discover sections for this genre sections = scraper.discover_genre_page_sections(genre) logger.info(f"Successfully discovered sections for {genre_slug}") return jsonify({ "success": True, "sections": sections, "genre": genre }) except Exception as e: logger.error(f"Error discovering sections for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "sections": {} }), 500 @app.route('/api/beatport/top-100', methods=['GET']) def get_beatport_top_100(): """Get Beatport Top 100 tracks""" try: logger.info("API request for Beatport Top 100") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '100')) enrich = request.args.get('enrich', 'true').lower() != 'false' # Scrape Top 100 tracks = scraper.scrape_top_100(limit=limit, enrich=enrich) logger.info(f"Successfully scraped {len(tracks)} tracks from Beatport Top 100") return jsonify({ "success": True, "tracks": tracks, "chart_name": "Beatport Top 100", "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching Beatport Top 100: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/genre-image//', methods=['GET']) def get_beatport_genre_image(genre_slug, genre_id): """Get image for a specific Beatport genre""" try: logger.info(f"API request for {genre_slug} genre image") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Construct genre URL genre_url = f"{scraper.base_url}/genre/{genre_slug}/{genre_id}" # Get genre image image_url = scraper.get_genre_image(genre_url) if image_url: logger.info(f"Found image for {genre_slug}") return jsonify({ "success": True, "image_url": image_url, "genre_slug": genre_slug, "genre_id": genre_id }) else: logger.info(f"No image found for {genre_slug}") return jsonify({ "success": False, "image_url": None, "genre_slug": genre_slug, "genre_id": genre_id }) except Exception as e: logger.error(f"Error fetching image for {genre_slug}: {e}") return jsonify({ "success": False, "error": str(e), "image_url": None }), 500 @app.route('/api/beatport/hype-top-100', methods=['GET']) def get_beatport_hype_top_100(): """Get Beatport Hype Top 100 - Improved with fixed URL""" try: logger.info("API request for Beatport Hype Top 100") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '100')) enrich = request.args.get('enrich', 'true').lower() != 'false' # Scrape Hype Top 100 using improved method tracks = scraper.scrape_hype_top_100(limit=limit, enrich=enrich) logger.info(f"Successfully scraped {len(tracks)} tracks from Beatport Hype Top 100") return jsonify({ "success": True, "tracks": tracks, "chart_name": "Beatport Hype Top 100", "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching Beatport Hype Top 100: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/top-100-releases', methods=['GET']) def get_beatport_top_100_releases(): """Get Beatport Top 100 Releases - New endpoint""" try: logger.info("API request for Beatport Top 100 Releases") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '100')) # Scrape Top 100 Releases using new method tracks = scraper.scrape_top_100_releases(limit=limit) logger.info(f"Successfully scraped {len(tracks)} tracks from Beatport Top 100 Releases") return jsonify({ "success": True, "tracks": tracks, "chart_name": "Top 100 New Releases", "count": len(tracks) }) except Exception as e: logger.error(f"Error fetching Beatport Top 100 Releases: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "count": 0 }), 500 @app.route('/api/beatport/homepage/new-releases', methods=['GET']) def get_beatport_homepage_new_releases(): """Get Beatport New Releases from homepage section""" try: limit = int(request.args.get('limit', 40)) logger.info(f"🆕 API request for Beatport homepage New Releases (limit: {limit})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get new releases from homepage new_releases = scraper.scrape_new_releases(limit=limit) logger.info(f"Successfully extracted {len(new_releases)} new releases from homepage") return jsonify({ "success": True, "tracks": new_releases, "track_count": len(new_releases), "source": "beatport_homepage_new_releases" }) except Exception as e: logger.error(f"Error getting Beatport homepage new releases: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "track_count": 0 }), 500 @app.route('/api/beatport/homepage/hype-picks', methods=['GET']) def get_beatport_homepage_hype_picks(): """Get Beatport Hype Picks from homepage section""" try: limit = int(request.args.get('limit', 40)) logger.info(f"API request for Beatport homepage Hype Picks (limit: {limit})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get hype picks from homepage hype_picks = scraper.scrape_hype_picks_homepage(limit=limit) logger.info(f"Successfully extracted {len(hype_picks)} hype picks from homepage") return jsonify({ "success": True, "tracks": hype_picks, "track_count": len(hype_picks), "source": "beatport_homepage_hype_picks" }) except Exception as e: logger.error(f"Error getting Beatport homepage hype picks: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "track_count": 0 }), 500 @app.route('/api/beatport/homepage/top-10-releases', methods=['GET']) def get_beatport_homepage_top_10_releases(): """Get Beatport Top 10 Releases from homepage section""" try: limit = int(request.args.get('limit', 10)) logger.info(f"API request for Beatport homepage Top 10 Releases (limit: {limit})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get top 10 releases from homepage top_10_releases = scraper.scrape_top_10_releases_homepage(limit=limit) logger.info(f"Successfully extracted {len(top_10_releases)} top 10 releases from homepage") return jsonify({ "success": True, "tracks": top_10_releases, "track_count": len(top_10_releases), "source": "beatport_homepage_top_10_releases" }) except Exception as e: logger.error(f"Error getting Beatport homepage top 10 releases: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "track_count": 0 }), 500 @app.route('/api/beatport/homepage/top-10-lists', methods=['GET']) def get_beatport_homepage_top10_lists(): """Get Beatport Top 10 Lists from homepage - both Beatport Top 10 and Hype Top 10""" try: logger.info("API request for Beatport homepage Top 10 Lists") # Check cache first cached_data = get_cached_beatport_data('homepage', 'top_10_lists') if cached_data: logger.info("Returning cached top 10 lists data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh top 10 lists data...") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get top 10 lists from homepage top10_lists = scraper.scrape_homepage_top10_lists() logger.info(f"Successfully extracted Beatport Top 10: {len(top10_lists['beatport_top10'])}, Hype Top 10: {len(top10_lists['hype_top10'])}") # Prepare response data response_data = { "success": True, "beatport_top10": top10_lists["beatport_top10"], "hype_top10": top10_lists["hype_top10"], "beatport_count": len(top10_lists["beatport_top10"]), "hype_count": len(top10_lists["hype_top10"]), "source": "beatport_homepage_top10_lists" } # Cache the successful response set_cached_beatport_data('homepage', 'top_10_lists', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error getting Beatport homepage top 10 lists: {e}") return jsonify({ "success": False, "error": str(e), "beatport_top10": [], "hype_top10": [], "beatport_count": 0, "hype_count": 0 }), 500 @app.route('/api/beatport/homepage/top-10-releases-cards', methods=['GET']) def get_beatport_homepage_top10_releases_cards(): """Get Beatport Top 10 Releases CARDS from homepage (not individual tracks)""" try: logger.info("API request for Beatport homepage Top 10 Releases CARDS") # Check cache first cached_data = get_cached_beatport_data('homepage', 'top_10_releases') if cached_data: logger.info("Returning cached top 10 releases data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh top 10 releases data...") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get top 10 releases from homepage top10_releases = scraper.scrape_homepage_top10_releases() logger.info(f"API extracted {len(top10_releases)} Top 10 Release Cards") # Debug: Log first release if any if top10_releases: logger.info(f"First release: {top10_releases[0].get('title', 'No title')} by {top10_releases[0].get('artist', 'No artist')}") else: logger.warning("No releases found by scraper") # Prepare response data response_data = { "success": True, "releases": top10_releases, "releases_count": len(top10_releases), "source": "beatport_homepage_top10_releases_cards" } # Cache the successful response set_cached_beatport_data('homepage', 'top_10_releases', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error getting Beatport homepage Top 10 Releases cards: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") return jsonify({ "success": False, "error": str(e), "releases": [], "releases_count": 0 }), 500 @app.route('/api/beatport/scrape-releases', methods=['POST']) def scrape_beatport_releases(): """General scraper endpoint - takes release URLs and returns tracks""" try: data = request.get_json() if not data: return jsonify({ "success": False, "error": "No JSON data provided", "tracks": [], "track_count": 0 }), 400 release_urls = data.get('release_urls', []) source_name = data.get('source_name', 'General Release Scraper') if not release_urls: return jsonify({ "success": False, "error": "No release URLs provided", "tracks": [], "track_count": 0 }), 400 logger.info(f"API request to scrape {len(release_urls)} release URLs with source: {source_name}") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Use our new general scraper function tracks = scraper.scrape_multiple_releases(release_urls, source_name) logger.info(f"Successfully extracted {len(tracks)} tracks from {len(release_urls)} releases") # Apply text cleaning to track data cleaned_tracks = [] for track in tracks: cleaned_track = track.copy() if 'title' in cleaned_track: cleaned_track['title'] = clean_beatport_text(cleaned_track['title']) if 'artist' in cleaned_track: cleaned_track['artist'] = clean_beatport_text(cleaned_track['artist']) if 'label' in cleaned_track: cleaned_track['label'] = clean_beatport_text(cleaned_track['label']) cleaned_tracks.append(cleaned_track) return jsonify({ "success": True, "tracks": cleaned_tracks, "track_count": len(cleaned_tracks), "source": source_name, "release_urls_processed": len(release_urls) }) except Exception as e: logger.error(f"Error scraping releases: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") return jsonify({ "success": False, "error": str(e), "tracks": [], "track_count": 0 }), 500 @app.route('/api/beatport/release-metadata', methods=['POST']) def get_beatport_release_metadata(): """Fetch structured release metadata for direct download modal (skip discovery)""" try: data = request.get_json() if not data: return jsonify({"success": False, "error": "No JSON data provided"}), 400 release_url = data.get('release_url', '') if not release_url: return jsonify({"success": False, "error": "No release_url provided"}), 400 logger.info(f"API request for release metadata: {release_url}") scraper = BeatportUnifiedScraper() result = scraper.get_release_metadata(release_url) if not result.get('success'): return jsonify(result), 404 # Apply text cleaning album = result['album'] artist = result['artist'] album['name'] = clean_beatport_text(album['name']) artist['name'] = clean_beatport_text(artist['name']) for track in result['tracks']: track['name'] = clean_beatport_text(track['name']) for a in track.get('artists', []): a['name'] = clean_beatport_text(a['name']) # Update the embedded album name too track['album']['name'] = album['name'] logger.info(f"Release metadata: {album['name']} by {artist['name']} ({len(result['tracks'])} tracks)") return jsonify(result) except Exception as e: logger.error(f"Error getting release metadata: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") return jsonify({"success": False, "error": str(e)}), 500 # Active enrichment tasks — progress tracked here, polled by frontend _enrichment_tasks = {} # enrichment_id -> {completed, total, current_track, done, tracks, error} _enrichment_tasks_lock = threading.Lock() @app.route('/api/beatport/enrich-tracks', methods=['POST']) def enrich_beatport_tracks(): """Start Beatport track enrichment. Returns immediately; poll /enrich-progress for updates.""" try: data = request.get_json() if not data: return jsonify({"success": False, "error": "No JSON data provided"}), 400 tracks = data.get('tracks', []) if not tracks: return jsonify({"success": False, "error": "No tracks provided"}), 400 enrichment_id = data.get('enrichment_id', str(uuid.uuid4())) logger.info(f"Enriching {len(tracks)} Beatport tracks with per-track metadata (id: {enrichment_id})") # --- Check enrichment cache (fast, do before spawning background) --- cached_results = {} uncached_tracks = [] uncached_indices = [] from core.metadata_cache import get_metadata_cache mcache = get_metadata_cache() for i, track in enumerate(tracks): url = track.get('url') or track.get('track_url') or '' if url: cached = mcache.get_entity('beatport', 'track', url) if cached: cached_results[i] = cached continue uncached_tracks.append(track) uncached_indices.append(i) cache_hits = len(cached_results) cache_misses = len(uncached_tracks) logger.info(f"Enrichment cache: {cache_hits} hits, {cache_misses} misses") # All cached — return immediately (no background task needed) if cache_misses == 0: merged = [None] * len(tracks) for idx, d in cached_results.items(): merged[idx] = d for i in range(len(merged)): if merged[i] is None: merged[i] = tracks[i] return jsonify({"success": True, "tracks": merged}) # --- Initialize progress tracker and start background task --- with _enrichment_tasks_lock: _enrichment_tasks[enrichment_id] = { 'completed': cache_hits, 'total': len(tracks), 'current_track': f'{cache_hits} tracks (cached)' if cache_hits > 0 else '', 'done': False, 'tracks': None, 'error': None, } def _run_enrichment(): try: def on_progress(completed, total, track_name): new_completed = cache_hits + completed with _enrichment_tasks_lock: task = _enrichment_tasks.get(enrichment_id) if task: task['completed'] = new_completed task['current_track'] = track_name else: logger.warning(f"on_progress: task {enrichment_id} not found in _enrichment_tasks!") scraper = BeatportUnifiedScraper() newly_enriched = scraper.enrich_chart_tracks(uncached_tracks, progress_callback=on_progress) # Clean and cache for track in newly_enriched: if track.get('title'): track['title'] = clean_beatport_text(track['title']) if track.get('artist'): track['artist'] = clean_beatport_text(track['artist']) if track.get('release_name'): track['release_name'] = clean_beatport_text(track['release_name']) if track.get('label'): track['label'] = clean_beatport_text(track['label']) url = track.get('url') or track.get('track_url') or '' if url: mcache.store_entity('beatport', 'track', url, track) # Merge in original order merged = [None] * len(tracks) for idx, d in cached_results.items(): merged[idx] = d for j, idx in enumerate(uncached_indices): if j < len(newly_enriched): merged[idx] = newly_enriched[j] for i in range(len(merged)): if merged[i] is None: merged[i] = tracks[i] logger.info(f"Enriched {len(merged)} tracks ({cache_hits} cached, {cache_misses} scraped)") with _enrichment_tasks_lock: task = _enrichment_tasks.get(enrichment_id) if task: task['done'] = True task['tracks'] = merged task['completed'] = len(tracks) except Exception as e: logger.error(f"Error enriching tracks: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") with _enrichment_tasks_lock: task = _enrichment_tasks.get(enrichment_id) if task: task['done'] = True task['error'] = str(e) task['tracks'] = tracks # Return originals as fallback threading.Thread(target=_run_enrichment, daemon=True).start() return jsonify({"success": True, "enrichment_id": enrichment_id, "async": True}) except Exception as e: logger.error(f"Error starting enrichment: {e}") import traceback logger.error(f"Full traceback: {traceback.format_exc()}") return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/beatport/enrich-progress/', methods=['GET']) def get_enrichment_progress(enrichment_id): """Poll enrichment progress. Returns current state; includes tracks when done.""" with _enrichment_tasks_lock: task = _enrichment_tasks.get(enrichment_id) if not task: return jsonify({"success": False, "error": "Unknown enrichment ID"}), 404 result = { 'success': True, 'completed': task['completed'], 'total': task['total'], 'current_track': task['current_track'], 'done': task['done'], } if task['done']: result['tracks'] = task['tracks'] result['error'] = task['error'] # Clean up — task is finished del _enrichment_tasks[enrichment_id] resp = jsonify(result) resp.headers['Cache-Control'] = 'no-store' return resp @app.route('/api/beatport/homepage/featured-charts', methods=['GET']) def get_beatport_homepage_featured_charts(): """Get Beatport Featured Charts from homepage section""" try: limit = int(request.args.get('limit', 20)) logger.info(f"API request for Beatport homepage Featured Charts (limit: {limit})") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get featured charts from homepage featured_charts = scraper.scrape_featured_charts(limit=limit) logger.info(f"Successfully extracted {len(featured_charts)} featured charts from homepage") return jsonify({ "success": True, "tracks": featured_charts, "track_count": len(featured_charts), "source": "beatport_homepage_featured_charts" }) except Exception as e: logger.error(f"Error getting Beatport homepage featured charts: {e}") return jsonify({ "success": False, "error": str(e), "tracks": [], "track_count": 0 }), 500 @app.route('/api/beatport/chart-sections', methods=['GET']) def get_beatport_chart_sections(): """Get dynamically discovered Beatport chart sections""" try: logger.info("API request for Beatport chart sections discovery") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Discover chart sections dynamically chart_sections = scraper.discover_chart_sections() logger.info("Successfully discovered chart sections") return jsonify({ "success": True, "chart_sections": chart_sections, "summary": chart_sections.get('summary', {}) }) except Exception as e: logger.error(f"Error discovering Beatport chart sections: {e}") return jsonify({ "success": False, "error": str(e), "chart_sections": {}, "summary": {} }), 500 @app.route('/api/beatport/dj-charts-improved', methods=['GET']) def get_beatport_dj_charts_improved(): """Get Beatport DJ Charts using improved method""" try: logger.info("API request for Beatport DJ Charts (improved)") # Initialize the Beatport scraper scraper = BeatportUnifiedScraper() # Get query parameters limit = int(request.args.get('limit', '20')) # Scrape DJ Charts using improved method charts = scraper.scrape_dj_charts(limit=limit) logger.info(f"Successfully scraped {len(charts)} DJ charts") return jsonify({ "success": True, "charts": charts, "chart_name": "Beatport DJ Charts", "count": len(charts) }) except Exception as e: logger.error(f"Error fetching Beatport DJ Charts: {e}") return jsonify({ "success": False, "error": str(e), "charts": [], "count": 0 }), 500 @app.route('/api/beatport/hype-picks') def get_beatport_hype_picks(): """Get Beatport Hype Picks for the rebuild slider grid (EXACT same pattern as new-releases)""" try: logger.info("Fetching Beatport hype picks...") # Check cache first cached_data = get_cached_beatport_data('homepage', 'hype_picks') if cached_data: logger.info("Returning cached hype picks data") response = jsonify(cached_data) return add_cache_headers(response, 3600) # 1 hour # Cache miss - scrape fresh data logger.info("Cache miss - scraping fresh hype picks data...") # Initialize scraper scraper = BeatportUnifiedScraper() # Get page and extract releases soup = scraper.get_page(scraper.base_url) if not soup: raise Exception("Could not fetch Beatport homepage") # Extract hype pick cards using data-testid selector (equivalent to new-releases CSS selector) hype_pick_cards = soup.select('[data-testid="hype-picks"]') releases = [] logger.info(f"Found {len(hype_pick_cards)} hype pick cards") for _i, card in enumerate(hype_pick_cards[:100]): # Limit to 100 for 10 slides (same as new-releases) release_data = {} # Extract title (exact same logic as new-releases) title_elem = card.select_one('[class*="title"], [class*="Title"], h1, h2, h3, h4, h5, h6') if title_elem: title_text = title_elem.get_text(strip=True) if title_text and len(title_text) > 2 and title_text not in ['Hype Picks', 'Buy', 'Play']: release_data['title'] = title_text # Extract artist (exact same logic as new-releases) artist_elem = card.select_one('[class*="artist"], [class*="Artist"], a[href*="/artist/"]') if artist_elem: artist_text = artist_elem.get_text(strip=True) if artist_text and len(artist_text) > 1: release_data['artist'] = artist_text # Extract label (exact same logic as new-releases) label_elem = card.select_one('[class*="label"], [class*="Label"], a[href*="/label/"]') if label_elem: label_text = label_elem.get_text(strip=True) if label_text and len(label_text) > 1: release_data['label'] = label_text # Extract URL (exact same logic as new-releases) url_link = card.select_one('a[href*="/release/"]') if url_link: href = url_link.get('href') if href: release_data['url'] = urljoin(scraper.base_url, href) # Extract image (exact same logic as new-releases) img = card.select_one('img') if img: src = img.get('src') or img.get('data-src') or img.get('data-lazy-src') if src: release_data['image_url'] = src # URL fallback for title (exact same logic as new-releases) if not release_data.get('title') and release_data.get('url'): url_parts = release_data['url'].split('/release/') if len(url_parts) > 1: slug = url_parts[1].split('/')[0] release_data['title'] = slug.replace('-', ' ').title() # Only add if we have essential data (exact same logic as new-releases) if release_data.get('title') and release_data.get('url'): # Add fallbacks for missing data (exact same logic as new-releases) if not release_data.get('artist'): release_data['artist'] = 'Various Artists' if not release_data.get('label'): release_data['label'] = 'Unknown Label' releases.append(release_data) logger.info(f"Successfully extracted {len(releases)} hype picks") # Prepare response data response_data = { 'success': True, 'releases': releases, 'count': len(releases), 'slides': (len(releases) + 9) // 10, # Calculate number of slides needed (same as new-releases) 'timestamp': datetime.now().isoformat() } # Cache the successful response set_cached_beatport_data('homepage', 'hype_picks', response_data) response = jsonify(response_data) return add_cache_headers(response, 3600) # 1 hour except Exception as e: logger.error(f"Error getting Beatport hype picks: {e}") return jsonify({ 'success': False, 'error': str(e), 'releases': [], 'count': 0 }), 500 @app.route('/api/beatport/discovery/start/', methods=['POST']) def start_beatport_discovery(url_hash): """Start Spotify discovery for Beatport chart tracks""" import json try: logger.info(f"Starting Beatport discovery for: {url_hash}") # Get chart data from request body data = request.get_json() or {} logger.debug(f"Raw request data: {data}") chart_data = data.get('chart_data') logger.debug(f"Chart data extracted: {chart_data is not None}") # Debug logging if chart_data: logger.debug(f"Chart data keys: {list(chart_data.keys()) if isinstance(chart_data, dict) else 'Not a dict'}") logger.debug(f"Chart name: {chart_data.get('name') if isinstance(chart_data, dict) else 'N/A'}") if isinstance(chart_data, dict) and 'tracks' in chart_data: logger.debug(f"Number of tracks: {len(chart_data['tracks'])}") if chart_data['tracks']: logger.debug(f"First track: {chart_data['tracks'][0]}") else: logger.warning("No chart data received") if not chart_data or not chart_data.get('tracks'): return jsonify({"error": "Chart data with tracks is required"}), 400 # Initialize Beatport chart state (similar to YouTube) if url_hash not in beatport_chart_states: beatport_chart_states[url_hash] = { 'chart': chart_data, 'phase': 'fresh', 'discovery_results': [], 'discovery_progress': 0, 'spotify_matches': 0, 'spotify_total': len(chart_data['tracks']), 'status': 'fresh', 'last_accessed': time.time() } state = beatport_chart_states[url_hash] state['last_accessed'] = time.time() if state['phase'] == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 # Update phase to discovering state['phase'] = 'discovering' state['status'] = 'discovering' state['discovery_progress'] = 0 state['spotify_matches'] = 0 # Add activity for discovery start chart_name = chart_data.get('name', 'Unknown Chart') track_count = len(chart_data['tracks']) add_activity_item("", "Beatport Discovery Started", f"'{chart_name}' - {track_count} tracks", "Now") # Start discovery worker (capture profile ID while we have Flask context) beatport_chart_states[url_hash]['_profile_id'] = get_current_profile_id() future = beatport_discovery_executor.submit(_run_beatport_discovery_worker, url_hash) state['discovery_future'] = future logger.info(f"Started Spotify discovery for Beatport chart: {chart_name}") return jsonify({"success": True, "message": "Discovery started", "status": "discovering"}) except Exception as e: logger.error(f"Error starting Beatport discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/discovery/status/', methods=['GET']) def get_beatport_discovery_status(url_hash): """Get real-time discovery status for a Beatport chart""" try: if url_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 state = beatport_chart_states[url_hash] state['last_accessed'] = time.time() response = { 'phase': state['phase'], 'status': state['status'], 'progress': state['discovery_progress'], 'spotify_matches': state['spotify_matches'], 'spotify_total': state['spotify_total'], 'results': state['discovery_results'], 'complete': state['phase'] == 'discovered' } return jsonify(response) except Exception as e: logger.error(f"Error getting Beatport discovery status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/discovery/update_match', methods=['POST']) def update_beatport_discovery_match(): """Update a Beatport discovery result with manually selected Spotify track""" try: data = request.get_json() identifier = data.get('identifier') # url_hash track_index = data.get('track_index') spotify_track = data.get('spotify_track') if not identifier or track_index is None or not spotify_track: return jsonify({'error': 'Missing required fields'}), 400 # Get the state state = beatport_chart_states.get(identifier) if not state: return jsonify({'error': 'Discovery state not found'}), 404 if track_index >= len(state['discovery_results']): return jsonify({'error': 'Invalid track index'}), 400 # Update the result result = state['discovery_results'][track_index] old_status = result.get('status') # Update with user-selected track result['status'] = 'Found' result['status_class'] = 'found' result['spotify_track'] = spotify_track['name'] result['spotify_artist'] = _join_artist_names(spotify_track['artists']) if isinstance(spotify_track['artists'], list) else _extract_artist_name(spotify_track['artists']) result['spotify_album'] = spotify_track['album'] result['spotify_id'] = spotify_track['id'] # Format duration (Beatport doesn't show duration in table, but store it anyway) duration_ms = spotify_track.get('duration_ms', 0) if duration_ms: minutes = duration_ms // 60000 seconds = (duration_ms % 60000) // 1000 result['duration'] = f"{minutes}:{seconds:02d}" else: result['duration'] = '0:00' # IMPORTANT: Also set spotify_data for sync/download compatibility. # Manual match from the fix modal — build a rich spotify_data (album # as dict with image info) matching the normal discovery shape, and # explicitly clear any prior wing-it flag since the user picked a # real metadata match. result['spotify_data'] = _build_fix_modal_spotify_data(spotify_track) result['wing_it_fallback'] = False result['manual_match'] = True # Flag for tracking # Update match count if status changed from not found/error if old_status != 'found' and old_status != 'Found': state['spotify_matches'] = state.get('spotify_matches', 0) + 1 logger.info(f"Manual match updated: beatport - {identifier} - track {track_index}") logger.info(f" → {result['spotify_artist']} - {result['spotify_track']}") return jsonify({'success': True, 'result': result}) except Exception as e: logger.error(f"Error updating Beatport discovery match: {e}") return jsonify({'error': str(e)}), 500 def clean_beatport_text(text): """Clean Beatport track/artist text for proper spacing""" if not text: return text import re # Fix common spacing issues text = re.sub(r'([a-z$!@#%&*])([A-Z])', r'\1 \2', text) # Add space between lowercase/symbols and uppercase text = re.sub(r'([a-zA-Z]),([a-zA-Z])', r'\1, \2', text) # Add space after comma text = re.sub(r'([a-zA-Z])(Mix|Remix|Extended|Version)\b', r'\1 \2', text) # Fix mix types text = re.sub(r'\s+', ' ', text) # Collapse multiple spaces text = text.strip() return text def _run_beatport_discovery_worker(url_hash): """Background worker for Beatport discovery process (Spotify preferred, iTunes fallback)""" _ew_state = {} try: _ew_state = _pause_enrichment_workers('Beatport discovery') state = beatport_chart_states[url_hash] chart = state['chart'] tracks = chart['tracks'] # Determine which provider to use discovery_source = _get_active_discovery_source() use_spotify = (discovery_source == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Initialize fallback client if needed itunes_client_instance = None if not use_spotify: itunes_client_instance = _get_metadata_fallback_client() logger.info(f"Starting {discovery_source.upper()} discovery for {len(tracks)} Beatport tracks...") # Store discovery source in state for frontend state['discovery_source'] = discovery_source # Process each track for discovery for i, track in enumerate(tracks): try: # Check for cancellation if state.get('phase') != 'discovering': logger.warning(f"Beatport discovery cancelled (phase changed to '{state.get('phase')}')") return # Update progress state['discovery_progress'] = int((i / len(tracks)) * 100) # Get track info from Beatport data (frontend sends 'name' and 'artists' fields) track_title = clean_beatport_text(track.get('name', 'Unknown Title')) track_artists = track.get('artists', ['Unknown Artist']) # Handle artists - could be a list or string if isinstance(track_artists, list): if len(track_artists) > 0 and isinstance(track_artists[0], str): # Handle case like ["CID,Taylr Renee"] - split on comma and clean track_artist = clean_beatport_text(track_artists[0].split(',')[0].strip()) else: track_artist = clean_beatport_text(track_artists[0] if track_artists else 'Unknown Artist') else: track_artist = clean_beatport_text(str(track_artists)) logger.debug(f"Searching {discovery_source.upper()} for: '{track_artist}' - '{track_title}'") # Check discovery cache first cache_key = _get_discovery_cache_key(track_title, track_artist) try: cache_db = get_database() cached_match = cache_db.get_discovery_cache_match(cache_key[0], cache_key[1], discovery_source) if cached_match and _validate_discovery_cache_artist(track_artist, cached_match): logger.debug(f"CACHE HIT [{i+1}/{len(tracks)}]: {track_artist} - {track_title}") # Convert artists from ['str'] to [{'name': 'str'}] for Beatport frontend format beatport_artists = cached_match.get('artists', []) if beatport_artists and isinstance(beatport_artists[0], str): cached_match['artists'] = [{'name': a} for a in beatport_artists] result_entry = { 'index': i, 'beatport_track': { 'title': track_title, 'artist': track_artist }, 'status': 'found', 'status_class': 'found', 'discovery_source': discovery_source, 'spotify_data': cached_match } state['spotify_matches'] += 1 state['discovery_results'].append(result_entry) continue except Exception as cache_err: logger.error(f"Cache lookup error: {cache_err}") # Use matching engine for track matching found_track = None best_confidence = 0.0 best_raw_track = None min_confidence = 0.9 # Higher threshold for Beatport to avoid bad matches # Generate search queries using matching engine (with fallback) try: temp_track = type('TempTrack', (), { 'name': track_title, 'artists': [track_artist], 'album': None })() search_queries = matching_engine.generate_download_queries(temp_track) logger.debug(f"Generated {len(search_queries)} search queries using matching engine") except Exception as e: logger.error(f"Matching engine failed for Beatport, falling back to basic queries: {e}") if use_spotify: search_queries = [ f"{track_artist} {track_title}", f'artist:"{track_artist}" track:"{track_title}"', f'"{track_artist}" "{track_title}"' ] else: search_queries = [ f"{track_artist} {track_title}", f"{track_title} {track_artist}", track_title ] for query_idx, search_query in enumerate(search_queries): try: logger.debug(f"Query {query_idx + 1}/{len(search_queries)}: {search_query} ({discovery_source.upper()})") search_results = None if use_spotify and not _spotify_rate_limited(): search_results = spotify_client.search_tracks(search_query, limit=10) else: search_results = itunes_client_instance.search_tracks(search_query, limit=10) if not search_results: continue # Score all results using the matching engine match, confidence, match_idx = _discovery_score_candidates( track_title, track_artist, 0, search_results ) if match and confidence > best_confidence and confidence >= min_confidence: best_confidence = confidence found_track = match if use_spotify and match.id: _cache = get_metadata_cache() best_raw_track = _cache.get_entity('spotify', 'track', match.id) else: best_raw_track = None logger.debug(f"New best Beatport match: {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") if best_confidence >= 0.9: logger.debug(f"High confidence match found ({best_confidence:.3f}), stopping search") break except Exception as e: logger.debug(f"Error in {discovery_source.upper()} search for query '{search_query}': {e}") continue # Strategy 4: Extended search with higher limit (last resort) if not found_track: logger.debug("Beatport Strategy 4: Extended search with limit=50") query = f"{track_artist} {track_title}" if use_spotify: extended_results = spotify_client.search_tracks(query, limit=50) else: extended_results = itunes_client_instance.search_tracks(query, limit=50) if extended_results: match, confidence, _ = _discovery_score_candidates( track_title, track_artist, 0, extended_results ) if match and confidence >= min_confidence: found_track = match best_confidence = confidence logger.debug(f"Strategy 4 Beatport match (extended): {match.artists[0]} - {match.name} (confidence: {confidence:.3f})") if found_track: logger.info(f"Final Beatport match: {found_track.artists[0]} - {found_track.name} (confidence: {best_confidence:.3f})") else: logger.warning(f"No suitable match found (best confidence was {best_confidence:.3f}, required {min_confidence:.3f})") # Create result entry result_entry = { 'index': i, # Add index for frontend table row identification 'beatport_track': { 'title': track_title, 'artist': track_artist }, 'status': 'found' if found_track else 'not_found', 'status_class': 'found' if found_track else 'not-found', 'discovery_source': discovery_source, 'confidence': best_confidence } if found_track: if use_spotify: # SPOTIFY result formatting # Debug: show available attributes logger.debug(f"Spotify track attributes: {dir(found_track)}") # Format artists correctly for frontend compatibility formatted_artists = [] if isinstance(found_track.artists, list): # If it's already a list of strings, convert to objects with 'name' property for artist in found_track.artists: if isinstance(artist, str): formatted_artists.append({'name': artist}) else: # If it's already an object, use as-is formatted_artists.append(artist) else: # Single artist case formatted_artists = [{'name': str(found_track.artists)}] # Use full album object from raw Spotify data if available album_data = best_raw_track.get('album', {}) if best_raw_track else {} if not album_data: # Fallback to string album name album_data = {'name': found_track.album, 'album_type': 'album', 'release_date': getattr(found_track, 'release_date', '') or '', 'images': []} result_entry['spotify_data'] = { 'name': found_track.name, 'artists': formatted_artists, # Now formatted as list of objects with 'name' property 'album': album_data, # Full album object with images 'id': found_track.id, 'source': 'spotify' } else: # ITUNES result formatting # Note: iTunes Track dataclass has 'artists' (list) and 'image_url', not 'artist' and 'artwork_url' result_artists = found_track.artists if hasattr(found_track, 'artists') else [] result_artist = result_artists[0] if result_artists else 'Unknown' result_name = found_track.name if hasattr(found_track, 'name') else 'Unknown' album_name = found_track.album if hasattr(found_track, 'album') else 'Unknown Album' image_url = found_track.image_url if hasattr(found_track, 'image_url') else '' track_id = found_track.id if hasattr(found_track, 'id') else '' # Format artists as list of objects for frontend compatibility formatted_artists = [{'name': result_artist}] # Build album data with artwork album_data = { 'name': album_name, 'album_type': 'album', 'release_date': getattr(found_track, 'release_date', '') or '', 'images': [{'url': image_url, 'height': 300, 'width': 300}] if image_url else [] } result_entry['spotify_data'] = { # Use same key for frontend compatibility 'name': result_name, 'artists': formatted_artists, 'album': album_data, 'id': track_id, 'source': discovery_source } state['spotify_matches'] += 1 # Save to discovery cache (normalize artists from [{name:str}] to [str] for canonical format) if best_confidence >= 0.75: try: cache_data = dict(result_entry['spotify_data']) cache_artists = cache_data.get('artists', []) if cache_artists and isinstance(cache_artists[0], dict): cache_data['artists'] = [a.get('name', '') for a in cache_artists] # Extract image URL for discovery pool display if 'image_url' not in cache_data: _bp_album = cache_data.get('album', {}) _bp_images = _bp_album.get('images', []) if isinstance(_bp_album, dict) else [] cache_data['image_url'] = _bp_images[0].get('url', '') if _bp_images else '' cache_db = get_database() cache_db.save_discovery_cache_match( cache_key[0], cache_key[1], discovery_source, best_confidence, cache_data, track_title, track_artist ) logger.debug(f"CACHE SAVED: {track_artist} - {track_title} (confidence: {best_confidence:.3f})") except Exception as cache_err: logger.error(f"Cache save error: {cache_err}") # Auto Wing It fallback for unmatched tracks if result_entry.get('status_class') == 'not-found': bp_t = result_entry.get('beatport_track', {}) stub = _build_discovery_wing_it_stub( bp_t.get('title', ''), bp_t.get('artist', ''), ) result_entry['status'] = 'found' result_entry['status_class'] = 'wing-it' result_entry['spotify_data'] = stub result_entry['match_data'] = stub result_entry['wing_it_fallback'] = True result_entry['confidence'] = 0 state['spotify_matches'] = state.get('spotify_matches', 0) + 1 state['wing_it_count'] = state.get('wing_it_count', 0) + 1 state['discovery_results'].append(result_entry) # Small delay to avoid rate limiting time.sleep(0.1) except Exception as e: logger.error(f"Error processing Beatport track {i}: {e}") # Add error result state['discovery_results'].append({ 'index': i, # Add index for frontend table row identification 'beatport_track': { 'title': track.get('name', 'Unknown'), # Changed from 'title' to 'name' to match track structure 'artist': track.get('artists', ['Unknown'])[0] if isinstance(track.get('artists'), list) else 'Unknown' }, 'status': 'error', 'status_class': 'error', # Add status class for CSS styling 'error': str(e), 'discovery_source': discovery_source }) # Mark discovery as complete state['discovery_progress'] = 100 state['phase'] = 'discovered' state['status'] = 'discovered' # Add activity for completion chart_name = chart.get('name', 'Unknown Chart') source_label = discovery_source.upper() add_activity_item("", f"Beatport Discovery Complete ({source_label})", f"'{chart_name}' - {state['spotify_matches']}/{len(tracks)} tracks found", "Now") logger.info(f"Beatport discovery complete ({source_label}): {state['spotify_matches']}/{len(tracks)} tracks found") # Sync discovery results back to mirrored playlist _sync_discovery_results_to_mirrored('beatport', url_hash, state.get('discovery_results', []), discovery_source, profile_id=state.get('_profile_id', 1)) except Exception as e: logger.error(f"Error in Beatport discovery worker: {e}") if url_hash in beatport_chart_states: beatport_chart_states[url_hash]['status'] = 'error' beatport_chart_states[url_hash]['phase'] = 'fresh' finally: _resume_enrichment_workers(_ew_state, 'Beatport discovery') @app.route('/api/beatport/sync/start/', methods=['POST']) def start_beatport_sync(url_hash): """Start sync process for a Beatport chart using discovered Spotify tracks""" try: logger.info(f"Beatport sync start requested for: {url_hash}") if url_hash not in beatport_chart_states: logger.warning(f"Beatport chart not found: {url_hash}") return jsonify({"error": "Beatport chart not found"}), 404 state = beatport_chart_states[url_hash] state['last_accessed'] = time.time() # Update access time logger.info(f"Beatport chart state: phase={state.get('phase')}, has_discovery_results={len(state.get('discovery_results', []))}") if state['phase'] not in ['discovered', 'sync_complete', 'download_complete']: logger.info(f"Beatport chart not ready for sync: {state['phase']}") return jsonify({"error": "Beatport chart not ready for sync"}), 400 # Convert discovery results to Spotify tracks format spotify_tracks = convert_beatport_results_to_spotify_tracks(state['discovery_results']) if not spotify_tracks: return jsonify({"error": "No Spotify matches found for sync"}), 400 # Create a temporary playlist ID for sync tracking sync_playlist_id = f"beatport_sync_{url_hash}_{int(time.time())}" # Initialize sync state state['sync_playlist_id'] = sync_playlist_id state['phase'] = 'syncing' state['sync_progress'] = {'status': 'starting', 'progress': 0} # Create sync job using existing infrastructure sync_data = { 'id': sync_playlist_id, 'name': state['chart']['name'], 'tracks': spotify_tracks, 'source': 'beatport', 'source_id': url_hash } # Add to sync states using existing sync system with sync_lock: sync_states[sync_playlist_id] = {"status": "starting", "progress": {}} # Start sync in background using existing thread pool future = sync_executor.submit(_run_sync_task, sync_playlist_id, sync_data['name'], spotify_tracks, None, get_current_profile_id()) state['sync_future'] = future logger.info(f"Started Beatport sync for chart: {state['chart']['name']}") return jsonify({"success": True, "sync_id": sync_playlist_id}) except Exception as e: logger.error(f"Error starting Beatport sync: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/sync/status/', methods=['GET']) def get_beatport_sync_status(url_hash): """Get sync status for a Beatport chart""" try: if url_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 state = beatport_chart_states[url_hash] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if not sync_playlist_id: return jsonify({"error": "No sync process found"}), 404 # Get sync status from sync states sync_state = sync_states.get(sync_playlist_id, {}) response = { 'status': sync_state.get('status', 'unknown'), 'progress': sync_state.get('progress', {}), 'sync_id': sync_playlist_id, 'complete': sync_state.get('status') == 'finished', 'error': sync_state.get('error') } # Check if sync completed successfully if sync_state.get('status') == 'finished': state['phase'] = 'sync_complete' # Extract playlist ID from sync result result = sync_state.get('result', {}) state['converted_spotify_playlist_id'] = result.get('spotify_playlist_id') chart_name = state.get('chart', {}).get('name', 'Unknown Chart') add_activity_item("", "Sync Complete", f"Beatport chart '{chart_name}' synced successfully", "Now") elif sync_state.get('status') == 'error': state['phase'] = 'discovered' # Revert on error chart_name = state.get('chart', {}).get('name', 'Unknown Chart') add_activity_item("", "Sync Failed", f"Beatport chart '{chart_name}' sync failed", "Now") return jsonify(response) except Exception as e: logger.error(f"Error getting Beatport sync status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/sync/cancel/', methods=['POST']) def cancel_beatport_sync(url_hash): """Cancel sync for a Beatport chart""" try: if url_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 state = beatport_chart_states[url_hash] state['last_accessed'] = time.time() # Update access time sync_playlist_id = state.get('sync_playlist_id') if sync_playlist_id and sync_playlist_id in sync_states: # Cancel the sync using existing sync infrastructure with sync_lock: sync_states[sync_playlist_id] = {"status": "cancelled"} # Cancel future if still running if 'sync_future' in state and state['sync_future']: state['sync_future'].cancel() # Revert Beatport state state['phase'] = 'discovered' state['sync_playlist_id'] = None state['sync_progress'] = {} logger.warning(f"Cancelled Beatport sync for: {url_hash}") return jsonify({"success": True}) except Exception as e: logger.error(f"Error cancelling Beatport sync: {e}") return jsonify({"error": str(e)}), 500 # =================================================================== # BEATPORT CHART PERSISTENCE API ENDPOINTS # =================================================================== @app.route('/api/beatport/charts', methods=['GET']) def get_beatport_charts(): """Get all persistent Beatport chart states for frontend hydration""" try: charts = [] current_time = time.time() # Clean up old charts (older than 24 hours) to_remove = [] for chart_hash, state in beatport_chart_states.items(): last_accessed = state.get('last_accessed', 0) if current_time - last_accessed > 86400: # 24 hours to_remove.append(chart_hash) else: # Include in response chart_info = { 'hash': chart_hash, 'name': state['chart']['name'], 'track_count': len(state['chart']['tracks']), 'phase': state.get('phase', 'fresh'), 'discovery_progress': state.get('discovery_progress', 0), 'spotify_matches': state.get('spotify_matches', 0), 'spotify_total': state.get('spotify_total', 0), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'last_accessed': last_accessed, 'chart_data': state['chart'] # Full chart data for restoration } charts.append(chart_info) # Remove old charts for chart_hash in to_remove: del beatport_chart_states[chart_hash] logger.info(f"Cleaned up old Beatport chart: {chart_hash}") logger.info(f"Returning {len(charts)} Beatport charts for hydration") return jsonify(charts) except Exception as e: logger.error(f"Error getting Beatport charts: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/charts/status/', methods=['GET']) def get_beatport_chart_status(chart_hash): """Get individual Beatport chart status with full state data""" try: if chart_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 state = beatport_chart_states[chart_hash] state['last_accessed'] = time.time() # Update access time # Return full state including discovery results for modal restoration response = { 'hash': chart_hash, 'phase': state.get('phase', 'fresh'), 'status': state.get('status', 'fresh'), 'discovery_progress': state.get('discovery_progress', 0), 'spotify_matches': state.get('spotify_matches', 0), 'spotify_total': state.get('spotify_total', 0), 'discovery_results': state.get('discovery_results', []), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), 'sync_playlist_id': state.get('sync_playlist_id'), 'sync_progress': state.get('sync_progress', {}), 'chart_data': state['chart'] # Full chart data } return jsonify(response) except Exception as e: logger.error(f"Error getting Beatport chart status: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/charts/update-phase/', methods=['POST']) def update_beatport_chart_phase(chart_hash): """Update Beatport chart phase (for modal close operations and reset)""" try: if chart_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 data = request.get_json() or {} new_phase = data.get('phase') is_reset = data.get('reset', False) if not new_phase: return jsonify({"error": "Phase is required"}), 400 state = beatport_chart_states[chart_hash] state['phase'] = new_phase state['last_accessed'] = time.time() # Handle reset operation - clear discovery data if is_reset and new_phase == 'fresh': state['discovery_results'] = [] state['discovery_progress'] = 0 state['spotify_matches'] = 0 state['status'] = 'fresh' state['converted_spotify_playlist_id'] = None state['download_process_id'] = None state['sync_playlist_id'] = None state['sync_progress'] = {} logger.info(f"Reset Beatport chart {chart_hash} to fresh state") else: # Handle other phase updates (like download phase transitions) converted_playlist_id = data.get('converted_spotify_playlist_id') if converted_playlist_id: state['converted_spotify_playlist_id'] = converted_playlist_id download_process_id = data.get('download_process_id') if download_process_id: state['download_process_id'] = download_process_id logger.info(f"Updated Beatport chart {chart_hash} phase to: {new_phase}") return jsonify({"success": True, "phase": new_phase}) except Exception as e: logger.error(f"Error updating Beatport chart phase: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/beatport/charts/delete/', methods=['DELETE']) def delete_beatport_chart(chart_hash): """Delete a Beatport chart from backend storage""" try: if chart_hash not in beatport_chart_states: return jsonify({"error": "Beatport chart not found"}), 404 chart_name = beatport_chart_states[chart_hash]['chart']['name'] del beatport_chart_states[chart_hash] logger.info(f"Deleted Beatport chart: {chart_name}") return jsonify({"success": True, "message": f"Deleted chart: {chart_name}"}) except Exception as e: logger.error(f"Error deleting Beatport chart: {e}") return jsonify({"error": str(e)}), 500 # ── Mirrored Playlists ──────────────────────────────────────────────── @app.route('/api/mirror-playlist', methods=['POST']) def mirror_playlist_endpoint(): """Save or update a mirrored playlist.""" try: data = request.get_json() if not data: return jsonify({"error": "No data received"}), 400 source = data.get('source') source_playlist_id = data.get('source_playlist_id') name = data.get('name') tracks = data.get('tracks', []) if not all([source, source_playlist_id, name]): return jsonify({"error": "source, source_playlist_id, and name are required"}), 400 database = get_database() profile_id = get_current_profile_id() playlist_id = database.mirror_playlist( source=source, source_playlist_id=str(source_playlist_id), name=name, tracks=tracks, profile_id=profile_id, description=data.get('description'), owner=data.get('owner'), image_url=data.get('image_url') ) if playlist_id is None: return jsonify({"error": "Failed to mirror playlist"}), 500 try: if automation_engine: automation_engine.emit('mirrored_playlist_created', { 'playlist_name': name, 'source': source, 'track_count': str(len(tracks)), }) except Exception: pass return jsonify({"success": True, "playlist_id": playlist_id}) except Exception as e: logger.error(f"Error mirroring playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists', methods=['GET']) def get_mirrored_playlists_endpoint(): """List all mirrored playlists for the active profile.""" try: database = get_database() profile_id = get_current_profile_id() playlists = database.get_mirrored_playlists(profile_id=profile_id) for pl in playlists: counts = database.get_mirrored_playlist_status_counts(pl['id']) pl['discovered_count'] = counts['discovered'] pl['total_count'] = counts['total'] pl['wishlisted_count'] = counts['wishlisted'] pl['in_library_count'] = counts['in_library'] return jsonify(playlists) except Exception as e: logger.error(f"Error getting mirrored playlists: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists/', methods=['GET']) def get_mirrored_playlist_endpoint(playlist_id): """Get a mirrored playlist with its tracks.""" try: database = get_database() playlist = database.get_mirrored_playlist(playlist_id) if not playlist: return jsonify({"error": "Playlist not found"}), 404 playlist['tracks'] = database.get_mirrored_playlist_tracks(playlist_id) return jsonify(playlist) except Exception as e: logger.error(f"Error getting mirrored playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists/', methods=['DELETE']) def delete_mirrored_playlist_endpoint(playlist_id): """Delete a mirrored playlist.""" try: database = get_database() if database.delete_mirrored_playlist(playlist_id): return jsonify({"success": True}) return jsonify({"error": "Playlist not found"}), 404 except Exception as e: logger.error(f"Error deleting mirrored playlist: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists//clear-discovery', methods=['POST']) def clear_mirrored_discovery_endpoint(playlist_id): """Clear discovery data for all tracks in a mirrored playlist, including discovery cache.""" try: database = get_database() # Clear discovery cache entries for these tracks so re-discovery does fresh lookups try: tracks = database.get_mirrored_playlist_tracks(playlist_id) if tracks: conn = database._get_connection() cursor = conn.cursor() for t in tracks: cache_key = _get_discovery_cache_key(t.get('track_name', ''), t.get('artist_name', '')) cursor.execute( "DELETE FROM discovery_match_cache WHERE normalized_title = ? AND normalized_artist = ?", (cache_key[0], cache_key[1]) ) conn.commit() logger.info(f"Cleared discovery cache for {len(tracks)} tracks in playlist {playlist_id}") except Exception as cache_err: logger.warning(f"Error clearing discovery cache: {cache_err}") cleared = database.clear_mirrored_playlist_discovery(playlist_id) return jsonify({"success": True, "cleared": cleared}) except Exception as e: logger.error(f"Error clearing mirrored discovery: {e}") return jsonify({"error": str(e)}), 500 # ==================== Discovery Pool ==================== @app.route('/api/discovery-pool', methods=['GET']) def get_discovery_pool(): """List matched and failed discovery tracks, optionally filtered by playlist.""" try: database = get_database() profile_id = get_current_profile_id() playlist_id = request.args.get('playlist_id', type=int) matched = database.get_discovery_pool_matched() failed = database.get_discovery_pool_failed(profile_id=profile_id, playlist_id=playlist_id) stats = database.get_discovery_pool_stats(profile_id=profile_id) # Playlist list for the filter dropdown playlists = database.get_mirrored_playlists(profile_id=profile_id) playlist_options = [{'id': p['id'], 'name': p['name']} for p in playlists] return jsonify({ 'matched': matched, 'failed': failed, 'stats': stats, 'playlists': playlist_options, }) except Exception as e: logger.error(f"Error getting discovery pool: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery-pool/fix', methods=['POST']) def fix_discovery_pool_track(): """Manually fix a failed discovery by linking a mirrored track to a Spotify/iTunes result.""" try: data = request.get_json() track_id = data.get('track_id') spotify_track = data.get('spotify_track') if not track_id or not spotify_track: return jsonify({"error": "track_id and spotify_track required"}), 400 database = get_database() # Build matched_data in the same format as the discovery flow artists = spotify_track.get('artists', []) album_raw = spotify_track.get('album', '') image_url = spotify_track.get('image_url', '') if not image_url and isinstance(album_raw, dict): images = album_raw.get('images', []) image_url = images[0].get('url', '') if images else '' # Ensure album carries the artwork too — download pipeline checks # album.images / album.image_url when extracting cover art. if isinstance(album_raw, dict): album_obj = dict(album_raw) if image_url and not album_obj.get('image_url'): album_obj['image_url'] = image_url if image_url and not album_obj.get('images'): album_obj['images'] = [{'url': image_url}] else: album_obj = {'name': album_raw or ''} if image_url: album_obj['image_url'] = image_url album_obj['images'] = [{'url': image_url}] matched_data = { 'id': spotify_track.get('id', ''), 'name': spotify_track.get('name', ''), 'artists': [{'name': a} if isinstance(a, str) else a for a in artists], 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } # Update the mirrored track's extra_data extra_data = { 'discovered': True, 'provider': 'spotify', 'confidence': 1.0, 'matched_data': matched_data, 'manual_match': True, } database.update_mirrored_track_extra_data(track_id, extra_data) # Also save to discovery cache so future discoveries hit the cache # Need to get the track's original name/artist for the cache key try: conn = database._get_connection() cursor = conn.cursor() cursor.execute("SELECT track_name, artist_name FROM mirrored_playlist_tracks WHERE id = ?", (track_id,)) row = cursor.fetchone() if row: cache_key = _get_discovery_cache_key(row['track_name'], row['artist_name']) database.save_discovery_cache_match( cache_key[0], cache_key[1], _get_active_discovery_source(), 1.0, matched_data, row['track_name'], row['artist_name'] ) except Exception: pass return jsonify({"success": True}) except Exception as e: logger.error(f"Error fixing discovery pool track: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery-pool/cache/', methods=['DELETE']) def delete_discovery_pool_cache_entry(entry_id): """Remove a single entry from the discovery match cache.""" try: database = get_database() if database.delete_discovery_cache_entry(entry_id): return jsonify({"success": True}) return jsonify({"error": "Entry not found"}), 404 except Exception as e: logger.error(f"Error deleting discovery cache entry: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/discovery-pool/rematch', methods=['POST']) def rematch_discovery_pool_track(): """Replace a discovery cache entry with a new match chosen by the user.""" try: data = request.get_json() cache_id = data.get('cache_id') original_title = (data.get('original_title') or '').strip() original_artist = (data.get('original_artist') or '').strip() spotify_track = data.get('spotify_track') if not cache_id: return jsonify({"error": "cache_id required"}), 400 database = get_database() # If no spotify_track provided, just delete the cache entry (phase 1 of rematch) if not spotify_track: database.delete_discovery_cache_entry(cache_id) return jsonify({"success": True, "action": "cache_cleared"}) # spotify_track provided — delete old cache and save new match (phase 2) database.delete_discovery_cache_entry(cache_id) # Build cache entry in same format as discovery flow artists = spotify_track.get('artists', []) album_raw = spotify_track.get('album', '') album_obj = album_raw if isinstance(album_raw, dict) else {'name': album_raw or ''} image_url = spotify_track.get('image_url', '') if not image_url and isinstance(album_raw, dict): images = album_raw.get('images', []) image_url = images[0].get('url', '') if images else '' matched_data = { 'id': spotify_track.get('id', ''), 'name': spotify_track.get('name', ''), 'artists': [{'name': a} if isinstance(a, str) else a for a in artists], 'album': album_obj, 'duration_ms': spotify_track.get('duration_ms', 0), 'image_url': image_url, 'source': 'spotify', } # Save to discovery cache normalized_title = matching_engine.normalize_string(original_title) if original_title else '' normalized_artist = matching_engine.normalize_string(original_artist) if original_artist else '' database.save_discovery_cache_match( normalized_title=normalized_title, normalized_artist=normalized_artist, provider='spotify', confidence=1.0, matched_data=matched_data, original_title=original_title, original_artist=original_artist, ) return jsonify({"success": True, "action": "rematched", "name": spotify_track.get('name', '')}) except Exception as e: logger.error(f"Error in discovery pool rematch: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists//prepare-discovery', methods=['POST']) def prepare_mirrored_discovery(playlist_id): """Register a mirrored playlist into youtube_playlist_states so the YouTube discovery pipeline can run.""" try: database = get_database() playlist = database.get_mirrored_playlist(playlist_id) if not playlist: return jsonify({"error": "Playlist not found"}), 404 tracks_data = database.get_mirrored_playlist_tracks(playlist_id) url_hash = f"mirrored_{playlist_id}" # Build track list in the format the YouTube discovery worker expects tracks = [] for t in tracks_data: # Parse extra_data if present extra = None if t.get('extra_data'): try: extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] except (json.JSONDecodeError, TypeError): pass tracks.append({ 'id': t.get('source_track_id') or f"mirrored_{t['id']}", 'db_track_id': t['id'], 'name': t['track_name'], 'artists': [t['artist_name']], 'album': t.get('album_name', ''), 'duration_ms': t.get('duration_ms', 0), 'extra_data': extra, }) # Determine current active metadata source for provider-mismatch detection _current_provider = _get_active_discovery_source() _use_spotify = (_current_provider == 'spotify') and spotify_client and spotify_client.is_spotify_authenticated() # Check for cached discovery results in extra_data pre_discovered_results = [] pre_discovered_count = 0 has_pending = False for idx, track in enumerate(tracks): extra = track.get('extra_data') if extra and extra.get('discovered'): cached_provider = extra.get('provider', 'spotify') # If the cached result was discovered by a different provider than the # currently active one, treat it as pending so re-discovery uses the # correct source (IDs, album data, images differ between providers). if cached_provider != _current_provider: has_pending = True dur = track.get('duration_ms', 0) pre_discovered_results.append({ 'index': idx, 'yt_track': track['name'], 'yt_artist': track['artists'][0] if track['artists'] else 'Unknown', 'status': 'Provider changed', 'status_class': 'not-found', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00', 'confidence': 0, }) continue # Previously found match — provider matches current source matched = extra.get('matched_data', {}) artists_raw = matched.get('artists', []) if artists_raw and isinstance(artists_raw[0], dict): artist_str = ', '.join(a.get('name', '') for a in artists_raw) else: artist_str = ', '.join(str(a) for a in artists_raw) if artists_raw else '' album_raw = matched.get('album', '') album_str = album_raw.get('name', '') if isinstance(album_raw, dict) else (str(album_raw) if album_raw else '') dur = track.get('duration_ms', 0) result = { 'index': idx, 'yt_track': track['name'], 'yt_artist': track['artists'][0] if track['artists'] else 'Unknown', 'status': 'Found', 'status_class': 'found', 'spotify_track': matched.get('name', ''), 'spotify_artist': artist_str, 'spotify_album': album_str, 'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00', 'discovery_source': extra.get('provider', 'spotify'), 'confidence': extra.get('confidence', 0), 'matched_data': matched, 'spotify_data': matched, } if extra.get('manual_match'): result['manual_match'] = True pre_discovered_results.append(result) pre_discovered_count += 1 elif extra and extra.get('discovery_attempted'): # Previously attempted but not found — also retry if provider changed cached_provider = extra.get('provider', 'spotify') if cached_provider != _current_provider: has_pending = True dur = track.get('duration_ms', 0) pre_discovered_results.append({ 'index': idx, 'yt_track': track['name'], 'yt_artist': track['artists'][0] if track['artists'] else 'Unknown', 'status': 'Provider changed' if cached_provider != _current_provider else 'Not Found', 'status_class': 'not-found', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00', 'discovery_source': cached_provider, 'confidence': 0, }) elif not extra or (not extra.get('discovered') and not extra.get('discovery_attempted')): # New track — no discovery data yet has_pending = True dur = track.get('duration_ms', 0) pre_discovered_results.append({ 'index': idx, 'yt_track': track['name'], 'yt_artist': track['artists'][0] if track['artists'] else 'Unknown', 'status': '🆕 Pending', 'status_class': 'not-found', 'spotify_track': '', 'spotify_artist': '', 'spotify_album': '', 'duration': f"{dur // 60000}:{(dur % 60000) // 1000:02d}" if dur else '0:00', 'confidence': 0, }) # Only treat as cached if at least one track was discovered by the current provider has_cached = any( t.get('extra_data') and (t['extra_data'].get('discovered') or t['extra_data'].get('discovery_attempted')) and t['extra_data'].get('provider', 'spotify') == _current_provider for t in tracks ) playlist_data = { 'id': url_hash, 'name': playlist['name'], 'tracks': tracks, 'track_count': len(tracks), 'url': f"mirrored://{playlist['source']}/{playlist['source_playlist_id']}", 'source': playlist['source'] } youtube_playlist_states[url_hash] = { 'playlist': playlist_data, 'phase': 'discovered' if has_cached else 'fresh', 'discovery_results': pre_discovered_results if has_cached else [], 'discovery_progress': 100 if has_cached else 0, 'spotify_matches': pre_discovered_count if has_cached else 0, 'spotify_total': len(tracks), 'status': 'complete' if has_cached else 'parsed', 'url': playlist_data['url'], 'sync_playlist_id': None, 'converted_spotify_playlist_id': None, 'download_process_id': None, 'created_at': time.time(), 'last_accessed': time.time(), 'discovery_future': None, 'sync_progress': {} } logger.info(f"Prepared mirrored playlist for discovery: {playlist['name']} ({len(tracks)} tracks, cached={has_cached}, matches={pre_discovered_count})") return jsonify({ "success": True, "url_hash": url_hash, "from_cache": has_cached, "cached_matches": pre_discovered_count, "total_tracks": len(tracks), "has_pending": has_pending, }) except Exception as e: logger.error(f"Error preparing mirrored discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists//retry-failed-discovery', methods=['POST']) def retry_failed_mirrored_discovery(playlist_id): """Re-run discovery only for tracks that failed or are pending in a mirrored playlist.""" try: url_hash = f"mirrored_{playlist_id}" state = youtube_playlist_states.get(url_hash) if not state: return jsonify({"error": "Discovery state not found. Run discovery first."}), 404 if state.get('phase') == 'discovering': return jsonify({"error": "Discovery already in progress"}), 400 tracks = state['playlist']['tracks'] results = state.get('discovery_results', []) # Build set of found track indices found_indices = set() kept_results = [] for r in results: if r.get('status_class') == 'found': found_indices.add(r.get('index', -1)) kept_results.append(r) already_found = len(found_indices) retry_count = len(tracks) - already_found if retry_count == 0: return jsonify({"success": True, "retry_count": 0, "already_found": already_found, "message": "All tracks already found"}) # Flag found tracks to skip, clear flag on others for i, track in enumerate(tracks): track['skip_discovery'] = (i in found_indices) # Keep only found results, remove failed/pending state['discovery_results'] = kept_results state['phase'] = 'discovering' state['status'] = 'discovering' state['discovery_progress'] = 0 # spotify_matches stays at found count (already_found) state['spotify_matches'] = already_found # Clear discovery_attempted in DB for failed tracks so they're retryable try: db = get_database() for i, track in enumerate(tracks): if i not in found_indices: db_track_id = track.get('db_track_id') if db_track_id: db.update_mirrored_track_extra_data(db_track_id, { 'discovered': False, 'discovery_attempted': False, }) except Exception as db_err: logger.error(f"Error clearing discovery_attempted in DB: {db_err}") # Submit worker future = youtube_discovery_executor.submit(_run_youtube_discovery_worker, url_hash) state['discovery_future'] = future logger.error(f"Retrying failed discovery for {url_hash}: {retry_count} tracks to retry, {already_found} already found") return jsonify({ "success": True, "retry_count": retry_count, "already_found": already_found, }) except Exception as e: logger.error(f"Error retrying failed discovery: {e}") return jsonify({"error": str(e)}), 500 @app.route('/api/mirrored-playlists/discovery-states', methods=['GET']) def get_mirrored_discovery_states(): """Return discovery states for any mirrored playlists that have active/completed discoveries.""" try: states = [] for url_hash, state in youtube_playlist_states.items(): if not url_hash.startswith('mirrored_'): continue states.append({ 'url_hash': url_hash, 'playlist_id': int(url_hash.replace('mirrored_', '')), 'playlist': state['playlist'], 'phase': state['phase'], 'status': state.get('status', ''), 'discovery_progress': state.get('discovery_progress', 0), 'spotify_matches': state.get('spotify_matches', 0), 'spotify_total': state.get('spotify_total', 0), 'discovery_results': state.get('discovery_results', []), 'converted_spotify_playlist_id': state.get('converted_spotify_playlist_id'), 'download_process_id': state.get('download_process_id'), }) return jsonify({"states": states}) except Exception as e: logger.error(f"Error getting mirrored discovery states: {e}") return jsonify({"error": str(e)}), 500 # ================================================================================================ # PLAYLIST EXPLORER # ================================================================================================ @app.route('/api/playlist-explorer/build-tree', methods=['POST']) def playlist_explorer_build_tree(): """Build a discovery tree from a mirrored playlist. Streams NDJSON: one line per artist with their albums. Works with Spotify, iTunes, or Deezer as the metadata source. Uses and populates the metadata cache to avoid redundant API calls.""" try: data = request.get_json() if not data: return jsonify({"success": False, "error": "No data provided"}), 400 playlist_id = data.get('playlist_id') mode = data.get('mode', 'albums') # 'albums' or 'discographies' if not playlist_id: return jsonify({"success": False, "error": "playlist_id is required"}), 400 if mode not in ('albums', 'discographies'): return jsonify({"success": False, "error": "mode must be 'albums' or 'discographies'"}), 400 database = get_database() playlist = database.get_mirrored_playlist(playlist_id) if not playlist: return jsonify({"success": False, "error": "Playlist not found"}), 404 tracks = database.get_mirrored_playlist_tracks(playlist_id) if not tracks: return jsonify({"success": False, "error": "Playlist has no tracks"}), 400 # Determine active metadata source — respect user's configured primary source_name = _get_active_discovery_source() if source_name == 'spotify' and spotify_client and spotify_client.is_spotify_authenticated(): active_client = spotify_client else: active_client = _get_metadata_fallback_client() source_name = _get_metadata_fallback_source() cache = get_metadata_cache() # Parse extra_data and group tracks by artist using discovered data artist_groups = {} for t in tracks: extra = {} if t.get('extra_data'): try: extra = json.loads(t['extra_data']) if isinstance(t['extra_data'], str) else t['extra_data'] except (json.JSONDecodeError, TypeError): pass # Only use discovery data if it matches the active metadata source is_discovered = extra.get('discovered', False) provider = (extra.get('provider') or '').lower() source_matches = provider == source_name or (provider in ('itunes', 'apple') and source_name == 'itunes') matched = extra.get('matched_data', {}) if (is_discovered and source_matches) else {} artists_list = matched.get('artists', []) primary_artist = artists_list[0] if artists_list else None # Artists can be dicts {"name": "X", "id": "Y"} or plain strings "X" if isinstance(primary_artist, dict): artist_name = primary_artist.get('name') or (t.get('artist_name') or '').strip() artist_id = primary_artist.get('id') or None elif isinstance(primary_artist, str): artist_name = primary_artist or (t.get('artist_name') or '').strip() artist_id = None else: artist_name = (t.get('artist_name') or '').strip() artist_id = None if not artist_name: continue key = artist_name.lower() if key not in artist_groups: artist_groups[key] = { 'name': artist_name, 'artist_id': artist_id, # Pre-resolved from discovery 'tracks': [], 'album_names': set(), 'discovered': extra.get('discovered', False), } # If we get an artist_id from a later track but didn't have one before, fill it in if artist_id and not artist_groups[key].get('artist_id'): artist_groups[key]['artist_id'] = artist_id artist_groups[key]['tracks'].append(t.get('track_name', '')) # Get album name from discovered data or playlist field album_name = '' album_data = matched.get('album') if isinstance(album_data, dict) and album_data.get('name'): album_name = album_data['name'] elif (t.get('album_name') or '').strip(): album_name = t['album_name'].strip() if album_name: artist_groups[key]['album_names'].add(album_name) def _normalize_for_match(title): import re return re.sub(r'\s*[\(\[][^)\]]*[\)\]]', '', title).strip().lower() def _fetch_artist_discography(artist_name, known_artist_id=None): """Fetch discography using the active client. Checks cache first, stores results after. If known_artist_id is provided (from discovery cache), skips the name search.""" # Check cache for this artist's discography cache_key = f"explorer_disco_{artist_name.lower().strip()}" cached = cache.get_entity(source_name, 'artist_discography', cache_key) if cache else None if cached and isinstance(cached, dict) and cached.get('albums'): logger.debug(f"Explorer: cache hit for '{artist_name}' discography") return cached artist_id = known_artist_id artist_image = None if artist_id: # Already have the ID from discovery — just fetch the artist image try: artist_info = active_client.get_artist(artist_id) if artist_info: if isinstance(artist_info, dict): images = artist_info.get('images') or [] artist_image = images[0].get('url') if images else None elif hasattr(artist_info, 'image_url'): artist_image = artist_info.image_url except Exception: pass else: # No pre-resolved ID — search by name try: search_results = active_client.search_artists(artist_name, limit=5) except Exception as e: return {'success': False, 'error': f'Search failed: {e}'} if not search_results: return {'success': False, 'error': f'"{artist_name}" not found'} # Find best match (exact first, then fuzzy) best = None for a in search_results: if a.name.lower().strip() == artist_name.lower().strip(): best = a break if not best: best = search_results[0] artist_id = best.id artist_image = best.image_url if hasattr(best, 'image_url') else None # Fetch albums try: # skip_cache only supported by spotify_client — other clients don't cache this call _skip = {'skip_cache': True} if hasattr(active_client, 'sp') else {} all_albums = active_client.get_artist_albums(artist_id, album_type='album,single', **_skip) except Exception as e: return {'success': False, 'error': f'Album fetch failed: {e}'} if not all_albums: return {'success': False, 'error': 'No albums found'} # Check which albums the user already owns owned_titles = set() try: db = get_database() with db._get_connection() as conn: cursor = conn.cursor() # Find all artists in DB matching this name cursor.execute("SELECT id FROM artists WHERE LOWER(name) = LOWER(?)", (artist_name,)) artist_rows = cursor.fetchall() for ar in artist_rows: cursor.execute("SELECT title FROM albums WHERE artist_id = ?", (ar['id'],)) for alb_row in cursor.fetchall(): owned_titles.add((alb_row['title'] or '').strip().lower()) except Exception: pass # Non-critical — owned badges just won't show # Build release list releases = [] for album in all_albums: # Skip albums where this artist isn't primary if hasattr(album, 'artist_ids') and album.artist_ids and album.artist_ids[0] != artist_id: continue releases.append({ 'title': album.name, 'year': album.release_date[:4] if album.release_date else None, 'image_url': album.image_url, 'spotify_id': album.id, 'track_count': album.total_tracks, 'album_type': (album.album_type or 'album').lower(), 'owned': (album.name or '').strip().lower() in owned_titles, }) result = { 'success': True, 'name': artist_name, # Required for metadata cache validation 'albums': releases, 'artist_image': artist_image, 'artist_id': artist_id, 'artist_name': artist_name, } # Store in cache if cache and releases: try: cache.store_entity(source_name, 'artist_discography', cache_key, result) except Exception: pass return result def generate(): yield json.dumps({ "type": "meta", "playlist_name": playlist.get('name', 'Unknown Playlist'), "playlist_image": playlist.get('image_url', ''), "total_artists": len(artist_groups), "total_tracks": len(tracks), "source": source_name, }) + '\n' total_albums = 0 for idx, (_key, group) in enumerate(artist_groups.items()): artist_name = group['name'] playlist_track_names = group['tracks'] playlist_album_names = group['album_names'] try: disco = _fetch_artist_discography(artist_name, group.get('artist_id')) if not disco.get('success'): yield json.dumps({ "type": "artist", "name": artist_name, "artist_id": None, "image_url": None, "playlist_tracks": playlist_track_names, "albums": [], "error": disco.get('error', 'Not found'), }) + '\n' time.sleep(0.1) continue # Tag each release with in_playlist flag # If no album names available, fall back to matching track names against single titles match_names = playlist_album_names if not match_names: match_names = set(playlist_track_names) all_releases = [] for release in disco.get('albums', []): r = dict(release) norm_title = _normalize_for_match(r['title']) r['in_playlist'] = any( _normalize_for_match(a) == norm_title or norm_title in _normalize_for_match(a) or _normalize_for_match(a) in norm_title for a in match_names ) all_releases.append(r) # Filter based on mode if mode == 'albums': filtered = [r for r in all_releases if r['in_playlist']] else: filtered = all_releases filtered.sort(key=lambda r: (not r.get('in_playlist', False), -(int(r.get('year') or 0)))) total_albums += len(filtered) yield json.dumps({ "type": "artist", "name": disco.get('artist_name', artist_name), "artist_id": disco.get('artist_id'), "image_url": disco.get('artist_image'), "playlist_tracks": playlist_track_names, "albums": filtered, }) + '\n' except Exception as e: logger.error(f"Explorer: error processing artist '{artist_name}': {e}") yield json.dumps({ "type": "artist", "name": artist_name, "artist_id": None, "image_url": None, "playlist_tracks": playlist_track_names, "albums": [], "error": str(e), }) + '\n' # Rate limit protection between artists if idx < len(artist_groups) - 1: time.sleep(0.2) get_database().mark_mirrored_playlist_explored(playlist_id) yield json.dumps({"type": "complete", "total_artists": len(artist_groups), "total_albums": total_albums}) + '\n' return Response(generate(), mimetype='application/x-ndjson', headers={ 'Cache-Control': 'no-cache', 'X-Accel-Buffering': 'no', }) except Exception as e: logger.error(f"Playlist Explorer build-tree error: {e}") import traceback traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/playlist-explorer/album-tracks/', methods=['GET']) def playlist_explorer_album_tracks(album_id): """Fetch track listing for an album. Works with active metadata source. Caches results.""" try: # Determine source spotify_available = spotify_client and spotify_client.is_spotify_authenticated() source_name = 'spotify' if spotify_available else _get_metadata_fallback_source() client = spotify_client if spotify_available else _get_metadata_fallback_client() if not client: return jsonify({"success": False, "error": "No metadata source available"}), 400 # Check cache cache = get_metadata_cache() cache_key = f"album_tracks_{album_id}" if cache: cached = cache.get_entity(source_name, 'album_tracks', cache_key) if cached and isinstance(cached, dict) and cached.get('tracks'): return jsonify(cached) album_data = client.get_album(album_id) if not album_data: return jsonify({"success": False, "error": "Album not found"}), 404 tracks_raw = album_data.get('tracks', {}).get('items', []) tracks = [] for t in tracks_raw: artists = ', '.join(a.get('name', '') for a in t.get('artists', [])) tracks.append({ 'name': t.get('name', 'Unknown'), 'track_number': t.get('track_number', 0), 'disc_number': t.get('disc_number', 1), 'duration_ms': t.get('duration_ms', 0), 'artists': artists, }) result = {"success": True, "name": album_data.get('name', ''), "tracks": tracks} # Store in cache if cache and tracks: try: cache.store_entity(source_name, 'album_tracks', cache_key, result) except Exception: pass return jsonify(result) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 def convert_beatport_results_to_spotify_tracks(discovery_results): """Convert Beatport discovery results to Spotify tracks format for sync""" spotify_tracks = [] for result in discovery_results: # Support both data formats: spotify_data (manual fixes) and individual fields (automatic discovery) if result.get('spotify_data'): spotify_data = result['spotify_data'] # Convert artists from objects to strings if needed artists = spotify_data['artists'] if isinstance(artists, list) and len(artists) > 0: if isinstance(artists[0], dict) and 'name' in artists[0]: # Convert from [{'name': 'Artist'}] to ['Artist'] artists = [artist['name'] for artist in artists] track = { 'id': spotify_data['id'], 'name': spotify_data['name'], 'artists': artists, 'album': spotify_data['album'], 'source': 'beatport' } if spotify_data.get('track_number'): track['track_number'] = spotify_data['track_number'] if spotify_data.get('disc_number'): track['disc_number'] = spotify_data['disc_number'] spotify_tracks.append(track) elif result.get('spotify_track') and result.get('status_class') == 'found': # Build from individual fields (automatic discovery format) album_val = result.get('spotify_album', '') album_dict = album_val if isinstance(album_val, dict) else { 'name': album_val or result.get('spotify_track', 'Unknown Album'), 'album_type': 'single', 'images': [], 'release_date': '', 'total_tracks': 1, } spotify_tracks.append({ 'id': result.get('spotify_id', 'unknown'), 'name': result.get('spotify_track', 'Unknown Track'), 'artists': [result.get('spotify_artist', 'Unknown Artist')] if result.get('spotify_artist') else ['Unknown Artist'], 'album': album_dict, 'source': 'beatport' }) return spotify_tracks # Beatport download missing tracks is handled frontend-only (like YouTube) # No backend endpoint needed - uses existing download modal infrastructure class WebMetadataUpdateWorker: """Web-based metadata update worker - EXACT port of dashboard.py MetadataUpdateWorker""" def __init__(self, artists, media_client, spotify_client, server_type, refresh_interval_days=30): self.artists = artists self.media_client = media_client # Can be plex_client or jellyfin_client self.spotify_client = spotify_client self.server_type = server_type # "plex" or "jellyfin" self.matching_engine = MusicMatchingEngine() self.refresh_interval_days = refresh_interval_days self.should_stop = False self.processed_count = 0 self.successful_count = 0 self.failed_count = 0 self.max_workers = 1 # DB-first: reuse existing metadata from SoulSync database try: from database.music_database import MusicDatabase self._db = MusicDatabase() except Exception: self._db = None self.thread_lock = threading.Lock() def stop(self): self.should_stop = True def get_artist_name(self, artist): """Get artist name consistently across Plex and Jellyfin""" return getattr(artist, 'title', 'Unknown Artist') def run(self): """Process all artists one by one - EXACT copy from dashboard.py""" global metadata_update_state try: # Load artists in background if not provided - EXACTLY like dashboard.py if self.artists is None: # Enable lightweight mode for Jellyfin to skip track caching if self.server_type == "jellyfin": self.media_client.set_metadata_only_mode(True) elif self.server_type == "navidrome": # Navidrome doesn't need special mode setting pass all_artists = self.media_client.get_all_artists() logger.debug(f"Raw artists returned: {[getattr(a, 'title', 'NO_TITLE') for a in (all_artists or [])]}") if not all_artists: metadata_update_state['status'] = 'error' metadata_update_state['error'] = f"No artists found in {self.server_type.title()} library" add_activity_item("", "Metadata Update", metadata_update_state['error'], "Now") return # Filter artists that need processing artists_to_process = [artist for artist in all_artists if self.artist_needs_processing(artist)] self.artists = artists_to_process # Emit loaded signal equivalent - EXACTLY like dashboard.py if len(artists_to_process) == 0: metadata_update_state['status'] = 'completed' metadata_update_state['completed_at'] = datetime.now() add_activity_item("", "Metadata Update", "All artists already have good metadata", "Now") return else: add_activity_item("", "Metadata Update", f"Processing {len(artists_to_process)} of {len(all_artists)} artists", "Now") if not artists_to_process: metadata_update_state['status'] = 'completed' metadata_update_state['completed_at'] = datetime.now() return total_artists = len(self.artists) metadata_update_state['total'] = total_artists # Process artists in parallel using ThreadPoolExecutor - EXACTLY like dashboard.py def process_single_artist(artist): """Process a single artist and return results""" if self.should_stop or metadata_update_state['status'] == 'stopping': return None artist_name = getattr(artist, 'title', 'Unknown Artist') # Double-check ignore flag right before processing if self.media_client.is_artist_ignored(artist): return (artist_name, True, "Skipped (ignored)") try: success, details = self.update_artist_metadata(artist) return (artist_name, success, details) except Exception as e: return (artist_name, False, f"Error: {str(e)}") # Process artists sequentially with rate limiting # (no ThreadPoolExecutor — API rate limits make parallelism counterproductive) import time for artist in self.artists: if self.should_stop or metadata_update_state['status'] == 'stopping': break result = process_single_artist(artist) if result is None: continue artist_name, success, details = result with self.thread_lock: self.processed_count += 1 if success: self.successful_count += 1 else: self.failed_count += 1 progress_percent = (self.processed_count / total_artists) * 100 metadata_update_state.update({ 'current_artist': artist_name, 'processed': self.processed_count, 'percentage': progress_percent, 'successful': self.successful_count, 'failed': self.failed_count }) # Rate limit: 1.5s between artists (this actually runs between artists now) time.sleep(1.5) # Mark as completed - equivalent to finished.emit metadata_update_state['status'] = 'completed' metadata_update_state['completed_at'] = datetime.now() metadata_update_state['current_artist'] = 'Completed' summary = f"Processed {self.processed_count} artists: {self.successful_count} updated, {self.failed_count} failed" add_activity_item("", "Metadata Complete", summary, "Now") except Exception as e: logger.error(f"Metadata update failed: {e}") metadata_update_state['status'] = 'error' metadata_update_state['error'] = str(e) add_activity_item("", "Metadata Error", str(e), "Now") def artist_needs_processing(self, artist): """Check if an artist needs metadata processing using age-based detection - EXACT copy from dashboard.py""" try: # Check if artist is manually ignored if self.media_client.is_artist_ignored(artist): return False # Use media client's age-based checking with configured interval return self.media_client.needs_update_by_age(artist, self.refresh_interval_days) except Exception as e: logger.error(f"Error checking artist {getattr(artist, 'title', 'Unknown')}: {e}") return True # Process if we can't determine status def _check_db_artist(self, artist_name): """Check SoulSync DB for existing artist metadata (genres, spotify_artist_id). NOTE: DB thumb_url is a Plex/Jellyfin internal path, NOT a downloadable URL. Photos must be checked via the media server object, not the DB. Returns (db_artist_dict, has_genres, spotify_artist_id) or (None, False, None) if not found.""" if not self._db: return None, False, None try: db_artists = self._db.search_artists(artist_name, limit=5) if not db_artists: return None, False, None # Find best name match best = None best_score = 0.0 norm_name = self.matching_engine.normalize_string(artist_name) for dba in db_artists: score = self.matching_engine.similarity_score( norm_name, self.matching_engine.normalize_string(dba.name)) if score > best_score: best_score = score best = dba if not best or best_score < 0.85: return None, False, None has_genres = bool(best.genres and len(best.genres) > 0) # Get spotify_artist_id from raw DB row (not in dataclass) spotify_artist_id = None try: raw = self._db.api_get_artist(best.id) if raw: spotify_artist_id = raw.get('spotify_artist_id') except Exception: pass return best, has_genres, spotify_artist_id except Exception: return None, False, None def update_artist_metadata(self, artist): """Update a single artist's metadata. Checks SoulSync DB first to avoid unnecessary API calls. DB-first strategy: - Genres: DB stores real genre strings → can apply directly, skip Spotify - spotify_artist_id: DB may have it from enrichment → skip search_artists() call - Photos/album art: DB thumb_url is a media-server internal path (not downloadable) so these MUST come from Spotify API """ try: artist_name = getattr(artist, 'title', 'Unknown Artist') # Skip processing for artists with no valid name if artist_name == 'Unknown Artist' or not artist_name or not artist_name.strip(): return False, "Skipped: No valid artist name" # DB-first: check what we already have cached db_artist, db_has_genres, db_spotify_id = self._check_db_artist(artist_name) # Check what the media server artist is currently missing needs_photo = not self.artist_has_valid_photo(artist) if self.server_type != "jellyfin" else True needs_genres = not getattr(artist, 'genres', None) needs_album_art = self.server_type == "plex" # If media server already has valid photo + genres + album art, skip entirely if not needs_photo and not needs_genres and not needs_album_art: self.media_client.update_artist_biography(artist) return True, "Already up to date" # Determine if we actually need Spotify # Photos and album art MUST come from Spotify (DB only has internal media server paths) # Genres CAN come from DB if available need_spotify = needs_photo or needs_album_art or (needs_genres and not db_has_genres) spotify_artist = None highest_score = 0.0 if need_spotify: # Try direct lookup by cached spotify_artist_id first (1 API call vs search) if db_spotify_id: try: from core.spotify_client import Artist as SpotifyArtistDC raw = self.spotify_client.get_artist(db_spotify_id) if raw and 'name' in raw: spotify_artist = SpotifyArtistDC.from_spotify_artist(raw) highest_score = 1.0 logger.debug(f"Metadata updater: direct Spotify lookup for '{artist_name}' via cached ID {db_spotify_id}") except Exception as e: logger.debug(f"Direct Spotify lookup failed for {db_spotify_id}: {e}") spotify_artist = None # Fall back to search if direct lookup didn't work if not spotify_artist: spotify_artists = self.spotify_client.search_artists(artist_name, limit=5) if not spotify_artists: # Spotify failed — apply DB genres if available, skip photos/art changes_made = [] if needs_genres and db_has_genres and db_artist: if self._apply_db_genres(artist, db_artist.genres): changes_made.append("genres (DB)") if changes_made: self.media_client.update_artist_biography(artist) return True, f"Updated {', '.join(changes_made)} (Spotify unavailable)" return False, "Not found on Spotify" # Find the best match best_match = None plex_artist_normalized = self.matching_engine.normalize_string(artist_name) for sa in spotify_artists: spotify_artist_normalized = self.matching_engine.normalize_string(sa.name) score = self.matching_engine.similarity_score(plex_artist_normalized, spotify_artist_normalized) if score > highest_score: highest_score = score best_match = sa if not best_match or highest_score < 0.7: # No good Spotify match — still try DB genres changes_made = [] if needs_genres and db_has_genres and db_artist: if self._apply_db_genres(artist, db_artist.genres): changes_made.append("genres (DB)") if changes_made: self.media_client.update_artist_biography(artist) return True, f"Updated {', '.join(changes_made)} (no Spotify match)" return False, f"No confident match found (best: '{getattr(best_match, 'name', 'N/A')}', score: {highest_score:.2f})" spotify_artist = best_match changes_made = [] # Update photo (always from Spotify — DB only has media server paths) if needs_photo and spotify_artist: photo_updated = self.update_artist_photo(artist, spotify_artist) if photo_updated: changes_made.append("photo") # Update genres — use DB if available, otherwise Spotify if needs_genres: if db_has_genres and db_artist: genres_updated = self._apply_db_genres(artist, db_artist.genres) if genres_updated: changes_made.append("genres (DB)") elif spotify_artist: # DB genres didn't result in changes, try Spotify for newer/different genres genres_updated = self.update_artist_genres(artist, spotify_artist) if genres_updated: changes_made.append("genres") elif spotify_artist: genres_updated = self.update_artist_genres(artist, spotify_artist) if genres_updated: changes_made.append("genres") # Update album artwork (only for Plex, always from Spotify) if self.server_type == "plex" and spotify_artist: albums_updated = self.update_album_artwork(artist, spotify_artist) if albums_updated > 0: changes_made.append(f"{albums_updated} album art") elif self.server_type != "plex": logger.info(f"Skipping album artwork updates for Jellyfin artist: {artist.title}") if changes_made: biography_updated = self.media_client.update_artist_biography(artist) if biography_updated: changes_made.append("timestamp") source = f"match: '{spotify_artist.name}', score: {highest_score:.2f}" if spotify_artist else "DB cache" details = f"Updated {', '.join(changes_made)} ({source})" return True, details else: self.media_client.update_artist_biography(artist) return True, "Already up to date" except Exception as e: return False, str(e) def _apply_db_genres(self, artist, genres): """Apply genres from DB cache to media server.""" try: if not genres: return False existing_genres = set(genre.tag if hasattr(genre, 'tag') else str(genre) for genre in (getattr(artist, 'genres', None) or [])) db_genres = set(g for g in genres if g and g.strip() and len(g.strip()) > 1) if db_genres and db_genres != existing_genres: return self.media_client.update_artist_genres(artist, list(db_genres)[:10]) return False except Exception: return False def update_artist_photo(self, artist, spotify_artist): """Update artist photo from Spotify - EXACT copy from dashboard.py""" try: # Check if artist already has a good photo (skip check for Jellyfin) if self.server_type != "jellyfin" and self.artist_has_valid_photo(artist): logger.info(f"Skipping {artist.title}: already has valid photo ({getattr(artist, 'thumb', 'None')})") return False # Get the image URL from Spotify if not spotify_artist.image_url: logger.warning(f"Skipping {artist.title}: no Spotify image URL available") return False logger.info(f"Processing {artist.title}: downloading from Spotify...") image_url = spotify_artist.image_url # Download and validate image response = requests.get(image_url, timeout=10) response.raise_for_status() # Validate and convert image (skip conversion for Jellyfin to preserve format) if self.server_type == "jellyfin": # For Jellyfin, use raw image data to preserve original format image_data = response.content logger.info(f"Using raw image data for Jellyfin ({len(image_data)} bytes)") else: # For other servers, validate and convert image_data = self.validate_and_convert_image(response.content) if not image_data: return False # Upload to media server using client's method return self.media_client.update_artist_poster(artist, image_data) except Exception as e: logger.error(f"Error updating photo for {getattr(artist, 'title', 'Unknown')}: {e}") return False def update_artist_genres(self, artist, spotify_artist): """Update artist genres from Spotify and albums - EXACT copy from dashboard.py""" try: # Get existing genres existing_genres = set(genre.tag if hasattr(genre, 'tag') else str(genre) for genre in (artist.genres or [])) # Get Spotify artist genres spotify_genres = set(spotify_artist.genres or []) # Get genres from all albums album_genres = set() try: for album in artist.albums(): if hasattr(album, 'genres') and album.genres: album_genres.update(genre.tag if hasattr(genre, 'tag') else str(genre) for genre in album.genres) except Exception: pass # Albums might not be accessible # Combine all genres (prioritize Spotify genres) all_genres = spotify_genres.union(album_genres) # Filter out empty/invalid genres all_genres = {g for g in all_genres if g and g.strip() and len(g.strip()) > 1} # Only update if we have new genres and they're different if all_genres and (not existing_genres or all_genres != existing_genres): # Convert to list and limit to 10 genres genre_list = list(all_genres)[:10] # Use media client API to update genres success = self.media_client.update_artist_genres(artist, genre_list) if success: return True else: return False else: return False except Exception as e: logger.error(f"Error updating genres for {getattr(artist, 'title', 'Unknown')}: {e}") return False def update_album_artwork(self, artist, spotify_artist): """Update album artwork for all albums by this artist from Spotify. DB thumb_url is a media-server internal path, so album art must come from Spotify.""" try: updated_count = 0 skipped_count = 0 # Get all albums for this artist try: albums = list(artist.albums()) except Exception: logger.error(f"Could not access albums for artist '{artist.title}'") return 0 if not albums: logger.warning(f"No albums found for artist '{artist.title}'") return 0 import time for album in albums: try: album_title = getattr(album, 'title', 'Unknown Album') # Check if album already has good artwork on the media server if self.album_has_valid_artwork(album): skipped_count += 1 continue # Rate limit between album API calls time.sleep(0.5) # Search for this specific album on Spotify album_query = f"album:{album_title} artist:{spotify_artist.name}" spotify_albums = self.spotify_client.search_albums(album_query, limit=3) if not spotify_albums: continue # Find the best matching album best_album = None highest_score = 0.0 plex_album_normalized = self.matching_engine.normalize_string(album_title) for spotify_album in spotify_albums: spotify_album_normalized = self.matching_engine.normalize_string(spotify_album.name) score = self.matching_engine.similarity_score(plex_album_normalized, spotify_album_normalized) if score > highest_score: highest_score = score best_album = spotify_album # If we found a good match with artwork, download it if best_album and highest_score > 0.7 and best_album.image_url: if self.download_and_upload_album_artwork(album, best_album.image_url): updated_count += 1 except Exception as e: logger.error(f"Error processing album '{getattr(album, 'title', 'Unknown')}': {e}") continue return updated_count except Exception as e: logger.error(f"Error updating album artwork for artist '{getattr(artist, 'title', 'Unknown')}': {e}") return 0 def album_has_valid_artwork(self, album): """Check if album has valid artwork - EXACT copy from dashboard.py""" try: if not hasattr(album, 'thumb') or not album.thumb: return False thumb_url = str(album.thumb) # Completely empty or None if not thumb_url or thumb_url.strip() == '': return False # Obvious placeholder text in URL obvious_placeholders = ['no-image', 'placeholder', 'missing', 'default-album', 'blank.jpg', 'empty.png'] thumb_lower = thumb_url.lower() for placeholder in obvious_placeholders: if placeholder in thumb_lower: return False # Extremely short URLs (likely broken) if len(thumb_url) < 20: return False return True except Exception as e: return True def download_and_upload_album_artwork(self, album, image_url): """Download artwork from Spotify and upload to media server - EXACT copy from dashboard.py""" try: # Download image from Spotify response = requests.get(image_url, timeout=10) response.raise_for_status() # Validate and convert image image_data = self.validate_and_convert_image(response.content) if not image_data: return False # Upload using media client success = self.media_client.update_album_poster(album, image_data) return success except Exception as e: logger.error(f"Error downloading/uploading artwork for album '{getattr(album, 'title', 'Unknown')}': {e}") return False def artist_has_valid_photo(self, artist): """Check if artist has a valid photo - EXACT copy from dashboard.py""" try: if not hasattr(artist, 'thumb') or not artist.thumb: return False thumb_url = str(artist.thumb) if 'default' in thumb_url.lower() or len(thumb_url) < 50: return False return True except Exception: return False def validate_and_convert_image(self, image_data): """Validate and convert image for media server compatibility - EXACT copy from dashboard.py""" try: from PIL import Image import io # Open and validate image image = Image.open(io.BytesIO(image_data)) # Check minimum dimensions width, height = image.size if width < 200 or height < 200: return None # Convert to JPEG for consistency if image.format != 'JPEG': buffer = io.BytesIO() image.convert('RGB').save(buffer, format='JPEG', quality=95) return buffer.getvalue() return image_data except Exception: return None def upload_artist_poster(self, artist, image_data): """Upload poster using media client - EXACT copy from dashboard.py""" try: # Use media client's update method if available if hasattr(self.media_client, 'update_artist_poster'): return self.media_client.update_artist_poster(artist, image_data) # Fallback for Plex: direct API call if self.server_type == "plex": import requests server = self.media_client.server upload_url = f"{server._baseurl}/library/metadata/{artist.ratingKey}/posters" headers = { 'X-Plex-Token': server._token, 'Content-Type': 'image/jpeg' } response = requests.post(upload_url, data=image_data, headers=headers) response.raise_for_status() # Refresh artist to see changes artist.refresh() return True # Jellyfin: Use Jellyfin API to upload artist image elif self.server_type == "jellyfin": import requests jellyfin_config = config_manager.get_jellyfin_config() jellyfin_base_url = jellyfin_config.get('base_url', '') jellyfin_token = jellyfin_config.get('api_key', '') if not jellyfin_base_url or not jellyfin_token: logger.warning("Jellyfin configuration missing for image upload") return False upload_url = f"{jellyfin_base_url.rstrip('/')}/Items/{artist.ratingKey}/Images/Primary" headers = { 'Authorization': f'MediaBrowser Token="{jellyfin_token}"', 'Content-Type': 'image/jpeg' } response = requests.post(upload_url, data=image_data, headers=headers) response.raise_for_status() return True # Navidrome: Currently not supported (Subsonic API doesn't support image uploads) elif self.server_type == "navidrome": logger.info("ℹ️ Navidrome does not support artist image uploads via Subsonic API") return False else: # Unknown server type return False except Exception as e: logger.error(f"Error uploading poster: {e}") return False # --- Docker Helper Functions --- def docker_resolve_url(url): """ Resolve localhost URLs to Docker host when running in container """ import os if os.path.exists('/.dockerenv') and 'localhost' in url: return url.replace('localhost', 'host.docker.internal') return url # --- Main Execution --- def start_oauth_callback_servers(): """Start dedicated OAuth callback servers for Spotify and Tidal""" import threading from http.server import HTTPServer, BaseHTTPRequestHandler import urllib.parse # Spotify callback server (port 8888 — for direct/local access only) _oauth_logger = get_logger("oauth_callback") class SpotifyCallbackHandler(BaseHTTPRequestHandler): def do_GET(self): try: parsed_url = urllib.parse.urlparse(self.path) # Health check at root — lets users verify the server is running if parsed_url.path == '/': self.send_response(200) self.send_header('Content-type', 'text/plain') self.end_headers() self.wfile.write(b'SoulSync Spotify OAuth callback server is running. Callback URL: /callback') return # Only process requests to /callback — ignore everything else if parsed_url.path != '/callback': self.send_response(404) self.send_header('Content-type', 'text/plain') self.end_headers() self.wfile.write(b'Not found. Spotify callback is at /callback') return query_params = urllib.parse.parse_qs(parsed_url.query) _oauth_logger.info(f"Spotify callback received: {self.path}") if 'code' in query_params: auth_code = query_params['code'][0] _oauth_logger.info(f"Received Spotify authorization code: {auth_code[:10]}...") # Manually trigger the token exchange using spotipy's auth manager try: from core.spotify_client import SpotifyClient from spotipy.oauth2 import SpotifyOAuth from config.settings import config_manager # Get Spotify config config = config_manager.get_spotify_config() configured_uri = config.get('redirect_uri', "http://127.0.0.1:8888/callback") _oauth_logger.info(f"Using redirect_uri for token exchange: {configured_uri}") # Create auth manager and exchange code for token auth_manager = SpotifyOAuth( client_id=config['client_id'], client_secret=config['client_secret'], redirect_uri=configured_uri, scope="user-library-read user-read-private playlist-read-private playlist-read-collaborative user-read-email user-follow-read", cache_path='config/.spotify_cache' ) # Extract the authorization code and exchange it for tokens token_info = auth_manager.get_access_token(auth_code) if token_info: # Reinitialize the global client with new tokens global spotify_client spotify_client = SpotifyClient() if spotify_client.is_spotify_authenticated(): # Clear rate limit ban + post-ban cooldown so Spotify is usable immediately from core.spotify_client import _clear_rate_limit _clear_rate_limit() spotify_client._invalidate_auth_cache() # Invalidate status cache so next poll picks up the new connection _status_cache_timestamps['spotify'] = 0 # Refresh enrichment worker's client so it picks up new auth if spotify_enrichment_worker and hasattr(spotify_enrichment_worker, 'client'): spotify_enrichment_worker.client.reload_config() spotify_enrichment_worker.client._invalidate_auth_cache() add_activity_item("", "Spotify Auth Complete", "Successfully authenticated with Spotify", "Now") self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(b'

Spotify Authentication Successful!

You can close this window.

') else: raise Exception("Token exchange succeeded but authentication validation failed") else: raise Exception("Failed to exchange authorization code for access token") except Exception as e: _oauth_logger.error(f"Spotify token processing error: {e}") add_activity_item("", "Spotify Auth Failed", f"Token processing failed: {str(e)}", "Now") self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Spotify Authentication Failed

{str(e)}

'.encode()) elif 'error' in query_params: error = query_params['error'][0] _oauth_logger.error(f"Spotify OAuth error returned by Spotify: {error}") _oauth_logger.error(f"Full callback URL: {self.path}") add_activity_item("", "Spotify Auth Failed", f"Spotify returned error: {error}", "Now") self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Spotify Authentication Failed

Spotify returned error: {error}

'.encode()) else: # No code AND no error — callback was hit without OAuth params _oauth_logger.error("Spotify callback received without OAuth parameters (no code or error)") _oauth_logger.error(f"Path: {self.path} | Query params: {query_params}") _oauth_logger.error("This usually means the redirect lost its query parameters (reverse proxy issue)") self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() msg = ( '

Spotify Authentication Failed

' '

The callback was received but no authorization code or error was included.

' '

If you are using a reverse proxy: Your proxy may be stripping query parameters ' 'during the redirect. Try setting your Spotify redirect URI to use port 8008 instead ' '(e.g. https://yourdomain.com/callback) — the main app handles callbacks too.

' ) self.wfile.write(msg.encode()) except Exception as e: # Top-level catch-all — ensures we ALWAYS send an HTTP response. # Without this, BaseHTTPRequestHandler silently closes the connection # on unhandled exceptions, producing ERR_EMPTY_RESPONSE in the browser. _oauth_logger.error(f"Unhandled error in Spotify callback handler: {e}", exc_info=True) try: self.send_response(500) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Internal Server Error

{str(e)}

'.encode()) except Exception: pass # Connection already broken, nothing more we can do def log_message(self, format, *args): pass # Suppress BaseHTTPRequestHandler access logs (we use our own logger) # Start Spotify callback server def run_spotify_server(): _env_val = os.environ.get('SOULSYNC_SPOTIFY_CALLBACK_PORT') spotify_port = int(_env_val) if _env_val else 8888 if _env_val: logger.info(f"[OAuth] SOULSYNC_SPOTIFY_CALLBACK_PORT={_env_val!r} — binding Spotify callback server on port {spotify_port}") else: logger.info(f"[OAuth] SOULSYNC_SPOTIFY_CALLBACK_PORT not set — using default port {spotify_port}") try: bind_addr = ('0.0.0.0', spotify_port) spotify_server = HTTPServer(bind_addr, SpotifyCallbackHandler) _oauth_logger.info(f"Spotify OAuth callback server listening on {bind_addr[0]}:{bind_addr[1]}") logger.info(f"Started Spotify OAuth callback server on {bind_addr[0]}:{bind_addr[1]}") spotify_server.serve_forever() except OSError as e: _oauth_logger.error(f"Failed to start Spotify callback server on port {spotify_port}: {e} — port may already be in use") logger.error(f"Failed to start Spotify callback server on port {spotify_port}: {e}") except Exception as e: _oauth_logger.error(f"Failed to start Spotify callback server: {e}") logger.error(f"Failed to start Spotify callback server: {e}") # Tidal callback server class TidalCallbackHandler(BaseHTTPRequestHandler): def do_GET(self): logger.info("TIDAL CALLBACK SERVER RECEIVED REQUEST ") parsed_url = urllib.parse.urlparse(self.path) query_params = urllib.parse.parse_qs(parsed_url.query) logger.info(f"Callback path: {self.path}") if 'code' in query_params: auth_code = query_params['code'][0] logger.info(f"Received Tidal authorization code: {auth_code[:10]}...") # Exchange the authorization code for tokens try: from core.tidal_client import TidalClient # Create a temporary client and set the stored PKCE values temp_client = TidalClient() # Restore the PKCE values from the auth request global tidal_oauth_state with tidal_oauth_lock: temp_client.code_verifier = tidal_oauth_state["code_verifier"] temp_client.code_challenge = tidal_oauth_state["code_challenge"] logger.info(f"Restored PKCE - verifier: {temp_client.code_verifier[:20] if temp_client.code_verifier else 'None'}... challenge: {temp_client.code_challenge[:20] if temp_client.code_challenge else 'None'}...") success = temp_client.fetch_token_from_code(auth_code) if success: # Reinitialize the global tidal client with new tokens global tidal_client tidal_client = TidalClient() if tidal_enrichment_worker: tidal_enrichment_worker.client = tidal_client add_activity_item("", "Tidal Auth Complete", "Successfully authenticated with Tidal", "Now") self.send_response(200) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(b'

Tidal Authentication Successful!

You can close this window.

') else: raise Exception("Failed to exchange authorization code for tokens") except Exception as e: logger.error(f"Tidal token processing error: {e}") add_activity_item("", "Tidal Auth Failed", f"Token processing failed: {str(e)}", "Now") self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Tidal Authentication Failed

{str(e)}

'.encode()) else: error = query_params.get('error', ['Unknown error'])[0] logger.error(f"Tidal OAuth error: {error}") add_activity_item("", "Tidal Auth Failed", f"OAuth error: {error}", "Now") self.send_response(400) self.send_header('Content-type', 'text/html') self.end_headers() self.wfile.write(f'

Tidal Authentication Failed

{error}

'.encode()) def log_message(self, format, *args): pass # Suppress server logs def run_tidal_server(): _env_val = os.environ.get('SOULSYNC_TIDAL_CALLBACK_PORT') tidal_port = int(_env_val) if _env_val else 8889 if _env_val: logger.info(f"[OAuth] SOULSYNC_TIDAL_CALLBACK_PORT={_env_val!r} — binding Tidal callback server on port {tidal_port}") else: logger.info(f"[OAuth] SOULSYNC_TIDAL_CALLBACK_PORT not set — using default port {tidal_port}") try: tidal_server = HTTPServer(('0.0.0.0', tidal_port), TidalCallbackHandler) logger.info(f"Started Tidal OAuth callback server on port {tidal_port}") logger.info(f"Tidal server listening on all interfaces, port {tidal_port}") tidal_server.serve_forever() except Exception as e: logger.error(f"Failed to start Tidal callback server: {e}") import traceback logger.error(f"Full error: {traceback.format_exc()}") # Start both servers in background threads spotify_thread = threading.Thread(target=run_spotify_server, daemon=True) tidal_thread = threading.Thread(target=run_tidal_server, daemon=True) spotify_thread.start() tidal_thread.start() logger.info("OAuth callback servers started") # ================================================================================================ # MUSICBRAINZ ENRICHMENT - PHASE 5 WEB UI INTEGRATION # ================================================================================================ # --- MusicBrainz Worker Initialization --- mb_worker = None try: from database.music_database import MusicDatabase mb_db = MusicDatabase() mb_worker = MusicBrainzWorker( database=mb_db, app_name="SoulSync", app_version="1.0", contact_email="" ) # Start worker automatically (can be paused via UI) mb_worker.start() if config_manager.get('musicbrainz_enrichment_paused', False): mb_worker.pause() logger.info("MusicBrainz enrichment worker initialized (paused — restored from config)") else: logger.info("MusicBrainz enrichment worker initialized and started") except Exception as e: logger.error(f"MusicBrainz worker initialization failed: {e}") mb_worker = None # --- MusicBrainz API Endpoints --- @app.route('/api/musicbrainz/status', methods=['GET']) def musicbrainz_status(): """Get MusicBrainz enrichment status for UI polling""" try: if mb_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = mb_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting MusicBrainz status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/musicbrainz/pause', methods=['POST']) def musicbrainz_pause(): """Pause MusicBrainz enrichment worker (finishes current match first)""" try: if mb_worker is None: return jsonify({'error': 'MusicBrainz worker not initialized'}), 400 mb_worker.pause() config_manager.set('musicbrainz_enrichment_paused', True) logger.info("MusicBrainz worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing MusicBrainz worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/musicbrainz/resume', methods=['POST']) def musicbrainz_resume(): """Resume MusicBrainz enrichment worker""" try: if mb_worker is None: return jsonify({'error': 'MusicBrainz worker not initialized'}), 400 mb_worker.resume() config_manager.set('musicbrainz_enrichment_paused', False) logger.info("MusicBrainz worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming MusicBrainz worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END MUSICBRAINZ INTEGRATION # ================================================================================================ # ================================================================================================ # AUDIODB ENRICHMENT - ARTIST METADATA & IMAGES # ================================================================================================ # --- AudioDB Worker Initialization --- audiodb_worker = None try: from database.music_database import MusicDatabase audiodb_db = MusicDatabase() audiodb_worker = AudioDBWorker(database=audiodb_db) audiodb_worker.start() if config_manager.get('audiodb_enrichment_paused', False): audiodb_worker.pause() logger.info("AudioDB enrichment worker initialized (paused — restored from config)") else: logger.info("AudioDB enrichment worker initialized and started") except Exception as e: logger.error(f"AudioDB worker initialization failed: {e}") audiodb_worker = None # --- AudioDB API Endpoints --- @app.route('/api/audiodb/status', methods=['GET']) def audiodb_status(): """Get AudioDB enrichment status for UI polling""" try: if audiodb_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = audiodb_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting AudioDB status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/audiodb/pause', methods=['POST']) def audiodb_pause(): """Pause AudioDB enrichment worker""" try: if audiodb_worker is None: return jsonify({'error': 'AudioDB worker not initialized'}), 400 audiodb_worker.pause() config_manager.set('audiodb_enrichment_paused', True) logger.info("AudioDB worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing AudioDB worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/audiodb/resume', methods=['POST']) def audiodb_resume(): """Resume AudioDB enrichment worker""" try: if audiodb_worker is None: return jsonify({'error': 'AudioDB worker not initialized'}), 400 audiodb_worker.resume() config_manager.set('audiodb_enrichment_paused', False) logger.info("AudioDB worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming AudioDB worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END AUDIODB INTEGRATION # ================================================================================================ # --- Discogs Worker Initialization --- discogs_worker = None try: from core.discogs_worker import DiscogsWorker from database.music_database import MusicDatabase discogs_db = MusicDatabase() discogs_worker = DiscogsWorker(database=discogs_db) discogs_worker.start() if config_manager.get('discogs_enrichment_paused', False): discogs_worker.pause() logger.info("Discogs enrichment worker initialized (paused — restored from config)") else: logger.info("Discogs enrichment worker initialized and started") except Exception as e: logger.error(f"Discogs worker initialization failed: {e}") discogs_worker = None # --- Discogs API Endpoints --- @app.route('/api/discogs/status', methods=['GET']) def discogs_status(): """Get Discogs enrichment status for UI polling""" try: if discogs_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, }), 200 return jsonify(discogs_worker.get_stats()), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/discogs/pause', methods=['POST']) def discogs_pause(): """Pause Discogs enrichment worker""" try: if discogs_worker is None: return jsonify({'error': 'Discogs worker not initialized'}), 400 discogs_worker.pause() config_manager.set('discogs_enrichment_paused', True) logger.info("Discogs worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/discogs/resume', methods=['POST']) def discogs_resume(): """Resume Discogs enrichment worker""" try: if discogs_worker is None: return jsonify({'error': 'Discogs worker not initialized'}), 400 discogs_worker.resume() config_manager.set('discogs_enrichment_paused', False) logger.info("Discogs worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 # ================================================================================================ # DEEZER ENRICHMENT INTEGRATION # ================================================================================================ # --- Deezer Worker Initialization --- deezer_worker = None try: from database.music_database import MusicDatabase deezer_db = MusicDatabase() deezer_worker = DeezerWorker(database=deezer_db) deezer_worker.start() if config_manager.get('deezer_enrichment_paused', False): deezer_worker.pause() logger.info("Deezer enrichment worker initialized (paused — restored from config)") else: logger.info("Deezer enrichment worker initialized and started") except Exception as e: logger.error(f"Deezer worker initialization failed: {e}") deezer_worker = None # --- Deezer API Endpoints --- @app.route('/api/deezer/status', methods=['GET']) def deezer_status(): """Get Deezer enrichment status for UI polling""" try: if deezer_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = deezer_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Deezer status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/deezer/pause', methods=['POST']) def deezer_pause(): """Pause Deezer enrichment worker""" try: if deezer_worker is None: return jsonify({'error': 'Deezer worker not initialized'}), 400 deezer_worker.pause() config_manager.set('deezer_enrichment_paused', True) logger.info("Deezer worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Deezer worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/deezer/resume', methods=['POST']) def deezer_resume(): """Resume Deezer enrichment worker""" try: if deezer_worker is None: return jsonify({'error': 'Deezer worker not initialized'}), 400 deezer_worker.resume() config_manager.set('deezer_enrichment_paused', False) logger.info("Deezer worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Deezer worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END DEEZER INTEGRATION # ================================================================================================ # ================================================================================================ # SPOTIFY ENRICHMENT INTEGRATION # ================================================================================================ # --- Spotify Worker Initialization --- spotify_enrichment_worker = None try: from database.music_database import MusicDatabase spotify_enrichment_db = MusicDatabase() spotify_enrichment_worker = SpotifyWorker(database=spotify_enrichment_db) if config_manager.get('spotify_enrichment_paused', False): spotify_enrichment_worker.paused = True # Set BEFORE start() to prevent race condition spotify_enrichment_worker.start() if spotify_enrichment_worker.paused: logger.info("Spotify enrichment worker initialized (paused — restored from config)") else: logger.info("Spotify enrichment worker initialized and started") except Exception as e: logger.error(f"Spotify enrichment worker initialization failed: {e}") spotify_enrichment_worker = None # --- API Rate Monitor Endpoints --- @app.route('/api/rate-monitor/history/', methods=['GET']) def get_rate_monitor_history(service_key): """Get 24-hour minute-bucketed call history for a service. Used by the detail modal graph.""" try: from core.api_call_tracker import api_call_tracker, RATE_LIMITS history = api_call_tracker.get_24h_history(service_key) return jsonify({ 'service': service_key, 'rate_limit': RATE_LIMITS.get(service_key.split(':')[0], 60), 'history': history, }) except Exception as e: return jsonify({'error': str(e)}), 500 # --- Spotify API Endpoints --- @app.route('/api/spotify-enrichment/status', methods=['GET']) def spotify_enrichment_status(): """Get Spotify enrichment status for UI polling""" try: if spotify_enrichment_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = spotify_enrichment_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Spotify enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/spotify-enrichment/pause', methods=['POST']) def spotify_enrichment_pause(): """Pause Spotify enrichment worker""" try: if spotify_enrichment_worker is None: return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400 spotify_enrichment_worker.pause() config_manager.set('spotify_enrichment_paused', True) # Drop any auto-pause marker so the post-download resume loop won't # override this explicit user pause. _download_auto_paused.discard('spotify-enrichment') logger.info("Spotify enrichment worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Spotify enrichment worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/spotify-enrichment/resume', methods=['POST']) def spotify_enrichment_resume(): """Resume Spotify enrichment worker""" try: if spotify_enrichment_worker is None: return jsonify({'error': 'Spotify enrichment worker not initialized'}), 400 # Block resume while Spotify is rate limited if _spotify_rate_limited(): return jsonify({'error': 'Cannot resume while Spotify is rate limited', 'rate_limited': True}), 429 spotify_enrichment_worker.resume() config_manager.set('spotify_enrichment_paused', False) _download_auto_paused.discard('spotify-enrichment') _download_yield_override.add('spotify-enrichment') # User override — don't re-pause during this download session logger.info("Spotify enrichment worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Spotify enrichment worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END SPOTIFY ENRICHMENT INTEGRATION # ================================================================================================ # ================================================================================================ # ITUNES ENRICHMENT INTEGRATION # ================================================================================================ # --- iTunes Worker Initialization --- itunes_enrichment_worker = None try: from database.music_database import MusicDatabase itunes_enrichment_db = MusicDatabase() itunes_enrichment_worker = iTunesWorker(database=itunes_enrichment_db) itunes_enrichment_worker.start() if config_manager.get('itunes_enrichment_paused', False): itunes_enrichment_worker.pause() logger.info("iTunes enrichment worker initialized (paused — restored from config)") else: logger.info("iTunes enrichment worker initialized and started") except Exception as e: logger.error(f"iTunes enrichment worker initialization failed: {e}") itunes_enrichment_worker = None # --- iTunes API Endpoints --- @app.route('/api/itunes-enrichment/status', methods=['GET']) def itunes_enrichment_status(): """Get iTunes enrichment status for UI polling""" try: if itunes_enrichment_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = itunes_enrichment_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting iTunes enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/itunes-enrichment/pause', methods=['POST']) def itunes_enrichment_pause(): """Pause iTunes enrichment worker""" try: if itunes_enrichment_worker is None: return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400 itunes_enrichment_worker.pause() config_manager.set('itunes_enrichment_paused', True) logger.info("iTunes enrichment worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing iTunes enrichment worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/itunes-enrichment/resume', methods=['POST']) def itunes_enrichment_resume(): """Resume iTunes enrichment worker""" try: if itunes_enrichment_worker is None: return jsonify({'error': 'iTunes enrichment worker not initialized'}), 400 itunes_enrichment_worker.resume() config_manager.set('itunes_enrichment_paused', False) logger.info("iTunes enrichment worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming iTunes enrichment worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END ITUNES ENRICHMENT INTEGRATION # ================================================================================================ # ================================================================================================ # LAST.FM ENRICHMENT WORKER # ================================================================================================ lastfm_worker = None try: from database.music_database import MusicDatabase lastfm_db = MusicDatabase() lastfm_worker = LastFMWorker(database=lastfm_db) lastfm_worker.start() if config_manager.get('lastfm_enrichment_paused', False): lastfm_worker.pause() logger.info("Last.fm enrichment worker initialized (paused — restored from config)") else: logger.info("Last.fm enrichment worker initialized and started") except Exception as e: logger.error(f"Last.fm worker initialization failed: {e}") lastfm_worker = None # --- Last.fm API Endpoints --- @app.route('/api/lastfm-enrichment/status', methods=['GET']) def lastfm_enrichment_status(): """Get Last.fm enrichment status for UI polling""" try: if lastfm_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = lastfm_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Last.fm enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/lastfm-enrichment/pause', methods=['POST']) def lastfm_enrichment_pause(): """Pause Last.fm enrichment worker""" try: if lastfm_worker is None: return jsonify({'error': 'Last.fm worker not initialized'}), 400 lastfm_worker.pause() config_manager.set('lastfm_enrichment_paused', True) # Drop any auto-pause marker so the post-download resume loop won't # override this explicit user pause. _download_auto_paused.discard('lastfm-enrichment') logger.info("Last.fm worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Last.fm worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/lastfm-enrichment/resume', methods=['POST']) def lastfm_enrichment_resume(): """Resume Last.fm enrichment worker""" try: if lastfm_worker is None: return jsonify({'error': 'Last.fm worker not initialized'}), 400 lastfm_worker.resume() config_manager.set('lastfm_enrichment_paused', False) _download_auto_paused.discard('lastfm-enrichment') _download_yield_override.add('lastfm-enrichment') # User override — don't re-pause during this download session logger.info("Last.fm worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Last.fm worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/artist//lastfm-top-tracks', methods=['GET']) def get_artist_lastfm_top_tracks(artist_id): """Get top tracks for an artist from Last.fm (lazy-loaded by frontend).""" try: artist_name = request.args.get('name', '') if not artist_name: return jsonify({'success': False, 'error': 'Artist name required'}), 400 if not lastfm_worker or not lastfm_worker.client: return jsonify({'success': True, 'tracks': []}) limit = int(request.args.get('limit', 100)) tracks = lastfm_worker.client.get_artist_top_tracks(artist_name, limit=min(limit, 100)) return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Error fetching Last.fm top tracks: {e}") return jsonify({'success': True, 'tracks': []}) # ================================================================================================ @app.route('/api/lastfm/auth-url', methods=['GET']) def lastfm_auth_url(): """Get the Last.fm authorization URL for scrobbling.""" try: api_key = config_manager.get('lastfm.api_key', '') if not api_key: return jsonify({'success': False, 'error': 'Last.fm API key not configured'}), 400 # Build callback URL callback = request.host_url.rstrip('/') + '/api/lastfm/callback' auth_url = f"https://www.last.fm/api/auth/?api_key={api_key}&cb={callback}" return jsonify({'success': True, 'url': auth_url}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/lastfm/callback', methods=['GET']) def lastfm_callback(): """Last.fm auth callback — exchanges token for session key.""" try: token = request.args.get('token') if not token: return "Error: No token received from Last.fm", 400 api_key = config_manager.get('lastfm.api_key', '') api_secret = config_manager.get('lastfm.api_secret', '') if not api_key or not api_secret: return "Error: Last.fm API key and secret must be configured in Settings", 400 from core.lastfm_client import LastFMClient client = LastFMClient(api_key=api_key, api_secret=api_secret) session_key = client.get_session_key(token) if session_key: config_manager.set('lastfm.session_key', session_key) return """

Last.fm Scrobbling Authorized!

You can close this window and return to SoulSync.

""" else: return "Error: Failed to get session key from Last.fm. Check your API key and secret.", 400 except Exception as e: return f"Error: {e}", 500 # END LAST.FM ENRICHMENT INTEGRATION # ================================================================================================ # ================================================================================================ # GENIUS ENRICHMENT WORKER # ================================================================================================ genius_worker = None try: from database.music_database import MusicDatabase genius_db = MusicDatabase() genius_worker = GeniusWorker(database=genius_db) if config_manager.get('genius_enrichment_paused', False): genius_worker.paused = True genius_worker.start() if genius_worker.paused: logger.info("Genius enrichment worker initialized (paused — restored from config)") else: logger.info("Genius enrichment worker initialized and started") except Exception as e: logger.error(f"Genius worker initialization failed: {e}") genius_worker = None # --- Genius API Endpoints --- @app.route('/api/genius-enrichment/status', methods=['GET']) def genius_enrichment_status(): """Get Genius enrichment status for UI polling""" try: if genius_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = genius_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Genius enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/genius-enrichment/pause', methods=['POST']) def genius_enrichment_pause(): """Pause Genius enrichment worker""" try: if genius_worker is None: return jsonify({'error': 'Genius worker not initialized'}), 400 genius_worker.pause() config_manager.set('genius_enrichment_paused', True) # Drop any auto-pause marker so the post-download resume loop won't # override this explicit user pause. _download_auto_paused.discard('genius-enrichment') logger.info("Genius worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Genius worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/genius-enrichment/resume', methods=['POST']) def genius_enrichment_resume(): """Resume Genius enrichment worker""" try: if genius_worker is None: return jsonify({'error': 'Genius worker not initialized'}), 400 genius_worker.resume() config_manager.set('genius_enrichment_paused', False) _download_auto_paused.discard('genius-enrichment') _download_yield_override.add('genius-enrichment') # User override — don't re-pause during this download session logger.info("Genius worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Genius worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END GENIUS ENRICHMENT INTEGRATION # ================================================================================================ # ================================================================================================ # TIDAL ENRICHMENT WORKER # ================================================================================================ tidal_enrichment_worker = None try: from database.music_database import MusicDatabase tidal_enrich_db = MusicDatabase() tidal_enrichment_worker = TidalWorker(database=tidal_enrich_db, client=tidal_client) tidal_enrichment_worker.start() if config_manager.get('tidal_enrichment_paused', False): tidal_enrichment_worker.pause() logger.info("Tidal enrichment worker initialized (paused — restored from config)") else: logger.info("Tidal enrichment worker initialized and started") except Exception as e: logger.error(f"Tidal worker initialization failed: {e}") tidal_enrichment_worker = None # --- Tidal Enrichment API Endpoints --- @app.route('/api/tidal-enrichment/status', methods=['GET']) def tidal_enrichment_status(): """Get Tidal enrichment status for UI polling""" try: if tidal_enrichment_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'authenticated': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = tidal_enrichment_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Tidal enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/tidal-enrichment/pause', methods=['POST']) def tidal_enrichment_pause(): """Pause Tidal enrichment worker""" try: if tidal_enrichment_worker is None: return jsonify({'error': 'Tidal worker not initialized'}), 400 tidal_enrichment_worker.pause() config_manager.set('tidal_enrichment_paused', True) logger.info("Tidal worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Tidal worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/tidal-enrichment/resume', methods=['POST']) def tidal_enrichment_resume(): """Resume Tidal enrichment worker""" try: if tidal_enrichment_worker is None: return jsonify({'error': 'Tidal worker not initialized'}), 400 tidal_enrichment_worker.resume() config_manager.set('tidal_enrichment_paused', False) logger.info("Tidal worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Tidal worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # QOBUZ ENRICHMENT WORKER # ================================================================================================ qobuz_enrichment_worker = None try: from database.music_database import MusicDatabase from core.qobuz_client import QobuzClient qobuz_enrich_db = MusicDatabase() qobuz_enrich_client = QobuzClient() # Separate client instance for thread safety qobuz_enrichment_worker = QobuzWorker(database=qobuz_enrich_db, client=qobuz_enrich_client) qobuz_enrichment_worker.start() if config_manager.get('qobuz_enrichment_paused', False): qobuz_enrichment_worker.pause() logger.info("Qobuz enrichment worker initialized (paused — restored from config)") else: logger.info("Qobuz enrichment worker initialized and started") except Exception as e: logger.error(f"Qobuz worker initialization failed: {e}") qobuz_enrichment_worker = None # --- Qobuz Enrichment API Endpoints --- @app.route('/api/qobuz-enrichment/status', methods=['GET']) def qobuz_enrichment_status(): """Get Qobuz enrichment status for UI polling""" try: if qobuz_enrichment_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'authenticated': False, 'current_item': None, 'stats': {'matched': 0, 'not_found': 0, 'pending': 0, 'errors': 0}, 'progress': {} }), 200 status = qobuz_enrichment_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Qobuz enrichment status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/qobuz-enrichment/pause', methods=['POST']) def qobuz_enrichment_pause(): """Pause Qobuz enrichment worker""" try: if qobuz_enrichment_worker is None: return jsonify({'error': 'Qobuz worker not initialized'}), 400 qobuz_enrichment_worker.pause() config_manager.set('qobuz_enrichment_paused', True) logger.info("Qobuz worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Qobuz worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/qobuz-enrichment/resume', methods=['POST']) def qobuz_enrichment_resume(): """Resume Qobuz enrichment worker""" try: if qobuz_enrichment_worker is None: return jsonify({'error': 'Qobuz worker not initialized'}), 400 qobuz_enrichment_worker.resume() config_manager.set('qobuz_enrichment_paused', False) logger.info("Qobuz worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Qobuz worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END TIDAL/QOBUZ ENRICHMENT INTEGRATION # ================================================================================================ # ================================================================================================ # HYDRABASE P2P MIRROR WORKER # ================================================================================================ # --- Hydrabase Worker & Client Initialization --- hydrabase_worker = None hydrabase_client = None try: def _get_hydrabase_ws_and_lock(): return (_hydrabase_ws, _hydrabase_lock) hydrabase_worker = HydrabaseWorker(get_ws_and_lock=_get_hydrabase_ws_and_lock) hydrabase_worker.start() hydrabase_client = HydrabaseClient(get_ws_and_lock=_get_hydrabase_ws_and_lock) logger.info("Hydrabase P2P mirror worker and metadata client initialized") # Update API blueprint references if hasattr(app, 'soulsync'): app.soulsync['hydrabase_client'] = hydrabase_client app.soulsync['hydrabase_worker'] = hydrabase_worker except Exception as e: logger.error(f"Hydrabase initialization failed: {e}") hydrabase_worker = None hydrabase_client = None # --- Hydrabase Auto-Reconnect --- try: _hydra_cfg = config_manager.get_hydrabase_config() if _hydra_cfg.get('auto_connect') and _hydra_cfg.get('url') and _hydra_cfg.get('api_key'): import websocket as _ws_mod _auto_ws = _ws_mod.create_connection( _hydra_cfg['url'], header={"x-api-key": _hydra_cfg['api_key']}, timeout=10 ) _hydrabase_ws = _auto_ws # Don't auto-enable dev mode — user must explicitly activate dev mode # Auto-connect just establishes the WebSocket for fallback/search tab use logger.info(f"Hydrabase auto-connected to {_hydra_cfg['url']}") except Exception as e: logger.error(f"Hydrabase auto-reconnect failed: {e}") # --- Hydrabase Worker API Endpoints --- @app.route('/api/hydrabase-worker/status', methods=['GET']) def hydrabase_worker_status(): """Get Hydrabase P2P mirror worker status for UI polling""" try: if hydrabase_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'queue_size': 0, 'stats': {'sent': 0, 'dropped': 0, 'errors': 0} }), 200 status = hydrabase_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting Hydrabase worker status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/hydrabase-worker/pause', methods=['POST']) def hydrabase_worker_pause(): """Pause Hydrabase P2P mirror worker""" try: if hydrabase_worker is None: return jsonify({'error': 'Hydrabase worker not initialized'}), 400 hydrabase_worker.pause() logger.info("Hydrabase worker paused via UI") return jsonify({'status': 'paused'}), 200 except Exception as e: logger.error(f"Error pausing Hydrabase worker: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/hydrabase-worker/resume', methods=['POST']) def hydrabase_worker_resume(): """Resume Hydrabase P2P mirror worker""" try: if hydrabase_worker is None: return jsonify({'error': 'Hydrabase worker not initialized'}), 400 hydrabase_worker.resume() logger.info("Hydrabase worker resumed via UI") return jsonify({'status': 'running'}), 200 except Exception as e: logger.error(f"Error resuming Hydrabase worker: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END HYDRABASE P2P MIRROR WORKER # ================================================================================================ # ================================================================================================ # LIBRARY REPAIR WORKER # ================================================================================================ from core.repair_worker import RepairWorker # =================================================================== # SoulID Worker — generates deterministic soul IDs for library entities # =================================================================== soulid_worker = None try: from core.soulid_worker import SoulIDWorker from database.music_database import MusicDatabase soulid_db = MusicDatabase() soulid_worker = SoulIDWorker(database=soulid_db) soulid_worker.start() logger.info("SoulID worker initialized and started") except Exception as e: logger.error(f"SoulID worker initialization failed: {e}") soulid_worker = None @app.route('/api/soulid/status', methods=['GET']) def soulid_status(): """Get SoulID worker status for UI polling.""" try: if soulid_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': False, 'idle': False, 'current_item': None, 'stats': {} }) return jsonify(soulid_worker.get_stats()) except Exception as e: return jsonify({'error': str(e)}), 500 # =================================================================== # Listening Stats Worker — polls media servers for play data # =================================================================== listening_stats_worker = None try: from core.listening_stats_worker import ListeningStatsWorker from database.music_database import MusicDatabase listening_stats_db = MusicDatabase() listening_stats_worker = ListeningStatsWorker( database=listening_stats_db, config_manager=config_manager, plex_client=plex_client, jellyfin_client=jellyfin_client, navidrome_client=navidrome_client, ) listening_stats_worker.start() logger.info("Listening stats worker initialized and started") except Exception as e: logger.error(f"Listening stats worker initialization failed: {e}") listening_stats_worker = None # --- Stats API Endpoints --- # Logic lives in core/stats/queries.py — these routes are thin handlers. from core.stats import queries as _stats_queries @app.route('/api/stats/cached', methods=['GET']) def stats_cached(): """Get all pre-computed stats for a time range from cache. Instant response.""" try: time_range = request.args.get('range', '7d') data = _stats_queries.get_cached_stats(get_database(), fix_artist_image_url, time_range) return jsonify({'success': True, **data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/overview', methods=['GET']) def stats_overview(): """Get aggregate listening stats for a time range.""" try: time_range = request.args.get('range', 'all') data = _stats_queries.get_overview(get_database(), time_range) return jsonify({'success': True, **data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/top-artists', methods=['GET']) def stats_top_artists(): """Get top artists by play count.""" try: time_range = request.args.get('range', 'all') limit = int(request.args.get('limit', 10)) artists = _stats_queries.get_top_artists(get_database(), fix_artist_image_url, time_range, limit) return jsonify({'success': True, 'artists': artists}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/top-albums', methods=['GET']) def stats_top_albums(): """Get top albums by play count.""" try: time_range = request.args.get('range', 'all') limit = int(request.args.get('limit', 10)) albums = _stats_queries.get_top_albums(get_database(), fix_artist_image_url, time_range, limit) return jsonify({'success': True, 'albums': albums}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/top-tracks', methods=['GET']) def stats_top_tracks(): """Get top tracks by play count.""" try: time_range = request.args.get('range', 'all') limit = int(request.args.get('limit', 10)) tracks = _stats_queries.get_top_tracks(get_database(), fix_artist_image_url, time_range, limit) return jsonify({'success': True, 'tracks': tracks}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/timeline', methods=['GET']) def stats_timeline(): """Get play count per time period for chart rendering.""" try: time_range = request.args.get('range', '30d') granularity = request.args.get('granularity', 'day') data = _stats_queries.get_timeline(get_database(), time_range, granularity) return jsonify({'success': True, 'timeline': data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/genres', methods=['GET']) def stats_genres(): """Get genre distribution by play count.""" try: time_range = request.args.get('range', 'all') data = _stats_queries.get_genres(get_database(), time_range) return jsonify({'success': True, 'genres': data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/library-health', methods=['GET']) def stats_library_health(): """Get library health metrics.""" try: data = _stats_queries.get_library_health(get_database()) return jsonify({'success': True, **data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/db-storage', methods=['GET']) def stats_db_storage(): """Get database storage breakdown by table.""" try: data = _stats_queries.get_db_storage(get_database()) return jsonify({'success': True, **data}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/recent', methods=['GET']) def stats_recent(): """Get recently played tracks.""" try: limit = int(request.args.get('limit', 20)) tracks = _stats_queries.get_recent_tracks(get_database(), limit) return jsonify({'success': True, 'tracks': tracks}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/stats/resolve-track', methods=['POST']) def stats_resolve_track(): """Resolve a track by title+artist to get its file_path for playback.""" try: data = request.get_json() title = data.get('title', '') artist = data.get('artist', '') if not title: return jsonify({'success': False, 'error': 'Title required'}), 400 track = _stats_queries.resolve_track(get_database(), fix_artist_image_url, title, artist) if track is None: return jsonify({'success': False, 'error': 'Track not found in library'}) return jsonify({'success': True, 'track': track}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/listening-stats/sync', methods=['POST']) def listening_stats_sync(): """Trigger an immediate listening stats poll.""" try: if not listening_stats_worker: return jsonify({'success': False, 'error': 'Listening stats worker not initialized'}), 400 _stats_queries.trigger_listening_sync(listening_stats_worker) return jsonify({'success': True, 'message': 'Sync started'}) except Exception as e: return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/listening-stats/status', methods=['GET']) def listening_stats_status(): """Get listening stats worker status.""" try: return jsonify(_stats_queries.get_listening_status(listening_stats_worker)) except Exception as e: return jsonify({'error': str(e)}), 500 # =================================================================== # Repair Worker — Library maintenance and repair jobs # =================================================================== repair_worker = None try: from database.music_database import MusicDatabase repair_db = MusicDatabase() transfer_path = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) repair_worker = RepairWorker(database=repair_db, transfer_folder=transfer_path) repair_worker.set_config_manager(config_manager) repair_worker.set_metadata_enhancer(_enhance_file_metadata) # --- Repair Job Progress Tracking (live progress like automation cards) --- repair_job_progress_states = {} # job_id (str) -> state dict repair_job_progress_lock = threading.Lock() def _repair_job_start(job_id, display_name): with repair_job_progress_lock: repair_job_progress_states[job_id] = { 'status': 'running', 'display_name': display_name, 'progress': 0, 'phase': 'Starting...', 'current_item': '', 'processed': 0, 'total': 0, 'log': [{'type': 'info', 'text': f'Starting {display_name}'}], 'started_at': datetime.now(timezone.utc).isoformat(), 'finished_at': None, } def _repair_job_progress(job_id, **kwargs): with repair_job_progress_lock: state = repair_job_progress_states.get(job_id) if not state: return for k, v in kwargs.items(): if k == 'log_line': state['log'].append({'type': kwargs.get('log_type', 'info'), 'text': v}) if len(state['log']) > 50: state['log'] = state['log'][-50:] elif k == 'scanned': state['processed'] = v if state.get('total', 0) > 0: state['progress'] = round(v / state['total'] * 100) elif k == 'total': state['total'] = v if state.get('processed', 0) > 0 and v > 0: state['progress'] = round(state['processed'] / v * 100) elif k != 'log_type': state[k] = v def _repair_job_finish(job_id, status, result): with repair_job_progress_lock: state = repair_job_progress_states.get(job_id) if not state: return state['status'] = status state['progress'] = 100 state['finished_at'] = datetime.now(timezone.utc).isoformat() summary = f'Done: {result.scanned} scanned, {result.auto_fixed} fixed, {result.findings_created} findings, {result.errors} errors' state['log'].append({'type': 'success' if status == 'finished' else 'error', 'text': summary}) try: socketio.emit('repair:progress', {job_id: dict(state)}) except Exception: pass repair_worker.register_progress_callbacks(_repair_job_start, _repair_job_progress, _repair_job_finish) # Store refs for WebSocket push loop repair_worker._progress_lock_ref = repair_job_progress_lock repair_worker._progress_states_ref = repair_job_progress_states repair_worker.start() logger.info("Repair worker initialized and started") except Exception as e: logger.error(f"Repair worker initialization failed: {e}") repair_worker = None # --- Repair Worker API Endpoints --- @app.route('/api/repair/status', methods=['GET']) def repair_status(): """Get repair worker status""" try: if repair_worker is None: return jsonify({ 'enabled': False, 'running': False, 'paused': True, 'idle': False, 'current_item': None, 'current_job': None, 'findings_pending': 0, 'stats': {'scanned': 0, 'repaired': 0, 'skipped': 0, 'errors': 0, 'pending': 0}, 'progress': {} }), 200 status = repair_worker.get_stats() return jsonify(status), 200 except Exception as e: logger.error(f"Error getting repair status: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/toggle', methods=['POST']) def repair_toggle(): """Toggle master enable/disable""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 new_state = repair_worker.toggle() logger.info("Repair worker %s via UI", "enabled" if new_state else "disabled") return jsonify({'enabled': new_state}), 200 except Exception as e: logger.error(f"Error toggling repair worker: {e}") return jsonify({'error': str(e)}), 500 # Backward compat aliases @app.route('/api/repair/pause', methods=['POST']) def repair_pause(): try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 repair_worker.pause() return jsonify({'status': 'paused'}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/repair/resume', methods=['POST']) def repair_resume(): try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 repair_worker.resume() return jsonify({'status': 'running'}), 200 except Exception as e: return jsonify({'error': str(e)}), 500 @app.route('/api/repair/jobs', methods=['GET']) def repair_jobs_list(): """Get all jobs with config and last run info""" try: if repair_worker is None: return jsonify({'jobs': []}), 200 jobs = repair_worker.get_all_job_info() return jsonify({'jobs': jobs}), 200 except Exception as e: logger.error(f"Error getting repair jobs: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/jobs//toggle', methods=['POST']) def repair_job_toggle(job_id): """Enable/disable a specific job""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} enabled = data.get('enabled') if enabled is None: # Toggle — get current state and flip it config = repair_worker.get_job_config(job_id) enabled = not config.get('enabled', False) repair_worker.set_job_enabled(job_id, enabled) logger.info("Repair job %s %s via UI", job_id, "enabled" if enabled else "disabled") return jsonify({'job_id': job_id, 'enabled': enabled}), 200 except Exception as e: logger.error(f"Error toggling repair job {job_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/jobs//settings', methods=['PUT']) def repair_job_settings(job_id): """Update job interval and/or settings""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} interval_hours = data.get('interval_hours') settings = data.get('settings') repair_worker.set_job_settings(job_id, interval_hours=interval_hours, settings=settings) logger.info("Repair job %s settings updated via UI", job_id) return jsonify({'success': True}), 200 except Exception as e: logger.error(f"Error updating repair job settings {job_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/jobs//run', methods=['POST']) def repair_job_run(job_id): """Trigger immediate run of a specific job""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 repair_worker.run_job_now(job_id) logger.info("Repair job %s triggered manually via UI", job_id) return jsonify({'success': True, 'job_id': job_id}), 200 except Exception as e: logger.error(f"Error running repair job {job_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings', methods=['GET']) def repair_findings_list(): """Get paginated findings with filters""" try: if repair_worker is None: return jsonify({'items': [], 'total': 0, 'page': 0, 'limit': 50}), 200 job_id = request.args.get('job_id') status = request.args.get('status') severity = request.args.get('severity') page = int(request.args.get('page', 0)) limit = int(request.args.get('limit', 50)) result = repair_worker.get_findings( job_id=job_id, status=status, severity=severity, page=page, limit=limit ) # Fix Plex/Jellyfin relative thumb URLs in finding details for item in result.get('items', []): details = item.get('details') if details and isinstance(details, dict): for key in ('album_thumb_url', 'artist_thumb_url'): if details.get(key): details[key] = fix_artist_image_url(details[key]) return jsonify(result), 200 except Exception as e: logger.error(f"Error getting repair findings: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings/counts', methods=['GET']) def repair_findings_counts(): """Get findings counts by status""" try: if repair_worker is None: return jsonify({'pending': 0, 'resolved': 0, 'dismissed': 0, 'total': 0}), 200 counts = repair_worker.get_findings_counts() return jsonify(counts), 200 except Exception as e: logger.error(f"Error getting findings counts: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/cache-health', methods=['GET']) def repair_cache_health(): """Get metadata cache health stats for the repair dashboard""" try: from core.metadata_cache import get_metadata_cache cache = get_metadata_cache() return jsonify(cache.get_health_stats()), 200 except Exception as e: logger.error(f"Error getting cache health: {e}") return jsonify({}), 500 @app.route('/api/repair/findings//fix', methods=['POST']) def repair_finding_fix(finding_id): """Execute the actual fix action for a finding""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} fix_action = data.get('fix_action') # e.g. 'staging' or 'delete' for orphan files result = repair_worker.fix_finding(finding_id, fix_action=fix_action) return jsonify(result), 200 if result.get('success') else 400 except Exception as e: logger.error(f"Error fixing finding {finding_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings//resolve', methods=['POST']) def repair_finding_resolve(finding_id): """Resolve a finding with optional action""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} action = data.get('action') success = repair_worker.resolve_finding(finding_id, action) return jsonify({'success': success}), 200 except Exception as e: logger.error(f"Error resolving finding {finding_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings//dismiss', methods=['POST']) def repair_finding_dismiss(finding_id): """Dismiss a finding""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 success = repair_worker.dismiss_finding(finding_id) return jsonify({'success': success}), 200 except Exception as e: logger.error(f"Error dismissing finding {finding_id}: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings/bulk-fix', methods=['POST']) def repair_findings_bulk_fix(): """Bulk fix all pending fixable findings matching filters""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} job_id = data.get('job_id') or None severity = data.get('severity') or None finding_ids = data.get('ids') or None fix_action = data.get('fix_action') or None result = repair_worker.bulk_fix_findings( job_id=job_id, severity=severity, finding_ids=finding_ids, fix_action=fix_action ) return jsonify({ 'success': True, 'fixed': result.get('fixed', 0), 'failed': result.get('failed', 0), 'total': result.get('total', 0), 'errors': result.get('errors', []) }), 200 except Exception as e: logger.error(f"Error bulk fixing findings: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings/bulk', methods=['POST']) def repair_findings_bulk(): """Bulk resolve or dismiss findings""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} finding_ids = data.get('ids', []) action = data.get('action', 'dismiss') if not finding_ids: return jsonify({'error': 'No finding IDs provided'}), 400 count = repair_worker.bulk_update_findings(finding_ids, action) return jsonify({'success': True, 'updated': count}), 200 except Exception as e: logger.error(f"Error bulk updating findings: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/findings/clear', methods=['POST']) def repair_findings_clear(): """Clear (delete) findings, optionally filtered by job_id and/or status""" try: if repair_worker is None: return jsonify({'error': 'Repair worker not initialized'}), 400 data = request.get_json(silent=True) or {} job_id = data.get('job_id') status = data.get('status') count = repair_worker.clear_findings(job_id=job_id, status=status) return jsonify({'success': True, 'deleted': count}), 200 except Exception as e: logger.error(f"Error clearing findings: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/history', methods=['GET']) def repair_history(): """Get job run history""" try: if repair_worker is None: return jsonify({'runs': []}), 200 job_id = request.args.get('job_id') limit = int(request.args.get('limit', 50)) runs = repair_worker.get_history(job_id=job_id, limit=limit) return jsonify({'runs': runs}), 200 except Exception as e: logger.error(f"Error getting repair history: {e}") return jsonify({'error': str(e)}), 500 @app.route('/api/repair/progress', methods=['GET']) def repair_job_progress(): """Get current repair job progress states (for initial page load)""" try: if repair_worker is None: return jsonify({}), 200 lock = getattr(repair_worker, '_progress_lock_ref', None) states = getattr(repair_worker, '_progress_states_ref', None) if lock is None or states is None: return jsonify({}), 200 with lock: result = {} for jid, state in states.items(): cp = dict(state) cp['log'] = list(state['log']) result[jid] = cp return jsonify(result), 200 except Exception as e: logger.error(f"Error getting repair progress: {e}") return jsonify({'error': str(e)}), 500 # ================================================================================================ # END LIBRARY REPAIR WORKER # ================================================================================================ # ================================================================================================ # IMPORT / STAGING SYSTEM # ================================================================================================ AUDIO_EXTENSIONS = {'.mp3', '.flac', '.ogg', '.opus', '.m4a', '.aac', '.wav', '.wma', '.aiff', '.aif', '.ape'} @app.route('/api/import/staging/files', methods=['GET']) def import_staging_files(): """Scan the staging folder and return audio files with tag metadata.""" try: staging_path = get_staging_path() os.makedirs(staging_path, exist_ok=True) files = [] for root, _dirs, filenames in os.walk(staging_path): for fname in filenames: ext = os.path.splitext(fname)[1].lower() if ext not in AUDIO_EXTENSIONS: continue full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) meta = read_staging_file_metadata(full_path, rel_path) files.append({ 'filename': fname, 'rel_path': rel_path, 'full_path': full_path, 'title': meta['title'], 'artist': meta['albumartist'] or meta['artist'] or 'Unknown Artist', 'album': meta['album'], 'track_number': meta['track_number'], 'disc_number': meta['disc_number'], 'extension': ext }) # Sort by filename files.sort(key=lambda f: f['filename'].lower()) return jsonify({'success': True, 'files': files, 'staging_path': staging_path}) except Exception as e: logger.error(f"Error scanning staging files: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/staging/groups', methods=['GET']) def import_staging_groups(): """Auto-detect album groups from staging files based on their tags. Groups files by (album_tag, artist) where both are non-empty and at least 2 files share the same album+artist combo. Returns groups sorted by file count descending. """ try: staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'groups': []}) # Scan files and group by album+artist tags album_groups = {} # (album_lower, artist_lower) -> {album, artist, files: []} for root, _dirs, filenames in os.walk(staging_path): for fname in filenames: ext = os.path.splitext(fname)[1].lower() if ext not in AUDIO_EXTENSIONS: continue full_path = os.path.join(root, fname) rel_path = os.path.relpath(full_path, staging_path) meta = read_staging_file_metadata(full_path, rel_path) album = meta['album'] artist = meta['albumartist'] or meta['artist'] if not album or not artist: continue key = (album.lower().strip(), artist.lower().strip()) if key not in album_groups: album_groups[key] = { 'album': album.strip(), 'artist': artist.strip(), 'files': [] } album_groups[key]['files'].append({ 'filename': fname, 'full_path': full_path, 'title': meta['title'], 'track_number': meta['track_number'], }) # Only return groups with 2+ files groups = [] for group in album_groups.values(): if len(group['files']) >= 2: group['files'].sort(key=lambda f: f.get('track_number') or 999) groups.append({ 'album': group['album'], 'artist': group['artist'], 'file_count': len(group['files']), 'files': group['files'], 'file_paths': [f['full_path'] for f in group['files']], }) # Sort by file count descending groups.sort(key=lambda g: g['file_count'], reverse=True) return jsonify({'success': True, 'groups': groups}) except Exception as e: logger.error(f"Error building staging groups: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/staging/hints', methods=['GET']) def import_staging_hints(): """Extract album search hints from staging folder (tags + folder names). Fast — no Spotify calls.""" try: staging_path = get_staging_path() if not os.path.isdir(staging_path): return jsonify({'success': True, 'hints': []}) # Collect hints from tags and folder structure tag_albums = {} # (album, artist) -> file count folder_hints = {} # subfolder name -> file count for root, _dirs, filenames in os.walk(staging_path): audio_files = [f for f in filenames if os.path.splitext(f)[1].lower() in AUDIO_EXTENSIONS] if not audio_files: continue # Folder-based hint: use immediate subfolder name relative to staging rel_dir = os.path.relpath(root, staging_path) if rel_dir != '.': # Use the top-level subfolder as the hint top_folder = rel_dir.split(os.sep)[0] folder_hints[top_folder] = folder_hints.get(top_folder, 0) + len(audio_files) # Tag-based hints for fname in audio_files: full_path = os.path.join(root, fname) try: from mutagen import File as MutagenFile tags = MutagenFile(full_path, easy=True) if tags: album = (tags.get('album') or [None])[0] artist = (tags.get('artist') or (tags.get('albumartist') or [None]))[0] if album: key = (album.strip(), (artist or '').strip()) tag_albums[key] = tag_albums.get(key, 0) + 1 except Exception: pass # Build search queries, prioritizing tag-based hints (more specific) queries = [] seen_queries_lower = set() # Tag-based: sort by file count descending for (album, artist), _count in sorted(tag_albums.items(), key=lambda x: -x[1]): q = f"{album} {artist}".strip() if artist else album if q.lower() not in seen_queries_lower: seen_queries_lower.add(q.lower()) queries.append(q) # Folder-based: parse "Artist - Album" pattern or use as-is for folder, _count in sorted(folder_hints.items(), key=lambda x: -x[1]): q = folder.replace('_', ' ') if q.lower() not in seen_queries_lower: seen_queries_lower.add(q.lower()) queries.append(q) # Cap at 5 queries to keep it fast queries = queries[:5] return jsonify({'success': True, 'hints': queries}) except Exception as e: logger.error(f"Error getting staging hints: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/search/albums', methods=['GET']) def import_search_albums(): """Search for albums using the active metadata provider.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 12)), 50) from core.metadata_service import get_primary_source if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, 'albums') albums = search_import_albums(query, limit=limit) return jsonify({'success': True, 'albums': albums}) except Exception as e: logger.error(f"Error searching albums for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/album/match', methods=['POST']) def import_album_match(): """Match staging files to an album's tracklist.""" try: data = request.get_json() or {} album_id = data.get('album_id') album_name = data.get('album_name', '') album_artist = data.get('album_artist', '') source = str(data.get('source') or '').strip().lower() # Optional: only match specific files (from auto-group selection) filter_file_paths = set(data.get('file_paths', [])) if not album_id: return jsonify({'success': False, 'error': 'Missing album_id'}), 400 payload = build_album_import_match_payload( album_id, album_name=album_name, album_artist=album_artist, file_paths=filter_file_paths, source=source or None, ) return jsonify(payload) except Exception as e: logger.error(f"Error matching album for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/album/process', methods=['POST']) def import_album_process(): """Process matched album files through the post-processing pipeline.""" try: data = request.get_json() or {} album = data.get('album', {}) matches = data.get('matches', []) if not album or not matches: return jsonify({'success': False, 'error': 'Missing album or matches data'}), 400 processed = 0 errors = [] album_name = album.get('name', album.get('album_name', 'Unknown Album')) artist_name = album.get('artist', album.get('artist_name', 'Unknown Artist')) album_id = album.get('id', album.get('album_id', '')) source = str(album.get('source') or data.get('source') or '').strip().lower() total_discs = max( ( match.get('track', {}).get('disc_number', 1) for match in matches if match.get('track') ), default=1, ) artist_context = resolve_album_artist_context(album, source=source) for match in matches: staging_file = match.get('staging_file') track = match.get('track') or {} if not staging_file or not track: continue file_path = staging_file.get('full_path', '') if not os.path.isfile(file_path): errors.append(f"File not found: {staging_file.get('filename', '?')}") continue track_name = track.get('name', 'Unknown Track') track_number = track.get('track_number', 1) disc_number = track.get('disc_number', 1) context_key = f"import_album_{album_id}_{track_number}_{uuid.uuid4().hex[:8]}" context = build_album_import_context( album, track, artist_context=artist_context, total_discs=total_discs, source=source, ) try: _post_process_matched_download(context_key, context, file_path) processed += 1 logger.info(f"Import processed: {track_number}. {track_name} from {album_name}") except Exception as proc_err: err_msg = f"{track_name}: {str(proc_err)}" errors.append(err_msg) logger.error(f"Import processing error: {err_msg}") add_activity_item("", "Album Imported", f"{album_name} by {artist_name} ({processed}/{len(matches)} tracks)", "Now") # Emit events through automation engine — same chain as download batches # batch_complete → auto-scan → library_scan_completed → auto-update DB if processed > 0: try: if automation_engine: automation_engine.emit('import_completed', { 'track_count': str(processed), 'album_name': album_name or '', 'artist': artist_name or '', }) automation_engine.emit('batch_complete', { 'playlist_name': f"Import: {album_name}" if album_name else 'Import', 'total_tracks': str(len(matches)), 'completed_tracks': str(processed), 'failed_tracks': str(len(errors)), }) except Exception: pass # Rebuild suggestions cache since staging contents changed if processed > 0: refresh_import_suggestions_cache() return jsonify({ 'success': True, 'processed': processed, 'total': len(matches), 'errors': errors }) except Exception as e: logger.error(f"Error processing album import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/search/tracks', methods=['GET']) def import_search_tracks(): """Search tracks using the configured metadata provider priority order.""" try: query = request.args.get('q', '').strip() if not query: return jsonify({'success': False, 'error': 'Missing query parameter'}), 400 limit = min(int(request.args.get('limit', 10)), 30) if get_primary_source() == 'hydrabase' and hydrabase_worker and dev_mode_enabled: hydrabase_worker.enqueue(query, 'tracks') tracks = search_import_tracks(query, limit=limit) return jsonify({'success': True, 'tracks': tracks}) except Exception as e: logger.error(f"Error searching tracks for import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 @app.route('/api/import/singles/process', methods=['POST']) def import_singles_process(): """Process individual staging files as singles through the post-processing pipeline.""" try: data = request.get_json() files = data.get('files', []) if not files: return jsonify({'success': False, 'error': 'No files provided'}), 400 processed = 0 errors = [] for file_info in files: file_path = file_info.get('full_path', '') if not os.path.isfile(file_path): errors.append(f"File not found: {file_info.get('filename', '?')}") continue title = file_info.get('title', '') artist = file_info.get('artist', '') manual_match = file_info.get('manual_match') if manual_match is not None and not isinstance(manual_match, dict): manual_match = None manual_match_source = '' manual_match_id = None if manual_match: manual_match_source = str(manual_match.get('source') or '').strip().lower() manual_match_id = str(manual_match.get('id') or '').strip() if not manual_match_id or not manual_match_source: errors.append(f"Malformed manual match for file: {file_info.get('filename', '?')}") continue # Fallback to filename parsing if no metadata if not title and not manual_match: parsed = parse_filename_metadata(file_info.get('filename', '')) title = parsed.get('title') or os.path.splitext(file_info.get('filename', 'Unknown'))[0] if not artist: artist = parsed.get('artist', '') from core.imports.resolution import get_single_track_import_context resolved = get_single_track_import_context( title, artist, override_id=manual_match_id, override_source=manual_match_source, ) context = normalize_import_context(resolved['context']) artist_data = get_import_context_artist(context) track_data = get_import_track_info(context) final_title = track_data.get('name', title) final_artist = artist_data.get('name', artist) context_key = f"import_single_{uuid.uuid4().hex[:8]}" try: _post_process_matched_download(context_key, context, file_path) processed += 1 logger.info( "Import single processed: %s by %s (source=%s)", final_title, final_artist, resolved.get('source') or 'local', ) except Exception as proc_err: err_msg = f"{title}: {str(proc_err)}" errors.append(err_msg) logger.error(f"Import single processing error: {err_msg}") add_activity_item("", "Singles Imported", f"{processed}/{len(files)} tracks processed", "Now") # Emit events through automation engine — same chain as download batches # batch_complete → auto-scan → library_scan_completed → auto-update DB if processed > 0: try: if automation_engine: automation_engine.emit('import_completed', { 'track_count': str(processed), 'album_name': '', 'artist': 'Various', }) automation_engine.emit('batch_complete', { 'playlist_name': 'Import: Singles', 'total_tracks': str(len(files)), 'completed_tracks': str(processed), 'failed_tracks': str(len(errors)), }) except Exception: pass # Rebuild suggestions cache since staging contents changed if processed > 0: refresh_import_suggestions_cache() return jsonify({ 'success': True, 'processed': processed, 'total': len(files), 'errors': errors }) except Exception as e: logger.error(f"Error processing singles import: {e}") return jsonify({'success': False, 'error': str(e)}), 500 # ── Auto-Import Worker ── auto_import_worker = None try: from core.auto_import_worker import AutoImportWorker _ai_db = get_database() _ai_staging = docker_resolve_path(config_manager.get('import.staging_path', './Staging')) _ai_transfer = docker_resolve_path(config_manager.get('soulseek.transfer_path', './Transfer')) auto_import_worker = AutoImportWorker( database=_ai_db, staging_path=_ai_staging, transfer_path=_ai_transfer, process_callback=_post_process_matched_download, config_manager=config_manager, automation_engine=automation_engine, ) if config_manager.get('auto_import.enabled', False): auto_import_worker.start() logger.info("Auto-import worker started") else: logger.info("Auto-import worker initialized (disabled)") except Exception as _ai_err: logger.error(f"Auto-import worker init failed: {_ai_err}") @app.route('/api/auto-import/status', methods=['GET']) def auto_import_status(): if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 return jsonify({"success": True, **auto_import_worker.get_status()}) @app.route('/api/auto-import/toggle', methods=['POST']) def auto_import_toggle(): if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 data = request.get_json() or {} enabled = data.get('enabled', not auto_import_worker.running) if enabled: config_manager.set('auto_import.enabled', True) if not auto_import_worker.running: auto_import_worker.start() else: config_manager.set('auto_import.enabled', False) auto_import_worker.stop() return jsonify({"success": True, "enabled": enabled}) @app.route('/api/auto-import/settings', methods=['GET', 'POST']) def auto_import_settings(): if request.method == 'GET': return jsonify({ "success": True, "enabled": config_manager.get('auto_import.enabled', False), "scan_interval": config_manager.get('auto_import.scan_interval', 60), "confidence_threshold": config_manager.get('auto_import.confidence_threshold', 0.9), "auto_process": config_manager.get('auto_import.auto_process', True), }) data = request.get_json() or {} for key in ['enabled', 'scan_interval', 'confidence_threshold', 'auto_process']: if key in data: config_manager.set(f'auto_import.{key}', data[key]) return jsonify({"success": True}) @app.route('/api/auto-import/results', methods=['GET']) def auto_import_results(): if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 status_filter = request.args.get('status') limit = request.args.get('limit', 50, type=int) results = auto_import_worker.get_results(status_filter=status_filter, limit=limit) return jsonify({"success": True, "results": results}) @app.route('/api/auto-import/approve/', methods=['POST']) def auto_import_approve(item_id): if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 return jsonify(auto_import_worker.approve_item(item_id)) @app.route('/api/auto-import/reject/', methods=['POST']) def auto_import_reject(item_id): if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 return jsonify(auto_import_worker.reject_item(item_id)) @app.route('/api/auto-import/scan-now', methods=['POST']) def auto_import_scan_now(): """Trigger an immediate scan cycle.""" if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 if not auto_import_worker.running: return jsonify({"success": False, "error": "Auto-import is not running"}), 400 # Run scan in background thread import threading threading.Thread(target=auto_import_worker._scan_cycle, daemon=True).start() return jsonify({"success": True}) @app.route('/api/auto-import/approve-all', methods=['POST']) def auto_import_approve_all(): """Approve all pending review items.""" if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 try: results = auto_import_worker.get_results(status_filter='pending_review', limit=200) count = 0 for r in results: result = auto_import_worker.approve_item(r['id']) if result.get('success'): count += 1 return jsonify({"success": True, "count": count}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/auto-import/clear-completed', methods=['POST']) def auto_import_clear_completed(): """Remove completed/imported items from history.""" if not auto_import_worker: return jsonify({"success": False, "error": "Auto-import not available"}), 500 try: db = get_database() with db._get_connection() as conn: cursor = conn.cursor() cursor.execute("DELETE FROM auto_import_history WHERE status IN ('completed', 'approved', 'failed', 'needs_identification', 'rejected')") count = cursor.rowcount conn.commit() return jsonify({"success": True, "count": count}) except Exception as e: return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" cache = get_import_suggestions_cache() return jsonify({ 'success': True, 'suggestions': cache['suggestions'], 'ready': cache['built'], }) # ================================================================================================ # END IMPORT / STAGING SYSTEM # ================================================================================================ # ================================================================================================ # WEBSOCKET (SOCKET.IO) EVENT HANDLERS AND BACKGROUND EMITTERS # ================================================================================================ def _build_status_payload(): """Build the same status payload used by GET /status, reading from the cache.""" download_mode = config_manager.get('download_source.mode', 'hybrid') soulseek_data = dict(_status_cache.get('soulseek', {})) soulseek_data['source'] = download_mode # Always include fresh rate limit info (it changes over time as ban expires) # Call is_rate_limited() first to ensure ban-end timestamp is recorded for cooldown spotify_data = dict(_status_cache.get('spotify', {})) is_rl = spotify_client.is_rate_limited() if spotify_client else False rate_limit_info = spotify_client.get_rate_limit_info() if is_rl else None cooldown_remaining = spotify_client.get_post_ban_cooldown_remaining() if spotify_client else 0 spotify_data['rate_limited'] = is_rl spotify_data['rate_limit'] = rate_limit_info if cooldown_remaining > 0: spotify_data['post_ban_cooldown'] = cooldown_remaining # Count active downloads for nav badge active_dl_count = 0 try: with tasks_lock: for t in download_tasks.values(): if t.get('status') in ('downloading', 'searching', 'post_processing', 'queued', 'pending'): active_dl_count += 1 except Exception: pass return { 'spotify': spotify_data, 'media_server': _status_cache.get('media_server', {}), 'soulseek': soulseek_data, 'active_media_server': config_manager.get_active_media_server(), 'enrichment': _get_enrichment_status(), 'active_downloads': active_dl_count, } def _build_watchlist_count_payload(profile_id=1): """Build the same payload used by GET /api/watchlist/count.""" try: database = get_database() count = database.get_watchlist_count(profile_id=profile_id) except Exception: count = 0 next_run_in_seconds = automation_engine.get_system_automation_next_run_seconds('scan_watchlist') if automation_engine else 0 return { 'success': True, 'count': count, 'next_run_in_seconds': next_run_in_seconds } def _hydrabase_reconnect_loop(): """Background thread that monitors Hydrabase connection and auto-reconnects if needed.""" global _hydrabase_ws _consecutive_failures = 0 while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(30) try: # Only attempt reconnect if auto_connect is enabled hydra_cfg = config_manager.get_hydrabase_config() if not hydra_cfg.get('auto_connect') or not hydra_cfg.get('url') or not hydra_cfg.get('api_key'): _consecutive_failures = 0 continue # Check if already connected try: if _hydrabase_ws is not None and _hydrabase_ws.connected: _consecutive_failures = 0 continue except Exception: pass # Socket in bad state — treat as disconnected # Disconnected with auto_connect enabled — try to reconnect # Back off: 30s, 60s, 120s, max 300s between attempts backoff = min(30 * (2 ** _consecutive_failures), 300) if _consecutive_failures > 0: socketio.sleep(backoff - 30) # Already slept 30s at top of loop import websocket try: with _hydrabase_lock: if _hydrabase_ws: try: _hydrabase_ws.close() except: pass ws = websocket.create_connection( hydra_cfg['url'], header={"x-api-key": hydra_cfg['api_key']}, timeout=10 ) _hydrabase_ws = ws _consecutive_failures = 0 logger.info(f"[Hydrabase] Auto-reconnected to {hydra_cfg['url']}") except Exception as e: _consecutive_failures += 1 if _consecutive_failures <= 3: logger.error(f"[Hydrabase] Reconnect attempt failed ({_consecutive_failures}): {e}") elif _consecutive_failures == 4: logger.error("[Hydrabase] Reconnect failing repeatedly — suppressing further logs until success") except Exception: pass # Don't crash the monitor loop def _emit_service_status_loop(): """Background thread that pushes service status every 5 seconds.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(5) try: socketio.emit('status:update', _build_status_payload()) except Exception as e: logger.debug(f"Error emitting service status: {e}") def _emit_watchlist_count_loop(): """Background thread that pushes watchlist count every 10 seconds to each profile room.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(10) try: database = get_database() profiles = database.get_all_profiles() for profile in profiles: pid = profile['id'] socketio.emit('watchlist:count', _build_watchlist_count_payload(profile_id=pid), room=f'profile:{pid}') except Exception as e: logger.debug(f"Error emitting watchlist count: {e}") def _emit_download_status_loop(): """Background thread that pushes download batch status every 2 seconds to subscribed rooms.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(2) try: live_transfers_lookup = get_cached_transfer_data() with tasks_lock: for batch_id, batch in download_batches.items(): try: status_data = _build_batch_status_data( batch_id, batch, live_transfers_lookup ) socketio.emit('downloads:batch_update', { 'batch_id': batch_id, 'data': status_data }, room=f'batch:{batch_id}') except Exception as e: logger.debug(f"Error building batch status for {batch_id}: {e}") except Exception as e: logger.debug(f"Error in download status emit loop: {e}") # --- Socket.IO event handlers --- @socketio.on('connect') def handle_connect(): logger.info("WebSocket client connected") @socketio.on('disconnect') def handle_disconnect(): logger.info("WebSocket client disconnected") @socketio.on('downloads:subscribe') def handle_download_subscribe(data): """Client subscribes to download batch updates by joining rooms.""" batch_ids = data.get('batch_ids', []) for bid in batch_ids: join_room(f'batch:{bid}') logger.debug(f"Client subscribed to batches: {batch_ids}") @socketio.on('downloads:unsubscribe') def handle_download_unsubscribe(data): """Client unsubscribes from download batch updates by leaving rooms.""" batch_ids = data.get('batch_ids', []) for bid in batch_ids: leave_room(f'batch:{bid}') logger.debug(f"Client unsubscribed from batches: {batch_ids}") @socketio.on('profile:join') def handle_profile_join(data): """Client joins a profile room for scoped WebSocket emits (watchlist/wishlist counts).""" profile_id = data.get('profile_id') if profile_id: # Leave any previous profile rooms old_id = data.get('old_profile_id') if old_id: leave_room(f'profile:{old_id}') join_room(f'profile:{profile_id}') logger.debug(f"Client joined profile room: profile:{profile_id}") # --- Phase 2: Dashboard emitters --- def _emit_system_stats_loop(): """Background thread that pushes system stats every 10 seconds.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(10) try: socketio.emit('dashboard:stats', _build_system_stats()) except Exception as e: logger.debug(f"Error emitting system stats: {e}") def _emit_activity_feed_loop(): """Background thread that pushes activity feed every 2 seconds.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(2) try: with activity_feed_lock: activities = activity_feed[-10:][::-1] socketio.emit('dashboard:activity', {'activities': activities}) except Exception as e: logger.debug(f"Error emitting activity feed: {e}") def _emit_db_stats_loop(): """Background thread that pushes database stats every 10 seconds.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(10) try: db = get_database() stats = db.get_database_info_for_server() socketio.emit('dashboard:db_stats', stats) except Exception as e: logger.debug(f"Error emitting db stats: {e}") def _emit_wishlist_count_loop(): """Background thread that pushes wishlist count every 10 seconds to each profile room.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(10) try: from core.wishlist_service import get_wishlist_service ws = get_wishlist_service() database = get_database() profiles = database.get_all_profiles() for profile in profiles: pid = profile['id'] count = ws.get_wishlist_count(profile_id=pid) socketio.emit('dashboard:wishlist_count', {'count': count}, room=f'profile:{pid}') except Exception as e: logger.debug(f"Error emitting wishlist count: {e}") # Note: Toasts are NOT on a timer — they emit instantly from add_activity_item() # --- Phase 3: Enrichment sidebar worker emitters --- def _has_active_downloads(): """Check if any download batches are currently active.""" try: with tasks_lock: for batch_data in download_batches.values(): if batch_data.get('phase') not in ('complete', 'error', 'cancelled', None): return True except Exception: pass return False # Track whether we auto-paused workers so we only resume ones we paused (not user-paused ones) _download_auto_paused = set() _download_yield_override = set() # Workers the user explicitly resumed during downloads — don't re-pause def _emit_rate_monitor_loop(): """Background thread that pushes API call rate data every 1 second for speedometer gauges. Also includes enrichment worker status so the combined cards have everything.""" # Map rate monitor service keys to enrichment status keys _enrichment_key_map = { 'spotify': 'spotify_enrichment', 'itunes': 'itunes_enrichment', 'deezer': 'deezer_enrichment', 'lastfm': 'lastfm', 'genius': 'genius', 'musicbrainz': 'musicbrainz', 'audiodb': 'audiodb', 'discogs': 'discogs', 'tidal': 'tidal_enrichment', 'qobuz': 'qobuz_enrichment', } while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(1) try: from core.api_call_tracker import api_call_tracker payload = api_call_tracker.get_all_rates() # Merge enrichment worker status into each service try: enrichment = _get_enrichment_status() for svc_key, entry in payload.items(): enr_key = _enrichment_key_map.get(svc_key) enr = enrichment.get(enr_key) if enr_key else None if enr: entry['worker'] = { 'status': 'not_configured' if not enr.get('configured') else 'paused' if enr.get('paused') else 'idle' if enr.get('idle') else 'running' if enr.get('running') else 'stopped', 'yield_reason': enr.get('yield_reason', ''), 'calls_1h': enr.get('calls_1h', 0), 'calls_24h': enr.get('calls_24h', 0), } if svc_key == 'spotify' and enr.get('daily_budget'): entry['worker']['daily_budget'] = enr['daily_budget'] except Exception: pass # Add Spotify rate limit state try: if spotify_client: rl_info = spotify_client.get_rate_limit_info() if rl_info: payload['spotify']['rate_limited'] = True payload['spotify']['rl_remaining'] = rl_info.get('remaining_seconds', 0) payload['spotify']['rl_endpoint'] = rl_info.get('endpoint', '') except Exception: pass socketio.emit('rate-monitor:update', payload) except Exception as e: logger.debug(f"Error emitting rate monitor: {e}") def _emit_enrichment_status_loop(): """Background thread that pushes all enrichment worker statuses every 2 seconds. Also auto-pauses rate-limited enrichment workers during active downloads.""" workers = { 'musicbrainz': lambda: mb_worker, 'audiodb': lambda: audiodb_worker, 'discogs': lambda: discogs_worker, 'deezer': lambda: deezer_worker, 'spotify-enrichment': lambda: spotify_enrichment_worker, 'itunes-enrichment': lambda: itunes_enrichment_worker, 'lastfm-enrichment': lambda: lastfm_worker, 'genius-enrichment': lambda: genius_worker, 'tidal-enrichment': lambda: tidal_enrichment_worker, 'qobuz-enrichment': lambda: qobuz_enrichment_worker, 'hydrabase': lambda: hydrabase_worker, 'soulid': lambda: soulid_worker, 'listening-stats': lambda: listening_stats_worker, 'repair': lambda: repair_worker, } # Workers to auto-pause during downloads (rate-limit sensitive services) yield_workers = { 'spotify-enrichment': lambda: spotify_enrichment_worker, 'lastfm-enrichment': lambda: lastfm_worker, 'genius-enrichment': lambda: genius_worker, } while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(2) # Auto-pause/resume rate-limited workers during downloads try: downloading = _has_active_downloads() if not downloading: _download_yield_override.clear() # Reset overrides when downloads finish for name, get_w in yield_workers.items(): w = get_w() if w is None: continue if downloading and not w.paused and name not in _download_yield_override: w.paused = True _download_auto_paused.add(name) logger.debug(f"Auto-paused {name} during active downloads") elif not downloading and name in _download_auto_paused: # Don't override an explicit user pause. If config says the worker # was paused via the UI, leave it paused and just drop the auto-pause # marker so the next auto-pause/resume cycle behaves normally. config_key = f"{name.replace('-', '_')}_paused" user_paused = config_manager.get(config_key, False) _download_auto_paused.discard(name) if not user_paused: w.paused = False logger.debug(f"Auto-resumed {name} after downloads finished") else: logger.debug(f"Downloads finished but {name} remains paused by user") except Exception as e: logger.debug(f"Error in download-yield check: {e}") for name, get_worker in workers.items(): try: worker = get_worker() if worker is None: continue status = worker.get_stats() # Flag workers that were auto-paused for downloads if name in _download_auto_paused: status['yield_reason'] = 'downloads' socketio.emit(f'enrichment:{name}', status) except Exception as e: logger.debug(f"Error emitting {name} status: {e}") def _emit_tool_progress_loop(): """Background thread that pushes all tool progress statuses every 1 second.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(1) # Stream status try: with stream_lock: socketio.emit('tool:stream', { "status": stream_state["status"], "progress": stream_state["progress"], "track_info": stream_state["track_info"], "error_message": stream_state["error_message"] }) except Exception as e: logger.debug(f"Error emitting stream status: {e}") # Quality Scanner try: with quality_scanner_lock: socketio.emit('tool:quality-scanner', dict(quality_scanner_state)) except Exception as e: logger.debug(f"Error emitting quality scanner status: {e}") # Duplicate Cleaner (add computed space_freed_mb) try: with duplicate_cleaner_lock: state_copy = duplicate_cleaner_state.copy() state_copy["space_freed_mb"] = duplicate_cleaner_state["space_freed"] / (1024 * 1024) socketio.emit('tool:duplicate-cleaner', state_copy) except Exception as e: logger.debug(f"Error emitting duplicate cleaner status: {e}") # Retag try: with retag_lock: socketio.emit('tool:retag', dict(retag_state)) except Exception as e: logger.debug(f"Error emitting retag status: {e}") # DB Update try: with db_update_lock: socketio.emit('tool:db-update', dict(db_update_state)) except Exception as e: logger.debug(f"Error emitting db update status: {e}") # Metadata Update (match HTTP wrapper: {success, status}) try: state_copy = metadata_update_state.copy() if state_copy.get('started_at'): state_copy['started_at'] = state_copy['started_at'].isoformat() if state_copy.get('completed_at'): state_copy['completed_at'] = state_copy['completed_at'].isoformat() socketio.emit('tool:metadata', {"success": True, "status": state_copy}) except Exception as e: logger.debug(f"Error emitting metadata status: {e}") # Logs (format activity_feed same as HTTP endpoint) try: with activity_feed_lock: recent = activity_feed[-50:][::-1] formatted = [] for a in recent: ts = a.get('time', 'Unknown') icon = a.get('icon', '•') title = a.get('title', 'Activity') sub = a.get('subtitle', '') formatted.append(f"[{ts}] {icon} {title} - {sub}" if sub else f"[{ts}] {icon} {title}") if not formatted: formatted = ["No recent activity.", "Sync and download operations..."] socketio.emit('tool:logs', {'logs': formatted}) except Exception as e: logger.debug(f"Error emitting logs: {e}") def _emit_live_log_loop(): """Background thread that tails app.log and pushes new lines via WebSocket.""" _last_pos = {} # {source: file_position} _active_source = 'app' log_map = { 'app': Path(_log_path), 'acoustid': _log_dir / 'acoustid.log', 'post_processing': _log_dir / 'post_processing.log', 'source_reuse': _log_dir / 'source_reuse.log', } while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(0.5) try: # Read which source clients want (stored by subscribe handler) source = getattr(_emit_live_log_loop, '_source', 'app') log_path = log_map.get(source, log_map['app']) if not os.path.exists(log_path): continue file_size = os.path.getsize(log_path) last_pos = _last_pos.get(source, 0) # File was truncated or rotated if file_size < last_pos: last_pos = 0 if file_size == last_pos: continue # No new data new_lines = [] with open(log_path, 'r', encoding='utf-8', errors='replace') as f: f.seek(last_pos) for line in f: stripped = line.rstrip() if stripped: new_lines.append(stripped) _last_pos[source] = f.tell() if new_lines: # Cap at 50 lines per push to avoid flooding socketio.emit('logs:live', { 'lines': new_lines[-50:], 'source': source, }) except Exception as e: logger.debug(f"Error in live log emitter: {e}") _emit_live_log_loop._source = 'app' @socketio.on('logs:subscribe') def handle_logs_subscribe(data): """Client subscribes to live log stream with optional source.""" source = data.get('source', 'app') _emit_live_log_loop._source = source join_room('logs:live') @socketio.on('logs:unsubscribe') def handle_logs_unsubscribe(data): leave_room('logs:live') @socketio.on('sync:subscribe') def handle_sync_subscribe(data): for pid in data.get('playlist_ids', []): join_room(f'sync:{pid}') @socketio.on('sync:unsubscribe') def handle_sync_unsubscribe(data): for pid in data.get('playlist_ids', []): leave_room(f'sync:{pid}') @socketio.on('discovery:subscribe') def handle_discovery_subscribe(data): for pid in data.get('ids', []): join_room(f'discovery:{pid}') @socketio.on('discovery:unsubscribe') def handle_discovery_unsubscribe(data): for pid in data.get('ids', []): leave_room(f'discovery:{pid}') def _emit_sync_progress_loop(): """Push sync progress to subscribed rooms every 1 second.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(1) try: with sync_lock: for pid, state in list(sync_states.items()): try: socketio.emit('sync:progress', { 'playlist_id': pid, **state }, room=f'sync:{pid}') except Exception: pass except Exception as e: logger.debug(f"Error in sync progress loop: {e}") def _emit_discovery_progress_loop(): """Push discovery progress to subscribed rooms every 1 second.""" platform_states = { 'tidal': lambda: tidal_discovery_states, 'deezer': lambda: deezer_discovery_states, 'youtube': lambda: youtube_playlist_states, 'beatport': lambda: beatport_chart_states, 'listenbrainz': lambda: listenbrainz_playlist_states, 'spotify_public': lambda: spotify_public_discovery_states, } while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(1) for platform_name, get_states in platform_states.items(): try: states_dict = get_states() for pid, state in list(states_dict.items()): try: phase = state.get('phase', '') if phase in ('', 'idle'): continue payload = { 'platform': platform_name, 'id': pid, 'phase': state.get('phase'), 'status': state.get('status', 'unknown'), 'progress': state.get('discovery_progress', 0), 'discovery_progress': state.get('discovery_progress', {}), 'spotify_matches': state.get('spotify_matches', 0), 'spotify_total': state.get('spotify_total', 0), 'results': state.get('discovery_results', state.get('results', [])), 'complete': state.get('phase') == 'discovered', } socketio.emit('discovery:progress', payload, room=f'discovery:{pid}') except Exception: pass except Exception as e: logger.debug(f"Error in {platform_name} discovery loop: {e}") def _emit_scan_status_loop(): """Push watchlist and media scan status every 2 seconds.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(2) # Watchlist scan try: state = watchlist_scan_state.copy() if state.get('started_at'): state['started_at'] = state['started_at'].isoformat() if state.get('completed_at'): state['completed_at'] = state['completed_at'].isoformat() state.pop('results', None) socketio.emit('scan:watchlist', {"success": True, **state}) except Exception as e: logger.debug(f"Error emitting watchlist scan: {e}") # Media scan try: if web_scan_manager: scan_status = web_scan_manager.get_scan_status() socketio.emit('scan:media', {"success": True, "status": scan_status}) except Exception as e: logger.debug(f"Error emitting media scan: {e}") # Wishlist stats (auto-processing detection + countdown refresh) try: next_run = automation_engine.get_system_automation_next_run_seconds('process_wishlist') if automation_engine else 0 socketio.emit('wishlist:stats', { "is_auto_processing": is_wishlist_actually_processing(), "next_run_in_seconds": next_run, }) except Exception as e: logger.debug(f"Error emitting wishlist stats: {e}") def _emit_automation_progress_loop(): """Push automation:progress events every 1 second for running automations.""" _auto_progress.emit_progress_loop( socketio, is_shutting_down=lambda: globals().get('IS_SHUTTING_DOWN', False), ) def _emit_repair_progress_loop(): """Push repair:progress events every 1 second for running repair jobs.""" while not globals().get('IS_SHUTTING_DOWN', False): socketio.sleep(1) try: if repair_worker is None: continue # Access the progress states set up during repair worker init lock = getattr(repair_worker, '_progress_lock_ref', None) states = getattr(repair_worker, '_progress_states_ref', None) if lock is None or states is None: continue with lock: active = {} stale = [] now = datetime.now() for jid, state in states.items(): if state['status'] == 'running': cp = dict(state) cp['log'] = list(state['log']) active[jid] = cp elif state['status'] in ('finished', 'error') and state.get('finished_at'): try: finished_time = datetime.fromisoformat(state['finished_at']) if (now - finished_time).total_seconds() > 60: stale.append(jid) except (ValueError, TypeError): stale.append(jid) # Still include recently finished states so frontend sees final state if jid not in stale: cp = dict(state) cp['log'] = list(state['log']) active[jid] = cp for jid in stale: del states[jid] if active: socketio.emit('repair:progress', active) except Exception as e: logger.debug(f"Error emitting repair progress: {e}") # ================================================================================================ # END WEBSOCKET HANDLERS # ================================================================================================ _runtime_start_lock = threading.Lock() _runtime_started = False def start_runtime_services(): """Start one-time server background services for direct and WSGI launches.""" global _runtime_started with _runtime_start_lock: if _runtime_started: return logger.info("Starting SoulSync runtime services...") # Dump SOULSYNC_* env vars for diagnostics (helps debug Docker/Unraid env issues) _soulsync_env = {k: v for k, v in os.environ.items() if k.startswith('SOULSYNC_')} if _soulsync_env: logger.info(f"[Startup] SOULSYNC environment variables: {_soulsync_env}") else: logger.warning("[Startup] No SOULSYNC_* environment variables detected") # Start OAuth callback servers logger.info("Starting OAuth callback servers...") start_oauth_callback_servers() # Startup diagnostics: Check and recover stuck flags logger.info("Running startup diagnostics...") stuck_flags_recovered = check_and_recover_stuck_flags() if stuck_flags_recovered: logger.warning("Recovered stuck flags from previous session") else: logger.warning("No stuck flags detected - system healthy") # Start simple background monitor when server starts logger.info("Starting simple background monitor...") start_simple_background_monitor() logger.info("Simple background monitor started (includes automatic search cleanup)") # Wishlist/watchlist timers are now managed by AutomationEngine system automations # Pre-build import suggestions cache in background logger.info("Pre-building import suggestions cache...") start_import_suggestions_cache() # Initialize app start time for uptime tracking app.start_time = time.time() # Register action handlers and start automation engine _register_automation_handlers() if automation_engine: try: logger.info("Starting automation engine...") automation_engine.start() logger.info("Automation engine started") try: automation_engine.emit('app_started', {}) except Exception: pass except AttributeError as e: logger.error(f"Automation engine failed to start: {e}") logger.info(" If using Docker, check that your volume mount is /app/data (not /app/database)") logger.error(f"Automation engine start error (possible stale Docker volume): {e}") except Exception as e: logger.error(f"Automation engine failed to start: {e}") logger.error(f"Automation engine start error: {e}") # Add startup activity add_activity_item("", "System Started", "SoulSync Web UI Server initialized", "Now") # Start WebSocket background emitters logger.info("Starting WebSocket background emitters...") # Phase 1: Global pollers socketio.start_background_task(_emit_service_status_loop) socketio.start_background_task(_emit_watchlist_count_loop) socketio.start_background_task(_emit_download_status_loop) # Phase 2: Dashboard pollers socketio.start_background_task(_emit_system_stats_loop) socketio.start_background_task(_emit_activity_feed_loop) socketio.start_background_task(_emit_db_stats_loop) socketio.start_background_task(_emit_wishlist_count_loop) # Phase 3: Enrichment sidebar workers socketio.start_background_task(_emit_enrichment_status_loop) # Phase 4: Tool progress pollers socketio.start_background_task(_emit_tool_progress_loop) # Phase 5: Sync/discovery progress + scans socketio.start_background_task(_emit_sync_progress_loop) socketio.start_background_task(_emit_discovery_progress_loop) socketio.start_background_task(_emit_scan_status_loop) # Phase 6: Automation progress socketio.start_background_task(_emit_automation_progress_loop) # Phase 7: Repair job progress socketio.start_background_task(_emit_repair_progress_loop) # Hydrabase auto-reconnect monitor socketio.start_background_task(_hydrabase_reconnect_loop) # API Rate Monitor — 1s push for speedometer gauges socketio.start_background_task(_emit_rate_monitor_loop) # Live log tail — streams new log lines to the log viewer socketio.start_background_task(_emit_live_log_loop) logger.info("WebSocket emitters started (Phase 1-7: global/dashboard/enrichment/tools/sync/automations/repair + rate monitor + live logs)") _runtime_started = True # Direct execution: python web_server.py (dev/Windows fallback) # Production should use: gunicorn -c gunicorn.conf.py wsgi:application if _DIRECT_RUN: logger.info("Starting SoulSync Web UI Server...") logger.info("Open your browser and navigate to http://127.0.0.1:8008") logger.info("Tip: For production, use gunicorn -c gunicorn.conf.py wsgi:application") start_runtime_services() socketio.run(app, host='0.0.0.0', port=8008, debug=False, allow_unsafe_werkzeug=True)