diff --git a/core/watchlist_scanner.py b/core/watchlist_scanner.py index 1c0c4827..6bdff96b 100644 --- a/core/watchlist_scanner.py +++ b/core/watchlist_scanner.py @@ -762,7 +762,7 @@ class WatchlistScanner: continue # Small delay between albums - time.sleep(0.3) + time.sleep(DELAY_BETWEEN_ALBUMS) except Exception as album_error: logger.warning(f"Error processing album: {album_error}") @@ -770,7 +770,7 @@ class WatchlistScanner: # Delay between artists if artist_idx < len(similar_artists): - time.sleep(1.0) + time.sleep(DELAY_BETWEEN_ARTISTS) except Exception as artist_error: logger.warning(f"Error processing artist {similar_artist.similar_artist_name}: {artist_error}") @@ -781,11 +781,127 @@ class WatchlistScanner: # Rotate discovery pool if needed (maintain 1000-2000 track limit) self.database.rotate_discovery_pool(max_tracks=2000, remove_count=500) + # Cache recent albums for discovery page + logger.info("Caching recent albums for discovery page...") + self.cache_discovery_recent_albums() + except Exception as e: logger.error(f"Error populating discovery pool: {e}") import traceback traceback.print_exc() + def cache_discovery_recent_albums(self): + """Cache recent albums from watchlist and similar artists for discover page""" + try: + from datetime import datetime, timedelta + import random + + logger.info("Caching recent albums for discover page...") + + # Clear existing cache + self.database.clear_discovery_recent_albums() + + cutoff_date = datetime.now() - timedelta(days=90) # 3 months + cached_count = 0 + albums_checked = 0 + + # Get watchlist artists (10 random for more variety) + watchlist_artists = self.database.get_watchlist_artists() + watchlist_sample = random.sample(watchlist_artists, min(10, len(watchlist_artists))) if watchlist_artists else [] + + # Get similar artists (10 random from top 30 for more variety) + similar_artists = self.database.get_top_similar_artists(limit=30) + similar_sample = random.sample(similar_artists, min(10, len(similar_artists))) if similar_artists else [] + + logger.info(f"Checking albums from {len(watchlist_sample)} watchlist + {len(similar_sample)} similar artists for recent releases") + + # Process watchlist artists + for artist in watchlist_sample: + try: + albums = self.spotify_client.get_artist_albums( + artist.spotify_artist_id, + album_type='album,single', + limit=20 + ) + + for album in albums: + try: + albums_checked += 1 + if hasattr(album, 'release_date') and album.release_date: + release_str = album.release_date + if len(release_str) >= 10: + release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") + if release_date >= cutoff_date: + album_data = { + 'album_spotify_id': album.id, + 'album_name': album.name, + 'artist_name': artist.artist_name, + 'artist_spotify_id': artist.spotify_artist_id, + 'album_cover_url': album.image_url if hasattr(album, 'image_url') else None, + 'release_date': release_str, + 'album_type': album.album_type if hasattr(album, 'album_type') else 'album' + } + if self.database.cache_discovery_recent_album(album_data): + cached_count += 1 + logger.debug(f"Cached recent album: {album.name} by {artist.artist_name} ({release_str})") + except Exception as e: + logger.warning(f"Error checking album for recent releases: {e}") + continue + + except Exception as e: + logger.debug(f"Error fetching albums for watchlist artist {artist.artist_name}: {e}") + continue + + # Rate limiting between artists + time.sleep(DELAY_BETWEEN_ARTISTS) + + # Process similar artists + for artist in similar_sample: + try: + albums = self.spotify_client.get_artist_albums( + artist.similar_artist_spotify_id, + album_type='album,single', + limit=20 + ) + + for album in albums: + try: + albums_checked += 1 + if hasattr(album, 'release_date') and album.release_date: + release_str = album.release_date + if len(release_str) >= 10: + release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") + if release_date >= cutoff_date: + album_data = { + 'album_spotify_id': album.id, + 'album_name': album.name, + 'artist_name': artist.similar_artist_name, + 'artist_spotify_id': artist.similar_artist_spotify_id, + 'album_cover_url': album.image_url if hasattr(album, 'image_url') else None, + 'release_date': release_str, + 'album_type': album.album_type if hasattr(album, 'album_type') else 'album' + } + if self.database.cache_discovery_recent_album(album_data): + cached_count += 1 + logger.debug(f"Cached recent album: {album.name} by {artist.similar_artist_name} ({release_str})") + except Exception as e: + logger.warning(f"Error checking album for recent releases: {e}") + continue + + except Exception as e: + logger.debug(f"Error fetching albums for similar artist {artist.similar_artist_name}: {e}") + continue + + # Rate limiting between artists + time.sleep(DELAY_BETWEEN_ARTISTS) + + logger.info(f"Cached {cached_count} recent albums from {albums_checked} albums checked (cutoff: {cutoff_date.strftime('%Y-%m-%d')})") + + except Exception as e: + logger.error(f"Error caching discovery recent albums: {e}") + import traceback + traceback.print_exc() + # Singleton instance _watchlist_scanner_instance = None diff --git a/database/music_database.py b/database/music_database.py index 5fe9cd0f..38d26ade 100644 --- a/database/music_database.py +++ b/database/music_database.py @@ -473,6 +473,21 @@ class MusicDatabase: ) """) + # Discovery Recent Albums cache - for discover page recent releases section + cursor.execute(""" + CREATE TABLE IF NOT EXISTS discovery_recent_albums ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + album_spotify_id TEXT NOT NULL UNIQUE, + album_name TEXT NOT NULL, + artist_name TEXT NOT NULL, + artist_spotify_id TEXT NOT NULL, + album_cover_url TEXT, + release_date TEXT NOT NULL, + album_type TEXT DEFAULT 'album', + cached_date TIMESTAMP DEFAULT CURRENT_TIMESTAMP + ) + """) + # Create indexes for performance cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_source ON similar_artists (source_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_similar_artists_spotify ON similar_artists (similar_artist_spotify_id)") @@ -483,6 +498,8 @@ class MusicDatabase: cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_pool_is_new ON discovery_pool (is_new_release)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_watchlist ON recent_releases (watchlist_artist_id)") cursor.execute("CREATE INDEX IF NOT EXISTS idx_recent_releases_date ON recent_releases (release_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_date ON discovery_recent_albums (release_date)") + cursor.execute("CREATE INDEX IF NOT EXISTS idx_discovery_recent_albums_artist ON discovery_recent_albums (artist_spotify_id)") logger.info("Discovery tables created successfully") @@ -2618,6 +2635,72 @@ class MusicDatabase: logger.error(f"Error getting discovery pool tracks: {e}") return [] + def cache_discovery_recent_album(self, album_data: Dict[str, Any]) -> bool: + """Cache a recent album for the discover page (from watchlist or similar artists)""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + INSERT OR REPLACE INTO discovery_recent_albums + (album_spotify_id, album_name, artist_name, artist_spotify_id, album_cover_url, release_date, album_type, cached_date) + VALUES (?, ?, ?, ?, ?, ?, ?, CURRENT_TIMESTAMP) + """, ( + album_data['album_spotify_id'], + album_data['album_name'], + album_data['artist_name'], + album_data['artist_spotify_id'], + album_data.get('album_cover_url'), + album_data['release_date'], + album_data.get('album_type', 'album') + )) + + conn.commit() + return True + + except Exception as e: + logger.error(f"Error caching discovery recent album: {e}") + return False + + def get_discovery_recent_albums(self, limit: int = 10) -> List[Dict[str, Any]]: + """Get cached recent albums for discover page""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + + cursor.execute(""" + SELECT * FROM discovery_recent_albums + ORDER BY release_date DESC + LIMIT ? + """, (limit,)) + + rows = cursor.fetchall() + return [{ + 'album_spotify_id': row['album_spotify_id'], + 'album_name': row['album_name'], + 'artist_name': row['artist_name'], + 'artist_spotify_id': row['artist_spotify_id'], + 'album_cover_url': row['album_cover_url'], + 'release_date': row['release_date'], + 'album_type': row['album_type'] + } for row in rows] + + except Exception as e: + logger.error(f"Error getting discovery recent albums: {e}") + return [] + + def clear_discovery_recent_albums(self) -> bool: + """Clear all cached recent albums""" + try: + with self._get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM discovery_recent_albums") + conn.commit() + return True + except Exception as e: + logger.error(f"Error clearing discovery recent albums: {e}") + return False + def add_recent_release(self, watchlist_artist_id: int, album_data: Dict[str, Any]) -> bool: """Add a recent release to the recent_releases table""" try: diff --git a/web_server.py b/web_server.py index f6994cd9..622f641f 100644 --- a/web_server.py +++ b/web_server.py @@ -14451,205 +14451,47 @@ def get_discover_hero(): @app.route('/api/discover/recent-releases', methods=['GET']) def get_discover_recent_releases(): - """Get recent albums/EPs from watchlist artists and similar artists (last 3 months)""" + """Get cached recent albums from watchlist and similar artists""" try: - from datetime import datetime, timedelta - import random - database = get_database() - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"success": True, "albums": []}) - - # Get watchlist artists (5 random) - watchlist_artists = database.get_watchlist_artists() - watchlist_sample = random.sample(watchlist_artists, min(5, len(watchlist_artists))) if watchlist_artists else [] - - # Get similar artists (5 random from top 20) - similar_artists = database.get_top_similar_artists(limit=20) - similar_sample = random.sample(similar_artists, min(5, len(similar_artists))) if similar_artists else [] - - recent_albums = [] - cutoff_date = datetime.now() - timedelta(days=90) # 3 months - - # Process watchlist artists - for artist in watchlist_sample: - try: - albums = spotify_client.get_artist_albums( - artist.spotify_artist_id, - album_type='album,single', - limit=20 - ) - - for album in albums: - try: - # Check if album is recent (last 3 months) - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: # Full date - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - recent_albums.append({ - "album_spotify_id": album.id, - "album_name": album.name, - "artist_name": artist.artist_name, - "artist_spotify_id": artist.spotify_artist_id, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "release_date": release_str, - "album_type": album.album_type if hasattr(album, 'album_type') else 'album' - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for watchlist artist {artist.artist_name}: {e}") - continue - - # Process similar artists - for artist in similar_sample: - try: - albums = spotify_client.get_artist_albums( - artist.similar_artist_spotify_id, - album_type='album,single', - limit=20 - ) + # Get cached recent albums + albums = database.get_discovery_recent_albums(limit=10) - for album in albums: - try: - # Check if album is recent (last 3 months) - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: # Full date - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - recent_albums.append({ - "album_spotify_id": album.id, - "album_name": album.name, - "artist_name": artist.similar_artist_name, - "artist_spotify_id": artist.similar_artist_spotify_id, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "release_date": release_str, - "album_type": album.album_type if hasattr(album, 'album_type') else 'album' - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for similar artist {artist.similar_artist_name}: {e}") - continue - - # Sort by release date (newest first) and limit to 10 - recent_albums.sort(key=lambda x: x['release_date'], reverse=True) - recent_albums = recent_albums[:10] - - return jsonify({"success": True, "albums": recent_albums}) + return jsonify({"success": True, "albums": albums}) except Exception as e: print(f"Error getting recent releases: {e}") - import traceback - traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/release-radar', methods=['GET']) def get_discover_release_radar(): - """Get release radar playlist - 50 tracks randomly selected from recent albums""" + """Get release radar playlist - 50 tracks from discovery pool (new releases only)""" try: - from datetime import datetime, timedelta import random database = get_database() - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"success": True, "tracks": []}) - - # Get watchlist artists (5 random) - watchlist_artists = database.get_watchlist_artists() - watchlist_sample = random.sample(watchlist_artists, min(5, len(watchlist_artists))) if watchlist_artists else [] + # Get new release tracks from discovery pool (is_new_release = True, last 30 days) + discovery_tracks = database.get_discovery_pool_tracks(limit=500, new_releases_only=True) - # Get similar artists (5 random from top 20) - similar_artists = database.get_top_similar_artists(limit=20) - similar_sample = random.sample(similar_artists, min(5, len(similar_artists))) if similar_artists else [] + if not discovery_tracks: + return jsonify({"success": True, "tracks": []}) + # Convert to JSON format all_tracks = [] - cutoff_date = datetime.now() - timedelta(days=90) # 3 months - - # Process watchlist artists - for artist in watchlist_sample: - try: - albums = spotify_client.get_artist_albums( - artist.spotify_artist_id, - album_type='album,single', - limit=20 - ) - - for album in albums: - try: - # Check if album is recent (last 3 months) - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: # Full date - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - # Get album tracks - album_data = spotify_client.get_album(album.id) - if album_data and 'tracks' in album_data: - for track in album_data['tracks']['items']: - all_tracks.append({ - "spotify_track_id": track['id'], - "track_name": track['name'], - "artist_name": artist.artist_name, - "album_name": album.name, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "duration_ms": track.get('duration_ms', 0), - "track_number": track.get('track_number', 0), - "track_data_json": track - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for watchlist artist {artist.artist_name}: {e}") - continue - - # Process similar artists - for artist in similar_sample: - try: - albums = spotify_client.get_artist_albums( - artist.similar_artist_spotify_id, - album_type='album,single', - limit=20 - ) - - for album in albums: - try: - # Check if album is recent (last 3 months) - if hasattr(album, 'release_date') and album.release_date: - release_str = album.release_date - if len(release_str) >= 10: # Full date - release_date = datetime.strptime(release_str[:10], "%Y-%m-%d") - if release_date >= cutoff_date: - # Get album tracks - album_data = spotify_client.get_album(album.id) - if album_data and 'tracks' in album_data: - for track in album_data['tracks']['items']: - all_tracks.append({ - "spotify_track_id": track['id'], - "track_name": track['name'], - "artist_name": artist.similar_artist_name, - "album_name": album.name, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "duration_ms": track.get('duration_ms', 0), - "track_number": track.get('track_number', 0), - "track_data_json": track - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for similar artist {artist.similar_artist_name}: {e}") - continue + for track in discovery_tracks: + all_tracks.append({ + "spotify_track_id": track.spotify_track_id, + "track_name": track.track_name, + "artist_name": track.artist_name, + "album_name": track.album_name, + "album_cover_url": track.album_cover_url, + "duration_ms": track.duration_ms, + "track_data_json": track.track_data_json + }) - # Randomly select 50 tracks + # Randomly select up to 50 tracks random.shuffle(all_tracks) selected_tracks = all_tracks[:50] @@ -14657,100 +14499,34 @@ def get_discover_release_radar(): except Exception as e: print(f"Error getting release radar: {e}") - import traceback - traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/discover/weekly', methods=['GET']) def get_discover_weekly(): - """Get discovery weekly playlist - 50 tracks randomly selected from any albums (not just recent)""" + """Get discovery weekly playlist - 50 tracks from discovery pool (all tracks, not just new)""" try: import random database = get_database() - if not spotify_client or not spotify_client.is_authenticated(): - return jsonify({"success": True, "tracks": []}) + # Get all tracks from discovery pool (not just new releases) + discovery_tracks = database.get_discovery_pool_tracks(limit=500, new_releases_only=False) - # Get watchlist artists (5 random) - watchlist_artists = database.get_watchlist_artists() - watchlist_sample = random.sample(watchlist_artists, min(5, len(watchlist_artists))) if watchlist_artists else [] - - # Get similar artists (5 random from top 20) - similar_artists = database.get_top_similar_artists(limit=20) - similar_sample = random.sample(similar_artists, min(5, len(similar_artists))) if similar_artists else [] + if not discovery_tracks: + return jsonify({"success": True, "tracks": []}) + # Convert to JSON format all_tracks = [] - - # Process watchlist artists - get tracks from any albums (not just recent) - for artist in watchlist_sample: - try: - albums = spotify_client.get_artist_albums( - artist.spotify_artist_id, - album_type='album', # Full albums only - limit=50 - ) - - # Select 2-3 random albums per artist - selected_albums = random.sample(albums, min(3, len(albums))) if albums else [] - - for album in selected_albums: - try: - # Get album tracks - album_data = spotify_client.get_album(album.id) - if album_data and 'tracks' in album_data: - for track in album_data['tracks']['items']: - all_tracks.append({ - "spotify_track_id": track['id'], - "track_name": track['name'], - "artist_name": artist.artist_name, - "album_name": album.name, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "duration_ms": track.get('duration_ms', 0), - "track_number": track.get('track_number', 0), - "track_data_json": track - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for watchlist artist {artist.artist_name}: {e}") - continue - - # Process similar artists - get tracks from any albums - for artist in similar_sample: - try: - albums = spotify_client.get_artist_albums( - artist.similar_artist_spotify_id, - album_type='album', # Full albums only - limit=50 - ) - - # Select 2-3 random albums per artist - selected_albums = random.sample(albums, min(3, len(albums))) if albums else [] - - for album in selected_albums: - try: - # Get album tracks - album_data = spotify_client.get_album(album.id) - if album_data and 'tracks' in album_data: - for track in album_data['tracks']['items']: - all_tracks.append({ - "spotify_track_id": track['id'], - "track_name": track['name'], - "artist_name": artist.similar_artist_name, - "album_name": album.name, - "album_cover_url": album.image_url if hasattr(album, 'image_url') else None, - "duration_ms": track.get('duration_ms', 0), - "track_number": track.get('track_number', 0), - "track_data_json": track - }) - except Exception as e: - continue - - except Exception as e: - print(f"Error fetching albums for similar artist {artist.similar_artist_name}: {e}") - continue + for track in discovery_tracks: + all_tracks.append({ + "spotify_track_id": track.spotify_track_id, + "track_name": track.track_name, + "artist_name": track.artist_name, + "album_name": track.album_name, + "album_cover_url": track.album_cover_url, + "duration_ms": track.duration_ms, + "track_data_json": track.track_data_json + }) # Randomly select 50 tracks random.shuffle(all_tracks) @@ -14760,8 +14536,6 @@ def get_discover_weekly(): except Exception as e: print(f"Error getting discovery weekly: {e}") - import traceback - traceback.print_exc() return jsonify({"success": False, "error": str(e)}), 500 @app.route('/api/metadata/start', methods=['POST']) @@ -17955,11 +17729,35 @@ if __name__ == '__main__': # Initialize app start time for uptime tracking import time app.start_time = time.time() - + # Add startup activity add_activity_item("🚀", "System Started", "SoulSync Web UI Server initialized", "Now") - + # Add a test activity to verify the system is working add_activity_item("🔧", "Debug Test", "Activity feed system test", "Now") - + + # Populate discovery pool at startup (background task) + def startup_populate_discovery(): + """Populate discovery pool at startup in background""" + try: + print("🎵 Populating discovery pool at startup...") + from core.watchlist_scanner import get_watchlist_scanner + if spotify_client and spotify_client.is_authenticated(): + scanner = get_watchlist_scanner(spotify_client) + scanner.populate_discovery_pool() + print("✅ Discovery pool populated successfully") + add_activity_item("🎵", "Discovery Pool", "Discovery data populated successfully", "Now") + else: + print("⚠️ Spotify not authenticated - skipping discovery pool population") + except Exception as e: + print(f"❌ Error populating discovery pool at startup: {e}") + import traceback + traceback.print_exc() + + # Run discovery pool population in background thread + import threading + discovery_thread = threading.Thread(target=startup_populate_discovery, daemon=True) + discovery_thread.start() + print("🔧 Discovery pool population started in background...") + app.run(host='0.0.0.0', port=8008, debug=False) diff --git a/webui/static/script.js b/webui/static/script.js index 432dda5d..5c075b33 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -24315,23 +24315,24 @@ async function loadDiscoverRecentReleases() { } const data = await response.json(); - if (!data.success || !data.releases || data.releases.length === 0) { + if (!data.success || !data.albums || data.albums.length === 0) { carousel.innerHTML = '
No recent releases found
${release.release_date}
+${album.artist_name}
+