database updater

pull/15/head
Broque Thomas 9 months ago
parent facc45e6f4
commit 2f535d8222

@ -24,6 +24,8 @@ from core.jellyfin_client import JellyfinClient
from core.soulseek_client import SoulseekClient
from core.tidal_client import TidalClient # Added import for Tidal
from core.matching_engine import MusicMatchingEngine
from core.database_update_worker import DatabaseUpdateWorker, DatabaseStatsWorker
from database.music_database import get_database
# --- Flask App Setup ---
base_dir = os.path.abspath(os.path.dirname(__file__))
@ -81,6 +83,19 @@ stream_lock = threading.Lock() # Prevent race conditions
stream_background_task = None
stream_executor = ThreadPoolExecutor(max_workers=1) # Only one stream at a time
db_update_executor = ThreadPoolExecutor(max_workers=1, thread_name_prefix="DBUpdate")
db_update_worker = None
db_update_state = {
"status": "idle", # idle, running, finished, error
"phase": "Idle",
"progress": 0,
"current_item": "",
"processed": 0,
"total": 0,
"error_message": ""
}
db_update_lock = threading.Lock()
# --- Global Matched Downloads Context Management ---
# Thread-safe storage for matched download contexts
# Key: slskd download ID, Value: dict containing Spotify artist/album data
@ -2278,6 +2293,120 @@ def start_simple_background_monitor():
import traceback
traceback.print_exc()
time.sleep(10)
# ===============================
# == DATABASE UPDATER API ==
# ===============================
def _db_update_progress_callback(current_item, processed, total, percentage):
with db_update_lock:
db_update_state.update({
"current_item": current_item,
"processed": processed,
"total": total,
"progress": percentage
})
def _db_update_phase_callback(phase):
with db_update_lock:
db_update_state["phase"] = phase
def _db_update_finished_callback(total_artists, total_albums, total_tracks, successful, failed):
with db_update_lock:
db_update_state["status"] = "finished"
db_update_state["phase"] = f"Completed: {successful} successful, {failed} failed."
def _db_update_error_callback(error_message):
with db_update_lock:
db_update_state["status"] = "error"
db_update_state["error_message"] = error_message
def _run_db_update_task(full_refresh, server_type):
"""The actual function that runs in the background thread."""
global db_update_worker
media_client = None
if server_type == "plex":
media_client = plex_client
elif server_type == "jellyfin":
media_client = jellyfin_client
if not media_client:
_db_update_error_callback(f"Media client for '{server_type}' not available.")
return
with db_update_lock:
db_update_worker = DatabaseUpdateWorker(
media_client=media_client,
full_refresh=full_refresh,
server_type=server_type
)
# Connect signals to callbacks
db_update_worker.progress_updated.connect(_db_update_progress_callback)
db_update_worker.phase_changed.connect(_db_update_phase_callback)
db_update_worker.finished.connect(_db_update_finished_callback)
db_update_worker.error.connect(_db_update_error_callback)
# This is a blocking call that runs the QThread's logic
db_update_worker.run()
@app.route('/api/database/stats', methods=['GET'])
def get_database_stats():
"""Endpoint to get current database statistics."""
try:
# This logic is adapted from DatabaseStatsWorker
db = get_database()
stats = db.get_database_info_for_server()
return jsonify(stats)
except Exception as e:
print(f"Error getting database stats: {e}")
return jsonify({"error": str(e)}), 500
@app.route('/api/database/update', methods=['POST'])
def start_database_update():
"""Endpoint to start the database update process."""
global db_update_worker
with db_update_lock:
if db_update_state["status"] == "running":
return jsonify({"success": False, "error": "An update is already in progress."}), 409
data = request.get_json()
full_refresh = data.get('full_refresh', False)
active_server = config_manager.get_active_media_server()
db_update_state.update({
"status": "running",
"phase": "Initializing...",
"progress": 0, "current_item": "", "processed": 0, "total": 0, "error_message": ""
})
# Submit the worker function to the executor
db_update_executor.submit(_run_db_update_task, full_refresh, active_server)
return jsonify({"success": True, "message": "Database update started."})
@app.route('/api/database/update/status', methods=['GET'])
def get_database_update_status():
"""Endpoint to poll for the current update status."""
with db_update_lock:
return jsonify(db_update_state)
@app.route('/api/database/update/stop', methods=['POST'])
def stop_database_update():
"""Endpoint to stop the current database update."""
global db_update_worker
with db_update_lock:
if db_update_worker and db_update_state["status"] == "running":
db_update_worker.stop()
db_update_state["status"] = "finished"
db_update_state["phase"] = "Update stopped by user."
return jsonify({"success": True, "message": "Stop request sent."})
else:
return jsonify({"success": False, "error": "No update is currently running."}), 404
monitor_thread = threading.Thread(target=simple_monitor)
monitor_thread.daemon = True

@ -222,6 +222,24 @@
<div class="tool-card" id="db-updater-card">
<h4 class="tool-card-title">Database Updater</h4>
<p class="tool-card-info">Last Full Refresh: <span id="db-last-refresh">Never</span></p>
<div class="tool-card-stats">
<div class="stat-item">
<span class="stat-item-label">Artists:</span>
<span class="stat-item-value" id="db-stat-artists">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Albums:</span>
<span class="stat-item-value" id="db-stat-albums">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Tracks:</span>
<span class="stat-item-value" id="db-stat-tracks">0</span>
</div>
<div class="stat-item">
<span class="stat-item-label">Size:</span>
<span class="stat-item-value" id="db-stat-size">0.0 MB</span>
</div>
</div>
<div class="tool-card-controls">
<select id="db-refresh-type">
<option value="incremental">Incremental Update</option>

@ -22,6 +22,8 @@ let currentFilterFormat = 'all';
let currentSortBy = 'quality_score';
let isSortReversed = false;
let searchAbortController = null;
let dbStatsInterval = null;
let dbUpdateStatusInterval = null;
// API endpoints
const API = {
@ -117,6 +119,9 @@ function navigateToPage(pageId) {
async function loadPageData(pageId) {
try {
// Stop any active polling when navigating away
stopDbStatsPolling();
stopDbUpdatePolling();
switch (pageId) {
case 'dashboard':
stopDownloadPolling();
@ -127,7 +132,6 @@ async function loadPageData(pageId) {
await loadSyncData();
break;
case 'downloads':
// --- FIX: Initialize first, THEN load data. This is the correct order. ---
initializeSearch();
initializeFilters();
await loadDownloadsData();
@ -3208,4 +3212,205 @@ function matchedDownloadAlbumTrack(albumIndex, trackIndex) {
openMatchingModal(trackData, false, albumData);
}
// ===========================================
// == DASHBOARD DATABASE UPDATER FUNCTIONALITY ==
// ===========================================
// --- State and Polling Management ---
function stopDbStatsPolling() {
if (dbStatsInterval) {
clearInterval(dbStatsInterval);
dbStatsInterval = null;
}
}
function stopDbUpdatePolling() {
if (dbUpdateStatusInterval) {
clearInterval(dbUpdateStatusInterval);
dbUpdateStatusInterval = null;
}
}
async function loadDashboardData() {
// Attach event listeners for the DB updater tool
const updateButton = document.getElementById('db-update-button');
if (updateButton) {
updateButton.addEventListener('click', handleDbUpdateButtonClick);
}
// Initial load of stats
await fetchAndUpdateDbStats();
// Start periodic refresh of stats (every 30 seconds)
stopDbStatsPolling(); // Ensure no duplicates
dbStatsInterval = setInterval(fetchAndUpdateDbStats, 30000);
// Also check the status of any ongoing update when the page loads
await checkAndUpdateDbProgress();
}
// --- Data Fetching and UI Updates ---
async function fetchAndUpdateDbStats() {
try {
const response = await fetch('/api/database/stats');
if (!response.ok) return;
const stats = await response.json();
// This function updates the stat cards in the top grid
updateDashboardStatCards(stats);
// This function updates the info within the DB Updater tool card
updateDbUpdaterCardInfo(stats);
} catch (error) {
console.warn('Could not fetch DB stats:', error);
}
}
function updateDashboardStatCards(stats) {
// You can expand this later to update the main stat cards
// For now, we focus on the updater tool itself.
}
function updateDbUpdaterCardInfo(stats) {
// Update the detailed stats within the DB Updater tool card
const lastRefreshEl = document.getElementById('db-last-refresh');
const artistsStatEl = document.getElementById('db-stat-artists');
const albumsStatEl = document.getElementById('db-stat-albums');
const tracksStatEl = document.getElementById('db-stat-tracks');
const sizeStatEl = document.getElementById('db-stat-size');
if (lastRefreshEl) {
if (stats.last_full_refresh) {
const date = new Date(stats.last_full_refresh);
lastRefreshEl.textContent = date.toLocaleString();
} else {
lastRefreshEl.textContent = 'Never';
}
}
if (artistsStatEl) artistsStatEl.textContent = stats.artists.toLocaleString() || '0';
if (albumsStatEl) albumsStatEl.textContent = stats.albums.toLocaleString() || '0';
if (tracksStatEl) tracksStatEl.textContent = stats.tracks.toLocaleString() || '0';
if (sizeStatEl) sizeStatEl.textContent = `${stats.database_size_mb.toFixed(2)} MB`;
// Update the title of the tool card to show which server is active
const toolCardTitle = document.querySelector('#db-updater-card .tool-card-title');
if (toolCardTitle && stats.server_source) {
const serverName = stats.server_source.charAt(0).toUpperCase() + stats.server_source.slice(1);
toolCardTitle.textContent = `${serverName} Database Updater`;
}
}
async function checkAndUpdateDbProgress() {
try {
const response = await fetch('/api/database/update/status');
if (!response.ok) return;
const state = await response.json();
updateDbProgressUI(state);
if (state.status === 'running') {
// If an update is running, start polling for progress
stopDbUpdatePolling();
dbUpdateStatusInterval = setInterval(checkAndUpdateDbProgress, 1000);
}
} catch (error) {
console.warn('Could not fetch DB update status:', error);
}
}
function updateDbProgressUI(state) {
const button = document.getElementById('db-update-button');
const phaseLabel = document.getElementById('db-phase-label');
const progressLabel = document.getElementById('db-progress-label');
const progressBar = document.getElementById('db-progress-bar');
const refreshSelect = document.getElementById('db-refresh-type');
if (!button || !phaseLabel || !progressLabel || !progressBar || !refreshSelect) return;
if (state.status === 'running') {
button.textContent = 'Stop Update';
button.disabled = false;
refreshSelect.disabled = true;
phaseLabel.textContent = state.phase || 'Processing...';
progressLabel.textContent = `${state.processed} / ${state.total} artists (${state.progress.toFixed(1)}%)`;
progressBar.style.width = `${state.progress}%`;
} else { // idle, finished, or error
stopDbUpdatePolling();
button.textContent = 'Update Database';
button.disabled = false;
refreshSelect.disabled = false;
if (state.status === 'error') {
phaseLabel.textContent = `Error: ${state.error_message}`;
progressBar.style.backgroundColor = '#ff4444'; // Red for error
} else {
phaseLabel.textContent = state.phase || 'Idle';
progressBar.style.backgroundColor = '#1db954'; // Green for normal
}
if (state.status === 'finished' || state.status === 'error') {
// Final stats refresh after completion/error
setTimeout(fetchAndUpdateDbStats, 500);
}
}
}
// --- Event Handlers ---
async function handleDbUpdateButtonClick() {
const button = document.getElementById('db-update-button');
const currentAction = button.textContent;
if (currentAction === 'Update Database') {
const refreshSelect = document.getElementById('db-refresh-type');
const isFullRefresh = refreshSelect.value === 'full';
if (isFullRefresh) {
const confirmed = confirm("⚠️ Full Refresh Warning!\n\nThis will clear and rebuild the database for the active server. It can take a long time. Are you sure you want to proceed?");
if (!confirmed) return;
}
try {
const response = await fetch('/api/database/update', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ full_refresh: isFullRefresh })
});
if (response.ok) {
showToast('Database update started!', 'success');
// Start polling immediately
stopDbUpdatePolling();
dbUpdateStatusInterval = setInterval(checkAndUpdateDbProgress, 1000);
} else {
const errorData = await response.json();
showToast(`Error: ${errorData.error}`, 'error');
}
} catch (error) {
showToast('Failed to start update process.', 'error');
}
} else { // "Stop Update"
try {
const response = await fetch('/api/database/update/stop', { method: 'POST' });
if (response.ok) {
showToast('Stop request sent.', 'info');
} else {
showToast('Failed to send stop request.', 'error');
}
} catch (error) {
showToast('Error sending stop request.', 'error');
}
}
}

@ -3770,3 +3770,34 @@ body {
100% { left: 100%; }
}
/* Styles for Stats inside the DB Updater Tool Card */
.tool-card-stats {
display: grid;
grid-template-columns: 1fr 1fr;
gap: 8px;
padding: 10px;
background: rgba(0, 0, 0, 0.2);
border-radius: 6px;
margin-top: 5px; /* Add some space above */
margin-bottom: 5px; /* Add some space below */
}
.stat-item {
display: flex;
justify-content: space-between;
align-items: center;
font-size: 12px;
}
.stat-item-label {
color: #b3b3b3;
}
.stat-item-value {
color: #ffffff;
font-weight: 600;
background-color: rgba(29, 185, 84, 0.1);
border-radius: 4px;
padding: 2px 6px;
}

Loading…
Cancel
Save