From bbf5af1ce1f7577427f189d4ae6f15f67ee0461b Mon Sep 17 00:00:00 2001 From: Broque Thomas <26755000+Nezreka@users.noreply.github.com> Date: Fri, 17 Apr 2026 19:37:42 -0700 Subject: [PATCH] Fix auto-import rescan race condition, coverage penalty, and UI Race condition: scanner re-scanned folders while post-processing was still moving files, causing partial matches and ghost failures. Now tracks in-progress paths and skips them on subsequent scans. Coverage penalty fix: individual tracks that match at 80%+ confidence now auto-import even when overall album coverage is low (e.g. 2 of 18 tracks present). Previously low coverage killed the entire import. Import page: stats bar, filter pills, Scan Now, Approve All, Clear History (clears imported + failed), live scan progress. --- core/auto_import_worker.py | 26 ++++++- web_server.py | 47 ++++++++++++ webui/index.html | 27 ++++++- webui/static/script.js | 142 +++++++++++++++++++++++++++++++++++-- webui/static/style.css | 104 +++++++++++++++++++++++++++ 5 files changed, 335 insertions(+), 11 deletions(-) diff --git a/core/auto_import_worker.py b/core/auto_import_worker.py index 2d169dba..d9dc934c 100644 --- a/core/auto_import_worker.py +++ b/core/auto_import_worker.py @@ -135,6 +135,7 @@ class AutoImportWorker: # State self._folder_snapshots: Dict[str, float] = {} # path -> mtime_sum + self._processing_paths: set = set() # Paths currently being processed (skip on rescan) self._current_folder = '' self._current_status = 'idle' self._stats = {'scanned': 0, 'auto_processed': 0, 'pending_review': 0, 'failed': 0} @@ -237,6 +238,11 @@ class AutoImportWorker: self._current_folder = candidate.name + # Skip folders currently being processed by a previous scan cycle + if candidate.path in self._processing_paths: + logger.debug(f"[Auto-Import] Skipping {candidate.name} — still processing from previous cycle") + continue + # Check if already processed if self._is_already_processed(candidate.folder_hash): continue @@ -248,6 +254,8 @@ class AutoImportWorker: self._stats['scanned'] += 1 logger.info(f"[Auto-Import] Processing folder: {candidate.name} ({len(candidate.audio_files)} files)") + # Mark as in-progress so next scan cycle skips this folder + self._processing_paths.add(candidate.path) try: # Phase 3: Identify identification = self._identify_folder(candidate) @@ -272,11 +280,21 @@ class AutoImportWorker: confidence = match_result['confidence'] status = 'matched' - if confidence >= threshold and auto_process: - # Phase 5: Auto-process - logger.info(f"[Auto-Import] High confidence ({confidence:.0%}) — auto-processing {candidate.name}") + # Check if individual track matches are strong even if overall confidence + # is low (e.g. only 2 of 18 album tracks present → low coverage kills + # overall score, but the 2 tracks match perfectly and should still import) + high_conf_matches = [m for m in match_result.get('matches', []) if m['confidence'] >= 0.8] + has_strong_individual_matches = len(high_conf_matches) > 0 + + if (confidence >= threshold or has_strong_individual_matches) and auto_process: + # Phase 5: Auto-process — process all tracks that matched + effective_conf = max(confidence, min(m['confidence'] for m in high_conf_matches) if high_conf_matches else 0) + logger.info(f"[Auto-Import] Processing {candidate.name} — " + f"overall: {confidence:.0%}, {len(high_conf_matches)} strong matches, " + f"{match_result.get('matched_count', 0)}/{match_result.get('total_tracks', '?')} tracks") success = self._process_matches(candidate, identification, match_result) status = 'completed' if success else 'failed' + confidence = max(confidence, effective_conf) if success: self._stats['auto_processed'] += 1 else: @@ -302,6 +320,8 @@ class AutoImportWorker: logger.error(f"[Auto-Import] Error processing {candidate.name}: {e}") self._record_result(candidate, 'failed', 0.0, error_message=str(e)) self._stats['failed'] += 1 + finally: + self._processing_paths.discard(candidate.path) # Rate limit between folders if self._interruptible_sleep(2): diff --git a/web_server.py b/web_server.py index 50a237c8..4f090307 100644 --- a/web_server.py +++ b/web_server.py @@ -53267,6 +53267,53 @@ def auto_import_reject(item_id): return jsonify(auto_import_worker.reject_item(item_id)) +@app.route('/api/auto-import/scan-now', methods=['POST']) +def auto_import_scan_now(): + """Trigger an immediate scan cycle.""" + if not auto_import_worker: + return jsonify({"success": False, "error": "Auto-import not available"}), 500 + if not auto_import_worker.running: + return jsonify({"success": False, "error": "Auto-import is not running"}), 400 + # Run scan in background thread + import threading + threading.Thread(target=auto_import_worker._scan_cycle, daemon=True).start() + return jsonify({"success": True}) + + +@app.route('/api/auto-import/approve-all', methods=['POST']) +def auto_import_approve_all(): + """Approve all pending review items.""" + if not auto_import_worker: + return jsonify({"success": False, "error": "Auto-import not available"}), 500 + try: + results = auto_import_worker.get_results(status_filter='pending_review', limit=200) + count = 0 + for r in results: + result = auto_import_worker.approve_item(r['id']) + if result.get('success'): + count += 1 + return jsonify({"success": True, "count": count}) + except Exception as e: + return jsonify({"success": False, "error": str(e)}), 500 + + +@app.route('/api/auto-import/clear-completed', methods=['POST']) +def auto_import_clear_completed(): + """Remove completed/imported items from history.""" + if not auto_import_worker: + return jsonify({"success": False, "error": "Auto-import not available"}), 500 + try: + db = get_database() + with db._get_connection() as conn: + cursor = conn.cursor() + cursor.execute("DELETE FROM auto_import_history WHERE status IN ('completed', 'approved', 'failed', 'needs_identification', 'rejected')") + count = cursor.rowcount + conn.commit() + return jsonify({"success": True, "count": count}) + except Exception as e: + return jsonify({"success": False, "error": str(e)}), 500 + + @app.route('/api/import/staging/suggestions', methods=['GET']) def import_staging_suggestions(): """Return cached import suggestions. If cache isn't built yet, returns partial/empty with a flag.""" diff --git a/webui/index.html b/webui/index.html index 8f03b600..7b41fac5 100644 --- a/webui/index.html +++ b/webui/index.html @@ -6048,6 +6048,10 @@ Auto-Import Disabled + + + + + + + +

Enable auto-import to watch your staging folder for new music.

-

Drop album folders into your staging directory and SoulSync will identify, match, and import them automatically.

+

Drop album folders or single tracks into your staging directory and SoulSync will identify, match, and import them automatically.

diff --git a/webui/static/script.js b/webui/static/script.js index 3becec08..36a17f0b 100644 --- a/webui/static/script.js +++ b/webui/static/script.js @@ -66449,6 +66449,7 @@ function importPageSwitchTab(tab) { // ── Auto-Import Tab ── let _autoImportPollInterval = null; +let _autoImportFilter = 'all'; function _autoImportStartPolling() { _autoImportStopPolling(); @@ -66499,14 +66500,43 @@ async function _autoImportLoadStatus() { const toggle = document.getElementById('auto-import-enabled'); const statusText = document.getElementById('auto-import-status-text'); const settingsRow = document.getElementById('auto-import-settings-row'); + const scanNowBtn = document.getElementById('auto-import-scan-now'); + const progressEl = document.getElementById('auto-import-progress'); + const progressText = document.getElementById('auto-import-progress-text'); if (toggle) toggle.checked = data.running; if (settingsRow) settingsRow.style.display = data.running ? '' : 'none'; + if (scanNowBtn) scanNowBtn.style.display = data.running ? '' : 'none'; + + // Live scan progress + if (progressEl) { + if (data.current_status === 'scanning') { + progressEl.style.display = ''; + if (progressText) { + const stats = data.stats || {}; + progressText.textContent = `Scanning: ${data.current_folder || '...'} (${stats.scanned || 0} processed)`; + } + } else { + progressEl.style.display = 'none'; + } + } + if (statusText) { if (data.paused) statusText.textContent = 'Paused'; - else if (data.current_status === 'scanning') statusText.textContent = `Scanning: ${data.current_folder || '...'}`; - else if (data.running) statusText.textContent = 'Watching'; - else statusText.textContent = 'Disabled'; + else if (data.current_status === 'scanning') statusText.textContent = 'Scanning...'; + else if (data.running) { + // Show last scan time + let watchText = 'Watching'; + if (data.last_scan_time) { + try { + const lastScan = new Date(data.last_scan_time); + const diffS = Math.floor((Date.now() - lastScan) / 1000); + if (diffS < 60) watchText = `Watching (scanned ${diffS}s ago)`; + else if (diffS < 3600) watchText = `Watching (scanned ${Math.floor(diffS / 60)}m ago)`; + } catch (e) {} + } + statusText.textContent = watchText; + } else statusText.textContent = 'Disabled'; statusText.className = 'auto-import-status ' + (data.running ? (data.current_status === 'scanning' ? 'scanning' : 'active') : 'disabled'); } } catch (e) {} @@ -66516,19 +66546,61 @@ async function _autoImportLoadResults() { const container = document.getElementById('auto-import-results'); if (!container) return; try { - const res = await fetch('/api/auto-import/results?limit=30'); + const res = await fetch('/api/auto-import/results?limit=100'); const data = await res.json(); if (!data.success || !data.results || data.results.length === 0) { - // Keep empty state if no results if (!container.querySelector('.auto-import-card')) { container.innerHTML = `
-

No imports yet. Drop album folders into your staging directory.

+

No imports yet. Drop album folders or single tracks into your staging directory.

`; } + // Hide stats and filters + const statsEl = document.getElementById('auto-import-stats'); + const filtersEl = document.getElementById('auto-import-filters'); + if (statsEl) statsEl.style.display = 'none'; + if (filtersEl) filtersEl.style.display = 'none'; return; } - container.innerHTML = data.results.map((r, idx) => { + // Compute stats + const allResults = data.results; + const importedCount = allResults.filter(r => r.status === 'completed' || r.status === 'approved').length; + const reviewCount = allResults.filter(r => r.status === 'pending_review').length; + const failedCount = allResults.filter(r => r.status === 'failed' || r.status === 'needs_identification').length; + + // Update stats + const statsEl = document.getElementById('auto-import-stats'); + if (statsEl) { + statsEl.style.display = ''; + document.getElementById('auto-import-stat-imported').textContent = `${importedCount} imported`; + document.getElementById('auto-import-stat-review').textContent = `${reviewCount} review`; + document.getElementById('auto-import-stat-failed').textContent = `${failedCount} failed`; + } + + // Show filters + const filtersEl = document.getElementById('auto-import-filters'); + if (filtersEl) { + filtersEl.style.display = ''; + // Show batch action buttons when applicable + const approveAllBtn = document.getElementById('auto-import-approve-all'); + const clearBtn = document.getElementById('auto-import-clear-completed'); + if (approveAllBtn) approveAllBtn.style.display = reviewCount > 0 ? '' : 'none'; + if (clearBtn) clearBtn.style.display = (importedCount + failedCount) > 0 ? '' : 'none'; + } + + // Apply filter + let filtered = allResults; + if (_autoImportFilter === 'pending') filtered = allResults.filter(r => r.status === 'pending_review'); + else if (_autoImportFilter === 'imported') filtered = allResults.filter(r => r.status === 'completed' || r.status === 'approved'); + else if (_autoImportFilter === 'failed') filtered = allResults.filter(r => r.status === 'failed' || r.status === 'needs_identification'); + + if (filtered.length === 0) { + const filterName = _autoImportFilter === 'pending' ? 'pending review' : _autoImportFilter; + container.innerHTML = `

No ${filterName} items.

`; + return; + } + + container.innerHTML = filtered.map((r, idx) => { const confPct = Math.round((r.confidence || 0) * 100); const confClass = confPct >= 90 ? 'high' : confPct >= 70 ? 'medium' : 'low'; const statusLabels = { @@ -66652,6 +66724,58 @@ async function _autoImportSaveSettings() { } catch (e) { showToast('Error', 'error'); } } +function _autoImportSetFilter(filter) { + _autoImportFilter = filter; + document.querySelectorAll('#auto-import-filters .adl-pill').forEach(p => + p.classList.toggle('active', p.dataset.filter === filter)); + _autoImportLoadResults(); +} + +async function _autoImportScanNow() { + try { + const res = await fetch('/api/auto-import/scan-now', { method: 'POST' }); + const data = await res.json(); + if (data.success) { + showToast('Scan triggered', 'success'); + _autoImportLoadStatus(); + } else { + showToast(data.error || 'Failed to trigger scan', 'error'); + } + } catch (e) { showToast('Error: ' + e.message, 'error'); } +} + +async function _autoImportApproveAll() { + const confirmed = await showConfirmDialog({ + title: 'Approve All', + message: 'Approve and import all pending review items?', + confirmText: 'Approve All', + }); + if (!confirmed) return; + try { + const res = await fetch('/api/auto-import/approve-all', { method: 'POST' }); + const data = await res.json(); + if (data.success) { + showToast(`Approved ${data.count || 0} items`, 'success'); + _autoImportLoadResults(); + } else { + showToast(data.error || 'Failed', 'error'); + } + } catch (e) { showToast('Error: ' + e.message, 'error'); } +} + +async function _autoImportClearCompleted() { + try { + const res = await fetch('/api/auto-import/clear-completed', { method: 'POST' }); + const data = await res.json(); + if (data.success) { + showToast(`Cleared ${data.count || 0} imported items`, 'success'); + _autoImportLoadResults(); + } else { + showToast(data.error || 'Failed', 'error'); + } + } catch (e) { showToast('Error: ' + e.message, 'error'); } +} + function _autoImportToggleDetail(idx) { const trackList = document.getElementById(`auto-import-tracks-${idx}`); if (trackList) { @@ -66659,6 +66783,10 @@ function _autoImportToggleDetail(idx) { } } window._autoImportToggleDetail = _autoImportToggleDetail; +window._autoImportSetFilter = _autoImportSetFilter; +window._autoImportScanNow = _autoImportScanNow; +window._autoImportApproveAll = _autoImportApproveAll; +window._autoImportClearCompleted = _autoImportClearCompleted; async function _autoImportApprove(id) { try { diff --git a/webui/static/style.css b/webui/static/style.css index 1c44924f..4a951000 100644 --- a/webui/static/style.css +++ b/webui/static/style.css @@ -58313,6 +58313,110 @@ body.reduce-effects *::after { font-size: 10px; font-weight: 600; color: rgba(255,255,255,0.5); } +/* Scan Now button */ +.auto-import-scan-now-btn { + background: rgba(255,255,255,0.06); + border: 1px solid rgba(255,255,255,0.1); + color: rgba(255,255,255,0.6); + font-size: 11px; + padding: 4px 12px; + border-radius: 6px; + cursor: pointer; + display: flex; + align-items: center; + gap: 5px; + transition: all 0.15s; + margin-left: auto; +} + +.auto-import-scan-now-btn:hover { + background: rgba(255,255,255,0.1); + color: rgba(255,255,255,0.9); +} + +/* Live progress */ +.auto-import-progress { + margin-top: 8px; + padding: 8px 12px; + background: rgba(var(--accent-rgb), 0.04); + border: 1px solid rgba(var(--accent-rgb), 0.1); + border-radius: 8px; +} + +.auto-import-progress-text { + font-size: 11px; + color: rgba(255,255,255,0.6); + margin-bottom: 4px; +} + +.auto-import-progress-bar { + height: 3px; + background: rgba(255,255,255,0.06); + border-radius: 2px; + overflow: hidden; +} + +.auto-import-progress-fill { + height: 100%; + background: rgba(var(--accent-rgb), 0.6); + border-radius: 2px; + width: 100%; + animation: adlPulse 1.5s ease-in-out infinite; +} + +/* Stats summary */ +.auto-import-stats { + display: flex; + gap: 16px; + padding: 8px 0; + margin-bottom: 4px; +} + +.auto-import-stat { + font-size: 12px; + color: rgba(255,255,255,0.5); + font-weight: 500; +} + +.auto-import-stat-review { color: #fbbf24; } +.auto-import-stat-failed { color: #f87171; } + +/* Filter pills */ +.auto-import-filters { + display: flex; + align-items: center; + gap: 6px; + padding: 6px 0; + margin-bottom: 8px; +} + +/* Batch action buttons */ +.auto-import-batch-btn { + font-size: 11px; + padding: 4px 12px; + border-radius: 6px; + border: 1px solid rgba(var(--accent-rgb), 0.3); + background: rgba(var(--accent-rgb), 0.08); + color: rgba(var(--accent-rgb), 1); + cursor: pointer; + transition: all 0.15s; +} + +.auto-import-batch-btn:hover { + background: rgba(var(--accent-rgb), 0.15); +} + +.auto-import-clear-btn { + border-color: rgba(255,255,255,0.1); + background: rgba(255,255,255,0.04); + color: rgba(255,255,255,0.5); +} + +.auto-import-clear-btn:hover { + background: rgba(255,255,255,0.08); + color: rgba(255,255,255,0.8); +} + .auto-import-card-meta { display: flex; gap: 8px; align-items: center; font-size: 10px; color: rgba(255,255,255,0.3); margin-top: 3px;