diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts index 870a36d..23bc6d7 100644 --- a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -396,10 +396,12 @@ export class UnifiedAPIServer { const backgroundQueueRunning = queueStatus.backgroundQueue?.running || 0; const backgroundQueuePending = queueStatus.backgroundQueue?.pending || 0; - // "Discovered" now represents files waiting for fast queue (validated but not yet processing) - // This merges the old "discovered" and "awaitingFastQueue" concepts - snapshot.totalDiscovered = fastQueuePending; - snapshot.awaitingFastQueue = fastQueuePending; // Keep for backward compatibility + // "Discovered" now represents unique files waiting for fast queue (validated but not yet processing) + // Use file count from container scheduler if available, otherwise fall back to task count + const filesStatus = queueStatus.files; + const uniqueFilesWaiting = filesStatus ? (filesStatus.fastWaiting + filesStatus.fastRunning) : fastQueuePending; + snapshot.totalDiscovered = uniqueFilesWaiting; + snapshot.awaitingFastQueue = uniqueFilesWaiting; // Keep for backward compatibility // Fix awaitingBackground: subtract files currently being processed by background workers // hashProcessing.size includes both waiting AND running files @@ -444,6 +446,14 @@ export class UnifiedAPIServer { const metrics = performanceMetrics.getMetrics(); snapshot.totalFailed = metrics.totalFailedFiles || 0; + // Add filtered count from pipeline stats + if (this.streamingPipeline) { + const pipelineStats = this.streamingPipeline.getStats(); + snapshot.totalFiltered = pipelineStats.filtered || 0; + } else { + snapshot.totalFiltered = 0; + } + return snapshot; }); diff --git a/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts b/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts index 630104b..b74fa26 100644 --- a/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts +++ b/packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts @@ -57,6 +57,14 @@ export interface UnifiedProcessingSnapshot { /** Files that completed fast queue but are waiting for background queue */ awaitingBackground: number; + // Filtering/duplicate metrics + /** Files filtered out due to unsupported extensions (non-media files) */ + totalFiltered?: number; + /** Files with same content hash as previously processed file */ + totalDuplicates?: number; + /** Number of unique content hashes seen */ + uniqueHashes?: number; + // Configuration fastQueueConcurrency?: number; // Number of fast queue workers backgroundQueueConcurrency?: number; // Number of background queue workers @@ -70,6 +78,10 @@ export class UnifiedProcessingStateManager { private maxDoneHistory = 100; // Keep last 100 completed files private totalProcessedCount = 0; // Track total number of completed files (not limited) + // Duplicate tracking + private seenHashes: Set = new Set(); + private duplicateCount = 0; // Files with same hash as a previously processed file + /** * Add a file to discovered state (file discovered) */ @@ -126,6 +138,13 @@ export class UnifiedProcessingStateManager { return; } + // Track duplicates - if hash already seen, increment duplicate counter + if (this.seenHashes.has(hash)) { + this.duplicateCount++; + } else { + this.seenHashes.add(hash); + } + // Move to hash processing queue this.hashProcessing.set(filePath, { filePath, @@ -273,6 +292,10 @@ export class UnifiedProcessingStateManager { // Files that completed fast queue but are waiting for/in background queue awaitingBackground: this.hashProcessing.size, + // Duplicate tracking (filtered count is added by API layer from pipeline stats) + totalDuplicates: this.duplicateCount, + uniqueHashes: this.seenHashes.size, + fastQueueConcurrency, backgroundQueueConcurrency }; @@ -296,6 +319,8 @@ export class UnifiedProcessingStateManager { this.hashProcessing.clear(); this.done = []; this.totalProcessedCount = 0; + this.seenHashes.clear(); + this.duplicateCount = 0; } /** @@ -366,4 +391,14 @@ export class UnifiedProcessingStateManager { getHashProcessingFiles(): Map { return this.hashProcessing; } + + /** + * Get duplicate tracking statistics + */ + getDuplicateStats(): { duplicateCount: number; uniqueHashes: number } { + return { + duplicateCount: this.duplicateCount, + uniqueHashes: this.seenHashes.size + }; + } } diff --git a/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts index 7b626cb..9328bdc 100644 --- a/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts +++ b/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts @@ -20,6 +20,7 @@ export class StreamingPipeline { private discoveredCount = 0; private validatedCount = 0; + private filteredCount = 0; // Files filtered out (unsupported extensions) private fastProcessedCount = 0; private backgroundProcessedCount = 0; @@ -106,6 +107,7 @@ export class StreamingPipeline { const ext = path.extname(filePath).toLowerCase(); if (!this.config.supportedExtensions.has(ext)) { // Skip unsupported file types - remove from discovered + this.filteredCount++; this.config.stateManager.removeFile(filePath); return; } @@ -353,6 +355,7 @@ export class StreamingPipeline { return { discovered: this.discoveredCount, validated: this.validatedCount, + filtered: this.filteredCount, fastProcessed: this.fastProcessedCount, backgroundProcessed: this.backgroundProcessedCount, queues: { @@ -396,6 +399,7 @@ export class StreamingPipeline { // Step 2: Reset counters this.discoveredCount = 0; this.validatedCount = 0; + this.filteredCount = 0; this.fastProcessedCount = 0; this.backgroundProcessedCount = 0; diff --git a/packages/meta-sort-ui/src/pages/Monitor.tsx b/packages/meta-sort-ui/src/pages/Monitor.tsx index c0084fc..fd7105d 100644 --- a/packages/meta-sort-ui/src/pages/Monitor.tsx +++ b/packages/meta-sort-ui/src/pages/Monitor.tsx @@ -10,7 +10,6 @@ import { import { formatMs, formatNumber, - formatUptime, formatBytes, getFilename } from '../utils/format'; @@ -21,6 +20,7 @@ function Monitor() { const [redisStats, setRedisStats] = useState(null); const [queue, setQueue] = useState([]); const [failedFiles, setFailedFiles] = useState([]); + const [showPipelineHelp, setShowPipelineHelp] = useState(false); const fetchData = useCallback(async () => { try { @@ -152,7 +152,26 @@ function Monitor() { {/* Pipeline Progress Bar - 6 Stage (including failed) */} {status && (
-

Processing Pipeline

+
+

Processing Pipeline

+ +
+ {showPipelineHelp && ( +
+
Discovered: Files found on disk, waiting to be processed.
+
Fast Queue: Files being processed for quick metadata extraction (hash, ffmpeg, etc.).
+
Awaiting BG: Files done with fast processing, queued for slower tasks.
+
Background: Files running slow tasks like full file hashing or TMDB lookups.
+
Complete: Files fully processed and available in the virtual filesystem.
+
Failed: Files that encountered errors during processing.
+
+ )}
@@ -319,27 +338,23 @@ function Monitor() {
)} -

System Stats

+

File Coverage

-
{redisStats ? formatNumber(redisStats.fileCount) : '-'}
-
Files in Redis
+
{status?.uniqueHashes ? formatNumber(status.uniqueHashes) : (redisStats ? formatNumber(redisStats.fileCount) : '-')}
+
Unique Files
-
{redisStats?.memoryUsage || '-'}
-
Redis Memory
+
{status?.totalFiltered !== undefined ? formatNumber(status.totalFiltered) : '-'}
+
Filtered (Non-Media)
-
{metrics ? formatUptime(metrics.uptime) : '-'}
-
Uptime
+
{status?.totalDuplicates !== undefined ? formatNumber(status.totalDuplicates) : '-'}
+
Duplicates (Same Hash)
-
- {metrics?.memoryUsage - ? `${Math.round(metrics.memoryUsage.heapUsed / 1024 / 1024)} MB` - : '-'} -
-
Node Heap
+
{status?.totalFailed !== undefined ? formatNumber(status.totalFailed) : '-'}
+
Failed
@@ -517,9 +532,71 @@ function Monitor() { } .pipeline-section h2 { + margin-bottom: 0; + } + + .pipeline-header { + display: flex; + align-items: center; + gap: 10px; margin-bottom: 16px; } + .help-button { + width: 20px; + height: 20px; + border-radius: 50%; + border: 1px solid var(--text-secondary); + background: transparent; + color: var(--text-secondary); + font-size: 12px; + font-weight: 600; + cursor: pointer; + display: flex; + align-items: center; + justify-content: center; + transition: all 0.2s ease; + } + + .help-button:hover { + background: var(--text-secondary); + color: var(--bg-primary); + } + + .pipeline-help { + background: var(--bg-tertiary); + border-radius: 8px; + padding: 12px 16px; + margin-bottom: 16px; + font-size: 0.85rem; + line-height: 1.6; + } + + .help-item { + display: flex; + align-items: center; + gap: 8px; + margin-bottom: 6px; + } + + .help-item:last-child { + margin-bottom: 0; + } + + .help-dot { + width: 10px; + height: 10px; + border-radius: 50%; + flex-shrink: 0; + } + + .help-dot.discovered { background: #9ca3af; } + .help-dot.fast { background: #fbbf24; } + .help-dot.awaiting-background { background: #f472b6; } + .help-dot.background { background: #a78bfa; } + .help-dot.done { background: #34d399; } + .help-dot.failed { background: #f87171; } + .pipeline-bar-container { margin-bottom: 16px; } diff --git a/packages/meta-sort-ui/src/types/index.ts b/packages/meta-sort-ui/src/types/index.ts index 72faa6d..cd03d8d 100644 --- a/packages/meta-sort-ui/src/types/index.ts +++ b/packages/meta-sort-ui/src/types/index.ts @@ -89,6 +89,12 @@ export interface ProcessingStatus { totalDone: number; /** Total number of failed files */ totalFailed?: number; + /** Files filtered out due to unsupported extensions (non-media files) */ + totalFiltered?: number; + /** Files with same content hash as previously processed file */ + totalDuplicates?: number; + /** Number of unique content hashes seen */ + uniqueHashes?: number; /** @deprecated Use totalDiscovered instead. Kept for backward compatibility. */ awaitingFastQueue: number; /** Files that completed fast queue but are waiting for background queue */