improve pipeline

packages/meta-sort-core/src/api/UnifiedAPIServer.ts

@@ -396,10 +396,12 @@ export class UnifiedAPIServer {
         const backgroundQueueRunning = queueStatus.backgroundQueue?.running || 0;
         const backgroundQueuePending = queueStatus.backgroundQueue?.pending || 0;
 
-        // "Discovered" now represents files waiting for fast queue (validated but not yet processing)
+        // "Discovered" now represents unique files waiting for fast queue (validated but not yet processing)
-        // This merges the old "discovered" and "awaitingFastQueue" concepts
+        // Use file count from container scheduler if available, otherwise fall back to task count
-        snapshot.totalDiscovered = fastQueuePending;
+        const filesStatus = queueStatus.files;
-        snapshot.awaitingFastQueue = fastQueuePending; // Keep for backward compatibility
+        const uniqueFilesWaiting = filesStatus ? (filesStatus.fastWaiting + filesStatus.fastRunning) : fastQueuePending;
+        snapshot.totalDiscovered = uniqueFilesWaiting;
+        snapshot.awaitingFastQueue = uniqueFilesWaiting; // Keep for backward compatibility
 
         // Fix awaitingBackground: subtract files currently being processed by background workers
         // hashProcessing.size includes both waiting AND running files
@@ -444,6 +446,14 @@ export class UnifiedAPIServer {
       const metrics = performanceMetrics.getMetrics();
       snapshot.totalFailed = metrics.totalFailedFiles || 0;
 
+      // Add filtered count from pipeline stats
+      if (this.streamingPipeline) {
+        const pipelineStats = this.streamingPipeline.getStats();
+        snapshot.totalFiltered = pipelineStats.filtered || 0;
+      } else {
+        snapshot.totalFiltered = 0;
+      }
+
       return snapshot;
     });
 

packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts

@@ -57,6 +57,14 @@ export interface UnifiedProcessingSnapshot {
   /** Files that completed fast queue but are waiting for background queue */
   awaitingBackground: number;
 
+  // Filtering/duplicate metrics
+  /** Files filtered out due to unsupported extensions (non-media files) */
+  totalFiltered?: number;
+  /** Files with same content hash as previously processed file */
+  totalDuplicates?: number;
+  /** Number of unique content hashes seen */
+  uniqueHashes?: number;
+
   // Configuration
   fastQueueConcurrency?: number;       // Number of fast queue workers
   backgroundQueueConcurrency?: number; // Number of background queue workers
@@ -70,6 +78,10 @@ export class UnifiedProcessingStateManager {
   private maxDoneHistory = 100; // Keep last 100 completed files
   private totalProcessedCount = 0; // Track total number of completed files (not limited)
 
+  // Duplicate tracking
+  private seenHashes: Set<string> = new Set();
+  private duplicateCount = 0; // Files with same hash as a previously processed file
+
   /**
    * Add a file to discovered state (file discovered)
    */
@@ -126,6 +138,13 @@ export class UnifiedProcessingStateManager {
       return;
     }
 
+    // Track duplicates - if hash already seen, increment duplicate counter
+    if (this.seenHashes.has(hash)) {
+      this.duplicateCount++;
+    } else {
+      this.seenHashes.add(hash);
+    }
+
     // Move to hash processing queue
     this.hashProcessing.set(filePath, {
       filePath,
@@ -273,6 +292,10 @@ export class UnifiedProcessingStateManager {
       // Files that completed fast queue but are waiting for/in background queue
       awaitingBackground: this.hashProcessing.size,
 
+      // Duplicate tracking (filtered count is added by API layer from pipeline stats)
+      totalDuplicates: this.duplicateCount,
+      uniqueHashes: this.seenHashes.size,
+
       fastQueueConcurrency,
       backgroundQueueConcurrency
     };
@@ -296,6 +319,8 @@ export class UnifiedProcessingStateManager {
     this.hashProcessing.clear();
     this.done = [];
     this.totalProcessedCount = 0;
+    this.seenHashes.clear();
+    this.duplicateCount = 0;
   }
 
   /**
@@ -366,4 +391,14 @@ export class UnifiedProcessingStateManager {
   getHashProcessingFiles(): Map<string, UnifiedFileState> {
     return this.hashProcessing;
   }
+
+  /**
+   * Get duplicate tracking statistics
+   */
+  getDuplicateStats(): { duplicateCount: number; uniqueHashes: number } {
+    return {
+      duplicateCount: this.duplicateCount,
+      uniqueHashes: this.seenHashes.size
+    };
+  }
 }

packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts

@@ -20,6 +20,7 @@ export class StreamingPipeline {
 
     private discoveredCount = 0;
     private validatedCount = 0;
+    private filteredCount = 0;  // Files filtered out (unsupported extensions)
     private fastProcessedCount = 0;
     private backgroundProcessedCount = 0;
 
@@ -106,6 +107,7 @@ export class StreamingPipeline {
             const ext = path.extname(filePath).toLowerCase();
             if (!this.config.supportedExtensions.has(ext)) {
                 // Skip unsupported file types - remove from discovered
+                this.filteredCount++;
                 this.config.stateManager.removeFile(filePath);
                 return;
             }
@@ -353,6 +355,7 @@ export class StreamingPipeline {
         return {
             discovered: this.discoveredCount,
             validated: this.validatedCount,
+            filtered: this.filteredCount,
             fastProcessed: this.fastProcessedCount,
             backgroundProcessed: this.backgroundProcessedCount,
             queues: {
@@ -396,6 +399,7 @@ export class StreamingPipeline {
         // Step 2: Reset counters
         this.discoveredCount = 0;
         this.validatedCount = 0;
+        this.filteredCount = 0;
         this.fastProcessedCount = 0;
         this.backgroundProcessedCount = 0;
 

packages/meta-sort-ui/src/pages/Monitor.tsx

@@ -10,7 +10,6 @@ import {
 import {
     formatMs,
     formatNumber,
-    formatUptime,
     formatBytes,
     getFilename
 } from '../utils/format';
@@ -21,6 +20,7 @@ function Monitor() {
     const [redisStats, setRedisStats] = useState<RedisStats | null>(null);
     const [queue, setQueue] = useState<QueueItem[]>([]);
     const [failedFiles, setFailedFiles] = useState<FailedFile[]>([]);
+    const [showPipelineHelp, setShowPipelineHelp] = useState(false);
 
     const fetchData = useCallback(async () => {
         try {
@@ -152,7 +152,26 @@ function Monitor() {
             {/* Pipeline Progress Bar - 6 Stage (including failed) */}
             {status && (
                 <div className="card pipeline-section">
+                    <div className="pipeline-header">
                         <h2>Processing Pipeline</h2>
+                        <button
+                            className="help-button"
+                            onClick={() => setShowPipelineHelp(!showPipelineHelp)}
+                            title="What do these mean?"
+                        >
+                            ?
+                        </button>
+                    </div>
+                    {showPipelineHelp && (
+                        <div className="pipeline-help">
+                            <div className="help-item"><span className="help-dot discovered"></span><strong>Discovered:</strong> Files found on disk, waiting to be processed.</div>
+                            <div className="help-item"><span className="help-dot fast"></span><strong>Fast Queue:</strong> Files being processed for quick metadata extraction (hash, ffmpeg, etc.).</div>
+                            <div className="help-item"><span className="help-dot awaiting-background"></span><strong>Awaiting BG:</strong> Files done with fast processing, queued for slower tasks.</div>
+                            <div className="help-item"><span className="help-dot background"></span><strong>Background:</strong> Files running slow tasks like full file hashing or TMDB lookups.</div>
+                            <div className="help-item"><span className="help-dot done"></span><strong>Complete:</strong> Files fully processed and available in the virtual filesystem.</div>
+                            <div className="help-item"><span className="help-dot failed"></span><strong>Failed:</strong> Files that encountered errors during processing.</div>
+                        </div>
+                    )}
                     <div className="pipeline-bar-container">
                         <div className="pipeline-bar">
                             <div className="bar-segment discovered" style={{ width: `${progress.discovered}%` }} />
@@ -319,27 +338,23 @@ function Monitor() {
                         </div>
                     )}
 
-                    <h3 className="section-title">System Stats</h3>
+                    <h3 className="section-title">File Coverage</h3>
                     <div className="stats-grid">
                         <div className="stat-box">
-                            <div className="stat-value">{redisStats ? formatNumber(redisStats.fileCount) : '-'}</div>
+                            <div className="stat-value">{status?.uniqueHashes ? formatNumber(status.uniqueHashes) : (redisStats ? formatNumber(redisStats.fileCount) : '-')}</div>
-                            <div className="stat-label">Files in Redis</div>
+                            <div className="stat-label">Unique Files</div>
                         </div>
                         <div className="stat-box">
-                            <div className="stat-value">{redisStats?.memoryUsage || '-'}</div>
+                            <div className="stat-value">{status?.totalFiltered !== undefined ? formatNumber(status.totalFiltered) : '-'}</div>
-                            <div className="stat-label">Redis Memory</div>
+                            <div className="stat-label">Filtered (Non-Media)</div>
                         </div>
                         <div className="stat-box">
-                            <div className="stat-value">{metrics ? formatUptime(metrics.uptime) : '-'}</div>
+                            <div className="stat-value">{status?.totalDuplicates !== undefined ? formatNumber(status.totalDuplicates) : '-'}</div>
-                            <div className="stat-label">Uptime</div>
+                            <div className="stat-label">Duplicates (Same Hash)</div>
                         </div>
                         <div className="stat-box">
-                            <div className="stat-value">
+                            <div className="stat-value">{status?.totalFailed !== undefined ? formatNumber(status.totalFailed) : '-'}</div>
-                                {metrics?.memoryUsage
+                            <div className="stat-label">Failed</div>
-                                    ? `${Math.round(metrics.memoryUsage.heapUsed / 1024 / 1024)} MB`
-                                    : '-'}
-                            </div>
-                            <div className="stat-label">Node Heap</div>
                         </div>
                     </div>
                 </div>
@@ -517,9 +532,71 @@ function Monitor() {
                 }
 
                 .pipeline-section h2 {
+                    margin-bottom: 0;
+                }
+
+                .pipeline-header {
+                    display: flex;
+                    align-items: center;
+                    gap: 10px;
                     margin-bottom: 16px;
                 }
 
+                .help-button {
+                    width: 20px;
+                    height: 20px;
+                    border-radius: 50%;
+                    border: 1px solid var(--text-secondary);
+                    background: transparent;
+                    color: var(--text-secondary);
+                    font-size: 12px;
+                    font-weight: 600;
+                    cursor: pointer;
+                    display: flex;
+                    align-items: center;
+                    justify-content: center;
+                    transition: all 0.2s ease;
+                }
+
+                .help-button:hover {
+                    background: var(--text-secondary);
+                    color: var(--bg-primary);
+                }
+
+                .pipeline-help {
+                    background: var(--bg-tertiary);
+                    border-radius: 8px;
+                    padding: 12px 16px;
+                    margin-bottom: 16px;
+                    font-size: 0.85rem;
+                    line-height: 1.6;
+                }
+
+                .help-item {
+                    display: flex;
+                    align-items: center;
+                    gap: 8px;
+                    margin-bottom: 6px;
+                }
+
+                .help-item:last-child {
+                    margin-bottom: 0;
+                }
+
+                .help-dot {
+                    width: 10px;
+                    height: 10px;
+                    border-radius: 50%;
+                    flex-shrink: 0;
+                }
+
+                .help-dot.discovered { background: #9ca3af; }
+                .help-dot.fast { background: #fbbf24; }
+                .help-dot.awaiting-background { background: #f472b6; }
+                .help-dot.background { background: #a78bfa; }
+                .help-dot.done { background: #34d399; }
+                .help-dot.failed { background: #f87171; }
+
                 .pipeline-bar-container {
                     margin-bottom: 16px;
                 }

packages/meta-sort-ui/src/types/index.ts

@@ -89,6 +89,12 @@ export interface ProcessingStatus {
     totalDone: number;
     /** Total number of failed files */
     totalFailed?: number;
+    /** Files filtered out due to unsupported extensions (non-media files) */
+    totalFiltered?: number;
+    /** Files with same content hash as previously processed file */
+    totalDuplicates?: number;
+    /** Number of unique content hashes seen */
+    uniqueHashes?: number;
     /** @deprecated Use totalDiscovered instead. Kept for backward compatibility. */
     awaitingFastQueue: number;
     /** Files that completed fast queue but are waiting for background queue */