initial commit

packages/meta-sort-core/.env.example

@@ -0,0 +1,102 @@
+# meta-sort-core Environment Configuration
+# Copy this file to .env and adjust values as needed
+
+# =============================================================================
+# Required Settings
+# =============================================================================
+
+# Comma-separated list of folders to watch for media files (REQUIRED)
+WATCH_FOLDER_LIST=/files
+
+# =============================================================================
+# Volume Paths (New Architecture)
+# =============================================================================
+
+# Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services
+# This is where Redis data, leader election locks, and service discovery files are stored
+META_CORE_PATH=/meta-core
+
+# Path to FILES_VOLUME - shared media volume
+# All file paths in the KV database are relative to this path
+FILES_PATH=/files
+
+# =============================================================================
+# KV Storage Configuration
+# =============================================================================
+
+# Use legacy etcd mode instead of Redis (default: false)
+# Set to 'true' to use etcd instead of Redis
+USE_ETCD=false
+
+# etcd API URL (only used if USE_ETCD=true)
+# ETCD_URL=http://localhost:2379
+
+# Direct Redis URL (skip leader election if provided)
+# If set, the service will connect directly to this Redis instance
+# instead of participating in leader election
+# REDIS_URL=redis://localhost:6379
+
+# Redis port for leader to listen on (default: 6379)
+REDIS_PORT=6379
+
+# =============================================================================
+# Service Identity
+# =============================================================================
+
+# Service name for leader election and service discovery
+SERVICE_NAME=meta-sort
+
+# Service version
+SERVICE_VERSION=1.0.0
+
+# =============================================================================
+# API Server
+# =============================================================================
+
+# FUSE API server port (default: 3000)
+FUSE_API_PORT=3000
+
+# FUSE API server host (default: localhost)
+FUSE_API_HOST=0.0.0.0
+
+# =============================================================================
+# Processing Configuration
+# =============================================================================
+
+# Maximum number of worker threads for Piscina worker pool
+# Default: CPU count
+# MAX_WORKER_THREADS=16
+
+# Interval in milliseconds for polling watch folders (0 to disable)
+# WATCH_FOLDER_POOLING_INTERVAL_MS=0
+
+# Metadata formats to generate: 'meta' (.meta YAML), 'jellyfin' (.nfo XML)
+METADATA_FORMATS=meta
+
+# =============================================================================
+# Cache Configuration
+# =============================================================================
+
+# Path to hash index cache
+INDEX_FOLDER_PATH=/data/cache/hash
+
+# Path to plugin cache folder
+CACHE_FOLDER_PATH=/data/cache/plugins
+
+# =============================================================================
+# Jellyfin Integration (Optional)
+# =============================================================================
+
+# Jellyfin server endpoint
+# JELLYFIN_ENDPOINT=http://localhost:8096
+
+# Jellyfin API key
+# JELLYFIN_API_KEY=your-api-key-here
+
+# =============================================================================
+# Polling Paths (Optional)
+# =============================================================================
+
+# JSON array of polling path configurations for paths that don't support inotify
+# Example: '[{"path": "/files/smb/downloads", "intervalMs": 60000}]'
+# POLLING_PATHS=[]

packages/meta-sort-core/package.json

@@ -0,0 +1,53 @@
+{
+  "name": "@meta-sort/core",
+  "version": "1.0.0",
+  "description": "Core file sorting and metadata extraction service",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "exec": "node dist/index.js",
+    "test": "mocha",
+    "lint": "eslint src --ext ts"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@fastify/cors": "^11.1.0",
+    "@fastify/static": "^8.3.0",
+    "@metazla/filename-tools": "github:worph/filename-tool",
+    "@metazla/meta-hash": "workspace:*",
+    "@metazla/meta-interface": "workspace:*",
+    "@worph/async-utils": "workspace:*",
+    "axios": "^1.6.7",
+    "bencode": "^4.0.0",
+    "chokidar": "^3.5.3",
+    "dockerode": "^4.0.2",
+    "dotenv": "^16.4.1",
+    "fastify": "^5.6.1",
+    "fluent-ffmpeg": "^2.1.2",
+    "franc-min": "^6.2.0",
+    "ioredis": "^5.4.1",
+    "iso-639-1": "^3.1.2",
+    "lodash": "^4.17.21",
+    "multiformats": "^13.1.0",
+    "p-queue": "^8.0.1",
+    "piscina": "^4.4.0",
+    "tslog": "^4.9.2",
+    "wanakana": "^5.3.1",
+    "xml2js": "^0.6.2",
+    "yaml": "^2.4.3"
+  },
+  "devDependencies": {
+    "@types/chai": "^4.3.11",
+    "@types/dockerode": "^3.3.31",
+    "@types/fluent-ffmpeg": "^2.1.24",
+    "@types/lodash": "^4.17.0",
+    "@types/mocha": "^10.0.6",
+    "@types/node": "^20.11.16",
+    "chai": "^5.0.3",
+    "mocha": "^10.2.0",
+    "typescript": "^5.3.3"
+  }
+}

packages/meta-sort-core/src/api/DuplicatesAPI.ts

@@ -0,0 +1,166 @@
+/**
+ * HTTP REST API Server for Duplicate Files
+ *
+ * Provides endpoints for accessing duplicate file detection results.
+ * Separate from FUSE API to maintain proper separation of concerns.
+ */
+
+import * as http from 'http';
+
+export interface DuplicatesAPIConfig {
+  /** Port to listen on (default: 3003) */
+  port?: number;
+  /** Host to bind to (default: 'localhost') */
+  host?: string;
+  /** Enable CORS (default: true) */
+  enableCors?: boolean;
+  /** Mount path for duplicates endpoint (default: '/duplicates') */
+  mountPath?: string;
+}
+
+export class DuplicatesAPI {
+  private server: http.Server | null = null;
+  private config: Required<DuplicatesAPIConfig>;
+
+  constructor(config: DuplicatesAPIConfig = {}) {
+    this.config = {
+      port: config.port ?? 3003,
+      host: config.host ?? 'localhost',
+      enableCors: config.enableCors ?? true,
+      mountPath: config.mountPath ?? '/duplicates'
+    };
+  }
+
+  /**
+   * Start the HTTP server
+   */
+  public async start(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.server = http.createServer(this.handleRequest.bind(this));
+
+      this.server.on('error', reject);
+
+      this.server.listen(this.config.port, this.config.host, () => {
+        console.log(`Duplicates API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`);
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Stop the HTTP server
+   */
+  public async stop(): Promise<void> {
+    if (!this.server) {
+      return;
+    }
+
+    return new Promise((resolve, reject) => {
+      this.server!.close((err) => {
+        if (err) {
+          reject(err);
+        } else {
+          console.log('Duplicates API stopped');
+          this.server = null;
+          resolve();
+        }
+      });
+    });
+  }
+
+  /**
+   * Handle incoming HTTP requests
+   */
+  private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    // CORS headers
+    if (this.config.enableCors) {
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+      if (req.method === 'OPTIONS') {
+        res.writeHead(204);
+        res.end();
+        return;
+      }
+    }
+
+    try {
+      const url = new URL(req.url!, `http://${req.headers.host}`);
+      const pathname = url.pathname;
+      const method = req.method!;
+
+      // Route the request
+      if (method === 'GET' && pathname === this.config.mountPath) {
+        await this.handleDuplicates(res);
+      } else if (method === 'GET' && pathname === '/health') {
+        this.handleHealth(res);
+      } else {
+        this.sendError(res, 404, 'Not Found');
+      }
+    } catch (error) {
+      console.error('Error handling request:', error);
+      this.sendError(res, 500, 'Internal Server Error', error);
+    }
+  }
+
+  /**
+   * Send JSON response
+   */
+  private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
+    res.writeHead(statusCode, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(data, null, 2));
+  }
+
+  /**
+   * Send error response
+   */
+  private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
+    this.sendJson(res, {
+      error: message,
+      details: error?.message ?? error
+    }, statusCode);
+  }
+
+  /**
+   * GET /duplicates
+   * Get duplicate files
+   * NOTE: This API is deprecated. Use UnifiedAPIServer's /api/duplicates endpoint instead.
+   * Disk-based duplicate logging has been removed for performance.
+   */
+  private async handleDuplicates(res: http.ServerResponse): Promise<void> {
+    try {
+      // Return empty results - duplicate detection now happens in-memory via UnifiedAPIServer
+      const hashDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
+      const titleDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
+
+      this.sendJson(res, {
+        hashDuplicates,
+        titleDuplicates,
+        stats: {
+          hashGroupCount: 0,
+          hashFileCount: 0,
+          titleGroupCount: 0,
+          titleFileCount: 0
+        },
+        deprecated: true,
+        message: 'This API is deprecated. Use UnifiedAPIServer /api/duplicates endpoint instead.'
+      });
+    } catch (error) {
+      console.error('Error in duplicates endpoint:', error);
+      this.sendError(res, 500, 'Failed to get duplicate data', error);
+    }
+  }
+
+  /**
+   * GET /health
+   * Health check endpoint
+   */
+  private handleHealth(res: http.ServerResponse): void {
+    this.sendJson(res, {
+      status: 'ok',
+      service: 'duplicates-api',
+      timestamp: new Date().toISOString()
+    });
+  }
+}

packages/meta-sort-core/src/api/FuseAPI.ts

@@ -0,0 +1,173 @@
+/**
+ * FUSE API for meta-mesh
+ *
+ * This API provides a virtual filesystem interface that can be consumed by a FUSE driver.
+ * It exposes the computed folder structure without creating actual hardlinks.
+ */
+
+import { HashMeta } from '@metazla/meta-interface';
+import { Stats } from 'fs';
+
+/**
+ * Represents a file or directory in the virtual filesystem
+ */
+export interface VirtualNode {
+  /** Virtual path in the output filesystem */
+  path: string;
+  /** Node type */
+  type: 'file' | 'directory';
+  /** File name */
+  name: string;
+  /** Parent directory path */
+  parent: string;
+}
+
+/**
+ * Represents a regular file in the virtual filesystem
+ */
+export interface VirtualFile extends VirtualNode {
+  type: 'file';
+  /** Source file path on the real filesystem (null for virtual files like .meta) */
+  sourcePath: string | null;
+  /** File size in bytes */
+  size: number;
+  /** Last modified time */
+  mtime: Date;
+  /** Creation time */
+  ctime: Date;
+  /** File permissions (octal) */
+  mode: number;
+  /** Associated metadata */
+  metadata?: HashMeta;
+}
+
+/**
+ * Represents a directory in the virtual filesystem
+ */
+export interface VirtualDirectory extends VirtualNode {
+  type: 'directory';
+  /** Child nodes (file/directory names) */
+  children: string[];
+  /** Directory permissions (octal) */
+  mode: number;
+  /** Last modified time */
+  mtime: Date;
+}
+
+/**
+ * File attributes returned by getattr() - compatible with FUSE stat structure
+ */
+export interface FileAttributes {
+  /** File size in bytes */
+  size: number;
+  /** File mode (permissions + type) */
+  mode: number;
+  /** Last modified time (Unix timestamp in seconds) */
+  mtime: number;
+  /** Last access time (Unix timestamp in seconds) */
+  atime: number;
+  /** Creation/change time (Unix timestamp in seconds) */
+  ctime: number;
+  /** Number of hard links */
+  nlink: number;
+  /** User ID */
+  uid: number;
+  /** Group ID */
+  gid: number;
+}
+
+/**
+ * Read result for file content
+ */
+export interface ReadResult {
+  /** For regular files: source path to read from. For virtual files: null */
+  sourcePath: string | null;
+  /** For virtual files: generated content. For regular files: null */
+  content: Buffer | null;
+  /** Total file size */
+  size: number;
+}
+
+/**
+ * FUSE API operations
+ */
+export interface IFuseAPI {
+  /**
+   * List directory contents
+   * @param path Virtual directory path (e.g., "/Anime/Naruto/Season 1")
+   * @returns Array of file/directory names in the directory, or null if not found
+   */
+  readdir(path: string): string[] | null;
+
+  /**
+   * Get file or directory attributes
+   * @param path Virtual file or directory path
+   * @returns File attributes, or null if not found
+   */
+  getattr(path: string): FileAttributes | null;
+
+  /**
+   * Check if a path exists
+   * @param path Virtual path
+   * @returns true if path exists, false otherwise
+   */
+  exists(path: string): boolean;
+
+  /**
+   * Get read information for a file
+   * @param path Virtual file path
+   * @returns Read result with source path or content, or null if not found
+   */
+  read(path: string): ReadResult | null;
+
+  /**
+   * Get metadata for a file
+   * @param path Virtual file path
+   * @returns HashMeta if available, null otherwise
+   */
+  getMetadata(path: string): HashMeta | null;
+
+  /**
+   * Get the complete virtual filesystem tree
+   * @returns Root directory node
+   */
+  getTree(): VirtualDirectory;
+
+  /**
+   * Refresh the virtual filesystem from current state
+   * Called after processing completes to update the VFS
+   */
+  refresh(): Promise<void>;
+}
+
+/**
+ * Events emitted by the FUSE API
+ */
+export interface FuseAPIEvents {
+  /** Emitted when the virtual filesystem is updated */
+  'vfs-updated': () => void;
+  /** Emitted when a file is added */
+  'file-added': (path: string) => void;
+  /** Emitted when a file is updated (e.g., metadata changed) */
+  'file-updated': (path: string) => void;
+  /** Emitted when a file is removed */
+  'file-removed': (path: string) => void;
+  /** Emitted when a directory is added */
+  'directory-added': (path: string) => void;
+}
+
+/**
+ * Configuration for FUSE API
+ */
+export interface FuseAPIConfig {
+  /** Base path for the virtual filesystem (default: "/") */
+  basePath?: string;
+  /** Default file permissions (default: 0o644) */
+  fileMode?: number;
+  /** Default directory permissions (default: 0o755) */
+  directoryMode?: number;
+  /** User ID for files (default: process.getuid()) */
+  uid?: number;
+  /** Group ID for files (default: process.getgid()) */
+  gid?: number;
+}

packages/meta-sort-core/src/api/FuseAPIServer.ts

@@ -0,0 +1,345 @@
+/**
+ * HTTP REST API Server for FUSE operations
+ *
+ * Exposes the VirtualFileSystem over HTTP for consumption by a FUSE driver.
+ */
+
+import * as http from 'http';
+import { VirtualFileSystem } from './VirtualFileSystem.js';
+import { FileAttributes, ReadResult } from './FuseAPI.js';
+
+export interface FuseAPIServerConfig {
+  /** Port to listen on (default: 3000) */
+  port?: number;
+  /** Host to bind to (default: 'localhost') */
+  host?: string;
+  /** Enable CORS (default: false) */
+  enableCors?: boolean;
+}
+
+export class FuseAPIServer {
+  private server: http.Server | null = null;
+  private vfs: VirtualFileSystem;
+  private config: Required<FuseAPIServerConfig>;
+
+  constructor(vfs: VirtualFileSystem, config: FuseAPIServerConfig = {}) {
+    this.vfs = vfs;
+    this.config = {
+      port: config.port ?? 3000,
+      host: config.host ?? 'localhost',
+      enableCors: config.enableCors ?? false
+    };
+  }
+
+  /**
+   * Start the HTTP server
+   */
+  public async start(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.server = http.createServer(this.handleRequest.bind(this));
+
+      this.server.on('error', reject);
+
+      this.server.listen(this.config.port, this.config.host, () => {
+        console.log(`FUSE API Server listening on http://${this.config.host}:${this.config.port}`);
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Stop the HTTP server
+   */
+  public async stop(): Promise<void> {
+    if (!this.server) {
+      return;
+    }
+
+    return new Promise((resolve, reject) => {
+      this.server!.close((err) => {
+        if (err) {
+          reject(err);
+        } else {
+          console.log('FUSE API Server stopped');
+          this.server = null;
+          resolve();
+        }
+      });
+    });
+  }
+
+  /**
+   * Handle incoming HTTP requests
+   */
+  private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    // CORS headers
+    if (this.config.enableCors) {
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+      if (req.method === 'OPTIONS') {
+        res.writeHead(204);
+        res.end();
+        return;
+      }
+    }
+
+    try {
+      const url = new URL(req.url!, `http://${req.headers.host}`);
+      const pathname = url.pathname;
+      const method = req.method!;
+
+      // Route the request
+      if (method === 'GET' && pathname === '/api/fuse/health') {
+        this.handleHealth(res);
+      } else if (method === 'GET' && pathname === '/api/fuse/stats') {
+        this.handleStats(res);
+      } else if (method === 'POST' && pathname === '/api/fuse/readdir') {
+        await this.handleReaddir(req, res);
+      } else if (method === 'POST' && pathname === '/api/fuse/getattr') {
+        await this.handleGetattr(req, res);
+      } else if (method === 'POST' && pathname === '/api/fuse/exists') {
+        await this.handleExists(req, res);
+      } else if (method === 'POST' && pathname === '/api/fuse/read') {
+        await this.handleRead(req, res);
+      } else if (method === 'POST' && pathname === '/api/fuse/metadata') {
+        await this.handleMetadata(req, res);
+      } else if (method === 'GET' && pathname === '/api/fuse/tree') {
+        this.handleTree(res);
+      } else if (method === 'GET' && pathname === '/api/fuse/files') {
+        this.handleFiles(res);
+      } else if (method === 'GET' && pathname === '/api/fuse/directories') {
+        this.handleDirectories(res);
+      } else if (method === 'POST' && pathname === '/api/fuse/refresh') {
+        await this.handleRefresh(req, res);
+      } else {
+        this.sendError(res, 404, 'Not Found');
+      }
+    } catch (error) {
+      console.error('Error handling request:', error);
+      this.sendError(res, 500, 'Internal Server Error', error);
+    }
+  }
+
+  /**
+   * Parse JSON body from request
+   */
+  private async parseBody(req: http.IncomingMessage): Promise<any> {
+    return new Promise((resolve, reject) => {
+      let body = '';
+      req.on('data', (chunk) => {
+        body += chunk.toString();
+      });
+      req.on('end', () => {
+        try {
+          resolve(JSON.parse(body));
+        } catch (error) {
+          reject(new Error('Invalid JSON'));
+        }
+      });
+      req.on('error', reject);
+    });
+  }
+
+  /**
+   * Send JSON response
+   */
+  private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
+    res.writeHead(statusCode, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(data));
+  }
+
+  /**
+   * Send error response
+   */
+  private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
+    this.sendJson(res, {
+      error: message,
+      details: error?.message ?? error
+    }, statusCode);
+  }
+
+  // API Handlers
+
+  /**
+   * GET /api/health
+   * Health check endpoint
+   */
+  private handleHealth(res: http.ServerResponse): void {
+    this.sendJson(res, {
+      status: 'ok',
+      timestamp: new Date().toISOString()
+    });
+  }
+
+  /**
+   * GET /api/stats
+   * Get VFS statistics
+   */
+  private handleStats(res: http.ServerResponse): void {
+    const stats = this.vfs.getStats();
+    this.sendJson(res, stats);
+  }
+
+  /**
+   * POST /api/readdir
+   * List directory contents
+   * Body: { path: string }
+   * Response: { entries: string[] } | { error: string }
+   */
+  private async handleReaddir(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const { path } = await this.parseBody(req);
+
+    if (typeof path !== 'string') {
+      this.sendError(res, 400, 'Missing or invalid "path" parameter');
+      return;
+    }
+
+    const entries = this.vfs.readdir(path);
+
+    if (entries === null) {
+      this.sendError(res, 404, 'Directory not found');
+      return;
+    }
+
+    this.sendJson(res, { entries });
+  }
+
+  /**
+   * POST /api/getattr
+   * Get file attributes
+   * Body: { path: string }
+   * Response: FileAttributes | { error: string }
+   */
+  private async handleGetattr(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const { path } = await this.parseBody(req);
+
+    if (typeof path !== 'string') {
+      this.sendError(res, 400, 'Missing or invalid "path" parameter');
+      return;
+    }
+
+    const attrs = this.vfs.getattr(path);
+
+    if (attrs === null) {
+      this.sendError(res, 404, 'Path not found');
+      return;
+    }
+
+    this.sendJson(res, attrs);
+  }
+
+  /**
+   * POST /api/exists
+   * Check if path exists
+   * Body: { path: string }
+   * Response: { exists: boolean }
+   */
+  private async handleExists(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const { path } = await this.parseBody(req);
+
+    if (typeof path !== 'string') {
+      this.sendError(res, 400, 'Missing or invalid "path" parameter');
+      return;
+    }
+
+    const exists = this.vfs.exists(path);
+    this.sendJson(res, { exists });
+  }
+
+  /**
+   * POST /api/read
+   * Read file
+   * Body: { path: string }
+   * Response: ReadResult | { error: string }
+   */
+  private async handleRead(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const { path } = await this.parseBody(req);
+
+    if (typeof path !== 'string') {
+      this.sendError(res, 400, 'Missing or invalid "path" parameter');
+      return;
+    }
+
+    const result = this.vfs.read(path);
+
+    if (result === null) {
+      this.sendError(res, 404, 'File not found');
+      return;
+    }
+
+    // Convert Buffer to base64 for JSON serialization
+    const response: any = {
+      sourcePath: result.sourcePath,
+      size: result.size
+    };
+
+    if (result.content !== null) {
+      response.content = result.content.toString('base64');
+      response.contentEncoding = 'base64';
+    }
+
+    this.sendJson(res, response);
+  }
+
+  /**
+   * POST /api/metadata
+   * Get file metadata
+   * Body: { path: string }
+   * Response: HashMeta | { error: string }
+   */
+  private async handleMetadata(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    const { path } = await this.parseBody(req);
+
+    if (typeof path !== 'string') {
+      this.sendError(res, 400, 'Missing or invalid "path" parameter');
+      return;
+    }
+
+    const metadata = this.vfs.getMetadata(path);
+
+    if (metadata === null) {
+      this.sendError(res, 404, 'Metadata not found');
+      return;
+    }
+
+    this.sendJson(res, metadata);
+  }
+
+  /**
+   * GET /api/tree
+   * Get complete VFS tree
+   */
+  private handleTree(res: http.ServerResponse): void {
+    const tree = this.vfs.getTree();
+    this.sendJson(res, tree);
+  }
+
+  /**
+   * GET /api/files
+   * Get all file paths
+   */
+  private handleFiles(res: http.ServerResponse): void {
+    const files = this.vfs.getAllFiles();
+    this.sendJson(res, { files });
+  }
+
+  /**
+   * GET /api/directories
+   * Get all directory paths
+   */
+  private handleDirectories(res: http.ServerResponse): void {
+    const directories = this.vfs.getAllDirectories();
+    this.sendJson(res, { directories });
+  }
+
+  /**
+   * POST /api/refresh
+   * Refresh the VFS (no-op, VFS is updated by processing pipeline)
+   */
+  private async handleRefresh(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    await this.vfs.refresh();
+    this.sendJson(res, { status: 'ok' });
+  }
+}

packages/meta-sort-core/src/api/MetricsAPI.ts

@@ -0,0 +1,160 @@
+/**
+ * HTTP REST API Server for Performance Metrics
+ *
+ * Provides a dedicated endpoint for exposing application performance metrics.
+ * Separate from FUSE API to maintain proper separation of concerns.
+ */
+
+import * as http from 'http';
+import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
+
+export interface MetricsAPIConfig {
+  /** Port to listen on (default: 3001) */
+  port?: number;
+  /** Host to bind to (default: 'localhost') */
+  host?: string;
+  /** Enable CORS (default: true for metrics) */
+  enableCors?: boolean;
+  /** Mount path for metrics endpoint (default: '/metrics') */
+  mountPath?: string;
+}
+
+export class MetricsAPI {
+  private server: http.Server | null = null;
+  private config: Required<MetricsAPIConfig>;
+
+  constructor(config: MetricsAPIConfig = {}) {
+    this.config = {
+      port: config.port ?? 3001,
+      host: config.host ?? 'localhost',
+      enableCors: config.enableCors ?? true,
+      mountPath: config.mountPath ?? '/metrics'
+    };
+  }
+
+  /**
+   * Start the HTTP server
+   */
+  public async start(): Promise<void> {
+    return new Promise((resolve, reject) => {
+      this.server = http.createServer(this.handleRequest.bind(this));
+
+      this.server.on('error', reject);
+
+      this.server.listen(this.config.port, this.config.host, () => {
+        console.log(`Metrics API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`);
+        resolve();
+      });
+    });
+  }
+
+  /**
+   * Stop the HTTP server
+   */
+  public async stop(): Promise<void> {
+    if (!this.server) {
+      return;
+    }
+
+    return new Promise((resolve, reject) => {
+      this.server!.close((err) => {
+        if (err) {
+          reject(err);
+        } else {
+          console.log('Metrics API stopped');
+          this.server = null;
+          resolve();
+        }
+      });
+    });
+  }
+
+  /**
+   * Handle incoming HTTP requests
+   */
+  private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
+    // CORS headers
+    if (this.config.enableCors) {
+      res.setHeader('Access-Control-Allow-Origin', '*');
+      res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
+      res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
+
+      if (req.method === 'OPTIONS') {
+        res.writeHead(204);
+        res.end();
+        return;
+      }
+    }
+
+    try {
+      const url = new URL(req.url!, `http://${req.headers.host}`);
+      const pathname = url.pathname;
+      const method = req.method!;
+
+      // Route the request
+      if (method === 'GET' && pathname === this.config.mountPath) {
+        this.handleMetrics(res);
+      } else if (method === 'GET' && pathname === '/health') {
+        this.handleHealth(res);
+      } else if (method === 'POST' && pathname === '/reset') {
+        this.handleReset(res);
+      } else {
+        this.sendError(res, 404, 'Not Found');
+      }
+    } catch (error) {
+      console.error('Error handling request:', error);
+      this.sendError(res, 500, 'Internal Server Error', error);
+    }
+  }
+
+  /**
+   * Send JSON response
+   */
+  private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
+    res.writeHead(statusCode, { 'Content-Type': 'application/json' });
+    res.end(JSON.stringify(data, null, 2));
+  }
+
+  /**
+   * Send error response
+   */
+  private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
+    this.sendJson(res, {
+      error: message,
+      details: error?.message ?? error
+    }, statusCode);
+  }
+
+  /**
+   * GET /metrics
+   * Get performance metrics
+   */
+  private handleMetrics(res: http.ServerResponse): void {
+    const metrics = performanceMetrics.getMetrics();
+    this.sendJson(res, metrics);
+  }
+
+  /**
+   * GET /health
+   * Health check endpoint
+   */
+  private handleHealth(res: http.ServerResponse): void {
+    this.sendJson(res, {
+      status: 'ok',
+      service: 'metrics-api',
+      timestamp: new Date().toISOString()
+    });
+  }
+
+  /**
+   * POST /reset
+   * Reset all metrics
+   */
+  private handleReset(res: http.ServerResponse): void {
+    performanceMetrics.reset();
+    this.sendJson(res, {
+      status: 'ok',
+      message: 'Metrics reset successfully'
+    });
+  }
+}

packages/meta-sort-core/src/api/README.md

@@ -0,0 +1,299 @@
+# FUSE API for Meta-Mesh
+
+This module provides a FUSE-compatible HTTP API that runs alongside meta-mesh's hardlink system. The API exposes a virtual filesystem that can be consumed by a future FUSE driver.
+
+## Overview
+
+The FUSE API runs **always-on alongside hardlinks**, providing:
+
+- **Dual output**: Both hardlinks (for local access) and virtual filesystem (for FUSE driver)
+- **Dynamic updates**: VirtualFileSystem rebuilds on every processing cycle
+- **Virtual .meta files**: Metadata files are generated on-the-fly without disk I/O
+- **HTTP REST API**: Can be consumed by local or remote FUSE drivers
+- **Zero configuration**: Always enabled, no opt-in required
+
+## Architecture
+
+### Components
+
+1. **VirtualFileSystem** - In-memory representation of the output directory structure
+2. **FuseAPIServer** - HTTP REST API server exposing FUSE operations
+3. **FuseAPI** - TypeScript interfaces defining the API contract
+
+### Data Flow
+
+```
+Input Files → File Processing → Metadata Extraction → Duplicate Removal
+    ↓
+Virtual Folder Structure Computation (diff generation)
+    ↓
+    ├─→ VirtualFileSystem Build (in-memory) → FUSE API Server → Future FUSE Driver
+    │
+    └─→ Hardlink Creation (on disk) → Direct File Access
+```
+
+Both outputs are generated in parallel from the same computed folder structure.
+
+## Configuration
+
+The FUSE API is **always enabled** and runs alongside hardlinks. You can optionally configure the server port and host:
+
+```bash
+# FUSE API Server configuration (optional)
+FUSE_API_PORT=3000        # Default: 3000
+FUSE_API_HOST=localhost   # Default: localhost
+```
+
+The API server starts automatically when meta-mesh starts.
+
+## API Endpoints
+
+The FUSE API Server exposes the following HTTP REST endpoints:
+
+### Health & Info
+
+- **GET /api/health** - Health check
+  ```json
+  { "status": "ok", "timestamp": "2025-01-15T10:30:00.000Z" }
+  ```
+
+- **GET /api/stats** - VFS statistics
+  ```json
+  {
+    "fileCount": 1250,
+    "directoryCount": 45,
+    "totalSize": 52428800,
+    "metaFileCount": 1250
+  }
+  ```
+
+### Directory Operations
+
+- **POST /api/readdir** - List directory contents
+  ```json
+  // Request
+  { "path": "/Anime/Naruto/Season 1" }
+
+  // Response
+  { "entries": ["Naruto S01E01.mkv", "Naruto S01E01.mkv.meta", ...] }
+  ```
+
+### File Operations
+
+- **POST /api/getattr** - Get file attributes
+  ```json
+  // Request
+  { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
+
+  // Response
+  {
+    "size": 524288000,
+    "mode": 33188,  // 0o100644 (regular file, rw-r--r--)
+    "mtime": 1705315200,
+    "atime": 1705315200,
+    "ctime": 1705315200,
+    "nlink": 1,
+    "uid": 1000,
+    "gid": 1000
+  }
+  ```
+
+- **POST /api/exists** - Check if path exists
+  ```json
+  // Request
+  { "path": "/Anime/Naruto" }
+
+  // Response
+  { "exists": true }
+  ```
+
+- **POST /api/read** - Read file
+  ```json
+  // Request
+  { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
+
+  // Response (regular file)
+  {
+    "sourcePath": "/data/input/naruto_01.mkv",
+    "size": 524288000
+  }
+
+  // Response (virtual .meta file)
+  {
+    "sourcePath": null,
+    "content": "..base64-encoded content..",
+    "contentEncoding": "base64",
+    "size": 1024
+  }
+  ```
+
+- **POST /api/metadata** - Get file metadata
+  ```json
+  // Request
+  { "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
+
+  // Response
+  {
+    "title": "Naruto",
+    "season": "01",
+    "episode": "01",
+    "anime": true,
+    "sha256": "abc123...",
+    // ... full HashMeta object
+  }
+  ```
+
+### Tree Operations
+
+- **GET /api/tree** - Get complete VFS tree structure
+- **GET /api/files** - Get all file paths
+- **GET /api/directories** - Get all directory paths
+- **POST /api/refresh** - Refresh VFS (no-op, VFS updates automatically)
+
+## Usage Example
+
+### Starting meta-mesh with FUSE API
+
+The FUSE API starts automatically with meta-mesh:
+
+```bash
+# Run meta-mesh (FUSE API starts automatically on port 3000)
+pnpm run start:mesh
+
+# Or customize the port
+export FUSE_API_PORT=8080
+pnpm run start:mesh
+```
+
+You'll see output like:
+```
+FUSE API Server started on http://localhost:3000
+VirtualFileSystem build took 15ms
+VirtualFileSystem: 1250 files (1250 .meta), 45 directories, 50.00 MB
+```
+
+### Consuming the API
+
+```typescript
+import { VirtualFileSystem, FuseAPIServer } from '@metazla/meta-mesh/api';
+
+// Create VFS instance
+const vfs = new VirtualFileSystem({
+  fileMode: 0o644,
+  directoryMode: 0o755,
+  uid: process.getuid(),
+  gid: process.getgid()
+});
+
+// Build from computed structure
+vfs.buildFromComputed(sourceToDestMap, metadataMap);
+
+// Start API server
+const server = new FuseAPIServer(vfs, {
+  port: 3000,
+  host: 'localhost'
+});
+
+await server.start();
+```
+
+### FUSE Driver Integration
+
+A FUSE driver would make HTTP requests to these endpoints:
+
+```typescript
+// Example pseudo-code for FUSE driver
+async function readdir(path: string): Promise<string[]> {
+  const response = await fetch('http://localhost:3000/api/readdir', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ path })
+  });
+
+  const { entries } = await response.json();
+  return entries;
+}
+
+async function read(path: string): Promise<Buffer> {
+  const response = await fetch('http://localhost:3000/api/read', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ path })
+  });
+
+  const result = await response.json();
+
+  if (result.sourcePath) {
+    // Regular file - read from source
+    return fs.readFileSync(result.sourcePath);
+  } else {
+    // Virtual file - use provided content
+    return Buffer.from(result.content, 'base64');
+  }
+}
+```
+
+## Virtual .meta Files
+
+`.meta` files are entirely virtual and generated on-the-fly:
+
+- **No disk I/O**: Content is serialized from in-memory metadata
+- **Always up-to-date**: Reflects current metadata state
+- **YAML format**: Uses `MetaMeshFormat.serialize()` for consistency
+- **No source path**: `sourcePath` is `null`, `content` contains the YAML
+
+## Events
+
+The VirtualFileSystem emits events for monitoring:
+
+```typescript
+vfs.on('vfs-updated', () => {
+  console.log('Virtual filesystem updated');
+});
+
+vfs.on('file-added', (path: string) => {
+  console.log(`File added: ${path}`);
+});
+
+vfs.on('file-removed', (path: string) => {
+  console.log(`File removed: ${path}`);
+});
+
+vfs.on('directory-added', (path: string) => {
+  console.log(`Directory added: ${path}`);
+});
+```
+
+## Performance
+
+The VirtualFileSystem is highly performant:
+
+- **In-memory**: All operations are O(1) or O(n) in-memory
+- **No disk I/O**: Except for reading source files when FUSE driver requests content
+- **Lazy .meta generation**: Content is only serialized when requested
+- **Fast updates**: Rebuilding the VFS takes milliseconds
+
+## Dual Output: Hardlinks + FUSE API
+
+Meta-mesh now provides **both** output methods simultaneously:
+
+| Feature | Hardlinks | FUSE API | Best For |
+|---------|-----------|----------|----------|
+| Disk usage | 1 inode per file | 0 (virtual) | Hardlinks: local use |
+| Update speed | Slow (file I/O) | Fast (in-memory) | FUSE: frequent updates |
+| .meta files | Written to disk | Virtual (on-demand) | FUSE: dynamic metadata |
+| Remote access | No | Yes (HTTP) | FUSE: network drives |
+| Compatibility | Any filesystem | Requires FUSE driver | Hardlinks: broad compat |
+| Access method | Direct file access | HTTP + FUSE driver | Hardlinks: native apps |
+
+**Use hardlinks** for local applications like Jellyfin, Plex, Kodi that read files directly.
+
+**Use FUSE API** for future FUSE drivers that want dynamic, virtual filesystem access.
+
+## Future Enhancements
+
+- **WebSocket support**: Real-time updates for FUSE drivers
+- **Authentication**: API key or JWT-based auth
+- **Compression**: Gzip/Brotli for API responses
+- **Caching**: ETag/Last-Modified headers
+- **Batch operations**: Multi-file operations in single request

packages/meta-sort-core/src/api/VirtualFileSystem.ts

@@ -0,0 +1,627 @@
+/**
+ * VirtualFileSystem - In-memory representation of the virtual output filesystem
+ *
+ * This class maintains the virtual directory structure and provides FUSE-compatible operations.
+ */
+
+import { EventEmitter } from 'events';
+import * as path from 'path';
+import * as fs from 'fs';
+import { HashMeta } from '@metazla/meta-interface';
+import {
+  IFuseAPI,
+  VirtualDirectory,
+  VirtualFile,
+  FileAttributes,
+  ReadResult,
+  FuseAPIConfig,
+  FuseAPIEvents
+} from './FuseAPI.js';
+import { MetaMeshFormat } from '../logic/MetaMeshFormat.js';
+import { MetadataFormatRegistry, IMetadataFormatHandler } from '../logic/MetadataFormatRegistry.js';
+import { config } from '../config/EnvConfig.js';
+
+interface VFSNode {
+  type: 'file' | 'directory';
+  name: string;
+  parent: string | null;
+  // For files
+  sourcePath?: string | null;
+  size?: number;
+  mtime?: Date;
+  ctime?: Date;
+  metadata?: HashMeta;
+  // For virtual metadata files
+  formatHandler?: IMetadataFormatHandler;
+  // For directories
+  children?: Set<string>;
+}
+
+export class VirtualFileSystem extends EventEmitter implements IFuseAPI {
+  private nodes: Map<string, VFSNode> = new Map();
+  private config: Required<FuseAPIConfig>;
+  private metaFormat: MetaMeshFormat = new MetaMeshFormat();
+  private formatRegistry: MetadataFormatRegistry = new MetadataFormatRegistry();
+  private activeFormatHandlers: IMetadataFormatHandler[] = [];
+
+  // Cached stats - updated incrementally on each modification
+  private cachedStats = {
+    fileCount: 0,
+    directoryCount: 0,
+    totalSize: 0,
+    metaFileCount: 0,
+    metadataFileCount: 0
+  };
+
+  constructor(fuseConfig: FuseAPIConfig = {}) {
+    super();
+    this.config = {
+      basePath: fuseConfig.basePath ?? '/',
+      fileMode: fuseConfig.fileMode ?? 0o644,
+      directoryMode: fuseConfig.directoryMode ?? 0o755,
+      uid: fuseConfig.uid ?? process.getuid(),
+      gid: fuseConfig.gid ?? process.getgid()
+    };
+
+    // Initialize active format handlers from configuration
+    this.activeFormatHandlers = this.formatRegistry.getHandlersForFormats(config.METADATA_FORMATS);
+
+    if (this.activeFormatHandlers.length > 0) {
+      console.log(`VirtualFileSystem: Active metadata formats: ${this.activeFormatHandlers.map(h => h.formatType).join(', ')}`);
+    } else {
+      console.log('VirtualFileSystem: No metadata formats configured');
+    }
+
+    // Initialize root directory
+    this.nodes.set('/', {
+      type: 'directory',
+      name: '',
+      parent: null,
+      children: new Set()
+    });
+
+    // Initialize cached stats with root directory
+    this.cachedStats.directoryCount = 1;
+  }
+
+  /**
+   * Build the virtual filesystem from a computed folder structure
+   * @param computed Map of source paths to destination paths
+   * @param metadata Map of source paths to HashMeta
+   * @param basePath Base path to strip from destination paths (e.g., OUTPUT_FOLDER)
+   */
+  public buildFromComputed(
+    computed: Map<string, string>,
+    metadata: Map<string, HashMeta>,
+    basePath?: string
+  ): void {
+    // Clear existing structure (except root)
+    this.nodes.clear();
+    this.nodes.set('/', {
+      type: 'directory',
+      name: '',
+      parent: null,
+      children: new Set()
+    });
+
+    // Reset cached stats (root directory will be counted)
+    this.cachedStats = {
+      fileCount: 0,
+      directoryCount: 1, // Root directory
+      totalSize: 0,
+      metaFileCount: 0,
+      metadataFileCount: 0
+    };
+
+    // Normalize basePath if provided
+    const normalizedBasePath = basePath ? path.normalize(basePath) : null;
+
+    // Add all files and their parent directories
+    for (const [sourcePath, destPath] of computed.entries()) {
+      // Strip basePath from destPath to create relative virtual path
+      let virtualPath = destPath;
+      if (normalizedBasePath && destPath.startsWith(normalizedBasePath)) {
+        virtualPath = destPath.slice(normalizedBasePath.length);
+        // Ensure path starts with /
+        if (!virtualPath.startsWith('/')) {
+          virtualPath = '/' + virtualPath;
+        }
+      }
+      this.addFile(virtualPath, sourcePath, metadata.get(sourcePath));
+    }
+
+    this.emit('vfs-updated');
+  }
+
+  /**
+   * Add a file to the virtual filesystem, creating parent directories as needed
+   */
+  public addFile(virtualPath: string, sourcePath: string, metadata?: HashMeta): void {
+    // Normalize path
+    virtualPath = this.normalizePath(virtualPath);
+
+    // Create parent directories
+    const dirname = path.dirname(virtualPath);
+    this.ensureDirectory(dirname);
+
+    // Try to use metadata for file stats (avoid disk I/O during fast load)
+    let size: number;
+    let mtime: Date;
+    let ctime: Date;
+
+    if (metadata && (metadata as any).sizeByte) {
+      // Use cached metadata (fast path - no disk I/O)
+      size = parseInt((metadata as any).sizeByte, 10);
+      // Use current time as placeholder for mtime/ctime (will be corrected on next scan)
+      mtime = new Date();
+      ctime = new Date();
+    } else {
+      // Fallback: stat the file (slow path - disk I/O)
+      try {
+        const stats = fs.statSync(sourcePath);
+        size = stats.size;
+        mtime = stats.mtime;
+        ctime = stats.ctime;
+      } catch (error) {
+        console.warn(`Cannot stat source file: ${sourcePath}`, error);
+        return;
+      }
+    }
+
+    // Add the file node
+    const filename = path.basename(virtualPath);
+    this.nodes.set(virtualPath, {
+      type: 'file',
+      name: filename,
+      parent: dirname,
+      sourcePath,
+      size,
+      mtime,
+      ctime,
+      metadata
+    });
+
+    // Add to parent's children
+    const parentNode = this.nodes.get(dirname);
+    if (parentNode && parentNode.type === 'directory') {
+      parentNode.children!.add(filename);
+    }
+
+    // Update cached stats
+    this.cachedStats.fileCount++;
+    this.cachedStats.totalSize += size;
+
+    // Also add corresponding metadata files based on configured formats
+    this.addMetadataFiles(virtualPath, metadata, mtime);
+
+    this.emit('file-added', virtualPath);
+  }
+
+  /**
+   * Add virtual metadata files for all configured formats
+   */
+  private addMetadataFiles(mediaPath: string, metadata: HashMeta | undefined, mtime: Date): void {
+    if (!metadata) return;
+
+    // Create a metadata file for each configured format
+    for (const handler of this.activeFormatHandlers) {
+      const metaPath = `${mediaPath}${handler.extension}`;
+      const dirname = path.dirname(metaPath);
+      const filename = path.basename(metaPath);
+
+      // Pre-generate metadata content to calculate size
+      const metaContent = handler.serialize(metadata);
+      const metaBuffer = Buffer.from(metaContent, 'utf-8');
+
+      // Add the virtual metadata file node
+      this.nodes.set(metaPath, {
+        type: 'file',
+        name: filename,
+        parent: dirname,
+        sourcePath: null, // Virtual file, no source
+        size: metaBuffer.length,
+        mtime: mtime,
+        ctime: mtime,
+        metadata,
+        formatHandler: handler
+      });
+
+      // Add to parent's children
+      const parentNode = this.nodes.get(dirname);
+      if (parentNode && parentNode.type === 'directory') {
+        parentNode.children!.add(filename);
+      }
+
+      // Update cached stats
+      this.cachedStats.fileCount++;
+      this.cachedStats.totalSize += metaBuffer.length;
+      this.cachedStats.metadataFileCount++;
+      // Legacy: Count .meta files separately for backward compatibility
+      if (handler.extension === '.meta') {
+        this.cachedStats.metaFileCount++;
+      }
+    }
+  }
+
+  /**
+   * Ensure a directory exists in the VFS, creating parent directories as needed
+   */
+  private ensureDirectory(dirPath: string): void {
+    dirPath = this.normalizePath(dirPath);
+
+    if (dirPath === '/' || this.nodes.has(dirPath)) {
+      return;
+    }
+
+    // Recursively create parent
+    const parentPath = path.dirname(dirPath);
+    this.ensureDirectory(parentPath);
+
+    // Create this directory
+    const dirName = path.basename(dirPath);
+    this.nodes.set(dirPath, {
+      type: 'directory',
+      name: dirName,
+      parent: parentPath,
+      children: new Set()
+    });
+
+    // Add to parent's children
+    const parentNode = this.nodes.get(parentPath);
+    if (parentNode && parentNode.type === 'directory') {
+      parentNode.children!.add(dirName);
+    }
+
+    // Update cached stats
+    this.cachedStats.directoryCount++;
+
+    this.emit('directory-added', dirPath);
+  }
+
+  /**
+   * Normalize path to use forward slashes and ensure leading slash
+   */
+  private normalizePath(p: string): string {
+    // Convert to forward slashes
+    p = p.split(path.sep).join('/');
+
+    // Ensure leading slash
+    if (!p.startsWith('/')) {
+      p = '/' + p;
+    }
+
+    // Remove trailing slash (except for root)
+    if (p !== '/' && p.endsWith('/')) {
+      p = p.slice(0, -1);
+    }
+
+    return p;
+  }
+
+  // IFuseAPI implementation
+
+  public readdir(dirPath: string): string[] | null {
+    dirPath = this.normalizePath(dirPath);
+    const node = this.nodes.get(dirPath);
+
+    if (!node || node.type !== 'directory') {
+      return null;
+    }
+
+    return Array.from(node.children!);
+  }
+
+  public getattr(filepath: string): FileAttributes | null {
+    filepath = this.normalizePath(filepath);
+    const node = this.nodes.get(filepath);
+
+    if (!node) {
+      return null;
+    }
+
+    if (node.type === 'directory') {
+      return {
+        size: 4096, // Standard directory size
+        mode: 0o040000 | this.config.directoryMode, // S_IFDIR | permissions
+        mtime: Date.now() / 1000,
+        atime: Date.now() / 1000,
+        ctime: Date.now() / 1000,
+        nlink: 2, // . and ..
+        uid: this.config.uid,
+        gid: this.config.gid
+      };
+    } else {
+      return {
+        size: node.size!,
+        mode: 0o100000 | this.config.fileMode, // S_IFREG | permissions
+        mtime: node.mtime!.getTime() / 1000,
+        atime: node.mtime!.getTime() / 1000,
+        ctime: node.ctime!.getTime() / 1000,
+        nlink: 1,
+        uid: this.config.uid,
+        gid: this.config.gid
+      };
+    }
+  }
+
+  public exists(filepath: string): boolean {
+    filepath = this.normalizePath(filepath);
+    return this.nodes.has(filepath);
+  }
+
+  public read(filepath: string): ReadResult | null {
+    filepath = this.normalizePath(filepath);
+    const node = this.nodes.get(filepath);
+
+    if (!node || node.type !== 'file') {
+      return null;
+    }
+
+    // Virtual metadata file (any format)
+    if (node.sourcePath === null && node.metadata && node.formatHandler) {
+      const metaContent = node.formatHandler.serialize(node.metadata);
+      const metaBuffer = Buffer.from(metaContent, 'utf-8');
+
+      return {
+        sourcePath: null,
+        content: metaBuffer,
+        size: metaBuffer.length
+      };
+    }
+
+    // Regular file - return source path for reading
+    return {
+      sourcePath: node.sourcePath!,
+      content: null,
+      size: node.size!
+    };
+  }
+
+  public getMetadata(filepath: string): HashMeta | null {
+    filepath = this.normalizePath(filepath);
+    const node = this.nodes.get(filepath);
+
+    if (!node || node.type !== 'file') {
+      return null;
+    }
+
+    return node.metadata ?? null;
+  }
+
+  public getTree(): VirtualDirectory {
+    const rootNode = this.nodes.get('/')!;
+
+    return {
+      path: '/',
+      type: 'directory',
+      name: '',
+      parent: '',
+      children: Array.from(rootNode.children!),
+      mode: this.config.directoryMode,
+      mtime: new Date()
+    };
+  }
+
+  public async refresh(): Promise<void> {
+    // Refresh is triggered by external calls after processing
+    // The buildFromComputed method handles the actual refresh
+    this.emit('vfs-updated');
+  }
+
+  /**
+   * Get all file paths in the VFS
+   */
+  public getAllFiles(): string[] {
+    const files: string[] = [];
+    for (const [filepath, node] of this.nodes.entries()) {
+      if (node.type === 'file') {
+        files.push(filepath);
+      }
+    }
+    return files;
+  }
+
+  /**
+   * Get all directory paths in the VFS
+   */
+  public getAllDirectories(): string[] {
+    const dirs: string[] = [];
+    for (const [dirpath, node] of this.nodes.entries()) {
+      if (node.type === 'directory') {
+        dirs.push(dirpath);
+      }
+    }
+    return dirs;
+  }
+
+  /**
+   * Get statistics about the VFS (returns cached values, updated incrementally)
+   */
+  public getStats(): {
+    fileCount: number;
+    directoryCount: number;
+    totalSize: number;
+    metaFileCount: number;
+    metadataFileCount: number;
+  } {
+    // Return cached stats - no iteration needed!
+    return { ...this.cachedStats };
+  }
+
+  /**
+   * Count files in VFS that have completed hash computation
+   * A file is considered "done" if hashStatus === 'complete' (all hashes computed without errors)
+   */
+  public countFilesWithHash(): number {
+    let count = 0;
+    for (const [filepath, node] of this.nodes.entries()) {
+      // Only count actual media files (not directories, not metadata files)
+      if (node.type === 'file' && node.sourcePath && node.metadata) {
+        // Check if file has completed processing (both light and hash phases)
+        if ((node.metadata as any).processingStatus === 'complete') {
+          count++;
+        }
+      }
+    }
+    return count;
+  }
+
+  /**
+   * Update metadata for an existing file in the VFS (e.g., when hash computation completes)
+   * This also regenerates the metadata files with updated content
+   */
+  public updateFileMetadata(virtualPath: string, metadata: HashMeta): void {
+    virtualPath = this.normalizePath(virtualPath);
+    const node = this.nodes.get(virtualPath);
+
+    if (!node || node.type !== 'directory') {
+      // Update file metadata
+      if (node) {
+        node.metadata = metadata;
+
+        // Regenerate metadata files with updated content
+        if (node.mtime) {
+          // Get old metadata file sizes before regenerating
+          const oldSizes = new Map<string, number>();
+          for (const handler of this.activeFormatHandlers) {
+            const metaPath = `${virtualPath}${handler.extension}`;
+            const oldMetaNode = this.nodes.get(metaPath);
+            if (oldMetaNode) {
+              oldSizes.set(metaPath, oldMetaNode.size ?? 0);
+            }
+          }
+
+          // Regenerate metadata files (this will overwrite the nodes)
+          this.addMetadataFilesUpdate(virtualPath, metadata, node.mtime, oldSizes);
+        }
+
+        this.emit('file-updated', virtualPath);
+      }
+    }
+  }
+
+  /**
+   * Add/update virtual metadata files for all configured formats (used during updates)
+   * This version adjusts cached stats based on size differences
+   */
+  private addMetadataFilesUpdate(
+    mediaPath: string,
+    metadata: HashMeta,
+    mtime: Date,
+    oldSizes: Map<string, number>
+  ): void {
+    // Create a metadata file for each configured format
+    for (const handler of this.activeFormatHandlers) {
+      const metaPath = `${mediaPath}${handler.extension}`;
+      const dirname = path.dirname(metaPath);
+      const filename = path.basename(metaPath);
+
+      // Pre-generate metadata content to calculate size
+      const metaContent = handler.serialize(metadata);
+      const metaBuffer = Buffer.from(metaContent, 'utf-8');
+      const newSize = metaBuffer.length;
+
+      const oldSize = oldSizes.get(metaPath);
+      const isNew = oldSize === undefined;
+
+      // Update the virtual metadata file node
+      this.nodes.set(metaPath, {
+        type: 'file',
+        name: filename,
+        parent: dirname,
+        sourcePath: null, // Virtual file, no source
+        size: newSize,
+        mtime: mtime,
+        ctime: mtime,
+        metadata,
+        formatHandler: handler
+      });
+
+      // Add to parent's children (no-op if already exists)
+      const parentNode = this.nodes.get(dirname);
+      if (parentNode && parentNode.type === 'directory') {
+        parentNode.children!.add(filename);
+      }
+
+      // Update cached stats
+      if (isNew) {
+        // New metadata file
+        this.cachedStats.fileCount++;
+        this.cachedStats.totalSize += newSize;
+        this.cachedStats.metadataFileCount++;
+        if (handler.extension === '.meta') {
+          this.cachedStats.metaFileCount++;
+        }
+      } else {
+        // Update existing - adjust size difference
+        this.cachedStats.totalSize += (newSize - oldSize);
+      }
+    }
+  }
+
+  /**
+   * Remove a file and its associated metadata files from the VFS
+   */
+  public removeFile(virtualPath: string): void {
+    virtualPath = this.normalizePath(virtualPath);
+    const node = this.nodes.get(virtualPath);
+
+    if (!node || node.type !== 'file') {
+      return;
+    }
+
+    // Update cached stats for main file
+    this.cachedStats.fileCount--;
+    this.cachedStats.totalSize -= node.size ?? 0;
+
+    // Remove the main file
+    this.nodes.delete(virtualPath);
+
+    // Remove from parent's children
+    const parentNode = this.nodes.get(node.parent!);
+    if (parentNode && parentNode.type === 'directory') {
+      parentNode.children!.delete(node.name);
+    }
+
+    // Remove associated metadata files (.nfo, .meta, etc.)
+    for (const handler of this.activeFormatHandlers) {
+      const metaPath = `${virtualPath}${handler.extension}`;
+      const metaNode = this.nodes.get(metaPath);
+
+      if (metaNode) {
+        // Update cached stats for metadata file
+        this.cachedStats.fileCount--;
+        this.cachedStats.totalSize -= metaNode.size ?? 0;
+        this.cachedStats.metadataFileCount--;
+        if (handler.extension === '.meta') {
+          this.cachedStats.metaFileCount--;
+        }
+
+        this.nodes.delete(metaPath);
+
+        // Remove from parent's children
+        const metaParent = this.nodes.get(metaNode.parent!);
+        if (metaParent && metaParent.type === 'directory') {
+          metaParent.children!.delete(metaNode.name);
+        }
+      }
+    }
+
+    this.emit('file-removed', virtualPath);
+  }
+
+  /**
+   * Check if a file exists in the VFS and return its metadata
+   */
+  public getFileMetadata(virtualPath: string): HashMeta | undefined {
+    virtualPath = this.normalizePath(virtualPath);
+    const node = this.nodes.get(virtualPath);
+    return node?.metadata;
+  }
+}
+
+// Type-safe event emitter
+export interface VirtualFileSystem {
+  on<K extends keyof FuseAPIEvents>(event: K, listener: FuseAPIEvents[K]): this;
+  emit<K extends keyof FuseAPIEvents>(event: K, ...args: Parameters<FuseAPIEvents[K]>): boolean;
+}

packages/meta-sort-core/src/api/index.ts

@@ -0,0 +1,11 @@
+/**
+ * API exports
+ *
+ * This module exports all API components including FUSE API, Metrics API, and Unified API Server.
+ */
+
+export { IFuseAPI, VirtualNode, VirtualFile, VirtualDirectory, FileAttributes, ReadResult, FuseAPIConfig, FuseAPIEvents } from './FuseAPI.js';
+export { VirtualFileSystem } from './VirtualFileSystem.js';
+export { FuseAPIServer, FuseAPIServerConfig } from './FuseAPIServer.js';
+export { MetricsAPI, MetricsAPIConfig } from './MetricsAPI.js';
+export { UnifiedAPIServer, UnifiedAPIServerConfig } from './UnifiedAPIServer.js';

packages/meta-sort-core/src/config/EnvConfig.ts

@@ -0,0 +1,216 @@
+import dotenv from 'dotenv';
+dotenv.config();
+
+export interface PollingPathConfig {
+    path: string;
+    intervalMs: number;
+}
+
+interface EnvConfig {
+    /** Comma-separated list of folders to watch for changes.(mandatory) */
+    WATCH_FOLDER_LIST: string;
+
+    /** Path to the index file within the NFO CID library. */
+    INDEX_FOLDER_PATH: string;
+
+    /** Interval in milliseconds to update/check for changes. (default 30000) */
+    UPDATE_INTERVAL_MS: number;
+
+    /** Interval in milliseconds for polling watch folders. Set to 0 to disable polling. (default 0) */
+    WATCH_FOLDER_POOLING_INTERVAL_MS: number;
+
+    /** CHMOD for new folder format (default 0o777) */
+    CHMOD_FOR_NEW_FOLDER: number;
+
+    /** Path to the cache folder. */
+    CACHE_FOLDER_PATH: string;
+
+    /** jellyfin */
+    JELLYFIN_ENDPOINT: string;
+    JELLYFIN_API_KEY: string;
+
+    /** FUSE API server port (default 3000) */
+    FUSE_API_PORT: number;
+
+    /** FUSE API server host (default 'localhost') */
+    FUSE_API_HOST: string;
+
+    /** Comma-separated list of metadata formats to generate in virtual filesystem (default 'meta'). Options: 'meta' (.meta YAML files), 'jellyfin' (.nfo XML files). Example: 'meta,jellyfin' or 'jellyfin' or '' (no metadata files) */
+    METADATA_FORMATS: string[];
+
+    /** Maximum number of worker threads for Piscina worker pool AND max concurrent file processing (default: CPU count) */
+    MAX_WORKER_THREADS?: number;
+
+    /** JSON array of polling path configurations. Example: '[{"path": "/data/watch/smb/downloads", "intervalMs": 60000}]' (optional) */
+    POLLING_PATHS?: PollingPathConfig[];
+
+    // ========================================================================
+    // KV Storage Configuration (NEW)
+    // ========================================================================
+
+    /** Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services (default '/meta-core') */
+    META_CORE_PATH: string;
+
+    /** Path to FILES_VOLUME - shared media volume (default '/files') */
+    FILES_PATH: string;
+
+    /** Service name for leader election and service discovery (default 'meta-sort') */
+    SERVICE_NAME: string;
+
+    /** Service version (default '1.0.0') */
+    SERVICE_VERSION: string;
+
+    /** Redis port (default 6379) */
+    REDIS_PORT: number;
+
+    /** Direct Redis URL (skip leader election if provided) */
+    REDIS_URL?: string;
+
+    /** Base URL for service discovery (e.g., 'http://localhost:3000' or 'http://meta-sort:80') */
+    BASE_URL?: string;
+
+    /**
+     * Hostname to advertise in lock file for other services to connect.
+     * Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution.
+     * If not set, auto-detects local IP (may be unstable across container restarts).
+     */
+    ADVERTISE_HOST?: string;
+
+    // ========================================================================
+    // TaskScheduler Configuration (Plugin Queue System)
+    // ========================================================================
+
+    /** Concurrency for fast plugin queue (plugins with <1s avg execution time). Default: 32 */
+    FAST_QUEUE_CONCURRENCY: number;
+
+    /** Concurrency for background plugin queue (plugins with >=1s avg execution time). Default: 8 */
+    BACKGROUND_QUEUE_CONCURRENCY: number;
+
+    /** Threshold in ms to classify plugin as fast vs background. Default: 1000 */
+    FAST_THRESHOLD_MS: number;
+
+    /** Minimum samples before using measured timing for classification. Default: 10 */
+    MIN_SAMPLES_FOR_MEASUREMENT: number;
+
+    // ========================================================================
+    // Container Plugins Configuration
+    // All plugins run as Docker containers for isolation and flexibility
+    // ========================================================================
+
+    /** Path to container plugins config file (default: '/app/plugins.yml') */
+    CONTAINER_PLUGINS_CONFIG: string;
+
+    /** Docker socket path (default: '/var/run/docker.sock') */
+    DOCKER_SOCKET_PATH: string;
+
+    /** Callback URL for container plugins (default: 'http://meta-sort:8180') */
+    CONTAINER_CALLBACK_URL: string;
+
+    /** meta-core URL for container plugins (default: 'http://meta-sort:9000') */
+    CONTAINER_META_CORE_URL: string;
+
+    /** Docker network for plugin containers (default: 'meta-network') */
+    CONTAINER_NETWORK: string;
+
+    /**
+     * Plugin file mounts - maps subpaths to host paths for /files in plugin containers.
+     * Format: "subpath:host_path,subpath2:host_path2"
+     * Example: "watch:/data/media,test:/app/test/media"
+     * Results in /files/watch and /files/test mounted in plugin containers (read-only).
+     */
+    PLUGIN_FILE_MOUNTS?: string;
+
+    /** Host path for plugin cache folder - each plugin gets its own subdirectory at /cache (optional, dev only) */
+    PLUGIN_CACHE_FOLDER?: string;
+
+    /**
+     * Host path for plugin output folders - each plugin gets /files/plugin/<id> mount (READ-WRITE)
+     * Plugins decide whether to use this folder - we always mount it if this env is set
+     */
+    PLUGIN_OUTPUT_FOLDER?: string;
+
+    /** Docker Compose project name for grouping containers in Docker Desktop (optional). When set, plugin containers appear grouped as a stack. */
+    PLUGIN_STACK_NAME?: string;
+}
+
+// Parse POLLING_PATHS from JSON string
+function parsePollingPaths(): PollingPathConfig[] | undefined {
+    if (!process.env.POLLING_PATHS) {
+        return undefined;
+    }
+
+    try {
+        const parsed = JSON.parse(process.env.POLLING_PATHS);
+        if (!Array.isArray(parsed)) {
+            console.error('POLLING_PATHS must be a JSON array');
+            return undefined;
+        }
+
+        // Validate structure
+        const valid = parsed.every(item =>
+            item && typeof item === 'object' &&
+            typeof item.path === 'string' &&
+            typeof item.intervalMs === 'number'
+        );
+
+        if (!valid) {
+            console.error('POLLING_PATHS entries must have {path: string, intervalMs: number}');
+            return undefined;
+        }
+
+        return parsed as PollingPathConfig[];
+    } catch (error) {
+        console.error('Failed to parse POLLING_PATHS:', error);
+        return undefined;
+    }
+}
+
+export const config: EnvConfig = {
+    WATCH_FOLDER_LIST: process.env.WATCH_FOLDER_LIST,
+    INDEX_FOLDER_PATH: process.env.INDEX_FOLDER_PATH || '/data/cache/hash',//default /data/cache/hash
+    UPDATE_INTERVAL_MS: parseInt(process.env.UPDATE_INTERVAL_MS || "30000", 10),//default 30000
+    WATCH_FOLDER_POOLING_INTERVAL_MS: parseInt(process.env.WATCH_FOLDER_POOLING_INTERVAL_MS || "0", 10),//default 0
+    CHMOD_FOR_NEW_FOLDER: process.env.CHMOD_FOR_NEW_FOLDER && parseInt(process.env.CHMOD_FOR_NEW_FOLDER, 8),//default 0o777 (this is the default for mkdir)
+    CACHE_FOLDER_PATH: process.env.CACHE_FOLDER_PATH || '/data/cache/plugins',//default /data/cache/plugins
+    JELLYFIN_ENDPOINT: process.env.JELLYFIN_ENDPOINT!,
+    JELLYFIN_API_KEY: process.env.JELLYFIN_API_KEY!,
+    FUSE_API_PORT: parseInt(process.env.FUSE_API_PORT || "3000", 10),//default 3000
+    FUSE_API_HOST: process.env.FUSE_API_HOST || 'localhost',//default localhost
+    METADATA_FORMATS: (process.env.METADATA_FORMATS || 'meta').split(',').map(f => f.trim()).filter(f => f.length > 0),//default 'meta'
+    MAX_WORKER_THREADS: process.env.MAX_WORKER_THREADS ? parseInt(process.env.MAX_WORKER_THREADS, 10) : undefined,
+    POLLING_PATHS: parsePollingPaths(),
+
+    // KV Storage Configuration
+    META_CORE_PATH: process.env.META_CORE_PATH || '/meta-core',
+    FILES_PATH: process.env.FILES_PATH || '/files',
+    SERVICE_NAME: process.env.SERVICE_NAME || 'meta-sort',
+    SERVICE_VERSION: process.env.SERVICE_VERSION || '1.0.0',
+    REDIS_PORT: parseInt(process.env.REDIS_PORT || "6379", 10),
+    REDIS_URL: process.env.REDIS_URL,
+    BASE_URL: process.env.BASE_URL,
+    ADVERTISE_HOST: process.env.ADVERTISE_HOST,
+
+    // TaskScheduler Configuration
+    FAST_QUEUE_CONCURRENCY: parseInt(process.env.FAST_QUEUE_CONCURRENCY || "32", 10),
+    BACKGROUND_QUEUE_CONCURRENCY: parseInt(process.env.BACKGROUND_QUEUE_CONCURRENCY || "8", 10),
+    FAST_THRESHOLD_MS: parseInt(process.env.FAST_THRESHOLD_MS || "1000", 10),
+    MIN_SAMPLES_FOR_MEASUREMENT: parseInt(process.env.MIN_SAMPLES_FOR_MEASUREMENT || "10", 10),
+
+    // Container Plugins Configuration
+    CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml',
+    DOCKER_SOCKET_PATH: process.env.DOCKER_SOCKET_PATH || '/var/run/docker.sock',
+    CONTAINER_CALLBACK_URL: process.env.CONTAINER_CALLBACK_URL || 'http://meta-sort:8180',
+    CONTAINER_META_CORE_URL: process.env.CONTAINER_META_CORE_URL || 'http://meta-sort:9000',
+    CONTAINER_NETWORK: process.env.CONTAINER_NETWORK || 'meta-network',
+    PLUGIN_FILE_MOUNTS: process.env.PLUGIN_FILE_MOUNTS,
+    PLUGIN_CACHE_FOLDER: process.env.PLUGIN_CACHE_FOLDER,
+    PLUGIN_OUTPUT_FOLDER: process.env.PLUGIN_OUTPUT_FOLDER,
+    PLUGIN_STACK_NAME: process.env.PLUGIN_STACK_NAME,
+};
+
+if(process.env.TEST!=='true') {
+    if (!config.WATCH_FOLDER_LIST) {
+        console.error('Invalid configuration: WATCH_FOLDER_LIST is required');
+    }
+}
+

packages/meta-sort-core/src/config/RenamingRule.ts

@@ -0,0 +1,62 @@
+import {
+    AnimeMeta, FileNameMeta,
+    FileStatMeta,
+    HashMeta,
+    LanguagesMeta, SubtitleFileMeta,
+    VersionMeta,
+    VideoMeta
+} from "@metazla/meta-interface";
+
+type Local =  VideoMeta & HashMeta & FileStatMeta & AnimeMeta & VersionMeta & LanguagesMeta & SubtitleFileMeta & FileNameMeta;
+
+/**
+ * return the path relative to the root folder
+ */
+export function renamingRule(metamesh: Local, filepath?: string): string | null {
+    let supported = ['video', 'subtitle', 'torrent'];
+
+    //check if filetype is one of the suported ones
+    if (!supported.includes(metamesh.fileType)) {
+        return null;
+    }
+
+    // Extract the file extension from the original file path
+    const extension = metamesh.extension;
+    if (!extension) {
+        throw new Error(`No extension found for file: ${filepath}`);
+    }
+
+    const title = metamesh?.titles?.eng || metamesh.originalTitle;
+    if (!title) {
+        if (metamesh.fileType !== "torrent") {
+            //don't warn for torrent files because they don't always have a title (eg representing a folder of files)
+            throw new Error(`No title found for file: ${filepath}`);
+        }
+        return null;
+    }
+
+
+    let newPath = "";
+
+    const season = metamesh.season ? ("S" + String(metamesh.season).padStart(2, '0')) : "";
+    const seasonSpace = metamesh.season ? ` ${season}` : "";
+    const episode = metamesh.episode ? ("E" + String(metamesh.episode).padStart(2, '0')) : "";
+    const version = metamesh.version ? ` ${metamesh.version}` : "";
+    const year = metamesh.movieYear ? ` (${metamesh.movieYear})` : "";
+    const subtitle = metamesh.fileType === "subtitle" && metamesh.subtitleLanguage ? `.${metamesh.subtitleLanguage}` : "";
+    const fileName = `${title}${seasonSpace}${episode}${year}${subtitle}${version}.${extension}`;
+
+    // Simplified structure: TV Shows for series, Movies for standalone content
+    // Note: season can be 0 (special episodes), so check for null/undefined explicitly
+    const hasSeason = metamesh.season !== null && metamesh.season !== undefined;
+    const hasEpisode = metamesh.episode !== null && metamesh.episode !== undefined;
+
+    if (metamesh.extra) {
+        newPath = `TV Shows/${title}/extra/${fileName}`;
+    } else if (hasSeason && hasEpisode) {
+        newPath = `TV Shows/${title}/${season}/${fileName}`;
+    } else {
+        newPath = `Movies/${title}${year}/${fileName}`;
+    }
+    return newPath;
+}

packages/meta-sort-core/src/config/SupportedFileTypes.ts

@@ -0,0 +1,68 @@
+/**
+ * Supported file extensions for Meta Mesh
+ * Application-specific configuration
+ */
+
+export const SUPPORTED_VIDEO_EXTENSIONS = new Set([
+    '.mkv',
+    '.mp4',
+    '.avi',
+    '.mov',
+    '.wmv',
+    '.flv',
+    '.webm',
+    '.m4v',
+    '.mpg',
+    '.mpeg',
+    '.m2ts',
+    '.ts',
+    '.vob',
+    '.3gp',
+    '.ogv',
+    '.divx',
+    '.xvid',
+    '.rm',
+    '.rmvb'
+]);
+
+export const SUPPORTED_SUBTITLE_EXTENSIONS = new Set([
+    '.srt',
+    '.ass',
+    '.ssa',
+    '.sub',
+    '.vtt',
+    '.idx'
+]);
+
+export const SUPPORTED_METADATA_EXTENSIONS = new Set([
+    '.torrent'
+]);
+
+export const SUPPORTED_IMAGE_EXTENSIONS = new Set([
+    '.jpg',
+    '.jpeg',
+    '.png',
+    '.webp',
+    '.gif',
+    '.bmp',
+    '.tiff',
+    '.tif'
+]);
+
+/**
+ * All supported file extensions
+ */
+export const SUPPORTED_EXTENSIONS = new Set([
+    ...SUPPORTED_VIDEO_EXTENSIONS,
+    ...SUPPORTED_SUBTITLE_EXTENSIONS,
+    ...SUPPORTED_METADATA_EXTENSIONS,
+    ...SUPPORTED_IMAGE_EXTENSIONS
+]);
+
+/**
+ * Check if file extension is supported
+ */
+export function isSupportedExtension(filePath: string): boolean {
+    const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase();
+    return SUPPORTED_EXTENSIONS.has(ext);
+}

packages/meta-sort-core/src/config/TargetHash.ts

@@ -0,0 +1,21 @@
+import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
+
+// Target hashes for Piscina worker (full-file hash computation)
+// Note: midhash256 is NOT included here because it's computed in the main thread
+// during light processing, not by the Piscina worker
+export const targetHash = [
+    CID_ALGORITHM_NAMES.sha1,        // For torrent verification
+    CID_ALGORITHM_NAMES.sha256,      // For IPFS compatibility
+    CID_ALGORITHM_NAMES.md5,         // For compatibility
+    CID_ALGORITHM_NAMES.sha3_256,    // Additional cryptographic hash
+    CID_ALGORITHM_NAMES.sha3_384,    // Additional cryptographic hash
+    CID_ALGORITHM_NAMES.crc32,       // Fast checksum
+    CID_ALGORITHM_NAMES.btih_v2      // BitTorrent v2 info hash (BEP 52)
+]
+
+// Target hashes for index-cid cache (includes all hashes we want to cache)
+// This includes midhash256 which is computed separately from the worker hashes
+export const targetHashForIndex = [
+    ...targetHash,
+    CID_ALGORITHM_NAMES.midhash256   // Fast hash for file identification (computed in light processing)
+]

packages/meta-sort-core/src/container-plugins/ConfigParser.ts

@@ -0,0 +1,372 @@
+/**
+ * Container Plugin Configuration Parser
+ *
+ * Parses plugins.yml configuration files for container plugins.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import YAML from 'yaml';
+import type {
+    ContainerPluginsConfig,
+    ContainerPluginConfig,
+    ContainerResourceLimits,
+    ContainerMount,
+    ContainerHealthCheck,
+} from './types.js';
+import type { TaskQueueType } from '../plugin-engine/types.js';
+
+/**
+ * Parse error with context
+ */
+export class ConfigParseError extends Error {
+    constructor(
+        message: string,
+        public readonly filePath: string,
+        public readonly cause?: Error
+    ) {
+        super(`${message} (file: ${filePath})`);
+        this.name = 'ConfigParseError';
+    }
+}
+
+/**
+ * Default configuration values
+ */
+const DEFAULTS = {
+    version: '1.0',
+    instances: 1,
+    enabled: true,
+    network: false,
+    healthCheck: {
+        interval: '30s',
+        timeout: '5s',
+        retries: 3,
+    },
+} as const;
+
+/**
+ * Substitute environment variables in a string
+ * Supports: ${VAR}, ${VAR:-default}, ${VAR:=default}
+ */
+function substituteEnvVars(value: string): string {
+    // Match ${VAR}, ${VAR:-default}, or ${VAR:=default}
+    const envPattern = /\$\{([A-Z_][A-Z0-9_]*)(?:(:?[-=])([^}]*))?\}/gi;
+
+    return value.replace(envPattern, (match, varName, operator, defaultValue) => {
+        const envValue = process.env[varName];
+
+        if (envValue !== undefined && envValue !== '') {
+            return envValue;
+        }
+
+        // Handle default values
+        if (operator === ':-' || operator === ':=') {
+            return defaultValue || '';
+        }
+        if (operator === '-' || operator === '=') {
+            // Only use default if variable is unset (not if empty)
+            if (envValue === undefined) {
+                return defaultValue || '';
+            }
+            return '';
+        }
+
+        // No default, return empty string
+        return '';
+    });
+}
+
+/**
+ * Recursively substitute env vars in an object
+ */
+function substituteEnvVarsInObject(obj: unknown): unknown {
+    if (typeof obj === 'string') {
+        return substituteEnvVars(obj);
+    }
+
+    if (Array.isArray(obj)) {
+        return obj.map(item => substituteEnvVarsInObject(item));
+    }
+
+    if (obj !== null && typeof obj === 'object') {
+        const result: Record<string, unknown> = {};
+        for (const [key, value] of Object.entries(obj)) {
+            result[key] = substituteEnvVarsInObject(value);
+        }
+        return result;
+    }
+
+    return obj;
+}
+
+/**
+ * Validate resource limits
+ */
+function validateResourceLimits(
+    resources: unknown,
+    pluginId: string
+): ContainerResourceLimits | undefined {
+    if (!resources || typeof resources !== 'object') {
+        return undefined;
+    }
+
+    const r = resources as Record<string, unknown>;
+    const result: ContainerResourceLimits = {};
+
+    if (r.memory !== undefined) {
+        if (typeof r.memory !== 'string') {
+            throw new Error(`Plugin '${pluginId}': resources.memory must be a string (e.g., '512m', '2g')`);
+        }
+        // Validate format
+        if (!/^\d+[kmgKMG]?$/.test(r.memory)) {
+            throw new Error(`Plugin '${pluginId}': invalid memory format '${r.memory}' (use e.g., '512m', '2g')`);
+        }
+        result.memory = r.memory;
+    }
+
+    if (r.cpus !== undefined) {
+        const cpus = Number(r.cpus);
+        if (isNaN(cpus) || cpus <= 0) {
+            throw new Error(`Plugin '${pluginId}': resources.cpus must be a positive number`);
+        }
+        result.cpus = cpus;
+    }
+
+    return Object.keys(result).length > 0 ? result : undefined;
+}
+
+/**
+ * Validate volume mounts
+ */
+function validateMounts(mounts: unknown, pluginId: string): ContainerMount[] | undefined {
+    if (!mounts) {
+        return undefined;
+    }
+
+    if (!Array.isArray(mounts)) {
+        throw new Error(`Plugin '${pluginId}': mounts must be an array`);
+    }
+
+    return mounts.map((mount, index) => {
+        if (!mount || typeof mount !== 'object') {
+            throw new Error(`Plugin '${pluginId}': mount[${index}] must be an object`);
+        }
+
+        const m = mount as Record<string, unknown>;
+
+        if (typeof m.source !== 'string' || !m.source) {
+            throw new Error(`Plugin '${pluginId}': mount[${index}].source is required`);
+        }
+
+        if (typeof m.target !== 'string' || !m.target) {
+            throw new Error(`Plugin '${pluginId}': mount[${index}].target is required`);
+        }
+
+        return {
+            source: m.source,
+            target: m.target,
+            readonly: m.readonly !== false, // Default to true
+        };
+    });
+}
+
+/**
+ * Validate health check config
+ */
+function validateHealthCheck(
+    healthCheck: unknown,
+    pluginId: string
+): ContainerHealthCheck | undefined {
+    if (!healthCheck || typeof healthCheck !== 'object') {
+        return undefined;
+    }
+
+    const h = healthCheck as Record<string, unknown>;
+    const result: ContainerHealthCheck = {};
+
+    if (h.interval !== undefined) {
+        if (typeof h.interval !== 'string') {
+            throw new Error(`Plugin '${pluginId}': healthCheck.interval must be a string (e.g., '30s')`);
+        }
+        result.interval = h.interval;
+    }
+
+    if (h.timeout !== undefined) {
+        if (typeof h.timeout !== 'string') {
+            throw new Error(`Plugin '${pluginId}': healthCheck.timeout must be a string (e.g., '5s')`);
+        }
+        result.timeout = h.timeout;
+    }
+
+    if (h.retries !== undefined) {
+        const retries = Number(h.retries);
+        if (isNaN(retries) || retries < 0 || !Number.isInteger(retries)) {
+            throw new Error(`Plugin '${pluginId}': healthCheck.retries must be a non-negative integer`);
+        }
+        result.retries = retries;
+    }
+
+    return Object.keys(result).length > 0 ? result : undefined;
+}
+
+/**
+ * Validate a single plugin configuration
+ */
+function validatePluginConfig(
+    pluginId: string,
+    config: unknown
+): ContainerPluginConfig {
+    if (!config || typeof config !== 'object') {
+        throw new Error(`Plugin '${pluginId}': configuration must be an object`);
+    }
+
+    const c = config as Record<string, unknown>;
+
+    // Required: image
+    if (typeof c.image !== 'string' || !c.image) {
+        throw new Error(`Plugin '${pluginId}': 'image' is required and must be a non-empty string`);
+    }
+
+    // Optional: enabled (default: true)
+    const enabled = c.enabled !== false;
+
+    // Optional: instances (default: 1)
+    let instances: number = DEFAULTS.instances;
+    if (c.instances !== undefined) {
+        instances = Number(c.instances);
+        if (isNaN(instances) || instances < 1 || !Number.isInteger(instances)) {
+            throw new Error(`Plugin '${pluginId}': 'instances' must be a positive integer`);
+        }
+    }
+
+    // Optional: defaultQueue
+    let defaultQueue: TaskQueueType | undefined;
+    if (c.defaultQueue !== undefined) {
+        if (c.defaultQueue !== 'fast' && c.defaultQueue !== 'background') {
+            throw new Error(`Plugin '${pluginId}': 'defaultQueue' must be 'fast' or 'background'`);
+        }
+        defaultQueue = c.defaultQueue;
+    }
+
+    // Optional: network (default: false)
+    const network = c.network === true;
+
+    return {
+        enabled,
+        image: c.image,
+        instances,
+        resources: validateResourceLimits(c.resources, pluginId),
+        config: c.config && typeof c.config === 'object'
+            ? c.config as Record<string, unknown>
+            : undefined,
+        network,
+        mounts: validateMounts(c.mounts, pluginId),
+        healthCheck: validateHealthCheck(c.healthCheck, pluginId),
+        defaultQueue,
+    };
+}
+
+/**
+ * Parse and validate container plugins configuration
+ */
+export function parseConfig(rawConfig: unknown, filePath: string): ContainerPluginsConfig {
+    if (!rawConfig || typeof rawConfig !== 'object') {
+        throw new ConfigParseError('Configuration must be an object', filePath);
+    }
+
+    const config = rawConfig as Record<string, unknown>;
+
+    // Optional: version
+    const version = typeof config.version === 'string'
+        ? config.version
+        : DEFAULTS.version;
+
+    // Required: plugins
+    if (!config.plugins || typeof config.plugins !== 'object') {
+        throw new ConfigParseError("'plugins' section is required and must be an object", filePath);
+    }
+
+    const plugins: Record<string, ContainerPluginConfig> = {};
+
+    for (const [pluginId, pluginConfig] of Object.entries(config.plugins)) {
+        // Validate plugin ID format
+        if (!/^[a-z][a-z0-9-]*$/.test(pluginId)) {
+            throw new ConfigParseError(
+                `Invalid plugin ID '${pluginId}': must be lowercase alphanumeric with hyphens, starting with a letter`,
+                filePath
+            );
+        }
+
+        try {
+            plugins[pluginId] = validatePluginConfig(pluginId, pluginConfig);
+        } catch (error) {
+            throw new ConfigParseError(
+                error instanceof Error ? error.message : String(error),
+                filePath
+            );
+        }
+    }
+
+    return { version, plugins };
+}
+
+/**
+ * Load and parse plugins.yml configuration file
+ */
+export async function loadConfig(configPath: string): Promise<ContainerPluginsConfig> {
+    // Check if file exists
+    try {
+        await fs.access(configPath, fs.constants.R_OK);
+    } catch {
+        throw new ConfigParseError(`Configuration file not found or not readable`, configPath);
+    }
+
+    // Read file
+    let content: string;
+    try {
+        content = await fs.readFile(configPath, 'utf-8');
+    } catch (error) {
+        throw new ConfigParseError(
+            'Failed to read configuration file',
+            configPath,
+            error instanceof Error ? error : undefined
+        );
+    }
+
+    // Parse YAML
+    let rawConfig: unknown;
+    try {
+        rawConfig = YAML.parse(content);
+    } catch (error) {
+        throw new ConfigParseError(
+            'Invalid YAML syntax',
+            configPath,
+            error instanceof Error ? error : undefined
+        );
+    }
+
+    // Substitute environment variables
+    const substituted = substituteEnvVarsInObject(rawConfig);
+
+    // Validate and return
+    return parseConfig(substituted, configPath);
+}
+
+/**
+ * Get enabled plugins from config
+ */
+export function getEnabledPlugins(
+    config: ContainerPluginsConfig
+): Array<{ id: string; config: ContainerPluginConfig }> {
+    return Object.entries(config.plugins)
+        .filter(([, pluginConfig]) => (pluginConfig as ContainerPluginConfig).enabled !== false)
+        .map(([id, pluginConfig]) => ({ id, config: pluginConfig as ContainerPluginConfig }));
+}
+
+/**
+ * Get default config path
+ */
+export function getDefaultConfigPath(): string {
+    return process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml';
+}

packages/meta-sort-core/src/container-plugins/ContainerManager.ts

@@ -0,0 +1,771 @@
+/**
+ * Container Manager
+ *
+ * Manages the lifecycle of plugin containers including spawning,
+ * health checking, configuration, and cleanup.
+ */
+
+import { EventEmitter } from 'events';
+import { DockerClient } from './DockerClient.js';
+import {
+    loadConfig,
+    getEnabledPlugins,
+} from './ConfigParser.js';
+import type {
+    ContainerPluginsConfig,
+    ContainerPluginConfig,
+    ContainerPluginManifest,
+    ContainerPluginInstance,
+    ContainerInstanceStatus,
+    ContainerManagerStatus,
+    PluginHealthResponse,
+    PluginConfigureResponse,
+} from './types.js';
+import { config } from '../config/EnvConfig.js';
+
+/**
+ * Container manager events
+ */
+export interface ContainerManagerEvents {
+    'plugin:starting': { pluginId: string; instanceIndex: number };
+    'plugin:started': { pluginId: string; instance: ContainerPluginInstance };
+    'plugin:healthy': { pluginId: string; instance: ContainerPluginInstance };
+    'plugin:unhealthy': { pluginId: string; instance: ContainerPluginInstance; error: string };
+    'plugin:stopped': { pluginId: string; instanceIndex: number };
+    'plugin:error': { pluginId: string; error: Error };
+    'manager:initialized': { pluginCount: number };
+    'manager:shutdown': {};
+}
+
+/**
+ * Container naming helper
+ */
+function getContainerName(pluginId: string, instanceIndex: number): string {
+    return `meta-plugin-${pluginId}-${instanceIndex}`;
+}
+
+/**
+ * Container Manager
+ */
+/**
+ * ============================================================================
+ * PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS
+ * ============================================================================
+ *
+ * Each plugin container has exactly 3 mount points (hardcoded):
+ *
+ *   /files   (READ-ONLY)  - Files to process, same structure as meta-sort
+ *   /output  (READ-WRITE) - Plugin-generated files
+ *   /cache   (READ-WRITE) - Plugin-specific cache
+ *
+ * Host paths are configured via environment variables:
+ *   - PLUGIN_FILE_MOUNTS: Maps subpaths under /files to host paths
+ *     Format: "subpath:host_path,subpath2:host_path2"
+ *     Example: "watch:/data/media,test:/app/test/media"
+ *     Results in: /files/watch, /files/test mounted in container
+ *
+ *   - PLUGIN_CACHE_FOLDER: Base path for /cache mounts
+ *   - PLUGIN_OUTPUT_FOLDER: Base path for /output mounts
+ *
+ * SECURITY: Plugins have read-only access to /files.
+ * Only /cache and /output are writable.
+ *
+ * ============================================================================
+ */
+
+/**
+ * File mount configuration - maps container path to host path
+ */
+interface FileMountConfig {
+    containerPath: string;
+    hostPath: string;
+}
+
+/**
+ * Parse PLUGIN_FILE_MOUNTS env var into mount configs
+ * Format: "container_path:host_path,container_path2:host_path2"
+ */
+function parseFileMounts(mountsStr?: string): FileMountConfig[] {
+    if (!mountsStr) {
+        return [];
+    }
+
+    const mounts: FileMountConfig[] = [];
+    const pairs = mountsStr.split(',').map(s => s.trim()).filter(s => s);
+
+    for (const pair of pairs) {
+        const [containerPath, hostPath] = pair.split(':').map(s => s.trim());
+        if (containerPath && hostPath) {
+            mounts.push({ containerPath, hostPath });
+        } else {
+            console.warn(`[ContainerManager] Invalid mount format: "${pair}" - expected "container_path:host_path"`);
+        }
+    }
+
+    return mounts;
+}
+
+export class ContainerManager extends EventEmitter {
+    private dockerClient: DockerClient;
+    private pluginsConfig: ContainerPluginsConfig | null = null;
+    private instances: Map<string, ContainerPluginInstance[]> = new Map();
+    private healthCheckIntervals: Map<string, NodeJS.Timeout> = new Map();
+    private roundRobinCounters: Map<string, number> = new Map();
+    private initialized: boolean = false;
+    private shutdownInProgress: boolean = false;
+    private fileMounts: FileMountConfig[] = [];
+    private pluginCacheFolder?: string;
+    private pluginOutputFolder?: string;
+    private stackName?: string;
+
+    constructor(
+        private configPath: string = config.CONTAINER_PLUGINS_CONFIG,
+        private network: string = config.CONTAINER_NETWORK,
+        private filesPath: string = config.FILES_PATH,
+        private callbackUrl: string = config.CONTAINER_CALLBACK_URL,
+        private metaCoreUrl: string = config.CONTAINER_META_CORE_URL,
+        dockerClient?: DockerClient
+    ) {
+        super();
+        this.dockerClient = dockerClient || new DockerClient(config.DOCKER_SOCKET_PATH);
+
+        // Parse file mounts from PLUGIN_FILE_MOUNTS env var
+        this.fileMounts = parseFileMounts(config.PLUGIN_FILE_MOUNTS);
+
+        // Configure plugin cache folder (READ-WRITE)
+        this.pluginCacheFolder = config.PLUGIN_CACHE_FOLDER;
+
+        // Configure plugin output folder (READ-WRITE for /plugin-files)
+        this.pluginOutputFolder = config.PLUGIN_OUTPUT_FOLDER;
+
+        // Configure stack name for Docker Desktop grouping
+        this.stackName = config.PLUGIN_STACK_NAME;
+    }
+
+    /**
+     * Initialize the container manager
+     */
+    async initialize(): Promise<void> {
+        if (this.initialized) {
+            return;
+        }
+
+        console.log('[ContainerManager] Initializing...');
+
+        // Log configured host mounts
+        console.log('[ContainerManager] Plugin mount configuration (hardcoded paths):');
+        console.log('  /files   (RO) - Files to process:');
+        if (this.fileMounts.length > 0) {
+            for (const mount of this.fileMounts) {
+                const targetPath = mount.containerPath.startsWith('/files')
+                    ? mount.containerPath
+                    : `/files/${mount.containerPath}`;
+                console.log(`    ${targetPath} -> ${mount.hostPath}`);
+            }
+        } else {
+            console.warn('    (not configured) - Set PLUGIN_FILE_MOUNTS');
+        }
+        console.log(`  /cache  (RW) - Plugin cache:    ${this.pluginCacheFolder || '(not configured)'}`);
+        console.log(`  /output (RW) - Plugin output:   ${this.pluginOutputFolder || '(not configured)'}`)
+
+        // Initialize Docker client
+        await this.dockerClient.initialize();
+
+        // Ensure network exists
+        await this.dockerClient.ensureNetwork(this.network);
+
+        // Load configuration
+        this.pluginsConfig = await loadConfig(this.configPath);
+
+        // Get enabled plugins
+        const enabledPlugins = getEnabledPlugins(this.pluginsConfig);
+
+        console.log(`[ContainerManager] Found ${enabledPlugins.length} enabled plugins`);
+
+        // Clean up any stale containers from previous runs
+        await this.cleanupStaleContainers();
+
+        // Spawn containers for each enabled plugin
+        for (const { id, config: pluginConfig } of enabledPlugins) {
+            try {
+                await this.spawnPlugin(id, pluginConfig);
+            } catch (error) {
+                console.error(`[ContainerManager] Failed to spawn plugin '${id}':`, error);
+                this.emit('plugin:error', {
+                    pluginId: id,
+                    error: error instanceof Error ? error : new Error(String(error)),
+                });
+            }
+        }
+
+        this.initialized = true;
+        this.emit('manager:initialized', { pluginCount: enabledPlugins.length });
+
+        console.log('[ContainerManager] Initialization complete');
+    }
+
+    /**
+     * Shutdown the container manager
+     */
+    async shutdown(): Promise<void> {
+        if (this.shutdownInProgress) {
+            return;
+        }
+
+        this.shutdownInProgress = true;
+        console.log('[ContainerManager] Shutting down...');
+
+        // Stop health checks
+        for (const interval of this.healthCheckIntervals.values()) {
+            clearInterval(interval);
+        }
+        this.healthCheckIntervals.clear();
+
+        // Stop all plugin containers
+        for (const [pluginId, instances] of this.instances) {
+            for (const instance of instances) {
+                try {
+                    await this.stopInstance(instance);
+                } catch (error) {
+                    console.error(
+                        `[ContainerManager] Error stopping ${instance.containerName}:`,
+                        error
+                    );
+                }
+            }
+        }
+
+        this.instances.clear();
+        this.initialized = false;
+        this.shutdownInProgress = false;
+        this.emit('manager:shutdown', {});
+
+        console.log('[ContainerManager] Shutdown complete');
+    }
+
+    /**
+     * Spawn all instances for a plugin
+     */
+    async spawnPlugin(pluginId: string, pluginConfig: ContainerPluginConfig): Promise<void> {
+        const instanceCount = pluginConfig.instances || 1;
+        const instances: ContainerPluginInstance[] = [];
+
+        console.log(
+            `[ContainerManager] Spawning ${instanceCount} instance(s) for plugin '${pluginId}'`
+        );
+
+        // Pull image if needed
+        const imageExists = await this.dockerClient.imageExists(pluginConfig.image);
+        if (!imageExists) {
+            console.log(`[ContainerManager] Pulling image ${pluginConfig.image}...`);
+            await this.dockerClient.pullImage(pluginConfig.image, (event) => {
+                if (event.status) {
+                    console.log(`[ContainerManager] Pull: ${event.status}`);
+                }
+            });
+        }
+
+        // Spawn each instance
+        for (let i = 0; i < instanceCount; i++) {
+            this.emit('plugin:starting', { pluginId, instanceIndex: i });
+
+            const instance = await this.spawnInstance(pluginId, pluginConfig, i);
+            instances.push(instance);
+
+            this.emit('plugin:started', { pluginId, instance });
+        }
+
+        this.instances.set(pluginId, instances);
+
+        // Start health checking
+        this.startHealthCheck(pluginId);
+
+        // Wait for at least one instance to be healthy
+        await this.waitForHealthy(pluginId, 30000);
+
+        // Fetch manifest from first healthy instance
+        const healthyInstance = this.getHealthyInstance(pluginId);
+        if (healthyInstance) {
+            try {
+                const manifest = await this.fetchManifest(healthyInstance);
+                // Update all instances with manifest
+                for (const inst of instances) {
+                    inst.manifest = manifest;
+                }
+            } catch (error) {
+                console.warn(
+                    `[ContainerManager] Failed to fetch manifest for '${pluginId}':`,
+                    error
+                );
+            }
+
+            // Send configuration
+            if (pluginConfig.config) {
+                try {
+                    await this.sendConfig(healthyInstance, pluginConfig.config);
+                } catch (error) {
+                    console.warn(
+                        `[ContainerManager] Failed to configure '${pluginId}':`,
+                        error
+                    );
+                }
+            }
+        }
+    }
+
+    /**
+     * Spawn a single container instance
+     */
+    private async spawnInstance(
+        pluginId: string,
+        pluginConfig: ContainerPluginConfig,
+        instanceIndex: number
+    ): Promise<ContainerPluginInstance> {
+        const containerName = getContainerName(pluginId, instanceIndex);
+
+        // Check if container already exists
+        const existingContainers = await this.dockerClient.listContainers({
+            name: [containerName],
+        });
+
+        if (existingContainers.length > 0) {
+            const existing = existingContainers[0];
+            // Remove existing container
+            console.log(`[ContainerManager] Removing existing container ${containerName}`);
+            await this.dockerClient.removeContainer(existing.Id, true);
+        }
+
+        /**
+         * PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS:
+         *   /files   (READ-ONLY)  - Files to process
+         *   /output  (READ-WRITE) - Plugin-generated files
+         *   /cache   (READ-WRITE) - Plugin-specific cache
+         */
+        const mounts: Array<{ source: string; target: string; readonly: boolean }> = [];
+
+        // /files/* (READ-ONLY) - Mount each configured subpath under /files
+        for (const fileMount of this.fileMounts) {
+            // fileMount.containerPath is the subpath (e.g., "watch", "test")
+            // Mount at /files/{subpath}
+            const targetPath = fileMount.containerPath.startsWith('/files')
+                ? fileMount.containerPath
+                : `/files/${fileMount.containerPath}`;
+            mounts.push({
+                source: fileMount.hostPath,
+                target: targetPath,
+                readonly: true,
+            });
+        }
+
+        if (this.fileMounts.length === 0) {
+            console.warn(`[ContainerManager] WARNING: No file mounts configured. Plugin '${pluginId}' won't have access to files.`);
+        }
+
+        // /cache (READ-WRITE) - Plugin-specific cache
+        if (this.pluginCacheFolder) {
+            mounts.push({
+                source: `${this.pluginCacheFolder}/${pluginId}`,
+                target: '/cache',
+                readonly: false,
+            });
+        }
+
+        // /output (READ-WRITE) - Plugin-generated files
+        if (this.pluginOutputFolder) {
+            mounts.push({
+                source: `${this.pluginOutputFolder}/${pluginId}`,
+                target: '/output',
+                readonly: false,
+            });
+        }
+
+        // Create container
+        const containerId = await this.dockerClient.createContainer({
+            image: pluginConfig.image,
+            name: containerName,
+            network: this.network,
+            resources: pluginConfig.resources,
+            mounts,
+            labels: {
+                'meta-mesh.plugin': 'true',
+                'meta-mesh.plugin.id': pluginId,
+                'meta-mesh.plugin.instance': String(instanceIndex),
+            },
+            env: {
+                PLUGIN_ID: pluginId,
+                META_CORE_URL: this.metaCoreUrl,
+                CALLBACK_URL: this.callbackUrl,
+            },
+            // Docker Desktop grouping
+            stackName: this.stackName,
+            serviceName: pluginId,
+        });
+
+        // Start container
+        await this.dockerClient.startContainer(containerId);
+
+        // Create instance record
+        const instance: ContainerPluginInstance = {
+            pluginId,
+            containerId,
+            containerName,
+            baseUrl: `http://${containerName}:8080`,
+            instanceIndex,
+            status: 'starting',
+            tasksProcessed: 0,
+            tasksFailed: 0,
+        };
+
+        console.log(`[ContainerManager] Started container ${containerName}`);
+
+        return instance;
+    }
+
+    /**
+     * Stop a container instance
+     */
+    private async stopInstance(instance: ContainerPluginInstance): Promise<void> {
+        console.log(`[ContainerManager] Stopping ${instance.containerName}...`);
+
+        try {
+            await this.dockerClient.stopContainer(instance.containerId, 10);
+        } catch (error) {
+            // Container might already be stopped
+        }
+
+        try {
+            await this.dockerClient.removeContainer(instance.containerId, true);
+        } catch (error) {
+            // Container might already be removed
+        }
+
+        instance.status = 'stopped';
+        this.emit('plugin:stopped', {
+            pluginId: instance.pluginId,
+            instanceIndex: instance.instanceIndex,
+        });
+    }
+
+    /**
+     * Start health check loop for a plugin
+     */
+    private startHealthCheck(pluginId: string, intervalMs: number = 30000): void {
+        // Clear existing interval if any
+        const existing = this.healthCheckIntervals.get(pluginId);
+        if (existing) {
+            clearInterval(existing);
+        }
+
+        const checkHealth = async () => {
+            const instances = this.instances.get(pluginId);
+            if (!instances) return;
+
+            for (const instance of instances) {
+                if (instance.status === 'stopped') continue;
+
+                try {
+                    const healthy = await this.checkHealth(instance);
+                    const previousStatus = instance.status;
+
+                    if (healthy) {
+                        instance.status = 'healthy';
+                        instance.lastHealthCheck = Date.now();
+
+                        if (previousStatus !== 'healthy') {
+                            this.emit('plugin:healthy', { pluginId, instance });
+                        }
+                    } else {
+                        instance.status = 'unhealthy';
+                        instance.lastError = 'Health check failed';
+
+                        if (previousStatus === 'healthy') {
+                            this.emit('plugin:unhealthy', {
+                                pluginId,
+                                instance,
+                                error: 'Health check failed',
+                            });
+                        }
+                    }
+                } catch (error) {
+                    instance.status = 'unhealthy';
+                    instance.lastError = error instanceof Error ? error.message : String(error);
+
+                    this.emit('plugin:unhealthy', {
+                        pluginId,
+                        instance,
+                        error: instance.lastError,
+                    });
+                }
+            }
+        };
+
+        // Initial check
+        checkHealth();
+
+        // Schedule periodic checks
+        const interval = setInterval(checkHealth, intervalMs);
+        this.healthCheckIntervals.set(pluginId, interval);
+    }
+
+    /**
+     * Check health of a container instance
+     */
+    async checkHealth(instance: ContainerPluginInstance): Promise<boolean> {
+        try {
+            const response = await fetch(`${instance.baseUrl}/health`, {
+                method: 'GET',
+                signal: AbortSignal.timeout(5000),
+            });
+
+            if (!response.ok) {
+                return false;
+            }
+
+            const data = (await response.json()) as PluginHealthResponse;
+            return data.status === 'healthy' && data.ready === true;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Wait for at least one healthy instance
+     */
+    private async waitForHealthy(pluginId: string, timeoutMs: number): Promise<void> {
+        const startTime = Date.now();
+        const checkInterval = 1000;
+
+        while (Date.now() - startTime < timeoutMs) {
+            const instances = this.instances.get(pluginId);
+            if (!instances) return;
+
+            for (const instance of instances) {
+                const healthy = await this.checkHealth(instance);
+                if (healthy) {
+                    instance.status = 'healthy';
+                    instance.lastHealthCheck = Date.now();
+                    this.emit('plugin:healthy', { pluginId, instance });
+                    return;
+                }
+            }
+
+            await new Promise((resolve) => setTimeout(resolve, checkInterval));
+        }
+
+        console.warn(`[ContainerManager] Timeout waiting for healthy instance of '${pluginId}'`);
+    }
+
+    /**
+     * Fetch manifest from a container instance
+     */
+    async fetchManifest(instance: ContainerPluginInstance): Promise<ContainerPluginManifest> {
+        const response = await fetch(`${instance.baseUrl}/manifest`, {
+            method: 'GET',
+            signal: AbortSignal.timeout(10000),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Failed to fetch manifest: ${response.status}`);
+        }
+
+        return response.json() as Promise<ContainerPluginManifest>;
+    }
+
+    /**
+     * Send configuration to a container instance
+     */
+    async sendConfig(
+        instance: ContainerPluginInstance,
+        pluginConfig: Record<string, unknown>
+    ): Promise<void> {
+        const response = await fetch(`${instance.baseUrl}/configure`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify({ config: pluginConfig }),
+            signal: AbortSignal.timeout(10000),
+        });
+
+        if (!response.ok) {
+            throw new Error(`Failed to configure plugin: ${response.status}`);
+        }
+
+        const data = (await response.json()) as PluginConfigureResponse;
+        if (!data.success) {
+            throw new Error(`Plugin configuration failed: ${data.error}`);
+        }
+    }
+
+    /**
+     * Get a healthy instance for a plugin (round-robin)
+     */
+    getHealthyInstance(pluginId: string): ContainerPluginInstance | null {
+        const instances = this.instances.get(pluginId);
+        if (!instances || instances.length === 0) {
+            return null;
+        }
+
+        const healthyInstances = instances.filter((i) => i.status === 'healthy');
+        if (healthyInstances.length === 0) {
+            return null;
+        }
+
+        // Round-robin selection
+        const counter = this.roundRobinCounters.get(pluginId) || 0;
+        const selected = healthyInstances[counter % healthyInstances.length];
+        this.roundRobinCounters.set(pluginId, counter + 1);
+
+        return selected;
+    }
+
+    /**
+     * Get all instances for a plugin
+     */
+    getInstances(pluginId: string): ContainerPluginInstance[] {
+        return this.instances.get(pluginId) || [];
+    }
+
+    /**
+     * Get all plugins
+     */
+    getPluginIds(): string[] {
+        return Array.from(this.instances.keys());
+    }
+
+    /**
+     * Check if a plugin is a container plugin
+     */
+    isContainerPlugin(pluginId: string): boolean {
+        return this.instances.has(pluginId);
+    }
+
+    /**
+     * Get plugin manifest
+     */
+    getPluginManifest(pluginId: string): ContainerPluginManifest | undefined {
+        const instances = this.instances.get(pluginId);
+        if (!instances || instances.length === 0) {
+            return undefined;
+        }
+        return instances[0].manifest;
+    }
+
+    /**
+     * Get manager status
+     */
+    getStatus(): ContainerManagerStatus {
+        const plugins: ContainerManagerStatus['plugins'] = [];
+
+        for (const [pluginId, instances] of this.instances) {
+            const pluginConfig = this.pluginsConfig?.plugins[pluginId];
+            const healthyCount = instances.filter((i) => i.status === 'healthy').length;
+
+            plugins.push({
+                pluginId,
+                enabled: pluginConfig?.enabled !== false,
+                image: pluginConfig?.image || 'unknown',
+                instances: instances.length,
+                healthyInstances: healthyCount,
+                manifest: instances[0]?.manifest,
+            });
+        }
+
+        const totalInstances = Array.from(this.instances.values()).reduce(
+            (sum, instances) => sum + instances.length,
+            0
+        );
+
+        const healthyInstances = Array.from(this.instances.values()).reduce(
+            (sum, instances) => sum + instances.filter((i) => i.status === 'healthy').length,
+            0
+        );
+
+        return {
+            initialized: this.initialized,
+            pluginCount: this.instances.size,
+            runningContainers: totalInstances,
+            healthyContainers: healthyInstances,
+            plugins,
+        };
+    }
+
+    /**
+     * Restart a plugin (stop and respawn all instances)
+     */
+    async restartPlugin(pluginId: string): Promise<void> {
+        console.log(`[ContainerManager] Restarting plugin '${pluginId}'...`);
+
+        const instances = this.instances.get(pluginId);
+        const pluginConfig = this.pluginsConfig?.plugins[pluginId];
+
+        if (!instances || !pluginConfig) {
+            throw new Error(`Plugin '${pluginId}' not found`);
+        }
+
+        // Stop health check
+        const interval = this.healthCheckIntervals.get(pluginId);
+        if (interval) {
+            clearInterval(interval);
+            this.healthCheckIntervals.delete(pluginId);
+        }
+
+        // Stop all instances
+        for (const instance of instances) {
+            await this.stopInstance(instance);
+        }
+
+        this.instances.delete(pluginId);
+
+        // Respawn
+        await this.spawnPlugin(pluginId, pluginConfig);
+    }
+
+    /**
+     * Clean up stale containers from previous runs
+     */
+    private async cleanupStaleContainers(): Promise<void> {
+        console.log('[ContainerManager] Cleaning up stale containers...');
+
+        const staleContainers = await this.dockerClient.listPluginContainers();
+
+        for (const container of staleContainers) {
+            console.log(
+                `[ContainerManager] Removing stale container ${container.Names?.[0] || container.Id}`
+            );
+            try {
+                await this.dockerClient.removeContainer(container.Id, true);
+            } catch (error) {
+                console.warn(
+                    `[ContainerManager] Failed to remove stale container:`,
+                    error
+                );
+            }
+        }
+    }
+
+    /**
+     * Record task completion for statistics
+     */
+    recordTaskCompletion(pluginId: string, instanceName: string, success: boolean): void {
+        const instances = this.instances.get(pluginId);
+        if (!instances) return;
+
+        const instance = instances.find((i) => i.containerName === instanceName);
+        if (!instance) return;
+
+        if (success) {
+            instance.tasksProcessed++;
+        } else {
+            instance.tasksFailed++;
+        }
+    }
+}
+
+/**
+ * Create a container manager instance
+ */
+export function createContainerManager(
+    configPath?: string,
+    network?: string
+): ContainerManager {
+    return new ContainerManager(configPath, network);
+}

packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts

@@ -0,0 +1,651 @@
+/**
+ * Container Plugin Scheduler
+ *
+ * Handles dispatching tasks to containerized plugins and
+ * processing completion callbacks.
+ */
+
+import { EventEmitter } from 'events';
+import PQueue from 'p-queue';
+import { randomUUID } from 'crypto';
+import type { ContainerManager } from './ContainerManager.js';
+import type {
+    ContainerTask,
+    ContainerTaskStatus,
+    PluginProcessRequest,
+    PluginProcessResponse,
+    PluginCallbackPayload,
+    CallbackStatus,
+} from './types.js';
+import type { TaskQueueType } from '../plugin-engine/types.js';
+import type { IKVClient } from '../kv/IKVClient.js';
+import { config } from '../config/EnvConfig.js';
+
+/**
+ * Scheduler events
+ */
+export interface ContainerSchedulerEvents {
+    'task:created': { task: ContainerTask };
+    'task:dispatched': { task: ContainerTask };
+    'task:completed': { task: ContainerTask };
+    'task:failed': { task: ContainerTask; error: string };
+    'task:timeout': { task: ContainerTask };
+    'file:complete': { fileHash: string; filePath: string };
+}
+
+/**
+ * Pending callback info
+ */
+interface PendingCallback {
+    task: ContainerTask;
+    resolve: (result: PluginCallbackPayload) => void;
+    reject: (error: Error) => void;
+    timeoutHandle: NodeJS.Timeout;
+}
+
+/**
+ * Container Plugin Scheduler
+ */
+export class ContainerPluginScheduler extends EventEmitter {
+    private containerManager: ContainerManager;
+    private fastQueue: PQueue;
+    private backgroundQueue: PQueue;
+    private pendingTasks: Map<string, ContainerTask> = new Map();
+    private pendingCallbacks: Map<string, PendingCallback> = new Map();
+    private fileTasks: Map<string, Set<string>> = new Map(); // fileHash -> taskIds
+    private fileCompletedPlugins: Map<string, Set<string>> = new Map(); // fileHash -> completed pluginIds
+    private dependencyWaiters: Map<string, Array<() => void>> = new Map(); // "fileHash:pluginId" -> resolve callbacks
+    private callbackUrl: string;
+    private metaCoreUrl: string;
+    private defaultTimeout: number = 60000; // 60 seconds
+    private kvClient: IKVClient | null = null;
+
+    constructor(
+        containerManager: ContainerManager,
+        options?: {
+            fastConcurrency?: number;
+            backgroundConcurrency?: number;
+            callbackUrl?: string;
+            metaCoreUrl?: string;
+            defaultTimeout?: number;
+            kvClient?: IKVClient;
+        }
+    ) {
+        super();
+        this.containerManager = containerManager;
+
+        this.fastQueue = new PQueue({
+            concurrency: options?.fastConcurrency ?? config.FAST_QUEUE_CONCURRENCY,
+        });
+
+        this.backgroundQueue = new PQueue({
+            concurrency: options?.backgroundConcurrency ?? config.BACKGROUND_QUEUE_CONCURRENCY,
+        });
+
+        this.callbackUrl = options?.callbackUrl ?? `${config.CONTAINER_CALLBACK_URL}/api/plugins/callback`;
+        this.metaCoreUrl = options?.metaCoreUrl ?? config.CONTAINER_META_CORE_URL;
+        this.defaultTimeout = options?.defaultTimeout ?? 60000;
+        this.kvClient = options?.kvClient ?? null;
+    }
+
+    /**
+     * Wait for dependency plugins to complete for a file
+     */
+    private async waitForDependencies(
+        fileHash: string,
+        dependencies: string[],
+        timeoutMs: number = 30000
+    ): Promise<void> {
+        if (!dependencies || dependencies.length === 0) {
+            return;
+        }
+
+        const completedPlugins = this.fileCompletedPlugins.get(fileHash) || new Set();
+        const pendingDeps = dependencies.filter(dep => !completedPlugins.has(dep));
+
+        if (pendingDeps.length === 0) {
+            return; // All dependencies already completed
+        }
+
+        // Wait for each pending dependency
+        const waitPromises = pendingDeps.map(depPluginId => {
+            return new Promise<void>((resolve, reject) => {
+                const key = `${fileHash}:${depPluginId}`;
+
+                // Check again in case it completed while we were setting up
+                const currentCompleted = this.fileCompletedPlugins.get(fileHash);
+                if (currentCompleted?.has(depPluginId)) {
+                    resolve();
+                    return;
+                }
+
+                // Set up timeout
+                const timeoutHandle = setTimeout(() => {
+                    // Remove waiter on timeout
+                    const waiters = this.dependencyWaiters.get(key);
+                    if (waiters) {
+                        const idx = waiters.indexOf(resolve);
+                        if (idx >= 0) waiters.splice(idx, 1);
+                    }
+                    // Resolve anyway to not block - plugin will get stale metadata but won't hang
+                    console.warn(`[ContainerScheduler] Timeout waiting for dependency '${depPluginId}' for file ${fileHash}`);
+                    resolve();
+                }, timeoutMs);
+
+                // Register waiter
+                if (!this.dependencyWaiters.has(key)) {
+                    this.dependencyWaiters.set(key, []);
+                }
+                this.dependencyWaiters.get(key)!.push(() => {
+                    clearTimeout(timeoutHandle);
+                    resolve();
+                });
+            });
+        });
+
+        await Promise.all(waitPromises);
+    }
+
+    /**
+     * Mark a plugin as completed for a file and notify waiters
+     */
+    private markPluginCompleted(fileHash: string, pluginId: string, filePath?: string): void {
+        // Track completion
+        if (!this.fileCompletedPlugins.has(fileHash)) {
+            this.fileCompletedPlugins.set(fileHash, new Set());
+        }
+        this.fileCompletedPlugins.get(fileHash)!.add(pluginId);
+
+        // Notify waiters
+        const key = `${fileHash}:${pluginId}`;
+        const waiters = this.dependencyWaiters.get(key);
+        if (waiters) {
+            for (const resolve of waiters) {
+                resolve();
+            }
+            this.dependencyWaiters.delete(key);
+        }
+
+        // Publish to Redis for external subscribers (e.g., meta-stremio)
+        this.publishPluginComplete(fileHash, pluginId, filePath || '');
+    }
+
+    /**
+     * Publish plugin completion event to Redis
+     * Channel: meta-sort:plugin:complete
+     *
+     * External services (like meta-stremio) can subscribe to this channel
+     * to be notified when specific plugins complete for a file.
+     */
+    private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise<void> {
+        if (!this.kvClient) {
+            return;
+        }
+
+        try {
+            const message = JSON.stringify({
+                fileHash,
+                pluginId,
+                filePath,
+                timestamp: Date.now()
+            });
+            await (this.kvClient as any).publish('meta-sort:plugin:complete', message);
+        } catch (error) {
+            // Silent fail - pub/sub is optional
+        }
+    }
+
+    /**
+     * Create a task for a container plugin
+     */
+    createTask(
+        pluginId: string,
+        fileHash: string,
+        filePath: string,
+        dependencies: string[] = [],
+        queue: TaskQueueType = 'fast'
+    ): ContainerTask {
+        const task: ContainerTask = {
+            id: randomUUID(),
+            fileHash,
+            filePath,
+            pluginId,
+            dependencies,
+            queue,
+            status: 'pending',
+            createdAt: Date.now(),
+        };
+
+        this.pendingTasks.set(task.id, task);
+
+        // Track tasks by file
+        if (!this.fileTasks.has(fileHash)) {
+            this.fileTasks.set(fileHash, new Set());
+        }
+        this.fileTasks.get(fileHash)!.add(task.id);
+
+        this.emit('task:created', { task });
+
+        return task;
+    }
+
+    /**
+     * Enqueue a task for execution
+     */
+    enqueueTask(task: ContainerTask): void {
+        const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
+
+        queue.add(async () => {
+            await this.executeTask(task);
+        });
+    }
+
+    /**
+     * Execute a task (dispatch to container)
+     */
+    private async executeTask(task: ContainerTask): Promise<void> {
+        // Wait for dependencies to complete first
+        if (task.dependencies && task.dependencies.length > 0) {
+            task.status = 'waiting';
+            await this.waitForDependencies(task.fileHash, task.dependencies);
+        }
+
+        const instance = this.containerManager.getHealthyInstance(task.pluginId);
+
+        if (!instance) {
+            task.status = 'failed';
+            task.error = `No healthy instance available for plugin '${task.pluginId}'`;
+            this.emit('task:failed', { task, error: task.error });
+            this.checkFileCompletion(task.fileHash);
+            return;
+        }
+
+        task.status = 'dispatched';
+        task.dispatchedAt = Date.now();
+        task.instanceName = instance.containerName;
+
+        this.emit('task:dispatched', { task });
+
+        try {
+            // Fetch fresh metadata after dependencies complete
+            // This ensures we get metadata written by dependency plugins
+            const existingMeta = await this.fetchExistingMetadata(task.fileHash);
+
+            // Build process request
+            const request: PluginProcessRequest = {
+                taskId: task.id,
+                cid: task.fileHash,
+                filePath: task.filePath,
+                callbackUrl: this.callbackUrl,
+                metaCoreUrl: this.metaCoreUrl,
+                existingMeta,
+            };
+
+            // Dispatch to container
+            const response = await this.dispatchToContainer(instance.baseUrl, request);
+
+            if (response.status !== 'accepted') {
+                throw new Error(response.error || 'Task rejected by plugin');
+            }
+
+            // Wait for callback with timeout
+            const manifest = instance.manifest;
+            const timeout = manifest?.timeout ?? this.defaultTimeout;
+
+            await this.waitForCallback(task, timeout);
+
+        } catch (error) {
+            task.status = 'failed';
+            task.error = error instanceof Error ? error.message : String(error);
+            task.completedAt = Date.now();
+
+            if (task.dispatchedAt) {
+                task.duration = task.completedAt - task.dispatchedAt;
+            }
+
+            this.containerManager.recordTaskCompletion(
+                task.pluginId,
+                task.instanceName!,
+                false
+            );
+
+            this.emit('task:failed', { task, error: task.error });
+        }
+
+        this.checkFileCompletion(task.fileHash);
+    }
+
+    /**
+     * Dispatch task to container plugin
+     */
+    private async dispatchToContainer(
+        baseUrl: string,
+        request: PluginProcessRequest
+    ): Promise<PluginProcessResponse> {
+        const response = await fetch(`${baseUrl}/process`, {
+            method: 'POST',
+            headers: { 'Content-Type': 'application/json' },
+            body: JSON.stringify(request),
+            signal: AbortSignal.timeout(10000), // 10s timeout for dispatch
+        });
+
+        if (!response.ok) {
+            throw new Error(`Plugin returned ${response.status}: ${await response.text()}`);
+        }
+
+        return response.json() as Promise<PluginProcessResponse>;
+    }
+
+    /**
+     * Wait for callback from container
+     */
+    private waitForCallback(task: ContainerTask, timeoutMs: number): Promise<PluginCallbackPayload> {
+        return new Promise((resolve, reject) => {
+            const timeoutHandle = setTimeout(() => {
+                this.pendingCallbacks.delete(task.id);
+                task.status = 'timeout';
+                task.completedAt = Date.now();
+                if (task.dispatchedAt) {
+                    task.duration = task.completedAt - task.dispatchedAt;
+                }
+                this.emit('task:timeout', { task });
+                reject(new Error(`Task timeout after ${timeoutMs}ms`));
+            }, timeoutMs);
+
+            this.pendingCallbacks.set(task.id, {
+                task,
+                resolve,
+                reject,
+                timeoutHandle,
+            });
+        });
+    }
+
+    /**
+     * Handle callback from container plugin
+     */
+    handleCallback(payload: PluginCallbackPayload): boolean {
+        const pending = this.pendingCallbacks.get(payload.taskId);
+
+        if (!pending) {
+            console.warn(
+                `[ContainerScheduler] Received callback for unknown task: ${payload.taskId}`
+            );
+            return false;
+        }
+
+        // Clear timeout
+        clearTimeout(pending.timeoutHandle);
+        this.pendingCallbacks.delete(payload.taskId);
+
+        const task = pending.task;
+        task.completedAt = Date.now();
+        task.duration = payload.duration;
+
+        if (payload.status === 'completed') {
+            task.status = 'completed';
+
+            this.containerManager.recordTaskCompletion(
+                task.pluginId,
+                task.instanceName!,
+                true
+            );
+
+            // Mark plugin as completed for dependency tracking
+            this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
+
+            this.emit('task:completed', { task });
+            pending.resolve(payload);
+
+        } else if (payload.status === 'failed') {
+            task.status = 'failed';
+            task.error = payload.error;
+
+            this.containerManager.recordTaskCompletion(
+                task.pluginId,
+                task.instanceName!,
+                false
+            );
+
+            // Still mark as completed for dependency purposes (don't block other plugins)
+            this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
+
+            this.emit('task:failed', { task, error: payload.error || 'Unknown error' });
+            pending.reject(new Error(payload.error || 'Task failed'));
+
+        } else if (payload.status === 'skipped') {
+            task.status = 'completed'; // Treat skipped as completed
+            task.error = payload.reason;
+
+            this.containerManager.recordTaskCompletion(
+                task.pluginId,
+                task.instanceName!,
+                true
+            );
+
+            // Mark plugin as completed for dependency tracking
+            this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
+
+            this.emit('task:completed', { task });
+            pending.resolve(payload);
+        }
+
+        return true;
+    }
+
+    /**
+     * Check if all tasks for a file are complete
+     */
+    private checkFileCompletion(fileHash: string): void {
+        const taskIds = this.fileTasks.get(fileHash);
+        if (!taskIds) return;
+
+        let allComplete = true;
+        let filePath: string | undefined;
+
+        for (const taskId of taskIds) {
+            const task = this.pendingTasks.get(taskId);
+            if (!task) continue;
+
+            filePath = task.filePath;
+
+            if (
+                task.status !== 'completed' &&
+                task.status !== 'failed' &&
+                task.status !== 'timeout'
+            ) {
+                allComplete = false;
+                break;
+            }
+        }
+
+        if (allComplete && filePath) {
+            // Clean up task tracking
+            for (const taskId of taskIds) {
+                this.pendingTasks.delete(taskId);
+            }
+            this.fileTasks.delete(fileHash);
+
+            // Clean up dependency tracking
+            this.fileCompletedPlugins.delete(fileHash);
+            // Clean up any stale dependency waiters for this file
+            for (const key of this.dependencyWaiters.keys()) {
+                if (key.startsWith(`${fileHash}:`)) {
+                    this.dependencyWaiters.delete(key);
+                }
+            }
+
+            this.emit('file:complete', { fileHash, filePath });
+        }
+    }
+
+    /**
+     * Fetch existing metadata for a file
+     */
+    private async fetchExistingMetadata(fileHash: string): Promise<Record<string, string>> {
+        try {
+            const response = await fetch(`${this.metaCoreUrl}/meta/${fileHash}`, {
+                method: 'GET',
+                signal: AbortSignal.timeout(5000),
+            });
+
+            if (!response.ok) {
+                return {};
+            }
+
+            const data = await response.json() as { metadata?: Record<string, string> };
+            return data.metadata || {};
+        } catch {
+            return {};
+        }
+    }
+
+    /**
+     * Get queue status
+     */
+    getQueueStatus(): {
+        fast: { pending: number; running: number };
+        background: { pending: number; running: number };
+        pendingTasks: number;
+        pendingCallbacks: number;
+    } {
+        return {
+            fast: {
+                pending: this.fastQueue.pending,
+                running: this.fastQueue.pending - this.fastQueue.size,
+            },
+            background: {
+                pending: this.backgroundQueue.pending,
+                running: this.backgroundQueue.pending - this.backgroundQueue.size,
+            },
+            pendingTasks: this.pendingTasks.size,
+            pendingCallbacks: this.pendingCallbacks.size,
+        };
+    }
+
+    /**
+     * Get pending tasks for a file
+     */
+    getPendingTasksForFile(fileHash: string): ContainerTask[] {
+        const taskIds = this.fileTasks.get(fileHash);
+        if (!taskIds) return [];
+
+        return Array.from(taskIds)
+            .map(id => this.pendingTasks.get(id))
+            .filter((t): t is ContainerTask => t !== undefined);
+    }
+
+    /**
+     * Cancel all tasks for a file
+     */
+    cancelFile(fileHash: string): void {
+        const taskIds = this.fileTasks.get(fileHash);
+        if (!taskIds) return;
+
+        for (const taskId of taskIds) {
+            const pending = this.pendingCallbacks.get(taskId);
+            if (pending) {
+                clearTimeout(pending.timeoutHandle);
+                pending.reject(new Error('Task cancelled'));
+                this.pendingCallbacks.delete(taskId);
+            }
+            this.pendingTasks.delete(taskId);
+        }
+
+        this.fileTasks.delete(fileHash);
+    }
+
+    /**
+     * Clear all queues
+     */
+    clear(): void {
+        this.fastQueue.clear();
+        this.backgroundQueue.clear();
+
+        for (const pending of this.pendingCallbacks.values()) {
+            clearTimeout(pending.timeoutHandle);
+            pending.reject(new Error('Scheduler cleared'));
+        }
+
+        this.pendingCallbacks.clear();
+        this.pendingTasks.clear();
+        this.fileTasks.clear();
+        this.fileCompletedPlugins.clear();
+        this.dependencyWaiters.clear();
+    }
+
+    /**
+     * Pause queues
+     */
+    pause(): void {
+        this.fastQueue.pause();
+        this.backgroundQueue.pause();
+    }
+
+    /**
+     * Resume queues
+     */
+    resume(): void {
+        this.fastQueue.start();
+        this.backgroundQueue.start();
+    }
+
+    /**
+     * Dispatch tasks to all enabled plugins for a file
+     * This creates and enqueues tasks for all plugins that should process this file
+     */
+    async dispatchAllPlugins(
+        fileHash: string,
+        filePath: string,
+        existingMeta: Record<string, string>
+    ): Promise<void> {
+        const status = this.containerManager.getStatus();
+
+        if (!status.initialized || status.pluginCount === 0) {
+            return; // No plugins available
+        }
+
+        console.log(`[ContainerScheduler] Dispatching tasks for ${fileHash} to ${status.pluginCount} plugins`);
+
+        // Get plugin info and dispatch tasks
+        for (const pluginStatus of status.plugins) {
+            if (!pluginStatus.enabled || pluginStatus.healthyInstances === 0) {
+                continue; // Skip disabled or unhealthy plugins
+            }
+
+            const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId);
+            const queue: TaskQueueType = manifest?.defaultQueue || 'fast';
+            const dependencies = manifest?.dependencies || [];
+
+            // Create and enqueue task
+            const task = this.createTask(
+                pluginStatus.pluginId,
+                fileHash,
+                filePath,
+                dependencies,
+                queue
+            );
+
+            // Store existing metadata in task for dispatch
+            (task as any).existingMeta = existingMeta;
+
+            this.enqueueTask(task);
+        }
+    }
+}
+
+/**
+ * Create a container plugin scheduler
+ */
+export function createContainerPluginScheduler(
+    containerManager: ContainerManager,
+    options?: {
+        fastConcurrency?: number;
+        backgroundConcurrency?: number;
+        callbackUrl?: string;
+        metaCoreUrl?: string;
+        kvClient?: IKVClient;
+    }
+): ContainerPluginScheduler {
+    return new ContainerPluginScheduler(containerManager, options);
+}

packages/meta-sort-core/src/container-plugins/DockerClient.ts

@@ -0,0 +1,417 @@
+/**
+ * Docker Client
+ *
+ * Wrapper around the Docker Engine API for container management.
+ * Requires: npm install dockerode @types/dockerode
+ */
+
+import Docker from 'dockerode';
+import type { Container, ContainerInfo, ContainerCreateOptions } from 'dockerode';
+import type { Readable } from 'stream';
+import type { ContainerSpawnOptions, ContainerResourceLimits } from './types.js';
+
+/**
+ * Docker client for container operations
+ */
+export class DockerClient {
+    private docker: Docker;
+    private initialized: boolean = false;
+
+    constructor(socketPath: string = '/var/run/docker.sock') {
+        this.docker = new Docker({ socketPath });
+    }
+
+    /**
+     * Initialize and verify Docker connection
+     */
+    async initialize(): Promise<void> {
+        try {
+            await this.docker.ping();
+            this.initialized = true;
+        } catch (error) {
+            throw new Error(`Failed to connect to Docker: ${error instanceof Error ? error.message : error}`);
+        }
+    }
+
+    /**
+     * Check if Docker is connected
+     */
+    isInitialized(): boolean {
+        return this.initialized;
+    }
+
+    /**
+     * Get Docker info
+     */
+    async getInfo(): Promise<any> {
+        return this.docker.info();
+    }
+
+    /**
+     * Pull a Docker image
+     */
+    async pullImage(imageName: string, onProgress?: (event: any) => void): Promise<void> {
+        return new Promise((resolve, reject) => {
+            this.docker.pull(imageName, (err: Error | null, stream: NodeJS.ReadableStream) => {
+                if (err) {
+                    reject(err);
+                    return;
+                }
+
+                this.docker.modem.followProgress(
+                    stream,
+                    (err: Error | null) => {
+                        if (err) {
+                            reject(err);
+                        } else {
+                            resolve();
+                        }
+                    },
+                    onProgress
+                );
+            });
+        });
+    }
+
+    /**
+     * Check if an image exists locally
+     */
+    async imageExists(imageName: string): Promise<boolean> {
+        try {
+            await this.docker.getImage(imageName).inspect();
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Create a container
+     */
+    async createContainer(options: ContainerSpawnOptions): Promise<string> {
+        // Build labels, adding Docker Compose labels if stackName is provided
+        const labels: Record<string, string> = {
+            'meta-mesh.plugin': 'true',
+            ...options.labels,
+        };
+
+        // Add compose labels for Docker Desktop grouping
+        if (options.stackName) {
+            labels['com.docker.compose.project'] = options.stackName;
+            if (options.serviceName) {
+                labels['com.docker.compose.service'] = options.serviceName;
+            }
+        }
+
+        const createOptions: ContainerCreateOptions = {
+            Image: options.image,
+            name: options.name,
+            Env: options.env
+                ? Object.entries(options.env).map(([k, v]) => `${k}=${v}`)
+                : undefined,
+            Labels: labels,
+            HostConfig: {
+                Binds: options.mounts?.map(m =>
+                    `${m.source}:${m.target}:${m.readonly ? 'ro' : 'rw'}`
+                ),
+                NetworkMode: options.network,
+                Memory: options.resources?.memory
+                    ? parseMemoryLimit(options.resources.memory)
+                    : undefined,
+                NanoCpus: options.resources?.cpus
+                    ? Math.floor(options.resources.cpus * 1e9)
+                    : undefined,
+                RestartPolicy: {
+                    Name: 'unless-stopped',
+                },
+            },
+            ExposedPorts: {
+                '8080/tcp': {},
+            },
+        };
+
+        const container = await this.docker.createContainer(createOptions);
+        return container.id;
+    }
+
+    /**
+     * Start a container
+     */
+    async startContainer(containerId: string): Promise<void> {
+        const container = this.docker.getContainer(containerId);
+        await container.start();
+    }
+
+    /**
+     * Stop a container
+     */
+    async stopContainer(containerId: string, timeout: number = 10): Promise<void> {
+        const container = this.docker.getContainer(containerId);
+        await container.stop({ t: timeout });
+    }
+
+    /**
+     * Remove a container
+     */
+    async removeContainer(containerId: string, force: boolean = false): Promise<void> {
+        const container = this.docker.getContainer(containerId);
+        await container.remove({ force });
+    }
+
+    /**
+     * Get container info
+     */
+    async inspectContainer(containerId: string): Promise<Docker.ContainerInspectInfo> {
+        const container = this.docker.getContainer(containerId);
+        return container.inspect();
+    }
+
+    /**
+     * Check if a container exists
+     */
+    async containerExists(containerId: string): Promise<boolean> {
+        try {
+            await this.inspectContainer(containerId);
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Check if a container is running
+     */
+    async isContainerRunning(containerId: string): Promise<boolean> {
+        try {
+            const info = await this.inspectContainer(containerId);
+            return info.State?.Running === true;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Get container IP address on a specific network
+     */
+    async getContainerIP(containerId: string, network: string): Promise<string | null> {
+        try {
+            const info = await this.inspectContainer(containerId);
+            const networkSettings = info.NetworkSettings?.Networks?.[network];
+            return networkSettings?.IPAddress || null;
+        } catch {
+            return null;
+        }
+    }
+
+    /**
+     * List containers with optional filters
+     */
+    async listContainers(filters?: {
+        label?: string[];
+        name?: string[];
+        status?: string[];
+    }): Promise<ContainerInfo[]> {
+        const dockerFilters: Record<string, string[]> = {};
+
+        if (filters?.label) {
+            dockerFilters.label = filters.label;
+        }
+        if (filters?.name) {
+            dockerFilters.name = filters.name;
+        }
+        if (filters?.status) {
+            dockerFilters.status = filters.status;
+        }
+
+        return this.docker.listContainers({
+            all: true,
+            filters: Object.keys(dockerFilters).length > 0 ? dockerFilters : undefined,
+        });
+    }
+
+    /**
+     * List plugin containers (created by meta-sort)
+     */
+    async listPluginContainers(): Promise<ContainerInfo[]> {
+        return this.listContainers({
+            label: ['meta-mesh.plugin=true'],
+        });
+    }
+
+    /**
+     * Get container logs
+     */
+    async getContainerLogs(
+        containerId: string,
+        options?: {
+            tail?: number;
+            since?: number;
+            stdout?: boolean;
+            stderr?: boolean;
+        }
+    ): Promise<string> {
+        const container = this.docker.getContainer(containerId);
+        const logs = await container.logs({
+            stdout: options?.stdout ?? true,
+            stderr: options?.stderr ?? true,
+            tail: options?.tail ?? 100,
+            since: options?.since,
+        });
+
+        // Convert buffer to string
+        if (Buffer.isBuffer(logs)) {
+            return logs.toString('utf-8');
+        }
+        return logs as unknown as string;
+    }
+
+    /**
+     * Stream container logs
+     */
+    async streamContainerLogs(
+        containerId: string,
+        onLog: (log: string) => void,
+        options?: {
+            stdout?: boolean;
+            stderr?: boolean;
+        }
+    ): Promise<() => void> {
+        const container = this.docker.getContainer(containerId);
+        const stream = await container.logs({
+            stdout: options?.stdout ?? true,
+            stderr: options?.stderr ?? true,
+            follow: true,
+            tail: 0,
+        });
+
+        const readable = stream as Readable;
+        readable.on('data', (chunk: Buffer) => {
+            onLog(chunk.toString('utf-8'));
+        });
+
+        return () => {
+            readable.destroy();
+        };
+    }
+
+    /**
+     * Check if a network exists
+     */
+    async networkExists(networkName: string): Promise<boolean> {
+        try {
+            const networks = await this.docker.listNetworks({
+                filters: { name: [networkName] },
+            });
+            return networks.some(n => n.Name === networkName);
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Create a network if it doesn't exist
+     */
+    async ensureNetwork(networkName: string): Promise<void> {
+        if (await this.networkExists(networkName)) {
+            return;
+        }
+
+        await this.docker.createNetwork({
+            Name: networkName,
+            Driver: 'bridge',
+            Labels: {
+                'meta-mesh': 'true',
+            },
+        });
+    }
+
+    /**
+     * Connect container to network
+     */
+    async connectToNetwork(containerId: string, networkName: string): Promise<void> {
+        const network = this.docker.getNetwork(networkName);
+        await network.connect({ Container: containerId });
+    }
+
+    /**
+     * Disconnect container from network
+     */
+    async disconnectFromNetwork(containerId: string, networkName: string): Promise<void> {
+        const network = this.docker.getNetwork(networkName);
+        await network.disconnect({ Container: containerId });
+    }
+
+    /**
+     * Execute a command in a container
+     */
+    async exec(
+        containerId: string,
+        cmd: string[],
+        options?: { workingDir?: string }
+    ): Promise<{ exitCode: number; output: string }> {
+        const container = this.docker.getContainer(containerId);
+
+        const exec = await container.exec({
+            Cmd: cmd,
+            AttachStdout: true,
+            AttachStderr: true,
+            WorkingDir: options?.workingDir,
+        });
+
+        const stream = await exec.start({ Detach: false });
+
+        return new Promise((resolve, reject) => {
+            let output = '';
+
+            stream.on('data', (chunk: Buffer) => {
+                output += chunk.toString('utf-8');
+            });
+
+            stream.on('end', async () => {
+                try {
+                    const info = await exec.inspect();
+                    resolve({
+                        exitCode: info.ExitCode ?? -1,
+                        output,
+                    });
+                } catch (err) {
+                    reject(err);
+                }
+            });
+
+            stream.on('error', reject);
+        });
+    }
+}
+
+/**
+ * Parse memory limit string to bytes
+ * Supports: '512m', '1g', '1024k'
+ */
+function parseMemoryLimit(limit: string): number {
+    const match = limit.toLowerCase().match(/^(\d+)([kmg]?)$/);
+    if (!match) {
+        throw new Error(`Invalid memory limit format: ${limit}`);
+    }
+
+    const value = parseInt(match[1], 10);
+    const unit = match[2] || '';
+
+    switch (unit) {
+        case 'k':
+            return value * 1024;
+        case 'm':
+            return value * 1024 * 1024;
+        case 'g':
+            return value * 1024 * 1024 * 1024;
+        default:
+            return value;
+    }
+}
+
+/**
+ * Create a new Docker client instance
+ */
+export function createDockerClient(socketPath?: string): DockerClient {
+    return new DockerClient(socketPath);
+}

packages/meta-sort-core/src/container-plugins/index.ts

@@ -0,0 +1,34 @@
+/**
+ * Container Plugins Module
+ *
+ * Provides support for running plugins as Docker containers.
+ */
+
+// Types
+export * from './types.js';
+
+// Configuration
+export {
+    loadConfig,
+    parseConfig,
+    getEnabledPlugins,
+    getDefaultConfigPath,
+    ConfigParseError,
+} from './ConfigParser.js';
+
+// Docker Client
+export { DockerClient, createDockerClient } from './DockerClient.js';
+
+// Container Manager
+export {
+    ContainerManager,
+    createContainerManager,
+    type ContainerManagerEvents,
+} from './ContainerManager.js';
+
+// Container Plugin Scheduler
+export {
+    ContainerPluginScheduler,
+    createContainerPluginScheduler,
+    type ContainerSchedulerEvents,
+} from './ContainerPluginScheduler.js';

packages/meta-sort-core/src/container-plugins/types.ts

@@ -0,0 +1,364 @@
+/**
+ * Container Plugin System Type Definitions
+ *
+ * Types for containerized plugins that run in Docker containers
+ * and communicate via HTTP APIs.
+ */
+
+import type { ConfigField, SchemaField, TaskQueueType } from '../plugin-engine/types.js';
+
+// =============================================================================
+// Plugin Configuration (stored in plugins.yml)
+// =============================================================================
+
+/**
+ * Resource limits for a container plugin
+ */
+export interface ContainerResourceLimits {
+    /** Memory limit (e.g., '512m', '2g') */
+    memory?: string;
+    /** CPU limit (e.g., 1.0, 2.5) */
+    cpus?: number;
+}
+
+/**
+ * Volume mount configuration
+ */
+export interface ContainerMount {
+    /** Source path (relative to FILES_PATH) */
+    source: string;
+    /** Target path inside container */
+    target: string;
+    /** Mount as read-only (default: true) */
+    readonly?: boolean;
+}
+
+/**
+ * Health check configuration
+ */
+export interface ContainerHealthCheck {
+    /** Health check interval (e.g., '30s', '1m') */
+    interval?: string;
+    /** Health check timeout (e.g., '5s') */
+    timeout?: string;
+    /** Number of retries before marking unhealthy */
+    retries?: number;
+}
+
+/**
+ * Single container plugin configuration (from plugins.yml)
+ */
+export interface ContainerPluginConfig {
+    /** Whether this plugin is enabled */
+    enabled?: boolean;
+    /** Docker image name and tag */
+    image: string;
+    /** Number of container instances to spawn (default: 1) */
+    instances?: number;
+    /** Resource limits */
+    resources?: ContainerResourceLimits;
+    /** Plugin configuration values (sent to /configure endpoint) */
+    config?: Record<string, unknown>;
+    /** Whether plugin needs external network access (default: false) */
+    network?: boolean;
+    /** Additional volume mounts */
+    mounts?: ContainerMount[];
+    /** Health check settings */
+    healthCheck?: ContainerHealthCheck;
+    /** Default queue override (normally from manifest) */
+    defaultQueue?: TaskQueueType;
+}
+
+/**
+ * Root configuration file structure (plugins.yml)
+ */
+export interface ContainerPluginsConfig {
+    /** Config version */
+    version?: string;
+    /** Plugin configurations keyed by plugin ID */
+    plugins: Record<string, ContainerPluginConfig>;
+}
+
+// =============================================================================
+// Plugin Manifest (returned by plugin container via GET /manifest)
+// =============================================================================
+
+/**
+ * File filter criteria for plugin
+ */
+export interface PluginFileFilter {
+    /** MIME type patterns (e.g., 'video/*', 'image/jpeg') */
+    mimeTypes?: string[];
+    /** File extension patterns (e.g., 'mkv', 'mp4') */
+    extensions?: string[];
+    /** Minimum file size in bytes */
+    minSize?: number;
+    /** Maximum file size in bytes */
+    maxSize?: number;
+}
+
+/**
+ * UI display hints
+ */
+export interface PluginUIConfig {
+    /** Color for UI display (e.g., '#4CAF50') */
+    color?: string;
+    /** Icon name (optional) */
+    icon?: string;
+}
+
+/**
+ * Container plugin manifest (from GET /manifest)
+ */
+export interface ContainerPluginManifest {
+    /** Unique plugin identifier */
+    id: string;
+    /** Human-readable name */
+    name: string;
+    /** Semantic version */
+    version: string;
+    /** Description */
+    description?: string;
+    /** Author or organization */
+    author?: string;
+    /** Source repository URL */
+    repository?: string;
+
+    /** Execution priority (lower = runs earlier, 10-100 range) */
+    priority?: number;
+    /** Default queue assignment */
+    defaultQueue?: TaskQueueType;
+    /** Max processing time in milliseconds */
+    timeout?: number;
+
+    /** Plugin IDs that must complete before this one */
+    dependencies?: string[];
+
+    /** File filter criteria */
+    filter?: PluginFileFilter;
+
+    /** Configuration schema (aliased as config for compatibility) */
+    config?: Record<string, ConfigField>;
+    /** Metadata output schema (aliased as schema for compatibility) */
+    schema?: Record<string, SchemaField>;
+
+    /** UI display color */
+    color?: string;
+
+    /** UI display hints */
+    ui?: PluginUIConfig;
+}
+
+// =============================================================================
+// Container Plugin Instance (runtime state)
+// =============================================================================
+
+/**
+ * Container instance status
+ */
+export type ContainerInstanceStatus = 'starting' | 'healthy' | 'unhealthy' | 'stopped' | 'error';
+
+/**
+ * Runtime state of a container plugin instance
+ */
+export interface ContainerPluginInstance {
+    /** Plugin ID */
+    pluginId: string;
+    /** Docker container ID */
+    containerId: string;
+    /** Container name (e.g., 'meta-plugin-ffmpeg-0') */
+    containerName: string;
+    /** HTTP base URL (e.g., 'http://meta-plugin-ffmpeg-0:8080') */
+    baseUrl: string;
+    /** Instance index (0, 1, 2, ...) */
+    instanceIndex: number;
+    /** Current status */
+    status: ContainerInstanceStatus;
+    /** Cached manifest from container */
+    manifest?: ContainerPluginManifest;
+    /** Last successful health check timestamp */
+    lastHealthCheck?: number;
+    /** Last error message */
+    lastError?: string;
+    /** Number of tasks processed */
+    tasksProcessed: number;
+    /** Number of failed tasks */
+    tasksFailed: number;
+}
+
+// =============================================================================
+// Plugin HTTP API Types
+// =============================================================================
+
+/**
+ * Health check response from GET /health
+ */
+export interface PluginHealthResponse {
+    status: 'healthy' | 'unhealthy';
+    ready: boolean;
+    version: string;
+    message?: string;
+}
+
+/**
+ * Configure request body for POST /configure
+ */
+export interface PluginConfigureRequest {
+    config: Record<string, unknown>;
+}
+
+/**
+ * Configure response from POST /configure
+ */
+export interface PluginConfigureResponse {
+    success: boolean;
+    error?: string;
+}
+
+/**
+ * Process request body for POST /process
+ */
+export interface PluginProcessRequest {
+    /** Unique task identifier */
+    taskId: string;
+    /** Content identifier (midhash256) */
+    cid: string;
+    /** Absolute path to file in container */
+    filePath: string;
+    /** Callback URL for completion notification */
+    callbackUrl: string;
+    /** meta-core API base URL */
+    metaCoreUrl: string;
+    /** Current metadata (flat key-value map) */
+    existingMeta: Record<string, string>;
+}
+
+/**
+ * Process response from POST /process (immediate)
+ */
+export interface PluginProcessResponse {
+    status: 'accepted' | 'rejected';
+    taskId: string;
+    error?: string;
+}
+
+/**
+ * Callback status
+ */
+export type CallbackStatus = 'completed' | 'failed' | 'skipped';
+
+/**
+ * Callback payload sent by plugin to meta-sort
+ */
+export interface PluginCallbackPayload {
+    /** Task identifier */
+    taskId: string;
+    /** Plugin ID */
+    pluginId: string;
+    /** Content identifier */
+    cid: string;
+    /** Task status */
+    status: CallbackStatus;
+    /** Processing duration in milliseconds */
+    duration: number;
+    /** Error message (if status is 'failed') */
+    error?: string;
+    /** Skip reason (if status is 'skipped') */
+    reason?: string;
+}
+
+// =============================================================================
+// Container Manager Types
+// =============================================================================
+
+/**
+ * Options for spawning a container
+ */
+export interface ContainerSpawnOptions {
+    /** Docker image */
+    image: string;
+    /** Container name */
+    name: string;
+    /** Environment variables */
+    env?: Record<string, string>;
+    /** Resource limits */
+    resources?: ContainerResourceLimits;
+    /** Volume mounts */
+    mounts?: Array<{
+        source: string;
+        target: string;
+        readonly?: boolean;
+    }>;
+    /** Docker network to join */
+    network?: string;
+    /** Labels for the container */
+    labels?: Record<string, string>;
+    /** Stack name for grouping containers in Docker Desktop (adds compose labels) */
+    stackName?: string;
+    /** Service name for compose grouping (usually the plugin ID) */
+    serviceName?: string;
+}
+
+/**
+ * Container manager status
+ */
+export interface ContainerManagerStatus {
+    /** Whether manager is initialized */
+    initialized: boolean;
+    /** Total number of plugin types */
+    pluginCount: number;
+    /** Total running containers */
+    runningContainers: number;
+    /** Healthy containers */
+    healthyContainers: number;
+    /** Per-plugin status */
+    plugins: Array<{
+        pluginId: string;
+        enabled: boolean;
+        image: string;
+        instances: number;
+        healthyInstances: number;
+        manifest?: ContainerPluginManifest;
+    }>;
+}
+
+// =============================================================================
+// Container Task Types
+// =============================================================================
+
+/**
+ * Container task status
+ */
+export type ContainerTaskStatus = 'pending' | 'waiting' | 'dispatched' | 'processing' | 'completed' | 'failed' | 'timeout';
+
+/**
+ * Container plugin task
+ */
+export interface ContainerTask {
+    /** Unique task ID (UUID) */
+    id: string;
+    /** File hash (midhash256) */
+    fileHash: string;
+    /** File path */
+    filePath: string;
+    /** Target plugin ID */
+    pluginId: string;
+    /** Plugin dependencies */
+    dependencies: string[];
+    /** Task queue */
+    queue: TaskQueueType;
+    /** Task status */
+    status: ContainerTaskStatus;
+    /** Timestamp when created */
+    createdAt: number;
+    /** Timestamp when dispatched to container */
+    dispatchedAt?: number;
+    /** Timestamp when completed */
+    completedAt?: number;
+    /** Container instance that processed this task */
+    instanceName?: string;
+    /** Processing duration in ms */
+    duration?: number;
+    /** Error message if failed */
+    error?: string;
+}

packages/meta-sort-core/src/index.ts

@@ -0,0 +1,308 @@
+import {FolderWatcher, PollingWatcher} from "@metazla/meta-hash";
+import {config} from "./config/EnvConfig.js";
+
+import os from 'os';
+import {existsSync} from 'fs';
+import {WatchedFileProcessor} from "./logic/WatchedFileProcessor.js";
+import {UnifiedAPIServer} from "./api/UnifiedAPIServer.js";
+import {StreamingPipeline} from "./logic/pipeline/StreamingPipeline.js";
+import {SUPPORTED_EXTENSIONS} from "./config/SupportedFileTypes.js";
+import {performanceMetrics} from "./metrics/PerformanceMetrics.js";
+import {KVManager} from "./kv/KVManager.js";
+import {getPluginManagerInstance, initializePluginManager} from "./logic/fileProcessor/FileProcessorPiscina.js";
+import {ContainerManager} from "./container-plugins/ContainerManager.js";
+import {ContainerPluginScheduler} from "./container-plugins/ContainerPluginScheduler.js";
+
+try {
+    // Get current user information
+    const userInfo = os.userInfo();
+    // Display the current user and UID
+    console.log(`[node] Current User: ${userInfo.username}`);
+    console.log(`[node] User ID (UID): ${userInfo.uid}`);
+}catch (error) {
+    /* ignore */
+}
+
+console.log(`BUILD_VERSION: ${process.env.BUILD_VERSION || 'dev'}`);
+
+//display the config
+console.log(config);
+
+process.umask(0);//This unable the process to create files with 777 permissions
+
+let fileProcessor = new WatchedFileProcessor();
+let folders = config.WATCH_FOLDER_LIST;
+
+// KV Manager for Redis-based storage with leader election (optional)
+let kvManager: KVManager | null = null;
+
+// Container Manager for containerized plugins
+let containerManager: ContainerManager | null = null;
+let containerPluginScheduler: ContainerPluginScheduler | null = null;
+
+// Calculate concurrency for pipeline stages
+const cpuCount = os.cpus().length;
+// Smart defaults optimized for network storage + HDD scenarios:
+// - Fast queue: CPU (fast operations: filename parsing, stats)
+// - Background queue: CPU / 2 (slow operations: SHA-256, FFmpeg)
+const defaultBackgroundWorkers = Math.max(1, Math.floor(cpuCount / 2));
+const validationConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS * 2 : cpuCount * 2;
+const fastQueueConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS : cpuCount;
+const backgroundQueueConcurrency = config.MAX_WORKER_THREADS || defaultBackgroundWorkers;
+
+console.log(`Pipeline concurrency: validation=${validationConcurrency}, fastQueue=${fastQueueConcurrency}, backgroundQueue=${backgroundQueueConcurrency} (cpuCount=${cpuCount}, defaultBackgroundWorkers=${defaultBackgroundWorkers})`);
+
+// Create streaming pipeline
+const pipeline = new StreamingPipeline({
+    supportedExtensions: SUPPORTED_EXTENSIONS,
+    validationConcurrency,
+    fastQueueConcurrency,
+    backgroundQueueConcurrency,
+    strictMimeValidation: false, // Skip MIME for speed
+    fileProcessor: fileProcessor.fileProcessor,
+    stateManager: fileProcessor.unifiedStateManager,
+    kvClient: fileProcessor.getKVClient(),
+    virtualFileSystem: fileProcessor.getVirtualFileSystem(),
+    metaDataToFolderStruct: fileProcessor.metaDataToFolderStruct
+});
+
+// Connect pipeline to fileProcessor for queue status monitoring
+fileProcessor.setPipeline(pipeline);
+
+// Create generic file discovery
+const folderWatcher = new FolderWatcher();
+
+// Parse folder list
+const folderList = folders.split(',').map(f => f.trim()).filter(f => f.length > 0);
+console.log(`Watching folders: ${folderList.join(', ')}`);
+
+// VFS and state manager are available immediately
+const vfs = fileProcessor.getVirtualFileSystem();
+const unifiedStateManager = fileProcessor.getUnifiedStateManager();
+const getDuplicateResult = () => fileProcessor.getDuplicateResult();
+
+// API Server will be initialized after KV is ready (deferred to async init)
+let apiServer: UnifiedAPIServer | null = null;
+
+// Shared progressive discovery and cleanup logic (used by both startup and manual scan)
+const runProgressiveDiscovery = async () => {
+    console.log(`[Discovery] Progressive scan mode: ${folderList.length} folders`);
+
+    // Process folders one at a time for progressive cleanup
+    for (const folder of folderList) {
+        console.log(`[Discovery] Scanning folder: ${folder}`);
+
+        // Track discovered files for this folder
+        const discoveredFilesInFolder = new Set<string>();
+
+        // Wrap discovery stream to track files
+        async function* trackDiscoveredFiles(stream: AsyncGenerator<string>) {
+            for await (const filePath of stream) {
+                discoveredFilesInFolder.add(filePath);
+                yield filePath;
+            }
+        }
+
+        const discoveryStream = folderWatcher.discoverFiles([folder]);
+        const trackedStream = trackDiscoveredFiles(discoveryStream);
+
+        await pipeline.start(trackedStream);
+
+        // Progressive cleanup: clean up after each folder completes
+        console.log(`[Discovery] Folder scan complete: ${folder}. Found ${discoveredFilesInFolder.size} files. Running cleanup...`);
+        await fileProcessor.cleanupStaleEntries([folder], discoveredFilesInFolder);
+        console.log(`[Discovery] Cleanup complete for folder: ${folder}`);
+    }
+
+    console.log(`[Discovery] All folders scanned and cleaned up`);
+};
+
+// Manual scan trigger (called from API)
+// Note: Stremio updates happen automatically via incremental add/remove notifications
+// during file processing (like FUSE), so no explicit refresh needed here
+const triggerScan = async () => {
+    console.log('Triggering manual scan from API...');
+    await runProgressiveDiscovery();
+};
+
+// Global error handlers - prevent crashes from unforeseen errors
+process.on('uncaughtException', (error: Error) => {
+    console.error('❌ [CRITICAL] Uncaught Exception:', error);
+    console.error('Stack:', error.stack);
+    // Log but don't exit - allow process to continue
+    // Monitor for worker health issues
+});
+
+process.on('unhandledRejection', (reason: any, promise: Promise<any>) => {
+    console.error('❌ [CRITICAL] Unhandled Rejection at:', promise);
+    console.error('Reason:', reason);
+    // Log but don't exit - allow process to continue
+});
+
+// Graceful shutdown
+process.on('SIGINT', async () => {
+    console.log('\nShutting down...');
+    if (apiServer) {
+        await apiServer.stop();
+    }
+
+    // Stop Container Manager if running
+    if (containerManager) {
+        console.log('[Shutdown] Stopping Container Manager...');
+        await containerManager.shutdown();
+    }
+
+    // Stop KV Manager if running
+    if (kvManager) {
+        console.log('[Shutdown] Stopping KV Manager...');
+        await kvManager.stop();
+    }
+
+    process.exit(0);
+});
+
+// Start initial discovery and processing (streaming pipeline)
+(async () => {
+    try {
+        // Initialize KV Manager if configured (Redis + leader election)
+        if (config.META_CORE_PATH) {
+            console.log('[Startup] Initializing KV Manager (Redis + leader election)...');
+            kvManager = new KVManager({
+                metaCorePath: config.META_CORE_PATH,
+                filesPath: config.FILES_PATH,
+                serviceName: config.SERVICE_NAME,
+                version: config.SERVICE_VERSION,
+                apiPort: config.FUSE_API_PORT,
+                redisPort: config.REDIS_PORT,
+                redisUrl: config.REDIS_URL,
+                baseUrl: config.BASE_URL,
+                capabilities: ['write', 'mount', 'monitor']
+            });
+
+            // Wait for KV to be ready
+            await kvManager.start();
+            await kvManager.waitForReady();
+
+            // Pass KV client to file processor
+            const client = kvManager.getClient();
+            if (client) {
+                fileProcessor.setKVClient(client);
+                console.log(`[Startup] KV Manager ready (role: ${kvManager.isLeader() ? 'LEADER' : 'FOLLOWER'})`);
+            }
+        }
+
+        // Start Unified API Server AFTER KV is ready (so kvClient is available)
+        const kvClient = fileProcessor.getKVClient();
+        apiServer = new UnifiedAPIServer(vfs, {
+            port: config.FUSE_API_PORT,
+            host: config.FUSE_API_HOST,
+            enableCors: true,
+        }, unifiedStateManager, getDuplicateResult, kvClient || undefined, triggerScan, backgroundQueueConcurrency, fastQueueConcurrency, () => fileProcessor.getQueueStatus(), kvManager || undefined, getPluginManagerInstance, () => fileProcessor.getTaskScheduler());
+
+        await apiServer.start();
+        console.log('Unified API Server started successfully');
+
+        // Initialize plugin manager early so it's available for API
+        console.log('[Startup] Initializing plugin manager...');
+        const pluginManager = await initializePluginManager();
+        console.log('[Startup] Plugin manager initialized');
+
+        // Initialize container plugins if config file exists
+        if (existsSync(config.CONTAINER_PLUGINS_CONFIG)) {
+            console.log('[Startup] Initializing container plugins...');
+            try {
+                containerManager = new ContainerManager();
+                await containerManager.initialize();
+
+                // Connect container manager to plugin manager
+                pluginManager.setContainerManager(containerManager);
+                await pluginManager.loadContainerPlugins();
+
+                // Create container plugin scheduler
+                containerPluginScheduler = new ContainerPluginScheduler(containerManager, {
+                    fastConcurrency: config.FAST_QUEUE_CONCURRENCY,
+                    backgroundConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY,
+                    kvClient: kvClient || undefined,
+                });
+
+                // Connect to API server (for callback handling)
+                apiServer.setContainerPluginManager(containerManager, containerPluginScheduler);
+                console.log('[Startup] ContainerPluginScheduler connected to API server');
+
+                const status = containerManager.getStatus();
+                console.log(`[Startup] Container plugins initialized: ${status.pluginCount} plugins, ${status.healthyContainers}/${status.runningContainers} healthy`);
+            } catch (error) {
+                console.error('[Startup] Failed to initialize container plugins:', error);
+                console.log('[Startup] Continuing without container plugins...');
+            }
+        } else {
+            console.log(`[Startup] Container plugins config not found at ${config.CONTAINER_PLUGINS_CONFIG}, skipping`);
+        }
+
+        // Initialize hash index cache (loads existing hashes from disk)
+        // This also initializes the TaskScheduler
+        console.log('[Startup] Initializing hash index cache...');
+        await fileProcessor.initialize();
+        console.log('[Startup] Hash index cache initialized');
+
+        // Connect ContainerPluginScheduler to TaskScheduler and Pipeline
+        if (containerPluginScheduler) {
+            const taskScheduler = fileProcessor.getTaskScheduler();
+            if (taskScheduler) {
+                taskScheduler.setContainerPluginScheduler(containerPluginScheduler);
+                console.log('[Startup] ContainerPluginScheduler connected to TaskScheduler');
+            }
+
+            // Connect to StreamingPipeline for file processing dispatch
+            pipeline.setContainerPluginScheduler(containerPluginScheduler);
+            console.log('[Startup] ContainerPluginScheduler connected to StreamingPipeline');
+        }
+
+        // First, rebuild VFS from KV to restore state from previous runs
+        console.log('[Startup] Rebuilding VirtualFileSystem from KV...');
+        await fileProcessor.rebuildVFSFromKV();
+        console.log('[Startup] VFS rebuild completed');
+
+        console.log('[Startup] Starting initial file discovery with progressive cleanup...');
+        await runProgressiveDiscovery();
+        console.log('[Startup] Initial discovery completed, processing continues in background');
+
+        // Log cache performance summary
+        const metrics = performanceMetrics.getMetrics();
+        if (metrics.cacheHits.midhash256 || metrics.cacheMisses.midhash256) {
+            const hits = metrics.cacheHits.midhash256 || 0;
+            const misses = metrics.cacheMisses.midhash256 || 0;
+            const total = hits + misses;
+            const hitRate = total > 0 ? ((hits / total) * 100).toFixed(1) : '0.0';
+            console.log(`[Cache] midhash256 cache: ${hits} hits, ${misses} misses (${hitRate}% hit rate)`);
+        }
+
+        // Setup file watching for changes
+        console.log('[Startup] Setting up file watching...');
+        folderWatcher.watch(folderList, {
+            onAdd: (filePath) => pipeline.handleFileAdded(filePath),
+            onChange: (filePath) => pipeline.handleFileChanged(filePath),
+            onUnlink: (filePath) => pipeline.handleFileDeleted(filePath)
+        }, {
+            interval: config.WATCH_FOLDER_POOLING_INTERVAL_MS || 30000,
+            stabilityThreshold: 30000,
+            pollInterval: 5000
+        });
+        console.log('[Startup] File watching active');
+    } catch (error) {
+        console.error('[Startup] Error during initial discovery:', error);
+    }
+})();
+
+// Start PollingWatcher for path-specific polling (if configured)
+if (config.POLLING_PATHS && config.POLLING_PATHS.length > 0) {
+    console.log(`PollingWatcher: ${config.POLLING_PATHS.length} paths configured`);
+    const pollingWatcher = new PollingWatcher(fileProcessor, config.POLLING_PATHS);
+    pollingWatcher.start().then(() => {
+        console.log('PollingWatcher started successfully');
+    }).catch((error) => {
+        console.error('Error starting PollingWatcher:', error);
+    });
+} else {
+    console.log('PollingWatcher: No polling paths configured (skipped)');
+}

packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts

@@ -0,0 +1,97 @@
+import {config} from "../config/EnvConfig.js";
+import axios from "axios";
+
+export class JellyfinAPI {
+
+    constructor() {
+
+    }
+
+    async getMetaData(filepath: string): Promise<any> {
+        console.log("geteMetaData");
+    }
+
+    private async sendAuthedRequest(url: string, method: string, data?: any, params?: any) {
+        return (await axios({
+            method: method,
+            url: url,
+            headers: {
+                'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
+            },
+            data: data,
+            params: params
+        })).data;
+    }
+
+    private async getRootUserId(): Promise<string> {
+        const users = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Users`, 'GET');
+        for (const user of users) {
+            if (user.Policy.IsAdministrator) {
+                return user.Id;
+            }
+        }
+        return users[0].Id;
+    }
+
+    private async getItemIdByFilePath(filePath: string): Promise<string | null> {
+        try {
+            let user = await this.getRootUserId();
+            const items = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Items`, 'GET', null, {
+                Path: encodeURIComponent(filePath),
+                userId: user
+            });
+
+            if (items.Items && items.Items.length > 0) {
+                return items.Items[0].Id;
+            } else {
+                console.log(`Item with file path "${filePath}" not found.`);
+                return null;
+            }
+        } catch (error) {
+            console.error('Error fetching item ID by file path:', error.message);
+            return null;
+        }
+    }
+
+    jellyApiAvailable(): boolean {
+        return !!config.JELLYFIN_API_KEY && !!config.JELLYFIN_ENDPOINT;
+    }
+
+    /** takes around 30s to complete (if light update) */
+    async refreshLibrary(): Promise<void>{
+        try {
+            const response = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Library/Refresh`, "POST",null, {
+                headers: {
+                    'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
+                }
+            });
+        } catch (error) {
+            console.error('Error refreshing library:', error.message);
+        }
+    }
+
+    async notifyJellyfin(filePath): Promise<void> {
+        const itemId = await this.getItemIdByFilePath(filePath);
+        if(!itemId) {
+            return;
+        }
+        // Fetch the public system info
+        const response = await axios.post(`${config.JELLYFIN_ENDPOINT}/Items/${itemId}/Refresh`, {
+
+        },{
+            headers: {
+                'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
+            }
+        });
+        console.log(response.data);
+    }
+
+    async systemEndpoint() {
+        const response = await axios.get(`${config.JELLYFIN_ENDPOINT}/System/Endpoint`, {
+            headers: {
+                'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
+            }
+        });
+        console.log(response.data);
+    }
+}

packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts

@@ -0,0 +1,278 @@
+import {readFile, writeFile} from 'fs/promises';
+import {Builder, Parser} from 'xml2js';
+import {mergeMetadata} from "../logic/tool/MetadataMerger.js";
+import {existsAsync} from "@metazla/meta-hash";
+import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface";
+import lodash from 'lodash';
+import * as xml2js from "xml2js";
+
+export class JellyfinNfo {
+
+    options = {explicitArray: true, ignoreAttrs: true};
+
+    /**
+     * Convert a JSON XML object back to a regular JSON object where arrays containing `set` are converted back to Set objects
+     * @param input - The input JSON XML object
+     */
+    private convertFromJsonXml(input: any): any {
+        const output: any = {};
+
+        for (const [key, value] of Object.entries(input)) {
+            if (!Array.isArray(value) || value.length === 0) {
+                throw new Error("Unexpected format: each property should be an array.");
+            }
+
+            const firstElement = value[0];
+
+            if (firstElement && typeof firstElement === 'object') {
+                if ('set' in firstElement) {
+                    // Convert array from 'set' back to Set object
+                    output[key] = new Set(firstElement.set);
+                } else {
+                    // Recursively convert from JsonXml for nested objects
+                    output[key] = this.convertFromJsonXml(firstElement);
+                }
+            } else {
+                // Primitive value, simply assign it as it is the only element in the array
+                output[key] = firstElement;
+            }
+        }
+
+        return output;
+    }
+
+    /**
+     * Filter only known jellyfin supported data
+     * @param data
+     */
+    filterData(data: any): any {
+        const supportedFields = ["originalTitle", "season", "episode", "videoType", "fileType"]
+        const result = {};
+        for (const key of supportedFields) {
+            if (data[key]) {
+                result[key] = data[key];
+            }
+        }
+        return result;
+    }
+
+    async read<T>(nfoPath: string): Promise<T> {
+        if (!nfoPath.endsWith('.nfo')) {
+            throw new Error(`NFO file path must end with '.nfo'`);
+        }
+        let metadata: T = {} as T;
+        if (await existsAsync(nfoPath)) {
+            try {
+                const nfoContent = await readFile(nfoPath, {encoding: 'utf8'});
+                const parser = new xml2js.Parser(this.options);
+                const result = await parser.parseStringPromise(nfoContent);
+                metadata = result.root || result.episodedetails || result.movie || {};
+                metadata = this.convertFromJsonXml(metadata);
+            } catch (error) {
+                console.error(`Error parsing NFO file ${nfoPath}`);
+            }
+        }
+
+        // we always read the file for meta data but ignore its content except for cid which are long to compute
+        return metadata;
+    }
+
+    async update<T extends FileStatMeta>(nfoPath: string, data: T): Promise<void> {
+        data = this.filterData(data);
+        if (!nfoPath.endsWith('.nfo')) {
+            throw new Error(`NFO file path must end with '.nfo'`);
+        }
+        if (data.fileType !== "video") {
+            return;//only process video files for writing jellyfin nfo
+        }
+        if (await existsAsync(nfoPath)) {
+            await this.updateNfoFile(nfoPath, data as any);
+        } else {
+            throw new Error(`NFO file does not exist at path: ${nfoPath}`);
+        }
+    }
+
+    public async createOrUpdate<T extends FileStatMeta>(nfoPath: string, data: T): Promise<void> {
+        data = this.filterData(data);
+        if (!nfoPath.endsWith('.nfo')) {
+            throw new Error(`NFO file path must end with '.nfo'`);
+        }
+        if (data.fileType !== "video") {
+            return;//only process video files for writing jellyfin nfo
+        }
+        if (await existsAsync(nfoPath)) {
+            await this.updateNfoFile(nfoPath, data as any);
+        } else {
+            // If the NFO file doesn't exist, just generate a new one
+            await this.generateNfoFile(nfoPath, data as any);
+        }
+    }
+
+    /**
+     * Convert a regular JSON object to an XML JSON object (add arrays)
+     * @param input
+     */
+    private convertToJsonXml(input: any): any {
+        const output: any = {};
+
+        // Use Object.entries to loop through each key-value pair in the input object
+        for (const [key, value] of Object.entries(input)) {
+            if (!this.assertLegalName(key)) {
+                throw new Error(`Illegal character in key: ${key}`);
+            }
+            if (value instanceof Set) {
+                // If the value is a Set, convert it to an array and then to XmlJson format
+                const array = Array.from(value as any);
+                if (array.length > 0) {
+                    output[key] = [{set: array}];
+                }
+            } else if (value instanceof Array) {
+                throw new Error("unsupported");
+            } else if (typeof value === 'object' && value !== null) {
+                // For nested objects, recursively convert
+                let jsonXml = this.convertToJsonXml(value);
+                if (jsonXml instanceof Array) {
+                    output[key] = jsonXml;
+                } else {
+                    output[key] = [jsonXml];
+                }
+            } else {
+                output[key] = [value];
+            }
+        }
+
+        return output;
+    }
+
+    private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
+        try {
+            const existingContent = await readFile(nfoFilePath, {encoding: "utf-8"});
+            const parser = new Parser(this.options);
+            let existingData = await parser.parseStringPromise(existingContent);
+            let hasChanged = false;
+
+            // 1st try to update the type of the metadata (root can be "root" or "movie" or "episodedetails") and
+            // according to the 'jelly' parameter and the isMovieOrTvShow function
+            const targetRoot = (this.isMovieOrTvShow(data) === "tvshow" ? 'episodedetails' : "movie");
+            if (existingData.root) {
+                existingData[targetRoot] = existingData.root;
+                delete existingData.root;
+                hasChanged = true;
+            } else if (existingData.movie && "movie" !== targetRoot) {
+                existingData[targetRoot] = existingData.movie;
+                delete existingData.movie;
+                hasChanged = true;
+            } else if (existingData.episodedetails && "episodedetails" !== targetRoot) {
+                existingData[targetRoot] = existingData.episodedetails;
+                delete existingData.episodedetails;
+                hasChanged = true;
+            }
+
+            //2nd update the data
+            let root = existingData[targetRoot];
+            if (typeof root !== 'object') {
+                root = {};//don't allow primitive type in root
+                hasChanged = true;
+            }
+            const convertedData = this.convertToJsonXml(data);
+            hasChanged = mergeMetadata(root, convertedData, nfoFilePath) || hasChanged;
+            root = this.sanitizeData(root);
+            existingData[targetRoot] = root;
+
+            if (hasChanged) {
+                //const builder = new Builder({ headless: true, renderOpts: { 'pretty': true } });
+                const builder = new Builder();
+                // Convert the updated object back to XML
+                try {
+                    const updatedContent = builder.buildObject(existingData);
+                    await writeFile(nfoFilePath, updatedContent, {encoding: "utf-8"});
+                } catch (error) {
+                    console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`);
+                }
+            }
+        } catch (error) {
+            console.error(`Failed to parse existing NFO content ${nfoFilePath} : ${error}`);
+            return;
+        }
+    }
+
+    private isMovieOrTvShow(data: VideoMeta): "tvshow" | "movie" {
+        if (data.videoType) {
+            return data.videoType;
+        } else if (data.episode) {
+            return "tvshow";
+        } else {
+            return "movie";
+        }
+    }
+
+    private assertLegalChar(str): boolean {
+        return str.match(/[\0\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/);
+    };
+
+    private assertLegalName(str: string) {
+        if (this.assertLegalChar(str)) {
+            return false;
+        }
+        const regex = /^([:A-Z_a-z\xC0-\xD6\xD8-\xF6\xF8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])([\x2D\.0-:A-Z_a-z\xB7\xC0-\xD6\xD8-\xF6\xF8-\u037D\u037F-\u1FFF\u200C\u200D\u203F\u2040\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])*$/;
+        if (!str.match(regex)) {
+            return false;
+        }
+        return true;
+    };
+
+    private sanitizeData(data: any): any {
+        const sanitize = (value: any): any => {
+            if (typeof value === 'string') {
+                // Escaping the string values
+                return lodash.escape(value);
+            } else if (Array.isArray(value)) {
+                throw new Error(`Arrays are not supported in NFO files`);
+            } else if (typeof value === 'object' && value !== null) {
+                // Recursively sanitize each property in the object
+                let result = {};
+                for (const key of Object.keys(value)) {
+                    if (this.assertLegalName(key)) {
+                        result[key] = value[key];
+                    }
+                }
+                return result;
+            }
+            return value;
+        };
+
+        return sanitize(data);
+    }
+
+    private async generateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
+        if (!nfoFilePath.endsWith('.nfo')) {
+            throw new Error(`NFO file path must end with '.nfo'`);
+        }
+        if (await existsAsync(nfoFilePath)) {
+            throw new Error(`NFO file already exists at path: ${nfoFilePath}`);
+        }
+
+        const rootElementName = data.videoType === "tvshow" ? 'episodedetails' : "movie";
+        /*const schemaAttributes = jelly ? {} : {
+            'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
+            'xsi:schemaLocation': 'http://www.example.com/schema http://www.example.com/schema/mediaFileSchema.xsd'
+        };*/
+        const schemaAttributes = {};
+
+        const convertedData = this.convertToJsonXml(data);
+        const sanitizedData = this.sanitizeData(convertedData);
+
+        // Prepare the data object with the correct root element and attributes
+        const obj = {
+            [rootElementName]: {
+                ...({$: schemaAttributes}),
+                ...sanitizedData
+            }
+        };
+
+        const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}});
+        const nfoContent = builder.buildObject(obj);
+        await writeFile(nfoFilePath, nfoContent, {encoding: "utf-8"});
+    }
+
+}

packages/meta-sort-core/src/jellyfin/README

@@ -0,0 +1,10 @@
+TODO:
+Jellyfin nfo format (only for video)
+    - nfo to metadata (this is used as lib)
+    - metadata to nfo
+    - create or update nfo
+
+meta yml to nfo
+    - nfo to metadata (this is used as lib)
+    - metadata to nfo
+    - create or update nfo

packages/meta-sort-core/src/jellyfin/nfoTool.ts

@@ -0,0 +1,22 @@
+import { parse, sep } from "path";
+
+/**
+ * Get the nfo file path for a given file path (replace the extension with .nfo)
+ * @param filePath
+ * @param ext - The extension to replace with, default is "nfo"
+ * @returns The nfo file path
+ */
+export function filePathReplace(filePath: string, ext: string = "nfo"): string {
+    const { dir, name } = parse(filePath);
+    return `${dir}${sep}${name}.${ext}`;
+}
+
+/**
+ * Get the nfo file path for a given folder path (add .nfo to the folder path)
+ * @param folderPath
+ * @param ext
+ * @returns The nfo file path
+ */
+export function filePathAdd(folderPath: string, ext: string = "nfo"): string {
+    return `${folderPath}.${ext}`;
+}

packages/meta-sort-core/src/kv/IKVClient.ts

@@ -0,0 +1,192 @@
+/**
+ * KV Client Interface - Abstraction layer for key-value storage
+ *
+ * Redis-based key-value storage with nested key architecture.
+ * Uses property-level granularity for efficient updates.
+ *
+ * Key Structure:
+ * /file/{hashId}/{property/path} → value
+ *
+ * Example:
+ * /file/midhash256:abc123/title → "Inception"
+ * /file/midhash256:abc123/video/codec → "h265"
+ */
+
+export interface KeyValuePair {
+    key: string;
+    value: string;
+}
+
+export interface IKVClient {
+    // ========================================================================
+    // Basic Operations
+    // ========================================================================
+
+    /**
+     * Set a key-value pair (value is JSON serialized)
+     */
+    set(key: string, value: any): Promise<void>;
+
+    /**
+     * Get a value by key (value is JSON deserialized)
+     */
+    get(key: string): Promise<any | null>;
+
+    /**
+     * Delete a key
+     */
+    delete(key: string): Promise<void>;
+
+    /**
+     * Count keys with a given prefix
+     */
+    countKeysWithPrefix(prefix: string): Promise<number>;
+
+    /**
+     * Health check - verifies KV store is accessible
+     */
+    health(): Promise<boolean>;
+
+    // ========================================================================
+    // Nested Key Architecture Methods
+    // ========================================================================
+
+    /**
+     * Set a single property value (raw string, not JSON)
+     */
+    setProperty(key: string, value: string): Promise<void>;
+
+    /**
+     * Get a single property value (raw string)
+     */
+    getProperty(key: string): Promise<string | null>;
+
+    /**
+     * Get all key-value pairs with a given prefix
+     */
+    getRange(prefix: string): Promise<KeyValuePair[]>;
+
+    /**
+     * Set multiple key-value pairs atomically (or as close as possible)
+     */
+    setMultiple(pairs: KeyValuePair[]): Promise<void>;
+
+    /**
+     * Delete all keys with a given prefix
+     */
+    deleteRange(prefix: string): Promise<number>;
+
+    // ========================================================================
+    // High-Level Metadata Operations
+    // ========================================================================
+
+    /**
+     * Store file metadata using nested key architecture
+     * Flattens the metadata object and stores each property as a separate key
+     */
+    setMetadataFlat(hashId: string, metadata: any, excludeFields?: string[]): Promise<void>;
+
+    /**
+     * Retrieve file metadata - reconstructs from individual property keys
+     */
+    getMetadataFlat(hashId: string): Promise<any | null>;
+
+    /**
+     * Get a specific property from a file's metadata
+     */
+    getMetadata(hashId: string, propertyPath: string): Promise<any | null>;
+
+    /**
+     * Delete all metadata for a file
+     */
+    deleteMetadataFlat(hashId: string): Promise<number>;
+
+    /**
+     * Get all unique hash IDs (files) stored in KV
+     */
+    getAllHashIds(): Promise<string[]>;
+
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+
+    /**
+     * Close connections and cleanup resources
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * KV Client configuration
+ */
+export interface KVClientConfig {
+    /** Backend type */
+    type: 'redis';
+
+    /** Connection URL (e.g., redis://localhost:6379) */
+    url: string;
+
+    /** Operation timeout in ms (default: 30000) */
+    timeout?: number;
+
+    /** Key prefix for all operations (default: '' for none) */
+    prefix?: string;
+}
+
+/**
+ * Leader lock file content format
+ */
+export interface LeaderLockInfo {
+    /** Hostname of the leader */
+    host: string;
+
+    /** Redis connection URL (e.g., redis://10.0.1.50:6379) */
+    api: string;
+
+    /** HTTP API URL for the leader service */
+    http: string;
+
+    /** Configured BASE_URL for stable service discovery (optional, falls back to http) */
+    baseUrl?: string;
+
+    /** Timestamp when leadership was acquired */
+    timestamp: number;
+
+    /** Process ID of the leader */
+    pid: number;
+}
+
+/**
+ * Service registration info
+ */
+export interface ServiceInfo {
+    /** Service name (e.g., 'meta-sort', 'meta-fuse') */
+    name: string;
+
+    /** Service version */
+    version: string;
+
+    /** HTTP API URL */
+    api: string;
+
+    /** Current status */
+    status: 'starting' | 'running' | 'stopping' | 'stopped';
+
+    /** Process ID */
+    pid: number;
+
+    /** Hostname */
+    hostname: string;
+
+    /** Start timestamp (ISO string) */
+    startedAt: string;
+
+    /** Last heartbeat timestamp (ISO string) */
+    lastHeartbeat: string;
+
+    /** Service capabilities */
+    capabilities: string[];
+
+    /** Named endpoints */
+    endpoints: Record<string, string>;
+}

packages/meta-sort-core/src/kv/KVManager.ts

@@ -0,0 +1,388 @@
+/**
+ * KV Manager - Unified manager for KV storage, leader election, and service discovery
+ *
+ * This is the main entry point for the KV subsystem. It:
+ * 1. Participates in leader election (spawns Redis if leader)
+ * 2. Creates and manages the KV client (Redis)
+ * 3. Handles service discovery registration
+ * 4. Manages reconnection on leader failure
+ *
+ * Usage:
+ * ```typescript
+ * const kvManager = new KVManager({
+ *     metaCorePath: '/meta-core',
+ *     filesPath: '/files',
+ *     serviceName: 'meta-sort',
+ *     apiPort: 3000
+ * });
+ *
+ * await kvManager.start();
+ * const client = kvManager.getClient();
+ * // Use client...
+ * await kvManager.stop();
+ * ```
+ */
+
+import { networkInterfaces } from 'os';
+import { LeaderElection } from './LeaderElection.js';
+import { ServiceDiscovery } from './ServiceDiscovery.js';
+import { RedisKVClient } from './RedisClient.js';
+import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
+
+interface KVManagerConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** Path to FILES_VOLUME (e.g., /files) */
+    filesPath: string;
+
+    /** Service name (e.g., 'meta-sort', 'meta-fuse') */
+    serviceName: string;
+
+    /** Service version */
+    version?: string;
+
+    /** HTTP API port */
+    apiPort: number;
+
+    /** Redis port (default: 6379) */
+    redisPort?: number;
+
+    /** Service capabilities */
+    capabilities?: string[];
+
+    /** Whether to skip leader election and use provided Redis URL */
+    redisUrl?: string;
+
+    /** Base URL for service discovery (overrides auto-detected URL) */
+    baseUrl?: string;
+
+    /**
+     * Hostname to advertise in lock file for other services to connect.
+     * Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution.
+     */
+    advertiseHost?: string;
+}
+
+export class KVManager {
+    private config: KVManagerConfig;
+    private leaderElection: LeaderElection | null = null;
+    private serviceDiscovery: ServiceDiscovery | null = null;
+    private kvClient: IKVClient | null = null;
+    private isStarted = false;
+    private isShuttingDown = false;
+
+    // Event callbacks
+    private onReadyCallbacks: (() => void)[] = [];
+    private onReconnectCallbacks: (() => void)[] = [];
+
+    constructor(config: KVManagerConfig) {
+        this.config = {
+            version: '1.0.0',
+            redisPort: 6379,
+            capabilities: [],
+            ...config
+        };
+    }
+
+    /**
+     * Get the local IP for API URL construction
+     */
+    private getLocalIP(): string {
+        const interfaces = networkInterfaces();
+
+        for (const name of Object.keys(interfaces)) {
+            const addrs = interfaces[name];
+            if (!addrs) continue;
+
+            for (const addr of addrs) {
+                if (addr.family === 'IPv4' && !addr.internal) {
+                    return addr.address;
+                }
+            }
+        }
+
+        return 'localhost';
+    }
+
+    /**
+     * Create Redis client from URL
+     */
+    private async createRedisClient(redisUrl: string): Promise<RedisKVClient> {
+        const client = new RedisKVClient({
+            url: redisUrl,
+            timeout: 30000,
+            prefix: '' // No prefix for shared access
+        });
+
+        await client.connect();
+        return client;
+    }
+
+    /**
+     * Start the KV manager
+     */
+    async start(): Promise<void> {
+        if (this.isStarted) {
+            console.warn('[KVManager] Already started');
+            return;
+        }
+
+        console.log(`[KVManager] Starting for ${this.config.serviceName}...`);
+
+        // Check for direct Redis URL (skip leader election)
+        if (this.config.redisUrl) {
+            console.log(`[KVManager] Using direct Redis URL: ${this.config.redisUrl}`);
+            this.kvClient = await this.createRedisClient(this.config.redisUrl);
+
+            // Initialize service discovery even with direct Redis URL
+            const apiUrl = this.config.baseUrl || `http://localhost:${this.config.apiPort}`;
+            this.serviceDiscovery = new ServiceDiscovery({
+                metaCorePath: this.config.metaCorePath,
+                serviceName: this.config.serviceName,
+                version: this.config.version!,
+                apiUrl,
+                capabilities: this.config.capabilities,
+                endpoints: {
+                    health: '/health',
+                    dashboard: '/',
+                    api: '/api',
+                    metrics: '/api/metrics',
+                    processing: '/api/processing/status'
+                }
+            });
+            await this.serviceDiscovery.start();
+
+            this.isStarted = true;
+            this.notifyReady();
+            return;
+        }
+
+        // Initialize leader election
+        const localIP = this.getLocalIP();
+
+        this.leaderElection = new LeaderElection({
+            metaCorePath: this.config.metaCorePath,
+            serviceName: this.config.serviceName,
+            apiPort: this.config.apiPort,
+            redisPort: this.config.redisPort,
+            baseUrl: this.config.baseUrl
+        });
+
+        // Setup leader election callbacks
+        this.leaderElection.onLeader(async () => {
+            console.log('[KVManager] We are the leader, connecting to local Redis...');
+
+            // Connect to local Redis (spawned by leader election)
+            const redisUrl = `redis://127.0.0.1:${this.config.redisPort}`;
+            this.kvClient = await this.createRedisClient(redisUrl);
+
+            this.notifyReady();
+        });
+
+        this.leaderElection.onFollower(async (leaderInfo: LeaderLockInfo) => {
+            console.log(`[KVManager] We are a follower, connecting to leader at ${leaderInfo.api}...`);
+
+            // Connect to leader's Redis
+            this.kvClient = await this.createRedisClient(leaderInfo.api);
+
+            this.notifyReady();
+        });
+
+        this.leaderElection.onLostLeader(async () => {
+            console.log('[KVManager] Leader lost, disconnecting...');
+
+            if (this.kvClient) {
+                await this.kvClient.close();
+                this.kvClient = null;
+            }
+
+            // Leader election will handle re-election and reconnection
+        });
+
+        // Initialize service discovery
+        // Use BASE_URL if provided, otherwise fall back to auto-detected local IP
+        const apiUrl = this.config.baseUrl || `http://${localIP}:${this.config.apiPort}`;
+
+        this.serviceDiscovery = new ServiceDiscovery({
+            metaCorePath: this.config.metaCorePath,
+            serviceName: this.config.serviceName,
+            version: this.config.version!,
+            apiUrl,
+            capabilities: this.config.capabilities,
+            endpoints: {
+                health: '/health',
+                dashboard: '/',
+                api: '/api',
+                metrics: '/api/metrics',
+                processing: '/api/processing/status'
+            }
+        });
+
+        // Start leader election first
+        await this.leaderElection.start();
+
+        // Start service discovery
+        await this.serviceDiscovery.start();
+
+        this.isStarted = true;
+    }
+
+    /**
+     * Stop the KV manager
+     */
+    async stop(): Promise<void> {
+        if (!this.isStarted) return;
+
+        console.log('[KVManager] Stopping...');
+        this.isShuttingDown = true;
+
+        // Stop service discovery
+        if (this.serviceDiscovery) {
+            await this.serviceDiscovery.stop();
+            this.serviceDiscovery = null;
+        }
+
+        // Close KV client
+        if (this.kvClient) {
+            await this.kvClient.close();
+            this.kvClient = null;
+        }
+
+        // Stop leader election (releases lock, stops Redis if leader)
+        if (this.leaderElection) {
+            await this.leaderElection.stop();
+            this.leaderElection = null;
+        }
+
+        this.isStarted = false;
+        console.log('[KVManager] Stopped');
+    }
+
+    /**
+     * Notify ready callbacks
+     */
+    private notifyReady(): void {
+        for (const callback of this.onReadyCallbacks) {
+            try {
+                callback();
+            } catch (error) {
+                console.error('[KVManager] Error in ready callback:', error);
+            }
+        }
+    }
+
+    /**
+     * Register a callback for when the KV client is ready
+     */
+    onReady(callback: () => void): this {
+        this.onReadyCallbacks.push(callback);
+
+        // If already ready, call immediately
+        if (this.kvClient) {
+            callback();
+        }
+
+        return this;
+    }
+
+    /**
+     * Register a callback for reconnection events
+     */
+    onReconnect(callback: () => void): this {
+        this.onReconnectCallbacks.push(callback);
+        return this;
+    }
+
+    /**
+     * Wait for the KV client to be ready
+     */
+    async waitForReady(timeoutMs: number = 30000): Promise<void> {
+        if (this.kvClient) return;
+
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                reject(new Error(`KV client not ready after ${timeoutMs}ms`));
+            }, timeoutMs);
+
+            this.onReady(() => {
+                clearTimeout(timeout);
+                resolve();
+            });
+        });
+    }
+
+    // ========================================================================
+    // Getters
+    // ========================================================================
+
+    /**
+     * Get the KV client (may be null if not yet connected)
+     */
+    getClient(): IKVClient | null {
+        return this.kvClient;
+    }
+
+    /**
+     * Get the KV client (throws if not connected)
+     */
+    requireClient(): IKVClient {
+        if (!this.kvClient) {
+            throw new Error('KV client not available. Call start() and wait for ready.');
+        }
+        return this.kvClient;
+    }
+
+    /**
+     * Check if this service is the leader
+     */
+    isLeader(): boolean {
+        return this.leaderElection?.isLeader() ?? false;
+    }
+
+    /**
+     * Check if this service is a follower
+     */
+    isFollower(): boolean {
+        return this.leaderElection?.isFollower() ?? false;
+    }
+
+    /**
+     * Get the current leader info
+     */
+    getLeaderInfo(): LeaderLockInfo | null {
+        return this.leaderElection?.getLeaderInfo() ?? null;
+    }
+
+    /**
+     * Get service discovery instance
+     */
+    getServiceDiscovery(): ServiceDiscovery | null {
+        return this.serviceDiscovery;
+    }
+
+    /**
+     * Discover another service
+     */
+    async discoverService(name: string) {
+        return this.serviceDiscovery?.discoverService(name) ?? null;
+    }
+
+    /**
+     * Check if the KV store is healthy
+     */
+    async isHealthy(): Promise<boolean> {
+        if (!this.kvClient) return false;
+        return this.kvClient.health();
+    }
+
+    /**
+     * Get paths configuration
+     */
+    getPaths() {
+        return {
+            metaCorePath: this.config.metaCorePath,
+            filesPath: this.config.filesPath
+        };
+    }
+}

packages/meta-sort-core/src/kv/LeaderElection.ts

@@ -0,0 +1,615 @@
+/**
+ * Leader Election using flock on shared filesystem
+ *
+ * This implements a distributed leader election mechanism using:
+ * 1. flock (advisory file locking) for mutual exclusion
+ * 2. Shared filesystem as the consensus layer
+ * 3. Separate info file for service discovery
+ *
+ * Architecture:
+ * - Service tries to acquire exclusive flock on /meta-core/locks/kv-leader.lock
+ * - If acquired → becomes LEADER, spawns Redis, writes endpoint to info file
+ * - If blocked → becomes FOLLOWER, reads leader endpoint from info file
+ * - flock automatically releases when process dies (no stale lock issues!)
+ *
+ * Key advantage over O_EXCL approach:
+ * - Lock is automatically released when process exits/crashes
+ * - No need for timestamp-based stale detection
+ * - No race conditions during takeover
+ */
+
+import { promises as fs } from 'fs';
+import { dirname } from 'path';
+import { hostname, networkInterfaces } from 'os';
+import { spawn, ChildProcess, execSync } from 'child_process';
+import * as IORedis from 'ioredis';
+import type { LeaderLockInfo } from './IKVClient.js';
+
+// ESM/CJS interop for ioredis
+const Redis = (IORedis as any).default ?? IORedis;
+
+interface LeaderElectionConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** Service name for identification */
+    serviceName: string;
+
+    /** HTTP API port */
+    apiPort: number;
+
+    /** Redis port (default: 6379) */
+    redisPort?: number;
+
+    /** Redis data directory (relative to META_CORE_VOLUME/db/redis) */
+    redisDataDir?: string;
+
+    /** Health check interval in ms (default: 5000) */
+    healthCheckInterval?: number;
+
+    /** Base URL for stable service discovery (e.g., http://localhost:8180) */
+    baseUrl?: string;
+}
+
+type LeaderRole = 'leader' | 'follower' | 'unknown';
+
+export class LeaderElection {
+    private config: LeaderElectionConfig;
+    private lockFilePath: string;
+    private infoFilePath: string;
+    private flockProcess: ChildProcess | null = null;
+    private role: LeaderRole = 'unknown';
+    private leaderInfo: LeaderLockInfo | null = null;
+    private redisProcess: ChildProcess | null = null;
+    private healthCheckTimer: NodeJS.Timeout | null = null;
+    private isShuttingDown = false;
+
+    // Event callbacks
+    private onBecomeLeader?: () => void;
+    private onBecomeFollower?: (leaderInfo: LeaderLockInfo) => void;
+    private onLeaderLost?: () => void;
+
+    constructor(config: LeaderElectionConfig) {
+        this.config = {
+            redisPort: 6379,
+            healthCheckInterval: 5000,
+            ...config
+        };
+
+        this.lockFilePath = `${this.config.metaCorePath}/locks/kv-leader.lock`;
+        this.infoFilePath = `${this.config.metaCorePath}/locks/kv-leader.info`;
+    }
+
+    /**
+     * Get the local machine's IP address
+     */
+    private getLocalIP(): string {
+        const interfaces = networkInterfaces();
+
+        // Prefer non-internal IPv4 addresses
+        for (const name of Object.keys(interfaces)) {
+            const addrs = interfaces[name];
+            if (!addrs) continue;
+
+            for (const addr of addrs) {
+                if (addr.family === 'IPv4' && !addr.internal) {
+                    return addr.address;
+                }
+            }
+        }
+
+        // Fallback to hostname or localhost
+        return hostname() || 'localhost';
+    }
+
+    /**
+     * Build leader info for this service
+     */
+    private buildLeaderInfo(): LeaderLockInfo {
+        const ip = this.getLocalIP();
+        return {
+            host: hostname(),
+            api: `redis://${ip}:${this.config.redisPort}`,
+            http: `http://${ip}:${this.config.apiPort}`,
+            baseUrl: this.config.baseUrl,
+            timestamp: Date.now(),
+            pid: process.pid
+        };
+    }
+
+    /**
+     * Ensure lock directory exists
+     */
+    private async ensureLockDir(): Promise<void> {
+        const lockDir = dirname(this.lockFilePath);
+        await fs.mkdir(lockDir, { recursive: true });
+    }
+
+    /**
+     * Try to acquire the leader lock using flock
+     * Returns true if lock acquired (we are leader), false otherwise
+     *
+     * Uses `flock -n -x` for non-blocking exclusive lock attempt.
+     * The lock is held by keeping a child process alive with the lock fd open.
+     */
+    private async tryAcquireLock(): Promise<boolean> {
+        await this.ensureLockDir();
+
+        // Ensure lock file exists (flock needs an existing file)
+        try {
+            await fs.writeFile(this.lockFilePath, '', { flag: 'a' });
+        } catch {
+            // Ignore if file already exists
+        }
+
+        return new Promise((resolve) => {
+            // Spawn a process that tries to acquire the lock and holds it
+            // Using flock with file descriptor: open fd 200 on lock file, then sleep forever
+            // The lock is held as long as fd 200 is open (process alive)
+            this.flockProcess = spawn('bash', [
+                '-c',
+                `exec 200>"${this.lockFilePath}" && flock -n -x 200 && echo LOCKED && while true; do sleep 86400; done`
+            ], {
+                stdio: ['ignore', 'pipe', 'pipe']
+            });
+
+            let resolved = false;
+
+            // Check stdout for "LOCKED" message indicating we got the lock
+            this.flockProcess.stdout?.on('data', (data) => {
+                const output = data.toString().trim();
+                if (output === 'LOCKED' && !resolved) {
+                    resolved = true;
+                    console.log('[LeaderElection] Acquired flock on', this.lockFilePath);
+                    resolve(true);
+                }
+            });
+
+            this.flockProcess.stderr?.on('data', (data) => {
+                const msg = data.toString().trim();
+                if (msg) {
+                    console.error('[LeaderElection] flock stderr:', msg);
+                }
+            });
+
+            // If process exits quickly, we didn't get the lock
+            this.flockProcess.on('exit', (code) => {
+                if (!resolved) {
+                    resolved = true;
+                    console.log('[LeaderElection] Could not acquire flock (lock held by another process)');
+                    this.flockProcess = null;
+                    resolve(false);
+                } else if (!this.isShuttingDown && this.role === 'leader') {
+                    // Lock process died unexpectedly while we were leader
+                    console.error('[LeaderElection] flock process died unexpectedly!');
+                    this.flockProcess = null;
+                    this.handleLockLost();
+                }
+            });
+
+            // Timeout: if we don't get the lock within 2 seconds, assume it's held
+            setTimeout(() => {
+                if (!resolved) {
+                    resolved = true;
+                    if (this.flockProcess) {
+                        this.flockProcess.kill();
+                        this.flockProcess = null;
+                    }
+                    console.log('[LeaderElection] Timeout acquiring flock (lock held by another process)');
+                    resolve(false);
+                }
+            }, 2000);
+        });
+    }
+
+    /**
+     * Handle losing the lock unexpectedly
+     */
+    private async handleLockLost(): Promise<void> {
+        if (this.isShuttingDown) return;
+
+        console.log('[LeaderElection] Lost leadership, cleaning up...');
+        this.role = 'unknown';
+
+        // Stop Redis
+        await this.stopRedis();
+
+        // Notify callback
+        this.onLeaderLost?.();
+
+        // Try to re-elect
+        const acquired = await this.tryAcquireLock();
+        if (acquired) {
+            await this.transitionToLeader();
+        } else {
+            await this.transitionToFollower();
+        }
+    }
+
+    /**
+     * Check if the lock is currently held by someone
+     * Uses flock -n which fails immediately if lock is held
+     */
+    private isLockHeld(): boolean {
+        try {
+            // Try to acquire lock non-blocking - if it succeeds, no one holds it
+            execSync(`flock -n -x "${this.lockFilePath}" -c "exit 0"`, {
+                timeout: 1000,
+                stdio: 'ignore'
+            });
+            // Lock was free (we briefly held it and released)
+            return false;
+        } catch {
+            // Lock is held by someone else
+            return true;
+        }
+    }
+
+    /**
+     * Read leader info from info file
+     */
+    private async readLeaderInfo(): Promise<LeaderLockInfo | null> {
+        try {
+            const content = await fs.readFile(this.infoFilePath, 'utf-8');
+            return JSON.parse(content) as LeaderLockInfo;
+        } catch {
+            return null;
+        }
+    }
+
+    /**
+     * Write leader info to info file (atomic write)
+     */
+    private async writeLeaderInfo(): Promise<void> {
+        this.leaderInfo = this.buildLeaderInfo();
+
+        // Atomic write: write to temp file, then rename
+        const tempPath = `${this.infoFilePath}.tmp`;
+        await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
+        await fs.rename(tempPath, this.infoFilePath);
+
+        console.log('[LeaderElection] Wrote leader info to', this.infoFilePath);
+    }
+
+    /**
+     * Update leader timestamp (heartbeat) using atomic write
+     */
+    private async updateLeaderTimestamp(): Promise<void> {
+        if (this.role !== 'leader' || !this.leaderInfo) return;
+
+        this.leaderInfo.timestamp = Date.now();
+        try {
+            // Atomic write: write to temp file, then rename
+            const tempPath = `${this.infoFilePath}.tmp`;
+            await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
+            await fs.rename(tempPath, this.infoFilePath);
+        } catch (error) {
+            console.error('[LeaderElection] Failed to update timestamp:', error);
+        }
+    }
+
+    /**
+     * Spawn Redis server as child process
+     */
+    private async spawnRedis(): Promise<void> {
+        const redisDataDir = `${this.config.metaCorePath}/db/redis`;
+
+        // Ensure Redis data directory exists
+        await fs.mkdir(redisDataDir, { recursive: true });
+
+        console.log(`[LeaderElection] Spawning Redis on port ${this.config.redisPort}...`);
+
+        // Spawn Redis server
+        this.redisProcess = spawn('redis-server', [
+            '--port', String(this.config.redisPort),
+            '--bind', '0.0.0.0',
+            '--dir', redisDataDir,
+            '--appendonly', 'yes',
+            '--appendfilename', 'appendonly.aof',
+            '--dbfilename', 'dump.rdb',
+            '--save', '60', '1', // Save after 60 seconds if at least 1 key changed
+            '--loglevel', 'warning'
+        ], {
+            stdio: ['ignore', 'pipe', 'pipe'],
+            detached: false
+        });
+
+        this.redisProcess.stdout?.on('data', (data) => {
+            console.log(`[Redis] ${data.toString().trim()}`);
+        });
+
+        this.redisProcess.stderr?.on('data', (data) => {
+            console.error(`[Redis] ${data.toString().trim()}`);
+        });
+
+        this.redisProcess.on('exit', (code, signal) => {
+            console.log(`[Redis] Process exited with code ${code}, signal ${signal}`);
+            this.redisProcess = null;
+
+            if (!this.isShuttingDown && this.role === 'leader') {
+                // Redis crashed - try to restart
+                console.log('[LeaderElection] Redis crashed, attempting restart...');
+                this.spawnRedis().catch(console.error);
+            }
+        });
+
+        // Wait for Redis to be ready
+        await this.waitForRedisReady();
+    }
+
+    /**
+     * Wait for Redis to be ready to accept connections
+     */
+    private async waitForRedisReady(): Promise<void> {
+        const maxAttempts = 30;
+        const ip = this.getLocalIP();
+
+        for (let i = 0; i < maxAttempts; i++) {
+            try {
+                const client = new Redis({
+                    host: ip,
+                    port: this.config.redisPort,
+                    connectTimeout: 1000,
+                    maxRetriesPerRequest: 1
+                });
+
+                const pong = await client.ping();
+                await client.quit();
+
+                if (pong === 'PONG') {
+                    console.log('[LeaderElection] Redis is ready');
+                    return;
+                }
+            } catch {
+                // Redis not ready yet
+            }
+
+            await new Promise(resolve => setTimeout(resolve, 1000));
+        }
+
+        throw new Error('Redis failed to start within timeout');
+    }
+
+    /**
+     * Stop Redis server
+     */
+    private async stopRedis(): Promise<void> {
+        if (this.redisProcess) {
+            console.log('[LeaderElection] Stopping Redis...');
+            this.redisProcess.kill('SIGTERM');
+
+            // Wait for graceful shutdown
+            await new Promise<void>((resolve) => {
+                const timeout = setTimeout(() => {
+                    if (this.redisProcess) {
+                        this.redisProcess.kill('SIGKILL');
+                    }
+                    resolve();
+                }, 5000);
+
+                this.redisProcess!.on('exit', () => {
+                    clearTimeout(timeout);
+                    resolve();
+                });
+            });
+
+            this.redisProcess = null;
+        }
+    }
+
+    /**
+     * Release the leader lock by killing the flock process
+     */
+    private async releaseLock(): Promise<void> {
+        if (this.flockProcess) {
+            console.log('[LeaderElection] Releasing flock...');
+
+            // Kill the flock holder process - this releases the lock
+            this.flockProcess.kill('SIGTERM');
+
+            // Wait briefly for graceful exit
+            await new Promise<void>((resolve) => {
+                const timeout = setTimeout(() => {
+                    if (this.flockProcess) {
+                        this.flockProcess.kill('SIGKILL');
+                    }
+                    resolve();
+                }, 1000);
+
+                if (this.flockProcess) {
+                    this.flockProcess.on('exit', () => {
+                        clearTimeout(timeout);
+                        resolve();
+                    });
+                } else {
+                    clearTimeout(timeout);
+                    resolve();
+                }
+            });
+
+            this.flockProcess = null;
+        }
+
+        // Clean up info file (optional, but good hygiene)
+        try {
+            await fs.unlink(this.infoFilePath);
+        } catch {
+            // Ignore if file doesn't exist
+        }
+    }
+
+    /**
+     * Start health check loop
+     */
+    private startHealthCheck(): void {
+        this.healthCheckTimer = setInterval(async () => {
+            if (this.isShuttingDown) return;
+
+            try {
+                if (this.role === 'leader') {
+                    // Update timestamp as heartbeat
+                    await this.updateLeaderTimestamp();
+
+                    // Verify we still hold the lock
+                    if (!this.flockProcess || this.flockProcess.exitCode !== null) {
+                        console.error('[LeaderElection] Lost flock process!');
+                        await this.handleLockLost();
+                    }
+                } else if (this.role === 'follower') {
+                    // Check if lock is still held (leader still alive)
+                    if (!this.isLockHeld()) {
+                        console.log('[LeaderElection] Lock is free, attempting to become leader...');
+                        this.onLeaderLost?.();
+
+                        // Try to become leader
+                        const acquired = await this.tryAcquireLock();
+                        if (acquired) {
+                            await this.transitionToLeader();
+                        } else {
+                            // Another service became leader - reconnect
+                            await this.transitionToFollower();
+                        }
+                    }
+                }
+            } catch (error) {
+                console.error('[LeaderElection] Health check error:', error);
+            }
+        }, this.config.healthCheckInterval);
+    }
+
+    /**
+     * Transition to leader role
+     */
+    private async transitionToLeader(): Promise<void> {
+        console.log('[LeaderElection] Becoming LEADER');
+        this.role = 'leader';
+
+        // Write leader info BEFORE spawning Redis so followers can find us
+        await this.writeLeaderInfo();
+
+        // Spawn Redis
+        await this.spawnRedis();
+
+        // Notify callback
+        this.onBecomeLeader?.();
+    }
+
+    /**
+     * Transition to follower role
+     */
+    private async transitionToFollower(): Promise<void> {
+        console.log('[LeaderElection] Becoming FOLLOWER');
+        this.role = 'follower';
+
+        // Wait a bit for leader to write info file
+        await new Promise(resolve => setTimeout(resolve, 500));
+
+        // Read leader info
+        this.leaderInfo = await this.readLeaderInfo();
+
+        if (this.leaderInfo) {
+            console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
+            this.onBecomeFollower?.(this.leaderInfo);
+        } else {
+            console.warn('[LeaderElection] Could not read leader info, will retry...');
+            // Retry reading after a delay
+            setTimeout(async () => {
+                this.leaderInfo = await this.readLeaderInfo();
+                if (this.leaderInfo) {
+                    console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
+                    this.onBecomeFollower?.(this.leaderInfo);
+                }
+            }, 2000);
+        }
+    }
+
+    /**
+     * Start the leader election process
+     */
+    async start(): Promise<void> {
+        console.log(`[LeaderElection] Starting election for ${this.config.serviceName}...`);
+        console.log(`[LeaderElection] Lock file: ${this.lockFilePath}`);
+        console.log(`[LeaderElection] Info file: ${this.infoFilePath}`);
+
+        // Try to become leader
+        const acquired = await this.tryAcquireLock();
+
+        if (acquired) {
+            await this.transitionToLeader();
+        } else {
+            await this.transitionToFollower();
+        }
+
+        // Start health check
+        this.startHealthCheck();
+    }
+
+    /**
+     * Stop and cleanup
+     */
+    async stop(): Promise<void> {
+        console.log('[LeaderElection] Stopping...');
+        this.isShuttingDown = true;
+
+        // Stop health check
+        if (this.healthCheckTimer) {
+            clearInterval(this.healthCheckTimer);
+            this.healthCheckTimer = null;
+        }
+
+        // If we were leader, cleanup
+        if (this.role === 'leader') {
+            await this.stopRedis();
+            await this.releaseLock();
+        }
+
+        this.role = 'unknown';
+        console.log('[LeaderElection] Stopped');
+    }
+
+    // ========================================================================
+    // Event Registration
+    // ========================================================================
+
+    onLeader(callback: () => void): this {
+        this.onBecomeLeader = callback;
+        return this;
+    }
+
+    onFollower(callback: (leaderInfo: LeaderLockInfo) => void): this {
+        this.onBecomeFollower = callback;
+        return this;
+    }
+
+    onLostLeader(callback: () => void): this {
+        this.onLeaderLost = callback;
+        return this;
+    }
+
+    // ========================================================================
+    // Getters
+    // ========================================================================
+
+    getRole(): LeaderRole {
+        return this.role;
+    }
+
+    isLeader(): boolean {
+        return this.role === 'leader';
+    }
+
+    isFollower(): boolean {
+        return this.role === 'follower';
+    }
+
+    getLeaderInfo(): LeaderLockInfo | null {
+        return this.leaderInfo;
+    }
+
+    getRedisUrl(): string | null {
+        if (this.leaderInfo) {
+            return this.leaderInfo.api;
+        }
+        return null;
+    }
+}

packages/meta-sort-core/src/kv/MetadataUtils.ts

@@ -0,0 +1,274 @@
+/**
+ * Utility functions for converting between nested JSON metadata and flat key-value pairs.
+ * Implements the hierarchical nested key architecture for KV storage (Redis, etc.)
+ */
+
+import type { KeyValuePair } from './IKVClient.js';
+
+/**
+ * Flattens a nested metadata object into an array of key-value pairs.
+ * Each property becomes a separate key with a hierarchical path.
+ *
+ * @param metadata - The nested metadata object to flatten
+ * @param prefix - The key prefix (e.g., "/file/{hashId}")
+ * @param excludeFields - Fields to exclude from flattening (e.g., internal tracking fields)
+ * @returns Array of key-value pairs suitable for KV storage
+ *
+ * @example
+ * flattenMetadata({ title: "Inception", video: { codec: "h265" } }, "/file/abc123")
+ * // Returns:
+ * // [
+ * //   { key: "/file/abc123/title", value: "Inception" },
+ * //   { key: "/file/abc123/video/codec", value: "h265" }
+ * // ]
+ */
+export function flattenMetadata(
+  metadata: any,
+  prefix: string,
+  excludeFields: string[] = []
+): KeyValuePair[] {
+  const pairs: KeyValuePair[] = [];
+
+  function flatten(obj: any, path: string) {
+    if (obj === null || obj === undefined) {
+      // Store null/undefined as empty string
+      pairs.push({ key: path, value: '' });
+      return;
+    }
+
+    if (typeof obj !== 'object') {
+      // Primitive value - store as string
+      pairs.push({ key: path, value: String(obj) });
+      return;
+    }
+
+    if (Array.isArray(obj)) {
+      // Handle arrays with numeric indices
+      obj.forEach((item, index) => {
+        flatten(item, `${path}/${index}`);
+      });
+      return;
+    }
+
+    // Handle objects recursively
+    Object.keys(obj).forEach((key) => {
+      // Skip excluded fields
+      if (excludeFields.includes(key)) {
+        return;
+      }
+
+      const value = obj[key];
+      const newPath = path ? `${path}/${key}` : key;
+
+      flatten(value, newPath);
+    });
+  }
+
+  flatten(metadata, prefix);
+  return pairs;
+}
+
+/**
+ * Reconstructs a nested metadata object from flat key-value pairs.
+ * Reverses the flattening process to restore the original structure.
+ *
+ * @param pairs - Array of key-value pairs from KV storage
+ * @param prefix - The key prefix to strip (e.g., "/file/{hashId}")
+ * @returns Reconstructed nested metadata object
+ *
+ * @example
+ * reconstructMetadata([
+ *   { key: "/file/abc123/title", value: "Inception" },
+ *   { key: "/file/abc123/video/codec", value: "h265" }
+ * ], "/file/abc123")
+ * // Returns: { title: "Inception", video: { codec: "h265" } }
+ */
+export function reconstructMetadata(
+  pairs: KeyValuePair[],
+  prefix: string
+): any {
+  const result: any = {};
+
+  pairs.forEach(({ key, value }) => {
+    // Strip the prefix to get the property path
+    if (!key.startsWith(prefix)) {
+      return; // Skip keys that don't match the prefix
+    }
+
+    const propertyPath = key.slice(prefix.length + 1); // +1 to skip the leading slash
+    if (!propertyPath) {
+      return; // Skip if no property path after prefix
+    }
+
+    const parts = propertyPath.split('/');
+    let current = result;
+
+    // Navigate/create the nested structure
+    for (let i = 0; i < parts.length; i++) {
+      const part = parts[i];
+      const isLast = i === parts.length - 1;
+
+      if (isLast) {
+        // Last part - set the value
+        current[part] = parseValue(value);
+      } else {
+        // Intermediate part - create object or array as needed
+        const nextPart = parts[i + 1];
+        const isNextPartNumeric = /^\d+$/.test(nextPart);
+
+        if (!(part in current)) {
+          // Create array if next part is numeric, otherwise create object
+          current[part] = isNextPartNumeric ? [] : {};
+        }
+
+        current = current[part];
+      }
+    }
+  });
+
+  // Post-process: Convert objects with only numeric keys to arrays
+  return convertNumericObjectsToArrays(result);
+}
+
+/**
+ * Recursively converts objects with only numeric sequential keys to arrays.
+ * This handles cases where arrays were stored with numeric indices.
+ */
+function convertNumericObjectsToArrays(obj: any): any {
+  if (obj === null || typeof obj !== 'object') {
+    return obj;
+  }
+
+  if (Array.isArray(obj)) {
+    // Already an array, recursively process elements
+    return obj.map(convertNumericObjectsToArrays);
+  }
+
+  const keys = Object.keys(obj);
+
+  // Check if all keys are numeric and sequential (0, 1, 2, ...)
+  const allNumeric = keys.every(key => /^\d+$/.test(key));
+
+  if (allNumeric && keys.length > 0) {
+    // Sort keys numerically
+    const sortedKeys = keys.map(Number).sort((a, b) => a - b);
+
+    // Check if keys are sequential starting from 0
+    const isSequential = sortedKeys.every((key, index) => key === index);
+
+    if (isSequential) {
+      // Convert to array
+      const arr = sortedKeys.map(index =>
+        convertNumericObjectsToArrays(obj[index.toString()])
+      );
+      return arr;
+    }
+  }
+
+  // Regular object - recursively process values
+  const result: any = {};
+  for (const key of keys) {
+    result[key] = convertNumericObjectsToArrays(obj[key]);
+  }
+  return result;
+}
+
+/**
+ * Parses a string value from KV storage into the appropriate JavaScript type.
+ * Handles numbers, booleans, null, and strings.
+ *
+ * @param value - The string value from KV storage
+ * @returns Parsed value with appropriate type
+ */
+function parseValue(value: string): any {
+  // Handle empty string as null
+  if (value === '') {
+    return null;
+  }
+
+  // Handle booleans
+  if (value === 'true') return true;
+  if (value === 'false') return false;
+
+  // Handle null
+  if (value === 'null') return null;
+
+  // Try to parse as number, but be conservative
+  // Only parse as number if it's a valid integer or floating point without leading zeros
+  // This preserves strings like "5.1" (channel layout), "23.976" (frame rate)
+  // Note: In actual storage, these will be stored as strings, so this only matters
+  // when the original value was numeric. For metadata consistency, we convert
+  // what looks like numbers to numbers.
+  const num = Number(value);
+  if (!isNaN(num) && value.trim() !== '' && /^-?\d+(\.\d+)?$/.test(value)) {
+    return num;
+  }
+
+  // Return as string
+  return value;
+}
+
+/**
+ * Extracts the hash ID from a file key.
+ *
+ * @param key - Full key like "/file/{hashId}/property/path"
+ * @returns The hash ID or null if not a valid file key
+ *
+ * @example
+ * extractHashId("/file/sha256:abc123/title") // Returns "sha256:abc123"
+ */
+export function extractHashId(key: string): string | null {
+  const match = key.match(/^\/file\/([^/]+)/);
+  return match ? match[1] : null;
+}
+
+/**
+ * Builds a property key path.
+ *
+ * @param hashId - The file's hash ID
+ * @param propertyPath - The property path (e.g., "video/codec")
+ * @returns Full KV key path
+ *
+ * @example
+ * buildPropertyKey("sha256:abc123", "video/codec")
+ * // Returns "/file/sha256:abc123/video/codec"
+ */
+export function buildPropertyKey(hashId: string, propertyPath: string): string {
+  return `/file/${hashId}/${propertyPath}`;
+}
+
+/**
+ * Builds a file prefix for range queries.
+ *
+ * @param hashId - The file's hash ID
+ * @returns Prefix for querying all properties of a file
+ *
+ * @example
+ * buildFilePrefix("sha256:abc123") // Returns "/file/sha256:abc123"
+ */
+export function buildFilePrefix(hashId: string): string {
+  return `/file/${hashId}`;
+}
+
+/**
+ * Groups key-value pairs by hash ID.
+ * Useful for batch processing multiple files' metadata.
+ *
+ * @param pairs - Array of key-value pairs
+ * @returns Map of hash ID to array of key-value pairs
+ */
+export function groupByHashId(pairs: KeyValuePair[]): Map<string, KeyValuePair[]> {
+  const grouped = new Map<string, KeyValuePair[]>();
+
+  pairs.forEach((pair) => {
+    const hashId = extractHashId(pair.key);
+    if (hashId) {
+      if (!grouped.has(hashId)) {
+        grouped.set(hashId, []);
+      }
+      grouped.get(hashId)!.push(pair);
+    }
+  });
+
+  return grouped;
+}

packages/meta-sort-core/src/kv/RedisClient.ts

@@ -0,0 +1,351 @@
+/**
+ * Redis KV Client - Implements IKVClient interface using Redis
+ *
+ * Uses Redis hashes for efficient nested key storage:
+ * - Each file's metadata is stored as a Redis hash
+ * - Hash key: file:{hashId}
+ * - Hash field: property/path (e.g., "video/codec")
+ *
+ * Benefits of Redis hashes:
+ * - Single command to get all file metadata (HGETALL)
+ * - Single command to set multiple properties (HMSET)
+ * - Atomic operations on file metadata
+ */
+
+import * as IORedis from 'ioredis';
+import type { IKVClient, KeyValuePair } from './IKVClient.js';
+import {
+    flattenMetadata,
+    reconstructMetadata,
+    buildFilePrefix,
+    buildPropertyKey,
+} from './MetadataUtils.js';
+
+// ESM/CJS interop for ioredis
+const Redis = (IORedis as any).default ?? IORedis;
+type RedisInstance = InstanceType<typeof Redis>;
+
+interface RedisClientConfig {
+    url: string;
+    timeout?: number;
+    prefix?: string;
+}
+
+export class RedisKVClient implements IKVClient {
+    private redis: RedisInstance;
+    private timeout: number;
+    private prefix: string;
+    private isConnected: boolean = false;
+
+    constructor(config: RedisClientConfig) {
+        this.timeout = config.timeout || 30000;
+        this.prefix = config.prefix || '';
+
+        // Parse Redis URL and create client
+        this.redis = new Redis(config.url, {
+            commandTimeout: this.timeout,
+            connectTimeout: this.timeout,
+            lazyConnect: true,
+            maxRetriesPerRequest: 3,
+            enableOfflineQueue: true,
+            retryStrategy: (times) => {
+                const delay = Math.min(times * 200, 2000);
+                console.log(`[Redis] Retry attempt ${times}, waiting ${delay}ms`);
+                return delay;
+            }
+        });
+
+        // Event handlers
+        this.redis.on('connect', () => {
+            this.isConnected = true;
+            console.log('[Redis] Connected');
+        });
+
+        this.redis.on('error', (err) => {
+            console.error('[Redis] Error:', err.message);
+        });
+
+        this.redis.on('close', () => {
+            this.isConnected = false;
+            console.log('[Redis] Connection closed');
+        });
+    }
+
+    /**
+     * Connect to Redis (call before using other methods)
+     */
+    async connect(): Promise<void> {
+        if (this.isConnected) return;
+        await this.redis.connect();
+    }
+
+    /**
+     * Build the full Redis key with optional prefix
+     */
+    private buildKey(key: string): string {
+        return this.prefix ? `${this.prefix}${key}` : key;
+    }
+
+    // ========================================================================
+    // Basic Operations
+    // ========================================================================
+
+    async set(key: string, value: any): Promise<void> {
+        const fullKey = this.buildKey(key);
+        await this.redis.set(fullKey, JSON.stringify(value));
+    }
+
+    async get(key: string): Promise<any | null> {
+        const fullKey = this.buildKey(key);
+        const value = await this.redis.get(fullKey);
+        if (value === null) return null;
+        try {
+            return JSON.parse(value);
+        } catch {
+            return value;
+        }
+    }
+
+    async delete(key: string): Promise<void> {
+        const fullKey = this.buildKey(key);
+        await this.redis.del(fullKey);
+    }
+
+    async countKeysWithPrefix(prefix: string): Promise<number> {
+        const fullPrefix = this.buildKey(prefix);
+        // Use SCAN to count keys (don't use KEYS in production)
+        let count = 0;
+        let cursor = '0';
+
+        do {
+            const [nextCursor, keys] = await this.redis.scan(
+                cursor,
+                'MATCH',
+                `${fullPrefix}*`,
+                'COUNT',
+                1000
+            );
+            cursor = nextCursor;
+            count += keys.length;
+        } while (cursor !== '0');
+
+        return count;
+    }
+
+    async health(): Promise<boolean> {
+        try {
+            const result = await this.redis.ping();
+            return result === 'PONG';
+        } catch {
+            return false;
+        }
+    }
+
+    // ========================================================================
+    // Nested Key Architecture Methods
+    // ========================================================================
+
+    async setProperty(key: string, value: string): Promise<void> {
+        const fullKey = this.buildKey(key);
+        await this.redis.set(fullKey, value);
+    }
+
+    async getProperty(key: string): Promise<string | null> {
+        const fullKey = this.buildKey(key);
+        return await this.redis.get(fullKey);
+    }
+
+    async getRange(prefix: string): Promise<KeyValuePair[]> {
+        const fullPrefix = this.buildKey(prefix);
+        const pairs: KeyValuePair[] = [];
+        let cursor = '0';
+
+        do {
+            const [nextCursor, keys] = await this.redis.scan(
+                cursor,
+                'MATCH',
+                `${fullPrefix}*`,
+                'COUNT',
+                1000
+            );
+            cursor = nextCursor;
+
+            if (keys.length > 0) {
+                // Get values for all found keys
+                const values = await this.redis.mget(...keys);
+                keys.forEach((key, index) => {
+                    const value = values[index];
+                    if (value !== null) {
+                        // Strip prefix to get original key
+                        const originalKey = this.prefix
+                            ? key.slice(this.prefix.length)
+                            : key;
+                        pairs.push({ key: originalKey, value });
+                    }
+                });
+            }
+        } while (cursor !== '0');
+
+        return pairs;
+    }
+
+    async setMultiple(pairs: KeyValuePair[]): Promise<void> {
+        if (pairs.length === 0) return;
+
+        // Use pipeline for efficient batch operations
+        const pipeline = this.redis.pipeline();
+
+        for (const pair of pairs) {
+            const fullKey = this.buildKey(pair.key);
+            pipeline.set(fullKey, pair.value);
+        }
+
+        await pipeline.exec();
+    }
+
+    async deleteRange(prefix: string): Promise<number> {
+        const fullPrefix = this.buildKey(prefix);
+        let deletedCount = 0;
+        let cursor = '0';
+
+        do {
+            const [nextCursor, keys] = await this.redis.scan(
+                cursor,
+                'MATCH',
+                `${fullPrefix}*`,
+                'COUNT',
+                1000
+            );
+            cursor = nextCursor;
+
+            if (keys.length > 0) {
+                const deleted = await this.redis.del(...keys);
+                deletedCount += deleted;
+            }
+        } while (cursor !== '0');
+
+        return deletedCount;
+    }
+
+    // ========================================================================
+    // High-Level Metadata Operations (Using Redis Hashes)
+    // ========================================================================
+
+    /**
+     * Store file metadata using Redis hash
+     * More efficient than individual keys - single HMSET command
+     */
+    async setMetadataFlat(
+        hashId: string,
+        metadata: any,
+        excludeFields: string[] = []
+    ): Promise<void> {
+        const prefix = buildFilePrefix(hashId);
+        const pairs = flattenMetadata(metadata, prefix, excludeFields);
+
+        if (pairs.length === 0) return;
+
+        // Convert to Redis hash format
+        // Hash key: file:{hashId}
+        // Hash fields: property paths (without /file/{hashId} prefix)
+        const hashKey = this.buildKey(`file:${hashId}`);
+        const hashData: Record<string, string> = {};
+
+        for (const pair of pairs) {
+            // Extract field name by removing the prefix
+            const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
+            if (field) {
+                hashData[field] = pair.value;
+            }
+        }
+
+        if (Object.keys(hashData).length > 0) {
+            await this.redis.hmset(hashKey, hashData);
+        }
+
+        // Also store in key index for getAllHashIds
+        await this.redis.sadd(this.buildKey('file:__index__'), hashId);
+    }
+
+    async getMetadataFlat(hashId: string): Promise<any | null> {
+        const hashKey = this.buildKey(`file:${hashId}`);
+        const hashData = await this.redis.hgetall(hashKey);
+
+        if (!hashData || Object.keys(hashData).length === 0) {
+            return null;
+        }
+
+        // Convert Redis hash back to KeyValuePair format for reconstruction
+        const prefix = buildFilePrefix(hashId);
+        const pairs: KeyValuePair[] = Object.entries(hashData).map(([field, value]) => ({
+            key: `${prefix}/${field}`,
+            value: value as string
+        }));
+
+        return reconstructMetadata(pairs, prefix);
+    }
+
+    async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
+        const hashKey = this.buildKey(`file:${hashId}`);
+        const value = await this.redis.hget(hashKey, propertyPath);
+
+        if (value === null) return null;
+
+        try {
+            return JSON.parse(value);
+        } catch {
+            return value;
+        }
+    }
+
+    async deleteMetadataFlat(hashId: string): Promise<number> {
+        const hashKey = this.buildKey(`file:${hashId}`);
+        const fieldCount = await this.redis.hlen(hashKey);
+
+        await this.redis.del(hashKey);
+        await this.redis.srem(this.buildKey('file:__index__'), hashId);
+
+        return fieldCount;
+    }
+
+    async getAllHashIds(): Promise<string[]> {
+        const indexKey = this.buildKey('file:__index__');
+        return await this.redis.smembers(indexKey);
+    }
+
+    // ========================================================================
+    // Pub/Sub Operations
+    // ========================================================================
+
+    /**
+     * Publish a message to a Redis channel
+     * Used for notifying other services (meta-fuse) about file changes
+     * Note: Channel names are NOT prefixed - they use their own namespace
+     */
+    async publish(channel: string, message: string): Promise<number> {
+        return await this.redis.publish(channel, message);
+    }
+
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+
+    async close(): Promise<void> {
+        await this.redis.quit();
+        this.isConnected = false;
+    }
+
+    /**
+     * Check if client is connected
+     */
+    isHealthy(): boolean {
+        return this.isConnected;
+    }
+
+    /**
+     * Get the underlying Redis client (for advanced operations)
+     */
+    getRedisClient(): RedisInstance {
+        return this.redis;
+    }
+}

packages/meta-sort-core/src/kv/ServiceDiscovery.ts

@@ -0,0 +1,312 @@
+/**
+ * Service Discovery using shared filesystem
+ *
+ * Each service registers itself in /meta-core/services/{service-name}.json
+ * with its API endpoint, status, and heartbeat timestamp.
+ *
+ * Services can discover each other by reading these JSON files.
+ * Stale detection: if lastHeartbeat > 60 seconds ago, service is considered dead.
+ */
+
+import { promises as fs } from 'fs';
+import { hostname } from 'os';
+import { join } from 'path';
+import type { ServiceInfo } from './IKVClient.js';
+
+interface ServiceDiscoveryConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** Service name */
+    serviceName: string;
+
+    /** Service version */
+    version: string;
+
+    /** HTTP API URL */
+    apiUrl: string;
+
+    /** Service capabilities */
+    capabilities?: string[];
+
+    /** Named endpoints */
+    endpoints?: Record<string, string>;
+
+    /** Heartbeat interval in ms (default: 30000) */
+    heartbeatInterval?: number;
+
+    /** Stale threshold in ms (default: 60000) */
+    staleThreshold?: number;
+}
+
+export class ServiceDiscovery {
+    private config: ServiceDiscoveryConfig;
+    private servicesDir: string;
+    private serviceFilePath: string;
+    private heartbeatTimer: NodeJS.Timeout | null = null;
+    private isShuttingDown = false;
+
+    constructor(config: ServiceDiscoveryConfig) {
+        this.config = {
+            heartbeatInterval: 30000,
+            staleThreshold: 60000,
+            capabilities: [],
+            endpoints: {},
+            ...config
+        };
+
+        this.servicesDir = `${this.config.metaCorePath}/services`;
+        this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}.json`;
+    }
+
+    /**
+     * Ensure services directory exists
+     */
+    private async ensureServicesDir(): Promise<void> {
+        await fs.mkdir(this.servicesDir, { recursive: true });
+    }
+
+    /**
+     * Build service info for this service
+     */
+    private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
+        return {
+            name: this.config.serviceName,
+            version: this.config.version,
+            api: this.config.apiUrl,
+            status,
+            pid: process.pid,
+            hostname: hostname(),
+            startedAt: new Date().toISOString(),
+            lastHeartbeat: new Date().toISOString(),
+            capabilities: this.config.capabilities || [],
+            endpoints: this.config.endpoints || {}
+        };
+    }
+
+    /**
+     * Register this service
+     */
+    async register(): Promise<void> {
+        await this.ensureServicesDir();
+
+        const info = this.buildServiceInfo('starting');
+        await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+
+        console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}`);
+    }
+
+    /**
+     * Update service status
+     */
+    async updateStatus(status: ServiceInfo['status']): Promise<void> {
+        try {
+            const content = await fs.readFile(this.serviceFilePath, 'utf-8');
+            const info = JSON.parse(content) as ServiceInfo;
+
+            info.status = status;
+            info.lastHeartbeat = new Date().toISOString();
+
+            await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+        } catch (error) {
+            console.error('[ServiceDiscovery] Failed to update status:', error);
+        }
+    }
+
+    /**
+     * Send heartbeat (update lastHeartbeat timestamp)
+     */
+    async heartbeat(): Promise<void> {
+        try {
+            const content = await fs.readFile(this.serviceFilePath, 'utf-8');
+            const info = JSON.parse(content) as ServiceInfo;
+
+            info.lastHeartbeat = new Date().toISOString();
+
+            await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+        } catch (error) {
+            // If file was deleted, re-register
+            if ((error as any).code === 'ENOENT') {
+                await this.register();
+                await this.updateStatus('running');
+            } else {
+                console.error('[ServiceDiscovery] Heartbeat failed:', error);
+            }
+        }
+    }
+
+    /**
+     * Start heartbeat loop
+     */
+    startHeartbeat(): void {
+        this.heartbeatTimer = setInterval(async () => {
+            if (this.isShuttingDown) return;
+            await this.heartbeat();
+        }, this.config.heartbeatInterval);
+    }
+
+    /**
+     * Stop heartbeat loop
+     */
+    stopHeartbeat(): void {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+    }
+
+    /**
+     * Unregister this service (on shutdown)
+     */
+    async unregister(): Promise<void> {
+        this.isShuttingDown = true;
+        this.stopHeartbeat();
+
+        try {
+            // Update status to stopped before removing
+            await this.updateStatus('stopped');
+
+            // Optionally remove the file (or keep for debugging)
+            // await fs.unlink(this.serviceFilePath);
+
+            console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}`);
+        } catch (error) {
+            console.error('[ServiceDiscovery] Failed to unregister:', error);
+        }
+    }
+
+    // ========================================================================
+    // Discovery Methods
+    // ========================================================================
+
+    /**
+     * Discover a service by name
+     */
+    async discoverService(name: string): Promise<ServiceInfo | null> {
+        const servicePath = join(this.servicesDir, `${name}.json`);
+
+        try {
+            const content = await fs.readFile(servicePath, 'utf-8');
+            const service = JSON.parse(content) as ServiceInfo;
+
+            // Check if service is alive (heartbeat within threshold)
+            const lastHeartbeat = new Date(service.lastHeartbeat).getTime();
+            const isAlive = Date.now() - lastHeartbeat < this.config.staleThreshold!;
+
+            if (!isAlive) {
+                console.warn(`[ServiceDiscovery] Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`);
+                return null;
+            }
+
+            return service;
+        } catch (error) {
+            if ((error as any).code === 'ENOENT') {
+                return null; // Service not registered
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Discover all registered services
+     */
+    async discoverAllServices(): Promise<ServiceInfo[]> {
+        try {
+            const files = await fs.readdir(this.servicesDir);
+            const services: ServiceInfo[] = [];
+
+            for (const file of files) {
+                if (!file.endsWith('.json')) continue;
+
+                const name = file.replace('.json', '');
+                const service = await this.discoverService(name);
+
+                if (service && service.status === 'running') {
+                    services.push(service);
+                }
+            }
+
+            return services;
+        } catch (error) {
+            if ((error as any).code === 'ENOENT') {
+                return []; // Services directory doesn't exist yet
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Wait for a service to become available
+     */
+    async waitForService(
+        name: string,
+        timeoutMs: number = 30000,
+        pollInterval: number = 1000
+    ): Promise<ServiceInfo> {
+        const startTime = Date.now();
+
+        while (Date.now() - startTime < timeoutMs) {
+            const service = await this.discoverService(name);
+            if (service && service.status === 'running') {
+                return service;
+            }
+
+            await new Promise(resolve => setTimeout(resolve, pollInterval));
+        }
+
+        throw new Error(`Service ${name} not available after ${timeoutMs}ms`);
+    }
+
+    /**
+     * Check if a service is healthy
+     */
+    async isServiceHealthy(name: string): Promise<boolean> {
+        const service = await this.discoverService(name);
+        if (!service) return false;
+
+        // Try to ping the service's health endpoint
+        const healthEndpoint = service.endpoints.health || '/health';
+        const url = `${service.api}${healthEndpoint}`;
+
+        try {
+            const response = await fetch(url, {
+                signal: AbortSignal.timeout(5000)
+            });
+            return response.ok;
+        } catch {
+            return false;
+        }
+    }
+
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+
+    /**
+     * Full startup sequence
+     */
+    async start(): Promise<void> {
+        await this.register();
+        await this.updateStatus('running');
+        this.startHeartbeat();
+    }
+
+    /**
+     * Full shutdown sequence
+     */
+    async stop(): Promise<void> {
+        await this.unregister();
+    }
+
+    // ========================================================================
+    // Getters
+    // ========================================================================
+
+    getServiceFilePath(): string {
+        return this.serviceFilePath;
+    }
+
+    getServicesDir(): string {
+        return this.servicesDir;
+    }
+}

packages/meta-sort-core/src/kv/index.ts

@@ -0,0 +1,18 @@
+/**
+ * KV Module - Unified key-value storage with leader election
+ *
+ * This module provides:
+ * - IKVClient interface for KV storage abstraction
+ * - RedisKVClient for Redis-based storage
+ * - LeaderElection for distributed leader election via flock
+ * - ServiceDiscovery for service registration and discovery
+ * - KVManager for unified management of all components
+ * - MetadataUtils for flattening/reconstructing nested metadata
+ */
+
+export * from './IKVClient.js';
+export * from './RedisClient.js';
+export * from './LeaderElection.js';
+export * from './ServiceDiscovery.js';
+export * from './KVManager.js';
+export * from './MetadataUtils.js';

packages/meta-sort-core/src/logic/DuplicateFinder.ts

@@ -0,0 +1,113 @@
+import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
+import {VideoMeta, HashMeta, VersionMeta, FileNameMeta, FileStatMeta} from "@metazla/meta-interface";
+import {renamingRule} from "../config/RenamingRule.js";
+
+type Local = VideoMeta & HashMeta & VersionMeta & FileNameMeta & FileStatMeta;
+
+export interface DuplicateGroup {
+    key: string;
+    files: string[];
+    virtualPath?: string;
+}
+
+export interface DuplicateResult {
+    hashDuplicates: DuplicateGroup[];
+    titleDuplicates: DuplicateGroup[];
+}
+
+export class DuplicateFinder {
+    async findDuplicates(database: Map<string, Local>): Promise<DuplicateResult> {
+        let [hashDuplicates, titleDuplicates] = await Promise.all([
+            this.findDuplicatesGeneric(database, this.hashKeyExtractor, 'remove'),
+            this.findDuplicatesGeneric(database, this.pathExtractor, 'version')
+        ]);
+
+        return {
+            hashDuplicates,
+            titleDuplicates
+        };
+    }
+
+    private hashKeyExtractor(metadata: Local): string | null {
+        return metadata[CID_ALGORITHM_NAMES.sha256] ?? null;
+    }
+
+    private pathExtractor(metadata: Local): string | null {
+        //case insensitive
+        try {
+            return renamingRule(metadata, '')?.toLowerCase() ?? null;
+        } catch (e) {
+            return null;
+        }
+    }
+
+    private async findDuplicatesGeneric(
+        database: Map<string, Local>,
+        keyExtractor: (metadata: Local) => string | null,
+        action?: 'remove' | 'version'
+    ): Promise<DuplicateGroup[]> {
+        const duplicates = new Map<string, string[]>();
+        const duplicateGroups: DuplicateGroup[] = [];
+
+        for (const [filePath, metadata] of database) {
+            const key = keyExtractor(metadata);
+            if (key) {
+                const files = duplicates.get(key) ?? [];
+                files.push(filePath);
+                duplicates.set(key, files);
+            }
+        }
+
+        for (const [key, files] of duplicates) {
+            if (files.length > 1) {
+                // Store duplicate group before modifying database
+                const sortedFiles = [...files].sort();
+
+                // Generate virtualPath for both hash and title duplicates
+                let virtualPath: string | undefined;
+                if (action === 'version') {
+                    // For title duplicates, use the key (normalized title path)
+                    virtualPath = key;
+                } else if (action === 'remove') {
+                    // For hash duplicates, use renamingRule on the first file's metadata
+                    const firstFile = sortedFiles[0];
+                    const metadata = database.get(firstFile);
+                    if (metadata) {
+                        try {
+                            virtualPath = renamingRule(metadata, '');
+                        } catch (e) {
+                            // If renamingRule fails, virtualPath remains undefined
+                        }
+                    }
+                }
+
+                duplicateGroups.push({
+                    key,
+                    files: sortedFiles,
+                    virtualPath
+                });
+
+                //alphabetical order
+                const remainingFiles = files.slice(1).sort();
+                //rename all problematic entries V2, V3, V4, V5
+                let i = 2;
+                for (const file of remainingFiles) {
+                    if (action === 'remove') {
+                        database.delete(file);
+                    } else if (action === 'version') {
+                        let definition = database.get(file);
+                        definition.version = "V" + i;
+                        if (definition.fileType !== "subtitle") {//allow multiple version of subtitle in the same folder
+                            definition.extra = "true";//mark as extra so it don't pollute the main list
+                        }
+                        i++;
+                    } else {
+                        throw new Error();
+                    }
+                }
+            }
+        }
+
+        return duplicateGroups;
+    }
+}

packages/meta-sort-core/src/logic/HashProcessingStateManager.ts

@@ -0,0 +1,143 @@
+/**
+ * HashProcessingStateManager
+ *
+ * Tracks the state of background hash computation operations:
+ * - Pending: Files queued for hash computation
+ * - Computing: Files currently having hashes computed
+ * - Complete: Recently completed hash computations
+ */
+
+export type HashStatus = 'pending' | 'computing' | 'complete' | 'error';
+
+export interface HashProcessingState {
+  filePath: string;
+  tempId?: string;
+  hashStatus: HashStatus;
+  queuedAt?: number;
+  startedAt?: number;
+  completedAt?: number;
+  processingTime?: number;
+  hash?: string;
+  error?: string;
+  virtualPath?: string;
+}
+
+export interface HashProcessingStatusSnapshot {
+  pending: HashProcessingState[];
+  computing: HashProcessingState[];
+  complete: HashProcessingState[];
+  totalPending: number;
+  totalComputing: number;
+  totalComplete: number;
+}
+
+export class HashProcessingStateManager {
+  private pending: Map<string, HashProcessingState> = new Map();
+  private computing: Map<string, HashProcessingState> = new Map();
+  private complete: HashProcessingState[] = [];
+  private maxCompleteHistory = 100; // Keep last 100 completed hashes
+
+  /**
+   * Add a file to the pending hash queue
+   */
+  addPending(filePath: string, tempId?: string, virtualPath?: string): void {
+    this.pending.set(filePath, {
+      filePath,
+      tempId,
+      virtualPath,
+      hashStatus: 'pending',
+      queuedAt: Date.now()
+    });
+  }
+
+  /**
+   * Move a file from pending to computing
+   */
+  startComputing(filePath: string): void {
+    const pendingState = this.pending.get(filePath);
+    this.pending.delete(filePath);
+
+    this.computing.set(filePath, {
+      filePath,
+      tempId: pendingState?.tempId,
+      virtualPath: pendingState?.virtualPath,
+      hashStatus: 'computing',
+      queuedAt: pendingState?.queuedAt,
+      startedAt: Date.now()
+    });
+  }
+
+  /**
+   * Move a file from computing to complete
+   */
+  completeComputing(
+    filePath: string,
+    processingTime: number,
+    hash?: string,
+    error?: string,
+    virtualPath?: string
+  ): void {
+    const computingState = this.computing.get(filePath);
+    this.computing.delete(filePath);
+
+    const completedState: HashProcessingState = {
+      filePath,
+      tempId: computingState?.tempId,
+      virtualPath: virtualPath || computingState?.virtualPath,
+      hashStatus: error ? 'error' : 'complete',
+      queuedAt: computingState?.queuedAt,
+      startedAt: computingState?.startedAt,
+      completedAt: Date.now(),
+      processingTime,
+      hash,
+      error
+    };
+
+    // Add to complete list (most recent first)
+    this.complete.unshift(completedState);
+
+    // Trim history if needed
+    if (this.complete.length > this.maxCompleteHistory) {
+      this.complete = this.complete.slice(0, this.maxCompleteHistory);
+    }
+  }
+
+  /**
+   * Remove a file from all states (e.g., when deleted)
+   */
+  removeFile(filePath: string): void {
+    this.pending.delete(filePath);
+    this.computing.delete(filePath);
+    this.complete = this.complete.filter(f => f.filePath !== filePath);
+  }
+
+  /**
+   * Get a snapshot of the current hash processing state
+   */
+  getSnapshot(): HashProcessingStatusSnapshot {
+    return {
+      pending: Array.from(this.pending.values()),
+      computing: Array.from(this.computing.values()),
+      complete: this.complete.slice(0, 50), // Return only last 50 for API
+      totalPending: this.pending.size,
+      totalComputing: this.computing.size,
+      totalComplete: this.complete.length
+    };
+  }
+
+  /**
+   * Clear all states (useful for testing or reset)
+   */
+  clear(): void {
+    this.pending.clear();
+    this.computing.clear();
+    this.complete = [];
+  }
+
+  /**
+   * Get the total number of files in all states
+   */
+  getTotalFileCount(): number {
+    return this.pending.size + this.computing.size + this.complete.length;
+  }
+}

packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts

@@ -0,0 +1,75 @@
+import * as path from 'path';
+import {
+    HashMeta,
+} from "@metazla/meta-interface";
+
+type Local =  HashMeta;
+
+export class MetaDataToFolderStruct {
+
+    public renamingRule: (metamesh: Local, filepath: string) => string | null;
+
+    constructor(renamingRule: (metamesh: Local, filepath: string) => string | null) {
+        this.renamingRule = renamingRule;
+    }
+
+    /**
+     * Compute virtual filesystem structure for FUSE/WebDAV
+     * Generates organized paths based on metadata without creating physical files
+     * @param data Map of file paths to metadata
+     * @private
+     */
+    private computeVirtualStructure(data: Map<string, Local>): Map<string, string> {
+        const virtualMap = new Map<string, string>();
+
+        for (const [filepath, metamesh] of data) {
+            // Compute the virtual path for the file
+            let virtualPath: string;
+            try {
+                virtualPath = this.renamingRule(metamesh, filepath);
+            }catch (e){
+                console.warn(e.message || e);
+            }
+            if (!virtualPath) {
+                continue;
+            }
+            virtualPath = this.sanitizePath(virtualPath);
+            virtualMap.set(filepath, virtualPath);
+        }
+
+        console.log(`Computed virtual structure for ${virtualMap.size} files`);
+        return virtualMap;
+    }
+
+    sanitizePath(newPath: string): string {
+        // First, isolate any potential Windows drive letter at the start of the path
+        const driveMatch = newPath.match(/^[a-zA-Z]:/);
+        const drive = driveMatch ? driveMatch[0] : '';
+
+        // Define illegal characters for Windows and potentially problematic ones for Unix
+        const illegalChars = /[<>:"|?*]/g;  // Null character added for Unix-like systems
+
+        // Replace illegal characters with an underscore in the path, excluding the drive part
+        const pathWithoutDrive = newPath.slice(drive.length);
+        const sanitizedPath = pathWithoutDrive.replace(illegalChars, '');
+
+        // Combine the drive (if any) and the sanitized path
+        const cleanPath = drive + sanitizedPath;
+
+        // Normalize the clean path to handle slashes appropriately
+        return path.normalize(cleanPath);
+    }
+
+    /**
+     * Generate virtual filesystem structure for FUSE/WebDAV
+     * Returns only the computed virtual paths without checking physical filesystem
+     * @param data Map of file paths to metadata
+     */
+    public generateVirtualStructure(data: Map<string, Local>): Map<string, string> {
+        const start = performance.now();
+        const computed = this.computeVirtualStructure(data);
+        console.log(`Compute virtual structure took ${performance.now() - start}ms`);
+        return computed;
+    }
+
+}

packages/meta-sort-core/src/logic/MetaMeshFormat.ts

@@ -0,0 +1,95 @@
+import { promises as fs } from 'fs';
+import yaml from 'yaml';
+const { parse, stringify } = yaml;
+/**
+ * File type:
+ * filename.<ext>.meta
+ * filename.meta (contains multiple metadata entry "mkv", "str" etc...)
+ * folder.meta
+ */
+
+export class MetaMeshFormat {
+    async createOrUpdate<T>(metaDataFilePath: string, data: T): Promise<void> {
+        if(!metaDataFilePath.endsWith('.meta')){
+            throw new Error(`File path must end with '.meta'`);
+        }
+        try {
+            // Check if the file exists
+            const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
+
+            if (fileExists) {
+                // If the file exists, update it
+                await this.update(metaDataFilePath, data);
+            } else {
+                // If the file does not exist, create it
+                await fs.writeFile(metaDataFilePath, stringify(data), 'utf8');
+            }
+        } catch (error) {
+            console.error(`Error in createOrUpdate: ${error}`);
+            throw error;
+        }
+    }
+
+    async read<T>(metaDataFilePath: string): Promise<T> {
+        if(!metaDataFilePath.endsWith('.meta')){
+            throw new Error(`File path must end with '.meta'`);
+        }
+        try {
+            const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
+
+            if (!fileExists) {
+                return null;
+            }
+            // Read the YAML file
+            const fileContent = await fs.readFile(metaDataFilePath, 'utf8');
+
+            // Parse the YAML content to a JavaScript object
+            return parse(fileContent) as T;
+        } catch (error) {
+            console.error(`Error in read: ${error}`);
+            throw error;
+        }
+    }
+
+    async update<T>(metaDataFilePath: string, data: T): Promise<void> {
+        if(!metaDataFilePath.endsWith('.meta')){
+            throw new Error(`File path must end with '.meta'`);
+        }
+        try {
+            // Read the existing file content
+            const existingData = await this.read<any>(metaDataFilePath) || {};
+
+            // Merge the existing data with the new data
+            const updatedData = { ...existingData, ...data };
+
+            // Write the updated data back to the file
+            await fs.writeFile(metaDataFilePath, stringify(updatedData), 'utf8');
+        } catch (error) {
+            console.error(`Error in update: ${error}`);
+            throw error;
+        }
+    }
+
+    async write<T>(metaDataFilePath: string, data: T): Promise<void> {
+        if(!metaDataFilePath.endsWith('.meta')){
+            throw new Error(`File path must end with '.meta'`);
+        }
+        try {
+            // Write the updated data back to the file
+            await fs.writeFile(metaDataFilePath, stringify(data), {
+                encoding:'utf8',
+                flag:'w' // Write mode : Overwrite
+            });
+        } catch (error) {
+            console.error(`Error in update: ${error}`);
+            throw error;
+        }
+    }
+
+    /**
+     * Serialize data to YAML string (for virtual .meta files)
+     */
+    serialize<T>(data: T): string {
+        return stringify(data);
+    }
+}

packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts

@@ -0,0 +1,217 @@
+import yaml from 'yaml';
+const { stringify: yamlStringify } = yaml;
+import { Builder } from 'xml2js';
+
+/**
+ * Supported metadata format types
+ */
+export enum MetadataFormatType {
+    META = 'meta',
+    JELLYFIN = 'jellyfin'
+}
+
+/**
+ * Interface for metadata format handlers
+ * Note: metadata parameter uses `any` type because it combines multiple metadata interfaces
+ * (HashMeta, JellyfinMeta, VideoMeta, etc.) and TypeScript doesn't have a single type for this union
+ */
+export interface IMetadataFormatHandler {
+    /**
+     * The file extension for this format (e.g., '.meta', '.nfo')
+     */
+    readonly extension: string;
+
+    /**
+     * The format type identifier
+     */
+    readonly formatType: MetadataFormatType;
+
+    /**
+     * Serialize metadata to string format
+     * @param metadata The metadata to serialize (combination of various metadata interfaces)
+     * @returns Serialized string content
+     */
+    serialize(metadata: any): string;
+}
+
+/**
+ * Handler for .meta YAML format
+ */
+export class MetaFormatHandler implements IMetadataFormatHandler {
+    readonly extension = '.meta';
+    readonly formatType = MetadataFormatType.META;
+
+    serialize(metadata: any): string {
+        return yamlStringify(metadata);
+    }
+}
+
+/**
+ * Handler for Jellyfin .nfo XML format
+ */
+export class JellyfinFormatHandler implements IMetadataFormatHandler {
+    readonly extension = '.nfo';
+    readonly formatType = MetadataFormatType.JELLYFIN;
+
+    serialize(metadata: any): string {
+        // Only include Jellyfin-compatible fields
+        const jellyfinData = this.filterJellyfinFields(metadata);
+
+        // Determine root element based on video type
+        const rootElementName = this.isMovieOrTvShow(jellyfinData) === "tvshow" ? 'episodedetails' : "movie";
+
+        // Convert to XML-friendly format
+        const xmlData = this.convertToJsonXml(jellyfinData);
+
+        // Build XML
+        const obj = {
+            [rootElementName]: xmlData
+        };
+
+        const builder = new Builder({
+            xmldec: { version: '1.0', encoding: 'UTF-8', standalone: true }
+        });
+
+        return builder.buildObject(obj);
+    }
+
+    private filterJellyfinFields(metadata: any): any {
+        const jellyfinFields: any = {};
+
+        // Video metadata
+        if (metadata.videoType) jellyfinFields.videoType = metadata.videoType;
+        if (metadata.season) jellyfinFields.season = metadata.season;
+        if (metadata.episode) jellyfinFields.episode = metadata.episode;
+        if (metadata.originalTitle) jellyfinFields.originalTitle = metadata.originalTitle;
+        if (metadata.movieYear) jellyfinFields.year = metadata.movieYear;
+
+        // Titles (first title or original title)
+        if (metadata.titles) {
+            const titleEntries = Object.entries(metadata.titles);
+            if (titleEntries.length > 0) {
+                jellyfinFields.title = titleEntries[0][1];
+            }
+        }
+
+        // IDs
+        if (metadata.imdbid) jellyfinFields.imdbid = metadata.imdbid;
+        if (metadata.tmdbid) jellyfinFields.tmdbid = metadata.tmdbid;
+        if (metadata.anidbid) jellyfinFields.anidbid = metadata.anidbid;
+
+        // Additional fields
+        if (metadata.rating) jellyfinFields.rating = metadata.rating;
+        if (metadata.plot) jellyfinFields.plot = metadata.plot;
+        if (metadata.studio) jellyfinFields.studio = metadata.studio;
+        if (metadata.releasedate) jellyfinFields.releasedate = metadata.releasedate;
+        if (metadata.mpaa) jellyfinFields.mpaa = metadata.mpaa;
+
+        // Sets (genres, tags, languages)
+        if (metadata.genres) jellyfinFields.genre = metadata.genres;
+        if (metadata.tags) jellyfinFields.tag = metadata.tags;
+        if (metadata.languages) jellyfinFields.language = metadata.languages;
+
+        // Art
+        if (metadata.art) {
+            if (metadata.art.poster) jellyfinFields.poster = metadata.art.poster;
+            if (metadata.art.fanart) jellyfinFields.fanart = metadata.art.fanart;
+        }
+
+        return jellyfinFields;
+    }
+
+    private isMovieOrTvShow(metadata: any): "tvshow" | "movie" {
+        if (metadata.videoType) {
+            return metadata.videoType;
+        } else if (metadata.episode) {
+            return "tvshow";
+        } else {
+            return "movie";
+        }
+    }
+
+    private convertToJsonXml(input: any): any {
+        const output: any = {};
+
+        for (const [key, value] of Object.entries(input)) {
+            if (value instanceof Set) {
+                // Convert Set to array of items for XML
+                const array = Array.from(value as any);
+                if (array.length > 0) {
+                    output[key] = array;
+                }
+            } else if (value instanceof Array) {
+                output[key] = value;
+            } else if (typeof value === 'object' && value !== null) {
+                // For nested objects, recursively convert
+                output[key] = [this.convertToJsonXml(value)];
+            } else {
+                output[key] = [value];
+            }
+        }
+
+        return output;
+    }
+}
+
+/**
+ * Registry for metadata format handlers
+ */
+export class MetadataFormatRegistry {
+    private handlers: Map<MetadataFormatType, IMetadataFormatHandler> = new Map();
+
+    constructor() {
+        // Register default handlers
+        this.registerHandler(new MetaFormatHandler());
+        this.registerHandler(new JellyfinFormatHandler());
+    }
+
+    /**
+     * Register a new format handler
+     */
+    registerHandler(handler: IMetadataFormatHandler): void {
+        this.handlers.set(handler.formatType, handler);
+    }
+
+    /**
+     * Get handler by format type
+     */
+    getHandler(formatType: MetadataFormatType): IMetadataFormatHandler | undefined {
+        return this.handlers.get(formatType);
+    }
+
+    /**
+     * Get handler by format name (case-insensitive)
+     */
+    getHandlerByName(formatName: string): IMetadataFormatHandler | undefined {
+        const normalizedName = formatName.toLowerCase();
+        for (const [type, handler] of this.handlers.entries()) {
+            if (type.toLowerCase() === normalizedName) {
+                return handler;
+            }
+        }
+        return undefined;
+    }
+
+    /**
+     * Get all registered handlers
+     */
+    getAllHandlers(): IMetadataFormatHandler[] {
+        return Array.from(this.handlers.values());
+    }
+
+    /**
+     * Get handlers for specified format names
+     */
+    getHandlersForFormats(formatNames: string[]): IMetadataFormatHandler[] {
+        const handlers: IMetadataFormatHandler[] = [];
+        for (const name of formatNames) {
+            const handler = this.getHandlerByName(name);
+            if (handler) {
+                handlers.push(handler);
+            } else {
+                console.warn(`Unknown metadata format: ${name}. Supported formats: ${Array.from(this.handlers.keys()).join(', ')}`);
+            }
+        }
+        return handlers;
+    }
+}

packages/meta-sort-core/src/logic/ProcessingStateManager.ts

@@ -0,0 +1,127 @@
+/**
+ * ProcessingStateManager
+ *
+ * Tracks the state of file processing operations:
+ * - Pending: Files queued for processing
+ * - Processing: Files currently being processed
+ * - Processed: Recently completed files
+ */
+
+export interface FileProcessingState {
+  filePath: string;
+  state: 'pending' | 'processing' | 'processed';
+  queuedAt?: number;
+  startedAt?: number;
+  completedAt?: number;
+  processingTime?: number;
+  error?: string;
+  virtualPath?: string;
+}
+
+export interface ProcessingStatusSnapshot {
+  pending: FileProcessingState[];
+  processing: FileProcessingState[];
+  processed: FileProcessingState[];
+  totalPending: number;
+  totalProcessing: number;
+  totalProcessed: number;
+}
+
+export class ProcessingStateManager {
+  private pending: Map<string, FileProcessingState> = new Map();
+  private processing: Map<string, FileProcessingState> = new Map();
+  private processed: FileProcessingState[] = [];
+  private maxProcessedHistory = 100; // Keep last 100 processed files
+
+  /**
+   * Add a file to the pending queue
+   */
+  addPending(filePath: string): void {
+    this.pending.set(filePath, {
+      filePath,
+      state: 'pending',
+      queuedAt: Date.now()
+    });
+  }
+
+  /**
+   * Move a file from pending to processing
+   */
+  startProcessing(filePath: string): void {
+    const pendingState = this.pending.get(filePath);
+    this.pending.delete(filePath);
+
+    this.processing.set(filePath, {
+      filePath,
+      state: 'processing',
+      queuedAt: pendingState?.queuedAt,
+      startedAt: Date.now()
+    });
+  }
+
+  /**
+   * Move a file from processing to processed
+   */
+  completeProcessing(filePath: string, processingTime: number, error?: string, virtualPath?: string): void {
+    const processingState = this.processing.get(filePath);
+    this.processing.delete(filePath);
+
+    const completedState: FileProcessingState = {
+      filePath,
+      state: 'processed',
+      queuedAt: processingState?.queuedAt,
+      startedAt: processingState?.startedAt,
+      completedAt: Date.now(),
+      processingTime,
+      error,
+      virtualPath
+    };
+
+    // Add to processed list (most recent first)
+    this.processed.unshift(completedState);
+
+    // Trim history if needed
+    if (this.processed.length > this.maxProcessedHistory) {
+      this.processed = this.processed.slice(0, this.maxProcessedHistory);
+    }
+  }
+
+  /**
+   * Remove a file from all states (e.g., when deleted)
+   */
+  removeFile(filePath: string): void {
+    this.pending.delete(filePath);
+    this.processing.delete(filePath);
+    this.processed = this.processed.filter(f => f.filePath !== filePath);
+  }
+
+  /**
+   * Get a snapshot of the current processing state
+   */
+  getSnapshot(): ProcessingStatusSnapshot {
+    return {
+      pending: Array.from(this.pending.values()),
+      processing: Array.from(this.processing.values()),
+      processed: this.processed.slice(0, 50), // Return only last 50 for API
+      totalPending: this.pending.size,
+      totalProcessing: this.processing.size,
+      totalProcessed: this.processed.length
+    };
+  }
+
+  /**
+   * Clear all states (useful for testing or reset)
+   */
+  clear(): void {
+    this.pending.clear();
+    this.processing.clear();
+    this.processed = [];
+  }
+
+  /**
+   * Get the total number of files in all states
+   */
+  getTotalFileCount(): number {
+    return this.pending.size + this.processing.size + this.processed.length;
+  }
+}

packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts

@@ -0,0 +1,354 @@
+/**
+ * UnifiedProcessingStateManager
+ *
+ * Tracks files through the complete processing pipeline with 4 states:
+ * 1. Pending: Files discovered in filesystem (not yet processed)
+ * 2. Light Processing: Quick metadata extraction + midhash256 computation (~20ms)
+ * 3. Hash Processing: SHA-256 computation for compatibility (slow, background)
+ * 4. Done: Fully processed files
+ *
+ * Key change from old system:
+ * - midhash256 (fast) is computed in lightProcessing and becomes the permanent file ID
+ * - SHA-256 (slow) is computed in hashProcessing as additional metadata
+ * - Files become accessible in VFS immediately after lightProcessing completes
+ */
+
+import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
+
+export type FileState = 'pending' | 'lightProcessing' | 'hashProcessing' | 'done';
+
+export interface UnifiedFileState {
+  filePath: string;
+  state: FileState;
+  hash?: string;  // midhash256 (set after lightProcessing)
+  virtualPath?: string;
+  error?: string;
+
+  // Retry tracking
+  retryCount?: number;
+  lastRetryAt?: number;
+
+  // Timestamps
+  discoveredAt?: number;
+  lightProcessingStartedAt?: number;
+  lightProcessingCompletedAt?: number;
+  hashProcessingStartedAt?: number;
+  hashProcessingCompletedAt?: number;
+
+  // Processing times
+  lightProcessingTime?: number;
+  hashProcessingTime?: number;
+  totalProcessingTime?: number;
+}
+
+export interface UnifiedProcessingSnapshot {
+  pending: UnifiedFileState[];
+  lightProcessing: UnifiedFileState[];
+  hashProcessing: UnifiedFileState[];
+  done: UnifiedFileState[];
+
+  totalPending: number;
+  totalLightProcessing: number;
+  totalHashProcessing: number;
+  totalDone: number;
+  totalFilesInVFS: number;         // Total number of files accessible in the Virtual File System
+
+  // Configuration
+  fastQueueConcurrency?: number;       // Number of fast queue workers
+  backgroundQueueConcurrency?: number; // Number of background queue workers
+}
+
+export class UnifiedProcessingStateManager {
+  private pending: Map<string, UnifiedFileState> = new Map();
+  private lightProcessing: Map<string, UnifiedFileState> = new Map();
+  private hashProcessing: Map<string, UnifiedFileState> = new Map();
+  private done: UnifiedFileState[] = [];
+  private maxDoneHistory = 100; // Keep last 100 completed files
+  private totalProcessedCount = 0; // Track total number of completed files (not limited)
+
+  /**
+   * Add a file to pending state (file discovered)
+   */
+  addPending(filePath: string): void {
+    this.pending.set(filePath, {
+      filePath,
+      state: 'pending',
+      discoveredAt: Date.now()
+    });
+  }
+
+  /**
+   * Move file from pending to light processing
+   */
+  startLightProcessing(filePath: string): void {
+    const pendingState = this.pending.get(filePath);
+    const existingState = this.lightProcessing.get(filePath);
+    this.pending.delete(filePath);
+
+    this.lightProcessing.set(filePath, {
+      filePath,
+      state: 'lightProcessing',
+      discoveredAt: pendingState?.discoveredAt || existingState?.discoveredAt,
+      lightProcessingStartedAt: Date.now(),
+      retryCount: existingState?.retryCount || 0
+    });
+  }
+
+  /**
+   * Complete light processing and move to hash processing queue
+   */
+  completeLightProcessing(
+    filePath: string,
+    hash: string,  // midhash256
+    virtualPath?: string,
+    error?: string
+  ): void {
+    const lightState = this.lightProcessing.get(filePath);
+    this.lightProcessing.delete(filePath);
+
+    const now = Date.now();
+    const lightProcessingTime = lightState?.lightProcessingStartedAt
+      ? now - lightState.lightProcessingStartedAt
+      : undefined;
+
+    // Record light processing time metric
+    if (lightProcessingTime !== undefined && !error) {
+      performanceMetrics.recordMetadataProcessing(lightProcessingTime);
+    }
+
+    if (error) {
+      // If error, mark as done with error
+      this.completeHashProcessing(filePath, hash, virtualPath, error);
+      return;
+    }
+
+    // Move to hash processing queue
+    this.hashProcessing.set(filePath, {
+      filePath,
+      state: 'hashProcessing',
+      hash,
+      virtualPath,
+      discoveredAt: lightState?.discoveredAt,
+      lightProcessingStartedAt: lightState?.lightProcessingStartedAt,
+      lightProcessingCompletedAt: now,
+      lightProcessingTime,
+      retryCount: lightState?.retryCount || 0
+    });
+  }
+
+  /**
+   * Start hash processing (SHA-256 computation)
+   */
+  startHashProcessing(filePath: string): void {
+    const hashState = this.hashProcessing.get(filePath);
+    if (hashState) {
+      hashState.hashProcessingStartedAt = Date.now();
+    }
+  }
+
+  /**
+   * Complete hash processing and move to done
+   */
+  completeHashProcessing(
+    filePath: string,
+    hash?: string,
+    virtualPath?: string,
+    error?: string
+  ): void {
+    const hashState = this.hashProcessing.get(filePath);
+    this.hashProcessing.delete(filePath);
+
+    const now = Date.now();
+    const hashProcessingTime = hashState?.hashProcessingStartedAt
+      ? now - hashState.hashProcessingStartedAt
+      : undefined;
+
+    const totalProcessingTime = hashState?.discoveredAt
+      ? now - hashState.discoveredAt
+      : undefined;
+
+    // Record hash processing time metric
+    if (hashProcessingTime !== undefined && !error) {
+      performanceMetrics.recordHashProcessing(hashProcessingTime);
+    }
+
+    const doneState: UnifiedFileState = {
+      filePath,
+      state: 'done',
+      hash: hash || hashState?.hash,
+      virtualPath: virtualPath || hashState?.virtualPath,
+      error,
+      discoveredAt: hashState?.discoveredAt,
+      lightProcessingStartedAt: hashState?.lightProcessingStartedAt,
+      lightProcessingCompletedAt: hashState?.lightProcessingCompletedAt,
+      hashProcessingStartedAt: hashState?.hashProcessingStartedAt,
+      hashProcessingCompletedAt: now,
+      lightProcessingTime: hashState?.lightProcessingTime,
+      hashProcessingTime,
+      totalProcessingTime
+    };
+
+    // Add to done list (most recent first)
+    this.done.unshift(doneState);
+
+    // Increment total processed counter (tracks all files, not limited)
+    this.totalProcessedCount++;
+
+    // Trim history if needed
+    if (this.done.length > this.maxDoneHistory) {
+      this.done = this.done.slice(0, this.maxDoneHistory);
+    }
+  }
+
+  /**
+   * Remove a file from all states (e.g., when deleted)
+   */
+  removeFile(filePath: string): void {
+    this.pending.delete(filePath);
+    this.lightProcessing.delete(filePath);
+    this.hashProcessing.delete(filePath);
+    this.done = this.done.filter(f => f.filePath !== filePath);
+  }
+
+  /**
+   * Get a snapshot of the current processing state
+   * @param vfsFileCount - Optional total number of files in the Virtual File System
+   * @param fastQueueConcurrency - Number of fast queue workers
+   * @param backgroundQueueConcurrency - Number of background queue workers
+   */
+  getSnapshot(vfsFileCount: number = 0, fastQueueConcurrency?: number, backgroundQueueConcurrency?: number): UnifiedProcessingSnapshot {
+    // Limit returned arrays to prevent slow API responses with large queues
+    const MAX_ITEMS_PER_STATE = 100;
+
+    // Efficiently extract only the needed items without creating full arrays
+    const getPendingSample = (): UnifiedFileState[] => {
+      const result: UnifiedFileState[] = [];
+      let count = 0;
+      for (const item of this.pending.values()) {
+        if (count >= MAX_ITEMS_PER_STATE) break;
+        result.push(item);
+        count++;
+      }
+      return result;
+    };
+
+    const getLightProcessingSample = (): UnifiedFileState[] => {
+      const result: UnifiedFileState[] = [];
+      let count = 0;
+      for (const item of this.lightProcessing.values()) {
+        if (count >= MAX_ITEMS_PER_STATE) break;
+        result.push(item);
+        count++;
+      }
+      return result;
+    };
+
+    const getHashProcessingSample = (): UnifiedFileState[] => {
+      const result: UnifiedFileState[] = [];
+      let count = 0;
+      for (const item of this.hashProcessing.values()) {
+        if (count >= MAX_ITEMS_PER_STATE) break;
+        result.push(item);
+        count++;
+      }
+      return result;
+    };
+
+    return {
+      pending: getPendingSample(),
+      lightProcessing: getLightProcessingSample(),
+      hashProcessing: getHashProcessingSample(),
+      done: this.done.slice(0, 50), // Return only last 50 for API
+
+      totalPending: this.pending.size,
+      totalLightProcessing: this.lightProcessing.size,
+      totalHashProcessing: this.hashProcessing.size,
+      totalDone: this.totalProcessedCount, // Total count of all processed files (not limited to history)
+      totalFilesInVFS: vfsFileCount, // Total number of files accessible in the Virtual File System
+
+      fastQueueConcurrency,
+      backgroundQueueConcurrency
+    };
+  }
+
+  /**
+   * Get the state of a file in any processing queue
+   */
+  getFileState(filePath: string): UnifiedFileState | undefined {
+    return this.pending.get(filePath) ||
+           this.lightProcessing.get(filePath) ||
+           this.hashProcessing.get(filePath);
+  }
+
+  /**
+   * Clear all states (useful for testing or reset)
+   */
+  clear(): void {
+    this.pending.clear();
+    this.lightProcessing.clear();
+    this.hashProcessing.clear();
+    this.done = [];
+    this.totalProcessedCount = 0;
+  }
+
+  /**
+   * Get the total number of files in all states
+   */
+  getTotalFileCount(): number {
+    return this.pending.size + this.lightProcessing.size + this.hashProcessing.size + this.done.length;
+  }
+
+  /**
+   * Retry processing for a file (move back to pending with incremented retry count)
+   */
+  retryProcessing(filePath: string): boolean {
+    const lightState = this.lightProcessing.get(filePath);
+    const hashState = this.hashProcessing.get(filePath);
+    const state = lightState || hashState;
+
+    if (!state) {
+      return false;
+    }
+
+    const retryCount = (state.retryCount || 0) + 1;
+
+    this.lightProcessing.delete(filePath);
+    this.hashProcessing.delete(filePath);
+
+    this.pending.set(filePath, {
+      filePath,
+      state: 'pending',
+      discoveredAt: state.discoveredAt,
+      retryCount,
+      lastRetryAt: Date.now()
+    });
+
+    return true;
+  }
+
+  /**
+   * Get the retry count for a file
+   */
+  getRetryCount(filePath: string): number {
+    const lightState = this.lightProcessing.get(filePath);
+    const hashState = this.hashProcessing.get(filePath);
+    const pendingState = this.pending.get(filePath);
+    return lightState?.retryCount || hashState?.retryCount || pendingState?.retryCount || 0;
+  }
+
+  /**
+   * Get all files currently in light processing
+   * Returns the internal Map for direct iteration
+   */
+  getLightProcessingFiles(): Map<string, UnifiedFileState> {
+    return this.lightProcessing;
+  }
+
+  /**
+   * Get all files currently in hash processing
+   * Returns the internal Map for direct iteration
+   */
+  getHashProcessingFiles(): Map<string, UnifiedFileState> {
+    return this.hashProcessing;
+  }
+}

packages/meta-sort-core/src/logic/WatchedFileProcessor.ts

@@ -0,0 +1,620 @@
+import {promises as fs} from 'fs';
+import {dirname, join, parse} from 'path';
+import {config} from "../config/EnvConfig.js";
+import {FileProcessorInterface} from "@metazla/meta-hash";
+import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js";
+import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js";
+import {FileType} from "@metazla/filename-tools";
+import {existsAsync} from "@metazla/meta-hash";
+import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js";
+import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js";
+import {renamingRule} from "../config/RenamingRule.js";
+import {MetaMeshFormat} from "./MetaMeshFormat.js";
+import {JellyfinAPI} from "../jellyfin/JellyfinAPI.js";
+import {VirtualFileSystem} from "../api/VirtualFileSystem.js";
+import {performanceMetrics} from "../metrics/PerformanceMetrics.js";
+import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js";
+import type {IKVClient} from "../kv/IKVClient.js";
+import PQueue from "p-queue";
+import os from 'os';
+import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js";
+import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js";
+import { from } from "@metazla/meta-interface";
+import { stat } from "fs/promises";
+import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js";
+
+export class WatchedFileProcessor implements FileProcessorInterface {
+    jellyfinAPI = new JellyfinAPI();
+    metaMeshFormat = new MetaMeshFormat();
+    metaDataToFolderStruct = new MetaDataToFolderStruct(renamingRule);
+    virtualFileSystem: VirtualFileSystem;
+    unifiedStateManager = new UnifiedProcessingStateManager();
+    //fileProcessor:FileAnalyzerInterface = new FileProcessor();//TODO only use for debug
+    fileProcessor:FileAnalyzerInterface = new FileProcessorPiscina();
+    duplicateFinder = new DuplicateFinder();
+    duplicateResult: DuplicateResult | null = null;
+    fileType = new FileType();
+    supported = ['video','subtitle','torrent'];
+
+    // Task-based plugin scheduler (replaces two-queue system)
+    private taskScheduler: TaskScheduler | null = null;
+
+    // Pre-processing queue for midhash256 computation (before plugins)
+    private preProcessQueue: PQueue;
+
+    // Reference to StreamingPipeline for queue status
+    private pipeline: StreamingPipeline | null = null;
+
+    constructor() {
+        // Always initialize VirtualFileSystem for FUSE API
+        this.virtualFileSystem = new VirtualFileSystem({
+            fileMode: 0o644,
+            directoryMode: config.CHMOD_FOR_NEW_FOLDER || 0o755
+        });
+
+        // Pass the unified state manager to the fileProcessor
+        if (this.fileProcessor instanceof FileProcessorPiscina) {
+            this.fileProcessor.setUnifiedStateManager(this.unifiedStateManager);
+        }
+
+        // Pre-processing queue for midhash256 computation
+        const cpuCount = os.cpus().length;
+        const preProcessConcurrency = config.MAX_WORKER_THREADS || cpuCount * 2;
+        this.preProcessQueue = new PQueue({concurrency: preProcessConcurrency, autoStart: true});
+
+        console.log(`WatchedFileProcessor: TaskScheduler-based system initialized`);
+        console.log(`  - Pre-process queue: ${preProcessConcurrency} workers (midhash256)`);
+        console.log(`  - TaskScheduler handles plugin execution (fast/background queues)`);
+    }
+
+    getVirtualFileSystem(): VirtualFileSystem {
+        return this.virtualFileSystem;
+    }
+
+    getUnifiedStateManager(): UnifiedProcessingStateManager {
+        return this.unifiedStateManager;
+    }
+
+    getDuplicateResult(): DuplicateResult | null {
+        return this.duplicateResult;
+    }
+
+    /**
+     * Get the KV client
+     */
+    getKVClient(): IKVClient | null {
+        if (this.fileProcessor instanceof FileProcessorPiscina) {
+            return this.fileProcessor.getKVClient();
+        }
+        return null;
+    }
+
+    /**
+     * Set the KV client
+     * This is called by KVManager after successful connection
+     */
+    setKVClient(client: IKVClient): void {
+        if (this.fileProcessor instanceof FileProcessorPiscina) {
+            this.fileProcessor.setKVClient(client);
+        }
+    }
+
+    /**
+     * Set the StreamingPipeline reference for queue status
+     * This is called by index.ts after creating the pipeline
+     */
+    setPipeline(pipeline: StreamingPipeline): void {
+        this.pipeline = pipeline;
+        console.log('[WatchedFileProcessor] Pipeline connected for queue status');
+    }
+
+    /**
+     * Get current queue status for monitoring UI
+     * Uses StreamingPipeline queues which do the actual processing:
+     * Pipeline flow: validation → lightProcessing → hashProcessing
+     *
+     * Mapping to UI:
+     * - Pre-Process = validation (quick extension checks, high concurrency)
+     * - Fast Queue = lightProcessing (midhash256 + metadata extraction)
+     * - Background Queue = hashProcessing (SHA-256 full file hash)
+     */
+    getQueueStatus() {
+        // Use StreamingPipeline stats if available (this is the actual processing engine)
+        if (this.pipeline) {
+            const pipelineStats = this.pipeline.getStats();
+            const queues = pipelineStats.queues;
+
+            // Map StreamingPipeline queues to the 3-queue architecture UI expects
+            // PQueue: .pending = currently running, .size = waiting in queue
+            // API expects: .pending = running, .size = waiting
+            const result = {
+                preProcessQueue: {
+                    // validation = pre-process (quick extension checks)
+                    size: queues.validation.size, // waiting in queue
+                    pending: queues.validation.pending, // running jobs
+                    isPaused: false
+                },
+                fastQueue: {
+                    // fastQueue = fast queue (midhash256 + metadata)
+                    running: queues.fastQueue.pending, // running jobs
+                    pending: queues.fastQueue.size, // waiting in queue
+                    size: queues.fastQueue.size,
+                    isPaused: false
+                },
+                backgroundQueue: {
+                    // backgroundQueue = background (SHA-256 computation)
+                    running: queues.backgroundQueue.pending, // running jobs
+                    pending: queues.backgroundQueue.size, // waiting in queue
+                    size: queues.backgroundQueue.size,
+                    isPaused: false
+                }
+            };
+            return result;
+        }
+
+        // Fallback: use TaskScheduler if available (for future plugin-based architecture)
+        if (this.taskScheduler) {
+            const status = this.taskScheduler.getQueueStatus();
+            return {
+                preProcessQueue: {
+                    size: this.preProcessQueue.pending,
+                    pending: this.preProcessQueue.size,
+                    isPaused: this.preProcessQueue.isPaused
+                },
+                fastQueue: {
+                    size: status.fast.ready,
+                    pending: status.fast.pending,
+                    running: status.fast.running,
+                    isPaused: status.fast.isPaused
+                },
+                backgroundQueue: {
+                    size: status.background.ready,
+                    pending: status.background.pending,
+                    running: status.background.running,
+                    isPaused: status.background.isPaused
+                }
+            };
+        }
+
+        // Fallback: empty status
+        return {
+            preProcessQueue: { size: 0, pending: 0, isPaused: false },
+            fastQueue: { size: 0, pending: 0, running: 0, isPaused: false },
+            backgroundQueue: { size: 0, pending: 0, running: 0, isPaused: false }
+        };
+    }
+
+    /**
+     * Get the TaskScheduler instance
+     */
+    getTaskScheduler(): TaskScheduler | null {
+        return this.taskScheduler;
+    }
+
+    async initialize() {
+        await this.fileProcessor.init();
+
+        // Initialize TaskScheduler with PluginManager using config values
+        const pluginManager = await initializePluginManager();
+        this.taskScheduler = createTaskScheduler(pluginManager, performanceMetrics, {
+            fastQueueConcurrency: config.FAST_QUEUE_CONCURRENCY,
+            backgroundQueueConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY,
+            fastThresholdMs: config.FAST_THRESHOLD_MS,
+            minSamplesForMeasurement: config.MIN_SAMPLES_FOR_MEASUREMENT,
+        });
+
+        // Set up event listeners for VFS updates
+        this.setupTaskSchedulerEvents();
+
+        console.log(`[WatchedFileProcessor] TaskScheduler initialized with:`);
+        console.log(`  - Fast queue concurrency: ${config.FAST_QUEUE_CONCURRENCY}`);
+        console.log(`  - Background queue concurrency: ${config.BACKGROUND_QUEUE_CONCURRENCY}`);
+        console.log(`  - Fast threshold: ${config.FAST_THRESHOLD_MS}ms`);
+        console.log(`  - Min samples for measurement: ${config.MIN_SAMPLES_FOR_MEASUREMENT}`);
+    }
+
+    /**
+     * Set up TaskScheduler event listeners for VFS integration and Redis pub/sub
+     */
+    private setupTaskSchedulerEvents(): void {
+        if (!this.taskScheduler) return;
+
+        // When a task completes, check if we should update VFS
+        this.taskScheduler.on('task:complete', (event: any) => {
+            const { task } = event;
+
+            // Update VFS when filename-parser completes (has basic metadata)
+            if (task.pluginId === 'filename-parser') {
+                this.updateVFSForFile(task.fileHash, task.filePath);
+            }
+
+            // Publish plugin completion event to Redis
+            this.publishPluginComplete(task.fileHash, task.pluginId, task.filePath);
+        });
+
+        // When all tasks for a file complete
+        this.taskScheduler.on('file:complete', (event: any) => {
+            const { fileHash, filePath } = event;
+            console.log(`[TaskScheduler] All plugins complete for ${filePath}`);
+
+            // Final VFS update with complete metadata
+            this.updateVFSForFile(fileHash, filePath);
+
+            // Update unified state manager
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeHashProcessing(filePath, fileHash);
+            }
+        });
+    }
+
+    /**
+     * Publish plugin completion event to Redis
+     * Channel: meta-sort:plugin:complete
+     */
+    private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise<void> {
+        const kvClient = this.getKVClient();
+        if (!kvClient || typeof (kvClient as any).publish !== 'function') {
+            return;
+        }
+
+        try {
+            const message = JSON.stringify({
+                fileHash,
+                pluginId,
+                filePath,
+                timestamp: Date.now()
+            });
+            await (kvClient as any).publish('meta-sort:plugin:complete', message);
+        } catch (error) {
+            // Silent fail - pub/sub is optional
+        }
+    }
+
+    /**
+     * Update VFS for a file with current metadata
+     */
+    private updateVFSForFile(fileHash: string, filePath: string): void {
+        const metadata = this.fileProcessor.getDatabase().get(filePath);
+        if (metadata && metadata.cid_midhash256) {
+            try {
+                const virtualPath = this.metaDataToFolderStruct.renamingRule(metadata as any, '');
+                this.virtualFileSystem.addFile(virtualPath, filePath, metadata);
+            } catch (e) {
+                // Renaming rule may fail for incomplete metadata
+            }
+        }
+    }
+
+    markPending(filePath: string): void {
+        // STATE 1: Mark as pending when file is discovered (before queue)
+        this.unifiedStateManager.addPending(filePath);
+    }
+
+    async queueFile(filePath: string): Promise<void> {
+        // TaskScheduler-based processing
+        // Phase 1: Pre-process (midhash256 computation)
+        // Phase 2: Plugin execution via TaskScheduler (fast/background queues)
+
+        // Debug: log queue state before adding
+        console.log(`[DEBUG queueFile] Before add: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`);
+
+        await this.preProcessQueue.add(async () => {
+            console.log(`[DEBUG queueFile] Task started: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`);
+            try {
+                // STATE: Start LIGHT processing
+                if (this.unifiedStateManager) {
+                    this.unifiedStateManager.startLightProcessing(filePath);
+                }
+
+                // STEP 1: Compute midhash256 (file identifier)
+                const fileStats = await stat(filePath);
+                const { computeMidHash256 } = await import('@metazla/meta-hash');
+
+                // Check cache first
+                const indexLine = (this.fileProcessor as any).indexManager?.getCidForFile(
+                    filePath,
+                    fileStats.size,
+                    fileStats.mtime.toISOString()
+                );
+
+                let midHash256: string;
+                if (indexLine?.cid_midhash256) {
+                    midHash256 = indexLine.cid_midhash256;
+                    performanceMetrics.recordCacheHit('midhash256');
+                } else {
+                    const computeStart = performance.now();
+                    midHash256 = await computeMidHash256(filePath);
+                    performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
+                    performanceMetrics.recordCacheMiss('midhash256');
+                }
+
+                // STEP 2: Initialize metadata with midhash256
+                const metadata: any = {
+                    cid_midhash256: midHash256,
+                    filePath,
+                    processingStatus: 'processing'
+                };
+
+                // Store in local database
+                this.fileProcessor.getDatabase().set(filePath, metadata);
+
+                // STEP 3: Create KV store for plugins
+                const metaDataNode = from(metadata);
+                const kv = new MetadataNodeKVStore(metaDataNode);
+
+                // STEP 4: Create and enqueue plugin tasks
+                if (this.taskScheduler) {
+                    const tasks = this.taskScheduler.createTasksForFile(filePath, midHash256, kv);
+                    this.taskScheduler.enqueueTasks(tasks);
+                }
+
+                // STATE: Light processing setup complete
+                if (this.unifiedStateManager) {
+                    this.unifiedStateManager.completeLightProcessing(
+                        filePath,
+                        midHash256,
+                        undefined // Virtual path will be computed by plugins
+                    );
+                }
+            } catch (error) {
+                console.error(`Failed pre-processing for ${filePath}:`, error);
+
+                // Mark processing as failed
+                if (this.unifiedStateManager) {
+                    this.unifiedStateManager.completeLightProcessing(
+                        filePath,
+                        '',
+                        undefined,
+                        error instanceof Error ? error.message : String(error)
+                    );
+                }
+            }
+        });
+    }
+
+    async processFile(current: number, queueSize: number, filePath: string): Promise<void> {
+        // This method is now a wrapper for queueFile for backwards compatibility
+        // In the new single-phase model, queueFile handles everything
+        await this.queueFile(filePath);
+    }
+
+    async finalize(): Promise<void> {
+        try {
+            console.log(`Virtual filesystem update start`);
+            const start = performance.now();
+            if (this.fileProcessor.getDatabase().size === 0) {
+                console.log(`No files to process`);
+                return;
+            }
+            let startTime = performance.now();
+            const newDuplicateResult = await this.duplicateFinder.findDuplicates(this.fileProcessor.getDatabase());//find duplicates and removes them to avoid conflict
+            const duplicateTime = Math.ceil(performance.now() - startTime);
+            const totalDuplicates = newDuplicateResult.hashDuplicates.length + newDuplicateResult.titleDuplicates.length;
+            console.log(`Duplicate find took ${duplicateTime}ms - found ${newDuplicateResult.hashDuplicates.length} hash groups, ${newDuplicateResult.titleDuplicates.length} title groups`);
+
+            // Only update duplicateResult if we found new duplicates, or if it's the first run
+            if (totalDuplicates > 0 || this.duplicateResult === null) {
+                this.duplicateResult = newDuplicateResult;
+            }
+
+            performanceMetrics.recordDuplicateDetection(duplicateTime, totalDuplicates);
+
+            startTime = performance.now();
+            console.log(`Generating virtual structure for ${this.fileProcessor.getDatabase().size} files`);
+            const virtualStructure = this.metaDataToFolderStruct.generateVirtualStructure(this.fileProcessor.getDatabase());
+            const virtualStructureTime = Math.ceil(performance.now() - startTime);
+            console.log(`Virtual structure generation took ${virtualStructureTime}ms`);
+            performanceMetrics.recordVirtualStructureGeneration(virtualStructureTime);
+
+            // Build VirtualFileSystem for FUSE/WebDAV API
+            startTime = performance.now();
+            this.virtualFileSystem.buildFromComputed(virtualStructure, this.fileProcessor.getDatabase());
+            const vfsBuildTime = Math.ceil(performance.now() - startTime);
+            console.log(`VirtualFileSystem build took ${vfsBuildTime}ms`);
+            performanceMetrics.recordVFSUpdate(vfsBuildTime);
+            const stats = this.virtualFileSystem.getStats();
+            console.log(`VirtualFileSystem: ${stats.fileCount} files (${stats.metaFileCount} .meta), ${stats.directoryCount} directories, ${(stats.totalSize / 1024 / 1024).toFixed(2)} MB`);
+
+            if(this.jellyfinAPI.jellyApiAvailable()) {
+                await this.jellyfinAPI.refreshLibrary();
+                console.log(`Jellyfin refreshed`);
+            }
+
+            console.log(`Virtual filesystem update took ${Math.ceil(performance.now() - start)}ms`);
+        } catch (error) {
+            console.error(error);
+        }
+    }
+
+    async canProcessFile(filePath: string) {
+        const fileTypeResult = await this.fileType.getFileType(filePath);
+        //check if filetype is one of the suported ones
+        return this.supported.includes(fileTypeResult);
+    }
+
+    async deleteFile(filePath: string) {
+        this.fileProcessor.deleteFile(filePath);
+        this.unifiedStateManager.removeFile(filePath);
+    }
+
+    /**
+     * Clean up stale entries from in-memory database for scanned directories
+     *
+     * This is called after discovery scans a folder. It removes files from
+     * in-memory DB that belong to the scanned folder but were not seen during
+     * the scan (meaning they no longer exist on disk).
+     *
+     * Design Philosophy:
+     * - Discovery is the source of truth for file existence
+     * - Empty/missing folders → remove all child paths from memory
+     * - KV keeps everything (distributed metadata archive)
+     *
+     * @param scannedFolders - Array of folder paths that were completely scanned
+     * @param discoveredFiles - Set of file paths that were found during scan
+     */
+    async cleanupStaleEntries(scannedFolders: string[], discoveredFiles: Set<string>): Promise<void> {
+        console.log(`[Cleanup] Starting cleanup for ${scannedFolders.length} scanned folders...`);
+        const startTime = performance.now();
+
+        let removedCount = 0;
+        const db = this.fileProcessor.getDatabase();
+        const removedHashIds: string[] = []; // Collect hashIds for Stremio notification
+
+        for (const folder of scannedFolders) {
+            const normalizedFolder = folder.endsWith('/') ? folder : folder + '/';
+
+            // Find all in-memory files that belong to this folder
+            const filesToCheck: string[] = [];
+            for (const [filePath] of db.entries()) {
+                if (filePath.startsWith(normalizedFolder) || filePath === folder) {
+                    filesToCheck.push(filePath);
+                }
+            }
+
+            // Remove files that weren't discovered (no longer exist)
+            for (const filePath of filesToCheck) {
+                if (!discoveredFiles.has(filePath)) {
+                    // File is in memory but wasn't seen during scan → removed/offline
+                    const metadata = db.get(filePath);
+
+                    // Collect hashId before deletion (for Stremio notification)
+                    if (metadata && metadata.cid_midhash256) {
+                        removedHashIds.push(metadata.cid_midhash256);
+                    }
+
+                    db.delete(filePath);
+                    this.unifiedStateManager.removeFile(filePath);
+                    removedCount++;
+                }
+            }
+        }
+
+        const elapsed = Math.ceil(performance.now() - startTime);
+        console.log(`[Cleanup] Removed ${removedCount} stale entries in ${elapsed}ms`);
+
+        // If we removed files, rebuild VFS to update structure
+        if (removedCount > 0) {
+            console.log('[Cleanup] Rebuilding VFS after cleanup...');
+            await this.finalize();
+
+            // Notify Stremio addon to remove videos (incremental updates like FUSE)
+            if (removedHashIds.length > 0) {
+                console.log(`[Cleanup] Notifying Stremio to remove ${removedHashIds.length} videos...`);
+                for (const hashId of removedHashIds) {
+                    this.notifyStremioRemove(hashId).catch(() => {
+                        // Silent fail - Stremio addon may not be running
+                    });
+                }
+            }
+        }
+    }
+
+    /**
+     * Rebuild VirtualFileSystem from KV data
+     * This is called on startup to restore the VFS from persistent storage
+     *
+     * Fast-load strategy: Load all metadata from KV WITHOUT disk validation.
+     * Discovery will later validate file existence and clean up stale entries.
+     *
+     * Design Philosophy:
+     * - KV = distributed metadata archive (permanent record)
+     * - In-memory DB = current VFS state (only present files)
+     * - Discovery = source of truth (validates and cleans)
+     */
+    async rebuildVFSFromKV(): Promise<void> {
+        const kvClient = this.getKVClient();
+        if (!kvClient) {
+            console.warn('[VFS Rebuild] KV client not available, skipping VFS rebuild');
+            return;
+        }
+
+        try {
+            console.log('[VFS Rebuild] Starting fast-load from KV...');
+            const startTime = performance.now();
+
+            // Get all hash IDs from KV
+            const hashIds = await kvClient.getAllHashIds();
+            console.log(`[VFS Rebuild] Found ${hashIds.length} files in KV`);
+
+            if (hashIds.length === 0) {
+                console.log('[VFS Rebuild] No files in KV, VFS remains empty');
+                return;
+            }
+
+            // Batch fetch all metadata in parallel (no disk validation)
+            console.log('[VFS Rebuild] Fetching metadata in parallel...');
+            const metadataList = await Promise.all(
+                hashIds.map(async (hashId) => {
+                    try {
+                        return await kvClient.getMetadataFlat(hashId);
+                    } catch (error) {
+                        console.error(`[VFS Rebuild] Failed to load metadata for ${hashId}:`, error);
+                        return null;
+                    }
+                })
+            );
+
+            // Populate in-memory database without disk validation
+            let loadedCount = 0;
+            for (const metadata of metadataList) {
+                if (metadata && metadata.filePath) {
+                    // Mark as unverified - discovery will validate later
+                    (metadata as any)._lastVerified = 0;
+                    this.fileProcessor.getDatabase().set(metadata.filePath, metadata);
+                    loadedCount++;
+                }
+            }
+
+            console.log(`[VFS Rebuild] Loaded ${loadedCount}/${hashIds.length} files into memory (unverified)`);
+
+            // Build VFS immediately - may contain stale entries until discovery validates
+            if (this.fileProcessor.getDatabase().size > 0) {
+                await this.finalize();
+                console.log(`[VFS Rebuild] VFS rebuilt in ${Math.ceil(performance.now() - startTime)}ms`);
+                console.log('[VFS Rebuild] Discovery will validate file existence in background');
+            } else {
+                console.log('[VFS Rebuild] No files loaded, VFS remains empty');
+            }
+        } catch (error) {
+            console.error('[VFS Rebuild] Error rebuilding VFS from KV:', error);
+        }
+    }
+
+    /**
+     * Notify Stremio addon to remove video (incremental update like FUSE)
+     */
+    private async notifyStremioRemove(hashId: string): Promise<void> {
+        try {
+            const response = await fetch('http://localhost:7000/remove', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ hashId }),
+                signal: AbortSignal.timeout(5000) // 5 second timeout
+            });
+
+            if (!response.ok) {
+                // Silent fail - Stremio may not be running
+                return;
+            }
+        } catch (error: any) {
+            // Silent fail - Stremio addon may not be running or network issue
+        }
+    }
+
+    /**
+     * Shutdown: clean up resources
+     */
+    async shutdown(): Promise<void> {
+        console.log('[WatchedFileProcessor] Shutting down...');
+
+        // Clear pre-process queue
+        this.preProcessQueue.clear();
+
+        // Shutdown TaskScheduler
+        if (this.taskScheduler) {
+            await this.taskScheduler.shutdown();
+        }
+
+        console.log('[WatchedFileProcessor] Shutdown complete');
+    }
+}

packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts

@@ -0,0 +1,37 @@
+import {HashMeta} from "@metazla/meta-interface";
+import {FileMetadata} from "../../types/FileMetadata.js";
+
+export interface FileAnalyzerInterface {
+    /**
+     * Full file processing - computes midhash256 and extracts all metadata
+     * Since midhash256 is instant (< 1s), files appear in VFS immediately with permanent ID
+     */
+    processFile(filePath: string, current: number, queueSize: number):Promise<void>;
+
+    /**
+     * Light processing phase - fast metadata extraction + midhash256
+     * File becomes accessible in VFS with permanent ID after this phase
+     */
+    processLightPhase(filePath: string, current: number, queueSize: number):Promise<void>;
+
+    /**
+     * Hash processing phase - background hash computation
+     * Computes full hashes (SHA-256, SHA-1, MD5, CRC32) and updates KV
+     */
+    processHashPhase(filePath: string, current: number, queueSize: number):Promise<void>;
+
+    /**
+     * Get the metadata database
+     */
+    getDatabase<T extends FileMetadata>(): Map<string, T>;
+
+    /**
+     * Delete a file from the database
+     */
+    deleteFile(filePath: string): Promise<void> | void;
+
+    /**
+     * Initialize the file analyzer
+     */
+    init():Promise<void>;
+}

packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts

@@ -0,0 +1,92 @@
+import {HashIndexManager} from "@metazla/meta-hash";
+import {config} from "../../config/EnvConfig.js";
+import {stat} from "fs/promises";
+import {targetHash} from "../../config/TargetHash.js";
+import {HashMeta} from "@metazla/meta-interface";
+import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js";
+import {FileProcessorInternal} from "./FileProcessorInternal.js";
+import {globalHashIndex} from "./SimpleFileToCid.js";
+import {FileMetadata} from "../../types/FileMetadata.js";
+
+
+
+export class FileProcessor implements FileAnalyzerInterface{
+    database = new Map<string, FileMetadata>();//Map<filePath,FileMetadata>
+    indexManager = globalHashIndex;
+    fileProcessor = new FileProcessorInternal();
+
+    constructor() {
+    }
+
+    async init() {
+        await this.indexManager.init(true);
+        console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry');
+    }
+
+    /**
+     * Quick metadata processing - extracts metadata without computing content hash
+     * Note: Plugin processing is now handled by container plugins via TaskScheduler
+     */
+    async processFileQuick(filePath: string): Promise<FileMetadata> {
+        // Initialize metadata with processing status
+        const metadata: FileMetadata = {
+            processingStatus: 'processing'
+        };
+
+        // Add to database immediately
+        // Plugin processing will be done asynchronously by container plugins
+        this.database.set(filePath, metadata);
+
+        return metadata;
+    }
+
+    getDatabase<T>(): Map<string, T> {
+        return this.database as Map<string, T>;
+    }
+
+    async processFile(filePath: string, current: number, queueSize: number) {
+        try {
+            if (this.database.has(filePath)) {
+                return;
+            }
+
+            const stats = await stat(filePath);
+            let indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+            let hash: HashMeta = {};
+            if(indexLine) {
+                for (const hashKey of targetHash) {
+                    if (indexLine[hashKey]) {
+                        hash[hashKey] = indexLine[hashKey];
+                    }
+                }
+            }
+
+            // --------------------------------------------
+            // Hash logic compute and get the hash - use cache etc...
+            // --------------------------------------------
+            const result = await this.fileProcessor.processFile(filePath,hash);
+            //const result = await this.fileProcessor.processFile(filePath,params.hash);
+
+            //save the metadata to the cache or disk
+            this.database.set(filePath, result.metadata);
+            this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), result.metadata);
+            //console.log(`File [${current}/${queueSize}] ${filePath} took ${result.time}ms`);
+        } catch (e) {
+            console.error(`Error processing file [No Thread] ${filePath}:`, e);
+        }
+    }
+
+    async processLightPhase(filePath: string, current: number, queueSize: number) {
+        // Legacy FileProcessor doesn't support split phases - just use processFile
+        await this.processFile(filePath, current, queueSize);
+    }
+
+    async processHashPhase(filePath: string, current: number, queueSize: number) {
+        // Legacy FileProcessor doesn't support split phases - no-op
+        // (processFile already did everything)
+    }
+
+    deleteFile(filePath: string) {
+        this.database.delete(filePath);
+    }
+}

packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts

@@ -0,0 +1,84 @@
+import { HashComputerFile } from "@metazla/meta-hash";
+import { targetHash } from "../../config/TargetHash.js";
+import { DebugMeta, HashMeta } from "@metazla/meta-interface";
+import { MetaMeshFormat } from "../MetaMeshFormat.js";
+
+type Local = HashMeta & DebugMeta;
+
+export class FileProcessorInternal {
+    hashComputer = new HashComputerFile(targetHash);
+    metaMeshFormat = new MetaMeshFormat();
+
+    async processFile(filePath: string, hash: HashMeta): Promise<{
+        time: number,
+        metadata: HashMeta,
+        hashTimings?: Record<string, number>,
+        pluginTimings?: Record<string, number>
+    }> {
+        const start = performance.now();
+        const hashTimings: Record<string, number> = {};
+        const pluginTimings: Record<string, number> = {};
+
+        // --------------------------------------------
+        // Hash logic compute and get the hash - use cache etc...
+        // --------------------------------------------
+        const metaDataPath = `${filePath}.meta`;
+        let metadata: HashMeta = await this.metaMeshFormat.read(metaDataPath) || {}; // Read the reference file
+        // try to update the sha values from the index
+        if (hash) {
+            for (let key in hash) {
+                if (!metadata[key] && hash[key]) {
+                    metadata[key] = hash[key];
+                }
+            }
+        }
+
+        // Time consuming - compute hashes and track timing
+        try {
+            // Determine which hashes need to be computed
+            const neededHashes = this.hashComputer['targetHash'].filter(hashName => !metadata[hashName]);
+
+            if (neededHashes.length > 0) {
+                // Measure total time for hash computation
+                const hashStart = performance.now();
+
+                // Compute all needed hashes at once (more efficient than individual)
+                await this.hashComputer.computeMissingHash(filePath, metadata);
+
+                const totalHashTime = Math.ceil(performance.now() - hashStart);
+
+                // Distribute the time evenly among computed hashes
+                // This is an approximation - actual individual hash times may vary
+                const avgTimePerHash = totalHashTime / neededHashes.length;
+                for (const hashAlgorithm of neededHashes) {
+                    hashTimings[hashAlgorithm] = Math.ceil(avgTimePerHash);
+                }
+            }
+        } catch (hashError) {
+            console.error(`[Worker] Hash computation failed for ${filePath}:`, hashError);
+            throw new Error(`Hash computation failed: ${hashError instanceof Error ? hashError.message : String(hashError)}`);
+        }
+
+        // Plugin processing is now handled by container plugins via TaskScheduler
+        // This internal processor only handles hash computation
+
+        // Validate all required hashes are present before returning
+        const missingHashesAfterCompute: string[] = [];
+        for (const hashKey of this.hashComputer['targetHash']) {
+            if (!metadata[hashKey]) {
+                missingHashesAfterCompute.push(hashKey);
+            }
+        }
+
+        if (missingHashesAfterCompute.length > 0) {
+            throw new Error(`[Worker] Hash validation failed - missing hashes: ${missingHashesAfterCompute.join(', ')}`);
+        }
+
+        return {
+            time: Math.ceil(performance.now() - start),
+            metadata,
+            hashTimings,
+            pluginTimings
+        };
+    }
+}

packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts

@@ -0,0 +1,594 @@
+import {Piscina} from "piscina";
+import {Input, Output} from "./worker.js";
+import {HashIndexManager} from "@metazla/meta-hash";
+import {config} from "../../config/EnvConfig.js";
+import {stat} from "fs/promises";
+// Note: Full hashes are now computed by the full-hash container plugin
+import {HashMeta} from "@metazla/meta-interface";
+import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js";
+import {globalHashIndex} from "./SimpleFileToCid.js";
+import {performanceMetrics} from "../../metrics/PerformanceMetrics.js";
+import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js";
+import {renamingRule} from "../../config/RenamingRule.js";
+import {FileMetadata, ProcessingStatus} from "../../types/FileMetadata.js";
+import { PluginManager } from "../../plugin-engine/index.js";
+import type {IKVClient} from "../../kv/IKVClient.js";
+import * as path from 'path';
+import { fileURLToPath } from 'url';
+import os from 'os';
+
+// Get the directory of this file to resolve plugin paths
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = path.dirname(__filename);
+
+// Singleton PluginManager instance
+let pluginManagerInstance: PluginManager | null = null;
+let pluginManagerInitPromise: Promise<void> | null = null;
+
+/**
+ * Get the plugin manager instance (for API access)
+ * Returns null if not yet initialized
+ */
+export function getPluginManagerInstance(): PluginManager | null {
+    return pluginManagerInstance;
+}
+
+/**
+ * Set the plugin manager instance (called by container initialization)
+ */
+export function setPluginManagerInstance(manager: PluginManager): void {
+    pluginManagerInstance = manager;
+}
+
+/**
+ * Initialize the plugin manager on startup
+ * Note: With container-based plugins, the PluginManager is initialized
+ * by the ContainerManager. This function creates a basic instance
+ * for state management if container plugins are not yet ready.
+ */
+export async function initializePluginManager(): Promise<PluginManager> {
+    if (pluginManagerInstance) {
+        return pluginManagerInstance;
+    }
+
+    // Create basic plugin manager for state management
+    const statePath = path.join(config.CACHE_FOLDER_PATH, 'plugins.json');
+    const cacheDir = path.join(config.CACHE_FOLDER_PATH, 'plugin-cache');
+
+    console.log(`[PluginManager] Initializing with:`);
+    console.log(`  - State path: ${statePath}`);
+    console.log(`  - Cache dir: ${cacheDir}`);
+
+    pluginManagerInstance = new PluginManager({
+        statePath,
+        cacheDir,
+    });
+
+    console.log(`[PluginManager] Basic instance created (waiting for container plugins)`);
+    return pluginManagerInstance;
+}
+
+export class FileProcessorPiscina implements FileAnalyzerInterface{
+    private piscina: Piscina;
+    database = new Map<string, FileMetadata>();//Map<filePath,FileMetadata>
+    indexManager = globalHashIndex;
+    private unifiedStateManager: UnifiedProcessingStateManager | null = null;
+
+    // Unified KV client (Redis)
+    private kvClient: IKVClient | null = null;
+
+    // Track running hash workers for abort capability
+    private runningHashTasks = new Map<string, AbortController>();
+
+    constructor() {
+        // Construct the URL for the current module
+        let distFolder = import.meta.dirname;
+        distFolder = distFolder.replace('src', 'dist');
+        distFolder = distFolder + "/worker.js";
+        const workerUrl = new URL(distFolder,"file://").href;
+
+        // Set maxThreads: Smart default uses half CPU cores to avoid starving the system
+        // (especially important for network storage + HDD scenarios)
+        const cpuCount = os.cpus().length;
+        const defaultWorkers = Math.max(1, Math.floor(cpuCount / 2));
+        const maxThreads = config.MAX_WORKER_THREADS || defaultWorkers;
+
+        this.piscina = new Piscina({
+            maxThreads,
+            //maxThreads: 1,//TODO note for DEBUG must be set to 1 to avoid memory issues
+            filename:workerUrl
+        });
+
+        console.log(`Piscina worker pool initialized with ${maxThreads} threads (cpuCount=${cpuCount}, defaultWorkers=${defaultWorkers})`);
+    }
+
+    async init() {
+        await this.indexManager.init(true);
+        console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry');
+    }
+
+    /**
+     * Single-phase metadata processing - computes midhash256 immediately and extracts metadata
+     * Files appear in VFS instantly with their permanent midhash256 ID
+     *
+     * @param filePath - Path to the file to process
+     * @param current - Current file number (for progress reporting)
+     * @param queueSize - Total queue size (for progress reporting)
+     * @returns FileMetadata with midhash256 and all metadata
+     */
+
+    setUnifiedStateManager(manager: UnifiedProcessingStateManager): void {
+        this.unifiedStateManager = manager;
+    }
+
+    getDatabase<T>(): Map<string, T> {
+        return this.database as Map<string, T>;
+    }
+
+    /**
+     * Abort all running hash workers immediately (for priority enforcement)
+     * @returns Number of workers aborted
+     */
+    abortAllHashWorkers(): number {
+        const abortedCount = this.runningHashTasks.size;
+
+        for (const [filePath, controller] of this.runningHashTasks.entries()) {
+            controller.abort();
+            console.log(`[Abort] Hash worker aborted: ${filePath}`);
+        }
+
+        this.runningHashTasks.clear();
+        return abortedCount;
+    }
+
+    /**
+     * Gracefully shutdown - close KV client
+     */
+    async shutdown(): Promise<void> {
+        if (this.kvClient) {
+            console.log('[FileProcessor] Closing KV client...');
+            await this.kvClient.close();
+            console.log('[FileProcessor] KV client closed');
+        }
+    }
+
+    async processFile(filePath: string, current: number, queueSize: number) {
+        const startTime = Date.now();
+
+        try {
+            // STATE: Start LIGHT processing (fast: midhash256 + metadata extraction)
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.startLightProcessing(filePath);
+            }
+
+            const stats = await stat(filePath);
+
+            // STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files)
+            let midHash256: string | undefined;
+            const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+
+            if (indexLine && indexLine.cid_midhash256) {
+                // Cache hit! Use cached midhash256 (no network I/O needed)
+                midHash256 = indexLine.cid_midhash256;
+                performanceMetrics.recordCacheHit('midhash256');
+            } else {
+                // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
+                const computeStart = performance.now();
+                const { computeMidHash256 } = await import('@metazla/meta-hash');
+                midHash256 = await computeMidHash256(filePath);
+                const computeTime = Math.ceil(performance.now() - computeStart);
+
+                // Record midhash256 computation time (excludes cache hits)
+                performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
+                performanceMetrics.recordCacheMiss('midhash256');
+            }
+
+            // Initialize metadata with midhash256
+            const metadata: FileMetadata = {
+                cid_midhash256: midHash256,
+                processingStatus: 'processing'
+            };
+
+            // STEP 2: Plugin metadata extraction is now handled by container plugins
+            // The TaskScheduler will dispatch tasks to container plugins asynchronously
+            // Basic metadata processing continues below
+
+            // Generate virtualPath from metadata (needed for VFS access)
+            let virtualPath: string | undefined;
+            try {
+                virtualPath = renamingRule(metadata as any, '');
+            } catch (e) {
+                // If renamingRule fails, virtualPath remains undefined
+            }
+
+            // STEP 3: Collision detection and store preliminary metadata in KV
+            let collisionDetected = false;
+            if (this.kvClient) {
+                try {
+                    const hashId = midHash256; // Use midhash256 as primary ID
+
+                    if (hashId) {
+                        // COLLISION DETECTION: Check if this midhash256 already exists
+                        const existingFilePath = await this.kvClient.getMetadata(hashId, 'filePath');
+
+                        if (existingFilePath && existingFilePath !== filePath) {
+                            // COLLISION DETECTED: Different file with same midhash256!
+                            collisionDetected = true;
+                            console.warn(`⚠️  MIDHASH256 COLLISION DETECTED!`);
+                            console.warn(`   Hash: ${hashId}`);
+                            console.warn(`   Existing file: ${existingFilePath}`);
+                            console.warn(`   New file:      ${filePath}`);
+                            console.warn(`   Action: Skipping VFS addition, forcing full hash computation`);
+
+                            // Do NOT store in KV with midhash256
+                            // File will be processed with full SHA-256 instead (see below)
+                        } else {
+                            // No collision or same file - safe to store
+                            const finalMetadata = {
+                                ...metadata,
+                                filePath
+                            };
+
+                            // Store using nested key architecture
+                            await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
+                        }
+                    }
+                } catch (metaError) {
+                    console.error(`Failed to store metadata in KV:`, metaError);
+                }
+            }
+
+            // STATE: Light processing complete, move to HASH processing queue
+            // If collision detected, skip VFS addition (virtualPath = undefined)
+            if (this.unifiedStateManager) {
+                if (collisionDetected) {
+                    // Collision: Do NOT complete light processing with virtualPath
+                    // This prevents file from being added to VFS
+                    this.unifiedStateManager.completeLightProcessing(
+                        filePath,
+                        midHash256,
+                        undefined  // No virtualPath = not added to VFS
+                    );
+                } else {
+                    // No collision: File becomes accessible in VFS
+                    this.unifiedStateManager.completeLightProcessing(
+                        filePath,
+                        midHash256,
+                        virtualPath
+                    );
+                }
+                // Start hash processing phase immediately
+                this.unifiedStateManager.startHashProcessing(filePath);
+            }
+
+            // STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the
+            // container plugin "full-hash" in background queue - not here.
+            // Only midhash256 is computed in-process for fast file identification.
+
+            // Mark processing as complete
+            metadata.processingStatus = 'complete';
+            this.database.set(filePath, metadata);
+
+            // STEP 5: Update metadata in KV with full hashes
+            if (this.kvClient) {
+                try {
+                    // Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
+                    let hashId: string;
+                    if (collisionDetected && metadata['cid_sha2-256']) {
+                        // Collision: Use full SHA-256 as primary ID
+                        hashId = metadata['cid_sha2-256'];
+                        console.log(`   Using SHA-256 as primary ID for collision: ${hashId}`);
+                    } else {
+                        // Normal: Use midhash256 as primary ID
+                        hashId = midHash256;
+                    }
+
+                    if (hashId) {
+                        // Prepare metadata with filePath added
+                        const finalMetadata = {
+                            ...metadata,
+                            filePath
+                        };
+
+                        // Store using nested key architecture
+                        await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
+                    }
+                } catch (metaError) {
+                    console.error(`Failed to store metadata in KV:`, metaError);
+                }
+            }
+
+            // STATE: Hash processing complete, move to done
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeHashProcessing(
+                    filePath,
+                    midHash256,
+                    virtualPath
+                );
+            }
+        } catch (e) {
+            console.error(`Error processing file ${filePath}:`, e);
+
+            // Mark processing as failed (complete hash processing with error)
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeHashProcessing(
+                    filePath,
+                    undefined,
+                    undefined,
+                    e instanceof Error ? e.message : String(e)
+                );
+            }
+        }
+    }
+
+    /**
+     * Light Processing Phase - Fast processing to make file accessible in VFS
+     * - Compute midhash256 (< 1s)
+     * - Extract metadata (filename, FFmpeg analysis)
+     * - Store in KV
+     * - File becomes accessible in VFS with midhash256 ID
+     *
+     * @deprecated Use TaskScheduler-based processing via WatchedFileProcessor.queueFile() instead.
+     * This method is kept for backward compatibility with StreamingPipeline.
+     */
+    async processLightPhase(filePath: string, current: number, queueSize: number) {
+        try {
+            // STATE: Start LIGHT processing
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.startLightProcessing(filePath);
+            }
+
+            const stats = await stat(filePath);
+
+            // STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files)
+            let midHash256: string | undefined;
+            const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+
+            if (indexLine && indexLine.cid_midhash256) {
+                // Cache hit! Use cached midhash256 (no network I/O needed)
+                midHash256 = indexLine.cid_midhash256;
+                performanceMetrics.recordCacheHit('midhash256');
+            } else {
+                // Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
+                const computeStart = performance.now();
+                const { computeMidHash256 } = await import('@metazla/meta-hash');
+                midHash256 = await computeMidHash256(filePath);
+                const computeTime = Math.ceil(performance.now() - computeStart);
+
+                // Record midhash256 computation time (excludes cache hits)
+                performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
+                performanceMetrics.recordCacheMiss('midhash256');
+            }
+
+            // Initialize metadata with midhash256
+            const metadata: FileMetadata = {
+                cid_midhash256: midHash256,
+                processingStatus: 'processing'
+            };
+
+            // STEP 2: Plugin metadata extraction is now handled by container plugins
+            // The TaskScheduler will dispatch tasks to container plugins asynchronously
+
+            // Generate virtualPath from metadata (needed for VFS access)
+            let virtualPath: string | undefined;
+            try {
+                virtualPath = renamingRule(metadata as any, '');
+            } catch (e) {
+                // If renamingRule fails, virtualPath remains undefined
+            }
+
+            // STEP 3: Store metadata in KV (file becomes accessible in VFS)
+            if (this.kvClient) {
+                try {
+                    const hashId = midHash256; // Use midhash256 as primary ID
+
+                    if (hashId) {
+                        // Prepare metadata with filePath added
+                        const finalMetadata = {
+                            ...metadata,
+                            filePath
+                        };
+
+                        // Store using nested key architecture
+                        await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
+                    }
+                } catch (metaError) {
+                    console.error(`Failed to store metadata in KV:`, metaError);
+                }
+            }
+
+            // Store in local database
+            this.database.set(filePath, metadata);
+
+            // STATE: Light processing complete, move to HASH processing queue
+            // File is now accessible in VFS with midhash256 ID
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeLightProcessing(
+                    filePath,
+                    midHash256,
+                    virtualPath
+                );
+            }
+        } catch (e) {
+            console.error(`Error in light processing for ${filePath}:`, e);
+
+            // Mark processing as failed
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeLightProcessing(
+                    filePath,
+                    '',
+                    undefined,
+                    e instanceof Error ? e.message : String(e)
+                );
+            }
+            throw e;
+        }
+    }
+
+    /**
+     * Hash Processing Phase - Background hash computation
+     * - Compute full hashes (SHA-256, SHA-1, MD5, CRC32)
+     * - Update KV with additional hash metadata
+     * - Mark processing as complete
+     * - Supports abortion for priority enforcement
+     *
+     * @deprecated Use the full-hash plugin via TaskScheduler instead.
+     * This method is kept for backward compatibility with StreamingPipeline.
+     */
+    async processHashPhase(filePath: string, current: number, queueSize: number) {
+        // Create abort controller for this task
+        const abortController = new AbortController();
+        this.runningHashTasks.set(filePath, abortController);
+
+        try {
+            // STATE: Start HASH processing
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.startHashProcessing(filePath);
+            }
+
+            const stats = await stat(filePath);
+            const metadata = this.database.get(filePath);
+            if (!metadata) {
+                throw new Error('Metadata not found - light processing may have failed');
+            }
+
+            const midHash256 = metadata.cid_midhash256;
+            if (!midHash256) {
+                throw new Error('midhash256 not found in metadata');
+            }
+
+            // Generate virtualPath (should already be done but ensure it's available)
+            let virtualPath: string | undefined;
+            try {
+                virtualPath = renamingRule(metadata as any, '');
+            } catch (e) {
+                // If renamingRule fails, virtualPath remains undefined
+            }
+
+            // Check if collision was detected during light processing
+            // If the midhash256 key exists in KV with a different file path, collision occurred
+            let collisionDetected = false;
+            if (this.kvClient) {
+                try {
+                    const existingFilePath = await this.kvClient.getMetadata(midHash256, 'filePath');
+                    if (existingFilePath && existingFilePath !== filePath) {
+                        collisionDetected = true;
+                        console.log(`   Collision detected in hash phase for ${filePath}`);
+                    }
+                } catch (e) {
+                    // Ignore errors - if key doesn't exist, no collision
+                }
+            }
+
+            // Check if aborted before processing
+            if (abortController.signal.aborted) {
+                throw new Error('AbortError');
+            }
+
+            // STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the
+            // container plugin "full-hash" in background queue - not here.
+            // Only midhash256 is computed in-process for fast file identification.
+
+            // Mark processing as complete
+            metadata.processingStatus = 'complete';
+            this.database.set(filePath, metadata);
+
+            // STEP 5: Update metadata in KV (full hashes will be added later by container plugin)
+            if (this.kvClient) {
+                try {
+                    // Use midhash256 as primary ID
+                    // Note: SHA-256 is computed later by the full-hash container plugin
+                    const hashId = midHash256;
+
+                    if (hashId) {
+                        // Prepare metadata with filePath added
+                        const finalMetadata = {
+                            ...metadata,
+                            filePath
+                        };
+
+                        // Store using nested key architecture
+                        await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
+                    }
+                } catch (metaError) {
+                    console.error(`Failed to store metadata in KV:`, metaError);
+                }
+            }
+
+            // STATE: Hash processing complete, move to done
+            if (this.unifiedStateManager) {
+                this.unifiedStateManager.completeHashProcessing(
+                    filePath,
+                    midHash256,
+                    virtualPath
+                );
+            }
+        } catch (e) {
+            // Handle abort errors gracefully
+            if (e instanceof Error && (e.message === 'AbortError' || e.name === 'AbortError')) {
+                // Task was aborted for priority - re-queue it
+                console.log(`[Abort] Hash processing aborted for ${filePath} - will retry when idle`);
+
+                // Don't mark as failed, just remove from running tasks
+                // The task will remain in hash queue and retry later
+                if (this.unifiedStateManager) {
+                    // Move back to hash processing start (will retry)
+                    this.unifiedStateManager.startHashProcessing(filePath);
+                }
+            } else {
+                console.error(`Error in hash processing for ${filePath}:`, e);
+
+                // Mark hash processing as failed for real errors
+                if (this.unifiedStateManager) {
+                    this.unifiedStateManager.completeHashProcessing(
+                        filePath,
+                        undefined,
+                        undefined,
+                        e instanceof Error ? e.message : String(e)
+                    );
+                }
+            }
+            throw e;
+        } finally {
+            // Always remove from running tasks
+            this.runningHashTasks.delete(filePath);
+        }
+    }
+
+    async deleteFile(filePath: string) {
+        const metadata = this.database.get(filePath);
+        this.database.delete(filePath);
+
+        // Delete from KV if configured (using nested key architecture)
+        if (this.kvClient && metadata) {
+            try {
+                const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
+
+                // Delete all properties for this file using nested key architecture
+                if (hashId) {
+                    const deletedCount = await this.kvClient.deleteMetadataFlat(hashId);
+                    console.log(`Metadata deleted from KV: /file/${hashId} (${deletedCount} keys)`);
+                }
+            } catch (metaError) {
+                // Log error but don't fail the entire process
+                console.error(`Failed to delete metadata from KV:`, metaError);
+            }
+        }
+    }
+
+    /**
+     * Set the KV client
+     * This is called by KVManager after successful connection
+     */
+    setKVClient(client: IKVClient): void {
+        this.kvClient = client;
+        console.log('[FileProcessor] KV client set');
+    }
+
+    /**
+     * Get the KV client
+     */
+    getKVClient(): IKVClient | null {
+        return this.kvClient;
+    }
+}

packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts

@@ -0,0 +1,31 @@
+import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
+import {HashIndexManager} from "@metazla/meta-hash";
+import {config} from "../../config/EnvConfig.js";
+import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
+import {stat} from "fs/promises";
+
+
+export const globalHashIndex = new HashIndexManager(config.INDEX_FOLDER_PATH, targetHashForIndex);
+export const globalHashComputer = new HashComputerFile(targetHash);
+
+class SimpleFileToCid{
+
+    async getcid(filePath: string): Promise<string | null> {
+        try {
+            const stats = await stat(filePath);
+            let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
+            if(indexLine) {
+                return indexLine[CID_ALGORITHM_NAMES.sha256];
+            }else {
+                let hashs = {};
+                await globalHashComputer.computeMissingHash(filePath, hashs);
+                globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
+                return hashs[CID_ALGORITHM_NAMES.sha256];
+            }
+        } catch (e) {
+            return null;
+        }
+    }
+}
+
+export const simpleFileToCid = new SimpleFileToCid();

packages/meta-sort-core/src/logic/fileProcessor/worker.ts

@@ -0,0 +1,20 @@
+import {FileProcessorInternal} from "./FileProcessorInternal.js";
+import {HashMeta} from "@metazla/meta-interface";
+
+const fileProcessor = new FileProcessorInternal();
+
+export interface Input {
+    filePath: string;
+    hash:HashMeta;
+}
+
+export interface Output {
+    time: number;
+    metadata: HashMeta;
+    hashTimings?: Record<string, number>; // Per-hash computation times in ms
+    pluginTimings?: Record<string, number>; // Per-plugin processing times in ms
+}
+
+export default async function processFile({ filePath,hash}: Input): Promise<Output> {
+    return fileProcessor.processFile(filePath,hash);
+}

packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts

@@ -0,0 +1,80 @@
+import {FileAnalyzerInterface} from "../fileProcessor/FileAnalyzerInterface.js";
+import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js";
+import type {IKVClient} from "../../kv/IKVClient.js";
+import {VirtualFileSystem} from "../../api/VirtualFileSystem.js";
+import {MetaDataToFolderStruct} from "../MetaDataToFolderStruct.js";
+import type {ContainerPluginScheduler} from "../../container-plugins/ContainerPluginScheduler.js";
+
+/**
+ * Configuration for StreamingPipeline
+ */
+export interface PipelineConfig {
+    /**
+     * Supported file extensions (application-specific)
+     */
+    supportedExtensions: Set<string>;
+
+    /**
+     * Concurrency for validation stage (extension + optional MIME check)
+     * Default: 32 (I/O bound - fast checks)
+     */
+    validationConcurrency: number;
+
+    /**
+     * Concurrency for fast queue stage (filename parsing, stats)
+     * Default: 16 (CPU/I/O balanced)
+     */
+    fastQueueConcurrency: number;
+
+    /**
+     * Concurrency for background queue stage (hash computation, FFmpeg)
+     * Default: 16 (CPU bound)
+     */
+    backgroundQueueConcurrency: number;
+
+    /**
+     * Enable strict MIME validation (reads file header)
+     * Default: false (skip for performance)
+     */
+    strictMimeValidation: boolean;
+
+    /**
+     * File processor (handles light and heavy processing)
+     */
+    fileProcessor: FileAnalyzerInterface;
+
+    /**
+     * State manager (tracks processing states)
+     */
+    stateManager: UnifiedProcessingStateManager;
+
+    /**
+     * Optional KV client for metadata storage
+     */
+    kvClient?: IKVClient;
+
+    /**
+     * Virtual filesystem (for instant file visibility)
+     */
+    virtualFileSystem: VirtualFileSystem;
+
+    /**
+     * Metadata to folder structure converter (for virtual path generation)
+     */
+    metaDataToFolderStruct: MetaDataToFolderStruct;
+
+    /**
+     * Optional container plugin scheduler for dispatching tasks to containerized plugins
+     */
+    containerPluginScheduler?: ContainerPluginScheduler;
+}
+
+/**
+ * Default pipeline configuration
+ */
+export const DEFAULT_PIPELINE_CONFIG: Partial<PipelineConfig> = {
+    validationConcurrency: 32,
+    fastQueueConcurrency: 16,
+    backgroundQueueConcurrency: 16,
+    strictMimeValidation: false
+};

packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts

@@ -0,0 +1,512 @@
+import * as path from 'path';
+import PQueue from 'p-queue';
+import {PipelineConfig} from './PipelineConfig.js';
+import {withTimeoutAndRetry, TimeoutConfig} from '../../utils/TimeoutWrapper.js';
+import {performanceMetrics} from '../../metrics/PerformanceMetrics.js';
+
+// Pub/sub channel for batched updates to meta-fuse
+const UPDATE_CHANNEL = 'meta-sort:file:batch';
+
+// Batch update interval (5 seconds)
+const BATCH_INTERVAL_MS = 5000;
+
+interface BatchChange {
+    action: 'add' | 'update' | 'remove';
+    hashId: string;
+}
+
+/**
+ * Streaming Pipeline for file processing
+ * Files flow through two stages as they're discovered:
+ * Discovery → Validation → Processing (midhash256 + metadata)
+ *
+ * With midhash256 being instant (< 1s), files appear in VFS immediately
+ *
+ * Publishes batched updates to meta-fuse via Redis pub/sub every 5 seconds.
+ */
+export class StreamingPipeline {
+    private validationQueue: PQueue;
+    private fastQueue: PQueue;
+    private backgroundQueue: PQueue;
+    private config: PipelineConfig;
+
+    private discoveredCount = 0;
+    private validatedCount = 0;
+    private fastProcessedCount = 0;
+    private backgroundProcessedCount = 0;
+
+    // Batch buffer for pub/sub notifications
+    private batchBuffer: BatchChange[] = [];
+    private batchTimer: NodeJS.Timeout | null = null;
+
+    // Retry configuration
+    private readonly MAX_PROCESSING_RETRIES = 10;
+
+    // Fast queue timeout: Fast metadata extraction + midhash256
+    private readonly fastQueueTimeoutConfig: TimeoutConfig = {
+        baseTimeout: 120000, // 2 minutes base timeout
+        timeoutMultiplier: 1.5, // Increase by 1.5x each retry
+        maxTimeout: 600000 // Max 10 minutes
+    };
+
+    // Background queue timeout: Full file hashing over NAS (can be very slow)
+    private readonly backgroundQueueTimeoutConfig: TimeoutConfig = {
+        baseTimeout: 7200000, // 2 hours base timeout (for NAS)
+        timeoutMultiplier: 1.5, // Increase by 1.5x each retry
+        maxTimeout: 14400000 // Max 4 hours
+    };
+
+    constructor(config: PipelineConfig) {
+        this.config = config;
+
+        // Stage 1: Validation (high concurrency - fast I/O checks)
+        this.validationQueue = new PQueue({
+            concurrency: config.validationConcurrency,
+            autoStart: true
+        });
+
+        // Stage 2: Fast Queue - metadata + midhash256 (high concurrency, fast)
+        this.fastQueue = new PQueue({
+            concurrency: config.fastQueueConcurrency,
+            autoStart: true
+        });
+
+        // Stage 3: Background Queue - SHA-256 computation (lower concurrency, CPU-intensive)
+        this.backgroundQueue = new PQueue({
+            concurrency: config.backgroundQueueConcurrency,
+            autoStart: true
+        });
+
+        console.log(`[Pipeline] Initialized with concurrency: validation=${config.validationConcurrency}, fastQueue=${config.fastQueueConcurrency}, backgroundQueue=${config.backgroundQueueConcurrency}`);
+
+        // Start batch publish timer
+        this.startBatchTimer();
+    }
+
+    /**
+     * Start the batch publish timer
+     * Publishes accumulated changes to meta-fuse every 5 seconds
+     */
+    private startBatchTimer(): void {
+        this.batchTimer = setInterval(() => {
+            this.flushBatch().catch(err => {
+                console.error('[Pipeline] Failed to flush batch:', err.message);
+            });
+        }, BATCH_INTERVAL_MS);
+    }
+
+    /**
+     * Stop the batch publish timer
+     */
+    stopBatchTimer(): void {
+        if (this.batchTimer) {
+            clearInterval(this.batchTimer);
+            this.batchTimer = null;
+        }
+    }
+
+    /**
+     * Queue a change for batched notification
+     */
+    private queueChange(action: 'add' | 'update' | 'remove', hashId: string): void {
+        // Deduplicate: remove existing entries for same hashId
+        this.batchBuffer = this.batchBuffer.filter(c => c.hashId !== hashId);
+        this.batchBuffer.push({ action, hashId });
+    }
+
+    /**
+     * Flush the batch buffer and publish to Redis
+     */
+    private async flushBatch(): Promise<void> {
+        if (this.batchBuffer.length === 0) {
+            return;
+        }
+
+        const changes = [...this.batchBuffer];
+        this.batchBuffer = [];
+
+        try {
+            const kvClient = this.config.kvClient;
+            if (kvClient && typeof (kvClient as any).publish === 'function') {
+                const message = JSON.stringify({
+                    timestamp: Date.now(),
+                    changes
+                });
+                await (kvClient as any).publish(UPDATE_CHANNEL, message);
+                console.log(`[Pipeline] Published batch update: ${changes.length} changes`);
+            }
+        } catch (error: any) {
+            // Put changes back in buffer on failure
+            this.batchBuffer = [...changes, ...this.batchBuffer];
+            console.warn(`[Pipeline] Failed to publish batch: ${error.message}`);
+        }
+    }
+
+    /**
+     * Start processing files from discovery stream
+     * Processing begins immediately as files are discovered
+     *
+     * @param discoveryStream - Async generator yielding file paths
+     */
+    async start(discoveryStream: AsyncGenerator<string>): Promise<void> {
+        console.log('[Pipeline] Starting file processing stream...');
+        const startTime = Date.now();
+
+        try {
+            // Consume discovery stream
+            for await (const filePath of discoveryStream) {
+                this.discoveredCount++;
+
+                // Stage 1: Validation (fire and forget)
+                this.validationQueue.add(() => this.validateFile(filePath))
+                    .catch(err => console.error(`[Pipeline] Validation error for ${filePath}:`, err.message));
+
+                // Log progress every 1000 files
+                if (this.discoveredCount % 1000 === 0) {
+                    this.logProgress();
+                }
+            }
+
+            const elapsed = Date.now() - startTime;
+            console.log(`[Pipeline] Discovery completed: ${this.discoveredCount} files found in ${elapsed}ms`);
+            console.log(`[Pipeline] Processing continues in background...`);
+            this.logProgress();
+        } catch (error: any) {
+            console.error(`[Pipeline] Error in discovery stream:`, error.message);
+            throw error;
+        }
+    }
+
+    /**
+     * Stage 1: Validation
+     * - Quick extension check (no file I/O)
+     * - Optional MIME validation (reads file header)
+     * - Mark file as pending
+     */
+    private async validateFile(filePath: string): Promise<void> {
+        try {
+            // Extension check (instant, no I/O)
+            const ext = path.extname(filePath).toLowerCase();
+            if (!this.config.supportedExtensions.has(ext)) {
+                // Skip unsupported file types
+                return;
+            }
+
+            // Optional strict MIME validation
+            // Note: Skipped for performance - extension check is sufficient
+            // If needed, could add MIME validation via external library
+
+            // Mark as pending
+            this.config.stateManager.addPending(filePath);
+            this.validatedCount++;
+
+            // Stage 2: Fast Queue - metadata + midhash256 (fire and forget)
+            this.fastQueue.add(() => this.processLightPhase(filePath))
+                .catch(err => console.error(`[Pipeline] Fast queue error for ${filePath}:`, err.message));
+        } catch (error: any) {
+            console.error(`[Pipeline] Validation failed for ${filePath}:`, error.message);
+        }
+    }
+
+    /**
+     * Stage 2: Fast Queue Phase
+     * - Extract metadata (filename parsing, FFmpeg analysis)
+     * - Compute midhash256 (< 1s)
+     * - Store to KV (file becomes accessible in VFS)
+     * - Queue for background processing
+     * - Includes timeout and retry logic
+     */
+    private async processLightPhase(filePath: string): Promise<void> {
+        try {
+            // Get current retry count
+            const retryCount = this.config.stateManager.getRetryCount(filePath);
+
+            // Call light processing with timeout
+            const current = this.fastProcessedCount + 1;
+            const queueSize = this.discoveredCount;
+
+            await withTimeoutAndRetry(
+                () => this.config.fileProcessor.processLightPhase(filePath, current, queueSize),
+                retryCount,
+                this.fastQueueTimeoutConfig,
+                `Fast queue processing file ${filePath}`
+            );
+
+            // After light processing, add to VFS
+            const metadata = this.config.fileProcessor.getDatabase().get(filePath);
+            if (metadata && metadata.cid_midhash256) {
+                // Generate virtual path from metadata
+                const virtualPath = this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath);
+
+                if (virtualPath) {
+                    // Add to VFS immediately (file appears with permanent midhash256 ID)
+                    this.config.virtualFileSystem.addFile(virtualPath, filePath, metadata);
+                    //console.log(`✓ File added to VFS: ${filePath} → ${virtualPath} (midhash256: ${metadata.cid_midhash256})`);
+
+                    // Queue for pub/sub notification to meta-fuse
+                    if (metadata.cid_midhash256) {
+                        this.queueChange('add', metadata.cid_midhash256);
+                    }
+
+                    // Notify Stremio addon to add video (incremental update like FUSE)
+                    if (metadata.cid_midhash256) {
+                        this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
+                            // Silent fail - Stremio addon may not be running
+                        });
+                    }
+                } else {
+                    // File cannot be added to VFS (e.g., unsupported type, missing title)
+                    console.log(`⚠ File skipped from VFS (no virtual path): ${filePath}`);
+                }
+
+                // Dispatch tasks to container plugins (fire and forget)
+                this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
+
+                // Stage 3: Queue for background processing (SHA-256 computation)
+                this.backgroundQueue.add(() => this.processHashPhase(filePath))
+                    .catch(err => console.error(`[Pipeline] Background queue error for ${filePath}:`, err.message));
+            }
+
+            this.fastProcessedCount++;
+        } catch (error: any) {
+            // Handle timeout or processing error
+            const retryCount = this.config.stateManager.getRetryCount(filePath);
+
+            if (retryCount < this.MAX_PROCESSING_RETRIES) {
+                // Retry: move back to pending queue with incremented retry count
+                const success = this.config.stateManager.retryProcessing(filePath);
+                if (success) {
+                    console.warn(`[Pipeline] Fast queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying with longer timeout. Error: ${error.message}`);
+
+                    // Re-queue for fast processing
+                    this.fastQueue.add(() => this.processLightPhase(filePath))
+                        .catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message));
+                    return;
+                }
+            }
+
+            // Max retries exceeded or retry failed - mark as permanently failed
+            console.error(`[Pipeline] Fast queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`);
+
+            // Record failed file in metrics
+            performanceMetrics.recordFailedFile(
+                filePath,
+                error.message || 'Unknown error',
+                retryCount + 1,
+                'processing'
+            );
+
+            // Remove from processing
+            this.config.stateManager.removeFile(filePath);
+        }
+    }
+
+    /**
+     * Stage 3: Background Queue Phase
+     * - Compute full hashes (SHA-256, SHA-1, MD5, CRC32)
+     * - Update KV with additional hash metadata
+     * - Mark processing as complete
+     * - Includes timeout and retry logic
+     */
+    private async processHashPhase(filePath: string): Promise<void> {
+        try {
+            // Get current retry count
+            const retryCount = this.config.stateManager.getRetryCount(filePath);
+
+            // Call hash processing with timeout
+            const current = this.backgroundProcessedCount + 1;
+            const queueSize = this.discoveredCount;
+
+            await withTimeoutAndRetry(
+                () => this.config.fileProcessor.processHashPhase(filePath, current, queueSize),
+                retryCount,
+                this.backgroundQueueTimeoutConfig,
+                `Background queue processing file ${filePath}`
+            );
+
+            this.backgroundProcessedCount++;
+        } catch (error: any) {
+            // Handle timeout or processing error
+            const retryCount = this.config.stateManager.getRetryCount(filePath);
+
+            if (retryCount < this.MAX_PROCESSING_RETRIES) {
+                // Retry: move back to background queue with incremented retry count
+                console.warn(`[Pipeline] Background queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying. Error: ${error.message}`);
+
+                // Re-queue for background processing
+                this.backgroundQueue.add(() => this.processHashPhase(filePath))
+                    .catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message));
+                return;
+            }
+
+            // Max retries exceeded - mark as permanently failed
+            console.error(`[Pipeline] Background queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`);
+
+            // Record failed file in metrics
+            performanceMetrics.recordFailedFile(
+                filePath,
+                error.message || 'Unknown error',
+                retryCount + 1,
+                'hash'
+            );
+
+            // Complete with error
+            this.config.stateManager.completeHashProcessing(filePath, undefined, undefined, error.message);
+        }
+    }
+
+    /**
+     * Handle file added event (from chokidar)
+     */
+    async handleFileAdded(filePath: string): Promise<void> {
+
+        // Process through validation stage
+        this.validationQueue.add(() => this.validateFile(filePath))
+            .catch(err => console.error(`[Pipeline] Error processing added file ${filePath}:`, err.message));
+    }
+
+    /**
+     * Handle file changed event (from chokidar)
+     */
+    async handleFileChanged(filePath: string): Promise<void> {
+
+        // Reprocess through validation stage
+        this.config.stateManager.removeFile(filePath);
+        this.validationQueue.add(() => this.validateFile(filePath))
+            .catch(err => console.error(`[Pipeline] Error processing changed file ${filePath}:`, err.message));
+    }
+
+    /**
+     * Handle file deleted event (from chokidar)
+     */
+    async handleFileDeleted(filePath: string): Promise<void> {
+        // Get hashId before removal for pub/sub notification
+        const metadata = this.config.fileProcessor.getDatabase().get(filePath);
+        const hashId = (metadata as any)?.cid_midhash256;
+
+        // Remove from state and call cleanup
+        if (this.config.fileProcessor.deleteFile) {
+            this.config.fileProcessor.deleteFile(filePath);
+        }
+        this.config.stateManager.removeFile(filePath);
+
+        // Queue for pub/sub notification to meta-fuse
+        if (hashId) {
+            this.queueChange('remove', hashId);
+        }
+    }
+
+    /**
+     * Get pipeline statistics
+     */
+    getStats() {
+        return {
+            discovered: this.discoveredCount,
+            validated: this.validatedCount,
+            fastProcessed: this.fastProcessedCount,
+            backgroundProcessed: this.backgroundProcessedCount,
+            queues: {
+                validation: {
+                    pending: this.validationQueue.pending,
+                    size: this.validationQueue.size
+                },
+                fastQueue: {
+                    pending: this.fastQueue.pending,
+                    size: this.fastQueue.size
+                },
+                backgroundQueue: {
+                    pending: this.backgroundQueue.pending,
+                    size: this.backgroundQueue.size
+                }
+            },
+            state: this.config.stateManager.getSnapshot()
+        };
+    }
+
+    /**
+     * Log current pipeline progress
+     */
+    private logProgress(): void {
+        const stats = this.getStats();
+    }
+
+    /**
+     * Notify Stremio addon to add video (incremental update like FUSE)
+     */
+    private async notifyStremioAdd(hashId: string): Promise<void> {
+        try {
+            const response = await fetch('http://localhost:7000/add', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ hashId }),
+                signal: AbortSignal.timeout(5000) // 5 second timeout
+            });
+
+            if (!response.ok) {
+                // Silent fail - Stremio may not be running
+                return;
+            }
+        } catch (error: any) {
+            // Silent fail - Stremio addon may not be running or network issue
+            // This is expected during development when Stremio isn't started
+        }
+    }
+
+    /**
+     * Notify Stremio addon to remove video (incremental update like FUSE)
+     */
+    private async notifyStremioRemove(hashId: string): Promise<void> {
+        try {
+            const response = await fetch('http://localhost:7000/remove', {
+                method: 'POST',
+                headers: { 'Content-Type': 'application/json' },
+                body: JSON.stringify({ hashId }),
+                signal: AbortSignal.timeout(5000) // 5 second timeout
+            });
+
+            if (!response.ok) {
+                // Silent fail - Stremio may not be running
+                return;
+            }
+        } catch (error: any) {
+            // Silent fail - Stremio addon may not be running or network issue
+        }
+    }
+
+    /**
+     * Set the container plugin scheduler (called after initialization)
+     */
+    setContainerPluginScheduler(scheduler: import('../../container-plugins/ContainerPluginScheduler.js').ContainerPluginScheduler): void {
+        (this.config as any).containerPluginScheduler = scheduler;
+        console.log('[Pipeline] Container plugin scheduler connected');
+    }
+
+    /**
+     * Dispatch tasks to container plugins for metadata extraction
+     * This is fire-and-forget - container plugins write results directly to meta-core
+     */
+    private dispatchContainerPluginTasks(filePath: string, hashId: string, existingMeta: Record<string, any>): void {
+        const scheduler = this.config.containerPluginScheduler;
+        if (!scheduler) {
+            return; // Container plugins not configured
+        }
+
+        try {
+            // Convert metadata to flat string map for plugin API
+            const metaFlat: Record<string, string> = {};
+            for (const [key, value] of Object.entries(existingMeta)) {
+                if (value != null) {
+                    metaFlat[key] = String(value);
+                }
+            }
+
+            // Dispatch tasks to all enabled plugins
+            scheduler.dispatchAllPlugins(hashId, filePath, metaFlat)
+                .catch(err => {
+                    console.error(`[Pipeline] Failed to dispatch container plugin tasks for ${filePath}:`, err.message);
+                });
+        } catch (error: any) {
+            console.error(`[Pipeline] Error dispatching container plugin tasks for ${filePath}:`, error.message);
+        }
+    }
+}

packages/meta-sort-core/src/logic/pipeline/index.ts

@@ -0,0 +1,2 @@
+export { StreamingPipeline } from './StreamingPipeline.js';
+export { PipelineConfig, DEFAULT_PIPELINE_CONFIG } from './PipelineConfig.js';

packages/meta-sort-core/src/logic/tool/MetadataMerger.ts

@@ -0,0 +1,36 @@
+import lodash from 'lodash';
+import {HashMeta} from "@metazla/meta-interface";
+
+/**
+ * Merge the new metadata into the old metadata
+ * @param existingMetadata
+ * @param newMetadata
+ * @param id
+ * @returns {boolean} true if the metadata has changed
+ */
+export function mergeMetadata(existingMetadata: any, newMetadata: any, id?:string): boolean{
+    let hasChanged = false;
+    const mergeLogic = (existingVal:any, newVal:any, key:string) => {
+        if ((lodash.isObject(existingVal) || lodash.isObject(newVal)) ||
+            (lodash.isArray(existingVal) || lodash.isArray(newVal))) {
+            // Recursively merge objects and arrays
+            return lodash.mergeWith(existingVal, newVal, mergeLogic);
+        }
+
+        if (existingVal) {
+            if (!lodash.isEqual(""+existingVal, ""+newVal)){
+                //conflict on primitive types
+                console.warn(`Conflict found for '${id}'. Existing value: '${existingVal}', New value: '${newVal}'.`);
+            }
+            // Keep the original value (especially in case of conflict)
+            return existingVal;
+        }else {
+            if(newVal !== undefined) {
+                hasChanged = true;
+            }
+            return newVal;
+        }
+    };
+    lodash.mergeWith(existingMetadata, newMetadata, mergeLogic);
+    return hasChanged;
+}

packages/meta-sort-core/src/metrics/PerformanceMetrics.ts

@@ -0,0 +1,582 @@
+/**
+ * Performance metrics tracker for meta-mesh
+ * Tracks detailed performance statistics and timing information
+ */
+
+export interface MetricEntry {
+  timestamp: number;
+  value: number;
+  label?: string;
+}
+
+export interface FailedFileEntry {
+  filePath: string;
+  reason: string;
+  timestamp: number;
+  retryCount: number;
+  stage: 'metadata' | 'hash' | 'processing';
+}
+
+export interface HashTimingStats {
+  total: number;
+  count: number;
+  average: number;
+  min: number;
+  max: number;
+}
+
+export interface PluginTimingStats {
+  total: number;
+  count: number;
+  average: number;
+  min: number;
+  max: number;
+}
+
+export interface PerformanceMetricsData {
+  // File processing metrics
+  totalFilesProcessed: number;
+  averageFileProcessingTime: number;
+  minFileProcessingTime: number;
+  maxFileProcessingTime: number;
+  recentFileProcessingTimes: MetricEntry[];
+
+  // Metadata (light processing) metrics
+  totalMetadataProcessed: number;
+  averageMetadataProcessingTime: number;
+  minMetadataProcessingTime: number;
+  maxMetadataProcessingTime: number;
+
+  // Hash processing metrics
+  totalHashProcessed: number;
+  averageHashProcessingTime: number;
+  minHashProcessingTime: number;
+  maxHashProcessingTime: number;
+
+  // Per-hash-algorithm timing metrics (excludes cache hits)
+  hashComputationTimes: Record<string, HashTimingStats>;
+
+  // Per-plugin timing metrics
+  pluginProcessingTimes: Record<string, PluginTimingStats>;
+
+  // VFS update metrics
+  totalVFSUpdates: number;
+  averageVFSUpdateTime: number;
+  lastVFSUpdateTime: number;
+  recentVFSUpdateTimes: MetricEntry[];
+
+  // Duplicate detection metrics
+  averageDuplicateDetectionTime: number;
+  lastDuplicateDetectionTime: number;
+  totalDuplicatesFound: number;
+
+  // Virtual structure generation metrics
+  averageVirtualStructureTime: number;
+  lastVirtualStructureTime: number;
+
+  // Failed files tracking
+  failedFiles: FailedFileEntry[];
+  totalFailedFiles: number;
+  totalMetadataFailures: number;
+  totalHashFailures: number;
+
+  // System metrics
+  uptime: number;
+  startTime: number;
+  lastActivityTime: number;
+
+  // Memory and performance
+  memoryUsage?: NodeJS.MemoryUsage;
+
+  // Processing rate
+  filesPerSecond: number;
+  totalProcessingTime: number;
+
+  // Cache metrics
+  cacheHits: Record<string, number>;
+  cacheMisses: Record<string, number>;
+  cacheHitRate: Record<string, number>;
+}
+
+export class PerformanceMetrics {
+  private startTime: number;
+  private lastActivityTime: number;
+
+  // File processing tracking - running statistics
+  private recentFileProcessing: MetricEntry[] = [];
+  private totalFilesProcessed: number = 0;
+  private fileProcessingSum: number = 0;
+  private fileProcessingMin: number = Infinity;
+  private fileProcessingMax: number = 0;
+
+  // Metadata (light processing) tracking - running statistics
+  private totalMetadataProcessed: number = 0;
+  private metadataProcessingSum: number = 0;
+  private metadataProcessingMin: number = Infinity;
+  private metadataProcessingMax: number = 0;
+
+  // Hash processing tracking - running statistics
+  private totalHashProcessed: number = 0;
+  private hashProcessingSum: number = 0;
+  private hashProcessingMin: number = Infinity;
+  private hashProcessingMax: number = 0;
+
+  // VFS update tracking - running statistics
+  private recentVFSUpdates: MetricEntry[] = [];
+  private totalVFSUpdates: number = 0;
+  private vfsUpdateSum: number = 0;
+  private lastVFSUpdateTime: number = 0;
+
+  // Duplicate detection tracking - running statistics
+  private totalDuplicatesFound: number = 0;
+  private duplicateDetectionSum: number = 0;
+  private duplicateDetectionCount: number = 0;
+  private lastDuplicateDetectionTime: number = 0;
+
+  // Virtual structure generation tracking - running statistics
+  private virtualStructureSum: number = 0;
+  private virtualStructureCount: number = 0;
+  private lastVirtualStructureTime: number = 0;
+
+  // Failed files tracking
+  private failedFiles: FailedFileEntry[] = [];
+  private totalMetadataFailures: number = 0;
+  private totalHashFailures: number = 0;
+
+  // Cache metrics tracking
+  private cacheHits: Record<string, number> = {};
+  private cacheMisses: Record<string, number> = {};
+
+  // Per-hash-algorithm timing tracking (excludes cache hits)
+  private hashTimings: Record<string, {
+    sum: number;
+    count: number;
+    min: number;
+    max: number;
+  }> = {};
+
+  // Per-plugin timing tracking
+  private pluginTimings: Record<string, {
+    sum: number;
+    count: number;
+    min: number;
+    max: number;
+  }> = {};
+
+  // Configuration
+  private maxRecentEntries: number = 100;
+  private maxFailedFilesHistory: number = 50;
+
+  constructor() {
+    this.startTime = Date.now();
+    this.lastActivityTime = this.startTime;
+  }
+
+  /**
+   * Record a file processing event
+   */
+  recordFileProcessing(filePath: string, timeMs: number): void {
+    // Update running statistics
+    this.fileProcessingSum += timeMs;
+    this.fileProcessingMin = Math.min(this.fileProcessingMin, timeMs);
+    this.fileProcessingMax = Math.max(this.fileProcessingMax, timeMs);
+    this.totalFilesProcessed++;
+
+    // Keep recent list for display only
+    this.recentFileProcessing.push({
+      timestamp: Date.now(),
+      value: timeMs,
+      label: filePath
+    });
+
+    if (this.recentFileProcessing.length > this.maxRecentEntries) {
+      this.recentFileProcessing.shift();
+    }
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a metadata (light processing) event
+   */
+  recordMetadataProcessing(timeMs: number): void {
+    // Update running statistics
+    this.metadataProcessingSum += timeMs;
+    this.metadataProcessingMin = Math.min(this.metadataProcessingMin, timeMs);
+    this.metadataProcessingMax = Math.max(this.metadataProcessingMax, timeMs);
+    this.totalMetadataProcessed++;
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a hash processing event
+   */
+  recordHashProcessing(timeMs: number): void {
+    // Update running statistics
+    this.hashProcessingSum += timeMs;
+    this.hashProcessingMin = Math.min(this.hashProcessingMin, timeMs);
+    this.hashProcessingMax = Math.max(this.hashProcessingMax, timeMs);
+    this.totalHashProcessed++;
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a VFS update event
+   */
+  recordVFSUpdate(timeMs: number): void {
+    // Update running statistics
+    this.vfsUpdateSum += timeMs;
+    this.lastVFSUpdateTime = timeMs;
+    this.totalVFSUpdates++;
+
+    // Keep recent list for display only
+    this.recentVFSUpdates.push({
+      timestamp: Date.now(),
+      value: timeMs
+    });
+
+    if (this.recentVFSUpdates.length > this.maxRecentEntries) {
+      this.recentVFSUpdates.shift();
+    }
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a duplicate detection event
+   */
+  recordDuplicateDetection(timeMs: number, duplicatesFound: number): void {
+    // Update running statistics
+    this.duplicateDetectionSum += timeMs;
+    this.duplicateDetectionCount++;
+    this.lastDuplicateDetectionTime = timeMs;
+    this.totalDuplicatesFound += duplicatesFound;
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a virtual structure generation event
+   */
+  recordVirtualStructureGeneration(timeMs: number): void {
+    // Update running statistics
+    this.virtualStructureSum += timeMs;
+    this.virtualStructureCount++;
+    this.lastVirtualStructureTime = timeMs;
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a cache hit event
+   */
+  recordCacheHit(cacheType: string): void {
+    if (!this.cacheHits[cacheType]) {
+      this.cacheHits[cacheType] = 0;
+    }
+    this.cacheHits[cacheType]++;
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a cache miss event
+   */
+  recordCacheMiss(cacheType: string): void {
+    if (!this.cacheMisses[cacheType]) {
+      this.cacheMisses[cacheType] = 0;
+    }
+    this.cacheMisses[cacheType]++;
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record hash computation time for a specific algorithm (excludes cache hits)
+   * @param hashAlgorithm - Hash algorithm name (e.g., 'cid_sha2-256', 'cid_midhash256')
+   * @param timeMs - Time taken to compute the hash in milliseconds
+   */
+  recordHashComputation(hashAlgorithm: string, timeMs: number): void {
+    if (!this.hashTimings[hashAlgorithm]) {
+      this.hashTimings[hashAlgorithm] = {
+        sum: 0,
+        count: 0,
+        min: Infinity,
+        max: 0
+      };
+    }
+
+    const stats = this.hashTimings[hashAlgorithm];
+    stats.sum += timeMs;
+    stats.count++;
+    stats.min = Math.min(stats.min, timeMs);
+    stats.max = Math.max(stats.max, timeMs);
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record plugin processing time
+   * @param pluginName - Name of the plugin
+   * @param timeMs - Time taken by the plugin in milliseconds
+   */
+  recordPluginProcessing(pluginName: string, timeMs: number): void {
+    if (!this.pluginTimings[pluginName]) {
+      this.pluginTimings[pluginName] = {
+        sum: 0,
+        count: 0,
+        min: Infinity,
+        max: 0
+      };
+    }
+
+    const stats = this.pluginTimings[pluginName];
+    stats.sum += timeMs;
+    stats.count++;
+    stats.min = Math.min(stats.min, timeMs);
+    stats.max = Math.max(stats.max, timeMs);
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Record a failed file processing event
+   */
+  recordFailedFile(filePath: string, reason: string, retryCount: number, stage: 'metadata' | 'hash' | 'processing'): void {
+    const failedEntry: FailedFileEntry = {
+      filePath,
+      reason,
+      timestamp: Date.now(),
+      retryCount,
+      stage
+    };
+
+    // Add to failed files list (most recent first)
+    this.failedFiles.unshift(failedEntry);
+
+    // Trim history if needed
+    if (this.failedFiles.length > this.maxFailedFilesHistory) {
+      this.failedFiles = this.failedFiles.slice(0, this.maxFailedFilesHistory);
+    }
+
+    // Update failure counters
+    if (stage === 'metadata') {
+      this.totalMetadataFailures++;
+    } else {
+      this.totalHashFailures++;
+    }
+
+    this.lastActivityTime = Date.now();
+  }
+
+  /**
+   * Get all metrics data - O(1) complexity using cached running statistics
+   */
+  getMetrics(): PerformanceMetricsData & {
+    // UI compatibility aliases
+    totalFilesDiscovered: number;
+    totalFilesFailed: number;
+    averageLightProcessingMs: number;
+    averageHashProcessingMs: number;
+  } {
+    const now = Date.now();
+    const uptime = now - this.startTime;
+    const totalFailedFiles = this.totalMetadataFailures + this.totalHashFailures;
+    const avgMetadata = this.totalMetadataProcessed > 0
+      ? this.metadataProcessingSum / this.totalMetadataProcessed
+      : 0;
+    const avgHash = this.totalHashProcessed > 0
+      ? this.hashProcessingSum / this.totalHashProcessed
+      : 0;
+
+    return {
+      // File processing metrics - O(1) from running statistics
+      totalFilesProcessed: this.totalFilesProcessed,
+
+      // UI compatibility aliases
+      totalFilesDiscovered: this.totalFilesProcessed + totalFailedFiles,
+      totalFilesFailed: totalFailedFiles,
+      averageLightProcessingMs: avgMetadata,
+      averageHashProcessingMs: avgHash,
+      averageFileProcessingTime: this.totalFilesProcessed > 0
+        ? this.fileProcessingSum / this.totalFilesProcessed
+        : 0,
+      minFileProcessingTime: this.fileProcessingMin === Infinity ? 0 : this.fileProcessingMin,
+      maxFileProcessingTime: this.fileProcessingMax,
+      recentFileProcessingTimes: this.recentFileProcessing.slice(-20), // Last 20 files
+
+      // Metadata (light processing) metrics - O(1) from running statistics
+      totalMetadataProcessed: this.totalMetadataProcessed,
+      averageMetadataProcessingTime: this.totalMetadataProcessed > 0
+        ? this.metadataProcessingSum / this.totalMetadataProcessed
+        : 0,
+      minMetadataProcessingTime: this.metadataProcessingMin === Infinity ? 0 : this.metadataProcessingMin,
+      maxMetadataProcessingTime: this.metadataProcessingMax,
+
+      // Hash processing metrics - O(1) from running statistics
+      totalHashProcessed: this.totalHashProcessed,
+      averageHashProcessingTime: this.totalHashProcessed > 0
+        ? this.hashProcessingSum / this.totalHashProcessed
+        : 0,
+      minHashProcessingTime: this.hashProcessingMin === Infinity ? 0 : this.hashProcessingMin,
+      maxHashProcessingTime: this.hashProcessingMax,
+
+      // Per-hash-algorithm timing metrics (excludes cache hits)
+      hashComputationTimes: Object.keys(this.hashTimings).reduce((acc, hashAlgorithm) => {
+        const stats = this.hashTimings[hashAlgorithm];
+        acc[hashAlgorithm] = {
+          total: stats.sum,
+          count: stats.count,
+          average: stats.count > 0 ? stats.sum / stats.count : 0,
+          min: stats.min === Infinity ? 0 : stats.min,
+          max: stats.max
+        };
+        return acc;
+      }, {} as Record<string, HashTimingStats>),
+
+      // Per-plugin timing metrics
+      pluginProcessingTimes: Object.keys(this.pluginTimings).reduce((acc, pluginName) => {
+        const stats = this.pluginTimings[pluginName];
+        acc[pluginName] = {
+          total: stats.sum,
+          count: stats.count,
+          average: stats.count > 0 ? stats.sum / stats.count : 0,
+          min: stats.min === Infinity ? 0 : stats.min,
+          max: stats.max
+        };
+        return acc;
+      }, {} as Record<string, PluginTimingStats>),
+
+      // VFS update metrics - O(1) from running statistics
+      totalVFSUpdates: this.totalVFSUpdates,
+      averageVFSUpdateTime: this.totalVFSUpdates > 0
+        ? this.vfsUpdateSum / this.totalVFSUpdates
+        : 0,
+      lastVFSUpdateTime: this.lastVFSUpdateTime,
+      recentVFSUpdateTimes: this.recentVFSUpdates.slice(-10), // Last 10 updates
+
+      // Duplicate detection metrics - O(1) from running statistics
+      averageDuplicateDetectionTime: this.duplicateDetectionCount > 0
+        ? this.duplicateDetectionSum / this.duplicateDetectionCount
+        : 0,
+      lastDuplicateDetectionTime: this.lastDuplicateDetectionTime,
+      totalDuplicatesFound: this.totalDuplicatesFound,
+
+      // Virtual structure generation metrics - O(1) from running statistics
+      averageVirtualStructureTime: this.virtualStructureCount > 0
+        ? this.virtualStructureSum / this.virtualStructureCount
+        : 0,
+      lastVirtualStructureTime: this.lastVirtualStructureTime,
+
+      // Failed files tracking
+      failedFiles: this.failedFiles,
+      totalFailedFiles: this.totalMetadataFailures + this.totalHashFailures,
+      totalMetadataFailures: this.totalMetadataFailures,
+      totalHashFailures: this.totalHashFailures,
+
+      // System metrics
+      uptime: uptime,
+      startTime: this.startTime,
+      lastActivityTime: this.lastActivityTime,
+
+      // Memory usage
+      memoryUsage: process.memoryUsage(),
+
+      // Processing rate
+      filesPerSecond: uptime > 0 ? (this.totalFilesProcessed / (uptime / 1000)) : 0,
+      totalProcessingTime: this.fileProcessingSum,
+
+      // Cache metrics
+      cacheHits: { ...this.cacheHits },
+      cacheMisses: { ...this.cacheMisses },
+      cacheHitRate: Object.keys(this.cacheHits).reduce((acc, key) => {
+        const hits = this.cacheHits[key] || 0;
+        const misses = this.cacheMisses[key] || 0;
+        const total = hits + misses;
+        acc[key] = total > 0 ? (hits / total) * 100 : 0;
+        return acc;
+      }, {} as Record<string, number>)
+    };
+  }
+
+  /**
+   * Get timing stats for a specific plugin
+   * @param pluginId - Plugin identifier
+   * @returns Plugin timing stats or null if no data
+   */
+  getPluginTiming(pluginId: string): PluginTimingStats | null {
+    const stats = this.pluginTimings[pluginId];
+    if (!stats || stats.count === 0) {
+      return null;
+    }
+    return {
+      total: stats.sum,
+      count: stats.count,
+      average: stats.sum / stats.count,
+      min: stats.min === Infinity ? 0 : stats.min,
+      max: stats.max
+    };
+  }
+
+  /**
+   * Reset all metrics
+   */
+  reset(): void {
+    // Reset file processing metrics
+    this.recentFileProcessing = [];
+    this.totalFilesProcessed = 0;
+    this.fileProcessingSum = 0;
+    this.fileProcessingMin = Infinity;
+    this.fileProcessingMax = 0;
+
+    // Reset metadata processing metrics
+    this.totalMetadataProcessed = 0;
+    this.metadataProcessingSum = 0;
+    this.metadataProcessingMin = Infinity;
+    this.metadataProcessingMax = 0;
+
+    // Reset hash processing metrics
+    this.totalHashProcessed = 0;
+    this.hashProcessingSum = 0;
+    this.hashProcessingMin = Infinity;
+    this.hashProcessingMax = 0;
+
+    // Reset VFS update metrics
+    this.recentVFSUpdates = [];
+    this.totalVFSUpdates = 0;
+    this.vfsUpdateSum = 0;
+    this.lastVFSUpdateTime = 0;
+
+    // Reset duplicate detection metrics
+    this.totalDuplicatesFound = 0;
+    this.duplicateDetectionSum = 0;
+    this.duplicateDetectionCount = 0;
+    this.lastDuplicateDetectionTime = 0;
+
+    // Reset virtual structure generation metrics
+    this.virtualStructureSum = 0;
+    this.virtualStructureCount = 0;
+    this.lastVirtualStructureTime = 0;
+
+    // Reset failed files tracking
+    this.failedFiles = [];
+    this.totalMetadataFailures = 0;
+    this.totalHashFailures = 0;
+
+    // Reset cache metrics
+    this.cacheHits = {};
+    this.cacheMisses = {};
+
+    // Reset hash computation timing
+    this.hashTimings = {};
+
+    // Reset plugin processing timing
+    this.pluginTimings = {};
+
+    this.startTime = Date.now();
+    this.lastActivityTime = this.startTime;
+  }
+}
+
+// Global singleton instance
+export const performanceMetrics = new PerformanceMetrics();

packages/meta-sort-core/src/plugin-engine/PluginContext.ts

@@ -0,0 +1,392 @@
+/**
+ * Plugin Context Implementation
+ *
+ * Provides KV store, cache, and logger implementations for plugins.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { existsSync, mkdirSync } from 'fs';
+import type { MetadataNode } from '@metazla/meta-interface';
+import type {
+    PluginKVStore,
+    PluginCache,
+    PluginLogger,
+    PluginContext,
+    PluginLifecycleContext,
+} from './types.js';
+
+// =============================================================================
+// KV Store Implementation
+// =============================================================================
+
+/**
+ * In-memory KV store implementation with path-style keys
+ */
+export class KVStore implements PluginKVStore {
+    private data: Map<string, string> = new Map();
+
+    constructor(initialData?: Map<string, string> | Record<string, string>) {
+        if (initialData) {
+            if (initialData instanceof Map) {
+                this.data = new Map(initialData);
+            } else {
+                for (const [key, value] of Object.entries(initialData)) {
+                    if (typeof value === 'string') {
+                        this.data.set(key, value);
+                    }
+                }
+            }
+        }
+    }
+
+    get(key: string): string | undefined {
+        return this.data.get(key);
+    }
+
+    set(key: string, value: string): void {
+        if (key === undefined || key === null || key === '') {
+            return;
+        }
+        if (value === undefined || value === null) {
+            return;
+        }
+        this.data.set(key, String(value));
+    }
+
+    delete(key: string): void {
+        this.data.delete(key);
+    }
+
+    keys(prefix?: string): string[] {
+        const allKeys = Array.from(this.data.keys());
+        if (!prefix) {
+            return allKeys;
+        }
+        return allKeys.filter(k => k.startsWith(prefix));
+    }
+
+    entries(prefix?: string): Array<[string, string]> {
+        const allEntries = Array.from(this.data.entries());
+        if (!prefix) {
+            return allEntries;
+        }
+        return allEntries.filter(([k]) => k.startsWith(prefix));
+    }
+
+    /**
+     * Get all data as a plain object (for serialization)
+     */
+    toObject(): Record<string, string> {
+        const obj: Record<string, string> = {};
+        this.data.forEach((value, key) => {
+            obj[key] = value;
+        });
+        return obj;
+    }
+
+    /**
+     * Get the underlying Map
+     */
+    toMap(): Map<string, string> {
+        return new Map(this.data);
+    }
+
+    /**
+     * Clear all data
+     */
+    clear(): void {
+        this.data.clear();
+    }
+}
+
+// =============================================================================
+// MetadataNode KV Store Adapter
+// =============================================================================
+
+/**
+ * KV store that wraps MetadataNode from @metazla/meta-interface.
+ * This ensures plugins output the same nested object format as the old processors.
+ *
+ * Path conversion:
+ * - 'fileType' -> metadata.at('fileType')
+ * - 'fileinfo/duration' -> metadata.at('fileinfo').at('duration')
+ * - 'fileinfo/streamdetails/video/0/codec' -> metadata.at('fileinfo').at('streamdetails').at('video').at('0').at('codec')
+ */
+export class MetadataNodeKVStore implements PluginKVStore {
+    private metadata: MetadataNode<any>;
+
+    constructor(metadata: MetadataNode<any>) {
+        this.metadata = metadata;
+    }
+
+    /**
+     * Navigate to a nested node using path segments
+     */
+    private navigate(key: string): MetadataNode<any> {
+        const parts = key.split('/');
+        let node = this.metadata;
+        for (const part of parts) {
+            node = node.at(part);
+        }
+        return node;
+    }
+
+    get(key: string): string | undefined {
+        try {
+            const value = this.navigate(key).get();
+            if (value === undefined || value === null) {
+                return undefined;
+            }
+            if (typeof value === 'string') {
+                return value;
+            }
+            if (typeof value === 'object') {
+                // Return undefined for objects (indicates path to nested data)
+                return undefined;
+            }
+            return String(value);
+        } catch {
+            return undefined;
+        }
+    }
+
+    set(key: string, value: string): void {
+        if (key === undefined || key === null || key === '') {
+            return;
+        }
+        if (value === undefined || value === null) {
+            return;
+        }
+        try {
+            this.navigate(key).set(String(value));
+        } catch (e) {
+            console.error(`[MetadataNodeKVStore] Error setting ${key}:`, e);
+        }
+    }
+
+    delete(key: string): void {
+        // MetadataNode doesn't have a delete method, so we set to undefined
+        try {
+            this.navigate(key).set(undefined as any);
+        } catch {
+            // Ignore errors
+        }
+    }
+
+    /**
+     * Get keys is not easily supported by MetadataNode.
+     * This returns an empty array - plugins should track their own keys if needed.
+     */
+    keys(prefix?: string): string[] {
+        // MetadataNode doesn't expose iteration over keys
+        // Plugins that need to iterate should use the metadata object directly
+        console.warn('[MetadataNodeKVStore] keys() not supported - use metadata.get() for object inspection');
+        return [];
+    }
+
+    entries(prefix?: string): Array<[string, string]> {
+        console.warn('[MetadataNodeKVStore] entries() not supported - use metadata.get() for object inspection');
+        return [];
+    }
+
+    /**
+     * Get the underlying MetadataNode
+     */
+    getMetadataNode(): MetadataNode<any> {
+        return this.metadata;
+    }
+
+    /**
+     * Add a value to a set (for RecordSet fields like genres, tags)
+     */
+    add(key: string, value: string): void {
+        if (key === undefined || key === null || key === '') {
+            return;
+        }
+        if (value === undefined || value === null) {
+            return;
+        }
+        try {
+            this.navigate(key).add(value);
+        } catch (e) {
+            console.error(`[MetadataNodeKVStore] Error adding to ${key}:`, e);
+        }
+    }
+
+    /**
+     * Get full object at a path (for inspecting nested structures)
+     */
+    getObject(key: string): any {
+        try {
+            return this.navigate(key).get();
+        } catch {
+            return undefined;
+        }
+    }
+}
+
+// =============================================================================
+// Cache Implementation
+// =============================================================================
+
+/**
+ * File-based cache implementation for plugins
+ */
+export class PluginCacheImpl implements PluginCache {
+    private cacheDir: string;
+    private initialized: boolean = false;
+
+    constructor(cacheDir: string) {
+        this.cacheDir = cacheDir;
+    }
+
+    private ensureDir(): void {
+        if (!this.initialized) {
+            if (!existsSync(this.cacheDir)) {
+                mkdirSync(this.cacheDir, { recursive: true });
+            }
+            this.initialized = true;
+        }
+    }
+
+    getPath(filename: string): string {
+        this.ensureDir();
+        return path.join(this.cacheDir, filename);
+    }
+
+    async readJson<T>(filename: string): Promise<T | null> {
+        try {
+            const filePath = this.getPath(filename);
+            const data = await fs.readFile(filePath, 'utf8');
+            return JSON.parse(data) as T;
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+                return null;
+            }
+            throw error;
+        }
+    }
+
+    async writeJson(filename: string, data: unknown): Promise<void> {
+        this.ensureDir();
+        const filePath = this.getPath(filename);
+        const dir = path.dirname(filePath);
+        if (!existsSync(dir)) {
+            await fs.mkdir(dir, { recursive: true });
+        }
+        await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf8');
+    }
+
+    async exists(filename: string): Promise<boolean> {
+        try {
+            await fs.access(this.getPath(filename));
+            return true;
+        } catch {
+            return false;
+        }
+    }
+
+    async delete(filename: string): Promise<void> {
+        try {
+            await fs.unlink(this.getPath(filename));
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+                throw error;
+            }
+        }
+    }
+
+    async clear(): Promise<void> {
+        try {
+            const files = await fs.readdir(this.cacheDir);
+            await Promise.all(
+                files.map(file => fs.unlink(path.join(this.cacheDir, file)).catch(() => {}))
+            );
+        } catch (error) {
+            if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
+                throw error;
+            }
+        }
+    }
+}
+
+// =============================================================================
+// Logger Implementation
+// =============================================================================
+
+/**
+ * Scoped logger implementation for plugins
+ */
+export class PluginLoggerImpl implements PluginLogger {
+    private prefix: string;
+
+    constructor(pluginId: string) {
+        this.prefix = `[plugin:${pluginId}]`;
+    }
+
+    debug(message: string, ...args: unknown[]): void {
+        console.debug(this.prefix, message, ...args);
+    }
+
+    info(message: string, ...args: unknown[]): void {
+        console.info(this.prefix, message, ...args);
+    }
+
+    warn(message: string, ...args: unknown[]): void {
+        console.warn(this.prefix, message, ...args);
+    }
+
+    error(message: string, ...args: unknown[]): void {
+        console.error(this.prefix, message, ...args);
+    }
+}
+
+// =============================================================================
+// Context Factory
+// =============================================================================
+
+/**
+ * Create a lifecycle context for a plugin
+ */
+export function createLifecycleContext(
+    pluginId: string,
+    config: Record<string, unknown>,
+    cacheDir: string
+): PluginLifecycleContext {
+    return {
+        config,
+        cacheDir,
+        log: new PluginLoggerImpl(pluginId),
+    };
+}
+
+/**
+ * Create a full plugin context for file processing
+ */
+export function createPluginContext(
+    pluginId: string,
+    filePath: string,
+    kv: PluginKVStore,
+    config: Record<string, unknown>,
+    cacheDir: string,
+    metadata?: any
+): PluginContext {
+    const ctx: PluginContext = {
+        kv,
+        filePath,
+        config,
+        cacheDir,
+        cache: new PluginCacheImpl(cacheDir),
+        log: new PluginLoggerImpl(pluginId),
+    };
+
+    // If kv is a MetadataNodeKVStore, extract the MetadataNode
+    if (metadata) {
+        ctx.metadata = metadata;
+    } else if (kv instanceof MetadataNodeKVStore) {
+        ctx.metadata = kv.getMetadataNode();
+    }
+
+    return ctx;
+}

packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts

@@ -0,0 +1,146 @@
+/**
+ * Plugin Executor
+ *
+ * Handles dependency resolution for the container-based plugin system.
+ */
+
+import type { PluginManifest } from './types.js';
+
+// =============================================================================
+// Dependency Resolution
+// =============================================================================
+
+/**
+ * Result of dependency resolution
+ */
+export interface DependencyResolution {
+    order: string[];
+    errors: string[];
+}
+
+/**
+ * Perform topological sort on plugins based on dependencies.
+ * Returns execution order (plugins that must run first come first).
+ *
+ * @param plugins Map of plugin id to manifest
+ * @param activePluginIds Set of active plugin ids to include in the order
+ */
+export function resolveDependencyOrder(
+    plugins: Map<string, PluginManifest>,
+    activePluginIds: Set<string>
+): DependencyResolution {
+    const errors: string[] = [];
+    const order: string[] = [];
+    const visited = new Set<string>();
+    const visiting = new Set<string>(); // For cycle detection
+
+    // Filter to only active plugins
+    const activePlugins = new Map<string, PluginManifest>();
+    activePluginIds.forEach(pluginId => {
+        const manifest = plugins.get(pluginId);
+        if (manifest) {
+            activePlugins.set(pluginId, manifest);
+        }
+    });
+
+    // DFS visit function
+    function visit(pluginId: string): boolean {
+        if (visited.has(pluginId)) {
+            return true;
+        }
+        if (visiting.has(pluginId)) {
+            errors.push(`Circular dependency detected involving plugin '${pluginId}'`);
+            return false;
+        }
+
+        const manifest = activePlugins.get(pluginId);
+        if (!manifest) {
+            // Plugin not found or not active
+            return true;
+        }
+
+        visiting.add(pluginId);
+
+        // Visit dependencies first
+        for (const depId of manifest.dependencies || []) {
+            if (!activePlugins.has(depId)) {
+                // Dependency is not active
+                if (plugins.has(depId)) {
+                    errors.push(`Plugin '${pluginId}' depends on '${depId}' which is not active`);
+                } else {
+                    errors.push(`Plugin '${pluginId}' depends on '${depId}' which does not exist`);
+                }
+                continue;
+            }
+            if (!visit(depId)) {
+                return false;
+            }
+        }
+
+        visiting.delete(pluginId);
+        visited.add(pluginId);
+        order.push(pluginId);
+
+        return true;
+    }
+
+    // Visit all active plugins
+    Array.from(activePluginIds).forEach(pluginId => {
+        if (!visited.has(pluginId)) {
+            visit(pluginId);
+        }
+    });
+
+    return { order, errors };
+}
+
+/**
+ * Check if a plugin can be activated (all dependencies are active)
+ */
+export function canActivate(
+    pluginId: string,
+    plugins: Map<string, PluginManifest>,
+    activePluginIds: Set<string>
+): { canActivate: boolean; missingDeps: string[] } {
+    const manifest = plugins.get(pluginId);
+    if (!manifest) {
+        return { canActivate: false, missingDeps: [] };
+    }
+
+    const missingDeps: string[] = [];
+    for (const depId of manifest.dependencies || []) {
+        if (!activePluginIds.has(depId)) {
+            missingDeps.push(depId);
+        }
+    }
+
+    return {
+        canActivate: missingDeps.length === 0,
+        missingDeps,
+    };
+}
+
+/**
+ * Check if a plugin can be deactivated (no active plugins depend on it)
+ */
+export function canDeactivate(
+    pluginId: string,
+    plugins: Map<string, PluginManifest>,
+    activePluginIds: Set<string>
+): { canDeactivate: boolean; dependents: string[] } {
+    const dependents: string[] = [];
+
+    activePluginIds.forEach(activeId => {
+        if (activeId === pluginId) return;
+
+        const manifest = plugins.get(activeId);
+        if (manifest?.dependencies?.includes(pluginId)) {
+            dependents.push(activeId);
+        }
+    });
+
+    return {
+        canDeactivate: dependents.length === 0,
+        dependents,
+    };
+}

packages/meta-sort-core/src/plugin-engine/PluginManager.ts

@@ -0,0 +1,406 @@
+/**
+ * Plugin Manager
+ *
+ * Main orchestrator for the container-based plugin system.
+ * Handles plugin manifests, lifecycle management, and state persistence.
+ * All plugins run as Docker containers - there is no local plugin execution.
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { existsSync, mkdirSync } from 'fs';
+import { EventEmitter } from 'events';
+import type {
+    PluginManifest,
+    PluginState,
+    PluginStateEntry,
+    PluginInfo,
+    LoadedPlugin,
+    PluginManagerEvent,
+    PluginKVStore,
+} from './types.js';
+import { KVStore } from './PluginContext.js';
+import {
+    resolveDependencyOrder,
+    canActivate,
+    canDeactivate,
+} from './PluginExecutor.js';
+import type { ContainerManager, ContainerPluginManifest } from '../container-plugins/index.js';
+
+// =============================================================================
+// Plugin Manager
+// =============================================================================
+
+export interface PluginManagerOptions {
+    /** Path to plugins.json state file */
+    statePath: string;
+    /** Base directory for plugin caches */
+    cacheDir: string;
+    /** Container manager for containerized plugins (required) */
+    containerManager?: ContainerManager;
+}
+
+export class PluginManager extends EventEmitter {
+    private statePath: string;
+    private cacheDir: string;
+
+    // Runtime state
+    private manifests: Map<string, PluginManifest> = new Map();
+    private loadedPlugins: Map<string, LoadedPlugin> = new Map();
+    private activePluginIds: Set<string> = new Set();
+    private configs: Map<string, Record<string, unknown>> = new Map();
+    private executionOrder: string[] = [];
+
+    // Container plugins (all plugins are container-based)
+    private containerManager: ContainerManager | null = null;
+
+    constructor(options: PluginManagerOptions) {
+        super();
+        this.statePath = options.statePath;
+        this.cacheDir = options.cacheDir;
+        this.containerManager = options.containerManager || null;
+
+        // Ensure cache directory exists
+        if (!existsSync(this.cacheDir)) {
+            mkdirSync(this.cacheDir, { recursive: true });
+        }
+    }
+
+    // =========================================================================
+    // Initialization
+    // =========================================================================
+
+    /**
+     * Initialize the plugin manager with container plugins.
+     * This loads plugin manifests from the ContainerManager and sets up the execution order.
+     */
+    async initialize(): Promise<void> {
+        if (!this.containerManager) {
+            console.warn('[PluginManager] No container manager set, no plugins will be available');
+            return;
+        }
+
+        await this.loadContainerPlugins();
+        console.log(`[PluginManager] Initialized with ${this.manifests.size} container plugins`);
+    }
+
+    // =========================================================================
+    // Lifecycle Management
+    // =========================================================================
+
+    /**
+     * Activate a plugin
+     */
+    async activate(pluginId: string): Promise<void> {
+        const manifest = this.manifests.get(pluginId);
+        if (!manifest) {
+            throw new Error(`Plugin '${pluginId}' not found`);
+        }
+        if (this.activePluginIds.has(pluginId)) {
+            return; // Already active
+        }
+
+        // Check dependencies
+        const { canActivate: can, missingDeps } = canActivate(
+            pluginId,
+            this.manifests,
+            this.activePluginIds
+        );
+        if (!can) {
+            throw new Error(
+                `Cannot activate '${pluginId}': missing dependencies: ${missingDeps.join(', ')}`
+            );
+        }
+
+        // Mark as active
+        this.activePluginIds.add(pluginId);
+        this.rebuildExecutionOrder();
+
+        // Save state
+        await this.saveState();
+
+        // Emit event
+        this.emit('plugin:activated', { pluginId });
+    }
+
+    /**
+     * Deactivate a plugin
+     */
+    async deactivate(pluginId: string): Promise<void> {
+        if (!this.activePluginIds.has(pluginId)) {
+            return; // Already inactive
+        }
+
+        // Check if other plugins depend on this one
+        const { canDeactivate: can, dependents } = canDeactivate(
+            pluginId,
+            this.manifests,
+            this.activePluginIds
+        );
+        if (!can) {
+            throw new Error(
+                `Cannot deactivate '${pluginId}': other plugins depend on it: ${dependents.join(', ')}`
+            );
+        }
+
+        // Mark as inactive
+        this.activePluginIds.delete(pluginId);
+        this.rebuildExecutionOrder();
+
+        // Save state
+        await this.saveState();
+
+        // Emit event
+        this.emit('plugin:deactivated', { pluginId });
+    }
+
+    /**
+     * Update plugin configuration
+     */
+    async updateConfig(pluginId: string, config: Record<string, unknown>): Promise<void> {
+        const manifest = this.manifests.get(pluginId);
+        if (!manifest) {
+            throw new Error(`Plugin '${pluginId}' not found`);
+        }
+
+        // Store the config
+        this.configs.set(pluginId, config);
+
+        // Save state
+        await this.saveState();
+
+        // Emit event
+        this.emit('plugin:config-changed', { pluginId, config });
+    }
+
+    // =========================================================================
+    // Container Plugin Support
+    // =========================================================================
+
+    /**
+     * Set the container manager (can be set after construction)
+     */
+    setContainerManager(containerManager: ContainerManager): void {
+        this.containerManager = containerManager;
+        console.log('[PluginManager] Container manager set');
+    }
+
+    /**
+     * Check if a plugin is a containerized plugin (all plugins are container-based)
+     */
+    isContainerPlugin(pluginId: string): boolean {
+        return this.manifests.has(pluginId);
+    }
+
+    /**
+     * Load plugin manifests from ContainerManager
+     * Should be called after container plugins are initialized
+     */
+    async loadContainerPlugins(): Promise<void> {
+        if (!this.containerManager) {
+            return;
+        }
+
+        const status = this.containerManager.getStatus();
+        if (!status.initialized) {
+            console.warn('[PluginManager] Container manager not initialized, skipping plugins');
+            return;
+        }
+
+        // Load state for plugin active status
+        const state = await this.loadState();
+
+        for (const pluginStatus of status.plugins) {
+            const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId);
+            if (!manifest) continue;
+
+            const pluginId = pluginStatus.pluginId;
+
+            // Convert container manifest to local manifest format
+            const localManifest: PluginManifest = {
+                id: manifest.id,
+                name: manifest.name,
+                version: manifest.version,
+                description: manifest.description || '',
+                author: manifest.author,
+                dependencies: manifest.dependencies || [],
+                schema: manifest.schema || {},
+                config: manifest.config,
+                priority: manifest.priority || 100,
+                color: manifest.color,
+                defaultQueue: manifest.defaultQueue || 'background',
+            };
+
+            this.manifests.set(pluginId, localManifest);
+
+            // Container plugins are always "loaded" but execution is remote
+            this.loadedPlugins.set(pluginId, {
+                manifest: localManifest,
+                instance: null as any, // No local instance for container plugins
+                status: 'loaded',
+                pluginDir: '', // No local dir
+            });
+
+            // Check if should be active
+            const stateEntry = state.plugins[pluginId];
+            if (stateEntry?.active !== false) {
+                // Default to active for plugins
+                this.activePluginIds.add(pluginId);
+            }
+
+            // Store config
+            const config = stateEntry?.config || {};
+            this.configs.set(pluginId, config);
+
+            console.log(`[PluginManager] Loaded plugin: ${pluginId}`);
+        }
+
+        // Rebuild execution order
+        this.rebuildExecutionOrder();
+        await this.saveState();
+    }
+
+    /**
+     * Get the container manager instance
+     */
+    getContainerManager(): ContainerManager | null {
+        return this.containerManager;
+    }
+
+    // =========================================================================
+    // State Management
+    // =========================================================================
+
+    /**
+     * Get all discovered plugins
+     */
+    getPlugins(): PluginInfo[] {
+        const plugins: PluginInfo[] = [];
+
+        this.manifests.forEach((manifest, pluginId) => {
+            const loaded = this.loadedPlugins.get(pluginId);
+            const config = this.configs.get(pluginId) || {};
+
+            plugins.push({
+                id: pluginId,
+                name: manifest.name,
+                version: manifest.version,
+                description: manifest.description,
+                priority: manifest.priority,
+                color: manifest.color,
+                dependencies: manifest.dependencies || [],
+                active: this.activePluginIds.has(pluginId),
+                config,
+                configSchema: manifest.config,
+                metadataSchema: manifest.schema,
+                status: loaded?.status || 'unloaded',
+                error: loaded?.error,
+                isContainer: true, // All plugins are container-based
+            });
+        });
+
+        return plugins;
+    }
+
+    /**
+     * Get only active plugins
+     */
+    getActivePlugins(): PluginInfo[] {
+        return this.getPlugins().filter(p => p.active);
+    }
+
+    /**
+     * Get execution order (dependency-sorted active plugin IDs)
+     */
+    getExecutionOrder(): string[] {
+        return [...this.executionOrder];
+    }
+
+    /**
+     * Create a new KV store for file processing
+     */
+    createKVStore(initialData?: Record<string, string>): KVStore {
+        return new KVStore(initialData);
+    }
+
+    // =========================================================================
+    // Shutdown
+    // =========================================================================
+
+    /**
+     * Shutdown: clear plugin state
+     */
+    async shutdown(): Promise<void> {
+        this.manifests.clear();
+        this.loadedPlugins.clear();
+        this.activePluginIds.clear();
+        this.executionOrder = [];
+        console.log('[PluginManager] Shutdown complete');
+    }
+
+    // =========================================================================
+    // Private Helpers
+    // =========================================================================
+
+    private rebuildExecutionOrder(): void {
+        const { order, errors } = resolveDependencyOrder(
+            this.manifests,
+            this.activePluginIds
+        );
+
+        if (errors.length > 0) {
+            for (const error of errors) {
+                console.warn(`Dependency resolution warning: ${error}`);
+            }
+        }
+
+        this.executionOrder = order;
+    }
+
+    private async loadState(): Promise<PluginState> {
+        try {
+            if (existsSync(this.statePath)) {
+                const content = await fs.readFile(this.statePath, 'utf8');
+                return JSON.parse(content) as PluginState;
+            }
+        } catch (error) {
+            console.error('Failed to load plugin state:', error);
+        }
+        return { plugins: {} };
+    }
+
+    private async saveState(): Promise<void> {
+        const state: PluginState = { plugins: {} };
+
+        this.manifests.forEach((_, pluginId) => {
+            state.plugins[pluginId] = {
+                active: this.activePluginIds.has(pluginId),
+                config: this.configs.get(pluginId) || {},
+            };
+        });
+
+        const dir = path.dirname(this.statePath);
+        if (!existsSync(dir)) {
+            await fs.mkdir(dir, { recursive: true });
+        }
+
+        await fs.writeFile(this.statePath, JSON.stringify(state, null, 2), 'utf8');
+    }
+}
+
+// =============================================================================
+// Factory Function
+// =============================================================================
+
+/**
+ * Create a plugin manager with default paths based on config
+ */
+export function createPluginManager(
+    statePath: string,
+    cacheDir: string
+): PluginManager {
+    return new PluginManager({
+        statePath,
+        cacheDir,
+    });
+}

packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts

@@ -0,0 +1,634 @@
+/**
+ * Task Scheduler
+ *
+ * Manages plugin task execution across fast and background queues.
+ * All plugins run as Docker containers - task execution is delegated to ContainerPluginScheduler.
+ */
+
+import { EventEmitter } from 'events';
+import PQueue from 'p-queue';
+import type {
+    PluginTask,
+    PluginTaskStatus,
+    PluginTaskResult,
+    TaskQueueType,
+    QueueStatus,
+    TaskSchedulerEvent,
+    PluginManifest,
+    PluginKVStore,
+} from './types.js';
+import type { PluginManager } from './PluginManager.js';
+import type { PerformanceMetrics, PluginTimingStats } from '../metrics/PerformanceMetrics.js';
+import type { ContainerPluginScheduler } from '../container-plugins/index.js';
+import { KVStore } from './PluginContext.js';
+
+// =============================================================================
+// Configuration
+// =============================================================================
+
+export interface TaskSchedulerConfig {
+    /** Concurrency for fast queue (default: 32) */
+    fastQueueConcurrency?: number;
+    /** Concurrency for background queue (default: 8) */
+    backgroundQueueConcurrency?: number;
+    /** Threshold in ms for fast classification (default: 1000) */
+    fastThresholdMs?: number;
+    /** Minimum samples before using measured timing (default: 10) */
+    minSamplesForMeasurement?: number;
+}
+
+const DEFAULT_CONFIG: Required<TaskSchedulerConfig> = {
+    fastQueueConcurrency: 32,
+    backgroundQueueConcurrency: 8,
+    fastThresholdMs: 1000,
+    minSamplesForMeasurement: 10,
+};
+
+// =============================================================================
+// Task Scheduler
+// =============================================================================
+
+export class TaskScheduler extends EventEmitter {
+    private pluginManager: PluginManager;
+    private performanceMetrics: PerformanceMetrics;
+    private config: Required<TaskSchedulerConfig>;
+
+    // Queues
+    private fastQueue: PQueue;
+    private backgroundQueue: PQueue;
+
+    // Task tracking
+    private pendingTasks: Map<string, PluginTask> = new Map();
+    private completedTasks: Set<string> = new Set();
+    private fileTasks: Map<string, Set<string>> = new Map(); // fileHash -> taskIds
+    private fileKVStores: Map<string, PluginKVStore> = new Map(); // fileHash -> KV store
+
+    // Container plugin support
+    private containerPluginScheduler: ContainerPluginScheduler | null = null;
+
+    // Statistics
+    private totalTasksCreated: number = 0;
+    private totalTasksCompleted: number = 0;
+    private totalTasksFailed: number = 0;
+
+    constructor(
+        pluginManager: PluginManager,
+        performanceMetrics: PerformanceMetrics,
+        config: TaskSchedulerConfig = {}
+    ) {
+        super();
+        this.pluginManager = pluginManager;
+        this.performanceMetrics = performanceMetrics;
+        this.config = { ...DEFAULT_CONFIG, ...config };
+
+        // Initialize fast queue
+        this.fastQueue = new PQueue({
+            concurrency: this.config.fastQueueConcurrency,
+        });
+
+        // Initialize background queue (starts paused)
+        this.backgroundQueue = new PQueue({
+            concurrency: this.config.backgroundQueueConcurrency,
+            autoStart: false,
+        });
+
+        // When fast queue becomes idle, start background queue
+        this.fastQueue.on('idle', () => {
+            this.onFastQueueIdle();
+        });
+    }
+
+    /**
+     * Set the container plugin scheduler (called after initialization)
+     */
+    setContainerPluginScheduler(scheduler: ContainerPluginScheduler): void {
+        this.containerPluginScheduler = scheduler;
+
+        // Forward container scheduler events to this scheduler
+        scheduler.on('task:completed', ({ task }) => {
+            this.onContainerTaskComplete(task.id, task.fileHash, true);
+        });
+        scheduler.on('task:failed', ({ task }) => {
+            this.onContainerTaskComplete(task.id, task.fileHash, false);
+        });
+        scheduler.on('file:complete', ({ fileHash, filePath }) => {
+            // Container file completion - check if local tasks are also done
+            this.checkFileComplete(fileHash);
+        });
+
+        console.log('[TaskScheduler] Container plugin scheduler set');
+    }
+
+    /**
+     * Handle completion of a container plugin task
+     */
+    private onContainerTaskComplete(taskId: string, fileHash: string, success: boolean): void {
+        // Move from pending to completed
+        this.pendingTasks.delete(taskId);
+        this.completedTasks.add(taskId);
+
+        if (success) {
+            this.totalTasksCompleted++;
+        } else {
+            this.totalTasksFailed++;
+        }
+
+        // Schedule dependent tasks
+        const task = { id: taskId, fileHash, pluginId: taskId.split(':')[1] } as PluginTask;
+        this.scheduleDependentTasks(task);
+
+        // Check if file is complete
+        this.checkFileComplete(fileHash);
+    }
+
+    // =========================================================================
+    // Task Creation
+    // =========================================================================
+
+    /**
+     * Create tasks for all active plugins for a file
+     * @param filePath - Absolute path to the file
+     * @param fileHash - File identity hash (midhash256)
+     * @param kv - KV store for the file (will be shared across all plugins)
+     */
+    createTasksForFile(filePath: string, fileHash: string, kv?: PluginKVStore): PluginTask[] {
+        const tasks: PluginTask[] = [];
+        const executionOrder = this.pluginManager.getExecutionOrder();
+        const plugins = this.pluginManager.getPlugins();
+        const pluginMap = new Map(plugins.map(p => [p.id, p]));
+
+        // Initialize file task tracking
+        if (!this.fileTasks.has(fileHash)) {
+            this.fileTasks.set(fileHash, new Set());
+        }
+        const fileTaskIds = this.fileTasks.get(fileHash)!;
+
+        // Store KV for this file (create new if not provided)
+        if (!this.fileKVStores.has(fileHash)) {
+            this.fileKVStores.set(fileHash, kv || new KVStore());
+        }
+
+        for (const pluginId of executionOrder) {
+            const plugin = pluginMap.get(pluginId);
+            if (!plugin || !plugin.active) continue;
+
+            const taskId = `${fileHash}:${pluginId}`;
+
+            // Skip if task already exists
+            if (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId)) {
+                continue;
+            }
+
+            const timing = this.performanceMetrics.getPluginTiming(pluginId);
+            const queue = this.classifyPlugin(pluginId);
+
+            const task: PluginTask = {
+                id: taskId,
+                fileHash,
+                filePath,
+                pluginId,
+                dependencies: plugin.dependencies || [],
+                queue,
+                priority: timing?.average || 0,
+                estimatedTimeMs: timing?.average || 0,
+                status: 'pending',
+                createdAt: Date.now(),
+            };
+
+            tasks.push(task);
+            fileTaskIds.add(taskId);
+            this.totalTasksCreated++;
+
+            this.emitEvent({ type: 'task:created', task });
+        }
+
+        return tasks;
+    }
+
+    /**
+     * Create a task for a specific plugin on a single file
+     * Used for recompute scenarios where we want to run just one plugin
+     * @param pluginId - Plugin to execute
+     * @param filePath - Absolute path to the file
+     * @param fileHash - File identity hash (midhash256)
+     * @param options - Runtime options merged into plugin config (e.g., { forceRecompute: true })
+     * @param kvData - Optional initial KV data (flattened metadata from Redis)
+     * @returns The created task, or null if plugin not found/not active
+     */
+    createTaskForPlugin(
+        pluginId: string,
+        filePath: string,
+        fileHash: string,
+        options?: Record<string, unknown>,
+        kvData?: Record<string, string>
+    ): PluginTask | null {
+        const plugins = this.pluginManager.getPlugins();
+        const plugin = plugins.find(p => p.id === pluginId);
+
+        if (!plugin || !plugin.active) {
+            console.warn(`[TaskScheduler] Plugin '${pluginId}' not found or not active`);
+            return null;
+        }
+
+        // Use a unique task ID for recompute tasks to avoid conflicts
+        const taskId = options?.forceRecompute
+            ? `${fileHash}:${pluginId}:recompute:${Date.now()}`
+            : `${fileHash}:${pluginId}`;
+
+        // Skip if non-recompute task already exists
+        if (!options?.forceRecompute && (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId))) {
+            return null;
+        }
+
+        // Initialize file task tracking if needed
+        if (!this.fileTasks.has(fileHash)) {
+            this.fileTasks.set(fileHash, new Set());
+        }
+        const fileTaskIds = this.fileTasks.get(fileHash)!;
+
+        // Initialize or update KV store for this file
+        // For recompute, we want to load existing metadata so plugin can see current state
+        if (!this.fileKVStores.has(fileHash)) {
+            this.fileKVStores.set(fileHash, new KVStore(kvData));
+        } else if (kvData) {
+            // Update existing KV store with provided data
+            const existingKv = this.fileKVStores.get(fileHash)!;
+            for (const [key, value] of Object.entries(kvData)) {
+                existingKv.set(key, value);
+            }
+        }
+
+        const timing = this.performanceMetrics.getPluginTiming(pluginId);
+        const queue = this.classifyPlugin(pluginId);
+
+        const task: PluginTask = {
+            id: taskId,
+            fileHash,
+            filePath,
+            pluginId,
+            dependencies: plugin.dependencies || [],
+            queue,
+            priority: timing?.average || 0,
+            estimatedTimeMs: timing?.average || 0,
+            status: 'pending',
+            createdAt: Date.now(),
+            options,
+        };
+
+        fileTaskIds.add(taskId);
+        this.totalTasksCreated++;
+        this.emitEvent({ type: 'task:created', task });
+
+        return task;
+    }
+
+    /**
+     * Create tasks for a specific plugin on multiple files
+     * Used for "recompute plugin on all files" feature
+     * @param pluginId - Plugin to execute
+     * @param files - Array of { filePath, fileHash, kvData? } objects
+     * @param options - Runtime options merged into plugin config
+     * @returns Array of created tasks
+     */
+    createTasksForPluginOnFiles(
+        pluginId: string,
+        files: Array<{ filePath: string; fileHash: string; kvData?: Record<string, string> }>,
+        options?: Record<string, unknown>
+    ): PluginTask[] {
+        const tasks: PluginTask[] = [];
+
+        for (const { filePath, fileHash, kvData } of files) {
+            const task = this.createTaskForPlugin(pluginId, filePath, fileHash, options, kvData);
+            if (task) {
+                tasks.push(task);
+            }
+        }
+
+        console.log(`[TaskScheduler] Created ${tasks.length} tasks for plugin '${pluginId}'`);
+        return tasks;
+    }
+
+    /**
+     * Enqueue a task for execution
+     * All plugins are container-based, so all tasks go to the container scheduler
+     */
+    enqueueTask(task: PluginTask): void {
+        this.pendingTasks.set(task.id, task);
+        this.enqueueContainerTask(task);
+    }
+
+    /**
+     * Enqueue multiple tasks
+     */
+    enqueueTasks(tasks: PluginTask[]): void {
+        for (const task of tasks) {
+            this.enqueueTask(task);
+        }
+    }
+
+    /**
+     * Enqueue a task to the container plugin scheduler
+     */
+    private enqueueContainerTask(task: PluginTask): void {
+        if (!this.containerPluginScheduler) {
+            console.warn(
+                `[TaskScheduler] Container plugin task ${task.id} cannot be scheduled: no container scheduler`
+            );
+            task.status = 'failed';
+            task.result = {
+                success: false,
+                timeMs: 0,
+                error: 'Container plugin scheduler not available',
+            };
+            this.pendingTasks.delete(task.id);
+            this.completedTasks.add(task.id);
+            this.totalTasksFailed++;
+            return;
+        }
+
+        // Create container task and enqueue
+        const containerTask = this.containerPluginScheduler.createTask(
+            task.pluginId,
+            task.fileHash,
+            task.filePath,
+            task.dependencies,
+            task.queue
+        );
+
+        // Map the local task ID to the container task ID
+        // Update our pending tasks map with the container task ID
+        this.pendingTasks.delete(task.id);
+        this.pendingTasks.set(containerTask.id, {
+            ...task,
+            id: containerTask.id, // Use container task ID
+        });
+
+        // Update file task tracking
+        const fileTaskIds = this.fileTasks.get(task.fileHash);
+        if (fileTaskIds) {
+            fileTaskIds.delete(task.id);
+            fileTaskIds.add(containerTask.id);
+        }
+
+        // Enqueue to container scheduler
+        this.containerPluginScheduler.enqueueTask(containerTask);
+    }
+
+    // =========================================================================
+    // Classification
+    // =========================================================================
+
+    /**
+     * Classify a plugin as fast or background based on timing data and manifest
+     */
+    classifyPlugin(pluginId: string): TaskQueueType {
+        const plugins = this.pluginManager.getPlugins();
+        const plugin = plugins.find(p => p.id === pluginId);
+        const timing = this.performanceMetrics.getPluginTiming(pluginId);
+
+        // Get manifest default queue (need to access via plugin manager internals)
+        // For now, we'll use a workaround - check if plugin has defaultQueue in manifest
+        const manifest = this.getPluginManifest(pluginId);
+        const defaultQueue = manifest?.defaultQueue || 'fast';
+
+        // If we have enough samples, use measured timing
+        if (timing && timing.count >= this.config.minSamplesForMeasurement) {
+            return timing.average < this.config.fastThresholdMs ? 'fast' : 'background';
+        }
+
+        // Otherwise use manifest default
+        return defaultQueue;
+    }
+
+    // =========================================================================
+    // Task Completion
+    // =========================================================================
+
+    private scheduleDependentTasks(completedTask: PluginTask): void {
+        // Container scheduler handles dependency resolution
+        // This method is kept for compatibility but doesn't need to do anything
+    }
+
+    private checkFileComplete(fileHash: string): void {
+        const fileTaskIds = this.fileTasks.get(fileHash);
+        if (!fileTaskIds) return;
+
+        // Check if all tasks for this file are completed
+        let filePath = '';
+        for (const taskId of fileTaskIds) {
+            if (!this.completedTasks.has(taskId)) {
+                return; // Still have pending/running tasks
+            }
+            // Try to get filePath from any task
+            if (!filePath) {
+                const task = this.pendingTasks.get(taskId);
+                if (task) filePath = task.filePath;
+            }
+        }
+
+        // All tasks complete
+        this.emitEvent({
+            type: 'file:complete',
+            fileHash,
+            filePath,
+        });
+
+        // Clean up file tracking
+        this.fileTasks.delete(fileHash);
+        this.fileKVStores.delete(fileHash);
+
+        // Clean up completed task IDs for this file to prevent memory leak
+        for (const taskId of fileTaskIds) {
+            this.completedTasks.delete(taskId);
+        }
+    }
+
+    // =========================================================================
+    // Queue Control
+    // =========================================================================
+
+    private onFastQueueIdle(): void {
+        this.emitEvent({ type: 'queue:fast:idle' });
+
+        // Start background queue if it has pending work
+        if (this.backgroundQueue.size > 0 || this.backgroundQueue.pending > 0) {
+            console.log('[TaskScheduler] Fast queue idle, starting background queue');
+            this.backgroundQueue.start();
+            this.emitEvent({ type: 'queue:background:started' });
+        }
+    }
+
+    // =========================================================================
+    // Status & Metrics
+    // =========================================================================
+
+    /**
+     * Get status of both queues
+     */
+    getQueueStatus(): { fast: QueueStatus; background: QueueStatus } {
+        return {
+            fast: this.buildQueueStatus('fast'),
+            background: this.buildQueueStatus('background'),
+        };
+    }
+
+    private buildQueueStatus(queueType: TaskQueueType): QueueStatus {
+        const queue = queueType === 'fast' ? this.fastQueue : this.backgroundQueue;
+
+        let pending = 0;
+        let ready = 0;
+        let running = 0;
+        let completed = 0;
+        let failed = 0;
+
+        // Count pending tasks for this queue type
+        for (const [, task] of this.pendingTasks) {
+            if (task.queue === queueType) {
+                if (task.status === 'pending') pending++;
+                else if (task.status === 'ready') ready++;
+                else if (task.status === 'running') running++;
+            }
+        }
+
+        // Count completed tasks
+        for (const taskId of this.completedTasks) {
+            const [, pluginId] = taskId.split(':');
+            if (this.classifyPlugin(pluginId) === queueType) {
+                completed++;
+            }
+        }
+
+        return {
+            pending,
+            ready: queue.size, // Tasks in queue waiting to run
+            running: queue.pending, // Tasks currently executing
+            completed,
+            failed, // Would need separate tracking
+            isPaused: queue.isPaused,
+        };
+    }
+
+    /**
+     * Get pending tasks for a specific file
+     */
+    getPendingTasksForFile(fileHash: string): PluginTask[] {
+        const tasks: PluginTask[] = [];
+        const fileTaskIds = this.fileTasks.get(fileHash);
+
+        if (fileTaskIds) {
+            for (const taskId of fileTaskIds) {
+                const task = this.pendingTasks.get(taskId);
+                if (task) {
+                    tasks.push(task);
+                }
+            }
+        }
+
+        return tasks;
+    }
+
+    /**
+     * Get the KV store for a file
+     */
+    getFileKVStore(fileHash: string): PluginKVStore | undefined {
+        return this.fileKVStores.get(fileHash);
+    }
+
+    /**
+     * Cancel processing for a file (removes all pending tasks)
+     */
+    cancelFile(fileHash: string): void {
+        const fileTaskIds = this.fileTasks.get(fileHash);
+        if (!fileTaskIds) return;
+
+        // Remove pending tasks
+        for (const taskId of fileTaskIds) {
+            this.pendingTasks.delete(taskId);
+        }
+
+        // Clean up file tracking
+        this.fileTasks.delete(fileHash);
+        this.fileKVStores.delete(fileHash);
+
+        console.log(`[TaskScheduler] Cancelled processing for file ${fileHash}`);
+    }
+
+    /**
+     * Get scheduler statistics
+     */
+    getStats(): {
+        totalTasksCreated: number;
+        totalTasksCompleted: number;
+        totalTasksFailed: number;
+        pendingTaskCount: number;
+        completedTaskCount: number;
+        filesInProgress: number;
+    } {
+        return {
+            totalTasksCreated: this.totalTasksCreated,
+            totalTasksCompleted: this.totalTasksCompleted,
+            totalTasksFailed: this.totalTasksFailed,
+            pendingTaskCount: this.pendingTasks.size,
+            completedTaskCount: this.completedTasks.size,
+            filesInProgress: this.fileTasks.size,
+        };
+    }
+
+    // =========================================================================
+    // Shutdown
+    // =========================================================================
+
+    /**
+     * Gracefully shutdown the scheduler
+     */
+    async shutdown(): Promise<void> {
+        console.log('[TaskScheduler] Shutting down...');
+
+        // Clear queues
+        this.fastQueue.clear();
+        this.backgroundQueue.clear();
+
+        // Wait for in-progress tasks to complete
+        await Promise.all([
+            this.fastQueue.onIdle(),
+            this.backgroundQueue.onIdle(),
+        ]);
+
+        // Clear tracking
+        this.pendingTasks.clear();
+        this.completedTasks.clear();
+        this.fileTasks.clear();
+        this.fileKVStores.clear();
+
+        console.log('[TaskScheduler] Shutdown complete');
+    }
+
+    // =========================================================================
+    // Helpers
+    // =========================================================================
+
+    private getPluginManifest(pluginId: string): PluginManifest | undefined {
+        // Access manifests from PluginManager
+        const manifests = (this.pluginManager as any).manifests as Map<string, PluginManifest>;
+        return manifests.get(pluginId);
+    }
+
+    private emitEvent(event: TaskSchedulerEvent): void {
+        this.emit(event.type, event);
+    }
+}
+
+// =============================================================================
+// Factory Function
+// =============================================================================
+
+/**
+ * Create a task scheduler instance
+ */
+export function createTaskScheduler(
+    pluginManager: PluginManager,
+    performanceMetrics: PerformanceMetrics,
+    config?: TaskSchedulerConfig
+): TaskScheduler {
+    return new TaskScheduler(pluginManager, performanceMetrics, config);
+}

packages/meta-sort-core/src/plugin-engine/index.ts

@@ -0,0 +1,59 @@
+/**
+ * Plugin System
+ *
+ * Main entry point for the meta-sort container-based plugin system.
+ * All plugins run as Docker containers - there is no local plugin execution.
+ */
+
+// Types
+export type {
+    // Manifest types
+    ConfigFieldType,
+    SchemaFieldType,
+    SelectOption,
+    ConfigField,
+    SchemaField,
+    PluginManifest,
+    TaskQueueType,
+    // State types
+    PluginStateEntry,
+    PluginState,
+    PluginStatus,
+    PluginInfo,
+    // Context types
+    PluginKVStore,
+    PluginCache,
+    PluginLogger,
+    PluginLifecycleContext,
+    PluginContext,
+    // Plugin interface
+    Plugin,
+    // Internal types
+    LoadedPlugin,
+    PluginManagerEvent,
+    // Task types
+    PluginTaskStatus,
+    PluginTaskResult,
+    PluginTask,
+    QueueStatus,
+    TaskSchedulerEvent,
+} from './types.js';
+
+// Context implementations (KVStore is still used for local state)
+export { KVStore, MetadataNodeKVStore } from './PluginContext.js';
+
+// Dependency resolution utilities
+export {
+    resolveDependencyOrder,
+    canActivate,
+    canDeactivate,
+} from './PluginExecutor.js';
+export type { DependencyResolution } from './PluginExecutor.js';
+
+// Plugin Manager
+export { PluginManager, createPluginManager } from './PluginManager.js';
+export type { PluginManagerOptions } from './PluginManager.js';
+
+// Task Scheduler
+export { TaskScheduler, createTaskScheduler } from './TaskScheduler.js';
+export type { TaskSchedulerConfig } from './TaskScheduler.js';

packages/meta-sort-core/src/plugin-engine/plugin.test.ts

@@ -0,0 +1,667 @@
+/**
+ * Plugin System Unit Tests
+ */
+
+import { expect } from 'chai';
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { existsSync, mkdirSync, rmSync } from 'fs';
+import { fileURLToPath } from 'url';
+
+import {
+    KVStore,
+    PluginCacheImpl,
+    PluginLoggerImpl,
+    parseManifest,
+    validateManifest,
+    resolveDependencyOrder,
+    canActivate,
+    canDeactivate,
+    PluginManager,
+} from './index.js';
+import type { PluginManifest, Plugin, PluginContext } from './types.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TEST_DIR = path.join(__dirname, '../../test-plugins');
+const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache');
+const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins.json');
+
+// =============================================================================
+// Test Helpers
+// =============================================================================
+
+async function setupTestDir() {
+    // Clean up
+    if (existsSync(TEST_DIR)) {
+        rmSync(TEST_DIR, { recursive: true });
+    }
+    if (existsSync(TEST_CACHE_DIR)) {
+        rmSync(TEST_CACHE_DIR, { recursive: true });
+    }
+    if (existsSync(TEST_STATE_PATH)) {
+        rmSync(TEST_STATE_PATH);
+    }
+
+    mkdirSync(TEST_DIR, { recursive: true });
+    mkdirSync(TEST_CACHE_DIR, { recursive: true });
+}
+
+async function cleanupTestDir() {
+    if (existsSync(TEST_DIR)) {
+        rmSync(TEST_DIR, { recursive: true });
+    }
+    if (existsSync(TEST_CACHE_DIR)) {
+        rmSync(TEST_CACHE_DIR, { recursive: true });
+    }
+    if (existsSync(TEST_STATE_PATH)) {
+        rmSync(TEST_STATE_PATH);
+    }
+}
+
+async function createTestPlugin(
+    id: string,
+    manifest: Partial<PluginManifest>,
+    processCode: string = 'ctx.kv.set("test/processed", "true");'
+): Promise<string> {
+    const pluginDir = path.join(TEST_DIR, id);
+    mkdirSync(pluginDir, { recursive: true });
+
+    // Write manifest
+    const fullManifest = {
+        id,
+        name: manifest.name || id,
+        version: manifest.version || '1.0.0',
+        ...manifest,
+    };
+
+    const manifestYaml = `id: ${fullManifest.id}
+name: ${fullManifest.name}
+version: ${fullManifest.version}
+${fullManifest.description ? `description: ${fullManifest.description}` : ''}
+${fullManifest.dependencies?.length ? `dependencies:\n${fullManifest.dependencies.map(d => `  - ${d}`).join('\n')}` : ''}
+`;
+
+    await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml);
+
+    // Write plugin code
+    const pluginCode = `
+export default {
+    async process(ctx) {
+        ${processCode}
+    }
+};
+`;
+    await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode);
+
+    return pluginDir;
+}
+
+// =============================================================================
+// KVStore Tests
+// =============================================================================
+
+describe('KVStore', () => {
+    it('should set and get values', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        expect(kv.get('file/name')).to.equal('test.mkv');
+    });
+
+    it('should return undefined for missing keys', () => {
+        const kv = new KVStore();
+        expect(kv.get('missing/key')).to.be.undefined;
+    });
+
+    it('should delete keys', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.delete('file/name');
+        expect(kv.get('file/name')).to.be.undefined;
+    });
+
+    it('should list all keys', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.set('video/codec', 'h264');
+        kv.set('audio/0/codec', 'aac');
+
+        const keys = kv.keys();
+        expect(keys).to.have.lengthOf(3);
+        expect(keys).to.include('file/name');
+        expect(keys).to.include('video/codec');
+        expect(keys).to.include('audio/0/codec');
+    });
+
+    it('should filter keys by prefix', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.set('audio/0/codec', 'aac');
+        kv.set('audio/0/language', 'eng');
+        kv.set('audio/1/codec', 'dts');
+
+        const audioKeys = kv.keys('audio/');
+        expect(audioKeys).to.have.lengthOf(3);
+        expect(audioKeys).to.include('audio/0/codec');
+        expect(audioKeys).to.include('audio/0/language');
+        expect(audioKeys).to.include('audio/1/codec');
+    });
+
+    it('should return entries', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.set('video/codec', 'h264');
+
+        const entries = kv.entries();
+        expect(entries).to.have.lengthOf(2);
+        expect(entries).to.deep.include(['file/name', 'test.mkv']);
+        expect(entries).to.deep.include(['video/codec', 'h264']);
+    });
+
+    it('should filter entries by prefix', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.set('audio/0/codec', 'aac');
+        kv.set('audio/1/codec', 'dts');
+
+        const audioEntries = kv.entries('audio/');
+        expect(audioEntries).to.have.lengthOf(2);
+    });
+
+    it('should initialize with data', () => {
+        const kv = new KVStore({ 'file/name': 'test.mkv', 'video/codec': 'h264' });
+        expect(kv.get('file/name')).to.equal('test.mkv');
+        expect(kv.get('video/codec')).to.equal('h264');
+    });
+
+    it('should convert to object', () => {
+        const kv = new KVStore();
+        kv.set('file/name', 'test.mkv');
+        kv.set('video/codec', 'h264');
+
+        const obj = kv.toObject();
+        expect(obj).to.deep.equal({
+            'file/name': 'test.mkv',
+            'video/codec': 'h264',
+        });
+    });
+
+    it('should ignore null/undefined values', () => {
+        const kv = new KVStore();
+        kv.set('test', null as any);
+        kv.set('test2', undefined as any);
+        expect(kv.keys()).to.have.lengthOf(0);
+    });
+
+    it('should ignore empty keys', () => {
+        const kv = new KVStore();
+        kv.set('', 'value');
+        expect(kv.keys()).to.have.lengthOf(0);
+    });
+});
+
+// =============================================================================
+// PluginCache Tests
+// =============================================================================
+
+describe('PluginCache', () => {
+    const cacheDir = path.join(TEST_CACHE_DIR, 'test-plugin');
+
+    beforeEach(async () => {
+        if (existsSync(cacheDir)) {
+            rmSync(cacheDir, { recursive: true });
+        }
+    });
+
+    afterEach(async () => {
+        if (existsSync(cacheDir)) {
+            rmSync(cacheDir, { recursive: true });
+        }
+    });
+
+    it('should write and read JSON', async () => {
+        const cache = new PluginCacheImpl(cacheDir);
+        const data = { foo: 'bar', num: 42 };
+
+        await cache.writeJson('test.json', data);
+        const result = await cache.readJson<typeof data>('test.json');
+
+        expect(result).to.deep.equal(data);
+    });
+
+    it('should return null for missing files', async () => {
+        const cache = new PluginCacheImpl(cacheDir);
+        const result = await cache.readJson('missing.json');
+        expect(result).to.be.null;
+    });
+
+    it('should check file existence', async () => {
+        const cache = new PluginCacheImpl(cacheDir);
+
+        expect(await cache.exists('test.json')).to.be.false;
+
+        await cache.writeJson('test.json', { foo: 'bar' });
+        expect(await cache.exists('test.json')).to.be.true;
+    });
+
+    it('should delete files', async () => {
+        const cache = new PluginCacheImpl(cacheDir);
+        await cache.writeJson('test.json', { foo: 'bar' });
+
+        await cache.delete('test.json');
+        expect(await cache.exists('test.json')).to.be.false;
+    });
+
+    it('should clear all files', async () => {
+        const cache = new PluginCacheImpl(cacheDir);
+        await cache.writeJson('test1.json', { a: 1 });
+        await cache.writeJson('test2.json', { b: 2 });
+
+        await cache.clear();
+
+        expect(await cache.exists('test1.json')).to.be.false;
+        expect(await cache.exists('test2.json')).to.be.false;
+    });
+
+    it('should get full path', () => {
+        const cache = new PluginCacheImpl(cacheDir);
+        const fullPath = cache.getPath('test.json');
+        expect(fullPath).to.equal(path.join(cacheDir, 'test.json'));
+    });
+});
+
+// =============================================================================
+// Manifest Parsing Tests
+// =============================================================================
+
+describe('Manifest Parsing', () => {
+    beforeEach(setupTestDir);
+    afterEach(cleanupTestDir);
+
+    it('should parse valid manifest', async () => {
+        const manifestPath = path.join(TEST_DIR, 'manifest.yml');
+        await fs.writeFile(
+            manifestPath,
+            `id: test-plugin
+name: Test Plugin
+version: 1.0.0
+description: A test plugin
+dependencies:
+  - other-plugin
+config:
+  apiKey:
+    type: string
+    required: true
+    secret: true
+  language:
+    type: string
+    default: en-US
+`
+        );
+
+        const manifest = await parseManifest(manifestPath);
+
+        expect(manifest.id).to.equal('test-plugin');
+        expect(manifest.name).to.equal('Test Plugin');
+        expect(manifest.version).to.equal('1.0.0');
+        expect(manifest.description).to.equal('A test plugin');
+        expect(manifest.dependencies).to.deep.equal(['other-plugin']);
+        expect(manifest.config?.apiKey?.type).to.equal('string');
+        expect(manifest.config?.apiKey?.secret).to.be.true;
+        expect(manifest.config?.language?.default).to.equal('en-US');
+    });
+
+    it('should validate manifest - valid', () => {
+        const manifest: PluginManifest = {
+            id: 'test-plugin',
+            name: 'Test Plugin',
+            version: '1.0.0',
+        };
+
+        const errors = validateManifest(manifest);
+        expect(errors).to.have.lengthOf(0);
+    });
+
+    it('should validate manifest - invalid id', () => {
+        const manifest: PluginManifest = {
+            id: 'Test_Plugin',
+            name: 'Test Plugin',
+            version: '1.0.0',
+        };
+
+        const errors = validateManifest(manifest);
+        expect(errors).to.have.lengthOf(1);
+        expect(errors[0]).to.include('Invalid plugin id');
+    });
+
+    it('should validate manifest - invalid version', () => {
+        const manifest: PluginManifest = {
+            id: 'test-plugin',
+            name: 'Test Plugin',
+            version: 'invalid',
+        };
+
+        const errors = validateManifest(manifest);
+        expect(errors).to.have.lengthOf(1);
+        expect(errors[0]).to.include('Invalid version');
+    });
+});
+
+// =============================================================================
+// Dependency Resolution Tests
+// =============================================================================
+
+describe('Dependency Resolution', () => {
+    it('should resolve simple dependency chain', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+            ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
+        ]);
+
+        const active = new Set(['a', 'b', 'c']);
+        const { order, errors } = resolveDependencyOrder(plugins, active);
+
+        expect(errors).to.have.lengthOf(0);
+        expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b'));
+        expect(order.indexOf('b')).to.be.lessThan(order.indexOf('c'));
+    });
+
+    it('should handle diamond dependency', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+            ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['a'] }],
+            ['d', { id: 'd', name: 'D', version: '1.0.0', dependencies: ['b', 'c'] }],
+        ]);
+
+        const active = new Set(['a', 'b', 'c', 'd']);
+        const { order, errors } = resolveDependencyOrder(plugins, active);
+
+        expect(errors).to.have.lengthOf(0);
+        expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b'));
+        expect(order.indexOf('a')).to.be.lessThan(order.indexOf('c'));
+        expect(order.indexOf('b')).to.be.lessThan(order.indexOf('d'));
+        expect(order.indexOf('c')).to.be.lessThan(order.indexOf('d'));
+    });
+
+    it('should detect circular dependencies', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['c'] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+            ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
+        ]);
+
+        const active = new Set(['a', 'b', 'c']);
+        const { errors } = resolveDependencyOrder(plugins, active);
+
+        expect(errors.length).to.be.greaterThan(0);
+        expect(errors[0]).to.include('Circular dependency');
+    });
+
+    it('should report missing dependencies', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['missing'] }],
+        ]);
+
+        const active = new Set(['a']);
+        const { errors } = resolveDependencyOrder(plugins, active);
+
+        expect(errors.length).to.be.greaterThan(0);
+        expect(errors[0]).to.include('does not exist');
+    });
+
+    it('should report inactive dependencies', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ]);
+
+        const active = new Set(['b']); // 'a' is not active
+        const { errors } = resolveDependencyOrder(plugins, active);
+
+        expect(errors.length).to.be.greaterThan(0);
+        expect(errors[0]).to.include('not active');
+    });
+});
+
+// =============================================================================
+// Activation/Deactivation Tests
+// =============================================================================
+
+describe('Activation Checks', () => {
+    it('should allow activation when dependencies are met', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ]);
+
+        const active = new Set(['a']);
+        const { canActivate: can, missingDeps } = canActivate('b', plugins, active);
+
+        expect(can).to.be.true;
+        expect(missingDeps).to.have.lengthOf(0);
+    });
+
+    it('should prevent activation when dependencies are missing', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ]);
+
+        const active = new Set<string>(); // 'a' is not active
+        const { canActivate: can, missingDeps } = canActivate('b', plugins, active);
+
+        expect(can).to.be.false;
+        expect(missingDeps).to.deep.equal(['a']);
+    });
+
+    it('should allow deactivation when no dependents', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ]);
+
+        const active = new Set(['a', 'b']);
+        const { canDeactivate: can, dependents } = canDeactivate('b', plugins, active);
+
+        expect(can).to.be.true;
+        expect(dependents).to.have.lengthOf(0);
+    });
+
+    it('should prevent deactivation when has dependents', () => {
+        const plugins = new Map<string, PluginManifest>([
+            ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+            ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ]);
+
+        const active = new Set(['a', 'b']);
+        const { canDeactivate: can, dependents } = canDeactivate('a', plugins, active);
+
+        expect(can).to.be.false;
+        expect(dependents).to.deep.equal(['b']);
+    });
+});
+
+// =============================================================================
+// Plugin Manager Integration Tests
+// =============================================================================
+
+describe('PluginManager', () => {
+    beforeEach(setupTestDir);
+    afterEach(async () => {
+        await cleanupTestDir();
+    });
+
+    it('should scan and discover plugins', async () => {
+        await createTestPlugin('test-plugin', {
+            name: 'Test Plugin',
+            version: '1.0.0',
+        });
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        const plugins = await manager.scanPlugins();
+
+        expect(plugins).to.have.lengthOf(1);
+        expect(plugins[0].id).to.equal('test-plugin');
+        expect(plugins[0].name).to.equal('Test Plugin');
+    });
+
+    it('should load and activate plugins', async () => {
+        await createTestPlugin('plugin-a', { name: 'Plugin A' });
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager.scanPlugins();
+        await manager.loadAll();
+
+        // Activate
+        await manager.activate('plugin-a');
+
+        const plugins = manager.getPlugins();
+        expect(plugins[0].active).to.be.true;
+
+        const order = manager.getExecutionOrder();
+        expect(order).to.include('plugin-a');
+    });
+
+    it('should respect dependency order in execution', async () => {
+        await createTestPlugin('plugin-a', { name: 'Plugin A' }, 'ctx.kv.set("order/a", Date.now().toString());');
+        await createTestPlugin(
+            'plugin-b',
+            { name: 'Plugin B', dependencies: ['plugin-a'] },
+            'ctx.kv.set("order/b", Date.now().toString());'
+        );
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager.scanPlugins();
+        await manager.loadAll();
+        await manager.activate('plugin-a');
+        await manager.activate('plugin-b');
+
+        const order = manager.getExecutionOrder();
+        expect(order.indexOf('plugin-a')).to.be.lessThan(order.indexOf('plugin-b'));
+    });
+
+    it('should process files through plugins', async () => {
+        await createTestPlugin(
+            'test-plugin',
+            { name: 'Test Plugin' },
+            `
+            ctx.kv.set('file/processed', 'true');
+            ctx.kv.set('file/path', ctx.filePath);
+            `
+        );
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager.scanPlugins();
+        await manager.loadAll();
+        await manager.activate('test-plugin');
+
+        const kv = manager.createKVStore();
+        const result = await manager.processFile('/test/file.mkv', kv);
+
+        expect(result.success).to.be.true;
+        expect(kv.get('file/processed')).to.equal('true');
+        expect(kv.get('file/path')).to.equal('/test/file.mkv');
+    });
+
+    it('should persist state across restarts', async () => {
+        await createTestPlugin('test-plugin', { name: 'Test Plugin' });
+
+        // First run
+        const manager1 = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager1.scanPlugins();
+        await manager1.loadAll();
+        await manager1.activate('test-plugin');
+        await manager1.updateConfig('test-plugin', { setting: 'value' });
+        await manager1.shutdown();
+
+        // Second run
+        const manager2 = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager2.scanPlugins();
+        await manager2.loadAll();
+
+        const plugins = manager2.getPlugins();
+        expect(plugins[0].active).to.be.true;
+        expect(plugins[0].config.setting).to.equal('value');
+
+        await manager2.shutdown();
+    });
+
+    it('should prevent deactivation of plugin with dependents', async () => {
+        await createTestPlugin('plugin-a', { name: 'Plugin A' });
+        await createTestPlugin('plugin-b', { name: 'Plugin B', dependencies: ['plugin-a'] });
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        await manager.scanPlugins();
+        await manager.loadAll();
+        await manager.activate('plugin-a');
+        await manager.activate('plugin-b');
+
+        try {
+            await manager.deactivate('plugin-a');
+            expect.fail('Should have thrown');
+        } catch (error) {
+            expect((error as Error).message).to.include('other plugins depend on it');
+        }
+
+        await manager.shutdown();
+    });
+
+    it('should emit events on activation/deactivation', async () => {
+        await createTestPlugin('test-plugin', { name: 'Test Plugin' });
+
+        const manager = new PluginManager({
+            pluginsDir: TEST_DIR,
+            statePath: TEST_STATE_PATH,
+            cacheDir: TEST_CACHE_DIR,
+        });
+
+        const events: string[] = [];
+        manager.on('plugin:activated', (e) => events.push(`activated:${e.pluginId}`));
+        manager.on('plugin:deactivated', (e) => events.push(`deactivated:${e.pluginId}`));
+
+        await manager.scanPlugins();
+        await manager.loadAll();
+        await manager.activate('test-plugin');
+        await manager.deactivate('test-plugin');
+
+        expect(events).to.deep.equal(['activated:test-plugin', 'deactivated:test-plugin']);
+
+        await manager.shutdown();
+    });
+});

packages/meta-sort-core/src/plugin-engine/types.ts

@@ -0,0 +1,429 @@
+/**
+ * Plugin System Type Definitions
+ *
+ * This module defines all types for the meta-sort plugin system.
+ */
+
+// =============================================================================
+// Plugin Manifest Types
+// =============================================================================
+
+/**
+ * Configuration field type in manifest
+ */
+export type ConfigFieldType = 'string' | 'number' | 'boolean' | 'select';
+
+/**
+ * Schema field type for editor UI
+ */
+export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
+
+/**
+ * Select option for config/schema fields
+ */
+export interface SelectOption {
+    value: string;
+    label: string;
+}
+
+/**
+ * Configuration field definition in manifest
+ */
+export interface ConfigField {
+    type: ConfigFieldType;
+    label?: string;
+    required?: boolean;
+    default?: unknown;
+    secret?: boolean;
+    options?: SelectOption[];
+}
+
+/**
+ * Metadata schema field definition (for editor UI)
+ */
+export interface SchemaField {
+    /** Display label */
+    label: string;
+    /** Field type for form control */
+    type?: SchemaFieldType;
+    /** Read-only field (not editable) */
+    readonly?: boolean;
+    /** Whether field is indexed (has {n} pattern) */
+    indexed?: boolean;
+    /** Options for select type */
+    options?: SelectOption[];
+    /** Hint text shown below field */
+    hint?: string;
+    /** URL template for clickable values (use {value} as placeholder) */
+    link?: string;
+    /** Pattern for JSON type to aggregate matching keys */
+    pattern?: string;
+    /** Minimum value for number type */
+    min?: number;
+    /** Maximum value for number type */
+    max?: number;
+    /** Placeholder text */
+    placeholder?: string;
+}
+
+/**
+ * Task queue type for plugin classification
+ */
+export type TaskQueueType = 'fast' | 'background';
+
+/**
+ * Plugin manifest (parsed from manifest.yml)
+ */
+export interface PluginManifest {
+    id: string;
+    name: string;
+    version: string;
+    description?: string;
+    /** Plugin author */
+    author?: string;
+    /** Display priority (lower = higher priority) */
+    priority?: number;
+    /** Color for UI display */
+    color?: string;
+    dependencies?: string[];
+    config?: Record<string, ConfigField>;
+    schema?: Record<string, SchemaField>;
+    /**
+     * Default queue for this plugin.
+     * - 'fast': High concurrency queue for quick operations (<1s)
+     * - 'background': Low concurrency queue for slow operations (>=1s)
+     * Default: 'fast'
+     */
+    defaultQueue?: TaskQueueType;
+}
+
+// =============================================================================
+// Plugin State Types
+// =============================================================================
+
+/**
+ * Plugin state entry (stored in plugins.json)
+ */
+export interface PluginStateEntry {
+    active: boolean;
+    config: Record<string, unknown>;
+}
+
+/**
+ * Complete plugin state file structure
+ */
+export interface PluginState {
+    plugins: Record<string, PluginStateEntry>;
+}
+
+/**
+ * Plugin status
+ */
+export type PluginStatus = 'loaded' | 'error' | 'unloaded';
+
+/**
+ * Plugin info (runtime representation)
+ */
+export interface PluginInfo {
+    id: string;
+    name: string;
+    version: string;
+    description?: string;
+    /** Display priority (lower = higher priority) */
+    priority?: number;
+    /** Color for UI display */
+    color?: string;
+    dependencies: string[];
+    active: boolean;
+    config: Record<string, unknown>;
+    configSchema?: Record<string, ConfigField>;
+    metadataSchema?: Record<string, SchemaField>;
+    status: PluginStatus;
+    error?: string;
+    /** True if this is a containerized plugin */
+    isContainer?: boolean;
+}
+
+// =============================================================================
+// Plugin Context Types (injected into plugins)
+// =============================================================================
+
+/**
+ * Key-value store for file metadata
+ */
+export interface PluginKVStore {
+    /**
+     * Get a value by key
+     * @param key Path-style key, e.g., "video/codec" or "audio/0/language"
+     */
+    get(key: string): string | undefined;
+
+    /**
+     * Set a value. Only string values are supported.
+     * @param key Path-style key
+     * @param value String value
+     */
+    set(key: string, value: string): void;
+
+    /**
+     * Delete a key
+     */
+    delete(key: string): void;
+
+    /**
+     * Get all keys, optionally filtered by prefix
+     * @param prefix Optional prefix filter, e.g., "audio/"
+     */
+    keys(prefix?: string): string[];
+
+    /**
+     * Get all key-value pairs, optionally filtered by prefix
+     */
+    entries(prefix?: string): Array<[string, string]>;
+
+    /**
+     * Add a value to a set (for RecordSet fields like genres, tags).
+     * Optional - only available with MetadataNodeKVStore.
+     */
+    add?(key: string, value: string): void;
+
+    /**
+     * Get the full object at a path (for inspecting nested structures).
+     * Optional - only available with MetadataNodeKVStore.
+     */
+    getObject?(key: string): any;
+}
+
+/**
+ * Cache utilities for persistent plugin storage
+ */
+export interface PluginCache {
+    /**
+     * Get full path for a cache file
+     * @param filename Relative filename within cache directory
+     */
+    getPath(filename: string): string;
+
+    /**
+     * Read and parse a JSON cache file
+     * @returns Parsed data or null if not found
+     */
+    readJson<T>(filename: string): Promise<T | null>;
+
+    /**
+     * Write data as JSON to cache file
+     */
+    writeJson(filename: string, data: unknown): Promise<void>;
+
+    /**
+     * Check if a cache file exists
+     */
+    exists(filename: string): Promise<boolean>;
+
+    /**
+     * Delete a cache file
+     */
+    delete(filename: string): Promise<void>;
+
+    /**
+     * Clear all files in the plugin's cache directory
+     */
+    clear(): Promise<void>;
+}
+
+/**
+ * Logger interface for plugins
+ */
+export interface PluginLogger {
+    debug(message: string, ...args: unknown[]): void;
+    info(message: string, ...args: unknown[]): void;
+    warn(message: string, ...args: unknown[]): void;
+    error(message: string, ...args: unknown[]): void;
+}
+
+/**
+ * Context passed to plugin lifecycle hooks
+ */
+export interface PluginLifecycleContext {
+    /** Plugin configuration from plugins.json */
+    config: Record<string, unknown>;
+    /** Absolute path to plugin's cache directory */
+    cacheDir: string;
+    /** Scoped logger */
+    log: PluginLogger;
+}
+
+/**
+ * Context passed to plugin process function
+ */
+export interface PluginContext extends PluginLifecycleContext {
+    /** Key-value storage for the current file's metadata */
+    kv: PluginKVStore;
+    /** Absolute path to the file being processed */
+    filePath: string;
+    /** Cache utilities for persistent storage */
+    cache: PluginCache;
+    /**
+     * Direct access to MetadataNode for plugins that need it.
+     * Use this for compatibility with existing libraries that expect MetadataNode.
+     * Only available when using MetadataNodeKVStore.
+     */
+    metadata?: any;
+}
+
+// =============================================================================
+// Plugin Interface
+// =============================================================================
+
+/**
+ * Plugin interface that all plugins must implement
+ */
+export interface Plugin {
+    /**
+     * Called once when the plugin is loaded at startup.
+     * Use for one-time initialization.
+     */
+    onLoad?(ctx: PluginLifecycleContext): Promise<void>;
+
+    /**
+     * Called when the plugin is activated.
+     * Triggered at startup (if previously active) or from dashboard.
+     */
+    onActivate?(ctx: PluginLifecycleContext): Promise<void>;
+
+    /**
+     * Called when the plugin is deactivated from the dashboard.
+     * Use for cleanup, releasing resources.
+     */
+    onDeactivate?(ctx: PluginLifecycleContext): Promise<void>;
+
+    /**
+     * Called when the plugin is unloaded at shutdown.
+     */
+    onUnload?(ctx: PluginLifecycleContext): Promise<void>;
+
+    /**
+     * Process a single file. Called for each file in the library.
+     * Must be implemented by all plugins.
+     */
+    process(ctx: PluginContext): Promise<void>;
+}
+
+// =============================================================================
+// Plugin Manager Types
+// =============================================================================
+
+/**
+ * Loaded plugin internal representation
+ */
+export interface LoadedPlugin {
+    manifest: PluginManifest;
+    instance: Plugin;
+    status: PluginStatus;
+    error?: string;
+    pluginDir: string;
+}
+
+/**
+ * Processing result for a single file
+ */
+export interface ProcessingResult {
+    success: boolean;
+    timings: Record<string, number>;
+    errors: Array<{ pluginId: string; error: string }>;
+}
+
+/**
+ * Plugin manager events
+ */
+export type PluginManagerEvent =
+    | { type: 'plugin:activated'; pluginId: string }
+    | { type: 'plugin:deactivated'; pluginId: string }
+    | { type: 'plugin:config-changed'; pluginId: string; config: Record<string, unknown> }
+    | { type: 'plugin:error'; pluginId: string; error: Error };
+
+// =============================================================================
+// Plugin Task Types (for TaskScheduler)
+// =============================================================================
+
+/**
+ * Status of a plugin task
+ */
+export type PluginTaskStatus = 'pending' | 'ready' | 'running' | 'completed' | 'failed';
+
+/**
+ * Result of a plugin task execution
+ */
+export interface PluginTaskResult {
+    success: boolean;
+    timeMs: number;
+    /** KV updates to apply (key-value pairs) */
+    kvUpdates?: Record<string, string>;
+    error?: string;
+}
+
+/**
+ * Plugin task representing a single plugin execution for a file
+ */
+export interface PluginTask {
+    /** Unique task ID: `${fileHash}:${pluginId}` */
+    id: string;
+    /** File identity hash (midhash256) */
+    fileHash: string;
+    /** Absolute path to the file */
+    filePath: string;
+    /** Plugin to execute */
+    pluginId: string;
+    /** Plugin IDs this task depends on */
+    dependencies: string[];
+    /** Target queue (from classification) */
+    queue: TaskQueueType;
+    /** Priority based on avg execution time (lower = higher priority) */
+    priority: number;
+    /** Estimated execution time in ms (from performanceMetrics) */
+    estimatedTimeMs: number;
+    /** Current task status */
+    status: PluginTaskStatus;
+    /** Timestamp when task was created */
+    createdAt: number;
+    /** Timestamp when task started executing */
+    startedAt?: number;
+    /** Timestamp when task completed */
+    completedAt?: number;
+    /** Execution result */
+    result?: PluginTaskResult;
+    /** Runtime options merged into plugin config (e.g., forceRecompute) */
+    options?: Record<string, unknown>;
+}
+
+/**
+ * Queue status information
+ */
+export interface QueueStatus {
+    /** Number of tasks pending (waiting for dependencies) */
+    pending: number;
+    /** Number of tasks ready (in queue, waiting for worker) */
+    ready: number;
+    /** Number of tasks currently running */
+    running: number;
+    /** Number of tasks completed */
+    completed: number;
+    /** Number of tasks failed */
+    failed: number;
+    /** Whether the queue is currently paused */
+    isPaused: boolean;
+}
+
+/**
+ * Task scheduler events
+ */
+export type TaskSchedulerEvent =
+    | { type: 'task:created'; task: PluginTask }
+    | { type: 'task:ready'; task: PluginTask }
+    | { type: 'task:started'; task: PluginTask }
+    | { type: 'task:complete'; task: PluginTask }
+    | { type: 'task:failed'; task: PluginTask; error: string }
+    | { type: 'file:complete'; fileHash: string; filePath: string }
+    | { type: 'queue:fast:idle' }
+    | { type: 'queue:background:started' }
+    | { type: 'queue:background:paused' };

packages/meta-sort-core/src/plugin-engine/validate.js

@@ -0,0 +1,210 @@
+/**
+ * Plugin System Validation Script
+ *
+ * Simple validation to ensure the plugin system works correctly.
+ * Run with: node dist/plugin2/validate.js
+ */
+
+import * as fs from 'fs/promises';
+import * as path from 'path';
+import { existsSync, mkdirSync, rmSync } from 'fs';
+import { fileURLToPath } from 'url';
+
+import {
+    KVStore,
+    PluginCacheImpl,
+    resolveDependencyOrder,
+    canActivate,
+    canDeactivate,
+    PluginManager,
+} from './index.js';
+
+const __dirname = path.dirname(fileURLToPath(import.meta.url));
+const TEST_DIR = path.join(__dirname, '../../test-plugins-validation');
+const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache-validation');
+const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins-validation.json');
+
+let passed = 0;
+let failed = 0;
+
+function assert(condition, message) {
+    if (condition) {
+        passed++;
+        console.log(`  ✓ ${message}`);
+    } else {
+        failed++;
+        console.log(`  ✗ ${message}`);
+    }
+}
+
+async function cleanup() {
+    if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true });
+    if (existsSync(TEST_CACHE_DIR)) rmSync(TEST_CACHE_DIR, { recursive: true });
+    if (existsSync(TEST_STATE_PATH)) rmSync(TEST_STATE_PATH);
+}
+
+async function setup() {
+    await cleanup();
+    mkdirSync(TEST_DIR, { recursive: true });
+    mkdirSync(TEST_CACHE_DIR, { recursive: true });
+}
+
+async function createTestPlugin(id, deps = []) {
+    const pluginDir = path.join(TEST_DIR, id);
+    mkdirSync(pluginDir, { recursive: true });
+
+    const manifestYaml = `id: ${id}
+name: ${id}
+version: 1.0.0
+${deps.length ? `dependencies:\n${deps.map(d => `  - ${d}`).join('\n')}` : ''}
+`;
+    await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml);
+
+    const pluginCode = `
+export default {
+    async process(ctx) {
+        ctx.kv.set("${id}/processed", "true");
+    }
+};
+`;
+    await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode);
+}
+
+async function testKVStore() {
+    console.log('\nTesting KVStore...');
+
+    const kv = new KVStore();
+    kv.set('file/name', 'test.mkv');
+    assert(kv.get('file/name') === 'test.mkv', 'set and get value');
+
+    assert(kv.get('missing/key') === undefined, 'missing key returns undefined');
+
+    kv.set('audio/0/codec', 'aac');
+    kv.set('audio/0/language', 'eng');
+    kv.set('audio/1/codec', 'dts');
+    const audioKeys = kv.keys('audio/');
+    assert(audioKeys.length === 3, 'keys with prefix filters correctly');
+
+    kv.delete('file/name');
+    assert(kv.get('file/name') === undefined, 'delete removes key');
+
+    const obj = kv.toObject();
+    assert(typeof obj === 'object', 'toObject returns object');
+}
+
+async function testPluginCache() {
+    console.log('\nTesting PluginCache...');
+
+    const cacheDir = path.join(TEST_CACHE_DIR, 'test-cache');
+    const cache = new PluginCacheImpl(cacheDir);
+
+    await cache.writeJson('test.json', { foo: 'bar' });
+    const data = await cache.readJson('test.json');
+    assert(data?.foo === 'bar', 'writeJson and readJson work');
+
+    const exists = await cache.exists('test.json');
+    assert(exists === true, 'exists returns true for existing file');
+
+    const notExists = await cache.exists('missing.json');
+    assert(notExists === false, 'exists returns false for missing file');
+
+    await cache.delete('test.json');
+    const afterDelete = await cache.exists('test.json');
+    assert(afterDelete === false, 'delete removes file');
+}
+
+async function testDependencyResolution() {
+    console.log('\nTesting Dependency Resolution...');
+
+    const plugins = new Map([
+        ['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
+        ['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
+        ['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
+    ]);
+
+    const active = new Set(['a', 'b', 'c']);
+    const { order, errors } = resolveDependencyOrder(plugins, active);
+
+    assert(errors.length === 0, 'no errors for valid dependency chain');
+    assert(order.indexOf('a') < order.indexOf('b'), 'a comes before b');
+    assert(order.indexOf('b') < order.indexOf('c'), 'b comes before c');
+
+    // Test can activate
+    const { canActivate: can, missingDeps } = canActivate('b', plugins, new Set(['a']));
+    assert(can === true, 'can activate when dependencies met');
+
+    const { canActivate: cannot, missingDeps: missing } = canActivate('b', plugins, new Set());
+    assert(cannot === false, 'cannot activate when dependencies missing');
+    assert(missing.includes('a'), 'reports missing dependency');
+
+    // Test can deactivate
+    const { canDeactivate: canDeact } = canDeactivate('c', plugins, active);
+    assert(canDeact === true, 'can deactivate when no dependents');
+
+    const { canDeactivate: cannotDeact, dependents } = canDeactivate('a', plugins, active);
+    assert(cannotDeact === false, 'cannot deactivate when has dependents');
+    assert(dependents.includes('b'), 'reports dependent plugin');
+}
+
+async function testPluginManager() {
+    console.log('\nTesting PluginManager...');
+
+    await createTestPlugin('plugin-a');
+    await createTestPlugin('plugin-b', ['plugin-a']);
+
+    const manager = new PluginManager({
+        pluginsDir: TEST_DIR,
+        statePath: TEST_STATE_PATH,
+        cacheDir: TEST_CACHE_DIR,
+    });
+
+    const plugins = await manager.scanPlugins();
+    assert(plugins.length === 2, 'scans and discovers plugins');
+
+    await manager.loadAll();
+    assert(manager.getPlugins().length === 2, 'loads all plugins');
+
+    await manager.activate('plugin-a');
+    await manager.activate('plugin-b');
+    assert(manager.getActivePlugins().length === 2, 'activates plugins');
+
+    const order = manager.getExecutionOrder();
+    assert(order.indexOf('plugin-a') < order.indexOf('plugin-b'), 'respects dependency order');
+
+    const kv = manager.createKVStore();
+    const result = await manager.processFile('/test/file.mkv', kv);
+    assert(result.success === true, 'processes file successfully');
+    assert(kv.get('plugin-a/processed') === 'true', 'plugin-a executed');
+    assert(kv.get('plugin-b/processed') === 'true', 'plugin-b executed');
+
+    await manager.deactivate('plugin-b');
+    assert(manager.getActivePlugins().length === 1, 'deactivates plugin');
+
+    await manager.shutdown();
+}
+
+async function run() {
+    console.log('Plugin System Validation\n========================');
+
+    try {
+        await setup();
+        await testKVStore();
+        await testPluginCache();
+        await testDependencyResolution();
+        await testPluginManager();
+        await cleanup();
+
+        console.log(`\n========================`);
+        console.log(`Results: ${passed} passed, ${failed} failed`);
+
+        if (failed > 0) {
+            process.exit(1);
+        }
+    } catch (error) {
+        console.error('\nValidation error:', error);
+        await cleanup();
+        process.exit(1);
+    }
+}
+
+run();

packages/meta-sort-core/src/types/FileMetadata.ts

@@ -0,0 +1,21 @@
+/**
+ * FileMetadata - Extended metadata for file processing
+ *
+ * Wraps HashMeta with application-specific fields for tracking
+ * file processing state.
+ */
+
+import { HashMeta } from '@metazla/meta-interface';
+
+export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
+
+/**
+ * Extended file metadata that includes processing state
+ * Extends the generic HashMeta with meta-mesh specific fields
+ *
+ * Note: No longer uses tempId - midhash256 is computed immediately and serves as permanent ID
+ */
+export interface FileMetadata extends HashMeta {
+  /** Current status of file processing */
+  processingStatus?: ProcessingStatus;
+}

packages/meta-sort-core/src/utils/TempIdGenerator.ts

@@ -0,0 +1,29 @@
+/**
+ * TempIdGenerator
+ *
+ * Generates temporary IDs for files before content hash is computed.
+ * Uses a deterministic approach based on file path, size, and mtime.
+ */
+
+import { createHash } from 'crypto';
+import { Stats } from 'fs';
+
+/**
+ * Generate a temporary ID for a file based on its path and stats
+ * This is deterministic - same file will always get same temp ID
+ *
+ * @param filePath - Absolute path to the file
+ * @param stats - File stats from fs.stat()
+ * @returns Temporary ID as MD5 hash (without prefix - prefix added by storage layer)
+ */
+export function generateTempId(filePath: string, stats: Stats): string {
+  // Create deterministic input from file metadata (NOT content)
+  const input = `${filePath}:${stats.size}:${stats.mtimeMs}`;
+
+  // Fast MD5 hash of the input
+  const hash = createHash('md5').update(input).digest('hex');
+
+  // Return hash without prefix - storage layer adds "tmp:" prefix
+  return hash;
+}
+

packages/meta-sort-core/src/utils/TimeoutWrapper.ts

@@ -0,0 +1,76 @@
+/**
+ * Timeout Wrapper Utility
+ *
+ * Provides timeout functionality for async operations with configurable retry logic
+ */
+
+export interface TimeoutConfig {
+  /** Base timeout in milliseconds (default: 30000ms = 30s) */
+  baseTimeout: number;
+  /** Timeout multiplier for each retry (default: 1.5x) */
+  timeoutMultiplier: number;
+  /** Maximum timeout in milliseconds (default: 600000ms = 10min) */
+  maxTimeout: number;
+}
+
+const DEFAULT_CONFIG: TimeoutConfig = {
+  baseTimeout: 30000, // 30 seconds
+  timeoutMultiplier: 1.5,
+  maxTimeout: 600000 // 10 minutes
+};
+
+/**
+ * Calculate timeout for a specific retry attempt
+ */
+export function calculateTimeout(retryCount: number, config: TimeoutConfig = DEFAULT_CONFIG): number {
+  const timeout = config.baseTimeout * Math.pow(config.timeoutMultiplier, retryCount);
+  return Math.min(timeout, config.maxTimeout);
+}
+
+/**
+ * Execute an async function with timeout
+ * @param fn - Function to execute
+ * @param timeoutMs - Timeout in milliseconds
+ * @param label - Label for error messages
+ * @returns Promise that rejects if timeout is exceeded
+ */
+export async function withTimeout<T>(
+  fn: () => Promise<T>,
+  timeoutMs: number,
+  label: string = 'Operation'
+): Promise<T> {
+  let timeoutHandle: NodeJS.Timeout;
+
+  const timeoutPromise = new Promise<never>((_, reject) => {
+    timeoutHandle = setTimeout(() => {
+      reject(new Error(`${label} timed out after ${timeoutMs}ms`));
+    }, timeoutMs);
+  });
+
+  try {
+    const result = await Promise.race([fn(), timeoutPromise]);
+    clearTimeout(timeoutHandle!);
+    return result;
+  } catch (error) {
+    clearTimeout(timeoutHandle!);
+    throw error;
+  }
+}
+
+/**
+ * Execute an async function with timeout and retry logic
+ * @param fn - Function to execute
+ * @param retryCount - Current retry count (0-based)
+ * @param config - Timeout configuration
+ * @param label - Label for error messages
+ * @returns Promise that resolves with result or rejects with timeout error
+ */
+export async function withTimeoutAndRetry<T>(
+  fn: () => Promise<T>,
+  retryCount: number,
+  config: TimeoutConfig = DEFAULT_CONFIG,
+  label: string = 'Operation'
+): Promise<T> {
+  const timeout = calculateTimeout(retryCount, config);
+  return withTimeout(fn, timeout, label);
+}

packages/meta-sort-core/tsconfig.json

@@ -0,0 +1,16 @@
+{
+  "compilerOptions": {
+    "target": "ESNext",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "resolveJsonModule": true,
+    "sourceMap": true,
+    "outDir": "./dist",
+    "esModuleInterop": true,
+    "forceConsistentCasingInFileNames": true,
+    "strict": false,
+    "skipLibCheck": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "**/*.spec.ts", "**/*.test.ts"]
+}