initial commit of meta-dup

README.md

@@ -0,0 +1,134 @@
+# meta-dup
+
+Standalone duplicate detection service for MetaMesh v2. Detects duplicate files by:
+- **Hash (SHA-256)**: Files with identical content
+- **Title**: Files with the same parsed title (from filename patterns)
+
+## Architecture
+
+meta-dup is a read-only service that:
+1. Subscribes to Redis Streams (`meta-sort:events`) for real-time updates
+2. Maintains an in-memory index of duplicates
+3. Provides REST API and web dashboard
+
+```
+meta-sort ──► Redis Streams ──► meta-dup ──► Dashboard (port 8183)
+              (events)          (consumer)
+```
+
+## API Endpoints
+
+| Endpoint | Method | Description |
+|----------|--------|-------------|
+| `/health` | GET | Health check with stats |
+| `/api/duplicates` | GET | All duplicates (hash + title) |
+| `/api/duplicates/hash` | GET | Hash duplicates only |
+| `/api/duplicates/title` | GET | Title duplicates only |
+| `/api/duplicates/stats` | GET | Duplicate statistics |
+| `/api/duplicates/rebuild` | POST | Force rebuild from Redis |
+
+## Response Format
+
+```json
+{
+  "hashDuplicates": [
+    {
+      "key": "sha256-hash",
+      "files": [
+        { "hashId": "...", "filePath": "/files/...", "sizeByte": 123456 }
+      ]
+    }
+  ],
+  "titleDuplicates": [...],
+  "stats": {
+    "hashGroupCount": 5,
+    "titleGroupCount": 3,
+    "totalFilesTracked": 100
+  },
+  "computedAt": 1234567890
+}
+```
+
+## Development
+
+### Build
+```bash
+cd packages/meta-dup
+pnpm install
+pnpm build
+```
+
+### Run (in Docker dev environment)
+```bash
+cd dev
+./start.sh  # Starts all services including meta-dup
+```
+
+### Rebuild
+```bash
+cd dev
+./reload-meta-dup.sh
+```
+
+### Tests
+```bash
+docker exec meta-test-runner /app/test/test.sh dup
+```
+
+## Configuration
+
+meta-dup uses service discovery via `/meta-core/locks/kv-leader.info` to find the Redis instance. No additional configuration required.
+
+### Environment Variables
+
+| Variable | Default | Description |
+|----------|---------|-------------|
+| `BASE_URL` | `http://localhost:8183` | External URL for service |
+| `SERVICE_NAME` | `meta-dup` | Service identifier |
+| `API_PORT` | `3000` | Internal API port (nginx proxies to 80) |
+
+## Package Structure
+
+```
+packages/meta-dup/
+├── package.json            # Workspace root
+├── pnpm-workspace.yaml
+└── packages/
+    ├── meta-dup-core/      # Backend service
+    │   ├── src/
+    │   │   ├── index.ts           # Entry point
+    │   │   ├── DuplicateIndex.ts  # Core duplicate detection
+    │   │   ├── api/
+    │   │   │   └── APIServer.ts   # Fastify REST API
+    │   │   └── kv/                # Redis client & leader discovery
+    │   └── package.json
+    └── meta-dup-ui/        # React dashboard
+        ├── src/
+        │   ├── App.tsx
+        │   └── main.tsx
+        └── package.json
+```
+
+## How Duplicate Detection Works
+
+### Hash Duplicates
+Files with the same `cid_sha2-256` (full file hash) are exact content duplicates.
+
+### Title Duplicates
+Files parsed to the same title (e.g., "Movie Name (2020)") from their filename patterns. Useful for finding different quality versions of the same content.
+
+## Redis Streams Integration
+
+meta-dup subscribes to `meta-sort:events` stream with consumer group `meta-dup-consumer`:
+
+```
+Event types:
+- batch: File batch processed (add/update duplicates)
+- reset: Full index rebuild required
+```
+
+On startup, meta-dup:
+1. Creates/joins consumer group
+2. Processes any pending entries
+3. Rebuilds full index from Redis `file:*` keys
+4. Starts listening for new events

package.json

@@ -0,0 +1,15 @@
+{
+  "name": "meta-dup",
+  "version": "1.0.0",
+  "private": true,
+  "description": "Standalone duplicate detection service for MetaMesh",
+  "scripts": {
+    "build": "pnpm -r build",
+    "dev": "pnpm -r --parallel dev",
+    "start": "pnpm --filter @meta-dup/core start",
+    "start:ui": "pnpm --filter @meta-dup/ui dev",
+    "test": "pnpm --filter @meta-dup/core test",
+    "lint": "pnpm -r lint"
+  },
+  "license": "MIT"
+}

packages/meta-dup-core/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "@meta-dup/core",
+  "version": "1.0.0",
+  "description": "Core duplicate detection service for meta-dup",
+  "main": "dist/index.js",
+  "type": "module",
+  "scripts": {
+    "build": "tsc",
+    "dev": "tsc --watch",
+    "start": "node dist/index.js",
+    "test": "vitest run",
+    "lint": "eslint src --ext ts"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "@fastify/cors": "^11.1.0",
+    "@fastify/static": "^8.0.0",
+    "dotenv": "^16.4.1",
+    "fastify": "^5.6.1",
+    "ioredis": "^5.4.1",
+    "tslog": "^4.9.2"
+  },
+  "devDependencies": {
+    "@types/node": "^20.11.16",
+    "typescript": "^5.3.3",
+    "vitest": "^1.0.0"
+  }
+}

packages/meta-dup-core/src/DuplicateIndex.ts

@@ -0,0 +1,334 @@
+/**
+ * DuplicateIndex - Core duplicate detection logic
+ *
+ * Maintains in-memory indexes for:
+ * - Hash duplicates: files with the same SHA-256 hash (exact copies)
+ * - Title duplicates: files with the same parsed title (different releases)
+ *
+ * Updates in real-time via Redis Streams events from meta-sort.
+ */
+
+import { Logger } from 'tslog';
+import type { FileMetadata, RedisClient } from './kv/RedisClient.js';
+
+const logger = new Logger({ name: 'DuplicateIndex' });
+
+/**
+ * Information about a file in a duplicate group
+ */
+export interface FileInfo {
+    hashId: string;
+    filePath: string;
+    title?: string;
+    sha256?: string;
+    sizeByte?: number;
+    year?: number;
+    season?: number;
+    episode?: number;
+    fileType?: string;
+}
+
+/**
+ * A group of duplicate files sharing the same key (hash or title)
+ */
+export interface DuplicateGroup {
+    key: string;           // hash or title
+    files: FileInfo[];     // Array of file info
+}
+
+/**
+ * Statistics about duplicate detection
+ */
+export interface DuplicateStats {
+    hashGroupCount: number;
+    hashFileCount: number;
+    titleGroupCount: number;
+    titleFileCount: number;
+    totalFilesTracked: number;
+    lastUpdated: string;
+}
+
+/**
+ * Full duplicate data response
+ */
+export interface DuplicateData {
+    hashDuplicates: DuplicateGroup[];
+    titleDuplicates: DuplicateGroup[];
+    stats: DuplicateStats;
+    computedAt: string;
+}
+
+export class DuplicateIndex {
+    // Primary indexes: key -> Set of hashIds
+    private byHash: Map<string, Set<string>> = new Map();
+    private byTitle: Map<string, Set<string>> = new Map();
+
+    // File info storage: hashId -> FileInfo
+    private fileInfo: Map<string, FileInfo> = new Map();
+
+    // Track when last updated
+    private lastUpdated: Date = new Date();
+
+    constructor() {
+        logger.info('DuplicateIndex initialized');
+    }
+
+    /**
+     * Add or update a file in the index
+     */
+    addFile(hashId: string, metadata: FileMetadata): void {
+        // First remove any existing entry to ensure clean state
+        this.removeFile(hashId);
+
+        // Extract relevant info
+        const info: FileInfo = {
+            hashId,
+            filePath: metadata.originalPath || metadata.sourcePath,
+            title: this.extractTitle(metadata),
+            sha256: metadata.sha256,
+            sizeByte: metadata.size,
+            year: metadata.year || metadata.movieYear,
+            season: metadata.season,
+            episode: metadata.episode,
+            fileType: metadata.fileType,
+        };
+
+        // Store file info
+        this.fileInfo.set(hashId, info);
+
+        // Index by SHA-256 hash (if available)
+        if (info.sha256) {
+            const hashSet = this.byHash.get(info.sha256) ?? new Set();
+            hashSet.add(hashId);
+            this.byHash.set(info.sha256, hashSet);
+        }
+
+        // Index by title (if available)
+        const titleKey = this.getTitleKey(metadata);
+        if (titleKey) {
+            const titleSet = this.byTitle.get(titleKey) ?? new Set();
+            titleSet.add(hashId);
+            this.byTitle.set(titleKey, titleSet);
+        }
+
+        this.lastUpdated = new Date();
+    }
+
+    /**
+     * Remove a file from the index
+     */
+    removeFile(hashId: string): void {
+        const info = this.fileInfo.get(hashId);
+        if (!info) return;
+
+        // Remove from hash index
+        if (info.sha256) {
+            const hashSet = this.byHash.get(info.sha256);
+            if (hashSet) {
+                hashSet.delete(hashId);
+                if (hashSet.size === 0) {
+                    this.byHash.delete(info.sha256);
+                }
+            }
+        }
+
+        // Remove from title index
+        if (info.title) {
+            const titleKey = info.title.toLowerCase();
+            const titleSet = this.byTitle.get(titleKey);
+            if (titleSet) {
+                titleSet.delete(hashId);
+                if (titleSet.size === 0) {
+                    this.byTitle.delete(titleKey);
+                }
+            }
+        }
+
+        // Remove file info
+        this.fileInfo.delete(hashId);
+        this.lastUpdated = new Date();
+    }
+
+    /**
+     * Extract title from metadata for duplicate detection
+     */
+    private extractTitle(metadata: FileMetadata): string | undefined {
+        // Prefer parsed title over filename
+        if (metadata.title) {
+            return metadata.title;
+        }
+        if (metadata.titles?.eng) {
+            return metadata.titles.eng;
+        }
+        if (metadata.originalTitle) {
+            return metadata.originalTitle;
+        }
+        return undefined;
+    }
+
+    /**
+     * Get normalized title key for indexing
+     * Includes year, season, episode for more precise matching
+     */
+    private getTitleKey(metadata: FileMetadata): string | null {
+        const title = this.extractTitle(metadata);
+        if (!title) return null;
+
+        // Build a normalized key
+        let key = title.toLowerCase().trim();
+
+        // Add year if available (for movies)
+        if (metadata.year || metadata.movieYear) {
+            key += `:${metadata.year || metadata.movieYear}`;
+        }
+
+        // Add season/episode if available (for TV shows)
+        if (metadata.season !== undefined) {
+            key += `:s${metadata.season}`;
+        }
+        if (metadata.episode !== undefined) {
+            key += `:e${metadata.episode}`;
+        }
+
+        return key;
+    }
+
+    /**
+     * Get all hash-based duplicates (files with same SHA-256)
+     */
+    getHashDuplicates(): DuplicateGroup[] {
+        const groups: DuplicateGroup[] = [];
+
+        for (const [hash, hashIds] of this.byHash) {
+            if (hashIds.size > 1) {
+                const files = this.getFileInfos(hashIds);
+                groups.push({
+                    key: hash,
+                    files: files.sort((a, b) => a.filePath.localeCompare(b.filePath)),
+                });
+            }
+        }
+
+        // Sort by number of duplicates (descending)
+        return groups.sort((a, b) => b.files.length - a.files.length);
+    }
+
+    /**
+     * Get all title-based duplicates (files with same parsed title)
+     */
+    getTitleDuplicates(): DuplicateGroup[] {
+        const groups: DuplicateGroup[] = [];
+
+        for (const [title, hashIds] of this.byTitle) {
+            if (hashIds.size > 1) {
+                const files = this.getFileInfos(hashIds);
+                groups.push({
+                    key: title,
+                    files: files.sort((a, b) => a.filePath.localeCompare(b.filePath)),
+                });
+            }
+        }
+
+        // Sort by number of duplicates (descending)
+        return groups.sort((a, b) => b.files.length - a.files.length);
+    }
+
+    /**
+     * Get FileInfo for a set of hashIds
+     */
+    private getFileInfos(hashIds: Set<string>): FileInfo[] {
+        const files: FileInfo[] = [];
+        for (const hashId of hashIds) {
+            const info = this.fileInfo.get(hashId);
+            if (info) {
+                files.push(info);
+            }
+        }
+        return files;
+    }
+
+    /**
+     * Get statistics about duplicates
+     */
+    getStats(): DuplicateStats {
+        const hashDuplicates = this.getHashDuplicates();
+        const titleDuplicates = this.getTitleDuplicates();
+
+        return {
+            hashGroupCount: hashDuplicates.length,
+            hashFileCount: hashDuplicates.reduce((sum, g) => sum + g.files.length, 0),
+            titleGroupCount: titleDuplicates.length,
+            titleFileCount: titleDuplicates.reduce((sum, g) => sum + g.files.length, 0),
+            totalFilesTracked: this.fileInfo.size,
+            lastUpdated: this.lastUpdated.toISOString(),
+        };
+    }
+
+    /**
+     * Get full duplicate data
+     */
+    getDuplicateData(): DuplicateData {
+        return {
+            hashDuplicates: this.getHashDuplicates(),
+            titleDuplicates: this.getTitleDuplicates(),
+            stats: this.getStats(),
+            computedAt: new Date().toISOString(),
+        };
+    }
+
+    /**
+     * Clear all indexes
+     */
+    clear(): void {
+        this.byHash.clear();
+        this.byTitle.clear();
+        this.fileInfo.clear();
+        this.lastUpdated = new Date();
+        logger.info('DuplicateIndex cleared');
+    }
+
+    /**
+     * Rebuild index from all files in Redis
+     */
+    async rebuildFromRedis(redisClient: RedisClient): Promise<void> {
+        logger.info('Rebuilding DuplicateIndex from Redis...');
+
+        // Clear existing data
+        this.clear();
+
+        // Load all files from Redis
+        const files = await redisClient.getAllFiles();
+
+        logger.info(`Loading ${files.size} files into DuplicateIndex...`);
+
+        // Add each file to the index
+        for (const [hashId, metadata] of files) {
+            this.addFile(hashId, metadata);
+        }
+
+        const stats = this.getStats();
+        logger.info(`DuplicateIndex rebuilt: ${stats.totalFilesTracked} files, ` +
+            `${stats.hashGroupCount} hash duplicates, ${stats.titleGroupCount} title duplicates`);
+    }
+
+    /**
+     * Handle file update from Redis
+     */
+    async onFileUpdate(
+        hashId: string,
+        action: 'add' | 'update' | 'remove',
+        redisClient: RedisClient
+    ): Promise<void> {
+        if (action === 'remove') {
+            this.removeFile(hashId);
+            logger.debug(`Removed file ${hashId} from index`);
+        } else {
+            // add or update - fetch latest metadata from Redis
+            const metadata = await redisClient.getFileByHashId(hashId);
+            if (metadata) {
+                this.addFile(hashId, metadata);
+                logger.debug(`${action === 'add' ? 'Added' : 'Updated'} file ${hashId} in index`);
+            }
+        }
+    }
+}

packages/meta-dup-core/src/api/APIServer.ts

@@ -0,0 +1,228 @@
+/**
+ * API Server for meta-dup
+ *
+ * REST API endpoints:
+ * - GET  /health              - Health check
+ * - GET  /api/duplicates      - Get all duplicates
+ * - GET  /api/duplicates/hash - Hash duplicates only
+ * - GET  /api/duplicates/title - Title duplicates only
+ * - GET  /api/duplicates/stats - Statistics
+ * - POST /api/duplicates/rebuild - Force rebuild from Redis
+ */
+
+import Fastify, { FastifyInstance, FastifyRequest, FastifyReply } from 'fastify';
+import cors from '@fastify/cors';
+import fastifyStatic from '@fastify/static';
+import { Logger } from 'tslog';
+import { DuplicateIndex } from '../DuplicateIndex.js';
+import { KVManager } from '../kv/KVManager.js';
+import { RedisClient } from '../kv/RedisClient.js';
+import { join, dirname } from 'path';
+import { fileURLToPath } from 'url';
+import { existsSync } from 'fs';
+
+const logger = new Logger({ name: 'APIServer' });
+
+const __filename = fileURLToPath(import.meta.url);
+const __dirname = dirname(__filename);
+
+export interface APIServerConfig {
+    port: number;
+    host: string;
+}
+
+export class APIServer {
+    private app: FastifyInstance;
+    private config: APIServerConfig;
+    private duplicateIndex: DuplicateIndex;
+    private kvManager: KVManager;
+    private redisClient: RedisClient | null = null;
+
+    constructor(
+        duplicateIndex: DuplicateIndex,
+        config: APIServerConfig,
+        kvManager: KVManager
+    ) {
+        this.duplicateIndex = duplicateIndex;
+        this.config = config;
+        this.kvManager = kvManager;
+
+        this.app = Fastify({
+            logger: false,
+        });
+
+        this.setupMiddleware();
+        this.setupRoutes();
+    }
+
+    /**
+     * Set Redis client (called after connection established)
+     */
+    setRedisClient(client: RedisClient): void {
+        this.redisClient = client;
+    }
+
+    /**
+     * Setup middleware
+     */
+    private setupMiddleware(): void {
+        // Enable CORS
+        this.app.register(cors, {
+            origin: true,
+            credentials: true,
+        });
+
+        // Serve static UI files if they exist
+        const uiDistPath = join(__dirname, '../../..', 'meta-dup-ui', 'dist');
+        if (existsSync(uiDistPath)) {
+            this.app.register(fastifyStatic, {
+                root: uiDistPath,
+                prefix: '/',
+            });
+            logger.info(`Serving UI from ${uiDistPath}`);
+        } else {
+            logger.info('UI dist not found, serving API only');
+        }
+    }
+
+    /**
+     * Setup routes
+     */
+    private setupRoutes(): void {
+        // Health check
+        this.app.get('/health', async (request: FastifyRequest, reply: FastifyReply) => {
+            const isHealthy = await this.kvManager.isHealthy();
+            const stats = this.duplicateIndex.getStats();
+
+            return {
+                status: isHealthy ? 'ok' : 'degraded',
+                service: 'meta-dup',
+                redis: isHealthy,
+                stats: {
+                    filesTracked: stats.totalFilesTracked,
+                    hashDuplicates: stats.hashGroupCount,
+                    titleDuplicates: stats.titleGroupCount,
+                },
+                timestamp: new Date().toISOString(),
+            };
+        });
+
+        // Get all duplicates
+        this.app.get('/api/duplicates', async (request: FastifyRequest, reply: FastifyReply) => {
+            return this.duplicateIndex.getDuplicateData();
+        });
+
+        // Get hash duplicates only
+        this.app.get('/api/duplicates/hash', async (request: FastifyRequest, reply: FastifyReply) => {
+            return {
+                duplicates: this.duplicateIndex.getHashDuplicates(),
+                computedAt: new Date().toISOString(),
+            };
+        });
+
+        // Get title duplicates only
+        this.app.get('/api/duplicates/title', async (request: FastifyRequest, reply: FastifyReply) => {
+            return {
+                duplicates: this.duplicateIndex.getTitleDuplicates(),
+                computedAt: new Date().toISOString(),
+            };
+        });
+
+        // Get statistics
+        this.app.get('/api/duplicates/stats', async (request: FastifyRequest, reply: FastifyReply) => {
+            return this.duplicateIndex.getStats();
+        });
+
+        // Force rebuild from Redis
+        this.app.post('/api/duplicates/rebuild', async (request: FastifyRequest, reply: FastifyReply) => {
+            if (!this.redisClient) {
+                reply.status(503);
+                return { error: 'Redis not connected' };
+            }
+
+            logger.info('Manual rebuild requested');
+            const startTime = Date.now();
+            await this.duplicateIndex.rebuildFromRedis(this.redisClient);
+            const elapsed = Date.now() - startTime;
+
+            return {
+                success: true,
+                message: 'Index rebuilt from Redis',
+                elapsedMs: elapsed,
+                stats: this.duplicateIndex.getStats(),
+            };
+        });
+
+        // Service discovery (for dashboard navigation)
+        this.app.get('/api/services', async (request: FastifyRequest, reply: FastifyReply) => {
+            const services: Array<{
+                name: string;
+                url: string;
+                api: string;
+                status: string;
+                role?: string;
+            }> = [];
+
+            try {
+                const serviceDiscovery = this.kvManager.getServiceDiscovery();
+                if (serviceDiscovery) {
+                    const allServices = await serviceDiscovery.discoverAllServices();
+
+                    for (const svc of allServices) {
+                        services.push({
+                            name: svc.name || 'Unknown',
+                            url: svc.baseUrl || '',
+                            api: svc.baseUrl || '',
+                            status: svc.status || 'unknown',
+                            role: svc.role,
+                        });
+                    }
+                }
+            } catch (error: any) {
+                logger.error('Error discovering services:', error);
+            }
+
+            return {
+                services,
+                current: 'meta-dup',
+            };
+        });
+
+        // Catch-all for SPA routing (serve index.html for non-API routes)
+        this.app.setNotFoundHandler(async (request: FastifyRequest, reply: FastifyReply) => {
+            const uiDistPath = join(__dirname, '../../..', 'meta-dup-ui', 'dist');
+            const indexPath = join(uiDistPath, 'index.html');
+
+            if (existsSync(indexPath) && !request.url.startsWith('/api/')) {
+                return reply.sendFile('index.html');
+            }
+
+            reply.status(404);
+            return { error: 'Not found' };
+        });
+    }
+
+    /**
+     * Start the server
+     */
+    async start(): Promise<void> {
+        try {
+            await this.app.listen({
+                port: this.config.port,
+                host: this.config.host,
+            });
+            logger.info(`API server listening on ${this.config.host}:${this.config.port}`);
+        } catch (error) {
+            logger.error('Failed to start API server:', error);
+            throw error;
+        }
+    }
+
+    /**
+     * Stop the server
+     */
+    async stop(): Promise<void> {
+        await this.app.close();
+        logger.info('API server stopped');
+    }
+}

packages/meta-dup-core/src/index.ts

@@ -0,0 +1,268 @@
+/**
+ * meta-dup-core entry point
+ *
+ * Standalone duplicate detection service that:
+ * - Consumes Redis Streams events from meta-sort
+ * - Detects duplicates by hash (SHA-256) and by title
+ * - Provides REST API and UI dashboard
+ *
+ * Uses Redis Streams for reliable event delivery from meta-sort.
+ */
+
+import 'dotenv/config';
+import { Logger } from 'tslog';
+import { KVManager } from './kv/KVManager.js';
+import { RedisClient, StreamMessage } from './kv/RedisClient.js';
+import { DuplicateIndex } from './DuplicateIndex.js';
+import { APIServer } from './api/APIServer.js';
+
+const logger = new Logger({ name: 'meta-dup' });
+
+// Redis Streams for reliable event delivery from meta-core
+const EVENTS_STREAM = 'file:events';
+const CONSUMER_GROUP = 'meta-dup-consumer';
+
+interface BatchUpdatePayload {
+    timestamp: number;
+    changes: Array<{
+        action: 'add' | 'update' | 'remove';
+        hashId: string;
+    }>;
+}
+
+interface ResetPayload {
+    timestamp: number;
+    action: 'reset';
+}
+
+/**
+ * Handle a stream message from Redis Streams
+ * Routes messages by type to appropriate handlers
+ *
+ * Supports both:
+ * - Direct file events from meta-core (add/change/delete/rename)
+ * - Legacy batch events (for backward compatibility)
+ */
+async function handleStreamMessage(
+    message: StreamMessage,
+    duplicateIndex: DuplicateIndex,
+    redisClient: RedisClient
+): Promise<void> {
+    try {
+        switch (message.type) {
+            // Direct file events from meta-core
+            case 'add':
+            case 'change': {
+                // File added or changed - look up by path and update index
+                // Note: We need to wait for meta-sort to process the file first
+                // For now, just log - the duplicate index is rebuilt on startup
+                logger.debug(`File ${message.type}: ${message.path}`);
+                break;
+            }
+
+            case 'delete': {
+                // File deleted - would need to find by path and remove
+                logger.debug(`File deleted: ${message.path}`);
+                break;
+            }
+
+            case 'rename': {
+                // File renamed - treated as delete + add
+                logger.debug(`File renamed: ${message.oldPath} -> ${message.path}`);
+                break;
+            }
+
+            // Legacy batch events (for backward compatibility)
+            case 'batch': {
+                if (!message.payload) break;
+                const batch: BatchUpdatePayload = JSON.parse(message.payload);
+                logger.debug(`Processing batch update: ${batch.changes.length} changes`);
+
+                for (const change of batch.changes) {
+                    await duplicateIndex.onFileUpdate(change.hashId, change.action, redisClient);
+                }
+                break;
+            }
+
+            case 'reset': {
+                if (!message.payload) break;
+                const reset: ResetPayload = JSON.parse(message.payload);
+                logger.info(`Processing reset event: ${reset.action}`);
+                await duplicateIndex.rebuildFromRedis(redisClient);
+                break;
+            }
+
+            case 'plugin:complete': {
+                // Plugin completion events - update the file's metadata
+                // This is when we get full hash for duplicate detection
+                if (message.payload) {
+                    try {
+                        const payload = JSON.parse(message.payload);
+                        if (payload.fileHash && payload.pluginId === 'fullhash') {
+                            // Full hash plugin completed - this is when we can detect duplicates
+                            await duplicateIndex.onFileUpdate(payload.fileHash, 'update', redisClient);
+                        }
+                    } catch {
+                        // Ignore parse errors
+                    }
+                }
+                break;
+            }
+
+            default:
+                logger.warn(`Unknown stream message type: ${message.type}`);
+        }
+    } catch (error: any) {
+        logger.error(`Error processing stream message ${message.id}:`, error.message);
+        throw error; // Re-throw to prevent ACK
+    }
+}
+
+async function main(): Promise<void> {
+    logger.info('Starting meta-dup...');
+    logger.info(`Node.js ${process.version}`);
+
+    // Configuration
+    const config = {
+        metaCorePath: process.env.META_CORE_PATH ?? '/meta-core',
+        filesPath: process.env.FILES_VOLUME ?? '/files',
+        redisUrl: process.env.REDIS_URL,
+        redisPrefix: process.env.REDIS_PREFIX ?? '',
+        apiPort: parseInt(process.env.API_PORT ?? '3000', 10),
+        apiHost: process.env.API_HOST ?? '0.0.0.0',
+        baseUrl: process.env.BASE_URL,
+        serviceVersion: process.env.SERVICE_VERSION ?? '1.0.0',
+    };
+
+    logger.info(`Config: META_CORE_PATH=${config.metaCorePath}`);
+
+    // Initialize KV Manager (handles leader discovery)
+    const kvManager = new KVManager({
+        metaCorePath: config.metaCorePath,
+        filesPath: config.filesPath,
+        serviceName: 'meta-dup',
+        apiPort: config.apiPort,
+        baseUrl: config.baseUrl,
+        redisPrefix: config.redisPrefix,
+    });
+
+    let redisClient: RedisClient | null = null;
+    let duplicateIndex: DuplicateIndex | null = null;
+    let apiServer: APIServer | null = null;
+
+    // Graceful shutdown
+    const shutdown = async (signal: string): Promise<void> => {
+        logger.info(`Received ${signal}, shutting down...`);
+
+        try {
+            // Stop stream consumer first
+            if (redisClient) {
+                redisClient.stopStreamConsumer();
+            }
+            if (apiServer) await apiServer.stop();
+            await kvManager.stop();
+            logger.info('Shutdown complete');
+            process.exit(0);
+        } catch (error) {
+            logger.error('Error during shutdown:', error);
+            process.exit(1);
+        }
+    };
+
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    process.on('SIGINT', () => shutdown('SIGINT'));
+
+    // Handle ready event from KV manager
+    kvManager.onReady(async () => {
+        logger.info('KV Manager ready, initializing DuplicateIndex...');
+
+        try {
+            // Get Redis client from KV manager
+            redisClient = kvManager.getClient();
+
+            if (!redisClient) {
+                logger.error('Redis client not available');
+                return;
+            }
+
+            // Initialize DuplicateIndex
+            duplicateIndex = new DuplicateIndex();
+
+            // Rebuild index from all file:* keys in Redis
+            logger.info('Building initial duplicate index from Redis...');
+            await duplicateIndex.rebuildFromRedis(redisClient);
+
+            // Initialize Redis Streams consumer
+            logger.info(`Initializing stream consumer for ${EVENTS_STREAM}...`);
+            await redisClient.initStreamConsumer(EVENTS_STREAM, CONSUMER_GROUP);
+
+            // Process any pending entries from crashed consumers
+            await redisClient.processPendingEntries(
+                EVENTS_STREAM,
+                CONSUMER_GROUP,
+                30000, // 30 second idle threshold
+                async (message: StreamMessage) => {
+                    await handleStreamMessage(message, duplicateIndex!, redisClient!);
+                }
+            );
+
+            // Start stream consumer in background
+            logger.info('Starting stream consumer...');
+            redisClient.startStreamConsumer(
+                EVENTS_STREAM,
+                CONSUMER_GROUP,
+                async (message: StreamMessage) => {
+                    await handleStreamMessage(message, duplicateIndex!, redisClient!);
+                },
+                5000 // 5 second block timeout
+            ).catch(error => {
+                logger.error('Stream consumer error:', error);
+            });
+
+            // Initialize API server
+            apiServer = new APIServer(duplicateIndex, {
+                port: config.apiPort,
+                host: config.apiHost,
+            }, kvManager);
+
+            // Set Redis client for rebuild endpoint
+            apiServer.setRedisClient(redisClient);
+
+            // Start API server
+            logger.info('Starting API server...');
+            await apiServer.start();
+
+            logger.info('meta-dup is ready!');
+            logger.info(`API: http://${config.apiHost}:${config.apiPort}`);
+
+            // Log initial stats
+            const stats = duplicateIndex.getStats();
+            logger.info(`Duplicates: ${stats.hashGroupCount} hash groups, ${stats.titleGroupCount} title groups`);
+            logger.info(`Total files tracked: ${stats.totalFilesTracked}`);
+        } catch (error) {
+            logger.error('Failed to initialize after KV ready:', error);
+        }
+    });
+
+    // Handle disconnect event
+    kvManager.onDisconnect(() => {
+        logger.warn('Redis disconnected, duplicate index will use cached data');
+    });
+
+    // Start KV Manager (initiates leader discovery)
+    try {
+        logger.info('Starting KV Manager...');
+        await kvManager.start();
+
+        // Wait for ready with timeout
+        await kvManager.waitForReady(60000);
+    } catch (error) {
+        logger.error('Failed to start meta-dup:', error);
+        process.exit(1);
+    }
+}
+
+main().catch((error) => {
+    logger.error('Fatal error:', error);
+    process.exit(1);
+});

packages/meta-dup-core/src/kv/IKVClient.ts

@@ -0,0 +1,109 @@
+/**
+ * KV Client Interface - Read-only abstraction for meta-dup
+ *
+ * meta-dup is a FOLLOWER service that only reads metadata from Redis.
+ * This interface provides the subset of operations needed for reading.
+ */
+
+export interface KeyValuePair {
+    key: string;
+    value: string;
+}
+
+export interface IKVClient {
+    /**
+     * Get a value by key (value is JSON deserialized)
+     */
+    get(key: string): Promise<any | null>;
+
+    /**
+     * Count keys with a given prefix
+     */
+    countKeysWithPrefix(prefix: string): Promise<number>;
+
+    /**
+     * Health check - verifies KV store is accessible
+     */
+    health(): Promise<boolean>;
+
+    /**
+     * Get all key-value pairs with a given prefix
+     */
+    getRange(prefix: string): Promise<KeyValuePair[]>;
+
+    /**
+     * Retrieve file metadata - reconstructs from individual property keys
+     */
+    getMetadataFlat(hashId: string): Promise<any | null>;
+
+    /**
+     * Get all unique hash IDs (files) stored in KV
+     */
+    getAllHashIds(): Promise<string[]>;
+
+    /**
+     * Subscribe to Redis pub/sub channel
+     */
+    subscribe(channel: string, callback: (message: string) => void): Promise<void>;
+
+    /**
+     * Unsubscribe from Redis pub/sub channel
+     */
+    unsubscribe(channel: string): Promise<void>;
+
+    /**
+     * Close connections and cleanup resources
+     */
+    close(): Promise<void>;
+}
+
+/**
+ * Leader lock file content format (kv-leader.info)
+ * Written by meta-core leader, read by other services
+ */
+export interface LeaderLockInfo {
+    /** Hostname of the leader */
+    hostname: string;
+
+    /** Base URL for the leader service (e.g., http://localhost:8180) */
+    baseUrl: string;
+
+    /** meta-core API URL (port 9000) */
+    apiUrl: string;
+
+    /** Redis connection URL (e.g., redis://10.0.1.50:6379) */
+    redisUrl: string;
+
+    /** WebDAV URL for file access */
+    webdavUrl: string;
+
+    /** Timestamp when leadership was acquired */
+    timestamp: number;
+
+    /** Process ID of the leader */
+    pid: number;
+}
+
+/**
+ * Service registration info (simplified)
+ * Full URLs are obtained via the /urls API endpoint
+ */
+export interface ServiceInfo {
+    /** Service name (e.g., 'meta-sort', 'meta-fuse') */
+    name: string;
+
+    /** Hostname */
+    hostname: string;
+
+    /** Base URL for the service */
+    baseUrl: string;
+
+    /** Current status */
+    status: 'running' | 'stale' | 'stopped';
+
+    /** Last heartbeat timestamp (ISO string) */
+    lastHeartbeat: string;
+
+    /** Role for meta-core instances: 'leader', 'follower', or undefined for other services */
+    role?: string;
+}

packages/meta-dup-core/src/kv/KVManager.ts

@@ -0,0 +1,359 @@
+/**
+ * KV Manager for meta-dup (FOLLOWER-only)
+ *
+ * Simplified KV manager that:
+ * 1. Discovers the leader via LeaderClient (reads kv-leader.info)
+ * 2. Creates Redis client connection
+ * 3. Handles reconnection on leader failure
+ *
+ * Unlike meta-sort's KVManager, this version never spawns Redis -
+ * it only connects as a client. Leader election is handled by meta-core.
+ */
+
+import { hostname } from 'os';
+import { Logger } from 'tslog';
+import { LeaderClient } from './LeaderClient.js';
+import { ServiceDiscovery } from './ServiceDiscovery.js';
+import { RedisClient } from './RedisClient.js';
+import type { LeaderLockInfo } from './IKVClient.js';
+
+const logger = new Logger({ name: 'KVManager' });
+
+interface KVManagerConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** Path to FILES_VOLUME (e.g., /files) */
+    filesPath: string;
+
+    /** Service name (for logging) */
+    serviceName?: string;
+
+    /** HTTP API port */
+    apiPort?: number;
+
+    /** Override Redis URL (skip leader discovery if provided) */
+    redisUrl?: string;
+
+    /** Redis key prefix (default: 'meta-sort:') */
+    redisPrefix?: string;
+
+    /** Base URL for service discovery (overrides auto-detected URL) */
+    baseUrl?: string;
+
+    /** meta-core API URL (e.g., http://localhost:9000) */
+    metaCoreUrl?: string;
+}
+
+export class KVManager {
+    private config: Required<KVManagerConfig>;
+    private leaderClient: LeaderClient | null = null;
+    private serviceDiscovery: ServiceDiscovery | null = null;
+    private redisClient: RedisClient | null = null;
+    private isStarted = false;
+    private isShuttingDown = false;
+
+    // Event callbacks
+    private onReadyCallbacks: (() => void)[] = [];
+    private onDisconnectCallbacks: (() => void)[] = [];
+
+    constructor(config: KVManagerConfig) {
+        this.config = {
+            serviceName: 'meta-dup',
+            apiPort: 3000,
+            redisUrl: '',
+            redisPrefix: '',
+            baseUrl: '',
+            metaCoreUrl: '',
+            ...config
+        };
+    }
+
+    /**
+     * Start the KV manager
+     */
+    async start(): Promise<void> {
+        if (this.isStarted) {
+            logger.warn('KVManager already started');
+            return;
+        }
+
+        logger.info(`Starting KVManager for ${this.config.serviceName}...`);
+
+        // Initialize and start service discovery for registration and heartbeat
+        const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
+        this.serviceDiscovery = new ServiceDiscovery({
+            metaCorePath: this.config.metaCorePath,
+            serviceName: this.config.serviceName,
+            apiUrl
+        });
+        await this.serviceDiscovery.start();
+
+        // If direct Redis URL provided, skip leader discovery
+        if (this.config.redisUrl) {
+            logger.info(`Using direct Redis URL: ${this.config.redisUrl}`);
+            await this.connectToRedis(this.config.redisUrl);
+            this.isStarted = true;
+            return;
+        }
+
+        // Use leader client
+        this.leaderClient = new LeaderClient({
+            metaCorePath: this.config.metaCorePath,
+            metaCoreUrl: this.config.metaCoreUrl || undefined
+        });
+
+        // Set up leader change callback
+        this.leaderClient.onChange(async () => {
+            logger.info('Leader changed, reconnecting...');
+            await this.reconnect();
+        });
+
+        // Wait for leader and connect
+        try {
+            const leaderInfo = await this.leaderClient.waitForLeader(30000);
+            logger.info(`Connecting to Redis at ${leaderInfo.redisUrl}...`);
+            await this.connectToRedis(leaderInfo.redisUrl);
+        } catch (error: any) {
+            logger.error(`Failed to connect to leader: ${error.message}`);
+            throw error;
+        }
+
+        // Start watching for leader changes
+        this.leaderClient.startWatching();
+
+        this.isStarted = true;
+    }
+
+    /**
+     * Reconnect to Redis after leader change
+     */
+    private async reconnect(): Promise<void> {
+        if (this.isShuttingDown || !this.leaderClient) {
+            return;
+        }
+
+        // Disconnect existing
+        await this.disconnectRedis();
+        this.notifyDisconnect();
+
+        // Wait for new leader and connect
+        try {
+            const leaderInfo = await this.leaderClient.waitForLeader(30000);
+            logger.info(`Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
+            await this.connectToRedis(leaderInfo.redisUrl);
+        } catch (error: any) {
+            logger.error(`Failed to reconnect: ${error.message}`);
+        }
+    }
+
+    /**
+     * Stop the KV manager
+     */
+    async stop(): Promise<void> {
+        if (!this.isStarted) return;
+
+        logger.info('Stopping KVManager...');
+        this.isShuttingDown = true;
+
+        // Stop leader client
+        if (this.leaderClient) {
+            this.leaderClient.close();
+            this.leaderClient = null;
+        }
+
+        // Stop service discovery (if it was started)
+        if (this.serviceDiscovery) {
+            await this.serviceDiscovery.stop();
+            this.serviceDiscovery = null;
+        }
+
+        // Disconnect Redis
+        await this.disconnectRedis();
+
+        this.isStarted = false;
+        logger.info('KVManager stopped');
+    }
+
+    /**
+     * Connect to Redis
+     */
+    private async connectToRedis(url: string): Promise<void> {
+        try {
+            // Disconnect existing client if any
+            await this.disconnectRedis();
+
+            // Create new client
+            this.redisClient = new RedisClient({
+                url,
+                prefix: this.config.redisPrefix,
+                filesVolume: this.config.filesPath
+            });
+
+            await this.redisClient.connect();
+            logger.info('Connected to Redis');
+
+            // Notify ready
+            this.notifyReady();
+        } catch (error: any) {
+            logger.error(`Failed to connect to Redis: ${error.message}`);
+            this.redisClient = null;
+        }
+    }
+
+    /**
+     * Disconnect from Redis
+     */
+    private async disconnectRedis(): Promise<void> {
+        if (this.redisClient) {
+            try {
+                await this.redisClient.disconnect();
+            } catch (error: any) {
+                logger.warn(`Error disconnecting Redis: ${error.message}`);
+            }
+            this.redisClient = null;
+        }
+    }
+
+    /**
+     * Notify ready callbacks
+     */
+    private notifyReady(): void {
+        for (const callback of this.onReadyCallbacks) {
+            try {
+                callback();
+            } catch (error: any) {
+                logger.error(`Error in ready callback: ${error.message}`);
+            }
+        }
+    }
+
+    /**
+     * Notify disconnect callbacks
+     */
+    private notifyDisconnect(): void {
+        for (const callback of this.onDisconnectCallbacks) {
+            try {
+                callback();
+            } catch (error: any) {
+                logger.error(`Error in disconnect callback: ${error.message}`);
+            }
+        }
+    }
+
+    // ========================================================================
+    // Event Registration
+    // ========================================================================
+
+    /**
+     * Register callback for when KV client is ready
+     */
+    onReady(callback: () => void): this {
+        this.onReadyCallbacks.push(callback);
+
+        // If already ready, call immediately
+        if (this.redisClient?.connected) {
+            callback();
+        }
+
+        return this;
+    }
+
+    /**
+     * Register callback for disconnection
+     */
+    onDisconnect(callback: () => void): this {
+        this.onDisconnectCallbacks.push(callback);
+        return this;
+    }
+
+    /**
+     * Wait for KV client to be ready
+     */
+    async waitForReady(timeoutMs: number = 30000): Promise<void> {
+        if (this.redisClient?.connected) return;
+
+        return new Promise((resolve, reject) => {
+            const timeout = setTimeout(() => {
+                reject(new Error(`KV client not ready after ${timeoutMs}ms`));
+            }, timeoutMs);
+
+            this.onReady(() => {
+                clearTimeout(timeout);
+                resolve();
+            });
+        });
+    }
+
+    // ========================================================================
+    // Getters
+    // ========================================================================
+
+    /**
+     * Get the Redis client (may be null if not connected)
+     */
+    getClient(): RedisClient | null {
+        return this.redisClient;
+    }
+
+    /**
+     * Get the Redis client (throws if not connected)
+     */
+    requireClient(): RedisClient {
+        if (!this.redisClient) {
+            throw new Error('Redis client not available. Ensure start() was called and leader is available.');
+        }
+        return this.redisClient;
+    }
+
+    /**
+     * Check if connected
+     */
+    isConnected(): boolean {
+        return this.redisClient?.connected ?? false;
+    }
+
+    /**
+     * Health check
+     */
+    async isHealthy(): Promise<boolean> {
+        if (!this.redisClient) return false;
+        try {
+            const stats = await this.redisClient.getStats();
+            return stats.connected;
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Get leader info
+     */
+    getLeaderInfo(): LeaderLockInfo | null {
+        return this.leaderClient?.getCachedLeaderInfo() ?? null;
+    }
+
+    /**
+     * Get leader client
+     */
+    getLeaderClient(): LeaderClient | null {
+        return this.leaderClient;
+    }
+
+    /**
+     * Get service discovery (for inter-service navigation)
+     */
+    getServiceDiscovery(): ServiceDiscovery | null {
+        return this.serviceDiscovery;
+    }
+
+    /**
+     * Get configured paths
+     */
+    getPaths() {
+        return {
+            metaCorePath: this.config.metaCorePath,
+            filesPath: this.config.filesPath
+        };
+    }
+}

packages/meta-dup-core/src/kv/LeaderClient.ts

@@ -0,0 +1,279 @@
+/**
+ * LeaderClient - Client for reading leader info from meta-core
+ *
+ * This replaces the old LeaderDiscovery.ts by delegating leader election
+ * to meta-core (Go sidecar) and just reading the results.
+ *
+ * Features:
+ * - Reads kv-leader.info file for leader URLs
+ * - Calls meta-core /urls API for current leader URLs
+ * - Watches for leader changes via file system watcher
+ */
+
+import { promises as fs } from 'fs';
+import { watch, FSWatcher } from 'fs';
+import { dirname } from 'path';
+import { Logger } from 'tslog';
+import type { LeaderLockInfo } from './IKVClient.js';
+
+const logger = new Logger({ name: 'LeaderClient' });
+
+export interface LeaderClientConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** meta-core API URL (e.g., http://meta-core:9000 or http://localhost:9000) */
+    metaCoreUrl?: string;
+}
+
+export interface URLsResponse {
+    hostname: string;
+    baseUrl: string;
+    apiUrl: string;
+    redisUrl: string;
+    webdavUrl: string;
+    isLeader: boolean;
+}
+
+export class LeaderClient {
+    private config: LeaderClientConfig;
+    private infoFilePath: string;
+    private leaderInfo: LeaderLockInfo | null = null;
+    private watcher: FSWatcher | null = null;
+    private onChangeCallbacks: (() => void)[] = [];
+
+    // URL caching
+    private cachedUrls: URLsResponse | null = null;
+    private urlsCacheTime: number = 0;
+    private readonly urlsCacheTTL: number = 5000; // 5 seconds
+
+    constructor(config: LeaderClientConfig) {
+        this.config = config;
+        this.infoFilePath = `${config.metaCorePath}/locks/kv-leader.info`;
+    }
+
+    /**
+     * Read API URL from file (plain text format)
+     */
+    private async getApiUrlFromFile(): Promise<string | null> {
+        try {
+            const content = await fs.readFile(this.infoFilePath, 'utf-8');
+            return content.trim() || null;
+        } catch (error: any) {
+            logger.error(`Failed to read API URL from file: ${error.message}`);
+            return null;
+        }
+    }
+
+    /**
+     * Fetch URLs from meta-core /urls API with caching
+     */
+    private async fetchUrls(apiUrl: string): Promise<URLsResponse | null> {
+        // Check cache
+        const now = Date.now();
+        if (this.cachedUrls && (now - this.urlsCacheTime) < this.urlsCacheTTL) {
+            return this.cachedUrls;
+        }
+
+        try {
+            const response = await fetch(`${apiUrl}/urls`, {
+                method: 'GET',
+                headers: { 'Accept': 'application/json' },
+                signal: AbortSignal.timeout(5000)
+            });
+
+            if (!response.ok) {
+                logger.error(`Failed to fetch URLs: ${response.status} ${response.statusText}`);
+                return null;
+            }
+
+            this.cachedUrls = await response.json() as URLsResponse;
+            this.urlsCacheTime = now;
+            return this.cachedUrls;
+        } catch (error: any) {
+            logger.error(`Error calling /urls API: ${error.message}`);
+            return null;
+        }
+    }
+
+    /**
+     * Read leader info from file and /urls API
+     */
+    async getLeaderInfo(): Promise<LeaderLockInfo | null> {
+        try {
+            // Read API URL from file (plain text)
+            const apiUrl = await this.getApiUrlFromFile();
+            if (!apiUrl) {
+                return null;
+            }
+
+            // Fetch full info from /urls API
+            const urls = await this.fetchUrls(apiUrl);
+            if (!urls) {
+                return null;
+            }
+
+            // Convert URLsResponse to LeaderLockInfo
+            this.leaderInfo = {
+                hostname: urls.hostname,
+                baseUrl: urls.baseUrl,
+                apiUrl: urls.apiUrl,
+                redisUrl: urls.redisUrl,
+                webdavUrl: urls.webdavUrl,
+                timestamp: Date.now(),
+                pid: 0 // Unknown for remote leader
+            };
+            return this.leaderInfo;
+        } catch (error: any) {
+            logger.error(`Failed to read leader info: ${error.message}`);
+            return null;
+        }
+    }
+
+    /**
+     * Get Redis URL from leader info
+     */
+    async getRedisUrl(): Promise<string | null> {
+        const info = await this.getLeaderInfo();
+        return info?.redisUrl ?? null;
+    }
+
+    /**
+     * Get WebDAV URL from leader info
+     */
+    async getWebdavUrl(): Promise<string | null> {
+        const info = await this.getLeaderInfo();
+        return info?.webdavUrl ?? null;
+    }
+
+    /**
+     * Get meta-core API URL from leader info
+     */
+    async getApiUrl(): Promise<string | null> {
+        const info = await this.getLeaderInfo();
+        return info?.apiUrl ?? null;
+    }
+
+    /**
+     * Call meta-core /urls API to get current URLs
+     * Useful for getting URLs when you don't want to read the file directly
+     */
+    async getUrls(): Promise<URLsResponse | null> {
+        // First try using configured metaCoreUrl
+        let apiUrl: string | null = this.config.metaCoreUrl ?? null;
+
+        // Fall back to reading from file
+        if (!apiUrl) {
+            apiUrl = await this.getApiUrlFromFile();
+        }
+
+        if (!apiUrl) {
+            logger.error('No meta-core API URL available');
+            return null;
+        }
+
+        return this.fetchUrls(apiUrl);
+    }
+
+    /**
+     * Wait for leader info to be available
+     */
+    async waitForLeader(timeoutMs: number = 30000): Promise<LeaderLockInfo> {
+        const startTime = Date.now();
+        const pollInterval = 500;
+
+        while (Date.now() - startTime < timeoutMs) {
+            const info = await this.getLeaderInfo();
+            if (info) {
+                logger.info(`Leader found: ${info.hostname} at ${info.redisUrl}`);
+                return info;
+            }
+
+            logger.debug('Waiting for leader...');
+            await new Promise(resolve => setTimeout(resolve, pollInterval));
+        }
+
+        throw new Error(`No leader found within ${timeoutMs}ms`);
+    }
+
+    /**
+     * Start watching for leader changes
+     */
+    startWatching(): void {
+        if (this.watcher) {
+            return; // Already watching
+        }
+
+        const lockDir = dirname(this.infoFilePath);
+
+        try {
+            this.watcher = watch(lockDir, (eventType, filename) => {
+                if (filename === 'kv-leader.info') {
+                    logger.debug(`Lock file ${eventType}, invalidating cache...`);
+                    // Invalidate cache to force fresh API call
+                    this.cachedUrls = null;
+                    this.urlsCacheTime = 0;
+
+                    this.getLeaderInfo().then(() => {
+                        this.notifyChange();
+                    }).catch((err) => logger.error(`Error reloading leader info: ${err.message}`));
+                }
+            });
+
+            this.watcher.on('error', (error: any) => {
+                logger.warn(`File watcher error: ${error.message}`);
+            });
+
+            logger.info(`Watching for leader changes in ${lockDir}`);
+        } catch (error: any) {
+            logger.error(`Failed to start watching: ${error.message}`);
+        }
+    }
+
+    /**
+     * Stop watching for leader changes
+     */
+    stopWatching(): void {
+        if (this.watcher) {
+            this.watcher.close();
+            this.watcher = null;
+            logger.debug('Stopped watching for leader changes');
+        }
+    }
+
+    /**
+     * Register callback for leader changes
+     */
+    onChange(callback: () => void): this {
+        this.onChangeCallbacks.push(callback);
+        return this;
+    }
+
+    /**
+     * Notify all change callbacks
+     */
+    private notifyChange(): void {
+        for (const callback of this.onChangeCallbacks) {
+            try {
+                callback();
+            } catch (error: any) {
+                logger.error(`Error in change callback: ${error.message}`);
+            }
+        }
+    }
+
+    /**
+     * Get cached leader info (without re-reading file)
+     */
+    getCachedLeaderInfo(): LeaderLockInfo | null {
+        return this.leaderInfo;
+    }
+
+    /**
+     * Clean up resources
+     */
+    close(): void {
+        this.stopWatching();
+        this.onChangeCallbacks = [];
+    }
+}

packages/meta-dup-core/src/kv/RedisClient.ts

@@ -0,0 +1,733 @@
+/**
+ * Redis Client for meta-dup
+ * Reads metadata from Redis (written by meta-sort)
+ *
+ * Key features:
+ * - FILES_VOLUME path resolution (prepends /files/ to relative paths)
+ * - Parses meta-sort hash format into structured metadata
+ * - Redis Streams support for real-time updates
+ */
+
+import { Redis } from 'ioredis';
+import { Logger } from 'tslog';
+import * as os from 'os';
+import type { IKVClient, KeyValuePair } from './IKVClient.js';
+
+const logger = new Logger({ name: 'RedisClient' });
+
+/**
+ * Stream message from Redis Streams
+ * Supports both:
+ * - Direct file events from meta-core (add/change/delete/rename)
+ * - Legacy batch events from meta-sort (batch/reset/plugin:complete)
+ */
+export interface StreamMessage {
+    id: string;
+    type: 'add' | 'change' | 'delete' | 'rename' | 'batch' | 'reset' | 'plugin:complete';
+    // Direct file event fields (from meta-core)
+    path?: string;
+    size?: string;
+    partialHash?: string;
+    oldPath?: string;
+    // Legacy payload field (for backward compatibility)
+    payload?: string;
+    timestamp: string;
+}
+
+/**
+ * Stream consumer callback
+ */
+export type StreamMessageHandler = (message: StreamMessage) => Promise<void>;
+
+/**
+ * File metadata structure for duplicate detection
+ */
+export interface FileMetadata {
+    // Path information
+    sourcePath: string;      // Full path: /files/media1/Movies/...
+    originalPath: string;    // Relative path from meta-sort: media1/Movies/...
+
+    // File stats
+    size: number;
+    mtime: number;
+    ctime: number;
+
+    // Identification
+    hashId?: string;         // midhash256 hash ID
+    sha256?: string;         // Full SHA-256 hash for duplicate detection
+
+    // Title information (for folder organization)
+    title?: string;
+    titles?: { eng?: string; [key: string]: string | undefined };
+    originalTitle?: string;
+    fileName?: string;
+
+    // Series metadata
+    season?: number;
+    episode?: number;
+    extra?: boolean;
+
+    // Movie metadata
+    movieYear?: number;
+    year?: number;
+
+    // File type
+    fileType?: string;       // 'video', 'subtitle', 'torrent', etc.
+    extension?: string;
+
+    // Version/variant info
+    version?: string;
+    subtitleLanguage?: string;
+
+    // Generic metadata container
+    [key: string]: unknown;
+}
+
+export interface RedisClientConfig {
+    url?: string;
+    prefix?: string;
+    filesVolume?: string;
+    reconnectInterval?: number;
+}
+
+export class RedisClient implements Partial<IKVClient> {
+    private client: Redis | null = null;
+    private subscriber: Redis | null = null;
+    private config: Required<RedisClientConfig>;
+    private isConnected = false;
+    private reconnectTimer: NodeJS.Timeout | null = null;
+    private subscriptions: Map<string, (message: string) => void> = new Map();
+
+    // Stream consumer state
+    private streamConsumerRunning = false;
+    private streamConsumerAbort: AbortController | null = null;
+    private consumerName: string;
+
+    constructor(config: RedisClientConfig = {}) {
+        this.config = {
+            url: config.url ?? process.env.REDIS_URL ?? 'redis://127.0.0.1:6379',
+            prefix: config.prefix ?? process.env.REDIS_PREFIX ?? '',
+            filesVolume: config.filesVolume ?? process.env.FILES_VOLUME ?? '/files',
+            reconnectInterval: config.reconnectInterval ?? 5000,
+        };
+
+        // Unique consumer name for this instance
+        this.consumerName = `${os.hostname()}-${process.pid}`;
+    }
+
+    /**
+     * Connect to Redis
+     */
+    async connect(): Promise<void> {
+        if (this.client && this.isConnected) {
+            return;
+        }
+
+        try {
+            this.client = new Redis(this.config.url, {
+                retryStrategy: (times: number) => {
+                    if (times > 10) {
+                        logger.warn('Max reconnection attempts reached');
+                        return null;
+                    }
+                    return Math.min(times * 500, 5000);
+                },
+                maxRetriesPerRequest: 3,
+            });
+
+            this.client.on('connect', () => {
+                logger.info('Connected to Redis');
+                this.isConnected = true;
+            });
+
+            this.client.on('error', (err: Error) => {
+                logger.error('Redis error:', err.message);
+                this.isConnected = false;
+            });
+
+            this.client.on('close', () => {
+                logger.warn('Redis connection closed');
+                this.isConnected = false;
+                this.scheduleReconnect();
+            });
+
+            // Wait for connection
+            await new Promise<void>((resolve, reject) => {
+                const timeout = setTimeout(() => {
+                    reject(new Error('Redis connection timeout'));
+                }, 10000);
+
+                this.client!.once('ready', () => {
+                    clearTimeout(timeout);
+                    this.isConnected = true;
+                    resolve();
+                });
+
+                this.client!.once('error', (err: Error) => {
+                    clearTimeout(timeout);
+                    reject(err);
+                });
+            });
+        } catch (error) {
+            logger.error('Failed to connect to Redis:', error);
+            this.scheduleReconnect();
+            throw error;
+        }
+    }
+
+    /**
+     * Schedule reconnection attempt
+     */
+    private scheduleReconnect(): void {
+        if (this.reconnectTimer) {
+            return;
+        }
+
+        this.reconnectTimer = setTimeout(async () => {
+            this.reconnectTimer = null;
+            try {
+                await this.connect();
+            } catch {
+                // Will retry via scheduleReconnect
+            }
+        }, this.config.reconnectInterval);
+    }
+
+    /**
+     * Disconnect from Redis
+     */
+    async disconnect(): Promise<void> {
+        // Stop stream consumer first
+        this.stopStreamConsumer();
+
+        if (this.reconnectTimer) {
+            clearTimeout(this.reconnectTimer);
+            this.reconnectTimer = null;
+        }
+
+        if (this.subscriber) {
+            await this.subscriber.quit();
+            this.subscriber = null;
+        }
+
+        if (this.client) {
+            await this.client.quit();
+            this.client = null;
+            this.isConnected = false;
+        }
+
+        this.subscriptions.clear();
+    }
+
+    /**
+     * Check if connected
+     */
+    get connected(): boolean {
+        return this.isConnected;
+    }
+
+    /**
+     * Get all file metadata from Redis
+     */
+    async getAllFiles(): Promise<Map<string, FileMetadata>> {
+        const files = new Map<string, FileMetadata>();
+
+        if (!this.client || !this.isConnected) {
+            logger.warn('Redis not connected, returning empty file list');
+            return files;
+        }
+
+        try {
+            // Get all file keys using the index
+            const indexKey = `${this.config.prefix}file:__index__`;
+            const hashIds = await this.client.smembers(indexKey);
+
+            if (hashIds.length === 0) {
+                // Fallback: scan for file keys directly
+                const pattern = `${this.config.prefix}file:*`;
+                const keys = await this.scanKeys(pattern);
+                logger.debug(`Found ${keys.length} file keys in Redis (via scan)`);
+
+                for (const key of keys) {
+                    if (key.includes('__index__')) continue;
+
+                    try {
+                        const data = await this.client.hgetall(key);
+                        if (data && Object.keys(data).length > 0) {
+                            const hashId = key.replace(`${this.config.prefix}file:`, '');
+                            const metadata = this.parseMetadata(data, hashId);
+                            files.set(hashId, metadata);
+                        }
+                    } catch (err) {
+                        logger.warn(`Failed to get metadata for key ${key}:`, err);
+                    }
+                }
+            } else {
+                logger.debug(`Found ${hashIds.length} files in index`);
+
+                // Fetch metadata for each hash ID
+                for (const hashId of hashIds) {
+                    try {
+                        const key = `${this.config.prefix}file:${hashId}`;
+                        const data = await this.client.hgetall(key);
+                        if (data && Object.keys(data).length > 0) {
+                            const metadata = this.parseMetadata(data, hashId);
+                            files.set(hashId, metadata);
+                        }
+                    } catch (err) {
+                        logger.warn(`Failed to get metadata for hashId ${hashId}:`, err);
+                    }
+                }
+            }
+
+            logger.info(`Loaded ${files.size} files from Redis`);
+        } catch (error) {
+            logger.error('Failed to get files from Redis:', error);
+        }
+
+        return files;
+    }
+
+    /**
+     * Get file metadata by hash ID
+     */
+    async getFileByHashId(hashId: string): Promise<FileMetadata | null> {
+        if (!this.client || !this.isConnected) {
+            return null;
+        }
+
+        try {
+            const key = `${this.config.prefix}file:${hashId}`;
+            const data = await this.client.hgetall(key);
+
+            if (data && Object.keys(data).length > 0) {
+                return this.parseMetadata(data, hashId);
+            }
+
+            return null;
+        } catch (error) {
+            logger.error(`Failed to get file ${hashId}:`, error);
+            return null;
+        }
+    }
+
+    /**
+     * Get all hash IDs from Redis index
+     */
+    async getAllHashIds(): Promise<string[]> {
+        if (!this.client || !this.isConnected) {
+            return [];
+        }
+
+        try {
+            const indexKey = `${this.config.prefix}file:__index__`;
+            return await this.client.smembers(indexKey);
+        } catch (error) {
+            logger.error('Failed to get hash IDs:', error);
+            return [];
+        }
+    }
+
+    /**
+     * Scan Redis keys matching pattern
+     */
+    private async scanKeys(pattern: string): Promise<string[]> {
+        if (!this.client) return [];
+
+        const keys: string[] = [];
+        let cursor = '0';
+
+        do {
+            const [newCursor, batch] = await this.client.scan(cursor, 'MATCH', pattern, 'COUNT', 1000);
+            cursor = newCursor;
+            keys.push(...batch);
+        } while (cursor !== '0');
+
+        return keys;
+    }
+
+    /**
+     * Parse metadata from Redis hash
+     * Handles meta-sort format and resolves paths with FILES_VOLUME
+     */
+    private parseMetadata(data: Record<string, string>, hashId?: string): FileMetadata {
+        // Get the original path (relative to FILES_VOLUME)
+        const originalPath = data.filePath ?? data.sourcePath ?? '';
+
+        // Resolve full source path by prepending FILES_VOLUME
+        let sourcePath = originalPath;
+        if (originalPath && !originalPath.startsWith('/')) {
+            // Relative path - prepend FILES_VOLUME
+            sourcePath = `${this.config.filesVolume}/${originalPath}`;
+        } else if (!originalPath.startsWith(this.config.filesVolume) && originalPath.startsWith('/')) {
+            // Absolute path but not in FILES_VOLUME - still prepend
+            sourcePath = `${this.config.filesVolume}${originalPath}`;
+        }
+
+        // Parse size
+        const size = parseInt(data.fileSize ?? data.size ?? data.sizeByte ?? '0', 10);
+
+        // Parse timestamps
+        let mtime = 0;
+        if (data.mtime) {
+            const parsed = Date.parse(data.mtime);
+            mtime = isNaN(parsed) ? parseFloat(data.mtime) : parsed;
+        }
+
+        // Parse titles - can be nested object or simple string
+        let titles: { eng?: string; [key: string]: string | undefined } | undefined;
+        if (data['titles/eng']) {
+            titles = { eng: data['titles/eng'] };
+        }
+
+        // Parse season/episode (can be 0 for specials)
+        const season = data.season !== undefined ? parseInt(data.season, 10) : undefined;
+        const episode = data.episode !== undefined ? parseInt(data.episode, 10) : undefined;
+
+        // Parse year
+        const movieYear = data.movieYear ? parseInt(data.movieYear, 10) : undefined;
+        const year = data.year ? parseInt(data.year, 10) : movieYear;
+
+        return {
+            // Path information
+            sourcePath,
+            originalPath,
+
+            // File stats
+            size,
+            mtime,
+            ctime: data.ctime ? parseFloat(data.ctime) : mtime,
+
+            // Identification
+            hashId: hashId ?? data.cid_midhash256 ?? data.hashId,
+            sha256: data['cid_sha2-256'] ?? data.sha256,
+
+            // Title information
+            title: data.title,
+            titles,
+            originalTitle: data.originalTitle,
+            fileName: data.fileName,
+
+            // Series metadata
+            season,
+            episode,
+            extra: data.extra === 'true',
+
+            // Movie metadata
+            movieYear,
+            year,
+
+            // File type
+            fileType: data.fileType,
+            extension: data.extension,
+
+            // Version/variant
+            version: data.version,
+            subtitleLanguage: data.subtitleLanguage,
+        };
+    }
+
+    // ========================================================================
+    // Redis Streams Consumer Methods
+    // ========================================================================
+
+    /**
+     * Initialize stream consumer group
+     * Creates the consumer group at position 0 to read all historical events
+     *
+     * @param stream - Stream name (e.g., 'meta-sort:events')
+     * @param group - Consumer group name (e.g., 'meta-dup-consumer')
+     */
+    async initStreamConsumer(stream: string, group: string): Promise<void> {
+        if (!this.client) {
+            throw new Error('Redis not connected');
+        }
+
+        try {
+            // Create consumer group at position 0 (read all historical events)
+            // MKSTREAM creates the stream if it doesn't exist
+            await this.client.xgroup('CREATE', stream, group, '0', 'MKSTREAM');
+            logger.info(`Created consumer group '${group}' for stream '${stream}'`);
+        } catch (error: any) {
+            // BUSYGROUP means group already exists - that's fine
+            if (error.message?.includes('BUSYGROUP')) {
+                logger.debug(`Consumer group '${group}' already exists`);
+            } else {
+                throw error;
+            }
+        }
+    }
+
+    /**
+     * Process pending entries from crashed consumers
+     * Uses XAUTOCLAIM to claim entries that have been idle too long
+     *
+     * @param stream - Stream name
+     * @param group - Consumer group name
+     * @param minIdleTime - Minimum idle time in ms before claiming (default: 30000)
+     * @param onMessage - Handler for each message
+     */
+    async processPendingEntries(
+        stream: string,
+        group: string,
+        minIdleTime: number = 30000,
+        onMessage: StreamMessageHandler
+    ): Promise<number> {
+        if (!this.client) {
+            return 0;
+        }
+
+        let processed = 0;
+        let cursor = '0-0';
+
+        try {
+            while (true) {
+                // XAUTOCLAIM claims idle entries and returns them
+                const result = await this.client.xautoclaim(
+                    stream,
+                    group,
+                    this.consumerName,
+                    minIdleTime,
+                    cursor,
+                    'COUNT',
+                    100
+                ) as [string, Array<[string, string[]]>, string[]];
+
+                const [nextCursor, entries] = result;
+                cursor = nextCursor;
+
+                if (!entries || entries.length === 0) {
+                    break;
+                }
+
+                for (const [id, fields] of entries) {
+                    try {
+                        const message = this.parseStreamEntry(id, fields);
+                        if (message) {
+                            await onMessage(message);
+                            // ACK the message after successful processing
+                            await this.client.xack(stream, group, id);
+                            processed++;
+                        }
+                    } catch (error: any) {
+                        logger.error(`Error processing pending entry ${id}:`, error.message);
+                    }
+                }
+
+                // If cursor is 0-0, we've processed all pending entries
+                if (cursor === '0-0') {
+                    break;
+                }
+            }
+
+            if (processed > 0) {
+                logger.info(`Processed ${processed} pending stream entries`);
+            }
+        } catch (error: any) {
+            logger.error('Error processing pending entries:', error.message);
+        }
+
+        return processed;
+    }
+
+    /**
+     * Start stream consumer loop
+     * Reads messages using XREADGROUP and calls the handler
+     *
+     * @param stream - Stream name
+     * @param group - Consumer group name
+     * @param onMessage - Handler for each message
+     * @param blockMs - Block timeout in ms (default: 5000)
+     */
+    async startStreamConsumer(
+        stream: string,
+        group: string,
+        onMessage: StreamMessageHandler,
+        blockMs: number = 5000
+    ): Promise<void> {
+        if (!this.client) {
+            throw new Error('Redis not connected');
+        }
+
+        if (this.streamConsumerRunning) {
+            logger.warn('Stream consumer already running');
+            return;
+        }
+
+        this.streamConsumerRunning = true;
+        this.streamConsumerAbort = new AbortController();
+
+        logger.info(`Starting stream consumer: ${stream} (group: ${group}, consumer: ${this.consumerName})`);
+
+        // Consumer loop
+        while (this.streamConsumerRunning && !this.streamConsumerAbort.signal.aborted) {
+            try {
+                // XREADGROUP blocks waiting for new messages
+                // '>' means only read new messages (not already delivered to this consumer)
+                const result = await (this.client.call(
+                    'XREADGROUP',
+                    'GROUP', group,
+                    this.consumerName,
+                    'BLOCK', blockMs,
+                    'COUNT', 10,
+                    'STREAMS', stream,
+                    '>'
+                ) as Promise<Array<[string, Array<[string, string[]]>]> | null>);
+
+                if (!result || result.length === 0) {
+                    continue; // Timeout, loop again
+                }
+
+                // Process each stream's messages
+                for (const [streamName, entries] of result) {
+                    for (const [id, fields] of entries) {
+                        try {
+                            const message = this.parseStreamEntry(id, fields);
+                            if (message) {
+                                await onMessage(message);
+                                // ACK the message after successful processing
+                                await this.client!.xack(stream, group, id);
+                            }
+                        } catch (error: any) {
+                            logger.error(`Error processing stream entry ${id}:`, error.message);
+                            // Don't ACK on error - message will be reprocessed
+                        }
+                    }
+                }
+            } catch (error: any) {
+                if (this.streamConsumerAbort?.signal.aborted) {
+                    break;
+                }
+                logger.error('Stream consumer error:', error.message);
+                // Brief pause before retrying
+                await new Promise(resolve => setTimeout(resolve, 1000));
+            }
+        }
+
+        logger.info('Stream consumer stopped');
+    }
+
+    /**
+     * Stop the stream consumer loop
+     */
+    stopStreamConsumer(): void {
+        if (!this.streamConsumerRunning) {
+            return;
+        }
+
+        logger.info('Stopping stream consumer...');
+        this.streamConsumerRunning = false;
+        this.streamConsumerAbort?.abort();
+        this.streamConsumerAbort = null;
+    }
+
+    /**
+     * Parse a stream entry into a StreamMessage
+     * Handles both direct file events (add/change/delete/rename) and legacy batch events
+     */
+    private parseStreamEntry(id: string, fields: string[]): StreamMessage | null {
+        // Fields come as flat array: ['type', 'add', 'path', '...', 'timestamp', '123']
+        const fieldMap: Record<string, string> = {};
+        for (let i = 0; i < fields.length; i += 2) {
+            fieldMap[fields[i]] = fields[i + 1];
+        }
+
+        if (!fieldMap.type) {
+            logger.warn(`Invalid stream entry ${id}: missing type`);
+            return null;
+        }
+
+        // For direct file events (add/change/delete/rename), path is required
+        // For legacy events (batch/reset/plugin:complete), payload is required
+        const directEventTypes = ['add', 'change', 'delete', 'rename'];
+        const isDirectEvent = directEventTypes.includes(fieldMap.type);
+
+        if (isDirectEvent && !fieldMap.path) {
+            logger.warn(`Invalid stream entry ${id}: missing path for ${fieldMap.type} event`);
+            return null;
+        }
+
+        if (!isDirectEvent && !fieldMap.payload) {
+            logger.warn(`Invalid stream entry ${id}: missing payload for ${fieldMap.type} event`);
+            return null;
+        }
+
+        return {
+            id,
+            type: fieldMap.type as StreamMessage['type'],
+            path: fieldMap.path,
+            size: fieldMap.size,
+            partialHash: fieldMap.partialHash,
+            oldPath: fieldMap.oldPath,
+            payload: fieldMap.payload,
+            timestamp: fieldMap.timestamp || '0',
+        };
+    }
+
+    /**
+     * Get stats about Redis data
+     */
+    async getStats(): Promise<{ totalKeys: number; fileCount: number; connected: boolean }> {
+        if (!this.client || !this.isConnected) {
+            return { totalKeys: 0, fileCount: 0, connected: false };
+        }
+
+        try {
+            // Try index first
+            const indexKey = `${this.config.prefix}file:__index__`;
+            const count = await this.client.scard(indexKey);
+
+            if (count > 0) {
+                return {
+                    totalKeys: count,
+                    fileCount: count,
+                    connected: true,
+                };
+            }
+
+            // Fallback to scanning
+            const filePattern = `${this.config.prefix}file:*`;
+            const fileKeys = await this.scanKeys(filePattern);
+            const actualFileKeys = fileKeys.filter(k => !k.includes('__index__'));
+
+            return {
+                totalKeys: actualFileKeys.length,
+                fileCount: actualFileKeys.length,
+                connected: true,
+            };
+        } catch {
+            return { totalKeys: 0, fileCount: 0, connected: false };
+        }
+    }
+
+    /**
+     * Health check
+     */
+    async health(): Promise<boolean> {
+        if (!this.client || !this.isConnected) {
+            return false;
+        }
+
+        try {
+            const result = await this.client.ping();
+            return result === 'PONG';
+        } catch {
+            return false;
+        }
+    }
+
+    /**
+     * Close connections
+     */
+    async close(): Promise<void> {
+        await this.disconnect();
+    }
+
+    /**
+     * Get the raw ioredis client for direct access if needed
+     * Use with caution - prefer using the typed methods above
+     */
+    getRawClient(): Redis | null {
+        return this.client;
+    }
+}

packages/meta-dup-core/src/kv/ServiceDiscovery.ts

@@ -0,0 +1,331 @@
+/**
+ * Service Discovery using shared filesystem
+ *
+ * Each service registers itself in /meta-core/services/{service-name}-{hostname}.json
+ * with its base URL, status, and heartbeat timestamp.
+ *
+ * Services can discover each other by reading these JSON files.
+ * Stale detection: if lastHeartbeat > 60 seconds ago, service is considered stale.
+ *
+ * Each service must call start() to register itself and begin heartbeat updates.
+ * meta-core runs a cleanup process that removes services with stale heartbeats.
+ */
+
+import { promises as fs } from 'fs';
+import { hostname } from 'os';
+import { join } from 'path';
+import { Logger } from 'tslog';
+import type { ServiceInfo } from './IKVClient.js';
+
+const logger = new Logger({ name: 'ServiceDiscovery' });
+
+interface ServiceDiscoveryConfig {
+    /** Path to META_CORE_VOLUME (e.g., /meta-core) */
+    metaCorePath: string;
+
+    /** Service name */
+    serviceName: string;
+
+    /** Base URL for the service */
+    apiUrl: string;
+
+    /** Heartbeat interval in ms (default: 30000) */
+    heartbeatInterval?: number;
+
+    /** Stale threshold in ms (default: 60000) */
+    staleThreshold?: number;
+}
+
+export class ServiceDiscovery {
+    private config: ServiceDiscoveryConfig;
+    private servicesDir: string;
+    private serviceFilePath: string;
+    private heartbeatTimer: NodeJS.Timeout | null = null;
+    private isShuttingDown = false;
+    private isStarted = false;
+    private currentHostname: string;
+
+    constructor(config: ServiceDiscoveryConfig) {
+        this.config = {
+            heartbeatInterval: 30000,
+            staleThreshold: 60000,
+            ...config
+        };
+
+        this.currentHostname = hostname();
+        this.servicesDir = `${this.config.metaCorePath}/services`;
+        // Use hostname-based file naming
+        this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`;
+    }
+
+    /**
+     * Ensure services directory exists
+     */
+    private async ensureServicesDir(): Promise<void> {
+        await fs.mkdir(this.servicesDir, { recursive: true });
+    }
+
+    /**
+     * Build service info for this service
+     */
+    private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
+        return {
+            name: this.config.serviceName,
+            hostname: this.currentHostname,
+            baseUrl: this.config.apiUrl,
+            status,
+            lastHeartbeat: new Date().toISOString()
+        };
+    }
+
+    /**
+     * Register this service by writing service info to the shared filesystem.
+     * meta-core's cleanup process monitors these files and removes stale entries.
+     */
+    async register(): Promise<void> {
+        await this.ensureServicesDir();
+
+        const info = this.buildServiceInfo('running');
+        await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+
+        logger.info(`Registered ${this.config.serviceName}-${this.currentHostname}`);
+    }
+
+    /**
+     * Update service status
+     */
+    async updateStatus(status: ServiceInfo['status']): Promise<void> {
+        try {
+            const content = await fs.readFile(this.serviceFilePath, 'utf-8');
+            const info = JSON.parse(content) as ServiceInfo;
+
+            info.status = status;
+            info.lastHeartbeat = new Date().toISOString();
+
+            await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+        } catch (error) {
+            logger.error('Failed to update status:', error);
+        }
+    }
+
+    /**
+     * Send heartbeat (update lastHeartbeat timestamp)
+     */
+    async heartbeat(): Promise<void> {
+        try {
+            const content = await fs.readFile(this.serviceFilePath, 'utf-8');
+            const info = JSON.parse(content) as ServiceInfo;
+
+            info.lastHeartbeat = new Date().toISOString();
+
+            await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
+        } catch (error: any) {
+            // If file was deleted, re-register
+            if (error.code === 'ENOENT') {
+                await this.register();
+            } else {
+                logger.error('Heartbeat failed:', error);
+            }
+        }
+    }
+
+    /**
+     * Start heartbeat loop
+     */
+    startHeartbeat(): void {
+        this.heartbeatTimer = setInterval(async () => {
+            if (this.isShuttingDown) return;
+            await this.heartbeat();
+        }, this.config.heartbeatInterval);
+    }
+
+    /**
+     * Stop heartbeat loop
+     */
+    stopHeartbeat(): void {
+        if (this.heartbeatTimer) {
+            clearInterval(this.heartbeatTimer);
+            this.heartbeatTimer = null;
+        }
+    }
+
+    /**
+     * Unregister this service (on shutdown)
+     */
+    async unregister(): Promise<void> {
+        this.isShuttingDown = true;
+        this.stopHeartbeat();
+
+        try {
+            // Remove the service file
+            await fs.unlink(this.serviceFilePath);
+            logger.info(`Unregistered ${this.config.serviceName}-${this.currentHostname}`);
+        } catch (error: any) {
+            if (error.code !== 'ENOENT') {
+                logger.error('Failed to unregister:', error);
+            }
+        }
+    }
+
+    /**
+     * Check if a service info is stale
+     */
+    private isStale(info: ServiceInfo): boolean {
+        const lastHeartbeat = new Date(info.lastHeartbeat).getTime();
+        return Date.now() - lastHeartbeat > this.config.staleThreshold!;
+    }
+
+    // ========================================================================
+    // Discovery Methods
+    // ========================================================================
+
+    /**
+     * Discover a service by name
+     * Looks for files matching pattern: {name}-*.json (hostname-based naming)
+     */
+    async discoverService(name: string): Promise<ServiceInfo | null> {
+        try {
+            // First try exact match for backward compatibility
+            const exactPath = join(this.servicesDir, `${name}.json`);
+            try {
+                const content = await fs.readFile(exactPath, 'utf-8');
+                const service = JSON.parse(content) as ServiceInfo;
+
+                if (this.isStale(service)) {
+                    service.status = 'stale';
+                }
+
+                return service.status === 'running' ? service : null;
+            } catch {
+                // Try hostname-based files
+            }
+
+            // Search for hostname-based files: name-*.json
+            const files = await fs.readdir(this.servicesDir);
+            const matchingFiles = files.filter(f =>
+                f.startsWith(`${name}-`) && f.endsWith('.json')
+            );
+
+            // Return first valid (non-stale) service
+            for (const file of matchingFiles) {
+                const filePath = join(this.servicesDir, file);
+                const content = await fs.readFile(filePath, 'utf-8');
+                const service = JSON.parse(content) as ServiceInfo;
+
+                if (this.isStale(service)) {
+                    service.status = 'stale';
+                    continue; // Skip stale services
+                }
+
+                if (service.status === 'running') {
+                    return service;
+                }
+            }
+
+            return null;
+        } catch (error: any) {
+            if (error.code === 'ENOENT') {
+                return null; // Service not registered
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Discover all registered services
+     */
+    async discoverAllServices(): Promise<ServiceInfo[]> {
+        try {
+            const files = await fs.readdir(this.servicesDir);
+            const services: ServiceInfo[] = [];
+
+            for (const file of files) {
+                if (!file.endsWith('.json')) continue;
+
+                const filePath = join(this.servicesDir, file);
+                try {
+                    const content = await fs.readFile(filePath, 'utf-8');
+                    const service = JSON.parse(content) as ServiceInfo;
+
+                    if (this.isStale(service)) {
+                        service.status = 'stale';
+                    }
+
+                    services.push(service);
+                } catch (error) {
+                    logger.error(`Failed to read ${file}:`, error);
+                }
+            }
+
+            return services;
+        } catch (error: any) {
+            if (error.code === 'ENOENT') {
+                return []; // Services directory doesn't exist yet
+            }
+            throw error;
+        }
+    }
+
+    /**
+     * Check if a service is healthy
+     */
+    async isServiceHealthy(name: string): Promise<boolean> {
+        const service = await this.discoverService(name);
+        if (!service) return false;
+
+        // Try to ping the service's health endpoint
+        const url = `${service.baseUrl}/health`;
+
+        try {
+            const response = await fetch(url, {
+                signal: AbortSignal.timeout(5000)
+            });
+            return response.ok;
+        } catch {
+            return false;
+        }
+    }
+
+    // ========================================================================
+    // Lifecycle
+    // ========================================================================
+
+    /**
+     * Full startup sequence: register service and start heartbeat loop.
+     */
+    async start(): Promise<void> {
+        if (this.isStarted) return;
+
+        await this.register();
+        this.startHeartbeat();
+        this.isStarted = true;
+
+        logger.info(`Started ${this.config.serviceName} at ${this.config.apiUrl}`);
+    }
+
+    /**
+     * Full shutdown sequence
+     */
+    async stop(): Promise<void> {
+        if (!this.isStarted) return;
+
+        await this.unregister();
+        this.isStarted = false;
+    }
+
+    // ========================================================================
+    // Getters
+    // ========================================================================
+
+    getServiceFilePath(): string {
+        return this.serviceFilePath;
+    }
+
+    getServicesDir(): string {
+        return this.servicesDir;
+    }
+
+    getIsStarted(): boolean {
+        return this.isStarted;
+    }
+}

packages/meta-dup-core/tsconfig.json

@@ -0,0 +1,20 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "NodeNext",
+    "lib": ["ES2022"],
+    "outDir": "./dist",
+    "rootDir": "./src",
+    "strict": true,
+    "esModuleInterop": true,
+    "skipLibCheck": true,
+    "forceConsistentCasingInFileNames": true,
+    "resolveJsonModule": true,
+    "declaration": true,
+    "declarationMap": true,
+    "sourceMap": true
+  },
+  "include": ["src/**/*"],
+  "exclude": ["node_modules", "dist"]
+}

packages/meta-dup-ui/dist/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>meta-dup - Duplicate Detection</title>
+    <script type="module" crossorigin src="/assets/index-GT6exvk4.js"></script>
+  </head>
+  <body>
+    <div id="root"></div>
+  </body>
+</html>

packages/meta-dup-ui/index.html

@@ -0,0 +1,13 @@
+<!DOCTYPE html>
+<html lang="en">
+  <head>
+    <meta charset="UTF-8" />
+    <link rel="icon" type="image/svg+xml" href="/vite.svg" />
+    <meta name="viewport" content="width=device-width, initial-scale=1.0" />
+    <title>meta-dup - Duplicate Detection</title>
+  </head>
+  <body>
+    <div id="root"></div>
+    <script type="module" src="/src/main.tsx"></script>
+  </body>
+</html>

packages/meta-dup-ui/package.json

@@ -0,0 +1,24 @@
+{
+  "name": "@meta-dup/ui",
+  "version": "1.0.0",
+  "description": "Dashboard UI for meta-dup",
+  "type": "module",
+  "scripts": {
+    "dev": "vite",
+    "build": "vite build",
+    "preview": "vite preview",
+    "lint": "eslint src --ext ts,tsx"
+  },
+  "license": "MIT",
+  "dependencies": {
+    "react": "^18.2.0",
+    "react-dom": "^18.2.0"
+  },
+  "devDependencies": {
+    "@types/react": "^18.2.55",
+    "@types/react-dom": "^18.2.19",
+    "@vitejs/plugin-react": "^4.2.1",
+    "typescript": "^5.3.3",
+    "vite": "^5.0.12"
+  }
+}

packages/meta-dup-ui/src/App.tsx

@@ -0,0 +1,653 @@
+import { useState, useEffect, useCallback } from 'react';
+import type { DuplicateData, DuplicateGroup, HealthStatus, FileInfo } from './types';
+import { formatNumber, formatDate, getFilename, truncateHash, formatBytes } from './utils/format';
+import ServiceNav from './components/ServiceNav';
+
+function App() {
+    const [duplicateData, setDuplicateData] = useState<DuplicateData | null>(null);
+    const [health, setHealth] = useState<HealthStatus | null>(null);
+    const [loading, setLoading] = useState(false);
+    const [error, setError] = useState<string | null>(null);
+    const [activeTab, setActiveTab] = useState<'hash' | 'title'>('hash');
+    const [expandedGroups, setExpandedGroups] = useState<Set<string>>(new Set());
+    const [rebuilding, setRebuilding] = useState(false);
+
+    const fetchDuplicates = useCallback(async () => {
+        try {
+            const res = await fetch('/api/duplicates');
+            if (res.ok) {
+                const data = await res.json();
+                setDuplicateData(data);
+                setError(null);
+            } else {
+                setError('Failed to fetch duplicate data');
+            }
+        } catch (err) {
+            console.error('Failed to fetch duplicates:', err);
+            setError('Failed to fetch duplicate data');
+        }
+    }, []);
+
+    const fetchHealth = useCallback(async () => {
+        try {
+            const res = await fetch('/health');
+            if (res.ok) {
+                const data = await res.json();
+                setHealth(data);
+            }
+        } catch (err) {
+            console.error('Failed to fetch health:', err);
+        }
+    }, []);
+
+    const rebuildIndex = async () => {
+        setRebuilding(true);
+        try {
+            const res = await fetch('/api/duplicates/rebuild', { method: 'POST' });
+            if (res.ok) {
+                await fetchDuplicates();
+            } else {
+                setError('Failed to rebuild index');
+            }
+        } catch (err) {
+            console.error('Failed to rebuild:', err);
+            setError('Failed to rebuild index');
+        } finally {
+            setRebuilding(false);
+        }
+    };
+
+    useEffect(() => {
+        fetchDuplicates();
+        fetchHealth();
+        const interval = setInterval(() => {
+            fetchDuplicates();
+            fetchHealth();
+        }, 10000); // Refresh every 10s
+        return () => clearInterval(interval);
+    }, [fetchDuplicates, fetchHealth]);
+
+    const toggleGroup = (key: string) => {
+        setExpandedGroups(prev => {
+            const next = new Set(prev);
+            if (next.has(key)) {
+                next.delete(key);
+            } else {
+                next.add(key);
+            }
+            return next;
+        });
+    };
+
+    const expandAll = () => {
+        const groups = activeTab === 'hash'
+            ? duplicateData?.hashDuplicates
+            : duplicateData?.titleDuplicates;
+        if (groups) {
+            setExpandedGroups(new Set(groups.map(g => g.key)));
+        }
+    };
+
+    const collapseAll = () => {
+        setExpandedGroups(new Set());
+    };
+
+    const renderFileInfo = (file: FileInfo, idx: number) => (
+        <div key={file.hashId} className="file-item">
+            <span className="file-index">{idx + 1}.</span>
+            <span className="file-name" title={file.filePath}>{getFilename(file.filePath)}</span>
+            <span className="file-path" title={file.filePath}>{file.filePath}</span>
+            {file.sizeByte && (
+                <span className="file-size">{formatBytes(file.sizeByte)}</span>
+            )}
+        </div>
+    );
+
+    const renderDuplicateGroup = (group: DuplicateGroup, type: 'hash' | 'title') => {
+        const isExpanded = expandedGroups.has(group.key);
+        const displayKey = type === 'hash'
+            ? truncateHash(group.key, 16)
+            : group.key;
+
+        return (
+            <div key={group.key} className={`duplicate-group ${isExpanded ? 'expanded' : ''}`}>
+                <div
+                    className="group-header"
+                    onClick={() => toggleGroup(group.key)}
+                >
+                    <span className="expand-icon">{isExpanded ? '▼' : '▶'}</span>
+                    <span className="group-key" title={group.key}>{displayKey}</span>
+                    <span className="group-count">{group.files.length} files</span>
+                </div>
+                {isExpanded && (
+                    <div className="group-files">
+                        {group.files.map((file, idx) => renderFileInfo(file, idx))}
+                    </div>
+                )}
+            </div>
+        );
+    };
+
+    const hashDuplicates = duplicateData?.hashDuplicates || [];
+    const titleDuplicates = duplicateData?.titleDuplicates || [];
+    const stats = duplicateData?.stats;
+
+    return (
+        <div className="app">
+            <header className="header">
+                <h1>meta-dup</h1>
+                <span className="subtitle">Duplicate Detection Service</span>
+                {health && (
+                    <span className={`status-badge ${health.status}`}>
+                        {health.status === 'ok' ? '● Connected' : '○ Degraded'}
+                    </span>
+                )}
+            </header>
+
+            <main className="main">
+                <ServiceNav />
+                {error && (
+                    <div className="error-banner">{error}</div>
+                )}
+
+                {/* Stats Cards */}
+                {stats && (
+                    <div className="stats-cards">
+                        <div className="stat-card hash">
+                            <div className="stat-icon">#</div>
+                            <div className="stat-content">
+                                <div className="stat-value">{formatNumber(stats.hashGroupCount)}</div>
+                                <div className="stat-label">Hash Duplicate Groups</div>
+                                <div className="stat-detail">{formatNumber(stats.hashFileCount)} total files</div>
+                            </div>
+                        </div>
+                        <div className="stat-card title">
+                            <div className="stat-icon">T</div>
+                            <div className="stat-content">
+                                <div className="stat-value">{formatNumber(stats.titleGroupCount)}</div>
+                                <div className="stat-label">Title Duplicate Groups</div>
+                                <div className="stat-detail">{formatNumber(stats.titleFileCount)} total files</div>
+                            </div>
+                        </div>
+                        <div className="stat-card info">
+                            <div className="stat-icon">i</div>
+                            <div className="stat-content">
+                                <div className="stat-value">{formatNumber(stats.totalFilesTracked)}</div>
+                                <div className="stat-label">Total Files Tracked</div>
+                                <div className="stat-detail">
+                                    Last updated: {formatDate(stats.lastUpdated)}
+                                </div>
+                            </div>
+                        </div>
+                    </div>
+                )}
+
+                {/* Action Bar */}
+                <div className="action-bar">
+                    <button
+                        className="btn btn-primary"
+                        onClick={rebuildIndex}
+                        disabled={rebuilding}
+                    >
+                        {rebuilding ? 'Rebuilding...' : 'Rebuild Index'}
+                    </button>
+                </div>
+
+                {/* Tabs */}
+                <div className="card duplicates-card">
+                    <div className="tabs-header">
+                        <div className="tabs">
+                            <button
+                                className={`tab ${activeTab === 'hash' ? 'active' : ''}`}
+                                onClick={() => setActiveTab('hash')}
+                            >
+                                Hash Duplicates ({hashDuplicates.length})
+                            </button>
+                            <button
+                                className={`tab ${activeTab === 'title' ? 'active' : ''}`}
+                                onClick={() => setActiveTab('title')}
+                            >
+                                Title Duplicates ({titleDuplicates.length})
+                            </button>
+                        </div>
+                        <div className="tab-actions">
+                            <button className="btn btn-small btn-secondary" onClick={expandAll}>
+                                Expand All
+                            </button>
+                            <button className="btn btn-small btn-secondary" onClick={collapseAll}>
+                                Collapse All
+                            </button>
+                        </div>
+                    </div>
+
+                    <div className="tab-content">
+                        {activeTab === 'hash' && (
+                            <div className="duplicates-list">
+                                {hashDuplicates.length === 0 ? (
+                                    <div className="empty-state">
+                                        <p>No hash duplicates found</p>
+                                        <p className="hint">Files with identical content (same SHA-256 hash)</p>
+                                    </div>
+                                ) : (
+                                    <>
+                                        <p className="list-description">
+                                            Files with identical content (exact byte-for-byte matches)
+                                        </p>
+                                        {hashDuplicates.map(group => renderDuplicateGroup(group, 'hash'))}
+                                    </>
+                                )}
+                            </div>
+                        )}
+
+                        {activeTab === 'title' && (
+                            <div className="duplicates-list">
+                                {titleDuplicates.length === 0 ? (
+                                    <div className="empty-state">
+                                        <p>No title duplicates found</p>
+                                        <p className="hint">Files with similar parsed titles</p>
+                                    </div>
+                                ) : (
+                                    <>
+                                        <p className="list-description">
+                                            Files with matching parsed titles (may be different quality/releases)
+                                        </p>
+                                        {titleDuplicates.map(group => renderDuplicateGroup(group, 'title'))}
+                                    </>
+                                )}
+                            </div>
+                        )}
+                    </div>
+                </div>
+            </main>
+
+            <style>{`
+                :root {
+                    --bg-primary: #0d1117;
+                    --bg-secondary: #161b22;
+                    --bg-tertiary: #21262d;
+                    --text-primary: #f0f6fc;
+                    --text-secondary: #8b949e;
+                    --border-color: #30363d;
+                    --accent-primary: #58a6ff;
+                    --accent-green: #3fb950;
+                    --error: #f85149;
+                }
+
+                * {
+                    margin: 0;
+                    padding: 0;
+                    box-sizing: border-box;
+                }
+
+                body {
+                    font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
+                    background: var(--bg-primary);
+                    color: var(--text-primary);
+                    min-height: 100vh;
+                }
+
+                .app {
+                    min-height: 100vh;
+                    display: flex;
+                    flex-direction: column;
+                }
+
+                .header {
+                    background: var(--bg-secondary);
+                    border-bottom: 1px solid var(--border-color);
+                    padding: 16px 24px;
+                    display: flex;
+                    align-items: center;
+                    gap: 16px;
+                }
+
+                .header h1 {
+                    font-size: 1.5rem;
+                    font-weight: 600;
+                }
+
+                .subtitle {
+                    color: var(--text-secondary);
+                    font-size: 0.9rem;
+                }
+
+                .status-badge {
+                    margin-left: auto;
+                    padding: 4px 12px;
+                    border-radius: 12px;
+                    font-size: 0.85rem;
+                }
+
+                .status-badge.ok {
+                    background: rgba(63, 185, 80, 0.1);
+                    color: var(--accent-green);
+                }
+
+                .status-badge.degraded {
+                    background: rgba(248, 81, 73, 0.1);
+                    color: var(--error);
+                }
+
+                .main {
+                    flex: 1;
+                    padding: 24px;
+                    max-width: 1400px;
+                    margin: 0 auto;
+                    width: 100%;
+                }
+
+                .error-banner {
+                    background: rgba(248, 81, 73, 0.1);
+                    border: 1px solid var(--error);
+                    color: var(--error);
+                    padding: 12px 16px;
+                    border-radius: 8px;
+                    margin-bottom: 20px;
+                }
+
+                /* Stats Cards */
+                .stats-cards {
+                    display: grid;
+                    grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
+                    gap: 16px;
+                    margin-bottom: 24px;
+                }
+
+                .stat-card {
+                    background: var(--bg-secondary);
+                    border-radius: 12px;
+                    padding: 20px;
+                    display: flex;
+                    gap: 16px;
+                    align-items: center;
+                    border: 1px solid var(--border-color);
+                }
+
+                .stat-card.hash {
+                    border-left: 4px solid #a78bfa;
+                }
+
+                .stat-card.title {
+                    border-left: 4px solid #fbbf24;
+                }
+
+                .stat-card.info {
+                    border-left: 4px solid #60a5fa;
+                }
+
+                .stat-icon {
+                    width: 48px;
+                    height: 48px;
+                    background: var(--bg-tertiary);
+                    border-radius: 12px;
+                    display: flex;
+                    align-items: center;
+                    justify-content: center;
+                    font-size: 1.5rem;
+                    font-weight: bold;
+                    color: var(--text-secondary);
+                }
+
+                .stat-card.hash .stat-icon { color: #a78bfa; }
+                .stat-card.title .stat-icon { color: #fbbf24; }
+                .stat-card.info .stat-icon { color: #60a5fa; }
+
+                .stat-content { flex: 1; }
+
+                .stat-value {
+                    font-size: 1.8rem;
+                    font-weight: 700;
+                    color: var(--text-primary);
+                }
+
+                .stat-label {
+                    color: var(--text-secondary);
+                    font-size: 0.9rem;
+                }
+
+                .stat-detail {
+                    color: var(--text-secondary);
+                    font-size: 0.8rem;
+                    margin-top: 4px;
+                    opacity: 0.7;
+                }
+
+                /* Action Bar */
+                .action-bar {
+                    display: flex;
+                    gap: 12px;
+                    margin-bottom: 20px;
+                }
+
+                /* Buttons */
+                .btn {
+                    padding: 10px 20px;
+                    border: none;
+                    border-radius: 8px;
+                    font-size: 1rem;
+                    cursor: pointer;
+                    transition: all 0.2s;
+                }
+
+                .btn-primary {
+                    background: var(--accent-primary);
+                    color: white;
+                }
+
+                .btn-primary:hover:not(:disabled) {
+                    background: #4393e6;
+                }
+
+                .btn-primary:disabled {
+                    opacity: 0.5;
+                    cursor: not-allowed;
+                }
+
+                .btn-secondary {
+                    background: var(--bg-tertiary);
+                    color: var(--text-primary);
+                    border: 1px solid var(--border-color);
+                }
+
+                .btn-secondary:hover {
+                    background: var(--bg-secondary);
+                }
+
+                .btn-small {
+                    padding: 6px 12px;
+                    font-size: 0.85rem;
+                }
+
+                /* Card */
+                .card {
+                    background: var(--bg-secondary);
+                    border-radius: 12px;
+                    border: 1px solid var(--border-color);
+                    padding: 20px;
+                }
+
+                /* Tabs */
+                .duplicates-card {
+                    min-height: 400px;
+                }
+
+                .tabs-header {
+                    display: flex;
+                    justify-content: space-between;
+                    align-items: center;
+                    margin-bottom: 20px;
+                    border-bottom: 1px solid var(--border-color);
+                    padding-bottom: 12px;
+                }
+
+                .tabs {
+                    display: flex;
+                    gap: 8px;
+                }
+
+                .tab {
+                    padding: 10px 20px;
+                    background: transparent;
+                    border: none;
+                    color: var(--text-secondary);
+                    font-size: 1rem;
+                    cursor: pointer;
+                    border-radius: 8px 8px 0 0;
+                    transition: all 0.2s;
+                }
+
+                .tab:hover {
+                    color: var(--text-primary);
+                    background: var(--bg-tertiary);
+                }
+
+                .tab.active {
+                    color: var(--accent-primary);
+                    background: var(--bg-tertiary);
+                    font-weight: 600;
+                }
+
+                .tab-actions {
+                    display: flex;
+                    gap: 8px;
+                }
+
+                /* Duplicates List */
+                .list-description {
+                    color: var(--text-secondary);
+                    font-size: 0.9rem;
+                    margin-bottom: 16px;
+                }
+
+                .duplicates-list {
+                    display: flex;
+                    flex-direction: column;
+                    gap: 8px;
+                }
+
+                .duplicate-group {
+                    background: var(--bg-tertiary);
+                    border-radius: 8px;
+                    overflow: hidden;
+                }
+
+                .group-header {
+                    display: flex;
+                    align-items: center;
+                    gap: 12px;
+                    padding: 12px 16px;
+                    cursor: pointer;
+                    transition: background 0.2s;
+                }
+
+                .group-header:hover {
+                    background: var(--bg-secondary);
+                }
+
+                .expand-icon {
+                    color: var(--text-secondary);
+                    font-size: 0.8rem;
+                    width: 16px;
+                }
+
+                .group-key {
+                    flex: 1;
+                    font-family: monospace;
+                    font-size: 0.9rem;
+                    color: var(--accent-primary);
+                }
+
+                .group-count {
+                    background: var(--bg-secondary);
+                    padding: 4px 10px;
+                    border-radius: 12px;
+                    font-size: 0.8rem;
+                    color: var(--text-secondary);
+                }
+
+                .group-files {
+                    background: var(--bg-secondary);
+                    padding: 8px 16px;
+                    border-top: 1px solid var(--border-color);
+                }
+
+                .file-item {
+                    display: grid;
+                    grid-template-columns: 30px 200px 1fr auto;
+                    gap: 12px;
+                    padding: 8px 0;
+                    border-bottom: 1px solid var(--border-color);
+                    font-size: 0.85rem;
+                    align-items: center;
+                }
+
+                .file-item:last-child {
+                    border-bottom: none;
+                }
+
+                .file-index {
+                    color: var(--text-secondary);
+                    text-align: right;
+                }
+
+                .file-name {
+                    font-weight: 500;
+                    overflow: hidden;
+                    text-overflow: ellipsis;
+                    white-space: nowrap;
+                }
+
+                .file-path {
+                    color: var(--text-secondary);
+                    font-family: monospace;
+                    font-size: 0.8rem;
+                    overflow: hidden;
+                    text-overflow: ellipsis;
+                    white-space: nowrap;
+                }
+
+                .file-size {
+                    color: var(--text-secondary);
+                    font-size: 0.8rem;
+                    white-space: nowrap;
+                }
+
+                /* Empty State */
+                .empty-state {
+                    text-align: center;
+                    padding: 60px 20px;
+                    color: var(--text-secondary);
+                }
+
+                .empty-state p {
+                    margin: 0;
+                }
+
+                .empty-state .hint {
+                    font-size: 0.9rem;
+                    margin-top: 8px;
+                    opacity: 0.7;
+                }
+
+                @media (max-width: 800px) {
+                    .tabs-header {
+                        flex-direction: column;
+                        gap: 12px;
+                        align-items: flex-start;
+                    }
+
+                    .file-item {
+                        grid-template-columns: 30px 1fr;
+                    }
+
+                    .file-path, .file-size {
+                        grid-column: 1 / -1;
+                        padding-left: 42px;
+                    }
+                }
+            `}</style>
+        </div>
+    );
+}
+
+export default App;

packages/meta-dup-ui/src/components/ServiceNav.tsx

@@ -0,0 +1,170 @@
+import { useState, useEffect } from 'react';
+
+interface ServiceInfo {
+  name: string;
+  url: string;
+  api: string;
+  status: string;
+  capabilities: string[];
+  version: string;
+  role?: string; // "leader", "follower", or undefined (for non-meta-core services)
+}
+
+interface ServicesResponse {
+  services: ServiceInfo[];
+  current: string;
+}
+
+const serviceIcons: Record<string, string> = {
+  'meta-sort': '📁',
+  'meta-fuse': '🗂️',
+  'meta-stremio': '🎬',
+  'meta-dup': '🔍',
+  'meta-orbit': '🌐',
+  'default': '📦'
+};
+
+function formatServiceName(name: string): string {
+  return name.split('-').map(word =>
+    word.charAt(0).toUpperCase() + word.slice(1)
+  ).join(' ');
+}
+
+function ServiceNav() {
+  const [services, setServices] = useState<ServiceInfo[]>([]);
+  const [loading, setLoading] = useState(true);
+  const currentService = 'meta-dup';
+
+  useEffect(() => {
+    const fetchServices = async () => {
+      try {
+        const response = await fetch('/api/services');
+        if (response.ok) {
+          const data: ServicesResponse = await response.json();
+          setServices(data.services || []);
+        }
+      } catch (error) {
+        console.error('Failed to fetch services:', error);
+      } finally {
+        setLoading(false);
+      }
+    };
+
+    fetchServices();
+    const interval = setInterval(fetchServices, 30000);
+    return () => clearInterval(interval);
+  }, []);
+
+  if (loading) return null;
+
+  // Filter out follower meta-core instances (only show leader)
+  const filteredServices = services.filter(
+    s => s.name !== 'meta-core' || s.role === 'leader'
+  );
+  const sortedServices = [...filteredServices].sort((a, b) => a.name.localeCompare(b.name));
+
+  if (sortedServices.length === 0) return null;
+
+  return (
+    <nav className="services-nav">
+      <span className="services-nav-label">Services:</span>
+      <div className="services-nav-items">
+        {sortedServices.map(service => {
+          const icon = serviceIcons[service.name] || serviceIcons.default;
+          const isActive = service.name === currentService;
+          return (
+            <a
+              key={service.name}
+              href={isActive ? '#' : service.url}
+              className={`service-link${isActive ? ' active' : ''}`}
+              onClick={(e) => {
+                e.preventDefault();
+                if (!isActive) {
+                  window.location.href = service.url;
+                }
+              }}
+            >
+              <span className="service-icon">{icon}</span>
+              <span>{formatServiceName(service.name)}</span>
+              <span className="service-status"></span>
+            </a>
+          );
+        })}
+      </div>
+
+      <style>{`
+        .services-nav {
+          background: var(--bg-secondary);
+          border-radius: 12px;
+          padding: 0.5rem;
+          margin-bottom: 1.5rem;
+          display: flex;
+          align-items: center;
+          gap: 0.5rem;
+          flex-wrap: wrap;
+          border: 1px solid var(--border-color);
+        }
+
+        .services-nav-label {
+          color: var(--text-secondary);
+          font-size: 0.85rem;
+          padding: 0.5rem 0.75rem;
+          white-space: nowrap;
+        }
+
+        .services-nav-items {
+          display: flex;
+          gap: 0.5rem;
+          flex-wrap: wrap;
+        }
+
+        .service-link {
+          display: inline-flex;
+          align-items: center;
+          gap: 0.4rem;
+          padding: 0.5rem 1rem;
+          border-radius: 8px;
+          text-decoration: none;
+          font-size: 0.9rem;
+          font-weight: 500;
+          transition: all 0.2s;
+          color: var(--text-primary);
+          background: var(--bg-tertiary);
+          border: 1px solid var(--border-color);
+        }
+
+        .service-link:hover {
+          background: var(--border-color);
+          border-color: rgba(78, 205, 196, 0.5);
+          text-decoration: none;
+        }
+
+        .service-link.active {
+          background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
+          border-color: rgba(78, 205, 196, 0.5);
+        }
+
+        .service-link .service-icon {
+          font-size: 1.1rem;
+        }
+
+        .service-link .service-status {
+          width: 6px;
+          height: 6px;
+          border-radius: 50%;
+          background: var(--accent-primary);
+          margin-left: 0.25rem;
+        }
+
+        @media (max-width: 600px) {
+          .services-nav {
+            flex-direction: column;
+            align-items: flex-start;
+          }
+        }
+      `}</style>
+    </nav>
+  );
+}
+
+export default ServiceNav;

packages/meta-dup-ui/src/main.tsx

@@ -0,0 +1,9 @@
+import React from 'react';
+import ReactDOM from 'react-dom/client';
+import App from './App';
+
+ReactDOM.createRoot(document.getElementById('root')!).render(
+  <React.StrictMode>
+    <App />
+  </React.StrictMode>
+);

packages/meta-dup-ui/src/types/index.ts

@@ -0,0 +1,48 @@
+/**
+ * Type definitions for meta-dup-ui
+ */
+
+export interface FileInfo {
+    hashId: string;
+    filePath: string;
+    title?: string;
+    sha256?: string;
+    sizeByte?: number;
+    year?: number;
+    season?: number;
+    episode?: number;
+    fileType?: string;
+}
+
+export interface DuplicateGroup {
+    key: string;
+    files: FileInfo[];
+}
+
+export interface DuplicateStats {
+    hashGroupCount: number;
+    hashFileCount: number;
+    titleGroupCount: number;
+    titleFileCount: number;
+    totalFilesTracked: number;
+    lastUpdated: string;
+}
+
+export interface DuplicateData {
+    hashDuplicates: DuplicateGroup[];
+    titleDuplicates: DuplicateGroup[];
+    stats: DuplicateStats;
+    computedAt: string;
+}
+
+export interface HealthStatus {
+    status: 'ok' | 'degraded';
+    service: string;
+    redis: boolean;
+    stats: {
+        filesTracked: number;
+        hashDuplicates: number;
+        titleDuplicates: number;
+    };
+    timestamp: string;
+}

packages/meta-dup-ui/src/utils/format.ts

@@ -0,0 +1,28 @@
+/**
+ * Formatting utilities
+ */
+
+export function formatNumber(num: number): string {
+    return num.toLocaleString();
+}
+
+export function formatBytes(bytes: number): string {
+    if (bytes === 0) return '0 B';
+    const k = 1024;
+    const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
+    const i = Math.floor(Math.log(bytes) / Math.log(k));
+    return parseFloat((bytes / Math.pow(k, i)).toFixed(2)) + ' ' + sizes[i];
+}
+
+export function formatDate(dateStr: string): string {
+    return new Date(dateStr).toLocaleString();
+}
+
+export function getFilename(path: string): string {
+    return path.split('/').pop() || path;
+}
+
+export function truncateHash(hash: string, length: number = 16): string {
+    if (hash.length <= length) return hash;
+    return `${hash.substring(0, length)}...`;
+}

packages/meta-dup-ui/tsconfig.json

@@ -0,0 +1,23 @@
+{
+  "compilerOptions": {
+    "target": "ES2022",
+    "useDefineForClassFields": true,
+    "lib": ["ES2022", "DOM", "DOM.Iterable"],
+    "module": "ESNext",
+    "skipLibCheck": true,
+    "moduleResolution": "bundler",
+    "allowImportingTsExtensions": true,
+    "resolveJsonModule": true,
+    "isolatedModules": true,
+    "noEmit": true,
+    "jsx": "react-jsx",
+    "strict": true,
+    "noUnusedLocals": true,
+    "noUnusedParameters": true,
+    "noFallthroughCasesInSwitch": true,
+    "allowSyntheticDefaultImports": true,
+    "esModuleInterop": true
+  },
+  "include": ["src"],
+  "references": [{ "path": "./tsconfig.node.json" }]
+}

packages/meta-dup-ui/tsconfig.node.json

@@ -0,0 +1,10 @@
+{
+  "compilerOptions": {
+    "composite": true,
+    "skipLibCheck": true,
+    "module": "ESNext",
+    "moduleResolution": "bundler",
+    "allowSyntheticDefaultImports": true
+  },
+  "include": ["vite.config.ts"]
+}

packages/meta-dup-ui/vite.config.ts

@@ -0,0 +1,23 @@
+import { defineConfig } from 'vite';
+import react from '@vitejs/plugin-react';
+
+export default defineConfig({
+  plugins: [react()],
+  server: {
+    port: 5175,
+    proxy: {
+      '/api': {
+        target: 'http://localhost:3000',
+        changeOrigin: true,
+      },
+      '/health': {
+        target: 'http://localhost:3000',
+        changeOrigin: true,
+      },
+    },
+  },
+  build: {
+    outDir: 'dist',
+    sourcemap: true,
+  },
+});

pnpm-workspace.yaml

@@ -0,0 +1,2 @@
+packages:
+  - 'packages/*'