metastar/meta-sort
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

meta-sort cleanup, add editor, remove duplicate

Browse Source
This commit is contained in:
Pierre
2026-02-07 16:12:46 +01:00
parent 725d22c98f
commit 27a866a10f
42 changed files with 5892 additions and 955 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/meta-sort-core/src/api/DuplicatesAPI.ts
-166
View File
@@ -1,166 +0,0 @@
/**
* HTTP REST API Server for Duplicate Files
*
* Provides endpoints for accessing duplicate file detection results.
* Separate from FUSE API to maintain proper separation of concerns.
*/
import * as http from 'http';
export interface DuplicatesAPIConfig {
/** Port to listen on (default: 3003) */
port?: number;
/** Host to bind to (default: 'localhost') */
host?: string;
/** Enable CORS (default: true) */
enableCors?: boolean;
/** Mount path for duplicates endpoint (default: '/duplicates') */
mountPath?: string;
}
export class DuplicatesAPI {
private server: http.Server | null = null;
private config: Required<DuplicatesAPIConfig>;
constructor(config: DuplicatesAPIConfig = {}) {
this.config = {
port: config.port ?? 3003,
host: config.host ?? 'localhost',
enableCors: config.enableCors ?? true,
mountPath: config.mountPath ?? '/duplicates'
};
}
/**
* Start the HTTP server
*/
public async start(): Promise<void> {
return new Promise((resolve, reject) => {
this.server = http.createServer(this.handleRequest.bind(this));
this.server.on('error', reject);
this.server.listen(this.config.port, this.config.host, () => {
console.log(`Duplicates API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`);
resolve();
});
});
}
/**
* Stop the HTTP server
*/
public async stop(): Promise<void> {
if (!this.server) {
return;
}
return new Promise((resolve, reject) => {
this.server!.close((err) => {
if (err) {
reject(err);
} else {
console.log('Duplicates API stopped');
this.server = null;
resolve();
}
});
});
}
/**
* Handle incoming HTTP requests
*/
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
if (this.config.enableCors) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
}
try {
const url = new URL(req.url!, `http://${req.headers.host}`);
const pathname = url.pathname;
const method = req.method!;
// Route the request
if (method === 'GET' && pathname === this.config.mountPath) {
await this.handleDuplicates(res);
} else if (method === 'GET' && pathname === '/health') {
this.handleHealth(res);
} else {
this.sendError(res, 404, 'Not Found');
}
} catch (error) {
console.error('Error handling request:', error);
this.sendError(res, 500, 'Internal Server Error', error);
}
}
/**
* Send JSON response
*/
private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
res.writeHead(statusCode, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(data, null, 2));
}
/**
* Send error response
*/
private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
this.sendJson(res, {
error: message,
details: error?.message ?? error
}, statusCode);
}
/**
* GET /duplicates
* Get duplicate files
* NOTE: This API is deprecated. Use UnifiedAPIServer's /api/duplicates endpoint instead.
* Disk-based duplicate logging has been removed for performance.
*/
private async handleDuplicates(res: http.ServerResponse): Promise<void> {
try {
// Return empty results - duplicate detection now happens in-memory via UnifiedAPIServer
const hashDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
const titleDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
this.sendJson(res, {
hashDuplicates,
titleDuplicates,
stats: {
hashGroupCount: 0,
hashFileCount: 0,
titleGroupCount: 0,
titleFileCount: 0
},
deprecated: true,
message: 'This API is deprecated. Use UnifiedAPIServer /api/duplicates endpoint instead.'
});
} catch (error) {
console.error('Error in duplicates endpoint:', error);
this.sendError(res, 500, 'Failed to get duplicate data', error);
}
}
/**
* GET /health
* Health check endpoint
*/
private handleHealth(res: http.ServerResponse): void {
this.sendJson(res, {
status: 'ok',
service: 'duplicates-api',
timestamp: new Date().toISOString()
});
}
}
packages/meta-sort-core/src/api/UnifiedAPIServer.ts
+3 -61
View File
@@ -10,7 +10,6 @@ import fastifyCors from '@fastify/cors';
import { VirtualFileSystem } from './VirtualFileSystem.js';
import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
import { UnifiedProcessingStateManager } from '../logic/UnifiedProcessingStateManager.js';
import { DuplicateResult } from '../logic/DuplicateFinder.js';
import type { IKVClient } from '../kv/IKVClient.js';
import type { KVManager } from '../kv/KVManager.js';
import type { ExtendedProcessingSnapshot, IKVClientWithPubSub } from '../types/ExtendedInterfaces.js';
@@ -35,7 +34,6 @@ export class UnifiedAPIServer {
private app: FastifyInstance;
private vfs: VirtualFileSystem;
private unifiedStateManager: UnifiedProcessingStateManager | null = null;
private getDuplicateResult: (() => DuplicateResult | null) | null = null;
private kvClient: IKVClient | null = null;
private kvManager: KVManager | null = null;
private getQueueStatus: (() => any) | null = null;
@@ -56,7 +54,6 @@ export class UnifiedAPIServer {
vfs: VirtualFileSystem,
config: UnifiedAPIServerConfig = {},
unifiedStateManager?: UnifiedProcessingStateManager,
getDuplicateResult?: () => DuplicateResult | null,
kvClient?: IKVClient,
backgroundQueueConcurrency?: number,
fastQueueConcurrency?: number,
@@ -67,7 +64,6 @@ export class UnifiedAPIServer {
) {
this.vfs = vfs;
this.unifiedStateManager = unifiedStateManager || null;
this.getDuplicateResult = getDuplicateResult || null;
this.kvClient = kvClient || null;
this.kvManager = kvManager || null;
this.getQueueStatus = getQueueStatus || null;
@@ -161,7 +157,6 @@ export class UnifiedAPIServer {
services: {
fuse: 'ok',
metrics: 'ok',
duplicates: 'ok',
unifiedProcessing: this.unifiedStateManager ? 'ok' : 'unavailable'
}
};
@@ -187,8 +182,7 @@ export class UnifiedAPIServer {
// Metrics API routes (under /api/metrics)
this.setupMetricsRoutes();
// Duplicates API routes (under /api/duplicates)
this.setupDuplicatesRoutes();
// NOTE: Duplicates API has been moved to meta-dup service (port 8183)
// NOTE: Metadata API routes (/api/metadata/*) and KV Browser routes (/api/kv/*)
// have been moved to meta-core Go API. The editor UI now uses meta-core's endpoints.
@@ -492,59 +486,8 @@ export class UnifiedAPIServer {
});
}
/**
* Setup Duplicates API routes
*/
private setupDuplicatesRoutes(): void {
// Get duplicate files (from stored duplicate data)
this.app.get('/api/duplicates', async (request, reply) => {
try {
// Get stored duplicate result from WatchedFileProcessor
const duplicateResult = this.getDuplicateResult ? this.getDuplicateResult() : null;
if (!duplicateResult) {
// No duplicates found yet (processing hasn't completed)
return {
hashDuplicates: [],
titleDuplicates: [],
stats: {
hashGroupCount: 0,
hashFileCount: 0,
titleGroupCount: 0,
titleFileCount: 0
}
};
}
return {
hashDuplicates: duplicateResult.hashDuplicates,
titleDuplicates: duplicateResult.titleDuplicates,
stats: {
hashGroupCount: duplicateResult.hashDuplicates.length,
hashFileCount: duplicateResult.hashDuplicates.reduce((sum, g) => sum + g.files.length, 0),
titleGroupCount: duplicateResult.titleDuplicates.length,
titleFileCount: duplicateResult.titleDuplicates.reduce((sum, g) => sum + g.files.length, 0)
}
};
} catch (error) {
console.error('Error retrieving duplicates:', error);
return reply.status(500).send({
error: 'Failed to retrieve duplicates',
details: (error as Error).message
});
}
});
// Refresh duplicates (trigger re-computation)
this.app.post('/api/duplicates/refresh', async (request, reply) => {
// Currently just acknowledges the request
// The duplicate detection runs automatically during processing
return {
status: 'ok',
message: 'Duplicate refresh triggered'
};
});
}
// NOTE: Duplicates API has been moved to meta-dup service (port 8183)
// Use http://localhost:8183/api/duplicates for duplicate detection
// NOTE: setupMetadataRoutes() and setupKVBrowserRoutes() have been removed.
// The metadata editor API (/api/metadata/*) and KV browser API (/api/kv/*)
@@ -1905,7 +1848,6 @@ export class UnifiedAPIServer {
console.log(`Unified API Server listening on http://${this.config.host}:${this.config.port}`);
console.log(` - FUSE API: http://${this.config.host}:${this.config.port}/api/fuse/*`);
console.log(` - Metrics API: http://${this.config.host}:${this.config.port}/api/metrics`);
console.log(` - Duplicates API: http://${this.config.host}:${this.config.port}/api/duplicates`);
if (this.kvClient) {
console.log(` - Metadata API: http://${this.config.host}:${this.config.port}/api/metadata/* (GET/PUT/DELETE/POST)`);
console.log(` - KV Browser API: http://${this.config.host}:${this.config.port}/api/kv/* (GET)`);
packages/meta-sort-core/src/api/api.test.ts
+3 -52
View File
@@ -9,7 +9,6 @@ import { expect } from 'chai';
import { UnifiedAPIServer } from './UnifiedAPIServer.js';
import { VirtualFileSystem } from './VirtualFileSystem.js';
import { UnifiedProcessingStateManager } from '../logic/UnifiedProcessingStateManager.js';
import { DuplicateResult } from '../logic/DuplicateFinder.js';
import type { IKVClient } from '../kv/IKVClient.js';
import type { FastifyInstance } from 'fastify';
@@ -205,25 +204,7 @@ class MockProcessingStateManager {
}
}
/**
* Mock Duplicate Result Provider
*/
function createMockDuplicateResult(): DuplicateResult {
return {
hashDuplicates: [
{
key: 'abc123',
files: ['/files/movie1.mp4', '/files/movie1-copy.mp4']
}
],
titleDuplicates: [
{
key: 'Movie Title',
files: ['/files/movie1.mp4', '/files/movie2.mp4']
}
]
};
}
// NOTE: Duplicates API has been moved to meta-dup service (port 8183)
// =============================================================================
// Test Suite
@@ -234,7 +215,6 @@ describe('UnifiedAPIServer', function() {
let app: FastifyInstance;
let mockKV: MockKVClient;
let mockStateManager: MockProcessingStateManager;
let duplicateResult = createMockDuplicateResult();
before(async function() {
// Create mocks
@@ -268,7 +248,6 @@ describe('UnifiedAPIServer', function() {
vfs,
{ port: 3000, host: 'localhost', enableCors: true },
mockStateManager as any,
() => duplicateResult,
mockKV as any,
4, // backgroundQueueConcurrency
16, // fastQueueConcurrency
@@ -396,36 +375,8 @@ describe('UnifiedAPIServer', function() {
});
});
// =========================================================================
// Duplicates API (Duplicates Page)
// =========================================================================
describe('Duplicates API (/api/duplicates)', function() {
it('GET /api/duplicates returns duplicate analysis', async function() {
const response = await app.inject({
method: 'GET',
url: '/api/duplicates'
});
expect(response.statusCode).to.equal(200);
const body = JSON.parse(response.payload);
expect(body).to.have.property('hashDuplicates');
expect(body).to.have.property('titleDuplicates');
expect(Array.isArray(body.hashDuplicates)).to.be.true;
expect(Array.isArray(body.titleDuplicates)).to.be.true;
});
it('POST /api/duplicates/refresh acknowledges request', async function() {
const response = await app.inject({
method: 'POST',
url: '/api/duplicates/refresh'
});
expect(response.statusCode).to.equal(200);
const body = JSON.parse(response.payload);
expect(body).to.have.property('status');
});
});
// NOTE: Duplicates API has been moved to meta-dup service (port 8183)
// See dev/test/suites/meta-dup.bats for duplicate detection tests
// =========================================================================
// Stats API (Dashboard)
packages/meta-sort-core/src/index.ts
+1 -2
View File
@@ -74,7 +74,6 @@ fileProcessor.setPipeline(pipeline);
// VFS and state manager are available immediately
const vfs = fileProcessor.getVirtualFileSystem();
const unifiedStateManager = fileProcessor.getUnifiedStateManager();
const getDuplicateResult = () => fileProcessor.getDuplicateResult();
// API Server will be initialized after KV is ready (deferred to async init)
let apiServer: UnifiedAPIServer | null = null;
@@ -164,7 +163,7 @@ process.on('SIGINT', async () => {
port: config.FUSE_API_PORT,
host: config.FUSE_API_HOST,
enableCors: true,
}, unifiedStateManager, getDuplicateResult, kvClient || undefined, backgroundQueueConcurrency, fastQueueConcurrency, () => fileProcessor.getQueueStatus(), kvManager || undefined, getPluginManagerInstance, () => fileProcessor.getTaskScheduler());
}, unifiedStateManager, kvClient || undefined, backgroundQueueConcurrency, fastQueueConcurrency, () => fileProcessor.getQueueStatus(), kvManager || undefined, getPluginManagerInstance, () => fileProcessor.getTaskScheduler());
await apiServer.start();
console.log('Unified API Server started successfully');
packages/meta-sort-core/src/logic/DuplicateFinder.ts
-113
View File
@@ -1,113 +0,0 @@
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
import {VideoMeta, HashMeta, VersionMeta, FileNameMeta, FileStatMeta} from "@metazla/meta-interface";
import {renamingRule} from "../config/RenamingRule.js";
type Local = VideoMeta & HashMeta & VersionMeta & FileNameMeta & FileStatMeta;
export interface DuplicateGroup {
key: string;
files: string[];
virtualPath?: string;
}
export interface DuplicateResult {
hashDuplicates: DuplicateGroup[];
titleDuplicates: DuplicateGroup[];
}
export class DuplicateFinder {
async findDuplicates(database: Map<string, Local>): Promise<DuplicateResult> {
let [hashDuplicates, titleDuplicates] = await Promise.all([
this.findDuplicatesGeneric(database, this.hashKeyExtractor, 'remove'),
this.findDuplicatesGeneric(database, this.pathExtractor, 'version')
]);
return {
hashDuplicates,
titleDuplicates
};
}
private hashKeyExtractor(metadata: Local): string | null {
return metadata[CID_ALGORITHM_NAMES.sha256] ?? null;
}
private pathExtractor(metadata: Local): string | null {
//case insensitive
try {
return renamingRule(metadata, '')?.toLowerCase() ?? null;
} catch (e) {
return null;
}
}
private async findDuplicatesGeneric(
database: Map<string, Local>,
keyExtractor: (metadata: Local) => string | null,
action?: 'remove' | 'version'
): Promise<DuplicateGroup[]> {
const duplicates = new Map<string, string[]>();
const duplicateGroups: DuplicateGroup[] = [];
for (const [filePath, metadata] of database) {
const key = keyExtractor(metadata);
if (key) {
const files = duplicates.get(key) ?? [];
files.push(filePath);
duplicates.set(key, files);
}
}
for (const [key, files] of duplicates) {
if (files.length > 1) {
// Store duplicate group before modifying database
const sortedFiles = [...files].sort();
// Generate virtualPath for both hash and title duplicates
let virtualPath: string | undefined;
if (action === 'version') {
// For title duplicates, use the key (normalized title path)
virtualPath = key;
} else if (action === 'remove') {
// For hash duplicates, use renamingRule on the first file's metadata
const firstFile = sortedFiles[0];
const metadata = database.get(firstFile);
if (metadata) {
try {
virtualPath = renamingRule(metadata, '');
} catch (e) {
// If renamingRule fails, virtualPath remains undefined
}
}
}
duplicateGroups.push({
key,
files: sortedFiles,
virtualPath
});
//alphabetical order
const remainingFiles = files.slice(1).sort();
//rename all problematic entries V2, V3, V4, V5
let i = 2;
for (const file of remainingFiles) {
if (action === 'remove') {
database.delete(file);
} else if (action === 'version') {
let definition = database.get(file);
definition.version = "V" + i;
if (definition.fileType !== "subtitle") {//allow multiple version of subtitle in the same folder
definition.extra = "true";//mark as extra so it don't pollute the main list
}
i++;
} else {
throw new Error();
}
}
}
}
return duplicateGroups;
}
}
packages/meta-sort-core/src/logic/WatchedFileProcessor.ts
+3 -20
View File
@@ -2,7 +2,6 @@ import {dirname, join, parse} from 'path';
import {config} from "../config/EnvConfig.js";
import {FileProcessorInterface} from "@metazla/meta-hash";
import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js";
import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js";
import {FileType} from "@metazla/filename-tools";
import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js";
import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js";
@@ -33,8 +32,6 @@ export class WatchedFileProcessor implements FileProcessorInterface {
virtualFileSystem: VirtualFileSystem;
unifiedStateManager = new UnifiedProcessingStateManager();
fileProcessor:FileAnalyzerInterface = new FileProcessorPiscina();
duplicateFinder = new DuplicateFinder();
duplicateResult: DuplicateResult | null = null;
fileType = new FileType();
supported = ['video','subtitle','torrent'];
@@ -77,10 +74,6 @@ export class WatchedFileProcessor implements FileProcessorInterface {
return this.unifiedStateManager;
}
getDuplicateResult(): DuplicateResult | null {
return this.duplicateResult;
}
/**
* Get the KV client
*/
@@ -415,20 +408,10 @@ export class WatchedFileProcessor implements FileProcessorInterface {
console.log(`No files to process`);
return;
}
// NOTE: Duplicate detection has been moved to meta-dup service
let startTime = performance.now();
const newDuplicateResult = await this.duplicateFinder.findDuplicates(this.fileProcessor.getDatabase());//find duplicates and removes them to avoid conflict
const duplicateTime = Math.ceil(performance.now() - startTime);
const totalDuplicates = newDuplicateResult.hashDuplicates.length + newDuplicateResult.titleDuplicates.length;
console.log(`Duplicate find took ${duplicateTime}ms - found ${newDuplicateResult.hashDuplicates.length} hash groups, ${newDuplicateResult.titleDuplicates.length} title groups`);
// Only update duplicateResult if we found new duplicates, or if it's the first run
if (totalDuplicates > 0 || this.duplicateResult === null) {
this.duplicateResult = newDuplicateResult;
}
performanceMetrics.recordDuplicateDetection(duplicateTime, totalDuplicates);
startTime = performance.now();
console.log(`Generating virtual structure for ${this.fileProcessor.getDatabase().size} files`);
const virtualStructure = this.metaDataToFolderStruct.generateVirtualStructure(this.fileProcessor.getDatabase());
const virtualStructureTime = Math.ceil(performance.now() - startTime);