From 2a7fba7a0be9c8b4f1df0900aac28c016f5ccb7d Mon Sep 17 00:00:00 2001 From: Pierre Date: Sun, 18 Jan 2026 16:48:06 +0100 Subject: [PATCH] UI and plugin fix --- .../src/api/UnifiedAPIServer.ts | 237 +++++++++- .../meta-sort-core/src/config/EnvConfig.ts | 18 +- .../src/container-plugins/ConfigParser.ts | 72 ++++ .../src/container-plugins/ContainerManager.ts | 227 ++++++++++ .../src/logic/WatchedFileProcessor.ts | 8 + .../fileProcessor/FileProcessorPiscina.ts | 6 + .../src/plugin-engine/PluginManager.ts | 41 +- packages/meta-sort-ui/src/pages/Monitor.tsx | 22 + packages/meta-sort-ui/src/pages/Plugins.tsx | 403 ++++++++++++++++++ .../src/lib/hash-compute/HashIndexManager.ts | 20 +- 10 files changed, 1039 insertions(+), 15 deletions(-) diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts index a24f1b6..ad252cd 100644 --- a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -1867,6 +1867,64 @@ export class UnifiedAPIServer { }); } }); + + // Clear all metadata and trigger rescan + this.app.post('/api/metadata/clear', async (request, reply) => { + if (!this.kvClient) { + return reply.status(503).send({ + status: 'error', + message: 'KV client not available' + }); + } + + try { + console.log('Clearing all metadata via API'); + + // Get all file keys and delete them + const hashIds = await this.kvClient.getAllHashIds(); + let deletedCount = 0; + + for (const hashId of hashIds) { + try { + await this.kvClient.delete(`/file/${hashId}`); + deletedCount++; + } catch (err) { + console.warn(`Failed to delete key /file/${hashId}:`, err); + } + } + + // Also delete the index key + try { + await this.kvClient.delete('/file/__index__'); + } catch (err) { + // Index might not exist + } + + console.log(`Cleared ${deletedCount} file metadata entries`); + + // Trigger rescan if callback is available + if (this.triggerScanCallback) { + console.log('Triggering rescan after metadata clear'); + this.triggerScanCallback().catch(error => { + console.error('Error during post-clear rescan:', error); + }); + } + + return { + status: 'ok', + message: `Cleared ${deletedCount} files`, + deletedCount, + rescanTriggered: !!this.triggerScanCallback + }; + } catch (error: any) { + console.error('Error clearing metadata:', error); + return reply.status(500).send({ + status: 'error', + message: 'Failed to clear metadata', + details: error.message + }); + } + }); } /** @@ -2083,7 +2141,7 @@ export class UnifiedAPIServer { } }); - // Clear plugin cache + // Clear plugin cache for a specific plugin this.app.post<{ Params: { pluginId: string } }>('/api/plugins/:pluginId/clear-cache', async (request, reply) => { const pluginManager = this.getPluginManager?.(); if (!pluginManager) { @@ -2099,6 +2157,55 @@ export class UnifiedAPIServer { }; }); + // Clear all plugin caches + this.app.post('/api/plugins/clear-cache', async (request, reply) => { + try { + const fs = await import('fs/promises'); + const fsSync = await import('fs'); + const path = await import('path'); + + const cacheDir = path.join(config.CACHE_FOLDER_PATH, 'plugin-cache'); + + let deletedFiles = 0; + let deletedDirs = 0; + + // Clear plugin cache directory + if (fsSync.existsSync(cacheDir)) { + const entries = await fs.readdir(cacheDir, { withFileTypes: true }); + for (const entry of entries) { + const entryPath = path.join(cacheDir, entry.name); + try { + if (entry.isDirectory()) { + await fs.rm(entryPath, { recursive: true, force: true }); + deletedDirs++; + } else { + await fs.unlink(entryPath); + deletedFiles++; + } + } catch (err) { + console.warn(`Failed to delete ${entryPath}:`, err); + } + } + } + + console.log(`Cleared plugin cache: ${deletedDirs} directories, ${deletedFiles} files`); + + return { + status: 'ok', + message: `Cleared plugin cache`, + deletedDirs, + deletedFiles + }; + } catch (error: any) { + console.error('Error clearing plugin cache:', error); + return reply.status(500).send({ + status: 'error', + message: 'Failed to clear plugin cache', + details: error.message + }); + } + }); + // Rescan plugins (reload from container manager) this.app.post('/api/plugins/rescan', async (request, reply) => { const pluginManager = this.getPluginManager?.(); @@ -2378,6 +2485,134 @@ export class UnifiedAPIServer { message: 'Use docker logs command directly for container logs' }); }); + + // Add new container plugin + this.app.post<{ + Body: { + pluginId: string; + image: string; + instances?: number; + resources?: { memory?: string; cpus?: number }; + config?: Record; + network?: boolean; + defaultQueue?: 'fast' | 'background'; + } + }>('/api/plugins/containers', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + const { pluginId, image, instances, resources, config: pluginConfig, network, defaultQueue } = request.body; + + if (!pluginId) { + return reply.status(400).send({ error: 'pluginId is required' }); + } + + if (!image) { + return reply.status(400).send({ error: 'image is required' }); + } + + try { + await this.containerManager.addPlugin(pluginId, image, { + instances, + resources, + config: pluginConfig, + network, + defaultQueue, + }); + + return { + success: true, + message: `Plugin '${pluginId}' added successfully` + }; + } catch (error: any) { + return reply.status(400).send({ + error: 'Failed to add plugin', + details: error.message + }); + } + }); + + // Remove container plugin + this.app.delete<{ + Params: { pluginId: string } + }>('/api/plugins/containers/:pluginId', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + const { pluginId } = request.params; + + try { + await this.containerManager.removePlugin(pluginId); + return { + success: true, + message: `Plugin '${pluginId}' removed successfully` + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to remove plugin', + details: error.message + }); + } + }); + + // Update container plugin + this.app.put<{ + Params: { pluginId: string }; + Body: { + image?: string; + instances?: number; + resources?: { memory?: string; cpus?: number }; + config?: Record; + network?: boolean; + defaultQueue?: 'fast' | 'background'; + } + }>('/api/plugins/containers/:pluginId', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + const { pluginId } = request.params; + const updates = request.body; + + if (!updates || Object.keys(updates).length === 0) { + return reply.status(400).send({ error: 'No updates provided' }); + } + + try { + await this.containerManager.updatePluginConfig(pluginId, updates); + return { + success: true, + message: `Plugin '${pluginId}' updated successfully` + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to update plugin', + details: error.message + }); + } + }); + + // Restart all container plugins + this.app.post('/api/plugins/containers/restart-all', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + + try { + await this.containerManager.restartAllPlugins(); + return { + success: true, + message: 'All plugins restarted successfully' + }; + } catch (error: any) { + return reply.status(500).send({ + error: 'Failed to restart plugins', + details: error.message + }); + } + }); } /** diff --git a/packages/meta-sort-core/src/config/EnvConfig.ts b/packages/meta-sort-core/src/config/EnvConfig.ts index b00dbc0..0b3ca62 100644 --- a/packages/meta-sort-core/src/config/EnvConfig.ts +++ b/packages/meta-sort-core/src/config/EnvConfig.ts @@ -115,6 +115,17 @@ interface EnvConfig { /** Docker Compose project name for grouping containers in Docker Desktop (optional). When set, plugin containers appear grouped as a stack. */ PLUGIN_STACK_NAME?: string; + + /** + * Host path for plugin cache folders. + * This is the actual host filesystem path that plugin containers will mount. + * Required for persistent plugin caches across container restarts. + * Example: "/d/workspace/MetaMesh/meta-root-v2/dev/cache/meta-sort/plugin-cache" + */ + PLUGIN_CACHE_HOST_PATH?: string; + + // NOTE: Plugin output files are written via WebDAV (WEBDAV_URL/plugin//) + // No PLUGIN_OUTPUT_HOST_PATH needed - plugins use HTTP PUT to write files } // Parse POLLING_PATHS from JSON string @@ -151,15 +162,15 @@ function parsePollingPaths(): PollingPathConfig[] | undefined { export const config: EnvConfig = { WATCH_FOLDER_LIST: process.env.WATCH_FOLDER_LIST, - INDEX_FOLDER_PATH: process.env.INDEX_FOLDER_PATH || '/data/cache/hash',//default /data/cache/hash + INDEX_FOLDER_PATH: process.env.INDEX_FOLDER_PATH || '/data/cache/hash-index',//default /data/cache/hash-index UPDATE_INTERVAL_MS: parseInt(process.env.UPDATE_INTERVAL_MS || "30000", 10),//default 30000 WATCH_FOLDER_POOLING_INTERVAL_MS: parseInt(process.env.WATCH_FOLDER_POOLING_INTERVAL_MS || "0", 10),//default 0 CHMOD_FOR_NEW_FOLDER: process.env.CHMOD_FOR_NEW_FOLDER && parseInt(process.env.CHMOD_FOR_NEW_FOLDER, 8),//default 0o777 (this is the default for mkdir) - CACHE_FOLDER_PATH: process.env.CACHE_FOLDER_PATH || '/data/cache/plugins',//default /data/cache/plugins + CACHE_FOLDER_PATH: process.env.CACHE_FOLDER_PATH || '/data/cache',//default /data/cache JELLYFIN_ENDPOINT: process.env.JELLYFIN_ENDPOINT!, JELLYFIN_API_KEY: process.env.JELLYFIN_API_KEY!, FUSE_API_PORT: parseInt(process.env.FUSE_API_PORT || "3000", 10),//default 3000 - FUSE_API_HOST: process.env.FUSE_API_HOST || 'localhost',//default localhost + FUSE_API_HOST: process.env.FUSE_API_HOST || '0.0.0.0',//default 0.0.0.0 METADATA_FORMATS: (process.env.METADATA_FORMATS || 'meta').split(',').map(f => f.trim()).filter(f => f.length > 0),//default 'meta' MAX_WORKER_THREADS: process.env.MAX_WORKER_THREADS ? parseInt(process.env.MAX_WORKER_THREADS, 10) : undefined, POLLING_PATHS: parsePollingPaths(), @@ -186,6 +197,7 @@ export const config: EnvConfig = { CONTAINER_NETWORK: process.env.CONTAINER_NETWORK || 'meta-network', PLUGIN_WEBDAV_URL: process.env.PLUGIN_WEBDAV_URL, PLUGIN_STACK_NAME: process.env.PLUGIN_STACK_NAME, + PLUGIN_CACHE_HOST_PATH: process.env.PLUGIN_CACHE_HOST_PATH, }; if(process.env.TEST!=='true') { diff --git a/packages/meta-sort-core/src/container-plugins/ConfigParser.ts b/packages/meta-sort-core/src/container-plugins/ConfigParser.ts index ea1f9f7..ee1ab32 100644 --- a/packages/meta-sort-core/src/container-plugins/ConfigParser.ts +++ b/packages/meta-sort-core/src/container-plugins/ConfigParser.ts @@ -370,3 +370,75 @@ export function getEnabledPlugins( export function getDefaultConfigPath(): string { return process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml'; } + +/** + * Convert ContainerPluginConfig back to YAML-friendly format + */ +function configToYamlObject(config: ContainerPluginConfig): Record { + const obj: Record = { + enabled: config.enabled, + image: config.image, + instances: config.instances, + }; + + if (config.resources) { + obj.resources = config.resources; + } + + if (config.config && Object.keys(config.config).length > 0) { + obj.config = config.config; + } + + if (config.network) { + obj.network = config.network; + } + + if (config.mounts && config.mounts.length > 0) { + obj.mounts = config.mounts; + } + + if (config.healthCheck) { + obj.healthCheck = config.healthCheck; + } + + if (config.defaultQueue) { + obj.defaultQueue = config.defaultQueue; + } + + return obj; +} + +/** + * Save container plugins configuration to YAML file + */ +export async function saveConfig( + configPath: string, + config: ContainerPluginsConfig +): Promise { + // Convert config to YAML-friendly format + const yamlConfig: Record = { + version: config.version || '1.0', + plugins: {} as Record, + }; + + for (const [pluginId, pluginConfig] of Object.entries(config.plugins)) { + (yamlConfig.plugins as Record)[pluginId] = configToYamlObject(pluginConfig); + } + + // Stringify to YAML + const yamlContent = YAML.stringify(yamlConfig, { + indent: 2, + lineWidth: 0, // Disable line wrapping + }); + + // Write to file + try { + await fs.writeFile(configPath, yamlContent, 'utf-8'); + } catch (error) { + throw new ConfigParseError( + 'Failed to write configuration file', + configPath, + error instanceof Error ? error : undefined + ); + } +} diff --git a/packages/meta-sort-core/src/container-plugins/ContainerManager.ts b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts index f093f47..7f56f41 100644 --- a/packages/meta-sort-core/src/container-plugins/ContainerManager.ts +++ b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts @@ -6,10 +6,12 @@ */ import { EventEmitter } from 'events'; +import { promises as fs } from 'fs'; import { DockerClient } from './DockerClient.js'; import { loadConfig, getEnabledPlugins, + saveConfig, } from './ConfigParser.js'; import type { ContainerPluginsConfig, @@ -298,9 +300,41 @@ export class ContainerManager extends EventEmitter { * Sees all mounts including SMB via rclone * * Note: No volume mounts for /files - plugins use HTTP/WebDAV client + * Cache: Each plugin gets a dedicated cache folder mounted at /cache */ const mounts: Array<{ source: string; target: string; readonly: boolean }> = []; + // Add cache mount for plugin persistence (survives container restarts) + // Use PLUGIN_CACHE_HOST_PATH if set (host filesystem path for Docker bind mounts) + // Otherwise fall back to internal path (only works if container shares filesystem) + if (config.PLUGIN_CACHE_HOST_PATH) { + const pluginCacheDir = `${config.PLUGIN_CACHE_HOST_PATH}/${pluginId}`; + // Create directory on internal path (which maps to host path) + const internalCacheDir = `${config.CACHE_FOLDER_PATH}/plugin-cache/${pluginId}`; + await fs.mkdir(internalCacheDir, { recursive: true }); + mounts.push({ + source: pluginCacheDir, + target: '/cache', + readonly: false, + }); + } else { + console.warn(`[ContainerManager] PLUGIN_CACHE_HOST_PATH not set. Plugin '${pluginId}' cache will not persist across restarts.`); + } + + // NOTE: Plugin output files are written via WebDAV (WEBDAV_URL/plugin//) + // No output mount needed - plugins use HTTP PUT to write files + + // Add any additional mounts from plugin config + if (pluginConfig.mounts) { + for (const mount of pluginConfig.mounts) { + mounts.push({ + source: mount.source, + target: mount.target, + readonly: mount.readonly ?? true, + }); + } + } + // No file mounts needed - plugins access files via WebDAV if (!this.webdavUrl) { console.warn(`[ContainerManager] WARNING: WebDAV not configured. Plugin '${pluginId}' won't have access to files. Set PLUGIN_WEBDAV_URL.`); @@ -687,6 +721,199 @@ export class ContainerManager extends EventEmitter { instance.tasksFailed++; } } + + /** + * Add a new plugin dynamically + */ + async addPlugin( + pluginId: string, + image: string, + pluginConfig?: Partial + ): Promise { + // Validate plugin ID format + if (!/^[a-z][a-z0-9-]*$/.test(pluginId)) { + throw new Error( + `Invalid plugin ID '${pluginId}': must be lowercase alphanumeric with hyphens, starting with a letter` + ); + } + + // Check if plugin already exists + if (this.instances.has(pluginId)) { + throw new Error(`Plugin '${pluginId}' already exists`); + } + + console.log(`[ContainerManager] Adding new plugin '${pluginId}' with image '${image}'`); + + // Build full plugin config + const fullConfig: ContainerPluginConfig = { + enabled: pluginConfig?.enabled !== false, + image, + instances: pluginConfig?.instances ?? 1, + resources: pluginConfig?.resources, + config: pluginConfig?.config, + network: pluginConfig?.network ?? false, + mounts: pluginConfig?.mounts, + healthCheck: pluginConfig?.healthCheck, + defaultQueue: pluginConfig?.defaultQueue, + }; + + // Update in-memory config + if (!this.pluginsConfig) { + this.pluginsConfig = { version: '1.0', plugins: {} }; + } + this.pluginsConfig.plugins[pluginId] = fullConfig; + + // Save to YAML file + await saveConfig(this.configPath, this.pluginsConfig); + + // Spawn the plugin containers + if (fullConfig.enabled) { + await this.spawnPlugin(pluginId, fullConfig); + } + + console.log(`[ContainerManager] Plugin '${pluginId}' added successfully`); + } + + /** + * Remove a plugin + */ + async removePlugin(pluginId: string): Promise { + console.log(`[ContainerManager] Removing plugin '${pluginId}'...`); + + const instances = this.instances.get(pluginId); + + if (instances) { + // Stop health check + const interval = this.healthCheckIntervals.get(pluginId); + if (interval) { + clearInterval(interval); + this.healthCheckIntervals.delete(pluginId); + } + + // Stop all instances + for (const instance of instances) { + await this.stopInstance(instance); + } + + this.instances.delete(pluginId); + } + + // Remove from config + if (this.pluginsConfig && this.pluginsConfig.plugins[pluginId]) { + delete this.pluginsConfig.plugins[pluginId]; + + // Save to YAML file + await saveConfig(this.configPath, this.pluginsConfig); + } + + // Reset round robin counter + this.roundRobinCounters.delete(pluginId); + + console.log(`[ContainerManager] Plugin '${pluginId}' removed successfully`); + } + + /** + * Restart all plugins + */ + async restartAllPlugins(): Promise { + console.log('[ContainerManager] Restarting all plugins...'); + + const pluginIds = Array.from(this.instances.keys()); + + for (const pluginId of pluginIds) { + try { + await this.restartPlugin(pluginId); + } catch (error) { + console.error(`[ContainerManager] Failed to restart plugin '${pluginId}':`, error); + } + } + + console.log('[ContainerManager] All plugins restarted'); + } + + /** + * Update plugin image and restart + */ + async updatePluginImage(pluginId: string, newImage: string): Promise { + console.log(`[ContainerManager] Updating plugin '${pluginId}' to image '${newImage}'...`); + + const pluginConfig = this.pluginsConfig?.plugins[pluginId]; + + if (!pluginConfig) { + throw new Error(`Plugin '${pluginId}' not found`); + } + + // Update image in config + pluginConfig.image = newImage; + + // Save to YAML file + if (this.pluginsConfig) { + await saveConfig(this.configPath, this.pluginsConfig); + } + + // Restart the plugin to use the new image + await this.restartPlugin(pluginId); + + console.log(`[ContainerManager] Plugin '${pluginId}' updated successfully`); + } + + /** + * Update plugin configuration + */ + async updatePluginConfig( + pluginId: string, + updates: Partial + ): Promise { + console.log(`[ContainerManager] Updating configuration for plugin '${pluginId}'...`); + + const pluginConfig = this.pluginsConfig?.plugins[pluginId]; + + if (!pluginConfig) { + throw new Error(`Plugin '${pluginId}' not found`); + } + + // Apply updates + if (updates.image !== undefined) { + pluginConfig.image = updates.image; + } + if (updates.instances !== undefined) { + pluginConfig.instances = updates.instances; + } + if (updates.resources !== undefined) { + pluginConfig.resources = updates.resources; + } + if (updates.config !== undefined) { + pluginConfig.config = updates.config; + } + if (updates.enabled !== undefined) { + pluginConfig.enabled = updates.enabled; + } + if (updates.network !== undefined) { + pluginConfig.network = updates.network; + } + if (updates.defaultQueue !== undefined) { + pluginConfig.defaultQueue = updates.defaultQueue; + } + + // Save to YAML file + if (this.pluginsConfig) { + await saveConfig(this.configPath, this.pluginsConfig); + } + + // Restart the plugin to apply changes + if (this.instances.has(pluginId)) { + await this.restartPlugin(pluginId); + } + + console.log(`[ContainerManager] Plugin '${pluginId}' configuration updated`); + } + + /** + * Get the current plugins configuration + */ + getPluginsConfig(): ContainerPluginsConfig | null { + return this.pluginsConfig; + } } /** diff --git a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts index 96942a3..871bc03 100644 --- a/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts +++ b/packages/meta-sort-core/src/logic/WatchedFileProcessor.ts @@ -329,6 +329,14 @@ export class WatchedFileProcessor implements FileProcessorInterface { midHash256 = await computeMidHash256(filePath); performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart)); performanceMetrics.recordCacheMiss('midhash256'); + + // Store in cache for future lookups + (this.fileProcessor as any).indexManager?.addFileCid( + filePath, + fileStats.size, + fileStats.mtime.toISOString(), + { cid_midhash256: midHash256 } + ); } // STEP 2: Initialize metadata with midhash256 diff --git a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts index 69e8fb1..ff640d7 100644 --- a/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts +++ b/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts @@ -181,6 +181,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ // Record midhash256 computation time (excludes cache hits) performanceMetrics.recordHashComputation('cid_midhash256', computeTime); performanceMetrics.recordCacheMiss('midhash256'); + + // Store in cache for future lookups + this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 }); } // Initialize metadata with midhash256 @@ -358,6 +361,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{ // Record midhash256 computation time (excludes cache hits) performanceMetrics.recordHashComputation('cid_midhash256', computeTime); performanceMetrics.recordCacheMiss('midhash256'); + + // Store in cache for future lookups + this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 }); } // Initialize metadata with midhash256 diff --git a/packages/meta-sort-core/src/plugin-engine/PluginManager.ts b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts index 61f8019..3efc576 100644 --- a/packages/meta-sort-core/src/plugin-engine/PluginManager.ts +++ b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts @@ -169,6 +169,19 @@ export class PluginManager extends EventEmitter { // Save state await this.saveState(); + // Send config to running container plugin + if (this.containerManager) { + const instance = this.containerManager.getHealthyInstance(pluginId); + if (instance) { + try { + await this.containerManager.sendConfig(instance, config); + console.log(`[PluginManager] Sent config update to container plugin '${pluginId}'`); + } catch (error) { + console.warn(`[PluginManager] Failed to send config to plugin '${pluginId}':`, error); + } + } + } + // Emit event this.emit('plugin:config-changed', { pluginId, config }); } @@ -210,6 +223,9 @@ export class PluginManager extends EventEmitter { // Load state for plugin active status const state = await this.loadState(); + // Track plugins that need config sent + const pluginsWithPersistedConfig: Array<{ pluginId: string; config: Record }> = []; + for (const pluginStatus of status.plugins) { const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId); if (!manifest) continue; @@ -248,9 +264,14 @@ export class PluginManager extends EventEmitter { this.activePluginIds.add(pluginId); } - // Store config - const config = stateEntry?.config || {}; - this.configs.set(pluginId, config); + // Store config from persisted state + const persistedConfig = stateEntry?.config || {}; + this.configs.set(pluginId, persistedConfig); + + // If there's persisted config, queue it for sending to the container + if (Object.keys(persistedConfig).length > 0) { + pluginsWithPersistedConfig.push({ pluginId, config: persistedConfig }); + } console.log(`[PluginManager] Loaded plugin: ${pluginId}`); } @@ -258,6 +279,20 @@ export class PluginManager extends EventEmitter { // Rebuild execution order this.rebuildExecutionOrder(); await this.saveState(); + + // Send persisted configs to running containers + // This ensures configs saved via API are applied after container restart + for (const { pluginId, config } of pluginsWithPersistedConfig) { + const instance = this.containerManager.getHealthyInstance(pluginId); + if (instance) { + try { + await this.containerManager.sendConfig(instance, config); + console.log(`[PluginManager] Sent persisted config to plugin '${pluginId}'`); + } catch (error) { + console.warn(`[PluginManager] Failed to send persisted config to plugin '${pluginId}':`, error); + } + } + } } /** diff --git a/packages/meta-sort-ui/src/pages/Monitor.tsx b/packages/meta-sort-ui/src/pages/Monitor.tsx index da77843..85a0849 100644 --- a/packages/meta-sort-ui/src/pages/Monitor.tsx +++ b/packages/meta-sort-ui/src/pages/Monitor.tsx @@ -103,6 +103,25 @@ function Monitor() { } }; + const clearMetadata = async () => { + if (!window.confirm('Are you sure you want to clear all metadata? This will delete all file metadata and trigger a full rescan.')) { + return; + } + try { + const response = await fetch('/api/metadata/clear', { method: 'POST' }); + const data = await response.json(); + if (response.ok) { + alert(`Cleared ${data.deletedCount} files. Rescan triggered.`); + } else { + alert(`Failed to clear metadata: ${data.message}`); + } + fetchData(); + } catch (err) { + console.error('Failed to clear metadata:', err); + alert('Failed to clear metadata'); + } + }; + // Calculate progress for 6-stage pipeline (including failed) // Discovered = files waiting for fast queue (validated but not yet processing) const getProgress = () => { @@ -187,6 +206,9 @@ function Monitor() { + diff --git a/packages/meta-sort-ui/src/pages/Plugins.tsx b/packages/meta-sort-ui/src/pages/Plugins.tsx index 2267830..05c3c07 100644 --- a/packages/meta-sort-ui/src/pages/Plugins.tsx +++ b/packages/meta-sort-ui/src/pages/Plugins.tsx @@ -6,6 +6,14 @@ import { } from '../types'; import { formatMs, formatNumber } from '../utils/format'; +interface AddPluginForm { + pluginId: string; + image: string; + instances: number; + memory: string; + cpus: string; +} + function Plugins() { const [pluginsData, setPluginsData] = useState(null); const [timings, setTimings] = useState([]); @@ -15,6 +23,18 @@ function Plugins() { const [error, setError] = useState(null); const [scanStatus, setScanStatus] = useState(null); + // Container plugin management state + const [showAddPluginModal, setShowAddPluginModal] = useState(false); + const [showRemoveConfirm, setShowRemoveConfirm] = useState(null); + const [addPluginForm, setAddPluginForm] = useState({ + pluginId: '', + image: '', + instances: 1, + memory: '256m', + cpus: '0.5', + }); + const [restartAllStatus, setRestartAllStatus] = useState(null); + const fetchPlugins = useCallback(async () => { try { const res = await fetch('/api/plugins'); @@ -164,6 +184,107 @@ function Plugins() { return timings.find(t => t.pluginId === pluginId); }; + // Container plugin management functions + const restartPlugin = async (pluginId: string) => { + setLoading(true); + try { + const res = await fetch(`/api/plugins/containers/${pluginId}/restart`, { method: 'POST' }); + if (!res.ok) { + const data = await res.json(); + throw new Error(data.details || data.error || 'Failed to restart plugin'); + } + await fetchPlugins(); + } catch (err: any) { + alert(err.message); + } finally { + setLoading(false); + } + }; + + const restartAllPlugins = async () => { + setLoading(true); + setRestartAllStatus('Restarting...'); + try { + const res = await fetch('/api/plugins/containers/restart-all', { method: 'POST' }); + if (!res.ok) { + const data = await res.json(); + throw new Error(data.details || data.error || 'Failed to restart plugins'); + } + setRestartAllStatus('Restarted'); + setTimeout(() => setRestartAllStatus(null), 3000); + await fetchPlugins(); + } catch (err: any) { + setRestartAllStatus('Failed'); + alert(err.message); + setTimeout(() => setRestartAllStatus(null), 3000); + } finally { + setLoading(false); + } + }; + + const addPlugin = async () => { + if (!addPluginForm.pluginId || !addPluginForm.image) { + alert('Plugin ID and Image are required'); + return; + } + + setLoading(true); + try { + const res = await fetch('/api/plugins/containers', { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + pluginId: addPluginForm.pluginId, + image: addPluginForm.image, + instances: addPluginForm.instances, + resources: { + memory: addPluginForm.memory, + cpus: parseFloat(addPluginForm.cpus) || 0.5, + }, + }), + }); + + if (!res.ok) { + const data = await res.json(); + throw new Error(data.details || data.error || 'Failed to add plugin'); + } + + setShowAddPluginModal(false); + setAddPluginForm({ + pluginId: '', + image: '', + instances: 1, + memory: '256m', + cpus: '0.5', + }); + await fetchPlugins(); + } catch (err: any) { + alert(err.message); + } finally { + setLoading(false); + } + }; + + const removePlugin = async (pluginId: string) => { + setLoading(true); + try { + const res = await fetch(`/api/plugins/containers/${pluginId}`, { method: 'DELETE' }); + if (!res.ok) { + const data = await res.json(); + throw new Error(data.details || data.error || 'Failed to remove plugin'); + } + setShowRemoveConfirm(null); + if (selectedPlugin === pluginId) { + setSelectedPlugin(null); + } + await fetchPlugins(); + } catch (err: any) { + alert(err.message); + } finally { + setLoading(false); + } + }; + const selectedPluginData = pluginsData?.plugins.find(p => p.id === selectedPlugin); // Group plugins by queue assignment @@ -202,6 +323,20 @@ function Plugins() {

Plugins

+ + +

Recompute will re-run this plugin on all files, bypassing cache checks. @@ -622,6 +773,115 @@ function Plugins() {

)} + {/* Add Plugin Modal */} + {showAddPluginModal && ( +
setShowAddPluginModal(false)}> +
e.stopPropagation()}> +
+

Add Container Plugin

+ +
+
+
+ + setAddPluginForm({ ...addPluginForm, pluginId: e.target.value })} + /> + Lowercase alphanumeric with hyphens (e.g., my-plugin) +
+
+ + setAddPluginForm({ ...addPluginForm, image: e.target.value })} + /> + Docker image name and tag +
+
+
+ + setAddPluginForm({ ...addPluginForm, instances: parseInt(e.target.value) || 1 })} + /> +
+
+ + setAddPluginForm({ ...addPluginForm, memory: e.target.value })} + /> +
+
+ + setAddPluginForm({ ...addPluginForm, cpus: e.target.value })} + /> +
+
+
+
+ + +
+
+
+ )} + + {/* Remove Confirmation Dialog */} + {showRemoveConfirm && ( +
setShowRemoveConfirm(null)}> +
e.stopPropagation()}> +
+

Remove Plugin

+ +
+
+

+ Are you sure you want to remove plugin '{showRemoveConfirm}'? +

+

+ This will stop the container and remove it from configuration. +

+
+
+ + +
+
+
+ )} + ); diff --git a/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts b/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts index 3575707..98cef38 100644 --- a/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts +++ b/packages/shared/meta-hash/src/lib/hash-compute/HashIndexManager.ts @@ -178,8 +178,10 @@ export class HashIndexManager { public async saveCacheToFile(): Promise { if (this.indexOpsInProgress || !this.hasChanged) { + console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`); return; } + console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`); this.hasChanged = false; this.indexOpsInProgress = true; const start = performance.now(); @@ -276,26 +278,28 @@ export class HashIndexManager { if (!filePath || !fileSize || !mtime || !hashs) { throw new Error('Invalid parameters'); } - for (const hash of this.targetHash) { - if (!hashs[hash]) { - throw new Error(`Missing hash ${hash}`); - } + // Check that at least one target hash is provided + const providedHashes = this.targetHash.filter(hash => hashs[hash]); + if (providedHashes.length === 0) { + console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`); + throw new Error('At least one target hash must be provided'); } + console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`); const size = fileSize + ""; const baseName = path.basename(filePath); const cacheKey = `${baseName}-${size}-${mtime}`; let indexLine = this.cache.get(cacheKey); if (!indexLine) { - //filter only the hashes we need + // Filter only the hashes we need (and are provided) let filteredHash = {}; - for (const hash of this.targetHash) { + for (const hash of providedHashes) { filteredHash[hash] = hashs[hash]; } const data = {path: baseName, size: size, mtime: mtime, ...filteredHash}; this.cache.set(cacheKey, data); } else { - //update the cache with the latest data - for (const hash of this.targetHash) { + // Update the cache with the provided hashes (don't overwrite with undefined) + for (const hash of providedHashes) { indexLine[hash] = hashs[hash]; } }