diff --git a/packages/meta-sort-core/.mocharc.json b/packages/meta-sort-core/.mocharc.json index f5eefc2..f9ee290 100644 --- a/packages/meta-sort-core/.mocharc.json +++ b/packages/meta-sort-core/.mocharc.json @@ -1,5 +1,10 @@ { - "spec": ["dist/api/api.test.js", "dist/kv/*.test.js"], + "spec": [ + "dist/api/api.test.js", + "dist/kv/*.test.js", + "dist/container-plugins/*.test.js", + "dist/discovery/*.test.js" + ], "timeout": 30000, "exit": true } diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts index 2dcd696..93df230 100644 --- a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -1027,7 +1027,9 @@ export class UnifiedAPIServer { instanceCount: p.instances, healthyCount: p.healthyInstances, enabled: p.enabled, - image: p.image + image: p.image, + kind: p.kind, + url: p.url })), queue: queueStatus || null }; @@ -1040,6 +1042,67 @@ export class UnifiedAPIServer { } }); + // Beacon v2 Scan card: enrichment plugins advertising on the network + // (metamesh.enrich/), classified against what this meta-sort runs. + this.app.post('/api/plugins/discover', async (_request, reply) => { + const leaderClient = this.kvManager?.getLeaderClient(); + const empty = { + cap: 'metamesh.enrich/*', disabled: true, + summary: { nodes: 0, capable: 0, configured: 0, boundElsewhere: 0, addable: 0 }, + candidates: [], + }; + if (!leaderClient) return reply.send(empty); + const status = this.containerManager?.getStatus(); + const configured: Array<[string, string]> = (status?.plugins ?? []) + .filter((p) => p.url) + .map((p) => [p.pluginId, p.url as string]); + const report = await leaderClient.scan('metamesh.enrich/*', configured, 1500); + // Same plugin id already loaded under another URL (e.g. our own spawned + // container of it): adding it would clash, so it counts as configured. + const loaded = new Set((status?.plugins ?? []).map((p) => p.pluginId)); + for (const c of report.candidates) { + if (c.state !== 'addable') continue; + const id = c.caps.map((cap) => cap.match(/^metamesh\.enrich\/([^@]+)/)?.[1]).find((x) => x && loaded.has(x)); + if (id) { + c.state = 'configured'; + c.configuredAs = id; + report.summary.addable--; + report.summary.configured++; + } + } + return reply.send(report); + }); + + // External plugin (one meta-sort did not spawn) — the Scan card's Add. + this.app.post<{ Body: { url?: string; name?: string } }>('/api/plugins/external', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + const url = request.body?.url; + if (!url) return reply.status(400).send({ error: 'url is required' }); + try { + const entry = await this.containerManager.addExternalPlugin(url, request.body?.name); + // Bring it into the processing pipeline (same as /api/plugins/rescan). + await this.getPluginManager?.()?.loadContainerPlugins(); + return { success: true, plugin: entry }; + } catch (error: any) { + return reply.status(400).send({ error: error.message }); + } + }); + + this.app.delete<{ Params: { pluginId: string } }>('/api/plugins/external/:pluginId', async (request, reply) => { + if (!this.containerManager) { + return reply.status(503).send({ error: 'Container manager not initialized' }); + } + try { + await this.containerManager.removeExternalPlugin(request.params.pluginId); + await this.getPluginManager?.()?.loadContainerPlugins(); + return { success: true }; + } catch (error: any) { + return reply.status(400).send({ error: error.message }); + } + }); + // Get container plugin manifest this.app.get<{ Params: { pluginId: string } diff --git a/packages/meta-sort-core/src/config/EnvConfig.ts b/packages/meta-sort-core/src/config/EnvConfig.ts index 7e2d2f2..af448c2 100644 --- a/packages/meta-sort-core/src/config/EnvConfig.ts +++ b/packages/meta-sort-core/src/config/EnvConfig.ts @@ -89,6 +89,13 @@ interface EnvConfig { /** Path to container plugins config file (default: '/app/plugins.yml') */ CONTAINER_PLUGINS_CONFIG: string; + /** + * External plugins added by URL from the Plugins page's beacon Scan card + * (default: '/state/external-plugins.json'). Kept apart + * from plugins.yml, which the store app mounts read-only. + */ + EXTERNAL_PLUGINS_PATH: string; + /** Docker socket path (default: '/var/run/docker.sock') */ DOCKER_SOCKET_PATH: string; @@ -156,6 +163,8 @@ export const config: EnvConfig = { // Container Plugins Configuration CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml', + EXTERNAL_PLUGINS_PATH: process.env.EXTERNAL_PLUGINS_PATH + || `${process.env.CACHE_FOLDER_PATH || '/data/cache'}/state/external-plugins.json`, DOCKER_SOCKET_PATH: process.env.DOCKER_SOCKET_PATH || '/var/run/docker.sock', CONTAINER_CALLBACK_URL: process.env.CONTAINER_CALLBACK_URL || 'http://meta-sort:8180', CONTAINER_META_CORE_URL: process.env.CONTAINER_META_CORE_URL || 'http://meta-sort', diff --git a/packages/meta-sort-core/src/container-plugins/ContainerManager.ts b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts index 1c61b20..cb47e7e 100644 --- a/packages/meta-sort-core/src/container-plugins/ContainerManager.ts +++ b/packages/meta-sort-core/src/container-plugins/ContainerManager.ts @@ -25,6 +25,9 @@ import type { PluginConfigureResponse, } from './types.js'; import { config } from '../config/EnvConfig.js'; +import { existsSync } from 'fs'; +import { hostname } from 'os'; +import { ExternalPluginStore, type ExternalPluginEntry } from './ExternalPluginStore.js'; /** * Service info structure from meta-core service discovery @@ -96,6 +99,11 @@ export class ContainerManager extends EventEmitter { private stackName?: string; private callbackUrl: string; private metaCoreUrl: string; + /** External plugins (added by URL); see ExternalPluginStore. */ + private externalStore: ExternalPluginStore; + private externalEntries: ExternalPluginEntry[] = []; + /** False when Docker is unreachable: only external plugins run then. */ + private dockerAvailable = false; constructor( private configPath: string = config.CONTAINER_PLUGINS_CONFIG, @@ -116,6 +124,12 @@ export class ContainerManager extends EventEmitter { // Configure stack name for Docker Desktop grouping this.stackName = config.PLUGIN_STACK_NAME; + this.externalStore = new ExternalPluginStore(config.EXTERNAL_PLUGINS_PATH); + } + + /** Swap the external-plugin store (tests). */ + setExternalStore(store: ExternalPluginStore): void { + this.externalStore = store; } /** @@ -183,22 +197,41 @@ export class ContainerManager extends EventEmitter { console.warn(' Plugins will not have access to files.'); } - // Initialize Docker client - await this.dockerClient.initialize(); + // Docker is needed only for the plugins we spawn. Without it (no + // socket, permission denied) external plugins still run. + try { + await this.dockerClient.initialize(); + await this.dockerClient.ensureNetwork(this.network); + this.dockerAvailable = true; + } catch (error) { + console.warn('[ContainerManager] Docker unavailable — only external plugins will run:', error); + } - // Ensure network exists - await this.dockerClient.ensureNetwork(this.network); - - // Load configuration - this.pluginsConfig = await loadConfig(this.configPath); + // Load configuration. A missing or broken plugins.yml means "no + // spawned plugins", not "no plugin manager": external plugins can + // still be added from the UI. + if (existsSync(this.configPath)) { + try { + this.pluginsConfig = await loadConfig(this.configPath); + } catch (error) { + console.error(`[ContainerManager] Ignoring ${this.configPath}:`, error); + } + } else { + console.log(`[ContainerManager] No ${this.configPath}; no spawned plugins`); + } + if (!this.pluginsConfig) { + this.pluginsConfig = { version: '1.0', plugins: {} }; + } // Get enabled plugins - const enabledPlugins = getEnabledPlugins(this.pluginsConfig); + const enabledPlugins = this.dockerAvailable ? getEnabledPlugins(this.pluginsConfig) : []; console.log(`[ContainerManager] Found ${enabledPlugins.length} enabled plugins`); - // Clean up any stale containers from previous runs - await this.cleanupStaleContainers(); + if (this.dockerAvailable) { + // Clean up any stale containers from previous runs + await this.cleanupStaleContainers(); + } // Spawn containers for each enabled plugin for (const { id, config: pluginConfig } of enabledPlugins) { @@ -213,6 +246,21 @@ export class ContainerManager extends EventEmitter { } } + // External plugins (added by URL). One unreachable at boot is + // registered anyway and picked up by the health loop when it appears. + this.externalEntries = await this.externalStore.load(); + for (const entry of this.externalEntries.filter((e) => e.enabled)) { + if (this.instances.has(entry.pluginId)) { + console.warn( + `[ContainerManager] External plugin '${entry.pluginId}' (${entry.url}) skipped: ` + + 'a spawned plugin already has that id' + ); + continue; + } + this.attachExternal(entry); + } + console.log(`[ContainerManager] ${this.externalEntries.length} external plugin(s) configured`); + this.initialized = true; this.emit('manager:initialized', { pluginCount: enabledPlugins.length }); @@ -415,6 +463,11 @@ export class ContainerManager extends EventEmitter { CALLBACK_URL: this.callbackUrl, WEBDAV_URL: this.webdavUrl || '', FILES_PATH: '/files', // Virtual path - files accessed via WebDAV + // Beacon v2: advertise the URL we reach it on (so a Scan shows it + // as configured), and only to this meta-sort (another one's + // Scan shows it as bound elsewhere, never addable). + BEACON_ADVERTISE_URL: `http://${containerName}:8080`, + BEACON_BINDS: hostname(), }, // Docker Desktop grouping stackName: this.stackName, @@ -427,6 +480,7 @@ export class ContainerManager extends EventEmitter { // Create instance record const instance: ContainerPluginInstance = { pluginId, + kind: 'container', containerId, containerName, baseUrl: `http://${containerName}:8080`, @@ -445,6 +499,12 @@ export class ContainerManager extends EventEmitter { * Stop a container instance */ private async stopInstance(instance: ContainerPluginInstance): Promise { + if (instance.kind === 'external') { + // Not ours to stop: just stop using it. + instance.status = 'stopped'; + this.emit('plugin:stopped', { pluginId: instance.pluginId, instanceIndex: instance.instanceIndex }); + return; + } console.log(`[ContainerManager] Stopping ${instance.containerName}...`); try { @@ -490,6 +550,11 @@ export class ContainerManager extends EventEmitter { if (healthy) { instance.status = 'healthy'; instance.lastHealthCheck = Date.now(); + // An external plugin unreachable at boot has no manifest + // yet; the scheduler needs one. + if (!instance.manifest) { + instance.manifest = await this.fetchManifest(instance).catch(() => undefined); + } if (previousStatus !== 'healthy') { this.emit('plugin:healthy', { pluginId, instance }); @@ -609,9 +674,12 @@ export class ContainerManager extends EventEmitter { throw new Error(`Failed to configure plugin: ${response.status}`); } + // Two contracts in the wild: `{ success: true }` and the TS plugins' + // `{ status: 'ok' }`. Before both were accepted every configure of a + // TS plugin "failed" (silently, behind a try/warn). const data = (await response.json()) as PluginConfigureResponse; - if (!data.success) { - throw new Error(`Plugin configuration failed: ${data.error}`); + if (!data.success && data.status !== 'ok') { + throw new Error(`Plugin configuration failed: ${data.error ?? JSON.stringify(data)}`); } } @@ -678,11 +746,14 @@ export class ContainerManager extends EventEmitter { for (const [pluginId, instances] of this.instances) { const pluginConfig = this.pluginsConfig?.plugins[pluginId]; const healthyCount = instances.filter((i) => i.status === 'healthy').length; + const external = instances[0]?.kind === 'external'; plugins.push({ pluginId, enabled: pluginConfig?.enabled !== false, - image: pluginConfig?.image || 'unknown', + image: external ? instances[0].baseUrl : pluginConfig?.image || 'unknown', + kind: external ? 'external' : 'container', + url: instances[0]?.baseUrl, instances: instances.length, healthyInstances: healthyCount, manifest: instances[0]?.manifest, @@ -717,6 +788,16 @@ export class ContainerManager extends EventEmitter { const instances = this.instances.get(pluginId); const pluginConfig = this.pluginsConfig?.plugins[pluginId]; + if (instances?.[0]?.kind === 'external') { + // Nothing to respawn: re-probe it and refresh its manifest. + for (const instance of instances) { + const healthy = await this.checkHealth(instance); + instance.status = healthy ? 'healthy' : 'unhealthy'; + if (healthy) instance.manifest = await this.fetchManifest(instance).catch(() => instance.manifest); + } + return; + } + if (!instances || !pluginConfig) { throw new Error(`Plugin '${pluginId}' not found`); } @@ -779,6 +860,93 @@ export class ContainerManager extends EventEmitter { } } + /** The external plugins, as persisted. */ + getExternalPlugins(): ExternalPluginEntry[] { + return [...this.externalEntries]; + } + + /** + * Register an external plugin (one meta-sort did not spawn) by URL — what + * the beacon Scan card's Add does. Its manifest id becomes its plugin id, + * so it must be reachable now and its id must not clash with a loaded one. + */ + async addExternalPlugin(rawUrl: string, name?: string): Promise { + const url = rawUrl.trim().replace(/\/+$/, ''); + if (!/^https?:\/\//.test(url)) { + throw new Error('url must start with http:// or https://'); + } + for (const list of this.instances.values()) { + const holder = list.find((i) => i.baseUrl.replace(/\/+$/, '') === url); + if (holder) throw new Error(`${url} is already registered as '${holder.pluginId}'`); + } + const probe: ContainerPluginInstance = { + pluginId: '?', kind: 'external', containerId: '', containerName: url, baseUrl: url, + instanceIndex: 0, status: 'starting', tasksProcessed: 0, tasksFailed: 0, + }; + let manifest: ContainerPluginManifest; + try { + manifest = await this.fetchManifest(probe); + } catch (error) { + throw new Error(`could not read ${url}/manifest: ${error instanceof Error ? error.message : error}`); + } + const pluginId = manifest?.id; + if (!pluginId || !/^[a-z][a-z0-9-]*$/.test(pluginId)) { + throw new Error(`${url}/manifest has no usable plugin id`); + } + if (this.instances.has(pluginId)) { + throw new Error(`a plugin with id '${pluginId}' is already loaded`); + } + const entry: ExternalPluginEntry = { + pluginId, url, enabled: true, name: name?.trim() || undefined, addedAt: new Date().toISOString(), + }; + this.externalEntries = [...this.externalEntries.filter((e) => e.pluginId !== pluginId), entry]; + await this.externalStore.save(this.externalEntries); + this.attachExternal(entry, manifest); + console.log(`[ContainerManager] External plugin '${pluginId}' added (${url})`); + return entry; + } + + /** Forget an external plugin. Throws if `pluginId` is not one. */ + async removeExternalPlugin(pluginId: string): Promise { + const known = this.externalEntries.some((e) => e.pluginId === pluginId); + const instances = this.instances.get(pluginId); + if (!known && !(instances?.[0]?.kind === 'external')) { + throw new Error(`'${pluginId}' is not an external plugin`); + } + if (instances?.[0]?.kind === 'external') { + const interval = this.healthCheckIntervals.get(pluginId); + if (interval) { + clearInterval(interval); + this.healthCheckIntervals.delete(pluginId); + } + for (const instance of instances) await this.stopInstance(instance); + this.instances.delete(pluginId); + this.roundRobinCounters.delete(pluginId); + } + this.externalEntries = this.externalEntries.filter((e) => e.pluginId !== pluginId); + await this.externalStore.save(this.externalEntries); + console.log(`[ContainerManager] External plugin '${pluginId}' removed`); + } + + /** One instance pointing at `entry.url`, kept alive by the health loop. */ + private attachExternal(entry: ExternalPluginEntry, manifest?: ContainerPluginManifest): void { + const instance: ContainerPluginInstance = { + pluginId: entry.pluginId, + kind: 'external', + containerId: '', + // The scheduler reports completions by this name; the URL is unique. + containerName: entry.url, + baseUrl: entry.url, + instanceIndex: 0, + status: 'starting', + manifest, + tasksProcessed: 0, + tasksFailed: 0, + }; + this.instances.set(entry.pluginId, [instance]); + this.startHealthCheck(entry.pluginId); + } + /** * Add a new plugin dynamically */ @@ -838,6 +1006,11 @@ export class ContainerManager extends EventEmitter { console.log(`[ContainerManager] Removing plugin '${pluginId}'...`); const instances = this.instances.get(pluginId); + // An external plugin is forgotten (and un-persisted), never stopped. + if (instances?.[0]?.kind === 'external' || this.externalEntries.some((e) => e.pluginId === pluginId)) { + await this.removeExternalPlugin(pluginId); + return; + } if (instances) { // Stop health check diff --git a/packages/meta-sort-core/src/container-plugins/ExternalPluginStore.ts b/packages/meta-sort-core/src/container-plugins/ExternalPluginStore.ts new file mode 100644 index 0000000..49f3104 --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/ExternalPluginStore.ts @@ -0,0 +1,52 @@ +/** + * External plugins — enrichment plugins meta-sort did NOT spawn, reached by + * URL (typically a MetaPlugin* store app found with the Plugins page's beacon + * Scan card). Persisted as JSON next to the cache, because the store app + * mounts plugins.yml read-only. + */ + +import { promises as fs } from 'fs'; +import { dirname } from 'path'; + +export interface ExternalPluginEntry { + /** The plugin's manifest id — its identity in the pipeline. */ + pluginId: string; + /** Base URL of the plugin's HTTP contract (/manifest, /health, /process). */ + url: string; + enabled: boolean; + /** Name it was added under (the advertising instance), display only. */ + name?: string; + addedAt?: string; +} + +export class ExternalPluginStore { + constructor(private readonly path: string) {} + + /** The persisted list; empty when the file is missing or unreadable. */ + async load(): Promise { + let raw: string; + try { + raw = await fs.readFile(this.path, 'utf-8'); + } catch { + return []; + } + try { + const parsed = JSON.parse(raw); + const list = Array.isArray(parsed?.plugins) ? parsed.plugins : []; + return list.filter( + (e: Partial) => typeof e?.pluginId === 'string' && typeof e?.url === 'string' + ).map((e: ExternalPluginEntry) => ({ ...e, enabled: e.enabled !== false })); + } catch (error) { + console.error(`[ExternalPluginStore] ${this.path} is not valid JSON; ignoring it:`, error); + return []; + } + } + + /** Atomic write: a crash mid-save leaves the previous file intact. */ + async save(entries: ExternalPluginEntry[]): Promise { + await fs.mkdir(dirname(this.path), { recursive: true }); + const tmp = `${this.path}.tmp`; + await fs.writeFile(tmp, JSON.stringify({ plugins: entries }, null, 2), 'utf-8'); + await fs.rename(tmp, this.path); + } +} diff --git a/packages/meta-sort-core/src/container-plugins/ExternalPlugins.test.ts b/packages/meta-sort-core/src/container-plugins/ExternalPlugins.test.ts new file mode 100644 index 0000000..3b6140d --- /dev/null +++ b/packages/meta-sort-core/src/container-plugins/ExternalPlugins.test.ts @@ -0,0 +1,74 @@ +import { strict as assert } from 'assert'; +import { createServer, type Server } from 'http'; +import { mkdtempSync, readFileSync } from 'fs'; +import { tmpdir } from 'os'; +import { join } from 'path'; +import { ContainerManager } from './ContainerManager.js'; +import { ExternalPluginStore } from './ExternalPluginStore.js'; + +/** + * External plugins are the meta-sort side of the beacon Scan card's Add: a + * plugin meta-sort did not spawn, reached by URL, identified by its manifest + * id, persisted outside the read-only plugins.yml. + */ +describe('external plugins', () => { + let server: Server; + let url: string; + + before(async () => { + server = createServer((req, res) => { + res.setHeader('Content-Type', 'application/json'); + if (req.url === '/manifest') { + res.end(JSON.stringify({ id: 'language', name: 'Language', version: '1.0.2', defaultQueue: 'fast' })); + } else if (req.url === '/health') { + res.end(JSON.stringify({ status: 'healthy', ready: true, version: '1.0.2' })); + } else { + res.statusCode = 404; + res.end('{}'); + } + }); + await new Promise((r) => server.listen(0, '127.0.0.1', () => r())); + const addr = server.address(); + url = `http://127.0.0.1:${typeof addr === 'object' && addr ? addr.port : 0}`; + }); + + after(() => server.close()); + + function manager(path: string): ContainerManager { + const m = new ContainerManager('/nonexistent/plugins.yml', 'none', '/files', 'http://cb', 'http://core'); + m.setExternalStore(new ExternalPluginStore(path)); + return m; + } + + it('adds by URL under the manifest id, persists, and refuses duplicates', async () => { + const path = join(mkdtempSync(join(tmpdir(), 'ext-')), 'state', 'external-plugins.json'); + const m = manager(path); + const entry = await m.addExternalPlugin(`${url}/`, 'metapluginlanguage-plugin'); + assert.equal(entry.pluginId, 'language'); + assert.equal(entry.url, url, 'trailing slash normalised'); + + const saved = JSON.parse(readFileSync(path, 'utf-8')); + assert.equal(saved.plugins[0].pluginId, 'language'); + + await assert.rejects(m.addExternalPlugin(url), /already registered/); + const status = m.getStatus().plugins.find((p) => p.pluginId === 'language'); + assert.equal(status?.kind, 'external'); + assert.equal(status?.url, url); + + await m.removeExternalPlugin('language'); + assert.equal(m.getStatus().plugins.length, 0); + assert.deepEqual(JSON.parse(readFileSync(path, 'utf-8')).plugins, []); + await assert.rejects(m.removeExternalPlugin('language'), /not an external plugin/); + }); + + it('rejects a URL whose manifest cannot be read', async () => { + const m = manager(join(mkdtempSync(join(tmpdir(), 'ext-')), 'x.json')); + await assert.rejects(m.addExternalPlugin('http://127.0.0.1:9/'), /could not read/); + await assert.rejects(m.addExternalPlugin('ftp://x'), /http/); + }); + + it('a missing or corrupt store reads as empty', async () => { + const dir = mkdtempSync(join(tmpdir(), 'ext-')); + assert.deepEqual(await new ExternalPluginStore(join(dir, 'missing.json')).load(), []); + }); +}); diff --git a/packages/meta-sort-core/src/container-plugins/types.ts b/packages/meta-sort-core/src/container-plugins/types.ts index 4fccd88..09d67a8 100644 --- a/packages/meta-sort-core/src/container-plugins/types.ts +++ b/packages/meta-sort-core/src/container-plugins/types.ts @@ -164,7 +164,12 @@ export type ContainerInstanceStatus = 'starting' | 'healthy' | 'unhealthy' | 'st export interface ContainerPluginInstance { /** Plugin ID */ pluginId: string; - /** Docker container ID */ + /** + * `container` = spawned by meta-sort from plugins.yml; `external` = added by + * URL (beacon Scan card), not ours to start or stop. Absent = container. + */ + kind?: 'container' | 'external'; + /** Docker container ID ('' for an external plugin) */ containerId: string; /** Container name (e.g., 'meta-plugin-ffmpeg-0') */ containerName: string; @@ -211,7 +216,9 @@ export interface PluginConfigureRequest { * Configure response from POST /configure */ export interface PluginConfigureResponse { - success: boolean; + /** Older contract. The TS plugins answer `{ status: 'ok' }` instead. */ + success?: boolean; + status?: string; error?: string; } @@ -315,7 +322,11 @@ export interface ContainerManagerStatus { plugins: Array<{ pluginId: string; enabled: boolean; + /** Docker image, or the plugin URL for an external plugin. */ image: string; + kind: 'container' | 'external'; + /** Base URL of the first instance. */ + url?: string; instances: number; healthyInstances: number; manifest?: ContainerPluginManifest; diff --git a/packages/meta-sort-core/src/discovery/meshdisco.test.ts b/packages/meta-sort-core/src/discovery/meshdisco.test.ts new file mode 100644 index 0000000..df1ea6b --- /dev/null +++ b/packages/meta-sort-core/src/discovery/meshdisco.test.ts @@ -0,0 +1,30 @@ +import { strict as assert } from 'assert'; +import { capMatches, classifyScan, suggestedName, type MeshNeighbor } from './meshdisco.js'; + +/** Mirrors meta-feeder-sdk `beacon::scan` tests: same rules, same JSON. */ +describe('beacon v2 scan classification', () => { + const node = (instance: string, cap: string, http?: string, binds?: string): MeshNeighbor => ({ + name: 'plugin', instance, caps: [cap], addr: '10.0.0.1', lastSeen: 0, + resources: [{ id: 'x', caps: [cap], ...(http ? { endpoints: { http } } : {}), ...(binds ? { binds } : {}) }], + }); + + it('classifies configured, addable and bound-elsewhere', () => { + const r = classifyScan('metamesh.enrich/*', 'metasort-app', [ + node('meta-plugin-language-0', 'metamesh.enrich/language', 'http://meta-plugin-language-0:8080/', 'metasort-app'), + node('metapluginffmpeg-plugin', 'metamesh.enrich/ffmpeg', 'http://metapluginffmpeg-plugin:8080'), + node('other-sort-plugin', 'metamesh.enrich/tmdb', 'http://x:8080', 'other-metasort'), + node('no-endpoint', 'metamesh.enrich/tmdb'), + node('a-service', 'metamesh.service/meta-core', 'http://core'), + ], [['language', 'http://meta-plugin-language-0:8080']]); + assert.deepEqual(r.summary, { nodes: 5, capable: 3, configured: 1, boundElsewhere: 1, addable: 1 }); + assert.equal(r.candidates[0].instance, 'metapluginffmpeg-plugin'); + assert.equal(r.candidates[1].configuredAs, 'language'); + assert.equal(r.candidates[2].state, 'bound-elsewhere'); + }); + + it('caps and names', () => { + assert.ok(capMatches('metamesh.enrich/*', 'metamesh.enrich/language@1')); + assert.ok(!capMatches('metamesh.enrich/*', 'metamesh.enrichx/a')); + assert.equal(suggestedName('MetaPlugin Language'), 'metaplugin-language'); + }); +}); diff --git a/packages/meta-sort-core/src/discovery/meshdisco.ts b/packages/meta-sort-core/src/discovery/meshdisco.ts index 7c1a414..b624931 100644 --- a/packages/meta-sort-core/src/discovery/meshdisco.ts +++ b/packages/meta-sort-core/src/discovery/meshdisco.ts @@ -102,6 +102,85 @@ export interface MeshNodeConfig { payload?: () => { status?: string; resources: BeaconResource[] }; } +/** How a discovered resource relates to this consumer (Scan card). */ +export type ScanState = 'configured' | 'addable' | 'bound-elsewhere'; + +export interface ScanCandidate { + instance: string; + name: string; + version?: string; + resourceId: string; + caps: string[]; + /** endpoints.http — what the consumer would register. */ + url: string; + binds?: string; + state: ScanState; + /** The list entry that already holds this URL (state `configured`). */ + configuredAs?: string; + /** A list-safe name to register it under. */ + suggestedName: string; +} + +/** Same JSON as meta-feeder-sdk `beacon::ScanReport`. */ +export interface ScanReport { + cap: string; + self: string; + durationMs: number; + summary: { nodes: number; capable: number; configured: number; boundElsewhere: number; addable: number }; + candidates: ScanCandidate[]; +} + +/** Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules. */ +export function suggestedName(instance: string): string { + const s = instance.trim().toLowerCase().replace(/[^a-z0-9._-]/g, '-'); + return s || 'plugin'; +} + +const normUrl = (u: string) => u.trim().replace(/\/+$/, ''); + +/** + * Classify every resource matching `pattern` against the consumer's + * `(name, url)` list. Mirrors meta-feeder-sdk `beacon::classify`. + */ +export function classifyScan( + pattern: string, + self: string, + neighbors: MeshNeighbor[], + configured: Array<[string, string]>, +): ScanReport { + const summary = { nodes: neighbors.length, capable: 0, configured: 0, boundElsewhere: 0, addable: 0 }; + const candidates: ScanCandidate[] = []; + for (const n of neighbors) { + for (const r of n.resources) { + if (!resourceMatches(r, pattern)) continue; + const http = resourceEndpoint(r, 'http'); + if (!http) continue; // nothing a consumer could register + const url = normUrl(http); + summary.capable++; + const hit = configured.find(([, u]) => normUrl(u) === url); + let state: ScanState; + if (hit) { state = 'configured'; summary.configured++; } + else if (r.binds && r.binds !== self) { state = 'bound-elsewhere'; summary.boundElsewhere++; } + else { state = 'addable'; summary.addable++; } + candidates.push({ + instance: n.instance, + name: n.name, + ...(n.version ? { version: n.version } : {}), + resourceId: r.id, + caps: r.caps ?? [], + url, + ...(r.binds ? { binds: r.binds } : {}), + state, + ...(hit ? { configuredAs: hit[0] } : {}), + suggestedName: suggestedName(n.instance), + }); + } + } + const rank: Record = { addable: 0, configured: 1, 'bound-elsewhere': 2 }; + candidates.sort((a, b) => rank[a.state] - rank[b.state] || a.instance.localeCompare(b.instance)); + return { cap: pattern, self, durationMs: 0, summary, candidates }; +} + /** * Does capability `cap` satisfy `pattern`? `*` matches everything; an `@N` on * the pattern must equal the cap's contract (none ignores it); `x/*` matches @@ -333,6 +412,19 @@ export class MeshNode { return [...best.values()].sort((a, b) => a.name.localeCompare(b.name)); } + /** + * The Scan card: probe everyone, wait `waitMs` for the replies, then + * classify what matches `pattern` against `configured` ([name, url]). + */ + async scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise { + const started = Date.now(); + this.probe(); + await new Promise((r) => setTimeout(r, waitMs)); + const report = classifyScan(pattern, this.cfg.instance, this.getNeighbors(), configured); + report.durationMs = Date.now() - started; + return report; + } + /** This node as a row, so a UI can render itself without waiting. */ self(): MeshNeighbor { const { status, resources } = this.payload(); @@ -557,6 +649,11 @@ export class MetaCoreLocator { return this.node.self(); } + /** The Scan card — see {@link MeshNode.scan}. */ + scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise { + return this.node.scan(pattern, configured, waitMs); + } + probe(want: string[] = []): void { this.node.probe(want); } diff --git a/packages/meta-sort-core/src/index.ts b/packages/meta-sort-core/src/index.ts index 6e9b5c3..048688d 100644 --- a/packages/meta-sort-core/src/index.ts +++ b/packages/meta-sort-core/src/index.ts @@ -1,7 +1,6 @@ import {config} from "./config/EnvConfig.js"; import os from 'os'; -import {existsSync} from 'fs'; import {WatchedFileProcessor} from "./logic/WatchedFileProcessor.js"; import {UnifiedAPIServer} from "./api/UnifiedAPIServer.js"; import {StreamingPipeline} from "./logic/pipeline/StreamingPipeline.js"; @@ -168,8 +167,11 @@ process.on('SIGINT', async () => { const pluginManager = await initializePluginManager(); console.log('[Startup] Plugin manager initialized'); - // Initialize container plugins if config file exists - if (existsSync(config.CONTAINER_PLUGINS_CONFIG)) { + // Container plugins: the ones spawned from plugins.yml AND the external + // ones added by URL (beacon Scan card). Always initialised — with no + // plugins.yml and no Docker it still serves the external ones, and the + // Plugins page can add the first one. + { console.log('[Startup] Initializing container plugins...'); try { // Get webdavUrlInternal from meta-core via LeaderClient (for container-to-container access) @@ -215,8 +217,6 @@ process.on('SIGINT', async () => { console.error('[Startup] Failed to initialize container plugins:', error); console.log('[Startup] Continuing without container plugins...'); } - } else { - console.log(`[Startup] Container plugins config not found at ${config.CONTAINER_PLUGINS_CONFIG}, skipping`); } // Initialize hash index cache (loads existing hashes from disk) diff --git a/packages/meta-sort-core/src/kv/LeaderClient.ts b/packages/meta-sort-core/src/kv/LeaderClient.ts index 261d87b..6a8794e 100644 --- a/packages/meta-sort-core/src/kv/LeaderClient.ts +++ b/packages/meta-sort-core/src/kv/LeaderClient.ts @@ -23,7 +23,7 @@ * api-mediated-access lockdown; everything routes over HTTP. */ -import { MetaCoreLocator, type MeshNeighbor } from '../discovery/meshdisco.js'; +import { MetaCoreLocator, type MeshNeighbor, type ScanReport } from '../discovery/meshdisco.js'; import type { LeaderLockInfo } from './IKVClient.js'; export interface LeaderClientConfig { @@ -273,6 +273,11 @@ export class LeaderClient { return this.locator.self(); } + /** Beacon v2 Scan card: probe, wait, classify `pattern` against `configured`. */ + scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise { + return this.locator.scan(pattern, configured, waitMs); + } + close(): void { this.onChangeCallbacks = []; void this.locator.stop(); diff --git a/packages/meta-sort-core/src/plugin-engine/PluginManager.ts b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts index 4c23998..b542be7 100644 --- a/packages/meta-sort-core/src/plugin-engine/PluginManager.ts +++ b/packages/meta-sort-core/src/plugin-engine/PluginManager.ts @@ -220,6 +220,17 @@ export class PluginManager extends EventEmitter { return; } + // Forget container-backed plugins that are gone (an external plugin + // removed from the Plugins page); they have no local dir. + const present = new Set(status.plugins.map((p) => p.pluginId)); + for (const [id, loaded] of this.loadedPlugins) { + if (loaded.pluginDir === '' && !present.has(id)) { + this.loadedPlugins.delete(id); + this.manifests.delete(id); + this.activePluginIds.delete(id); + } + } + // Load state for plugin active status const state = await this.loadState(); diff --git a/packages/meta-sort-ui/src/components/BeaconScanCard.tsx b/packages/meta-sort-ui/src/components/BeaconScanCard.tsx new file mode 100644 index 0000000..8a0cd24 --- /dev/null +++ b/packages/meta-sort-ui/src/components/BeaconScanCard.tsx @@ -0,0 +1,60 @@ +import { useEffect, useRef } from 'react'; +import '../meta-beacon-scan.js'; + +/** + * The Plugins page's beacon v2 Scan card: enrichment plugins advertising on + * the network (`metamesh.enrich/`, typically MetaPlugin* store apps), with + * an Add per new one. A thin wrapper around the framework-free + * element shared with meta-share and meta-gateway; the + * --mm-scan-* tokens map this dashboard's palette onto it. + * + * Add registers the plugin as an EXTERNAL plugin (POST /api/plugins/external): + * meta-sort calls it by URL and never starts or stops it. + */ + +declare global { + // eslint-disable-next-line @typescript-eslint/no-namespace + namespace JSX { + interface IntrinsicElements { + 'meta-beacon-scan': React.DetailedHTMLProps, HTMLElement> & { + 'scan-endpoint'?: string; + 'add-endpoint'?: string; + 'add-extra'?: string; + label?: string; + }; + } + } +} + +const TOKENS = { + display: 'block', + '--mm-scan-fg': 'var(--text-primary)', + '--mm-scan-bg': 'var(--bg-secondary)', + '--mm-scan-border': 'var(--border-color)', + '--mm-scan-accent': 'var(--accent-primary)', + '--mm-scan-err': 'var(--accent-secondary)', +} as React.CSSProperties; + +function BeaconScanCard({ onAdded }: { onAdded: () => void }) { + const ref = useRef(null); + + useEffect(() => { + const el = ref.current; + if (!el) return; + const handler = () => onAdded(); + el.addEventListener('beacon-added', handler); + return () => el.removeEventListener('beacon-added', handler); + }, [onAdded]); + + return ( + + ); +} + +export default BeaconScanCard; diff --git a/packages/meta-sort-ui/src/meta-beacon-scan.js b/packages/meta-sort-ui/src/meta-beacon-scan.js new file mode 100644 index 0000000..a04cea6 --- /dev/null +++ b/packages/meta-sort-ui/src/meta-beacon-scan.js @@ -0,0 +1,277 @@ +/** + * — "what plugins are on the network?", as a card. + * + * The Scan button POSTs to the host's own discover endpoint, which probes the + * beacon v2 group (239.255.99.1:9099), waits for the replies, and classifies + * every resource matching the host's capability (transport plugins for + * meta-share, feeders for meta-gateway, enrichment plugins for meta-sort): + * + * 10 nodes on the network · 3 transport plugins · 3 already configured + * + * Each `addable` row gets an Add button that POSTs {name, url} to the host's + * existing add endpoint. Response shape: meta-feeder-sdk `beacon::ScanReport` + * (docs/project-architecture/beacon-v2.md). + * + * ⚠ MIRRORED FILE. Byte-identical copies live in meta-share, meta-gateway and + * meta-sort-ui; see `scripts/check-mirrors.sh`. Edit one → edit all. + * + * Shadow DOM for the same reason as : the three hosts use + * incompatible token vocabularies. Style it from the host with + * --mm-scan-fg, --mm-scan-bg, --mm-scan-border, --mm-scan-accent, + * --mm-scan-ok, --mm-scan-err + * + * Attributes: + * scan-endpoint POST → ScanReport (required) + * add-endpoint POST {name, url, ...add-extra} (required) + * add-extra JSON object merged into the add body (optional) + * label what the host calls a capable resource, plural + * (default "plugins") + * + * Events (bubbling, composed): + * beacon-added detail = { candidate, response }, after a successful Add + */ + +const STATE_TEXT = { + addable: "new", + configured: "configured", + "bound-elsewhere": "bound elsewhere", +}; + +const TEMPLATE = ` + +
+
+ Beacon discovery + plugins advertising themselves on this network + + +
+ + +
    +
    +`; + +function h(tag, cls, text) { + const n = document.createElement(tag); + if (cls) n.className = cls; + if (text != null) n.textContent = text; + return n; +} + +async function readError(res) { + const body = await res.text().catch(() => ""); + try { + const j = JSON.parse(body); + return j.error || j.message || body || `HTTP ${res.status}`; + } catch { + return body || `HTTP ${res.status}`; + } +} + +class MetaBeaconScan extends HTMLElement { + constructor() { + super(); + this.attachShadow({ mode: "open" }); + this.shadowRoot.innerHTML = TEMPLATE; + this._scanBtn = this.shadowRoot.querySelector("button.scan"); + this._summary = this.shadowRoot.querySelector(".summary"); + this._error = this.shadowRoot.querySelector(".error"); + this._list = this.shadowRoot.querySelector("ul"); + this._busy = false; + } + + connectedCallback() { + if (this._wired) return; + this._wired = true; + this._scanBtn.addEventListener("click", () => this.scan()); + } + + get label() { + return this.getAttribute("label") || "plugins"; + } + + async scan() { + const endpoint = this.getAttribute("scan-endpoint"); + if (!endpoint || this._busy) return; + this._busy = true; + this._scanBtn.disabled = true; + this._scanBtn.textContent = "Scanning…"; + this._error.hidden = true; + try { + const res = await fetch(endpoint, { method: "POST", headers: { Accept: "application/json" } }); + if (!res.ok) throw new Error(await readError(res)); + this._render(await res.json()); + } catch (e) { + // Keep the last result visible; just say what went wrong. + this._error.textContent = `Scan failed: ${e.message}`; + this._error.hidden = false; + } finally { + this._busy = false; + this._scanBtn.disabled = false; + this._scanBtn.textContent = "Scan again"; + } + } + + _render(report) { + const s = report.summary || {}; + this._summary.replaceChildren(); + const part = (n, text) => { + const span = h("span"); + span.append(h("b", null, String(n ?? 0)), ` ${text}`); + return span; + }; + const parts = [ + part(s.nodes, s.nodes === 1 ? "node on the network" : "nodes on the network"), + part(s.capable, this.label), + part(s.configured, "already configured"), + part(s.addable, "new"), + ]; + if (s.boundElsewhere) parts.push(part(s.boundElsewhere, "bound to another consumer")); + parts.forEach((p, i) => { + if (i) this._summary.append(" · "); + this._summary.append(p); + }); + this._summary.hidden = false; + + this._list.replaceChildren(...(report.candidates || []).map((c) => this._row(c))); + } + + _row(c) { + const li = h("li"); + const who = h("div", "who"); + who.append(h("span", "name", c.instance)); + if (c.name && c.name !== c.instance) who.append(h("span", "muted", c.name)); + if (c.version) who.append(h("span", "muted", `v${c.version}`)); + const chips = h("span", "chips"); + for (const cap of c.caps || []) chips.append(h("span", "chip", cap)); + who.append(chips); + + const actions = h("div", "row-actions"); + const state = h("span", `state ${c.state}`, STATE_TEXT[c.state] || c.state); + if (c.state === "configured" && c.configuredAs) state.title = `listed as "${c.configuredAs}"`; + if (c.state === "bound-elsewhere" && c.binds) state.title = `belongs to ${c.binds}`; + actions.append(state); + if (c.state === "addable") { + const add = h("button", "add", "Add"); + add.type = "button"; + add.addEventListener("click", () => this._add(c, add, li)); + actions.append(add); + } + + const where = h("div"); + where.append(h("code", null, c.url)); + li.append(who, actions, where); + return li; + } + + async _add(candidate, btn, li) { + const endpoint = this.getAttribute("add-endpoint"); + if (!endpoint) return; + let extra = {}; + try { + extra = JSON.parse(this.getAttribute("add-extra") || "{}"); + } catch { + /* a malformed attribute must not block the add */ + } + btn.disabled = true; + btn.textContent = "Adding…"; + li.querySelector(".row-err")?.remove(); + try { + const res = await fetch(endpoint, { + method: "POST", + headers: { "Content-Type": "application/json", Accept: "application/json" }, + body: JSON.stringify({ ...extra, name: candidate.suggestedName, url: candidate.url }), + }); + if (!res.ok) throw new Error(await readError(res)); + const text = await res.text(); + let response = null; + try { + response = text ? JSON.parse(text) : null; + } catch { + response = text; + } + this.dispatchEvent( + new CustomEvent("beacon-added", { bubbles: true, composed: true, detail: { candidate, response } }), + ); + btn.textContent = "Added"; + } catch (e) { + btn.disabled = false; + btn.textContent = "Add"; + li.append(h("div", "row-err", e.message)); + } + } +} + +if (!customElements.get("meta-beacon-scan")) { + customElements.define("meta-beacon-scan", MetaBeaconScan); +} + +export default MetaBeaconScan; diff --git a/packages/meta-sort-ui/src/pages/Plugins.tsx b/packages/meta-sort-ui/src/pages/Plugins.tsx index 20b3809..d577891 100644 --- a/packages/meta-sort-ui/src/pages/Plugins.tsx +++ b/packages/meta-sort-ui/src/pages/Plugins.tsx @@ -5,6 +5,7 @@ import { PluginTimingsResponse } from '../types'; import { formatMs, formatNumber } from '../utils/format'; +import BeaconScanCard from '../components/BeaconScanCard'; interface AddPluginForm { pluginId: string; @@ -33,6 +34,8 @@ function Plugins() { cpus: '0.5', }); const [restartAllStatus, setRestartAllStatus] = useState(null); + // Plugins meta-sort reaches by URL but did not spawn (added from the Scan card). + const [externalUrls, setExternalUrls] = useState>({}); const fetchPlugins = useCallback(async () => { try { @@ -41,6 +44,14 @@ function Plugins() { const data = await res.json(); setPluginsData(data); setError(null); + fetch('/api/plugins/containers') + .then((r) => (r.ok ? r.json() : null)) + .then((c: { plugins?: Array<{ id: string; kind?: string; url?: string }> } | null) => { + const map: Record = {}; + for (const p of c?.plugins ?? []) if (p.kind === 'external' && p.url) map[p.id] = p.url; + setExternalUrls(map); + }) + .catch(() => {}); } else if (res.status === 503) { setError('Plugin manager not yet initialized. Process a file to initialize plugins.'); } @@ -370,6 +381,9 @@ function Plugins() {
    {plugin.name} v{plugin.version} + {externalUrls[plugin.id] && ( + external + )}