From eee9773f23daea8aada61ad495d22535f9c50689 Mon Sep 17 00:00:00 2001 From: Pierre Date: Sat, 7 Mar 2026 14:44:17 +0100 Subject: [PATCH] bugfix event --- Dockerfile | 8 +- .../src/api/UnifiedAPIServer.ts | 60 +-- packages/meta-sort-core/src/kv/KVManager.ts | 45 +-- .../meta-sort-core/src/kv/ServiceDiscovery.ts | 341 ------------------ .../src/kv/ServiceRegistration.ts | 199 ++++++++++ packages/meta-sort-core/src/kv/index.ts | 4 +- .../src/components/ServiceNav.tsx | 20 +- 7 files changed, 236 insertions(+), 441 deletions(-) delete mode 100644 packages/meta-sort-core/src/kv/ServiceDiscovery.ts create mode 100644 packages/meta-sort-core/src/kv/ServiceRegistration.ts diff --git a/Dockerfile b/Dockerfile index 170b0e2..4bd7085 100644 --- a/Dockerfile +++ b/Dockerfile @@ -27,13 +27,17 @@ RUN npm run build # Stage 2: Build Backend using pnpm workspace FROM node:21-alpine AS backend-builder -# Install pnpm -RUN npm install -g pnpm +# Install pnpm via corepack (version from package.json packageManager field) +ENV PNPM_HOME="/pnpm" +ENV PATH="$PNPM_HOME:$PATH" +ENV COREPACK_ENABLE_DOWNLOAD_PROMPT=0 +RUN npm install -g corepack@latest && corepack enable WORKDIR /build # Copy workspace configuration COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./ +RUN corepack install # Copy all packages for the workspace COPY packages/shared/ ./packages/shared/ diff --git a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts index 8470d7d..242547b 100644 --- a/packages/meta-sort-core/src/api/UnifiedAPIServer.ts +++ b/packages/meta-sort-core/src/api/UnifiedAPIServer.ts @@ -201,10 +201,7 @@ export class UnifiedAPIServer { this.setupStatsRoutes(); } - // Service discovery routes (under /api/services) - if (this.kvManager) { - this.setupServicesRoutes(); - } + // Note: /api/services is handled by nginx proxy to meta-core (centralized service discovery) // Plugin management routes (under /api/plugins) // Always register - routes check for plugin manager at request time @@ -864,61 +861,6 @@ export class UnifiedAPIServer { }); } - /** - * Setup Services Discovery API routes - * Returns list of discovered services for dashboard navigation - */ - private setupServicesRoutes(): void { - if (!this.kvManager) { - return; - } - - // Get discovered services (for inter-service navigation) - this.app.get('/api/services', async (request, reply) => { - const services: Array<{ - name: string; - url: string; - api: string; - status: string; - role?: string; - }> = []; - - try { - const serviceDiscovery = this.kvManager!.getServiceDiscovery(); - if (serviceDiscovery) { - const allServices = await serviceDiscovery.discoverAllServices(); - - for (const svc of allServices) { - services.push({ - name: svc.name || 'Unknown', - url: svc.baseUrl || '', - api: svc.baseUrl || '', - status: svc.status || 'unknown', - role: svc.role, - }); - } - } - } catch (error: any) { - console.error('[Services API] Error discovering services:', error); - } - - // Get leader info - const leaderInfo = this.kvManager!.getLeaderInfo(); - - return { - services, - current: 'meta-sort', - leader: leaderInfo ? { - hostname: leaderInfo.hostname, - baseUrl: leaderInfo.baseUrl, - apiUrl: leaderInfo.apiUrl, - redisUrl: leaderInfo.redisUrl, - webdavUrl: leaderInfo.webdavUrl, - } : null, - }; - }); - } - /** * Setup Plugin Management API routes */ diff --git a/packages/meta-sort-core/src/kv/KVManager.ts b/packages/meta-sort-core/src/kv/KVManager.ts index 0da180f..cd7b60b 100644 --- a/packages/meta-sort-core/src/kv/KVManager.ts +++ b/packages/meta-sort-core/src/kv/KVManager.ts @@ -28,7 +28,7 @@ import { hostname } from 'os'; import { LeaderClient } from './LeaderClient.js'; -import { ServiceDiscovery } from './ServiceDiscovery.js'; +import { ServiceRegistration } from './ServiceRegistration.js'; import { RedisKVClient } from './RedisClient.js'; import type { IKVClient, LeaderLockInfo } from './IKVClient.js'; import { configure as configureWebdav } from '../webdav/WebdavClient.js'; @@ -59,7 +59,7 @@ interface KVManagerConfig { export class KVManager { private config: KVManagerConfig; private leaderClient: LeaderClient | null = null; - private serviceDiscovery: ServiceDiscovery | null = null; + private serviceRegistration: ServiceRegistration | null = null; private kvClient: IKVClient | null = null; private isStarted = false; private isShuttingDown = false; @@ -124,9 +124,10 @@ export class KVManager { console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`); this.kvClient = await this.createRedisClient(leaderInfo.redisUrl); - // Configure WebDAV client with leader's WebDAV URL - if (leaderInfo.webdavUrl) { - configureWebdav(leaderInfo.webdavUrl); + // Configure WebDAV client with leader's WebDAV URL (prefer internal for container-to-container) + const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl; + if (webdavUrl) { + configureWebdav(webdavUrl); } } catch (error) { console.error('[KVManager] Failed to connect to leader:', error); @@ -136,15 +137,15 @@ export class KVManager { // Start watching for leader changes this.leaderClient.startWatching(); - // Initialize and start service discovery for registration and heartbeat + // Initialize and start service registration for heartbeat const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`; - this.serviceDiscovery = new ServiceDiscovery({ + this.serviceRegistration = new ServiceRegistration({ metaCorePath: this.config.metaCorePath, serviceName: this.config.serviceName, version: '1.0.0', apiUrl }); - await this.serviceDiscovery.start(); + await this.serviceRegistration.start(); this.isStarted = true; this.notifyReady(); @@ -170,9 +171,10 @@ export class KVManager { console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`); this.kvClient = await this.createRedisClient(leaderInfo.redisUrl); - // Reconfigure WebDAV client with new leader's WebDAV URL - if (leaderInfo.webdavUrl) { - configureWebdav(leaderInfo.webdavUrl); + // Reconfigure WebDAV client with new leader's WebDAV URL (prefer internal for container-to-container) + const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl; + if (webdavUrl) { + configureWebdav(webdavUrl); } // Notify reconnect callbacks @@ -209,10 +211,10 @@ export class KVManager { this.kvClient = null; } - // Service discovery stop (if it was started) - if (this.serviceDiscovery) { - await this.serviceDiscovery.stop(); - this.serviceDiscovery = null; + // Service registration stop (if it was started) + if (this.serviceRegistration) { + await this.serviceRegistration.stop(); + this.serviceRegistration = null; } this.isStarted = false; @@ -308,17 +310,10 @@ export class KVManager { } /** - * Get service discovery instance + * Get service registration instance */ - getServiceDiscovery(): ServiceDiscovery | null { - return this.serviceDiscovery; - } - - /** - * Discover another service - */ - async discoverService(name: string) { - return this.serviceDiscovery?.discoverService(name) ?? null; + getServiceRegistration(): ServiceRegistration | null { + return this.serviceRegistration; } /** diff --git a/packages/meta-sort-core/src/kv/ServiceDiscovery.ts b/packages/meta-sort-core/src/kv/ServiceDiscovery.ts deleted file mode 100644 index 89ce165..0000000 --- a/packages/meta-sort-core/src/kv/ServiceDiscovery.ts +++ /dev/null @@ -1,341 +0,0 @@ -/** - * Service Discovery using shared filesystem - * - * Each service registers itself in /meta-core/services/{service-name}-{hostname}.json - * with its base URL, status, and heartbeat timestamp. - * - * Services can discover each other by reading these JSON files. - * Stale detection: if lastHeartbeat > 60 seconds ago, service is considered stale. - * - * Each service must call start() to register itself and begin heartbeat updates. - * meta-core runs a cleanup process that removes services with stale heartbeats. - */ - -import { promises as fs } from 'fs'; -import { hostname } from 'os'; -import { join, basename } from 'path'; -import type { ServiceInfo } from './IKVClient.js'; -import { isNodeError } from '../types/ExtendedInterfaces.js'; - -interface ServiceDiscoveryConfig { - /** Path to META_CORE_VOLUME (e.g., /meta-core) */ - metaCorePath: string; - - /** Service name */ - serviceName: string; - - /** Service version (optional, not used in simplified format) */ - version?: string; - - /** Base URL for the service */ - apiUrl: string; - - /** Heartbeat interval in ms (default: 30000) */ - heartbeatInterval?: number; - - /** Stale threshold in ms (default: 60000) */ - staleThreshold?: number; -} - -export class ServiceDiscovery { - private config: ServiceDiscoveryConfig; - private servicesDir: string; - private serviceFilePath: string; - private heartbeatTimer: NodeJS.Timeout | null = null; - private isShuttingDown = false; - private currentHostname: string; - - constructor(config: ServiceDiscoveryConfig) { - this.config = { - heartbeatInterval: 30000, - staleThreshold: 60000, - ...config - }; - - this.currentHostname = hostname(); - this.servicesDir = `${this.config.metaCorePath}/services`; - // Use hostname-based file naming - this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`; - } - - /** - * Ensure services directory exists - */ - private async ensureServicesDir(): Promise { - await fs.mkdir(this.servicesDir, { recursive: true }); - } - - /** - * Build service info for this service - */ - private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo { - return { - name: this.config.serviceName, - hostname: this.currentHostname, - baseUrl: this.config.apiUrl, - status, - lastHeartbeat: new Date().toISOString() - }; - } - - /** - * Register this service by writing service info to the shared filesystem. - * meta-core's cleanup process monitors these files and removes stale entries. - */ - async register(): Promise { - await this.ensureServicesDir(); - - const info = this.buildServiceInfo('running'); - await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); - - console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}-${this.currentHostname}`); - } - - /** - * Update service status - */ - async updateStatus(status: ServiceInfo['status']): Promise { - try { - const content = await fs.readFile(this.serviceFilePath, 'utf-8'); - const info = JSON.parse(content) as ServiceInfo; - - info.status = status; - info.lastHeartbeat = new Date().toISOString(); - - await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); - } catch (error) { - console.error('[ServiceDiscovery] Failed to update status:', error); - } - } - - /** - * Send heartbeat (update lastHeartbeat timestamp) - */ - async heartbeat(): Promise { - try { - const content = await fs.readFile(this.serviceFilePath, 'utf-8'); - const info = JSON.parse(content) as ServiceInfo; - - info.lastHeartbeat = new Date().toISOString(); - - await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); - } catch (error) { - // If file was deleted, re-register - if (isNodeError(error) && error.code === 'ENOENT') { - await this.register(); - } else { - console.error('[ServiceDiscovery] Heartbeat failed:', error); - } - } - } - - /** - * Start heartbeat loop - */ - startHeartbeat(): void { - this.heartbeatTimer = setInterval(async () => { - if (this.isShuttingDown) return; - await this.heartbeat(); - }, this.config.heartbeatInterval); - } - - /** - * Stop heartbeat loop - */ - stopHeartbeat(): void { - if (this.heartbeatTimer) { - clearInterval(this.heartbeatTimer); - this.heartbeatTimer = null; - } - } - - /** - * Unregister this service (on shutdown) - */ - async unregister(): Promise { - this.isShuttingDown = true; - this.stopHeartbeat(); - - try { - // Remove the service file - await fs.unlink(this.serviceFilePath); - console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}-${this.currentHostname}`); - } catch (error) { - if (!isNodeError(error) || error.code !== 'ENOENT') { - console.error('[ServiceDiscovery] Failed to unregister:', error); - } - } - } - - /** - * Check if a service info is stale - */ - private isStale(info: ServiceInfo): boolean { - const lastHeartbeat = new Date(info.lastHeartbeat).getTime(); - return Date.now() - lastHeartbeat > this.config.staleThreshold!; - } - - // ======================================================================== - // Discovery Methods - // ======================================================================== - - /** - * Discover a service by name - * Looks for files matching pattern: {name}-*.json (hostname-based naming) - */ - async discoverService(name: string): Promise { - try { - // First try exact match for backward compatibility - const exactPath = join(this.servicesDir, `${name}.json`); - try { - const content = await fs.readFile(exactPath, 'utf-8'); - const service = JSON.parse(content) as ServiceInfo; - - if (this.isStale(service)) { - service.status = 'stale'; - } - - return service.status === 'running' ? service : null; - } catch { - // Try hostname-based files - } - - // Search for hostname-based files: name-*.json - const files = await fs.readdir(this.servicesDir); - const matchingFiles = files.filter(f => - f.startsWith(`${name}-`) && f.endsWith('.json') - ); - - // Return first valid (non-stale) service - for (const file of matchingFiles) { - const filePath = join(this.servicesDir, file); - const content = await fs.readFile(filePath, 'utf-8'); - const service = JSON.parse(content) as ServiceInfo; - - if (this.isStale(service)) { - service.status = 'stale'; - continue; // Skip stale services - } - - if (service.status === 'running') { - return service; - } - } - - return null; - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - return null; // Service not registered - } - throw error; - } - } - - /** - * Discover all registered services - */ - async discoverAllServices(): Promise { - try { - const files = await fs.readdir(this.servicesDir); - const services: ServiceInfo[] = []; - - for (const file of files) { - if (!file.endsWith('.json')) continue; - - const filePath = join(this.servicesDir, file); - try { - const content = await fs.readFile(filePath, 'utf-8'); - const service = JSON.parse(content) as ServiceInfo; - - if (this.isStale(service)) { - service.status = 'stale'; - } - - services.push(service); - } catch (error) { - console.error(`[ServiceDiscovery] Failed to read ${file}:`, error); - } - } - - return services; - } catch (error) { - if (isNodeError(error) && error.code === 'ENOENT') { - return []; // Services directory doesn't exist yet - } - throw error; - } - } - - /** - * Wait for a service to become available - */ - async waitForService( - name: string, - timeoutMs: number = 30000, - pollInterval: number = 1000 - ): Promise { - const startTime = Date.now(); - - while (Date.now() - startTime < timeoutMs) { - const service = await this.discoverService(name); - if (service && service.status === 'running') { - return service; - } - - await new Promise(resolve => setTimeout(resolve, pollInterval)); - } - - throw new Error(`Service ${name} not available after ${timeoutMs}ms`); - } - - /** - * Check if a service is healthy - */ - async isServiceHealthy(name: string): Promise { - const service = await this.discoverService(name); - if (!service) return false; - - // Try to ping the service's health endpoint - const url = `${service.baseUrl}/health`; - - try { - const response = await fetch(url, { - signal: AbortSignal.timeout(5000) - }); - return response.ok; - } catch { - return false; - } - } - - // ======================================================================== - // Lifecycle - // ======================================================================== - - /** - * Full startup sequence: register service and start heartbeat loop. - */ - async start(): Promise { - await this.register(); - this.startHeartbeat(); - } - - /** - * Full shutdown sequence - */ - async stop(): Promise { - await this.unregister(); - } - - // ======================================================================== - // Getters - // ======================================================================== - - getServiceFilePath(): string { - return this.serviceFilePath; - } - - getServicesDir(): string { - return this.servicesDir; - } -} diff --git a/packages/meta-sort-core/src/kv/ServiceRegistration.ts b/packages/meta-sort-core/src/kv/ServiceRegistration.ts new file mode 100644 index 0000000..2251308 --- /dev/null +++ b/packages/meta-sort-core/src/kv/ServiceRegistration.ts @@ -0,0 +1,199 @@ +/** + * Service Registration using shared filesystem + * + * Each service registers itself in /meta-core/services/{service-name}-{hostname}.json + * with its base URL, status, and heartbeat timestamp. + * + * Service discovery is centralized in meta-core - this class only handles registration. + * For service discovery, use the /api/services endpoint (proxied to meta-core via nginx). + * + * Each service must call start() to register itself and begin heartbeat updates. + * meta-core runs a cleanup process that removes services with stale heartbeats. + */ + +import { promises as fs } from 'fs'; +import { hostname } from 'os'; +import type { ServiceInfo } from './IKVClient.js'; +import { isNodeError } from '../types/ExtendedInterfaces.js'; + +interface ServiceRegistrationConfig { + /** Path to META_CORE_VOLUME (e.g., /meta-core) */ + metaCorePath: string; + + /** Service name */ + serviceName: string; + + /** Service version (optional, not used in simplified format) */ + version?: string; + + /** Base URL for the service */ + apiUrl: string; + + /** Heartbeat interval in ms (default: 30000) */ + heartbeatInterval?: number; + + /** Stale threshold in ms (default: 60000) */ + staleThreshold?: number; +} + +export class ServiceRegistration { + private config: ServiceRegistrationConfig; + private servicesDir: string; + private serviceFilePath: string; + private heartbeatTimer: NodeJS.Timeout | null = null; + private isShuttingDown = false; + private currentHostname: string; + + constructor(config: ServiceRegistrationConfig) { + this.config = { + heartbeatInterval: 30000, + staleThreshold: 60000, + ...config + }; + + this.currentHostname = hostname(); + this.servicesDir = `${this.config.metaCorePath}/services`; + // Use hostname-based file naming + this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`; + } + + /** + * Ensure services directory exists + */ + private async ensureServicesDir(): Promise { + await fs.mkdir(this.servicesDir, { recursive: true }); + } + + /** + * Build service info for this service + */ + private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo { + return { + name: this.config.serviceName, + hostname: this.currentHostname, + baseUrl: this.config.apiUrl, + status, + lastHeartbeat: new Date().toISOString() + }; + } + + /** + * Register this service by writing service info to the shared filesystem. + * meta-core's cleanup process monitors these files and removes stale entries. + */ + async register(): Promise { + await this.ensureServicesDir(); + + const info = this.buildServiceInfo('running'); + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + + console.log(`[ServiceRegistration] Registered ${this.config.serviceName}-${this.currentHostname}`); + } + + /** + * Update service status + */ + async updateStatus(status: ServiceInfo['status']): Promise { + try { + const content = await fs.readFile(this.serviceFilePath, 'utf-8'); + const info = JSON.parse(content) as ServiceInfo; + + info.status = status; + info.lastHeartbeat = new Date().toISOString(); + + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + } catch (error) { + console.error('[ServiceRegistration] Failed to update status:', error); + } + } + + /** + * Send heartbeat (update lastHeartbeat timestamp) + */ + async heartbeat(): Promise { + try { + const content = await fs.readFile(this.serviceFilePath, 'utf-8'); + const info = JSON.parse(content) as ServiceInfo; + + info.lastHeartbeat = new Date().toISOString(); + + await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2)); + } catch (error) { + // If file was deleted, re-register + if (isNodeError(error) && error.code === 'ENOENT') { + await this.register(); + } else { + console.error('[ServiceRegistration] Heartbeat failed:', error); + } + } + } + + /** + * Start heartbeat loop + */ + startHeartbeat(): void { + this.heartbeatTimer = setInterval(async () => { + if (this.isShuttingDown) return; + await this.heartbeat(); + }, this.config.heartbeatInterval); + } + + /** + * Stop heartbeat loop + */ + stopHeartbeat(): void { + if (this.heartbeatTimer) { + clearInterval(this.heartbeatTimer); + this.heartbeatTimer = null; + } + } + + /** + * Unregister this service (on shutdown) + */ + async unregister(): Promise { + this.isShuttingDown = true; + this.stopHeartbeat(); + + try { + // Remove the service file + await fs.unlink(this.serviceFilePath); + console.log(`[ServiceRegistration] Unregistered ${this.config.serviceName}-${this.currentHostname}`); + } catch (error) { + if (!isNodeError(error) || error.code !== 'ENOENT') { + console.error('[ServiceRegistration] Failed to unregister:', error); + } + } + } + + // ======================================================================== + // Lifecycle + // ======================================================================== + + /** + * Full startup sequence: register service and start heartbeat loop. + */ + async start(): Promise { + await this.register(); + this.startHeartbeat(); + } + + /** + * Full shutdown sequence + */ + async stop(): Promise { + await this.unregister(); + } + + // ======================================================================== + // Getters + // ======================================================================== + + getServiceFilePath(): string { + return this.serviceFilePath; + } + + getServicesDir(): string { + return this.servicesDir; + } +} diff --git a/packages/meta-sort-core/src/kv/index.ts b/packages/meta-sort-core/src/kv/index.ts index fd40721..0852123 100644 --- a/packages/meta-sort-core/src/kv/index.ts +++ b/packages/meta-sort-core/src/kv/index.ts @@ -5,7 +5,7 @@ * - IKVClient interface for KV storage abstraction * - RedisKVClient for Redis-based storage * - LeaderClient for reading leader info from meta-core - * - ServiceDiscovery for service registration and discovery + * - ServiceRegistration for service registration (discovery is centralized in meta-core) * - KVManager for unified management of all components * - MetadataUtils for flattening/reconstructing nested metadata */ @@ -13,6 +13,6 @@ export * from './IKVClient.js'; export * from './RedisClient.js'; export * from './LeaderClient.js'; -export * from './ServiceDiscovery.js'; +export * from './ServiceRegistration.js'; export * from './KVManager.js'; export * from './MetadataUtils.js'; diff --git a/packages/meta-sort-ui/src/components/ServiceNav.tsx b/packages/meta-sort-ui/src/components/ServiceNav.tsx index db407b0..c88f65e 100644 --- a/packages/meta-sort-ui/src/components/ServiceNav.tsx +++ b/packages/meta-sort-ui/src/components/ServiceNav.tsx @@ -2,12 +2,10 @@ import { useState, useEffect } from 'react'; interface ServiceInfo { name: string; - url: string; - api: string; + hostname: string; + baseUrl: string; status: string; - capabilities: string[]; - version: string; - role?: string; // "leader", "follower", or undefined (for non-meta-core services) + lastHeartbeat: string; } interface ServicesResponse { @@ -19,6 +17,8 @@ const serviceIcons: Record = { 'meta-sort': '📁', 'meta-fuse': '🗂️', 'meta-stremio': '🎬', + 'meta-core': '⚙️', + 'meta-dup': '🔍', 'meta-orbit': '🌐', 'default': '📦' }; @@ -56,11 +56,7 @@ function ServiceNav() { if (loading) return null; - // Filter out follower meta-core instances (only show leader) - const filteredServices = services.filter( - s => s.name !== 'meta-core' || s.role === 'leader' - ); - const sortedServices = [...filteredServices].sort((a, b) => a.name.localeCompare(b.name)); + const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name)); if (sortedServices.length === 0) return null; @@ -74,12 +70,12 @@ function ServiceNav() { return ( { e.preventDefault(); if (!isActive) { - window.location.href = service.url; + window.location.href = service.baseUrl; } }} >