metastar/meta-sort
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

leader discovery fix and code cleaning

Browse Source
This commit is contained in:
Pierre
2026-02-01 13:39:42 +01:00
parent f220972cb2
commit 34ff1375c6
13 changed files with 4961 additions and 878 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/meta-sort-core/src/api/UnifiedAPIServer.ts
+7 -15
View File
@@ -988,8 +988,6 @@ export class UnifiedAPIServer {
url: string;
api: string;
status: string;
capabilities: string[];
version: string;
}> = [];
try {
@@ -998,17 +996,11 @@ export class UnifiedAPIServer {
const allServices = await serviceDiscovery.discoverAllServices();
for (const svc of allServices) {
// Build dashboard URL from API URL
const apiUrl = svc.api || '';
const dashboardPath = svc.endpoints?.dashboard || '/';
services.push({
name: svc.name || 'Unknown',
url: apiUrl + dashboardPath,
api: apiUrl,
url: svc.baseUrl || '',
api: svc.baseUrl || '',
status: svc.status || 'unknown',
capabilities: svc.capabilities || [],
version: svc.version || '',
});
}
}
@@ -1018,17 +1010,17 @@ export class UnifiedAPIServer {
// Get leader info
const leaderInfo = this.kvManager!.getLeaderInfo();
const isLeader = this.kvManager!.isLeader();
return {
services,
current: 'meta-sort',
leader: leaderInfo ? {
host: leaderInfo.host,
api: leaderInfo.api,
http: leaderInfo.http,
hostname: leaderInfo.hostname,
baseUrl: leaderInfo.baseUrl,
apiUrl: leaderInfo.apiUrl,
redisUrl: leaderInfo.redisUrl,
webdavUrl: leaderInfo.webdavUrl,
} : null,
isLeader,
};
});
}
packages/meta-sort-core/src/index.ts
+17 -13
View File
@@ -141,26 +141,22 @@ process.on('SIGINT', async () => {
// Start initial discovery and processing (streaming pipeline)
(async () => {
try {
// Validate META_CORE_URL is configured
if (!config.META_CORE_URL) {
console.error('[Startup] ERROR: META_CORE_URL is required. Set META_CORE_URL environment variable.');
console.error('[Startup] Example: META_CORE_URL=http://meta-core');
// Validate META_CORE_PATH is configured (required for leader discovery)
if (!config.META_CORE_PATH) {
console.error('[Startup] ERROR: META_CORE_PATH is required. Set META_CORE_PATH environment variable.');
console.error('[Startup] Example: META_CORE_PATH=/meta-core');
process.exit(1);
}
// Initialize KV Manager if configured (Redis + leader election)
// Initialize KV Manager (Redis via leader discovery)
if (config.META_CORE_PATH) {
console.log('[Startup] Initializing KV Manager (Redis + leader election)...');
console.log('[Startup] Initializing KV Manager (leader discovery)...');
kvManager = new KVManager({
metaCorePath: config.META_CORE_PATH,
filesPath: config.FILES_PATH,
serviceName: config.SERVICE_NAME,
version: config.SERVICE_VERSION,
apiPort: config.FUSE_API_PORT,
redisPort: config.REDIS_PORT,
redisUrl: config.REDIS_URL,
baseUrl: config.BASE_URL,
capabilities: ['write', 'mount', 'monitor']
});
// Wait for KV to be ready
@@ -172,7 +168,7 @@ process.on('SIGINT', async () => {
if (client) {
fileProcessor.setKVClient(client);
pipeline.setKVClient(client);
console.log(`[Startup] KV Manager ready (role: ${kvManager.isLeader() ? 'LEADER' : 'FOLLOWER'})`);
console.log('[Startup] KV Manager ready');
}
}
@@ -266,9 +262,17 @@ process.on('SIGINT', async () => {
}
// Connect to meta-core for file events via SSE (Architecture V3)
console.log(`[Startup] Connecting to meta-core at ${config.META_CORE_URL}...`);
// Get meta-core URL from leader discovery
const leaderInfo = kvManager?.getLeaderInfo();
if (!leaderInfo) {
console.error('[Startup] ERROR: No leader info available. Cannot connect to meta-core.');
console.error('[Startup] Make sure meta-core is running and kv-leader.info exists.');
process.exit(1);
}
const metaCoreUrl = leaderInfo.baseUrl;
console.log(`[Startup] Connecting to meta-core at ${metaCoreUrl}...`);
eventSubscriber = new EventSubscriber({
metaCoreUrl: config.META_CORE_URL,
metaCoreUrl: metaCoreUrl,
pipeline: pipeline,
requestInitialScan: true, // Request initial scan from meta-core
reconnectDelayMs: 5000,
packages/meta-sort-core/src/kv/IKVClient.ts
+19 -29
View File
@@ -139,20 +139,24 @@ export interface KVClientConfig {
}
/**
* Leader lock file content format
* Leader lock file content format (kv-leader.info)
* Written by meta-core leader, read by other services
*/
export interface LeaderLockInfo {
/** Hostname of the leader */
host: string;
hostname: string;
/** Base URL for the leader service (e.g., http://localhost:8180) */
baseUrl: string;
/** meta-core API URL (port 9000) */
apiUrl: string;
/** Redis connection URL (e.g., redis://10.0.1.50:6379) */
api: string;
redisUrl: string;
/** HTTP API URL for the leader service */
http: string;
/** Configured BASE_URL for stable service discovery (optional, falls back to http) */
baseUrl?: string;
/** WebDAV URL for file access */
webdavUrl: string;
/** Timestamp when leadership was acquired */
timestamp: number;
@@ -162,36 +166,22 @@ export interface LeaderLockInfo {
}
/**
* Service registration info
* Service registration info (simplified)
* Full URLs are obtained via the /urls API endpoint
*/
export interface ServiceInfo {
/** Service name (e.g., 'meta-sort', 'meta-fuse') */
name: string;
/** Service version */
version: string;
/** HTTP API URL */
api: string;
/** Current status */
status: 'starting' | 'running' | 'stopping' | 'stopped';
/** Process ID */
pid: number;
/** Hostname */
hostname: string;
/** Start timestamp (ISO string) */
startedAt: string;
/** Base URL for the service */
baseUrl: string;
/** Current status */
status: 'running' | 'stale' | 'stopped';
/** Last heartbeat timestamp (ISO string) */
lastHeartbeat: string;
/** Service capabilities */
capabilities: string[];
/** Named endpoints */
endpoints: Record<string, string>;
}
packages/meta-sort-core/src/kv/KVManager.ts
+90 -151
View File
@@ -1,11 +1,14 @@
/**
* KV Manager - Unified manager for KV storage, leader election, and service discovery
* KV Manager - Unified manager for KV storage and service discovery
*
* This is the main entry point for the KV subsystem. It:
* 1. Participates in leader election (spawns Redis if leader)
* 1. Uses LeaderClient to read leader info from meta-core
* 2. Creates and manages the KV client (Redis)
* 3. Handles service discovery registration
* 4. Manages reconnection on leader failure
* 4. Manages reconnection on leader changes
*
* Note: Leader election is now handled by meta-core (Go sidecar).
* This service only reads the leader info and connects to Redis.
*
* Usage:
* ```typescript
@@ -23,8 +26,8 @@
* ```
*/
import { networkInterfaces } from 'os';
import { LeaderElection } from './LeaderElection.js';
import { hostname } from 'os';
import { LeaderClient } from './LeaderClient.js';
import { ServiceDiscovery } from './ServiceDiscovery.js';
import { RedisKVClient } from './RedisClient.js';
import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
@@ -39,34 +42,22 @@ interface KVManagerConfig {
/** Service name (e.g., 'meta-sort', 'meta-fuse') */
serviceName: string;
/** Service version */
version?: string;
/** HTTP API port */
apiPort: number;
/** Redis port (default: 6379) */
redisPort?: number;
/** Service capabilities */
capabilities?: string[];
/** Whether to skip leader election and use provided Redis URL */
redisUrl?: string;
/** Base URL for service discovery (overrides auto-detected URL) */
baseUrl?: string;
/**
* Hostname to advertise in lock file for other services to connect.
* Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution.
*/
advertiseHost?: string;
/** Optional direct Redis URL (bypasses leader discovery) */
redisUrl?: string;
/** meta-core API URL (e.g., http://localhost:9000) */
metaCoreUrl?: string;
}
export class KVManager {
private config: KVManagerConfig;
private leaderElection: LeaderElection | null = null;
private leaderClient: LeaderClient | null = null;
private serviceDiscovery: ServiceDiscovery | null = null;
private kvClient: IKVClient | null = null;
private isStarted = false;
@@ -77,32 +68,7 @@ export class KVManager {
private onReconnectCallbacks: (() => void)[] = [];
constructor(config: KVManagerConfig) {
this.config = {
version: '1.0.0',
redisPort: 6379,
capabilities: [],
...config
};
}
/**
* Get the local IP for API URL construction
*/
private getLocalIP(): string {
const interfaces = networkInterfaces();
for (const name of Object.keys(interfaces)) {
const addrs = interfaces[name];
if (!addrs) continue;
for (const addr of addrs) {
if (addr.family === 'IPv4' && !addr.internal) {
return addr.address;
}
}
}
return 'localhost';
this.config = config;
}
/**
@@ -130,106 +96,86 @@ export class KVManager {
console.log(`[KVManager] Starting for ${this.config.serviceName}...`);
// Check for direct Redis URL (skip leader election)
// Check for direct Redis URL (skip leader discovery)
if (this.config.redisUrl) {
console.log(`[KVManager] Using direct Redis URL: ${this.config.redisUrl}`);
this.kvClient = await this.createRedisClient(this.config.redisUrl);
// NOTE: Service discovery is handled by Go meta-core sidecar
// The Go binary writes to /meta-core/services/{serviceName}.json with correct internal IPs
// TypeScript ServiceDiscovery would overwrite with external BASE_URL, breaking internal discovery
// Keep ServiceDiscovery instance for reading/discovery only, but don't call start()
const apiUrl = this.config.baseUrl || `http://localhost:${this.config.apiPort}`;
this.serviceDiscovery = new ServiceDiscovery({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
version: this.config.version!,
apiUrl,
capabilities: this.config.capabilities,
endpoints: {
health: '/health',
dashboard: '/',
api: '/api',
metrics: '/api/metrics',
processing: '/api/processing/status'
}
});
// Don't call start() - Go meta-core handles registration
// await this.serviceDiscovery.start();
this.isStarted = true;
this.notifyReady();
return;
}
// Initialize leader election
const localIP = this.getLocalIP();
this.leaderElection = new LeaderElection({
// Initialize leader client
this.leaderClient = new LeaderClient({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
apiPort: this.config.apiPort,
redisPort: this.config.redisPort,
baseUrl: this.config.baseUrl
metaCoreUrl: this.config.metaCoreUrl
});
// Setup leader election callbacks
this.leaderElection.onLeader(async () => {
console.log('[KVManager] We are the leader, connecting to local Redis...');
// Connect to local Redis (spawned by leader election)
const redisUrl = `redis://127.0.0.1:${this.config.redisPort}`;
this.kvClient = await this.createRedisClient(redisUrl);
this.notifyReady();
// Watch for leader changes
this.leaderClient.onChange(async () => {
console.log('[KVManager] Leader changed, reconnecting...');
await this.reconnect();
});
this.leaderElection.onFollower(async (leaderInfo: LeaderLockInfo) => {
console.log(`[KVManager] We are a follower, connecting to leader at ${leaderInfo.api}...`);
// Wait for leader and connect
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
} catch (error) {
console.error('[KVManager] Failed to connect to leader:', error);
throw error;
}
// Connect to leader's Redis
this.kvClient = await this.createRedisClient(leaderInfo.api);
this.notifyReady();
});
this.leaderElection.onLostLeader(async () => {
console.log('[KVManager] Leader lost, disconnecting...');
if (this.kvClient) {
await this.kvClient.close();
this.kvClient = null;
}
// Leader election will handle re-election and reconnection
});
// Initialize service discovery
// Use BASE_URL if provided, otherwise fall back to auto-detected local IP
const apiUrl = this.config.baseUrl || `http://${localIP}:${this.config.apiPort}`;
// Start watching for leader changes
this.leaderClient.startWatching();
// Initialize service discovery (meta-core handles actual registration)
// We keep this for service discovery (reading other services)
const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
this.serviceDiscovery = new ServiceDiscovery({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
version: this.config.version!,
apiUrl,
capabilities: this.config.capabilities,
endpoints: {
health: '/health',
dashboard: '/',
api: '/api',
metrics: '/api/metrics',
processing: '/api/processing/status'
}
version: '1.0.0',
apiUrl
});
// Start leader election first
await this.leaderElection.start();
// Start service discovery
await this.serviceDiscovery.start();
// Note: We don't start ServiceDiscovery - meta-core handles registration
this.isStarted = true;
this.notifyReady();
}
/**
* Reconnect to Redis after leader change
*/
private async reconnect(): Promise<void> {
if (this.isShuttingDown || !this.leaderClient) {
return;
}
// Close existing connection
if (this.kvClient) {
await this.kvClient.close();
this.kvClient = null;
}
// Wait for new leader
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
// Notify reconnect callbacks
for (const callback of this.onReconnectCallbacks) {
try {
callback();
} catch (error) {
console.error('[KVManager] Error in reconnect callback:', error);
}
}
} catch (error) {
console.error('[KVManager] Failed to reconnect:', error);
}
}
/**
@@ -241,10 +187,10 @@ export class KVManager {
console.log('[KVManager] Stopping...');
this.isShuttingDown = true;
// Stop service discovery
if (this.serviceDiscovery) {
await this.serviceDiscovery.stop();
this.serviceDiscovery = null;
// Stop leader client
if (this.leaderClient) {
this.leaderClient.close();
this.leaderClient = null;
}
// Close KV client
@@ -253,10 +199,10 @@ export class KVManager {
this.kvClient = null;
}
// Stop leader election (releases lock, stops Redis if leader)
if (this.leaderElection) {
await this.leaderElection.stop();
this.leaderElection = null;
// Service discovery stop (if it was started)
if (this.serviceDiscovery) {
await this.serviceDiscovery.stop();
this.serviceDiscovery = null;
}
this.isStarted = false;
@@ -337,25 +283,18 @@ export class KVManager {
return this.kvClient;
}
/**
* Check if this service is the leader
*/
isLeader(): boolean {
return this.leaderElection?.isLeader() ?? false;
}
/**
* Check if this service is a follower
*/
isFollower(): boolean {
return this.leaderElection?.isFollower() ?? false;
}
/**
* Get the current leader info
*/
getLeaderInfo(): LeaderLockInfo | null {
return this.leaderElection?.getLeaderInfo() ?? null;
return this.leaderClient?.getCachedLeaderInfo() ?? null;
}
/**
* Get the leader client
*/
getLeaderClient(): LeaderClient | null {
return this.leaderClient;
}
/**
packages/meta-sort-core/src/kv/LeaderClient.ts
+216
View File
@@ -0,0 +1,216 @@
/**
* LeaderClient - Client for reading leader info from meta-core
*
* This replaces the old LeaderElection.ts by delegating leader election
* to meta-core (Go sidecar) and just reading the results.
*
* Features:
* - Reads kv-leader.info file for leader URLs
* - Calls meta-core /urls API for current leader URLs
* - Watches for leader changes via file system watcher
*/
import { promises as fs } from 'fs';
import { watch, FSWatcher } from 'fs';
import { dirname } from 'path';
import type { LeaderLockInfo } from './IKVClient.js';
export interface LeaderClientConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** meta-core API URL (e.g., http://meta-core:9000 or http://localhost:9000) */
metaCoreUrl?: string;
}
export interface URLsResponse {
hostname: string;
baseUrl: string;
apiUrl: string;
redisUrl: string;
webdavUrl: string;
isLeader: boolean;
}
export class LeaderClient {
private config: LeaderClientConfig;
private infoFilePath: string;
private leaderInfo: LeaderLockInfo | null = null;
private watcher: FSWatcher | null = null;
private onChangeCallbacks: (() => void)[] = [];
constructor(config: LeaderClientConfig) {
this.config = config;
this.infoFilePath = `${config.metaCorePath}/locks/kv-leader.info`;
}
/**
* Read leader info from the info file
*/
async getLeaderInfo(): Promise<LeaderLockInfo | null> {
try {
const content = await fs.readFile(this.infoFilePath, 'utf-8');
this.leaderInfo = JSON.parse(content) as LeaderLockInfo;
return this.leaderInfo;
} catch (error) {
console.error('[LeaderClient] Failed to read leader info:', error);
return null;
}
}
/**
* Get Redis URL from leader info
*/
async getRedisUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.redisUrl ?? null;
}
/**
* Get WebDAV URL from leader info
*/
async getWebdavUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.webdavUrl ?? null;
}
/**
* Get meta-core API URL from leader info
*/
async getApiUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.apiUrl ?? null;
}
/**
* Call meta-core /urls API to get current URLs
* Useful for getting URLs when you don't want to read the file directly
*/
async getUrls(): Promise<URLsResponse | null> {
// First try using configured metaCoreUrl
let apiUrl = this.config.metaCoreUrl;
// Fall back to reading from leader info
if (!apiUrl) {
const info = await this.getLeaderInfo();
apiUrl = info?.apiUrl;
}
if (!apiUrl) {
console.error('[LeaderClient] No meta-core API URL available');
return null;
}
try {
const response = await fetch(`${apiUrl}/urls`, {
method: 'GET',
headers: { 'Accept': 'application/json' },
signal: AbortSignal.timeout(5000)
});
if (!response.ok) {
console.error('[LeaderClient] Failed to get URLs:', response.status, response.statusText);
return null;
}
return await response.json() as URLsResponse;
} catch (error) {
console.error('[LeaderClient] Error calling /urls API:', error);
return null;
}
}
/**
* Wait for leader info to be available
*/
async waitForLeader(timeoutMs: number = 30000): Promise<LeaderLockInfo> {
const startTime = Date.now();
const pollInterval = 500;
while (Date.now() - startTime < timeoutMs) {
const info = await this.getLeaderInfo();
if (info) {
console.log(`[LeaderClient] Leader found: ${info.hostname} at ${info.redisUrl}`);
return info;
}
console.log('[LeaderClient] Waiting for leader...');
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
throw new Error(`No leader found within ${timeoutMs}ms`);
}
/**
* Start watching for leader changes
*/
startWatching(): void {
if (this.watcher) {
return; // Already watching
}
const lockDir = dirname(this.infoFilePath);
try {
this.watcher = watch(lockDir, (eventType, filename) => {
if (filename === 'kv-leader.info') {
console.log('[LeaderClient] Leader info changed, reloading...');
this.getLeaderInfo().then(() => {
this.notifyChange();
}).catch(console.error);
}
});
console.log(`[LeaderClient] Watching for leader changes in ${lockDir}`);
} catch (error) {
console.error('[LeaderClient] Failed to start watching:', error);
}
}
/**
* Stop watching for leader changes
*/
stopWatching(): void {
if (this.watcher) {
this.watcher.close();
this.watcher = null;
console.log('[LeaderClient] Stopped watching for leader changes');
}
}
/**
* Register callback for leader changes
*/
onChange(callback: () => void): this {
this.onChangeCallbacks.push(callback);
return this;
}
/**
* Notify all change callbacks
*/
private notifyChange(): void {
for (const callback of this.onChangeCallbacks) {
try {
callback();
} catch (error) {
console.error('[LeaderClient] Error in change callback:', error);
}
}
}
/**
* Get cached leader info (without re-reading file)
*/
getCachedLeaderInfo(): LeaderLockInfo | null {
return this.leaderInfo;
}
/**
* Clean up resources
*/
close(): void {
this.stopWatching();
this.onChangeCallbacks = [];
}
}
packages/meta-sort-core/src/kv/LeaderElection.ts
-615
View File
@@ -1,615 +0,0 @@
/**
* Leader Election using flock on shared filesystem
*
* This implements a distributed leader election mechanism using:
* 1. flock (advisory file locking) for mutual exclusion
* 2. Shared filesystem as the consensus layer
* 3. Separate info file for service discovery
*
* Architecture:
* - Service tries to acquire exclusive flock on /meta-core/locks/kv-leader.lock
* - If acquired → becomes LEADER, spawns Redis, writes endpoint to info file
* - If blocked → becomes FOLLOWER, reads leader endpoint from info file
* - flock automatically releases when process dies (no stale lock issues!)
*
* Key advantage over O_EXCL approach:
* - Lock is automatically released when process exits/crashes
* - No need for timestamp-based stale detection
* - No race conditions during takeover
*/
import { promises as fs } from 'fs';
import { dirname } from 'path';
import { hostname, networkInterfaces } from 'os';
import { spawn, ChildProcess, execSync } from 'child_process';
import * as IORedis from 'ioredis';
import type { LeaderLockInfo } from './IKVClient.js';
// ESM/CJS interop for ioredis
const Redis = (IORedis as any).default ?? IORedis;
interface LeaderElectionConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** Service name for identification */
serviceName: string;
/** HTTP API port */
apiPort: number;
/** Redis port (default: 6379) */
redisPort?: number;
/** Redis data directory (relative to META_CORE_VOLUME/db/redis) */
redisDataDir?: string;
/** Health check interval in ms (default: 5000) */
healthCheckInterval?: number;
/** Base URL for stable service discovery (e.g., http://localhost:8180) */
baseUrl?: string;
}
type LeaderRole = 'leader' | 'follower' | 'unknown';
export class LeaderElection {
private config: LeaderElectionConfig;
private lockFilePath: string;
private infoFilePath: string;
private flockProcess: ChildProcess | null = null;
private role: LeaderRole = 'unknown';
private leaderInfo: LeaderLockInfo | null = null;
private redisProcess: ChildProcess | null = null;
private healthCheckTimer: NodeJS.Timeout | null = null;
private isShuttingDown = false;
// Event callbacks
private onBecomeLeader?: () => void;
private onBecomeFollower?: (leaderInfo: LeaderLockInfo) => void;
private onLeaderLost?: () => void;
constructor(config: LeaderElectionConfig) {
this.config = {
redisPort: 6379,
healthCheckInterval: 5000,
...config
};
this.lockFilePath = `${this.config.metaCorePath}/locks/kv-leader.lock`;
this.infoFilePath = `${this.config.metaCorePath}/locks/kv-leader.info`;
}
/**
* Get the local machine's IP address
*/
private getLocalIP(): string {
const interfaces = networkInterfaces();
// Prefer non-internal IPv4 addresses
for (const name of Object.keys(interfaces)) {
const addrs = interfaces[name];
if (!addrs) continue;
for (const addr of addrs) {
if (addr.family === 'IPv4' && !addr.internal) {
return addr.address;
}
}
}
// Fallback to hostname or localhost
return hostname() || 'localhost';
}
/**
* Build leader info for this service
*/
private buildLeaderInfo(): LeaderLockInfo {
const ip = this.getLocalIP();
return {
host: hostname(),
api: `redis://${ip}:${this.config.redisPort}`,
http: `http://${ip}:${this.config.apiPort}`,
baseUrl: this.config.baseUrl,
timestamp: Date.now(),
pid: process.pid
};
}
/**
* Ensure lock directory exists
*/
private async ensureLockDir(): Promise<void> {
const lockDir = dirname(this.lockFilePath);
await fs.mkdir(lockDir, { recursive: true });
}
/**
* Try to acquire the leader lock using flock
* Returns true if lock acquired (we are leader), false otherwise
*
* Uses `flock -n -x` for non-blocking exclusive lock attempt.
* The lock is held by keeping a child process alive with the lock fd open.
*/
private async tryAcquireLock(): Promise<boolean> {
await this.ensureLockDir();
// Ensure lock file exists (flock needs an existing file)
try {
await fs.writeFile(this.lockFilePath, '', { flag: 'a' });
} catch {
// Ignore if file already exists
}
return new Promise((resolve) => {
// Spawn a process that tries to acquire the lock and holds it
// Using flock with file descriptor: open fd 200 on lock file, then sleep forever
// The lock is held as long as fd 200 is open (process alive)
this.flockProcess = spawn('bash', [
'-c',
`exec 200>"${this.lockFilePath}" && flock -n -x 200 && echo LOCKED && while true; do sleep 86400; done`
], {
stdio: ['ignore', 'pipe', 'pipe']
});
let resolved = false;
// Check stdout for "LOCKED" message indicating we got the lock
this.flockProcess.stdout?.on('data', (data) => {
const output = data.toString().trim();
if (output === 'LOCKED' && !resolved) {
resolved = true;
console.log('[LeaderElection] Acquired flock on', this.lockFilePath);
resolve(true);
}
});
this.flockProcess.stderr?.on('data', (data) => {
const msg = data.toString().trim();
if (msg) {
console.error('[LeaderElection] flock stderr:', msg);
}
});
// If process exits quickly, we didn't get the lock
this.flockProcess.on('exit', (code) => {
if (!resolved) {
resolved = true;
console.log('[LeaderElection] Could not acquire flock (lock held by another process)');
this.flockProcess = null;
resolve(false);
} else if (!this.isShuttingDown && this.role === 'leader') {
// Lock process died unexpectedly while we were leader
console.error('[LeaderElection] flock process died unexpectedly!');
this.flockProcess = null;
this.handleLockLost();
}
});
// Timeout: if we don't get the lock within 2 seconds, assume it's held
setTimeout(() => {
if (!resolved) {
resolved = true;
if (this.flockProcess) {
this.flockProcess.kill();
this.flockProcess = null;
}
console.log('[LeaderElection] Timeout acquiring flock (lock held by another process)');
resolve(false);
}
}, 2000);
});
}
/**
* Handle losing the lock unexpectedly
*/
private async handleLockLost(): Promise<void> {
if (this.isShuttingDown) return;
console.log('[LeaderElection] Lost leadership, cleaning up...');
this.role = 'unknown';
// Stop Redis
await this.stopRedis();
// Notify callback
this.onLeaderLost?.();
// Try to re-elect
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
await this.transitionToFollower();
}
}
/**
* Check if the lock is currently held by someone
* Uses flock -n which fails immediately if lock is held
*/
private isLockHeld(): boolean {
try {
// Try to acquire lock non-blocking - if it succeeds, no one holds it
execSync(`flock -n -x "${this.lockFilePath}" -c "exit 0"`, {
timeout: 1000,
stdio: 'ignore'
});
// Lock was free (we briefly held it and released)
return false;
} catch {
// Lock is held by someone else
return true;
}
}
/**
* Read leader info from info file
*/
private async readLeaderInfo(): Promise<LeaderLockInfo | null> {
try {
const content = await fs.readFile(this.infoFilePath, 'utf-8');
return JSON.parse(content) as LeaderLockInfo;
} catch {
return null;
}
}
/**
* Write leader info to info file (atomic write)
*/
private async writeLeaderInfo(): Promise<void> {
this.leaderInfo = this.buildLeaderInfo();
// Atomic write: write to temp file, then rename
const tempPath = `${this.infoFilePath}.tmp`;
await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
await fs.rename(tempPath, this.infoFilePath);
console.log('[LeaderElection] Wrote leader info to', this.infoFilePath);
}
/**
* Update leader timestamp (heartbeat) using atomic write
*/
private async updateLeaderTimestamp(): Promise<void> {
if (this.role !== 'leader' || !this.leaderInfo) return;
this.leaderInfo.timestamp = Date.now();
try {
// Atomic write: write to temp file, then rename
const tempPath = `${this.infoFilePath}.tmp`;
await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
await fs.rename(tempPath, this.infoFilePath);
} catch (error) {
console.error('[LeaderElection] Failed to update timestamp:', error);
}
}
/**
* Spawn Redis server as child process
*/
private async spawnRedis(): Promise<void> {
const redisDataDir = `${this.config.metaCorePath}/db/redis`;
// Ensure Redis data directory exists
await fs.mkdir(redisDataDir, { recursive: true });
console.log(`[LeaderElection] Spawning Redis on port ${this.config.redisPort}...`);
// Spawn Redis server
this.redisProcess = spawn('redis-server', [
'--port', String(this.config.redisPort),
'--bind', '0.0.0.0',
'--dir', redisDataDir,
'--appendonly', 'yes',
'--appendfilename', 'appendonly.aof',
'--dbfilename', 'dump.rdb',
'--save', '60', '1', // Save after 60 seconds if at least 1 key changed
'--loglevel', 'warning'
], {
stdio: ['ignore', 'pipe', 'pipe'],
detached: false
});
this.redisProcess.stdout?.on('data', (data) => {
console.log(`[Redis] ${data.toString().trim()}`);
});
this.redisProcess.stderr?.on('data', (data) => {
console.error(`[Redis] ${data.toString().trim()}`);
});
this.redisProcess.on('exit', (code, signal) => {
console.log(`[Redis] Process exited with code ${code}, signal ${signal}`);
this.redisProcess = null;
if (!this.isShuttingDown && this.role === 'leader') {
// Redis crashed - try to restart
console.log('[LeaderElection] Redis crashed, attempting restart...');
this.spawnRedis().catch(console.error);
}
});
// Wait for Redis to be ready
await this.waitForRedisReady();
}
/**
* Wait for Redis to be ready to accept connections
*/
private async waitForRedisReady(): Promise<void> {
const maxAttempts = 30;
const ip = this.getLocalIP();
for (let i = 0; i < maxAttempts; i++) {
try {
const client = new Redis({
host: ip,
port: this.config.redisPort,
connectTimeout: 1000,
maxRetriesPerRequest: 1
});
const pong = await client.ping();
await client.quit();
if (pong === 'PONG') {
console.log('[LeaderElection] Redis is ready');
return;
}
} catch {
// Redis not ready yet
}
await new Promise(resolve => setTimeout(resolve, 1000));
}
throw new Error('Redis failed to start within timeout');
}
/**
* Stop Redis server
*/
private async stopRedis(): Promise<void> {
if (this.redisProcess) {
console.log('[LeaderElection] Stopping Redis...');
this.redisProcess.kill('SIGTERM');
// Wait for graceful shutdown
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
if (this.redisProcess) {
this.redisProcess.kill('SIGKILL');
}
resolve();
}, 5000);
this.redisProcess!.on('exit', () => {
clearTimeout(timeout);
resolve();
});
});
this.redisProcess = null;
}
}
/**
* Release the leader lock by killing the flock process
*/
private async releaseLock(): Promise<void> {
if (this.flockProcess) {
console.log('[LeaderElection] Releasing flock...');
// Kill the flock holder process - this releases the lock
this.flockProcess.kill('SIGTERM');
// Wait briefly for graceful exit
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
if (this.flockProcess) {
this.flockProcess.kill('SIGKILL');
}
resolve();
}, 1000);
if (this.flockProcess) {
this.flockProcess.on('exit', () => {
clearTimeout(timeout);
resolve();
});
} else {
clearTimeout(timeout);
resolve();
}
});
this.flockProcess = null;
}
// Clean up info file (optional, but good hygiene)
try {
await fs.unlink(this.infoFilePath);
} catch {
// Ignore if file doesn't exist
}
}
/**
* Start health check loop
*/
private startHealthCheck(): void {
this.healthCheckTimer = setInterval(async () => {
if (this.isShuttingDown) return;
try {
if (this.role === 'leader') {
// Update timestamp as heartbeat
await this.updateLeaderTimestamp();
// Verify we still hold the lock
if (!this.flockProcess || this.flockProcess.exitCode !== null) {
console.error('[LeaderElection] Lost flock process!');
await this.handleLockLost();
}
} else if (this.role === 'follower') {
// Check if lock is still held (leader still alive)
if (!this.isLockHeld()) {
console.log('[LeaderElection] Lock is free, attempting to become leader...');
this.onLeaderLost?.();
// Try to become leader
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
// Another service became leader - reconnect
await this.transitionToFollower();
}
}
}
} catch (error) {
console.error('[LeaderElection] Health check error:', error);
}
}, this.config.healthCheckInterval);
}
/**
* Transition to leader role
*/
private async transitionToLeader(): Promise<void> {
console.log('[LeaderElection] Becoming LEADER');
this.role = 'leader';
// Write leader info BEFORE spawning Redis so followers can find us
await this.writeLeaderInfo();
// Spawn Redis
await this.spawnRedis();
// Notify callback
this.onBecomeLeader?.();
}
/**
* Transition to follower role
*/
private async transitionToFollower(): Promise<void> {
console.log('[LeaderElection] Becoming FOLLOWER');
this.role = 'follower';
// Wait a bit for leader to write info file
await new Promise(resolve => setTimeout(resolve, 500));
// Read leader info
this.leaderInfo = await this.readLeaderInfo();
if (this.leaderInfo) {
console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
this.onBecomeFollower?.(this.leaderInfo);
} else {
console.warn('[LeaderElection] Could not read leader info, will retry...');
// Retry reading after a delay
setTimeout(async () => {
this.leaderInfo = await this.readLeaderInfo();
if (this.leaderInfo) {
console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
this.onBecomeFollower?.(this.leaderInfo);
}
}, 2000);
}
}
/**
* Start the leader election process
*/
async start(): Promise<void> {
console.log(`[LeaderElection] Starting election for ${this.config.serviceName}...`);
console.log(`[LeaderElection] Lock file: ${this.lockFilePath}`);
console.log(`[LeaderElection] Info file: ${this.infoFilePath}`);
// Try to become leader
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
await this.transitionToFollower();
}
// Start health check
this.startHealthCheck();
}
/**
* Stop and cleanup
*/
async stop(): Promise<void> {
console.log('[LeaderElection] Stopping...');
this.isShuttingDown = true;
// Stop health check
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
this.healthCheckTimer = null;
}
// If we were leader, cleanup
if (this.role === 'leader') {
await this.stopRedis();
await this.releaseLock();
}
this.role = 'unknown';
console.log('[LeaderElection] Stopped');
}
// ========================================================================
// Event Registration
// ========================================================================
onLeader(callback: () => void): this {
this.onBecomeLeader = callback;
return this;
}
onFollower(callback: (leaderInfo: LeaderLockInfo) => void): this {
this.onBecomeFollower = callback;
return this;
}
onLostLeader(callback: () => void): this {
this.onLeaderLost = callback;
return this;
}
// ========================================================================
// Getters
// ========================================================================
getRole(): LeaderRole {
return this.role;
}
isLeader(): boolean {
return this.role === 'leader';
}
isFollower(): boolean {
return this.role === 'follower';
}
getLeaderInfo(): LeaderLockInfo | null {
return this.leaderInfo;
}
getRedisUrl(): string | null {
if (this.leaderInfo) {
return this.leaderInfo.api;
}
return null;
}
}
packages/meta-sort-core/src/kv/ServiceDiscovery.ts
+82 -52
View File
@@ -1,16 +1,19 @@
/**
* Service Discovery using shared filesystem
*
* Each service registers itself in /meta-core/services/{service-name}.json
* with its API endpoint, status, and heartbeat timestamp.
* Each service registers itself in /meta-core/services/{service-name}-{hostname}.json
* with its base URL, status, and heartbeat timestamp.
*
* Services can discover each other by reading these JSON files.
* Stale detection: if lastHeartbeat > 60 seconds ago, service is considered dead.
* Stale detection: if lastHeartbeat > 60 seconds ago, service is considered stale.
*
* Note: meta-core (Go sidecar) handles actual service registration.
* This class is primarily used for discovering other services.
*/
import { promises as fs } from 'fs';
import { hostname } from 'os';
import { join } from 'path';
import { join, basename } from 'path';
import type { ServiceInfo } from './IKVClient.js';
import { isNodeError } from '../types/ExtendedInterfaces.js';
@@ -21,18 +24,12 @@ interface ServiceDiscoveryConfig {
/** Service name */
serviceName: string;
/** Service version */
version: string;
/** Service version (optional, not used in simplified format) */
version?: string;
/** HTTP API URL */
/** Base URL for the service */
apiUrl: string;
/** Service capabilities */
capabilities?: string[];
/** Named endpoints */
endpoints?: Record<string, string>;
/** Heartbeat interval in ms (default: 30000) */
heartbeatInterval?: number;
@@ -46,18 +43,19 @@ export class ServiceDiscovery {
private serviceFilePath: string;
private heartbeatTimer: NodeJS.Timeout | null = null;
private isShuttingDown = false;
private currentHostname: string;
constructor(config: ServiceDiscoveryConfig) {
this.config = {
heartbeatInterval: 30000,
staleThreshold: 60000,
capabilities: [],
endpoints: {},
...config
};
this.currentHostname = hostname();
this.servicesDir = `${this.config.metaCorePath}/services`;
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}.json`;
// Use hostname-based file naming
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`;
}
/**
@@ -73,28 +71,25 @@ export class ServiceDiscovery {
private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
return {
name: this.config.serviceName,
version: this.config.version,
api: this.config.apiUrl,
hostname: this.currentHostname,
baseUrl: this.config.apiUrl,
status,
pid: process.pid,
hostname: hostname(),
startedAt: new Date().toISOString(),
lastHeartbeat: new Date().toISOString(),
capabilities: this.config.capabilities || [],
endpoints: this.config.endpoints || {}
lastHeartbeat: new Date().toISOString()
};
}
/**
* Register this service
* Note: In the new architecture, meta-core handles registration.
* This is kept for backwards compatibility and testing.
*/
async register(): Promise<void> {
await this.ensureServicesDir();
const info = this.buildServiceInfo('starting');
const info = this.buildServiceInfo('running');
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}`);
console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}-${this.currentHostname}`);
}
/**
@@ -129,7 +124,6 @@ export class ServiceDiscovery {
// If file was deleted, re-register
if (isNodeError(error) && error.code === 'ENOENT') {
await this.register();
await this.updateStatus('running');
} else {
console.error('[ServiceDiscovery] Heartbeat failed:', error);
}
@@ -164,42 +158,72 @@ export class ServiceDiscovery {
this.stopHeartbeat();
try {
// Update status to stopped before removing
await this.updateStatus('stopped');
// Optionally remove the file (or keep for debugging)
// await fs.unlink(this.serviceFilePath);
console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}`);
// Remove the service file
await fs.unlink(this.serviceFilePath);
console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}-${this.currentHostname}`);
} catch (error) {
console.error('[ServiceDiscovery] Failed to unregister:', error);
if (!isNodeError(error) || error.code !== 'ENOENT') {
console.error('[ServiceDiscovery] Failed to unregister:', error);
}
}
}
/**
* Check if a service info is stale
*/
private isStale(info: ServiceInfo): boolean {
const lastHeartbeat = new Date(info.lastHeartbeat).getTime();
return Date.now() - lastHeartbeat > this.config.staleThreshold!;
}
// ========================================================================
// Discovery Methods
// ========================================================================
/**
* Discover a service by name
* Looks for files matching pattern: {name}-*.json (hostname-based naming)
*/
async discoverService(name: string): Promise<ServiceInfo | null> {
const servicePath = join(this.servicesDir, `${name}.json`);
try {
const content = await fs.readFile(servicePath, 'utf-8');
const service = JSON.parse(content) as ServiceInfo;
// First try exact match for backward compatibility
const exactPath = join(this.servicesDir, `${name}.json`);
try {
const content = await fs.readFile(exactPath, 'utf-8');
const service = JSON.parse(content) as ServiceInfo;
// Check if service is alive (heartbeat within threshold)
const lastHeartbeat = new Date(service.lastHeartbeat).getTime();
const isAlive = Date.now() - lastHeartbeat < this.config.staleThreshold!;
if (this.isStale(service)) {
service.status = 'stale';
}
if (!isAlive) {
console.warn(`[ServiceDiscovery] Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`);
return null;
return service.status === 'running' ? service : null;
} catch {
// Try hostname-based files
}
return service;
// Search for hostname-based files: name-*.json
const files = await fs.readdir(this.servicesDir);
const matchingFiles = files.filter(f =>
f.startsWith(`${name}-`) && f.endsWith('.json')
);
// Return first valid (non-stale) service
for (const file of matchingFiles) {
const filePath = join(this.servicesDir, file);
const content = await fs.readFile(filePath, 'utf-8');
const service = JSON.parse(content) as ServiceInfo;
if (this.isStale(service)) {
service.status = 'stale';
continue; // Skip stale services
}
if (service.status === 'running') {
return service;
}
}
return null;
} catch (error) {
if (isNodeError(error) && error.code === 'ENOENT') {
return null; // Service not registered
@@ -219,11 +243,18 @@ export class ServiceDiscovery {
for (const file of files) {
if (!file.endsWith('.json')) continue;
const name = file.replace('.json', '');
const service = await this.discoverService(name);
const filePath = join(this.servicesDir, file);
try {
const content = await fs.readFile(filePath, 'utf-8');
const service = JSON.parse(content) as ServiceInfo;
if (this.isStale(service)) {
service.status = 'stale';
}
if (service && service.status === 'running') {
services.push(service);
} catch (error) {
console.error(`[ServiceDiscovery] Failed to read ${file}:`, error);
}
}
@@ -266,8 +297,7 @@ export class ServiceDiscovery {
if (!service) return false;
// Try to ping the service's health endpoint
const healthEndpoint = service.endpoints.health || '/health';
const url = `${service.api}${healthEndpoint}`;
const url = `${service.baseUrl}/health`;
try {
const response = await fetch(url, {
@@ -285,10 +315,10 @@ export class ServiceDiscovery {
/**
* Full startup sequence
* Note: In the new architecture, meta-core handles registration.
*/
async start(): Promise<void> {
await this.register();
await this.updateStatus('running');
this.startHeartbeat();
}
packages/meta-sort-core/src/kv/index.ts
+3 -3
View File
@@ -1,10 +1,10 @@
/**
* KV Module - Unified key-value storage with leader election
* KV Module - Unified key-value storage with leader client
*
* This module provides:
* - IKVClient interface for KV storage abstraction
* - RedisKVClient for Redis-based storage
* - LeaderElection for distributed leader election via flock
* - LeaderClient for reading leader info from meta-core
* - ServiceDiscovery for service registration and discovery
* - KVManager for unified management of all components
* - MetadataUtils for flattening/reconstructing nested metadata
@@ -12,7 +12,7 @@
export * from './IKVClient.js';
export * from './RedisClient.js';
export * from './LeaderElection.js';
export * from './LeaderClient.js';
export * from './ServiceDiscovery.js';
export * from './KVManager.js';
export * from './MetadataUtils.js';