leader discovery fix and code cleaning
This commit is contained in:
Pierre
@@ -303,9 +303,11 @@ export class APIServer {
|
||||
const info = this.kvManager.getLeaderInfo();
|
||||
if (info) {
|
||||
leaderInfo = {
|
||||
host: info.host,
|
||||
api: info.api,
|
||||
http: info.http,
|
||||
hostname: info.hostname,
|
||||
baseUrl: info.baseUrl,
|
||||
apiUrl: info.apiUrl,
|
||||
redisUrl: info.redisUrl,
|
||||
webdavUrl: info.webdavUrl,
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -314,7 +316,6 @@ export class APIServer {
|
||||
services,
|
||||
current: 'meta-fuse',
|
||||
leader: leaderInfo,
|
||||
isLeader: false, // meta-fuse is never the leader
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -73,12 +73,8 @@ async function main(): Promise<void> {
|
||||
metaCorePath: config.metaCorePath,
|
||||
filesPath: config.filesPath,
|
||||
serviceName: 'meta-fuse',
|
||||
version: config.serviceVersion,
|
||||
apiPort: config.apiPort,
|
||||
redisUrl: config.redisUrl,
|
||||
redisPrefix: config.redisPrefix,
|
||||
baseUrl: config.baseUrl,
|
||||
capabilities: ['read', 'vfs'],
|
||||
});
|
||||
|
||||
let redisClient: RedisClient | null = null;
|
||||
|
||||
@@ -58,17 +58,24 @@ export interface IKVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Leader lock file content format
|
||||
* Leader lock file content format (kv-leader.info)
|
||||
* Written by meta-core leader, read by other services
|
||||
*/
|
||||
export interface LeaderLockInfo {
|
||||
/** Hostname of the leader */
|
||||
host: string;
|
||||
hostname: string;
|
||||
|
||||
/** Base URL for the leader service (e.g., http://localhost:8180) */
|
||||
baseUrl: string;
|
||||
|
||||
/** meta-core API URL (port 9000) */
|
||||
apiUrl: string;
|
||||
|
||||
/** Redis connection URL (e.g., redis://10.0.1.50:6379) */
|
||||
api: string;
|
||||
redisUrl: string;
|
||||
|
||||
/** HTTP API URL for the leader service */
|
||||
http: string;
|
||||
/** WebDAV URL for file access */
|
||||
webdavUrl: string;
|
||||
|
||||
/** Timestamp when leadership was acquired */
|
||||
timestamp: number;
|
||||
@@ -78,36 +85,22 @@ export interface LeaderLockInfo {
|
||||
}
|
||||
|
||||
/**
|
||||
* Service registration info stored in JSON file
|
||||
* Service registration info (simplified)
|
||||
* Full URLs are obtained via the /urls API endpoint
|
||||
*/
|
||||
export interface ServiceInfo {
|
||||
/** Service name */
|
||||
/** Service name (e.g., 'meta-sort', 'meta-fuse') */
|
||||
name: string;
|
||||
|
||||
/** Service version */
|
||||
version: string;
|
||||
|
||||
/** HTTP API URL (e.g., http://10.0.1.50:3000) */
|
||||
api: string;
|
||||
|
||||
/** Service status */
|
||||
status: 'starting' | 'running' | 'stopping' | 'stopped';
|
||||
|
||||
/** Process ID */
|
||||
pid: number;
|
||||
|
||||
/** Hostname */
|
||||
hostname: string;
|
||||
|
||||
/** Service start time (ISO string) */
|
||||
startedAt: string;
|
||||
/** Base URL for the service */
|
||||
baseUrl: string;
|
||||
|
||||
/** Last heartbeat time (ISO string) */
|
||||
/** Current status */
|
||||
status: 'running' | 'stale' | 'stopped';
|
||||
|
||||
/** Last heartbeat timestamp (ISO string) */
|
||||
lastHeartbeat: string;
|
||||
|
||||
/** Service capabilities */
|
||||
capabilities: string[];
|
||||
|
||||
/** Named endpoints */
|
||||
endpoints: Record<string, string>;
|
||||
}
|
||||
|
||||
@@ -2,16 +2,17 @@
|
||||
* KV Manager for meta-fuse (FOLLOWER-only)
|
||||
*
|
||||
* Simplified KV manager that:
|
||||
* 1. Discovers the leader via lock file
|
||||
* 1. Discovers the leader via LeaderClient (reads kv-leader.info)
|
||||
* 2. Creates Redis client connection
|
||||
* 3. Handles reconnection on leader failure
|
||||
*
|
||||
* Unlike meta-sort's KVManager, this version never spawns Redis -
|
||||
* it only connects as a client.
|
||||
* it only connects as a client. Leader election is handled by meta-core.
|
||||
*/
|
||||
|
||||
import { hostname } from 'os';
|
||||
import { Logger } from 'tslog';
|
||||
import { LeaderDiscovery } from './LeaderDiscovery.js';
|
||||
import { LeaderClient } from './LeaderClient.js';
|
||||
import { ServiceDiscovery } from './ServiceDiscovery.js';
|
||||
import { RedisClient } from './RedisClient.js';
|
||||
import type { LeaderLockInfo } from './IKVClient.js';
|
||||
@@ -28,9 +29,6 @@ interface KVManagerConfig {
|
||||
/** Service name (for logging) */
|
||||
serviceName?: string;
|
||||
|
||||
/** Service version */
|
||||
version?: string;
|
||||
|
||||
/** HTTP API port */
|
||||
apiPort?: number;
|
||||
|
||||
@@ -43,13 +41,13 @@ interface KVManagerConfig {
|
||||
/** Base URL for service discovery (overrides auto-detected URL) */
|
||||
baseUrl?: string;
|
||||
|
||||
/** Service capabilities */
|
||||
capabilities?: string[];
|
||||
/** meta-core API URL (e.g., http://localhost:9000) */
|
||||
metaCoreUrl?: string;
|
||||
}
|
||||
|
||||
export class KVManager {
|
||||
private config: Required<KVManagerConfig>;
|
||||
private leaderDiscovery: LeaderDiscovery | null = null;
|
||||
private leaderClient: LeaderClient | null = null;
|
||||
private serviceDiscovery: ServiceDiscovery | null = null;
|
||||
private redisClient: RedisClient | null = null;
|
||||
private isStarted = false;
|
||||
@@ -62,12 +60,11 @@ export class KVManager {
|
||||
constructor(config: KVManagerConfig) {
|
||||
this.config = {
|
||||
serviceName: 'meta-fuse',
|
||||
version: '1.0.0',
|
||||
apiPort: 3000,
|
||||
redisUrl: '',
|
||||
redisPrefix: 'meta-sort:',
|
||||
baseUrl: '',
|
||||
capabilities: ['read', 'vfs'],
|
||||
metaCoreUrl: '',
|
||||
...config
|
||||
};
|
||||
}
|
||||
@@ -83,24 +80,15 @@ export class KVManager {
|
||||
|
||||
logger.info(`Starting KVManager for ${this.config.serviceName}...`);
|
||||
|
||||
// Initialize service discovery
|
||||
const apiUrl = this.config.baseUrl || `http://localhost:${this.config.apiPort}`;
|
||||
// Initialize service discovery (meta-core handles actual registration)
|
||||
// We keep this for service discovery (reading other services)
|
||||
const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
|
||||
this.serviceDiscovery = new ServiceDiscovery({
|
||||
metaCorePath: this.config.metaCorePath,
|
||||
serviceName: this.config.serviceName,
|
||||
version: this.config.version,
|
||||
apiUrl,
|
||||
baseUrl: this.config.baseUrl || undefined,
|
||||
capabilities: this.config.capabilities,
|
||||
endpoints: {
|
||||
health: '/api/fuse/health',
|
||||
dashboard: '/',
|
||||
stats: '/api/fuse/stats',
|
||||
}
|
||||
apiUrl
|
||||
});
|
||||
|
||||
// Start service discovery
|
||||
await this.serviceDiscovery.start();
|
||||
// Note: We don't start ServiceDiscovery - meta-core handles registration
|
||||
|
||||
// If direct Redis URL provided, skip leader discovery
|
||||
if (this.config.redisUrl) {
|
||||
@@ -110,29 +98,56 @@ export class KVManager {
|
||||
return;
|
||||
}
|
||||
|
||||
// Use leader discovery
|
||||
this.leaderDiscovery = new LeaderDiscovery({
|
||||
metaCorePath: this.config.metaCorePath
|
||||
// Use leader client
|
||||
this.leaderClient = new LeaderClient({
|
||||
metaCorePath: this.config.metaCorePath,
|
||||
metaCoreUrl: this.config.metaCoreUrl || undefined
|
||||
});
|
||||
|
||||
// Set up leader discovery callbacks
|
||||
this.leaderDiscovery.onLeaderFound(async (info: LeaderLockInfo) => {
|
||||
logger.info(`Connecting to leader at ${info.api}...`);
|
||||
await this.connectToRedis(info.api);
|
||||
// Set up leader change callback
|
||||
this.leaderClient.onChange(async () => {
|
||||
logger.info('Leader changed, reconnecting...');
|
||||
await this.reconnect();
|
||||
});
|
||||
|
||||
this.leaderDiscovery.onLeaderLost(async () => {
|
||||
logger.warn('Leader lost, disconnecting...');
|
||||
await this.disconnectRedis();
|
||||
this.notifyDisconnect();
|
||||
});
|
||||
// Wait for leader and connect
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
logger.info(`Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
await this.connectToRedis(leaderInfo.redisUrl);
|
||||
} catch (error: any) {
|
||||
logger.error(`Failed to connect to leader: ${error.message}`);
|
||||
throw error;
|
||||
}
|
||||
|
||||
// Start leader discovery
|
||||
await this.leaderDiscovery.start();
|
||||
// Start watching for leader changes
|
||||
this.leaderClient.startWatching();
|
||||
|
||||
this.isStarted = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Reconnect to Redis after leader change
|
||||
*/
|
||||
private async reconnect(): Promise<void> {
|
||||
if (this.isShuttingDown || !this.leaderClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Disconnect existing
|
||||
await this.disconnectRedis();
|
||||
this.notifyDisconnect();
|
||||
|
||||
// Wait for new leader and connect
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
logger.info(`Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
await this.connectToRedis(leaderInfo.redisUrl);
|
||||
} catch (error: any) {
|
||||
logger.error(`Failed to reconnect: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop the KV manager
|
||||
*/
|
||||
@@ -142,18 +157,18 @@ export class KVManager {
|
||||
logger.info('Stopping KVManager...');
|
||||
this.isShuttingDown = true;
|
||||
|
||||
// Stop service discovery
|
||||
// Stop leader client
|
||||
if (this.leaderClient) {
|
||||
this.leaderClient.close();
|
||||
this.leaderClient = null;
|
||||
}
|
||||
|
||||
// Stop service discovery (if it was started)
|
||||
if (this.serviceDiscovery) {
|
||||
await this.serviceDiscovery.stop();
|
||||
this.serviceDiscovery = null;
|
||||
}
|
||||
|
||||
// Stop leader discovery
|
||||
if (this.leaderDiscovery) {
|
||||
await this.leaderDiscovery.stop();
|
||||
this.leaderDiscovery = null;
|
||||
}
|
||||
|
||||
// Disconnect Redis
|
||||
await this.disconnectRedis();
|
||||
|
||||
@@ -313,10 +328,17 @@ export class KVManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get leader info (if using leader discovery)
|
||||
* Get leader info
|
||||
*/
|
||||
getLeaderInfo(): LeaderLockInfo | null {
|
||||
return this.leaderDiscovery?.getLeaderInfo() ?? null;
|
||||
return this.leaderClient?.getCachedLeaderInfo() ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get leader client
|
||||
*/
|
||||
getLeaderClient(): LeaderClient | null {
|
||||
return this.leaderClient;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* LeaderClient - Client for reading leader info from meta-core
|
||||
*
|
||||
* This replaces the old LeaderDiscovery.ts by delegating leader election
|
||||
* to meta-core (Go sidecar) and just reading the results.
|
||||
*
|
||||
* Features:
|
||||
* - Reads kv-leader.info file for leader URLs
|
||||
* - Calls meta-core /urls API for current leader URLs
|
||||
* - Watches for leader changes via file system watcher
|
||||
*/
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import { watch, FSWatcher } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import { Logger } from 'tslog';
|
||||
import type { LeaderLockInfo } from './IKVClient.js';
|
||||
|
||||
const logger = new Logger({ name: 'LeaderClient' });
|
||||
|
||||
export interface LeaderClientConfig {
|
||||
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
||||
metaCorePath: string;
|
||||
|
||||
/** meta-core API URL (e.g., http://meta-core:9000 or http://localhost:9000) */
|
||||
metaCoreUrl?: string;
|
||||
}
|
||||
|
||||
export interface URLsResponse {
|
||||
hostname: string;
|
||||
baseUrl: string;
|
||||
apiUrl: string;
|
||||
redisUrl: string;
|
||||
webdavUrl: string;
|
||||
isLeader: boolean;
|
||||
}
|
||||
|
||||
export class LeaderClient {
|
||||
private config: LeaderClientConfig;
|
||||
private infoFilePath: string;
|
||||
private leaderInfo: LeaderLockInfo | null = null;
|
||||
private watcher: FSWatcher | null = null;
|
||||
private onChangeCallbacks: (() => void)[] = [];
|
||||
|
||||
constructor(config: LeaderClientConfig) {
|
||||
this.config = config;
|
||||
this.infoFilePath = `${config.metaCorePath}/locks/kv-leader.info`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read leader info from the info file
|
||||
*/
|
||||
async getLeaderInfo(): Promise<LeaderLockInfo | null> {
|
||||
try {
|
||||
const content = await fs.readFile(this.infoFilePath, 'utf-8');
|
||||
this.leaderInfo = JSON.parse(content) as LeaderLockInfo;
|
||||
return this.leaderInfo;
|
||||
} catch (error: any) {
|
||||
logger.error(`Failed to read leader info: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Redis URL from leader info
|
||||
*/
|
||||
async getRedisUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.redisUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get WebDAV URL from leader info
|
||||
*/
|
||||
async getWebdavUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.webdavUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get meta-core API URL from leader info
|
||||
*/
|
||||
async getApiUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.apiUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call meta-core /urls API to get current URLs
|
||||
* Useful for getting URLs when you don't want to read the file directly
|
||||
*/
|
||||
async getUrls(): Promise<URLsResponse | null> {
|
||||
// First try using configured metaCoreUrl
|
||||
let apiUrl = this.config.metaCoreUrl;
|
||||
|
||||
// Fall back to reading from leader info
|
||||
if (!apiUrl) {
|
||||
const info = await this.getLeaderInfo();
|
||||
apiUrl = info?.apiUrl;
|
||||
}
|
||||
|
||||
if (!apiUrl) {
|
||||
logger.error('No meta-core API URL available');
|
||||
return null;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${apiUrl}/urls`, {
|
||||
method: 'GET',
|
||||
headers: { 'Accept': 'application/json' },
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
logger.error(`Failed to get URLs: ${response.status} ${response.statusText}`);
|
||||
return null;
|
||||
}
|
||||
|
||||
return await response.json() as URLsResponse;
|
||||
} catch (error: any) {
|
||||
logger.error(`Error calling /urls API: ${error.message}`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for leader info to be available
|
||||
*/
|
||||
async waitForLeader(timeoutMs: number = 30000): Promise<LeaderLockInfo> {
|
||||
const startTime = Date.now();
|
||||
const pollInterval = 500;
|
||||
|
||||
while (Date.now() - startTime < timeoutMs) {
|
||||
const info = await this.getLeaderInfo();
|
||||
if (info) {
|
||||
logger.info(`Leader found: ${info.hostname} at ${info.redisUrl}`);
|
||||
return info;
|
||||
}
|
||||
|
||||
logger.debug('Waiting for leader...');
|
||||
await new Promise(resolve => setTimeout(resolve, pollInterval));
|
||||
}
|
||||
|
||||
throw new Error(`No leader found within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for leader changes
|
||||
*/
|
||||
startWatching(): void {
|
||||
if (this.watcher) {
|
||||
return; // Already watching
|
||||
}
|
||||
|
||||
const lockDir = dirname(this.infoFilePath);
|
||||
|
||||
try {
|
||||
this.watcher = watch(lockDir, (eventType, filename) => {
|
||||
if (filename === 'kv-leader.info') {
|
||||
logger.debug(`Lock file ${eventType}, checking for leader...`);
|
||||
this.getLeaderInfo().then(() => {
|
||||
this.notifyChange();
|
||||
}).catch((err) => logger.error(`Error reloading leader info: ${err.message}`));
|
||||
}
|
||||
});
|
||||
|
||||
this.watcher.on('error', (error: any) => {
|
||||
logger.warn(`File watcher error: ${error.message}`);
|
||||
});
|
||||
|
||||
logger.info(`Watching for leader changes in ${lockDir}`);
|
||||
} catch (error: any) {
|
||||
logger.error(`Failed to start watching: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop watching for leader changes
|
||||
*/
|
||||
stopWatching(): void {
|
||||
if (this.watcher) {
|
||||
this.watcher.close();
|
||||
this.watcher = null;
|
||||
logger.debug('Stopped watching for leader changes');
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Register callback for leader changes
|
||||
*/
|
||||
onChange(callback: () => void): this {
|
||||
this.onChangeCallbacks.push(callback);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify all change callbacks
|
||||
*/
|
||||
private notifyChange(): void {
|
||||
for (const callback of this.onChangeCallbacks) {
|
||||
try {
|
||||
callback();
|
||||
} catch (error: any) {
|
||||
logger.error(`Error in change callback: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached leader info (without re-reading file)
|
||||
*/
|
||||
getCachedLeaderInfo(): LeaderLockInfo | null {
|
||||
return this.leaderInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources
|
||||
*/
|
||||
close(): void {
|
||||
this.stopWatching();
|
||||
this.onChangeCallbacks = [];
|
||||
}
|
||||
}
|
||||
@@ -1,276 +0,0 @@
|
||||
/**
|
||||
* Leader Discovery for meta-fuse (FOLLOWER-only)
|
||||
*
|
||||
* This is a simplified version of leader election that only discovers
|
||||
* the existing leader. meta-fuse never spawns Redis - it only connects
|
||||
* to the leader's Redis instance.
|
||||
*
|
||||
* Architecture:
|
||||
* - Leader holds flock on META_CORE_PATH/locks/kv-leader.lock
|
||||
* - Leader writes info to META_CORE_PATH/locks/kv-leader.info
|
||||
* - This class reads the info file to discover the leader
|
||||
* - Watches for leader changes via health checks
|
||||
* - Triggers reconnection callbacks on leader failure/change
|
||||
*/
|
||||
|
||||
import { promises as fs, watch } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import { Logger } from 'tslog';
|
||||
import type { LeaderLockInfo } from './IKVClient.js';
|
||||
|
||||
const logger = new Logger({ name: 'LeaderDiscovery' });
|
||||
|
||||
interface LeaderDiscoveryConfig {
|
||||
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
||||
metaCorePath: string;
|
||||
|
||||
/** Health check interval in ms (default: 5000) */
|
||||
healthCheckInterval?: number;
|
||||
|
||||
/** Max consecutive failures before marking leader as lost (default: 3) */
|
||||
maxFailures?: number;
|
||||
}
|
||||
|
||||
export class LeaderDiscovery {
|
||||
private config: Required<LeaderDiscoveryConfig>;
|
||||
private lockFilePath: string;
|
||||
private leaderInfo: LeaderLockInfo | null = null;
|
||||
private healthCheckTimer: NodeJS.Timeout | null = null;
|
||||
private fileWatcher: ReturnType<typeof watch> | null = null;
|
||||
private isStarted = false;
|
||||
private consecutiveFailures = 0;
|
||||
|
||||
// Event callbacks
|
||||
private onLeaderFoundCallback?: (info: LeaderLockInfo) => void;
|
||||
private onLeaderLostCallback?: () => void;
|
||||
|
||||
constructor(config: LeaderDiscoveryConfig) {
|
||||
this.config = {
|
||||
healthCheckInterval: 5000,
|
||||
maxFailures: 3,
|
||||
...config
|
||||
};
|
||||
|
||||
this.lockFilePath = `${this.config.metaCorePath}/locks/kv-leader.info`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for leader
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isStarted) {
|
||||
logger.warn('LeaderDiscovery already started');
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info(`Starting leader discovery, lock file: ${this.lockFilePath}`);
|
||||
|
||||
// Ensure lock directory exists (for file watching)
|
||||
await this.ensureLockDir();
|
||||
|
||||
// Try to read existing leader info
|
||||
await this.checkForLeader();
|
||||
|
||||
// Start file watcher for immediate detection of leader changes
|
||||
this.startFileWatcher();
|
||||
|
||||
// Start health check for ongoing monitoring
|
||||
this.startHealthCheck();
|
||||
|
||||
this.isStarted = true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop watching
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
if (!this.isStarted) return;
|
||||
|
||||
logger.info('Stopping leader discovery');
|
||||
|
||||
if (this.healthCheckTimer) {
|
||||
clearInterval(this.healthCheckTimer);
|
||||
this.healthCheckTimer = null;
|
||||
}
|
||||
|
||||
if (this.fileWatcher) {
|
||||
this.fileWatcher.close();
|
||||
this.fileWatcher = null;
|
||||
}
|
||||
|
||||
this.isStarted = false;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure lock directory exists
|
||||
*/
|
||||
private async ensureLockDir(): Promise<void> {
|
||||
const lockDir = dirname(this.lockFilePath);
|
||||
try {
|
||||
await fs.mkdir(lockDir, { recursive: true });
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'EEXIST') {
|
||||
logger.warn(`Could not create lock directory: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check for leader by reading lock file
|
||||
*/
|
||||
private async checkForLeader(): Promise<void> {
|
||||
try {
|
||||
const content = await fs.readFile(this.lockFilePath, 'utf-8');
|
||||
const info = JSON.parse(content) as LeaderLockInfo;
|
||||
|
||||
// Validate leader info
|
||||
if (info && info.api && info.http) {
|
||||
const isNew = !this.leaderInfo || this.leaderInfo.api !== info.api;
|
||||
this.leaderInfo = info;
|
||||
this.consecutiveFailures = 0;
|
||||
|
||||
if (isNew) {
|
||||
logger.info(`Leader found: ${info.host} at ${info.api}`);
|
||||
this.onLeaderFoundCallback?.(info);
|
||||
}
|
||||
}
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
// Lock file doesn't exist - no leader yet
|
||||
if (this.leaderInfo) {
|
||||
logger.warn('Lock file removed, leader lost');
|
||||
this.handleLeaderLost();
|
||||
}
|
||||
} else {
|
||||
logger.warn(`Error reading lock file: ${error.message}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if current leader is still healthy
|
||||
*/
|
||||
private async isLeaderHealthy(): Promise<boolean> {
|
||||
if (!this.leaderInfo) return false;
|
||||
|
||||
try {
|
||||
const response = await fetch(`${this.leaderInfo.http}/health`, {
|
||||
signal: AbortSignal.timeout(5000)
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle leader being lost
|
||||
*/
|
||||
private handleLeaderLost(): void {
|
||||
this.leaderInfo = null;
|
||||
this.onLeaderLostCallback?.();
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching lock file for changes
|
||||
*/
|
||||
private startFileWatcher(): void {
|
||||
try {
|
||||
const lockDir = dirname(this.lockFilePath);
|
||||
this.fileWatcher = watch(lockDir, async (eventType, filename) => {
|
||||
if (filename === 'kv-leader.info') {
|
||||
logger.debug(`Lock file ${eventType}, checking for leader...`);
|
||||
await this.checkForLeader();
|
||||
}
|
||||
});
|
||||
|
||||
this.fileWatcher.on('error', (error) => {
|
||||
logger.warn(`File watcher error: ${error.message}`);
|
||||
});
|
||||
} catch (error: any) {
|
||||
logger.warn(`Could not start file watcher: ${error.message}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start health check loop
|
||||
*/
|
||||
private startHealthCheck(): void {
|
||||
this.healthCheckTimer = setInterval(async () => {
|
||||
if (!this.leaderInfo) {
|
||||
// No leader known, try to find one
|
||||
await this.checkForLeader();
|
||||
return;
|
||||
}
|
||||
|
||||
// Check if current leader is healthy
|
||||
const healthy = await this.isLeaderHealthy();
|
||||
|
||||
if (healthy) {
|
||||
this.consecutiveFailures = 0;
|
||||
} else {
|
||||
this.consecutiveFailures++;
|
||||
logger.warn(`Leader health check failed (${this.consecutiveFailures}/${this.config.maxFailures})`);
|
||||
|
||||
if (this.consecutiveFailures >= this.config.maxFailures) {
|
||||
logger.warn('Leader appears dead, marking as lost');
|
||||
this.handleLeaderLost();
|
||||
|
||||
// Try to find new leader
|
||||
await this.checkForLeader();
|
||||
}
|
||||
}
|
||||
}, this.config.healthCheckInterval);
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Event Registration
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Register callback for when leader is found/changed
|
||||
*/
|
||||
onLeaderFound(callback: (info: LeaderLockInfo) => void): this {
|
||||
this.onLeaderFoundCallback = callback;
|
||||
|
||||
// If we already have leader info, call immediately
|
||||
if (this.leaderInfo) {
|
||||
callback(this.leaderInfo);
|
||||
}
|
||||
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Register callback for when leader is lost
|
||||
*/
|
||||
onLeaderLost(callback: () => void): this {
|
||||
this.onLeaderLostCallback = callback;
|
||||
return this;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Getters
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Get current leader info
|
||||
*/
|
||||
getLeaderInfo(): LeaderLockInfo | null {
|
||||
return this.leaderInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Redis URL of current leader
|
||||
*/
|
||||
getRedisUrl(): string | null {
|
||||
return this.leaderInfo?.api ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a leader is known
|
||||
*/
|
||||
hasLeader(): boolean {
|
||||
return this.leaderInfo !== null;
|
||||
}
|
||||
}
|
||||
@@ -1,11 +1,14 @@
|
||||
/**
|
||||
* Service Discovery using shared filesystem
|
||||
*
|
||||
* Each service registers itself in /meta-core/services/{service-name}.json
|
||||
* with its API endpoint, status, and heartbeat timestamp.
|
||||
* Each service registers itself in /meta-core/services/{service-name}-{hostname}.json
|
||||
* with its base URL, status, and heartbeat timestamp.
|
||||
*
|
||||
* Services can discover each other by reading these JSON files.
|
||||
* Stale detection: if lastHeartbeat > 60 seconds ago, service is considered dead.
|
||||
* Stale detection: if lastHeartbeat > 60 seconds ago, service is considered stale.
|
||||
*
|
||||
* Note: meta-core (Go sidecar) handles actual service registration.
|
||||
* This class is primarily used for discovering other services.
|
||||
*/
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
@@ -23,20 +26,8 @@ interface ServiceDiscoveryConfig {
|
||||
/** Service name */
|
||||
serviceName: string;
|
||||
|
||||
/** Service version */
|
||||
version: string;
|
||||
|
||||
/** HTTP API URL (can be overridden by baseUrl) */
|
||||
apiUrl?: string;
|
||||
|
||||
/** Base URL for service discovery (overrides auto-detected URL) */
|
||||
baseUrl?: string;
|
||||
|
||||
/** Service capabilities */
|
||||
capabilities?: string[];
|
||||
|
||||
/** Named endpoints */
|
||||
endpoints?: Record<string, string>;
|
||||
/** Base URL for the service */
|
||||
apiUrl: string;
|
||||
|
||||
/** Heartbeat interval in ms (default: 30000) */
|
||||
heartbeatInterval?: number;
|
||||
@@ -52,18 +43,19 @@ export class ServiceDiscovery {
|
||||
private heartbeatTimer: NodeJS.Timeout | null = null;
|
||||
private isShuttingDown = false;
|
||||
private isStarted = false;
|
||||
private currentHostname: string;
|
||||
|
||||
constructor(config: ServiceDiscoveryConfig) {
|
||||
this.config = {
|
||||
heartbeatInterval: 30000,
|
||||
staleThreshold: 60000,
|
||||
capabilities: [],
|
||||
endpoints: {},
|
||||
...config
|
||||
};
|
||||
|
||||
this.currentHostname = hostname();
|
||||
this.servicesDir = `${this.config.metaCorePath}/services`;
|
||||
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}.json`;
|
||||
// Use hostname-based file naming
|
||||
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -79,28 +71,25 @@ export class ServiceDiscovery {
|
||||
private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
|
||||
return {
|
||||
name: this.config.serviceName,
|
||||
version: this.config.version,
|
||||
api: this.config.baseUrl || this.config.apiUrl || '',
|
||||
hostname: this.currentHostname,
|
||||
baseUrl: this.config.apiUrl,
|
||||
status,
|
||||
pid: process.pid,
|
||||
hostname: hostname(),
|
||||
startedAt: new Date().toISOString(),
|
||||
lastHeartbeat: new Date().toISOString(),
|
||||
capabilities: this.config.capabilities || [],
|
||||
endpoints: this.config.endpoints || {}
|
||||
lastHeartbeat: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Register this service
|
||||
* Note: In the new architecture, meta-core handles registration.
|
||||
* This is kept for backwards compatibility and testing.
|
||||
*/
|
||||
async register(): Promise<void> {
|
||||
await this.ensureServicesDir();
|
||||
|
||||
const info = this.buildServiceInfo('starting');
|
||||
const info = this.buildServiceInfo('running');
|
||||
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
|
||||
|
||||
logger.info(`Registered ${this.config.serviceName}`);
|
||||
logger.info(`Registered ${this.config.serviceName}-${this.currentHostname}`);
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -135,7 +124,6 @@ export class ServiceDiscovery {
|
||||
// If file was deleted, re-register
|
||||
if (error.code === 'ENOENT') {
|
||||
await this.register();
|
||||
await this.updateStatus('running');
|
||||
} else {
|
||||
logger.error('Heartbeat failed:', error);
|
||||
}
|
||||
@@ -170,39 +158,72 @@ export class ServiceDiscovery {
|
||||
this.stopHeartbeat();
|
||||
|
||||
try {
|
||||
// Update status to stopped before removing
|
||||
await this.updateStatus('stopped');
|
||||
|
||||
logger.info(`Unregistered ${this.config.serviceName}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to unregister:', error);
|
||||
// Remove the service file
|
||||
await fs.unlink(this.serviceFilePath);
|
||||
logger.info(`Unregistered ${this.config.serviceName}-${this.currentHostname}`);
|
||||
} catch (error: any) {
|
||||
if (error.code !== 'ENOENT') {
|
||||
logger.error('Failed to unregister:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if a service info is stale
|
||||
*/
|
||||
private isStale(info: ServiceInfo): boolean {
|
||||
const lastHeartbeat = new Date(info.lastHeartbeat).getTime();
|
||||
return Date.now() - lastHeartbeat > this.config.staleThreshold!;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Discovery Methods
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Discover a service by name
|
||||
* Looks for files matching pattern: {name}-*.json (hostname-based naming)
|
||||
*/
|
||||
async discoverService(name: string): Promise<ServiceInfo | null> {
|
||||
const servicePath = join(this.servicesDir, `${name}.json`);
|
||||
|
||||
try {
|
||||
const content = await fs.readFile(servicePath, 'utf-8');
|
||||
const service = JSON.parse(content) as ServiceInfo;
|
||||
// First try exact match for backward compatibility
|
||||
const exactPath = join(this.servicesDir, `${name}.json`);
|
||||
try {
|
||||
const content = await fs.readFile(exactPath, 'utf-8');
|
||||
const service = JSON.parse(content) as ServiceInfo;
|
||||
|
||||
// Check if service is alive (heartbeat within threshold)
|
||||
const lastHeartbeat = new Date(service.lastHeartbeat).getTime();
|
||||
const isAlive = Date.now() - lastHeartbeat < this.config.staleThreshold!;
|
||||
if (this.isStale(service)) {
|
||||
service.status = 'stale';
|
||||
}
|
||||
|
||||
if (!isAlive) {
|
||||
logger.warn(`Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`);
|
||||
return null;
|
||||
return service.status === 'running' ? service : null;
|
||||
} catch {
|
||||
// Try hostname-based files
|
||||
}
|
||||
|
||||
return service;
|
||||
// Search for hostname-based files: name-*.json
|
||||
const files = await fs.readdir(this.servicesDir);
|
||||
const matchingFiles = files.filter(f =>
|
||||
f.startsWith(`${name}-`) && f.endsWith('.json')
|
||||
);
|
||||
|
||||
// Return first valid (non-stale) service
|
||||
for (const file of matchingFiles) {
|
||||
const filePath = join(this.servicesDir, file);
|
||||
const content = await fs.readFile(filePath, 'utf-8');
|
||||
const service = JSON.parse(content) as ServiceInfo;
|
||||
|
||||
if (this.isStale(service)) {
|
||||
service.status = 'stale';
|
||||
continue; // Skip stale services
|
||||
}
|
||||
|
||||
if (service.status === 'running') {
|
||||
return service;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
} catch (error: any) {
|
||||
if (error.code === 'ENOENT') {
|
||||
return null; // Service not registered
|
||||
@@ -222,11 +243,18 @@ export class ServiceDiscovery {
|
||||
for (const file of files) {
|
||||
if (!file.endsWith('.json')) continue;
|
||||
|
||||
const name = file.replace('.json', '');
|
||||
const service = await this.discoverService(name);
|
||||
const filePath = join(this.servicesDir, file);
|
||||
try {
|
||||
const content = await fs.readFile(filePath, 'utf-8');
|
||||
const service = JSON.parse(content) as ServiceInfo;
|
||||
|
||||
if (this.isStale(service)) {
|
||||
service.status = 'stale';
|
||||
}
|
||||
|
||||
if (service && service.status === 'running') {
|
||||
services.push(service);
|
||||
} catch (error) {
|
||||
logger.error(`Failed to read ${file}:`, error);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -247,8 +275,7 @@ export class ServiceDiscovery {
|
||||
if (!service) return false;
|
||||
|
||||
// Try to ping the service's health endpoint
|
||||
const healthEndpoint = service.endpoints.health || '/health';
|
||||
const url = `${service.api}${healthEndpoint}`;
|
||||
const url = `${service.baseUrl}/health`;
|
||||
|
||||
try {
|
||||
const response = await fetch(url, {
|
||||
@@ -266,16 +293,16 @@ export class ServiceDiscovery {
|
||||
|
||||
/**
|
||||
* Full startup sequence
|
||||
* Note: In the new architecture, meta-core handles registration.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isStarted) return;
|
||||
|
||||
await this.register();
|
||||
await this.updateStatus('running');
|
||||
this.startHeartbeat();
|
||||
this.isStarted = true;
|
||||
|
||||
logger.info(`Started ${this.config.serviceName} at ${this.config.baseUrl || this.config.apiUrl}`);
|
||||
logger.info(`Started ${this.config.serviceName} at ${this.config.apiUrl}`);
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
Reference in New Issue
Block a user