initial commit

2026-01-11 17:26:07 +01:00
commit ff4fbf136f
49 changed files with 13118 additions and 0 deletions
@@ -0,0 +1,168 @@
+/**
+ * meta-fuse-core entry point
+ *
+ * Standalone FUSE API server that reads metadata from Redis,
+ * computes virtual paths, and exposes organized folder structure.
+ */
+
+import 'dotenv/config';
+import { Logger } from 'tslog';
+import { KVManager } from './kv/KVManager.js';
+import { RedisClient } from './kv/RedisClient.js';
+import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
+import { APIServer } from './api/APIServer.js';
+
+const logger = new Logger({ name: 'meta-fuse' });
+
+// Pub/sub channel for batched updates from meta-sort
+const UPDATE_CHANNEL = 'meta-sort:file:batch';
+
+interface BatchUpdateMessage {
+    timestamp: number;
+    changes: Array<{
+        action: 'add' | 'update' | 'remove';
+        hashId: string;
+    }>;
+}
+
+async function main(): Promise<void> {
+    logger.info('Starting meta-fuse...');
+    logger.info(`Node.js ${process.version}`);
+
+    // Configuration
+    const config = {
+        metaCorePath: process.env.META_CORE_PATH ?? '/meta-core',
+        filesPath: process.env.FILES_VOLUME ?? '/files',
+        redisUrl: process.env.REDIS_URL,
+        redisPrefix: process.env.REDIS_PREFIX ?? 'meta-sort:',
+        apiPort: parseInt(process.env.API_PORT ?? '3000', 10),
+        apiHost: process.env.API_HOST ?? '0.0.0.0',
+        vfsRefreshInterval: parseInt(process.env.VFS_REFRESH_INTERVAL ?? '30000', 10),
+        baseUrl: process.env.BASE_URL,
+        serviceVersion: process.env.SERVICE_VERSION ?? '1.0.0',
+    };
+
+    logger.info(`Config: META_CORE_PATH=${config.metaCorePath}, FILES_VOLUME=${config.filesPath}`);
+
+    // Initialize KV Manager (handles leader discovery)
+    const kvManager = new KVManager({
+        metaCorePath: config.metaCorePath,
+        filesPath: config.filesPath,
+        serviceName: 'meta-fuse',
+        version: config.serviceVersion,
+        apiPort: config.apiPort,
+        redisUrl: config.redisUrl,
+        redisPrefix: config.redisPrefix,
+        baseUrl: config.baseUrl,
+        capabilities: ['read', 'vfs'],
+    });
+
+    let redisClient: RedisClient | null = null;
+    let vfs: VirtualFileSystem | null = null;
+    let apiServer: APIServer | null = null;
+
+    // Graceful shutdown
+    const shutdown = async (signal: string): Promise<void> => {
+        logger.info(`Received ${signal}, shutting down...`);
+
+        try {
+            if (apiServer) await apiServer.stop();
+            if (vfs) await vfs.stop();
+            await kvManager.stop();
+            logger.info('Shutdown complete');
+            process.exit(0);
+        } catch (error) {
+            logger.error('Error during shutdown:', error);
+            process.exit(1);
+        }
+    };
+
+    process.on('SIGTERM', () => shutdown('SIGTERM'));
+    process.on('SIGINT', () => shutdown('SIGINT'));
+
+    // Handle ready event from KV manager
+    kvManager.onReady(async () => {
+        logger.info('KV Manager ready, initializing VFS...');
+
+        try {
+            // Get Redis client from KV manager
+            redisClient = kvManager.getClient();
+
+            if (!redisClient) {
+                logger.error('Redis client not available');
+                return;
+            }
+
+            // Initialize VFS
+            vfs = new VirtualFileSystem(redisClient, {
+                fileMode: parseInt(process.env.FUSE_FILE_MODE ?? '644', 8),
+                directoryMode: parseInt(process.env.FUSE_DIR_MODE ?? '755', 8),
+                uid: parseInt(process.env.PUID ?? '1000', 10),
+                gid: parseInt(process.env.PGID ?? '1000', 10),
+                refreshInterval: config.vfsRefreshInterval,
+                configDir: process.env.CONFIG_DIR ?? '/meta-fuse/config',
+            });
+
+            // Start VFS
+            logger.info('Starting VirtualFileSystem...');
+            await vfs.start();
+
+            // Subscribe to batch updates from meta-sort
+            logger.info(`Subscribing to ${UPDATE_CHANNEL}...`);
+            await redisClient.subscribe(UPDATE_CHANNEL, async (message: string) => {
+                try {
+                    const batch: BatchUpdateMessage = JSON.parse(message);
+                    logger.debug(`Received batch update: ${batch.changes.length} changes`);
+
+                    // Process each change incrementally
+                    for (const change of batch.changes) {
+                        await vfs!.onFileUpdate(change.hashId, change.action);
+                    }
+                } catch (error: any) {
+                    logger.error(`Error processing batch update: ${error.message}`);
+                }
+            });
+
+            // Initialize API server (with kvManager for service discovery)
+            apiServer = new APIServer(vfs, {
+                port: config.apiPort,
+                host: config.apiHost,
+            }, kvManager);
+
+            // Start API server
+            logger.info('Starting API server...');
+            await apiServer.start();
+
+            logger.info('meta-fuse is ready!');
+            logger.info(`API: http://${config.apiHost}:${config.apiPort}`);
+
+            // Log initial stats
+            const stats = vfs.getStats();
+            logger.info(`VFS: ${stats.fileCount} files, ${stats.directoryCount} directories`);
+        } catch (error) {
+            logger.error('Failed to initialize after KV ready:', error);
+        }
+    });
+
+    // Handle disconnect event
+    kvManager.onDisconnect(() => {
+        logger.warn('Redis disconnected, VFS will use cached data');
+    });
+
+    // Start KV Manager (initiates leader discovery)
+    try {
+        logger.info('Starting KV Manager...');
+        await kvManager.start();
+
+        // Wait for ready with timeout
+        await kvManager.waitForReady(60000);
+    } catch (error) {
+        logger.error('Failed to start meta-fuse:', error);
+        process.exit(1);
+    }
+}
+
+main().catch((error) => {
+    logger.error('Fatal error:', error);
+    process.exit(1);
+});