bug fix on file dispaly, document update

packages/meta-fuse-core/src/index.ts

@@ -151,6 +151,10 @@ async function main(): Promise<void> {
                 filesPath: config.filesPath,
             });
 
+            // Wire up StateBuilder to VFS for refresh operations
+            // This enables vfs.refresh() to replay the event stream
+            vfs.setStreamingStateBuilder(stateBuilder);
+
             // Bootstrap: Replay meta:events stream from position 0
             // This rebuilds state from all historical events
             logger.info(`Replaying ${EVENTS_STREAM} stream from position 0 (streaming bootstrap)...`);
@@ -166,21 +170,9 @@ async function main(): Promise<void> {
             );
 
             const bootstrapTime = Date.now() - bootstrapStart;
-            let stats = vfs.getStats();
+            const stats = vfs.getStats();
             logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
 
-            // Fallback: If stream was empty but Redis has files, bootstrap from Redis keys
-            // This handles the case where meta-core restarted and cleared the stream
-            if (stats.fileCount === 0) {
-                const indexCount = await redisClient.getIndexCount();
-                if (indexCount > 0) {
-                    logger.info(`Stream was empty but Redis has ${indexCount} files - bootstrapping from Redis keys...`);
-                    await vfs.refresh();
-                    stats = vfs.getStats();
-                    logger.info(`Redis key bootstrap complete: ${stats.fileCount} files`);
-                }
-            }
-
             // Start live stream consumer from where replay left off
             logger.info(`Starting live stream consumer from ${lastId}...`);
             redisClient.startSimpleStreamConsumer(

packages/meta-fuse-core/src/kv/IKVClient.ts

@@ -74,9 +74,12 @@ export interface LeaderLockInfo {
     /** Redis connection URL (e.g., redis://10.0.1.50:6379) */
     redisUrl: string;
 
-    /** WebDAV URL for file access */
+    /** WebDAV URL for file access (external, via nginx/HTTPS) */
     webdavUrl: string;
 
+    /** WebDAV URL for internal container-to-container access (direct to port 9000) */
+    webdavUrlInternal: string;
+
     /** Timestamp when leadership was acquired */
     timestamp: number;
 

packages/meta-fuse-core/src/kv/LeaderClient.ts

@@ -32,6 +32,7 @@ export interface URLsResponse {
     apiUrl: string;
     redisUrl: string;
     webdavUrl: string;
+    webdavUrlInternal: string;
     isLeader: boolean;
 }
 
@@ -120,6 +121,7 @@ export class LeaderClient {
                 apiUrl: urls.apiUrl,
                 redisUrl: urls.redisUrl,
                 webdavUrl: urls.webdavUrl,
+                webdavUrlInternal: urls.webdavUrlInternal,
                 timestamp: Date.now(),
                 pid: 0 // Unknown for remote leader
             };
@@ -139,13 +141,22 @@ export class LeaderClient {
     }
 
     /**
-     * Get WebDAV URL from leader info
+     * Get WebDAV URL from leader info (external, via nginx/HTTPS)
      */
     async getWebdavUrl(): Promise<string | null> {
         const info = await this.getLeaderInfo();
         return info?.webdavUrl ?? null;
     }
 
+    /**
+     * Get internal WebDAV URL from leader info (direct to port 9000)
+     * Use this for container-to-container communication
+     */
+    async getWebdavUrlInternal(): Promise<string | null> {
+        const info = await this.getLeaderInfo();
+        return info?.webdavUrlInternal ?? null;
+    }
+
     /**
      * Get meta-core API URL from leader info
      */

packages/meta-fuse-core/src/kv/RedisClient.ts

@@ -463,6 +463,76 @@ export class RedisClient implements Partial<IKVClient> {
         }
     }
 
+    /**
+     * Scan flat keys to build file metadata for all indexed files
+     * This is used as a fallback when the meta:events stream is empty or unavailable.
+     *
+     * Algorithm:
+     * 1. Get all hash IDs from the index
+     * 2. For each hash ID, fetch VFS-relevant properties using MGET
+     * 3. Return a Map of hashId to flat properties
+     *
+     * This replaces the broken getAllFiles() which used HGETALL on flat keys.
+     *
+     * @param vfsRelevantProperties - Set of property names that are VFS-relevant
+     * @returns Map of hashId to properties (only files with filePath)
+     */
+    async scanFlatKeysForFiles(vfsRelevantProperties: Set<string>): Promise<Map<string, Record<string, string>>> {
+        const result = new Map<string, Record<string, string>>();
+
+        if (!this.client || !this.isConnected) {
+            logger.warn('Redis not connected, returning empty result');
+            return result;
+        }
+
+        try {
+            // Get all hash IDs from the index
+            const hashIds = await this.getAllHashIds();
+            logger.info(`Scanning flat keys for ${hashIds.length} files...`);
+
+            if (hashIds.length === 0) {
+                return result;
+            }
+
+            // Batch process hashIds to avoid overwhelming Redis
+            const batchSize = 100;
+            const propertiesArray = Array.from(vfsRelevantProperties);
+
+            for (let i = 0; i < hashIds.length; i += batchSize) {
+                const batch = hashIds.slice(i, i + batchSize);
+
+                // Process each hashId in the batch
+                const batchPromises = batch.map(async (hashId) => {
+                    const props = await this.getProperties(hashId, propertiesArray);
+
+                    // Only include files that have a filePath
+                    if (props.has('filePath')) {
+                        const propsObj: Record<string, string> = {};
+                        for (const [key, value] of props) {
+                            propsObj[key] = value;
+                        }
+                        return { hashId, props: propsObj };
+                    }
+                    return null;
+                });
+
+                const batchResults = await Promise.all(batchPromises);
+
+                for (const item of batchResults) {
+                    if (item) {
+                        result.set(item.hashId, item.props);
+                    }
+                }
+            }
+
+            logger.info(`Flat key scan complete: ${result.size} files with filePath`);
+            return result;
+        } catch (error: any) {
+            logger.error('Failed to scan flat keys:', error.message);
+            return result;
+        }
+    }
+
     /**
      * Scan Redis keys matching pattern
      */

packages/meta-fuse-core/src/vfs/StreamingStateBuilder.ts

@@ -101,6 +101,21 @@ export class StreamingStateBuilder {
         logger.info(`Updated VFS-relevant properties: ${this.vfsRelevantProps.size}`);
     }
 
+    /**
+     * Get the set of VFS-relevant properties
+     * Used by VFS refresh for flat key scanning fallback
+     */
+    getVfsRelevantProperties(): Set<string> {
+        return this.vfsRelevantProps;
+    }
+
+    /**
+     * Get the current VFS callback
+     */
+    getVFSCallback(): VFSUpdateCallback | undefined {
+        return this.vfsCallback;
+    }
+
     /**
      * Process a stream message event
      * This is the main entry point for event processing

packages/meta-fuse-core/src/vfs/VirtualFileSystem.ts

@@ -15,7 +15,7 @@ import { TemplateEngine } from './template/TemplateEngine.js';
 import { ConditionEvaluator } from './template/ConditionEvaluator.js';
 import { ConfigStorage } from '../config/ConfigStorage.js';
 import type { RenamingConfig, RenamingRule, PreviewItem, PreviewResponse } from './types/RenamingRuleTypes.js';
-import type { VFSUpdateCallback } from './StreamingStateBuilder.js';
+import type { VFSUpdateCallback, StreamingStateBuilder } from './StreamingStateBuilder.js';
 
 const logger = new Logger({ name: 'VirtualFileSystem' });
 
@@ -72,6 +72,7 @@ export class VirtualFileSystem implements VFSUpdateCallback {
     private config: Required<VFSConfig>;
     private lastRefresh: Date | null = null;
     private useStreamingMode: boolean = false;  // Flag to indicate streaming mode
+    private stateBuilder: StreamingStateBuilder | null = null;  // For stream replay during refresh
 
     private cachedStats = {
         fileCount: 0,
@@ -128,6 +129,15 @@ export class VirtualFileSystem implements VFSUpdateCallback {
         logger.info('VFS streaming mode enabled');
     }
 
+    /**
+     * Set the streaming state builder for refresh operations
+     * This enables refresh() to replay the event stream instead of using broken HGETALL
+     */
+    setStreamingStateBuilder(builder: StreamingStateBuilder): void {
+        this.stateBuilder = builder;
+        logger.debug('StreamingStateBuilder set for VFS refresh');
+    }
+
     /**
      * Convert flat properties map to FileMetadata
      * Used when processing streaming events
@@ -281,46 +291,144 @@ export class VirtualFileSystem implements VFSUpdateCallback {
     }
 
     /**
-     * Refresh the VFS from Redis
-     * Called once on startup for initial bootstrap.
-     * Subsequent updates come via Redis Streams (onFileUpdate).
-     * Uses template-based rules if configured, otherwise falls back to MetaDataToFolderStruct
+     * Refresh the VFS from Redis by replaying the event stream
+     *
+     * This method replays the meta:events stream from position 0, applying updates
+     * incrementally without wiping the existing VFS state. Files not seen during
+     * the replay are considered deleted and removed.
+     *
+     * Design principles:
+     * - Do NOT wipe the internal model (no initRoot() call)
+     * - Track which files are seen during replay
+     * - Remove only files that are no longer in the stream (deleted)
+     * - Single code path - reuses StreamingStateBuilder event processing
      */
     async refresh(): Promise<void> {
-        logger.debug('Loading VFS from Redis (bootstrap)');
+        logger.info('Refreshing VFS from event stream...');
+        const startTime = Date.now();
+
+        // Load rules configuration
+        this.loadRulesConfig();
+
+        // Check if we have a streaming state builder configured
+        if (!this.stateBuilder) {
+            logger.warn('No StreamingStateBuilder set - cannot refresh from stream');
+            logger.warn('VFS refresh skipped. Set streaming state builder via setStreamingStateBuilder()');
+            return;
+        }
 
         try {
-            // Load rules configuration
-            this.loadRulesConfig();
+            // Track which hashIds we see during this refresh
+            const seenHashIds = new Set<string>();
 
-            // Get all files from Redis (keyed by sourcePath)
-            const files = await this.redisClient.getAllFiles();
+            // Create a tracking callback that records seen hashIds
+            const originalCallback = this.stateBuilder.getVFSCallback();
+            const trackingVFS: VFSUpdateCallback = {
+                onPropertyChange: (hashId, property, value) => {
+                    this.onPropertyChange(hashId, property, value);
+                },
+                onPropertyDelete: (hashId, property) => {
+                    this.onPropertyDelete(hashId, property);
+                },
+                onFileDelete: (hashId) => {
+                    this.onFileDelete(hashId);
+                },
+                onFileComplete: (hashId, metadata) => {
+                    seenHashIds.add(hashId);
+                    this.onFileComplete(hashId, metadata);
+                },
+            };
 
-            if (files.size === 0) {
-                logger.info('No files found in Redis');
-                this.initRoot();
-                return;
-            }
+            // Temporarily set tracking callback
+            this.stateBuilder.setVFSCallback(trackingVFS);
 
-            // Rebuild VFS with computed paths
-            this.initRoot();
+            // Clear state builder's internal state (but NOT VFS nodes)
+            this.stateBuilder.clear();
 
-            for (const [sourcePath, metadata] of files) {
-                // Compute virtual path using rules or fallback
-                const virtualPath = this.computeVirtualPath(metadata, sourcePath);
-                if (virtualPath) {
-                    await this.addFile(virtualPath, metadata);
-                    this.sourcePathToVirtualPath.set(sourcePath, virtualPath);
+            // Replay meta:events stream from position 0
+            const lastId = await this.redisClient.replayStream(
+                'meta:events',
+                '0',
+                async (message) => {
+                    await this.stateBuilder!.processEvent(message);
+                },
+                100 // Batch size
+            );
+
+            // Fallback: If no files found from stream replay, scan flat keys directly
+            // This handles the case where meta:events stream isn't being populated
+            // but data exists in flat key format
+            if (seenHashIds.size === 0) {
+                const indexCount = await this.redisClient.getIndexCount();
+                if (indexCount > 0) {
+                    logger.info(`Stream replay found no files but index has ${indexCount} - scanning flat keys...`);
+
+                    // Get VFS-relevant properties from state builder
+                    const vfsRelevantProps = this.stateBuilder.getVfsRelevantProperties();
+
+                    // Scan flat keys
+                    const flatKeyFiles = await this.redisClient.scanFlatKeysForFiles(vfsRelevantProps);
+
+                    // Process each file through onFileComplete
+                    for (const [hashId, props] of flatKeyFiles) {
+                        seenHashIds.add(hashId);
+                        this.onFileComplete(hashId, props);
+                    }
+
+                    logger.info(`Flat key scan populated ${seenHashIds.size} files`);
                 }
             }
 
+            // Restore original callback
+            this.stateBuilder.setVFSCallback(originalCallback || this);
+
+            // Remove files not seen during replay (these were deleted)
+            const toRemove: string[] = [];
+            for (const [hashId, virtualPath] of this.hashIdToVirtualPath) {
+                if (!seenHashIds.has(hashId)) {
+                    logger.debug(`Removing deleted file: ${hashId} -> ${virtualPath}`);
+                    toRemove.push(hashId);
+                }
+            }
+
+            for (const hashId of toRemove) {
+                const virtualPath = this.hashIdToVirtualPath.get(hashId);
+                if (virtualPath) {
+                    this.removeFile(virtualPath);
+                }
+                this.hashIdToVirtualPath.delete(hashId);
+            }
+
             this.lastRefresh = new Date();
-            logger.info(`VFS refreshed: ${this.cachedStats.fileCount} files, ${this.cachedStats.directoryCount} directories`);
+            this.updateStats();
+            const elapsed = Date.now() - startTime;
+            logger.info(`VFS refresh complete: ${this.cachedStats.fileCount} files, ${this.cachedStats.directoryCount} directories in ${elapsed}ms`);
+            logger.info(`Replay ended at stream position: ${lastId}, removed ${toRemove.length} deleted files`);
         } catch (error) {
             logger.error('Failed to refresh VFS:', error);
         }
     }
 
+    /**
+     * Update cached stats from current nodes
+     */
+    private updateStats(): void {
+        let fileCount = 0;
+        let directoryCount = 0;
+        let totalSize = 0;
+
+        for (const node of this.nodes.values()) {
+            if (node.type === 'file') {
+                fileCount++;
+                totalSize += node.size ?? 0;
+            } else {
+                directoryCount++;
+            }
+        }
+
+        this.cachedStats = { fileCount, directoryCount, totalSize };
+    }
+
     /**
      * Load rules configuration from file
      * ConfigStorage creates the default config file if it doesn't exist