bug fix on file dispaly, document update
This commit is contained in:
Pierre
@@ -92,30 +92,37 @@ Meta-Fuse is a read-only process in the MetaMesh ecosystem that:
|
|||||||
|
|
||||||
### KV Leader Election
|
### KV Leader Election
|
||||||
|
|
||||||
All services in MetaMesh use a shared KV wrapper with leader election via flock(2):
|
All services in MetaMesh use leader discovery via meta-core's lock file and HTTP API:
|
||||||
|
|
||||||
```
|
```
|
||||||
/meta-core/locks/
|
/meta-core/locks/
|
||||||
└── kv-leader.lock # flock-based leader election + endpoint info
|
├── kv-leader.lock # flock-based leader election
|
||||||
|
└── kv-leader.info # Leader API URL (plain text)
|
||||||
|
|
||||||
Lock File Format (JSON):
|
Lock Info File Format (plain text):
|
||||||
|
http://meta-core:9000
|
||||||
|
|
||||||
|
/urls API Response (JSON):
|
||||||
{
|
{
|
||||||
"host": "meta-sort-instance-1",
|
"hostname": "meta-core-dev",
|
||||||
"api": "redis://10.0.1.50:6379",
|
"baseUrl": "http://localhost:8083",
|
||||||
"http": "http://10.0.1.50:3000",
|
"apiUrl": "http://meta-core:9000",
|
||||||
"timestamp": 1703808000000,
|
"redisUrl": "redis://meta-core:6379",
|
||||||
"pid": 12345
|
"webdavUrl": "http://localhost:8083/webdav",
|
||||||
|
"webdavUrlInternal": "http://meta-core:9000/webdav",
|
||||||
|
"isLeader": true
|
||||||
}
|
}
|
||||||
|
|
||||||
Leader Election Flow:
|
Leader Discovery Flow:
|
||||||
1. meta-sort attempts exclusive flock on kv-leader.lock
|
1. meta-core acquires flock on kv-leader.lock
|
||||||
2. Winner (leader) spawns Redis, writes endpoint to lock file
|
2. Winner (leader) spawns Redis, writes API URL to kv-leader.info
|
||||||
3. meta-fuse reads lock file to discover Redis endpoint
|
3. meta-fuse reads kv-leader.info to find API URL
|
||||||
4. meta-fuse connects as read-only client
|
4. meta-fuse calls /urls API to get Redis URL and other endpoints
|
||||||
5. On leader failure, flock auto-releases, new leader elected
|
5. meta-fuse connects to Redis as read-only client
|
||||||
|
6. On leader failure, flock auto-releases, new leader elected
|
||||||
```
|
```
|
||||||
|
|
||||||
**Note**: meta-fuse never becomes leader - it only reads the lock file to discover the active Redis endpoint managed by meta-sort.
|
**Note**: meta-fuse never becomes leader - it reads the lock file and calls the /urls API to discover the active Redis endpoint managed by meta-core.
|
||||||
|
|
||||||
## Core Features
|
## Core Features
|
||||||
|
|
||||||
@@ -150,15 +157,26 @@ meta-fuse/
|
|||||||
│ │ ├── src/
|
│ │ ├── src/
|
||||||
│ │ │ ├── api/ # Fastify REST API (APIServer)
|
│ │ │ ├── api/ # Fastify REST API (APIServer)
|
||||||
│ │ │ │ └── APIServer.ts
|
│ │ │ │ └── APIServer.ts
|
||||||
|
│ │ │ ├── config/ # Configuration management
|
||||||
|
│ │ │ │ └── ConfigStorage.ts
|
||||||
│ │ │ ├── kv/ # KV client wrapper (FOLLOWER mode only)
|
│ │ │ ├── kv/ # KV client wrapper (FOLLOWER mode only)
|
||||||
│ │ │ │ ├── IKVClient.ts # Read-only interface
|
│ │ │ │ ├── IKVClient.ts # Read-only interface
|
||||||
│ │ │ │ ├── KVManager.ts # Leader discovery, connection management
|
│ │ │ │ ├── KVManager.ts # Leader discovery, connection management
|
||||||
│ │ │ │ ├── LeaderDiscovery.ts # Lock file reading
|
│ │ │ │ ├── LeaderClient.ts # Lock file reading + /urls API
|
||||||
│ │ │ │ └── RedisClient.ts # Redis connection with pub/sub
|
│ │ │ │ ├── RedisClient.ts # Redis connection with Streams support
|
||||||
|
│ │ │ │ └── ServiceDiscovery.ts # Service discovery client
|
||||||
│ │ │ ├── vfs/ # Virtual filesystem logic
|
│ │ │ ├── vfs/ # Virtual filesystem logic
|
||||||
│ │ │ │ ├── VirtualFileSystem.ts # In-memory VFS representation
|
│ │ │ │ ├── VirtualFileSystem.ts # In-memory VFS representation
|
||||||
│ │ │ │ ├── MetaDataToFolderStruct.ts # Folder organization
|
│ │ │ │ ├── StreamingStateBuilder.ts # Event-driven state management
|
||||||
│ │ │ │ └── RenamingRule.ts # Virtual path rules
|
│ │ │ │ ├── RulesPropertyExtractor.ts # Property filtering from rules
|
||||||
|
│ │ │ │ ├── MetaDataToFolderStruct.ts # Folder organization
|
||||||
|
│ │ │ │ ├── RenamingRule.ts # Virtual path rules
|
||||||
|
│ │ │ │ ├── defaults/ # Default renaming rules
|
||||||
|
│ │ │ │ ├── template/ # Template engine components
|
||||||
|
│ │ │ │ │ ├── TemplateEngine.ts # Variable interpolation
|
||||||
|
│ │ │ │ │ └── ConditionEvaluator.ts # Rule condition evaluation
|
||||||
|
│ │ │ │ └── types/ # TypeScript type definitions
|
||||||
|
│ │ │ │ └── RenamingRuleTypes.ts
|
||||||
│ │ │ └── index.ts # Entry point
|
│ │ │ └── index.ts # Entry point
|
||||||
│ │ ├── package.json
|
│ │ ├── package.json
|
||||||
│ │ └── tsconfig.json
|
│ │ └── tsconfig.json
|
||||||
@@ -174,6 +192,11 @@ meta-fuse/
|
|||||||
│ ├── src/
|
│ ├── src/
|
||||||
│ └── package.json
|
│ └── package.json
|
||||||
│
|
│
|
||||||
|
├── docs/ # Architecture documentation
|
||||||
|
│ ├── vfs-rebuild-architecture.md
|
||||||
|
│ ├── streaming-architecture.md
|
||||||
|
│ └── api-reference.md
|
||||||
|
│
|
||||||
├── docker/
|
├── docker/
|
||||||
│ ├── nginx.conf # Reverse proxy config
|
│ ├── nginx.conf # Reverse proxy config
|
||||||
│ ├── wsgidav.yaml # WebDAV server config
|
│ ├── wsgidav.yaml # WebDAV server config
|
||||||
@@ -191,21 +214,27 @@ meta-fuse/
|
|||||||
|
|
||||||
| Component | File | Purpose |
|
| Component | File | Purpose |
|
||||||
|-----------|------|---------|
|
|-----------|------|---------|
|
||||||
| Entry Point | `src/index.ts` | Initializes KV manager, VFS, API server, pub/sub listener |
|
| Entry Point | `src/index.ts` | Initializes KV manager, VFS, API server, stream consumer |
|
||||||
| API Server | `src/api/APIServer.ts` | Fastify REST API for FUSE operations (readdir, getattr, read) |
|
| API Server | `src/api/APIServer.ts` | Fastify REST API for FUSE operations and rules management |
|
||||||
| KV Manager | `src/kv/KVManager.ts` | **FOLLOWER-only**: leader discovery, Redis connection, reconnection loop |
|
| KV Manager | `src/kv/KVManager.ts` | **FOLLOWER-only**: leader discovery, Redis connection, reconnection loop |
|
||||||
| Leader Discovery | `src/kv/LeaderDiscovery.ts` | Reads `/meta-core/locks/kv-leader.lock` to find active Redis |
|
| Leader Client | `src/kv/LeaderClient.ts` | Reads `/meta-core/locks/kv-leader.info`, calls `/urls` API |
|
||||||
| Redis Client | `src/kv/RedisClient.ts` | Wrapper around ioredis with pub/sub support |
|
| Redis Client | `src/kv/RedisClient.ts` | Wrapper around ioredis with Redis Streams support |
|
||||||
|
| Service Discovery | `src/kv/ServiceDiscovery.ts` | Discovers all MetaMesh services via service files |
|
||||||
| KV Interface | `src/kv/IKVClient.ts` | Read-only interface (get, scan, subscribe) |
|
| KV Interface | `src/kv/IKVClient.ts` | Read-only interface (get, scan, subscribe) |
|
||||||
| Virtual FS | `src/vfs/VirtualFileSystem.ts` | In-memory VFS representation with caching |
|
| Virtual FS | `src/vfs/VirtualFileSystem.ts` | In-memory VFS representation with caching |
|
||||||
|
| Streaming State Builder | `src/vfs/StreamingStateBuilder.ts` | Processes `meta:events` stream, builds VFS state incrementally |
|
||||||
|
| Rules Property Extractor | `src/vfs/RulesPropertyExtractor.ts` | Extracts VFS-relevant properties from renaming rules |
|
||||||
| Folder Organizer | `src/vfs/MetaDataToFolderStruct.ts` | Converts flat metadata to organized folder structure |
|
| Folder Organizer | `src/vfs/MetaDataToFolderStruct.ts` | Converts flat metadata to organized folder structure |
|
||||||
| Renaming Rules | `src/vfs/RenamingRule.ts` | Rules for virtual path organization |
|
| Renaming Rules | `src/vfs/RenamingRule.ts` | Rules for virtual path organization |
|
||||||
|
| Template Engine | `src/vfs/template/TemplateEngine.ts` | Variable interpolation for renaming templates |
|
||||||
|
| Condition Evaluator | `src/vfs/template/ConditionEvaluator.ts` | Evaluates rule conditions against file metadata |
|
||||||
|
|
||||||
**Note**: Unlike meta-sort, meta-fuse's KVManager is simplified:
|
**Note**: Unlike meta-sort, meta-fuse's KVManager is simplified:
|
||||||
- **Never spawns Redis** (always FOLLOWER)
|
- **Never spawns Redis** (always FOLLOWER)
|
||||||
- **Only reads metadata** (read-only interface)
|
- **Only reads metadata** (read-only interface)
|
||||||
- **Discovers leader** via lock file at `/meta-core/locks/kv-leader.lock`
|
- **Discovers leader** via lock file at `/meta-core/locks/kv-leader.info`
|
||||||
- **No service discovery registration** (minimal footprint)
|
- **Calls `/urls` API** to get Redis URL, WebDAV URL, and other endpoints
|
||||||
|
- **Uses streaming mode** - processes `meta:events` stream for real-time updates
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
@@ -286,12 +315,28 @@ services:
|
|||||||
|
|
||||||
| Method | Endpoint | Description |
|
| Method | Endpoint | Description |
|
||||||
|--------|----------|-------------|
|
|--------|----------|-------------|
|
||||||
| GET | `/api/fuse/health` | Health check |
|
| **Health & Status** |
|
||||||
|
| GET | `/health` | Health check |
|
||||||
|
| GET | `/api/health` | Health check (alias) |
|
||||||
|
| GET | `/api/fuse/health` | Health check (alias) |
|
||||||
| GET | `/api/fuse/stats` | Filesystem statistics |
|
| GET | `/api/fuse/stats` | Filesystem statistics |
|
||||||
|
| **FUSE Operations** |
|
||||||
| POST | `/api/fuse/readdir` | List directory contents |
|
| POST | `/api/fuse/readdir` | List directory contents |
|
||||||
| POST | `/api/fuse/getattr` | Get file/directory attributes |
|
| POST | `/api/fuse/getattr` | Get file/directory attributes |
|
||||||
| POST | `/api/fuse/read` | Read file content |
|
|
||||||
| POST | `/api/fuse/exists` | Check path existence |
|
| POST | `/api/fuse/exists` | Check path existence |
|
||||||
|
| POST | `/api/fuse/read` | Read file content (returns source path or WebDAV URL) |
|
||||||
|
| POST | `/api/fuse/metadata` | Get full metadata for a path |
|
||||||
|
| GET | `/api/fuse/files` | List all virtual files |
|
||||||
|
| GET | `/api/fuse/directories` | List all virtual directories |
|
||||||
|
| POST | `/api/fuse/refresh` | Trigger VFS refresh |
|
||||||
|
| **Renaming Rules** |
|
||||||
|
| GET | `/api/fuse/rules` | Get current renaming rules configuration |
|
||||||
|
| PUT | `/api/fuse/rules` | Update renaming rules configuration |
|
||||||
|
| POST | `/api/fuse/rules/preview` | Preview how files would be renamed |
|
||||||
|
| POST | `/api/fuse/rules/validate` | Validate a single rule |
|
||||||
|
| GET | `/api/fuse/rules/variables` | Get list of available template variables |
|
||||||
|
| **Service Discovery** |
|
||||||
|
| GET | `/api/services` | List all discovered MetaMesh services |
|
||||||
|
|
||||||
### Example Responses
|
### Example Responses
|
||||||
|
|
||||||
@@ -474,15 +519,19 @@ cargo build --release
|
|||||||
|
|
||||||
### KV Data Structure
|
### KV Data Structure
|
||||||
|
|
||||||
meta-fuse reads metadata stored by meta-sort using the `meta-sort:` prefix:
|
meta-fuse reads metadata stored by meta-sort using flat Redis keys:
|
||||||
|
|
||||||
```
|
```
|
||||||
# File metadata stored by meta-sort (hash-based keys)
|
# File metadata stored by meta-sort (flat key format)
|
||||||
meta-sort:file:{midhash256}:title → "Inception"
|
file:{hashId}/title → "Inception"
|
||||||
meta-sort:file:{midhash256}:year → "2010"
|
file:{hashId}/year → "2010"
|
||||||
meta-sort:file:{midhash256}:filePath → "media1/Movies/Inception (2010)/Inception.mkv"
|
file:{hashId}/filePath → "media1/Movies/Inception (2010)/Inception.mkv"
|
||||||
meta-sort:file:{midhash256}:size → 4831838208
|
file:{hashId}/size → 4831838208
|
||||||
meta-sort:file:{midhash256}:video/codec → "h265"
|
file:{hashId}/video/codec → "h265"
|
||||||
|
file:{hashId}/titles/eng → "Inception"
|
||||||
|
|
||||||
|
# File index for enumeration
|
||||||
|
file:__index__ → SET of all hashIds
|
||||||
|
|
||||||
# VFS paths are computed dynamically from metadata
|
# VFS paths are computed dynamically from metadata
|
||||||
# meta-fuse builds virtual paths like:
|
# meta-fuse builds virtual paths like:
|
||||||
@@ -490,28 +539,45 @@ meta-sort:file:{midhash256}:video/codec → "h265"
|
|||||||
# → resolves to sourcePath: /files/media1/Movies/Inception (2010)/Inception.mkv
|
# → resolves to sourcePath: /files/media1/Movies/Inception (2010)/Inception.mkv
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**Key Format**: `file:{hashId}/{property}` where:
|
||||||
|
- `hashId` is the midhash256 content identifier
|
||||||
|
- `property` can be flat (e.g., `title`) or nested (e.g., `titles/eng`)
|
||||||
|
|
||||||
**Note**: File paths in Redis are **relative to FILES_VOLUME** (`/files`). meta-fuse prepends the FILES_VOLUME path when resolving actual file locations.
|
**Note**: File paths in Redis are **relative to FILES_VOLUME** (`/files`). meta-fuse prepends the FILES_VOLUME path when resolving actual file locations.
|
||||||
|
|
||||||
### Real-Time Updates via Pub/Sub
|
### Real-Time Updates via Redis Streams
|
||||||
|
|
||||||
meta-fuse subscribes to Redis pub/sub channel `meta-sort:file:batch` for real-time metadata updates from meta-sort:
|
meta-fuse uses the `meta:events` Redis Stream for real-time metadata updates. The streaming architecture provides:
|
||||||
|
- **Reliable delivery** - Messages persist until consumed
|
||||||
|
- **Replay capability** - Can rebuild state from stream position 0
|
||||||
|
- **Memory efficiency** - Only VFS-relevant properties are fetched
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
interface BatchUpdateMessage {
|
// Stream message format (meta:events)
|
||||||
timestamp: number;
|
interface StreamMessage {
|
||||||
changes: Array<{
|
id: string; // Stream entry ID (e.g., "1703808000000-0")
|
||||||
action: 'add' | 'update' | 'remove';
|
type: 'set' | 'del'; // Operation type
|
||||||
hashId: string;
|
key: string; // Redis key (e.g., "file:abc123/title")
|
||||||
}>;
|
ts: string; // Timestamp
|
||||||
}
|
}
|
||||||
```
|
```
|
||||||
|
|
||||||
When meta-sort adds/updates/removes files, it publishes batch updates. meta-fuse processes these to:
|
**Startup Sequence**:
|
||||||
1. Invalidate affected caches
|
1. **Streaming Bootstrap**: Replay `meta:events` stream from position 0
|
||||||
2. Rebuild virtual paths for modified files
|
2. **Build State**: Process each event, fetch only VFS-relevant properties
|
||||||
3. Update directory listings without full refresh
|
3. **Go Live**: Continue consuming new events from last processed position
|
||||||
|
|
||||||
This ensures the virtual filesystem stays synchronized with metadata changes in near real-time.
|
**Event Processing Pipeline**:
|
||||||
|
1. Parse key to extract `hashId` and `property`
|
||||||
|
2. Check if property is VFS-relevant (based on renaming rules)
|
||||||
|
3. If relevant, fetch property value from Redis
|
||||||
|
4. Update internal state and notify VFS
|
||||||
|
5. When file has `filePath`, it appears in VFS
|
||||||
|
|
||||||
|
This streaming architecture enables:
|
||||||
|
- Sub-second startup (no HGETALL/SCAN)
|
||||||
|
- Memory-efficient state (~500 bytes/file)
|
||||||
|
- Real-time VFS updates as files are processed
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -537,10 +603,13 @@ journalctl -u meta-fuse-driver -f
|
|||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Check lock file exists
|
# Check lock file exists
|
||||||
cat /data/apps/meta-core/db/redis.lock
|
cat /meta-core/locks/kv-leader.info
|
||||||
|
|
||||||
|
# Check /urls API
|
||||||
|
curl http://localhost:8083/api/urls
|
||||||
|
|
||||||
# Verify redis is running (on leader)
|
# Verify redis is running (on leader)
|
||||||
redis-cli ping
|
docker exec meta-core-dev redis-cli ping
|
||||||
|
|
||||||
# Check meta-fuse logs
|
# Check meta-fuse logs
|
||||||
docker logs meta-fuse | grep "KV"
|
docker logs meta-fuse | grep "KV"
|
||||||
@@ -562,14 +631,20 @@ docker logs meta-fuse | grep "wsgidav"
|
|||||||
### Files Not Appearing
|
### Files Not Appearing
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Check KV has data
|
# Check Redis has file data (flat key format)
|
||||||
redis-cli keys "meta-fuse:vfs:*"
|
docker exec meta-core-dev redis-cli keys "file:*" | head -20
|
||||||
|
|
||||||
|
# Check file index
|
||||||
|
docker exec meta-core-dev redis-cli scard "file:__index__"
|
||||||
|
|
||||||
# Verify source files exist
|
# Verify source files exist
|
||||||
ls -la /data/watch/
|
ls -la /files/watch/
|
||||||
|
|
||||||
# Check API stats
|
# Check API stats
|
||||||
curl http://localhost:3000/api/fuse/stats
|
curl http://localhost:3000/api/fuse/stats
|
||||||
|
|
||||||
|
# Check streaming state builder stats
|
||||||
|
docker logs meta-fuse-dev | grep "State builder"
|
||||||
```
|
```
|
||||||
|
|
||||||
## Technology Stack
|
## Technology Stack
|
||||||
@@ -597,14 +672,25 @@ Meta-Fuse is designed to work seamlessly with other MetaMesh services:
|
|||||||
Ensure meta-sort is running and has processed files:
|
Ensure meta-sort is running and has processed files:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# On meta-sort host
|
# Check meta-sort processing status
|
||||||
curl http://localhost:3000/api/processing/status
|
curl http://localhost:8180/api/processing/status
|
||||||
|
|
||||||
# Verify KV has metadata
|
# Verify Redis has metadata (via meta-core)
|
||||||
redis-cli keys "meta-sort:file:*"
|
docker exec meta-core-dev redis-cli scard "file:__index__"
|
||||||
|
|
||||||
|
# Check meta:events stream has events
|
||||||
|
docker exec meta-core-dev redis-cli xlen "meta:events"
|
||||||
```
|
```
|
||||||
|
|
||||||
Meta-fuse will automatically discover the KV database through the shared lock file.
|
Meta-fuse will automatically discover the KV database through the shared lock file at `/meta-core/locks/kv-leader.info`.
|
||||||
|
|
||||||
|
## Documentation
|
||||||
|
|
||||||
|
For detailed architecture documentation, see the `docs/` directory:
|
||||||
|
|
||||||
|
- [VFS Rebuild Architecture](docs/vfs-rebuild-architecture.md) - How VFS state is built from Redis Streams
|
||||||
|
- [Streaming Architecture](docs/streaming-architecture.md) - Event processing pipeline details
|
||||||
|
- [API Reference](docs/api-reference.md) - Complete REST API documentation
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,671 @@
|
|||||||
|
# API Reference
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
meta-fuse exposes a REST API for FUSE operations, VFS management, and service discovery. The API server is built with Fastify and listens on port 3000 by default.
|
||||||
|
|
||||||
|
**Base URL**: `http://localhost:3000`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Health & Status
|
||||||
|
|
||||||
|
### GET /health
|
||||||
|
|
||||||
|
Health check endpoint.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "ok",
|
||||||
|
"timestamp": "2024-01-15T10:30:00.000Z",
|
||||||
|
"service": "meta-fuse"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /api/health
|
||||||
|
|
||||||
|
Alias for `/health`.
|
||||||
|
|
||||||
|
### GET /api/fuse/health
|
||||||
|
|
||||||
|
Alias for `/health`.
|
||||||
|
|
||||||
|
### GET /api/fuse/stats
|
||||||
|
|
||||||
|
Get VFS statistics and state information.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"fileCount": 5123,
|
||||||
|
"directoryCount": 847,
|
||||||
|
"totalSize": 15847392847362,
|
||||||
|
"lastRefresh": "2024-01-15T10:30:00.000Z",
|
||||||
|
"categories": {
|
||||||
|
"Movies": 1523,
|
||||||
|
"TV": 2847,
|
||||||
|
"Anime": 753
|
||||||
|
},
|
||||||
|
"kvConnection": "connected",
|
||||||
|
"fuseMount": "mounted"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## FUSE Operations
|
||||||
|
|
||||||
|
### POST /api/fuse/readdir
|
||||||
|
|
||||||
|
List contents of a virtual directory.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/Movies/Action"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"entries": [
|
||||||
|
"Die Hard (1988)",
|
||||||
|
"Mad Max Fury Road (2015)",
|
||||||
|
"The Dark Knight (2008)"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (404 Not Found):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "Directory not found"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/getattr
|
||||||
|
|
||||||
|
Get file or directory attributes.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/Movies/Action/Die Hard (1988)/Die Hard.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK) - File:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"size": 4831838208,
|
||||||
|
"mode": "file",
|
||||||
|
"permissions": 644,
|
||||||
|
"uid": 1000,
|
||||||
|
"gid": 1000,
|
||||||
|
"atime": "2024-01-15T10:30:00Z",
|
||||||
|
"mtime": "2024-01-15T10:30:00Z",
|
||||||
|
"ctime": "2024-01-15T10:30:00Z",
|
||||||
|
"sourcePath": "/files/watch/downloads/Die.Hard.1988.1080p.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK) - Directory:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"mode": "directory",
|
||||||
|
"permissions": 755,
|
||||||
|
"uid": 1000,
|
||||||
|
"gid": 1000
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (404 Not Found):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "Path not found"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/exists
|
||||||
|
|
||||||
|
Check if a path exists in the VFS.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/Movies/Action/Die Hard (1988)"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"exists": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/read
|
||||||
|
|
||||||
|
Get information for reading a file. Returns source path and optionally WebDAV URL.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/Movies/Action/Die Hard (1988)/Die Hard.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK) - Regular file:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"sourcePath": "/files/watch/downloads/Die.Hard.1988.1080p.mkv",
|
||||||
|
"size": 4831838208
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK) - File with WebDAV access:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"sourcePath": "/files/watch/downloads/Die.Hard.1988.1080p.mkv",
|
||||||
|
"size": 4831838208,
|
||||||
|
"webdavUrl": "http://meta-core/webdav/watch/downloads/Die.Hard.1988.1080p.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK) - Virtual file with content:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"sourcePath": null,
|
||||||
|
"size": 256,
|
||||||
|
"content": "VGhpcyBpcyBhIHZpcnR1YWwgZmlsZQ==",
|
||||||
|
"contentEncoding": "base64"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (404 Not Found):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "File not found"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/metadata
|
||||||
|
|
||||||
|
Get full metadata for a virtual path.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"path": "/Movies/Action/Die Hard (1988)/Die Hard.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"hashId": "bafkreiabc123...",
|
||||||
|
"filePath": "watch/downloads/Die.Hard.1988.1080p.mkv",
|
||||||
|
"title": "Die Hard",
|
||||||
|
"year": "1988",
|
||||||
|
"fileType": "video",
|
||||||
|
"size": "4831838208",
|
||||||
|
"extension": "mkv",
|
||||||
|
"video/codec": "h265",
|
||||||
|
"video/resolution": "1920x1080",
|
||||||
|
"audio/codec": "aac",
|
||||||
|
"audio/channels": "6"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (404 Not Found):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "Metadata not found"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /api/fuse/files
|
||||||
|
|
||||||
|
List all virtual files in the VFS.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"files": [
|
||||||
|
"/Movies/Action/Die Hard (1988)/Die Hard.mkv",
|
||||||
|
"/Movies/Action/Mad Max Fury Road (2015)/Mad Max Fury Road.mkv",
|
||||||
|
"/TV/Breaking Bad/Season 01/S01E01 - Pilot.mkv"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /api/fuse/directories
|
||||||
|
|
||||||
|
List all virtual directories in the VFS.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"directories": [
|
||||||
|
"/Movies",
|
||||||
|
"/Movies/Action",
|
||||||
|
"/Movies/Action/Die Hard (1988)",
|
||||||
|
"/TV",
|
||||||
|
"/TV/Breaking Bad",
|
||||||
|
"/TV/Breaking Bad/Season 01"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/refresh
|
||||||
|
|
||||||
|
Trigger a VFS refresh. Re-reads metadata and rebuilds virtual paths.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"status": "ok"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Renaming Rules
|
||||||
|
|
||||||
|
### GET /api/fuse/rules
|
||||||
|
|
||||||
|
Get the current renaming rules configuration.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"config": {
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"name": "Movies",
|
||||||
|
"enabled": true,
|
||||||
|
"priority": 100,
|
||||||
|
"template": "/Movies/{genre}/{title} ({year})/{fileName}",
|
||||||
|
"conditions": {
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" },
|
||||||
|
{ "field": "mediaType", "operator": "equals", "value": "movie" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "TV Shows",
|
||||||
|
"enabled": true,
|
||||||
|
"priority": 90,
|
||||||
|
"template": "/TV/{title}/Season {season:pad2}/{title} - S{season:pad2}E{episode:pad2}.{extension}",
|
||||||
|
"conditions": {
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" },
|
||||||
|
{ "field": "mediaType", "operator": "equals", "value": "tv" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"defaultRule": {
|
||||||
|
"name": "Default",
|
||||||
|
"template": "/Other/{fileName}",
|
||||||
|
"conditions": { "operator": "and", "conditions": [] }
|
||||||
|
},
|
||||||
|
"lastModified": "2024-01-15T10:30:00Z"
|
||||||
|
},
|
||||||
|
"lastModified": "2024-01-15T10:30:00Z"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### PUT /api/fuse/rules
|
||||||
|
|
||||||
|
Update the renaming rules configuration. Automatically triggers VFS refresh.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"config": {
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"name": "Movies",
|
||||||
|
"enabled": true,
|
||||||
|
"priority": 100,
|
||||||
|
"template": "/Movies/{title} ({year})/{fileName}",
|
||||||
|
"conditions": {
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (200 OK):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"success": true,
|
||||||
|
"refreshed": true
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (400 Bad Request) - Validation errors:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"success": false,
|
||||||
|
"errors": [
|
||||||
|
"Rule \"Movies\": Unknown variable 'unknownVar' in template",
|
||||||
|
"Rule \"TV Shows\": Invalid condition operator 'matches'"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/rules/preview
|
||||||
|
|
||||||
|
Preview how files would be renamed with the given rules.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"rules": [
|
||||||
|
{
|
||||||
|
"name": "Movies",
|
||||||
|
"template": "/New Movies/{title} ({year})/{fileName}",
|
||||||
|
"conditions": {
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"limit": 10
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"previews": [
|
||||||
|
{
|
||||||
|
"hashId": "bafkreiabc123...",
|
||||||
|
"currentPath": "/Movies/Action/Die Hard (1988)/Die Hard.mkv",
|
||||||
|
"newPath": "/New Movies/Die Hard (1988)/Die Hard.mkv",
|
||||||
|
"ruleName": "Movies"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"hashId": "bafkreixyz789...",
|
||||||
|
"currentPath": "/TV/Breaking Bad/Season 01/S01E01.mkv",
|
||||||
|
"newPath": null,
|
||||||
|
"ruleName": null,
|
||||||
|
"reason": "No matching rule"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"total": 5123,
|
||||||
|
"sampled": 10
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### POST /api/fuse/rules/validate
|
||||||
|
|
||||||
|
Validate a single renaming rule.
|
||||||
|
|
||||||
|
**Request Body**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"rule": {
|
||||||
|
"name": "Test Rule",
|
||||||
|
"template": "/Test/{title|originalTitle} ({year})/{fileName}",
|
||||||
|
"conditions": {
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
},
|
||||||
|
"sampleMetadata": {
|
||||||
|
"title": "Inception",
|
||||||
|
"originalTitle": "Inception",
|
||||||
|
"year": "2010",
|
||||||
|
"fileName": "Inception.mkv"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (valid rule):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"valid": true,
|
||||||
|
"errors": [],
|
||||||
|
"warnings": [],
|
||||||
|
"sampleOutput": "/Test/Inception (2010)/Inception.mkv"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Response** (invalid rule):
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"valid": false,
|
||||||
|
"errors": [
|
||||||
|
"Unknown variable 'unknownField' in template"
|
||||||
|
],
|
||||||
|
"warnings": [
|
||||||
|
"Variable 'director' is rarely available in metadata"
|
||||||
|
],
|
||||||
|
"sampleOutput": null
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### GET /api/fuse/rules/variables
|
||||||
|
|
||||||
|
Get list of available template variables.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"variables": [
|
||||||
|
{
|
||||||
|
"name": "title",
|
||||||
|
"description": "Media title",
|
||||||
|
"example": "Inception"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "originalTitle",
|
||||||
|
"description": "Original language title",
|
||||||
|
"example": "Inception"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "year",
|
||||||
|
"description": "Release year",
|
||||||
|
"example": "2010"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "season",
|
||||||
|
"description": "Season number (TV shows)",
|
||||||
|
"example": "1",
|
||||||
|
"modifiers": ["pad2", "pad3"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "episode",
|
||||||
|
"description": "Episode number (TV shows)",
|
||||||
|
"example": "5",
|
||||||
|
"modifiers": ["pad2", "pad3"]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "fileName",
|
||||||
|
"description": "Original filename",
|
||||||
|
"example": "Movie.2024.1080p.mkv"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "extension",
|
||||||
|
"description": "File extension",
|
||||||
|
"example": "mkv"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "fileType",
|
||||||
|
"description": "Type of file",
|
||||||
|
"example": "video"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "genre",
|
||||||
|
"description": "Primary genre",
|
||||||
|
"example": "Action"
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Service Discovery
|
||||||
|
|
||||||
|
### GET /api/services
|
||||||
|
|
||||||
|
List all discovered MetaMesh services.
|
||||||
|
|
||||||
|
**Response**:
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"services": [
|
||||||
|
{
|
||||||
|
"name": "meta-sort",
|
||||||
|
"url": "http://meta-sort-dev",
|
||||||
|
"api": "http://meta-sort-dev/api",
|
||||||
|
"status": "healthy",
|
||||||
|
"role": "writer"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "meta-fuse",
|
||||||
|
"url": "http://meta-fuse-dev",
|
||||||
|
"api": "http://meta-fuse-dev/api",
|
||||||
|
"status": "healthy",
|
||||||
|
"role": "reader"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"name": "meta-stremio",
|
||||||
|
"url": "http://meta-stremio-dev",
|
||||||
|
"api": "http://meta-stremio-dev/api",
|
||||||
|
"status": "healthy",
|
||||||
|
"role": "reader"
|
||||||
|
}
|
||||||
|
],
|
||||||
|
"current": "meta-fuse",
|
||||||
|
"leader": {
|
||||||
|
"hostname": "meta-core-dev",
|
||||||
|
"baseUrl": "http://localhost:8083",
|
||||||
|
"apiUrl": "http://meta-core:9000",
|
||||||
|
"redisUrl": "redis://meta-core:6379",
|
||||||
|
"webdavUrl": "http://localhost:8083/webdav"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Error Responses
|
||||||
|
|
||||||
|
All endpoints may return these error responses:
|
||||||
|
|
||||||
|
### 400 Bad Request
|
||||||
|
|
||||||
|
Missing or invalid parameters.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "Missing or invalid \"path\" parameter"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 404 Not Found
|
||||||
|
|
||||||
|
Resource not found.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "File not found"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 500 Internal Server Error
|
||||||
|
|
||||||
|
Server-side error.
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "Failed to process request"
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Template Syntax
|
||||||
|
|
||||||
|
### Basic Variables
|
||||||
|
|
||||||
|
```
|
||||||
|
{variableName} → Replaced with metadata value
|
||||||
|
{title} → "Inception"
|
||||||
|
{year} → "2010"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Variable with Fallback
|
||||||
|
|
||||||
|
```
|
||||||
|
{title|originalTitle} → Uses title, falls back to originalTitle
|
||||||
|
{genre|"Unknown"} → Uses genre, falls back to literal "Unknown"
|
||||||
|
```
|
||||||
|
|
||||||
|
### Variable Modifiers
|
||||||
|
|
||||||
|
```
|
||||||
|
{season:pad2} → "01" (zero-padded to 2 digits)
|
||||||
|
{episode:pad3} → "001" (zero-padded to 3 digits)
|
||||||
|
{title:lower} → "inception" (lowercase)
|
||||||
|
{title:upper} → "INCEPTION" (uppercase)
|
||||||
|
```
|
||||||
|
|
||||||
|
### Condition Operators
|
||||||
|
|
||||||
|
| Operator | Description | Example |
|
||||||
|
|----------|-------------|---------|
|
||||||
|
| `equals` | Exact match | `{ "field": "fileType", "operator": "equals", "value": "video" }` |
|
||||||
|
| `notEquals` | Not equal | `{ "field": "fileType", "operator": "notEquals", "value": "subtitle" }` |
|
||||||
|
| `contains` | Contains substring | `{ "field": "title", "operator": "contains", "value": "Star" }` |
|
||||||
|
| `startsWith` | Starts with | `{ "field": "fileName", "operator": "startsWith", "value": "[" }` |
|
||||||
|
| `endsWith` | Ends with | `{ "field": "extension", "operator": "endsWith", "value": "mkv" }` |
|
||||||
|
| `exists` | Field exists | `{ "field": "season", "operator": "exists" }` |
|
||||||
|
| `notExists` | Field missing | `{ "field": "season", "operator": "notExists" }` |
|
||||||
|
| `greaterThan` | Numeric > | `{ "field": "year", "operator": "greaterThan", "value": "2000" }` |
|
||||||
|
| `lessThan` | Numeric < | `{ "field": "season", "operator": "lessThan", "value": "10" }` |
|
||||||
|
|
||||||
|
### Condition Groups
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"operator": "and",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "fileType", "operator": "equals", "value": "video" },
|
||||||
|
{
|
||||||
|
"operator": "or",
|
||||||
|
"conditions": [
|
||||||
|
{ "field": "mediaType", "operator": "equals", "value": "movie" },
|
||||||
|
{ "field": "mediaType", "operator": "equals", "value": "tv" }
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Related Documentation
|
||||||
|
|
||||||
|
- [README](../README.md) - Service overview
|
||||||
|
- [VFS Rebuild Architecture](vfs-rebuild-architecture.md) - State management
|
||||||
|
- [Streaming Architecture](streaming-architecture.md) - Event processing
|
||||||
@@ -0,0 +1,455 @@
|
|||||||
|
# Streaming Architecture
|
||||||
|
|
||||||
|
## Overview
|
||||||
|
|
||||||
|
This document describes the streaming event processing system used by meta-fuse to maintain VFS state. The architecture uses Redis Streams for reliable, ordered event delivery with replay capability.
|
||||||
|
|
||||||
|
## Core Components
|
||||||
|
|
||||||
|
### StreamingStateBuilder
|
||||||
|
|
||||||
|
**File**: `src/vfs/StreamingStateBuilder.ts`
|
||||||
|
|
||||||
|
The StreamingStateBuilder is the central component for processing stream events and maintaining file metadata state.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface StreamingStateBuilderConfig {
|
||||||
|
/** Redis client for fetching property values */
|
||||||
|
redisClient: RedisClient;
|
||||||
|
/** Renaming rules config for extracting VFS-relevant properties */
|
||||||
|
rulesConfig: RenamingConfig;
|
||||||
|
/** VFS update callback */
|
||||||
|
vfsCallback?: VFSUpdateCallback;
|
||||||
|
/** FILES_VOLUME path prefix */
|
||||||
|
filesPath?: string;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Responsibilities**:
|
||||||
|
- Process events from `meta:events` stream
|
||||||
|
- Filter events by VFS-relevant properties
|
||||||
|
- Fetch property values from Redis
|
||||||
|
- Maintain in-memory state of all files
|
||||||
|
- Notify VFS when files are complete or change
|
||||||
|
|
||||||
|
### RulesPropertyExtractor
|
||||||
|
|
||||||
|
**File**: `src/vfs/RulesPropertyExtractor.ts`
|
||||||
|
|
||||||
|
Analyzes renaming rules configuration to determine which metadata properties are needed for VFS path computation.
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
class RulesPropertyExtractor {
|
||||||
|
// Extract all VFS-relevant properties from rules config
|
||||||
|
extractVfsProperties(config: RenamingConfig): Set<string>;
|
||||||
|
|
||||||
|
// Get core properties that are always required
|
||||||
|
getCoreProperties(): Set<string>;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Core Properties** (always required):
|
||||||
|
- `filePath` - Source file location
|
||||||
|
- `size`, `fileSize` - File size
|
||||||
|
- `mtime`, `ctime` - Timestamps
|
||||||
|
- `fileName`, `extension` - Filename components
|
||||||
|
|
||||||
|
**Rule-Derived Properties** (extracted from templates and conditions):
|
||||||
|
- Template variables: `{title}`, `{season:pad2}`, `{episode:pad2}`
|
||||||
|
- Condition fields: `fileType`, `year`, `mediaType`
|
||||||
|
|
||||||
|
### VFSUpdateCallback
|
||||||
|
|
||||||
|
**File**: `src/vfs/StreamingStateBuilder.ts:32-37`
|
||||||
|
|
||||||
|
Interface for VFS to receive updates from StreamingStateBuilder:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface VFSUpdateCallback {
|
||||||
|
// Called when a property value changes
|
||||||
|
onPropertyChange(hashId: string, property: string, value: string): void;
|
||||||
|
|
||||||
|
// Called when a property is deleted
|
||||||
|
onPropertyDelete(hashId: string, property: string): void;
|
||||||
|
|
||||||
|
// Called when a file is completely removed (filePath deleted)
|
||||||
|
onFileDelete(hashId: string): void;
|
||||||
|
|
||||||
|
// Called when a file becomes complete (has filePath) or updates
|
||||||
|
onFileComplete(hashId: string, metadata: Record<string, string>): void;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Event Processing Pipeline
|
||||||
|
|
||||||
|
### Stream Message Format
|
||||||
|
|
||||||
|
**File**: `src/kv/RedisClient.ts:30-43`
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
interface StreamMessage {
|
||||||
|
id: string; // Stream entry ID (e.g., "1703808000000-0")
|
||||||
|
type: 'set' | 'del'; // Operation type
|
||||||
|
key?: string; // Redis key (e.g., "file:abc123/title")
|
||||||
|
ts?: string; // Timestamp
|
||||||
|
timestamp: string; // Fallback timestamp
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### 5-Step Event Processing
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Step 1: Receive Stream Message │
|
||||||
|
│ ────────────────────────────────────────────────────────────── │
|
||||||
|
│ {id: "1703808000-0", type: "set", key: "file:abc123/title"} │
|
||||||
|
└───────────────────────────────┬─────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Step 2: Parse Key │
|
||||||
|
│ ────────────────────────────────────────────────────────────── │
|
||||||
|
│ parseMetaEventKey("file:abc123/title") │
|
||||||
|
│ → { hashId: "abc123", property: "title" } │
|
||||||
|
└───────────────────────────────┬─────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Step 3: Check VFS Relevance │
|
||||||
|
│ ────────────────────────────────────────────────────────────── │
|
||||||
|
│ isVfsRelevantProperty("title", vfsRelevantProps) │
|
||||||
|
│ → true (if "title" is in renaming rules) │
|
||||||
|
│ → false → skip, increment propertiesSkipped │
|
||||||
|
└───────────────────────────────┬─────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Step 4: Fetch Property Value │
|
||||||
|
│ ────────────────────────────────────────────────────────────── │
|
||||||
|
│ await redisClient.getProperty("abc123", "title") │
|
||||||
|
│ → "Inception" │
|
||||||
|
└───────────────────────────────┬─────────────────────────────────┘
|
||||||
|
│
|
||||||
|
▼
|
||||||
|
┌─────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Step 5: Update State & Notify VFS │
|
||||||
|
│ ────────────────────────────────────────────────────────────── │
|
||||||
|
│ updateProperty("abc123", "title", "Inception") │
|
||||||
|
│ vfsCallback.onPropertyChange("abc123", "title", "Inception") │
|
||||||
|
│ │
|
||||||
|
│ If file just became complete (now has filePath): │
|
||||||
|
│ vfsCallback.onFileComplete("abc123", metadata) │
|
||||||
|
└─────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Code Implementation
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// StreamingStateBuilder.ts:163-206
|
||||||
|
private async handlePropertySet(hashId: string, property: string): Promise<void> {
|
||||||
|
// Step 3: Check if this property is VFS-relevant
|
||||||
|
if (!isVfsRelevantProperty(property, this.vfsRelevantProps)) {
|
||||||
|
this.stats.propertiesSkipped++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Step 4: Fetch the property value from Redis
|
||||||
|
const value = await this.redisClient.getProperty(hashId, property);
|
||||||
|
if (value === null) {
|
||||||
|
return; // Property was set then deleted before we could fetch
|
||||||
|
}
|
||||||
|
|
||||||
|
this.stats.propertiesFetched++;
|
||||||
|
|
||||||
|
// Check if file was complete BEFORE updating state
|
||||||
|
const wasComplete = this.isFileComplete(hashId);
|
||||||
|
|
||||||
|
// Step 5: Update internal state
|
||||||
|
this.updateProperty(hashId, property, value);
|
||||||
|
|
||||||
|
// Notify VFS of changes
|
||||||
|
if (this.vfsCallback) {
|
||||||
|
this.vfsCallback.onPropertyChange(hashId, property, value);
|
||||||
|
|
||||||
|
// If file just became complete (has filePath now), notify VFS
|
||||||
|
if (!wasComplete && this.isFileComplete(hashId)) {
|
||||||
|
this.stats.filesComplete++;
|
||||||
|
const metadata = this.getFileMetadata(hashId);
|
||||||
|
if (metadata) {
|
||||||
|
this.vfsCallback.onFileComplete(hashId, metadata);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Bootstrap vs Live Phases
|
||||||
|
|
||||||
|
### Bootstrap Phase
|
||||||
|
|
||||||
|
**When**: Application startup
|
||||||
|
|
||||||
|
**Goal**: Rebuild complete VFS state from event history
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// index.ts:158-174
|
||||||
|
// Bootstrap: Replay meta:events stream from position 0
|
||||||
|
const lastId = await redisClient.replayStream(
|
||||||
|
EVENTS_STREAM,
|
||||||
|
'0', // Start from beginning
|
||||||
|
async (message: StreamMessage) => {
|
||||||
|
await stateBuilder.processEvent(message);
|
||||||
|
},
|
||||||
|
100 // Batch size
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Characteristics**:
|
||||||
|
- Reads from stream position `0` (all history)
|
||||||
|
- Non-blocking batch reads (no XREADGROUP)
|
||||||
|
- Processes all events sequentially
|
||||||
|
- Returns last processed event ID
|
||||||
|
|
||||||
|
### Live Phase
|
||||||
|
|
||||||
|
**When**: After bootstrap completes
|
||||||
|
|
||||||
|
**Goal**: Process new events in real-time
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// index.ts:177-187
|
||||||
|
redisClient.startSimpleStreamConsumer(
|
||||||
|
EVENTS_STREAM,
|
||||||
|
lastId, // Resume from bootstrap position
|
||||||
|
async (message: StreamMessage) => {
|
||||||
|
await stateBuilder.processEvent(message);
|
||||||
|
},
|
||||||
|
5000 // 5 second block timeout
|
||||||
|
);
|
||||||
|
```
|
||||||
|
|
||||||
|
**Characteristics**:
|
||||||
|
- Resumes from last bootstrap position
|
||||||
|
- Uses blocking reads (XREAD BLOCK)
|
||||||
|
- Processes events as they arrive
|
||||||
|
- Runs until shutdown
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Memory Efficiency
|
||||||
|
|
||||||
|
### State Structure
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
type FileState = Map<string, string>; // property → value
|
||||||
|
type FilesState = Map<string, FileState>; // hashId → properties
|
||||||
|
```
|
||||||
|
|
||||||
|
### Memory Per File
|
||||||
|
|
||||||
|
| Component | Size |
|
||||||
|
|-----------|------|
|
||||||
|
| hashId (64 char) | ~128 bytes |
|
||||||
|
| Map overhead | ~56 bytes |
|
||||||
|
| 5-10 properties (avg) | ~200-400 bytes |
|
||||||
|
| **Total per file** | **~400-600 bytes** |
|
||||||
|
|
||||||
|
### Comparison with Full Load
|
||||||
|
|
||||||
|
| Approach | Memory per File | 10K Files |
|
||||||
|
|----------|-----------------|-----------|
|
||||||
|
| Full HGETALL (all props) | ~2-5 KB | 20-50 MB |
|
||||||
|
| Streaming (VFS props only) | ~500 bytes | 5 MB |
|
||||||
|
| **Savings** | **75-90%** | **15-45 MB** |
|
||||||
|
|
||||||
|
### Why It Works
|
||||||
|
|
||||||
|
1. **Property Filtering**: Only fetch properties used in renaming rules
|
||||||
|
2. **Lazy Loading**: Properties fetched on-demand from events
|
||||||
|
3. **No Duplication**: Values stored once in state map
|
||||||
|
4. **Garbage Collection**: File state removed when file deleted
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Property Filtering Details
|
||||||
|
|
||||||
|
### Extracting VFS-Relevant Properties
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// RulesPropertyExtractor.ts:132-155
|
||||||
|
extractVfsProperties(config: RenamingConfig): Set<string> {
|
||||||
|
const properties = new Set<string>(CORE_PROPERTIES);
|
||||||
|
|
||||||
|
// Extract from all enabled rules
|
||||||
|
for (const rule of config.rules) {
|
||||||
|
if (!rule.enabled) continue;
|
||||||
|
|
||||||
|
// From template: {title|originalTitle}, {season:pad2}
|
||||||
|
const templateVars = templateEngine.extractVariables(rule.template);
|
||||||
|
properties.add(...templateVars);
|
||||||
|
|
||||||
|
// From conditions: { field: 'fileType', value: 'video' }
|
||||||
|
const conditionFields = extractConditionFields(rule.conditions);
|
||||||
|
properties.add(...conditionFields);
|
||||||
|
}
|
||||||
|
|
||||||
|
return properties;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Property Path Matching
|
||||||
|
|
||||||
|
Handles both exact and nested property matching:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// RulesPropertyExtractor.ts:85-111
|
||||||
|
function isVfsRelevantProperty(property: string, vfsProperties: Set<string>): boolean {
|
||||||
|
const normalized = normalizePropertyPath(property);
|
||||||
|
|
||||||
|
// Direct match
|
||||||
|
if (vfsProperties.has(normalized)) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Nested property match
|
||||||
|
// e.g., if we track 'titles', then 'titles/eng' matches
|
||||||
|
for (const vfsProp of vfsProperties) {
|
||||||
|
if (normalized.startsWith(vfsProp + '.')) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
if (vfsProp.startsWith(normalized + '.')) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Statistics and Monitoring
|
||||||
|
|
||||||
|
### StreamingStateBuilder Stats
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// StreamingStateBuilder.ts:68-74
|
||||||
|
private stats = {
|
||||||
|
eventsProcessed: 0, // Total events received
|
||||||
|
propertiesFetched: 0, // Properties fetched from Redis
|
||||||
|
propertiesSkipped: 0, // Non-VFS-relevant properties skipped
|
||||||
|
filesComplete: 0, // Files that became complete
|
||||||
|
lastEventId: '0', // Last processed event ID
|
||||||
|
};
|
||||||
|
```
|
||||||
|
|
||||||
|
### Accessing Stats
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const stats = stateBuilder.getStats();
|
||||||
|
// Returns:
|
||||||
|
{
|
||||||
|
eventsProcessed: 15234,
|
||||||
|
propertiesFetched: 8934,
|
||||||
|
propertiesSkipped: 6300,
|
||||||
|
filesComplete: 5123,
|
||||||
|
lastEventId: "1703808000000-0",
|
||||||
|
fileCount: 5123, // Complete files (have filePath)
|
||||||
|
stateSize: 5234 // Total files in state
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Interpreting Stats
|
||||||
|
|
||||||
|
| Metric | Meaning |
|
||||||
|
|--------|---------|
|
||||||
|
| `eventsProcessed` | Total stream events consumed |
|
||||||
|
| `propertiesFetched` | VFS-relevant property GET calls |
|
||||||
|
| `propertiesSkipped` | Non-relevant events filtered |
|
||||||
|
| `filesComplete` | Files added to VFS |
|
||||||
|
| `fileCount` | Current files in VFS |
|
||||||
|
| Fetch ratio | `propertiesFetched / eventsProcessed` (lower is better) |
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Error Handling
|
||||||
|
|
||||||
|
### Event Processing Errors
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// StreamingStateBuilder.ts:123-136
|
||||||
|
async processEvent(message: StreamMessage): Promise<void> {
|
||||||
|
this.stats.eventsProcessed++;
|
||||||
|
this.stats.lastEventId = message.id;
|
||||||
|
|
||||||
|
if ((message.type === 'set' || message.type === 'del') && message.key) {
|
||||||
|
await this.processMetaEvent(message);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ignore legacy event types - not supported in flat key architecture
|
||||||
|
logger.debug(`Ignoring legacy event type: ${message.type}`);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Property Fetch Failures
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// StreamingStateBuilder.ts:171-177
|
||||||
|
const value = await this.redisClient.getProperty(hashId, property);
|
||||||
|
if (value === null) {
|
||||||
|
// Property was set then deleted before we could fetch it
|
||||||
|
// Or it's a different data type - skip silently
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### Stream Consumer Recovery
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// index.ts:185-187
|
||||||
|
.catch(error => {
|
||||||
|
logger.error('Stream consumer error:', error);
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
On error, the consumer loop continues retrying with brief pauses.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Configuration
|
||||||
|
|
||||||
|
### StreamingStateBuilder Configuration
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
const stateBuilder = new StreamingStateBuilder({
|
||||||
|
redisClient,
|
||||||
|
rulesConfig: vfs.getRulesConfig(),
|
||||||
|
vfsCallback: vfs,
|
||||||
|
filesPath: '/files',
|
||||||
|
});
|
||||||
|
```
|
||||||
|
|
||||||
|
### Dynamic Rules Update
|
||||||
|
|
||||||
|
When renaming rules change:
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// StreamingStateBuilder.ts:99-102
|
||||||
|
updateRulesConfig(rulesConfig: RenamingConfig): void {
|
||||||
|
this.vfsRelevantProps = this.propertyExtractor.extractVfsProperties(rulesConfig);
|
||||||
|
logger.info(`Updated VFS-relevant properties: ${this.vfsRelevantProps.size}`);
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Related Documentation
|
||||||
|
|
||||||
|
- [VFS Rebuild Architecture](vfs-rebuild-architecture.md) - High-level architecture overview
|
||||||
|
- [API Reference](api-reference.md) - REST API documentation
|
||||||
|
- [Redis Key Format](../../../meta-core/docs/redis-format.md) - Flat key storage format
|
||||||
+310
-426
@@ -2,7 +2,7 @@
|
|||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
This document describes the architecture and design philosophy for the Virtual Filesystem (VFS) rebuild process in MetaMesh. The system uses a **two-phase strategy** to provide fast startup times while maintaining data accuracy through continuous validation.
|
This document describes the architecture and design philosophy for the Virtual Filesystem (VFS) rebuild process in MetaMesh. The system uses a **streaming-based strategy** with Redis Streams to provide fast startup times while maintaining data accuracy through event-driven updates.
|
||||||
|
|
||||||
## Design Philosophy
|
## Design Philosophy
|
||||||
|
|
||||||
@@ -10,276 +10,242 @@ The VFS rebuild architecture is built on three core principles:
|
|||||||
|
|
||||||
| Component | Role | Responsibility |
|
| Component | Role | Responsibility |
|
||||||
|-----------|------|----------------|
|
|-----------|------|----------------|
|
||||||
| **etcd** | Distributed Metadata Archive | Permanent record of all files (even if deleted from disk) |
|
| **Redis** | Metadata Storage | Flat key storage of all file properties |
|
||||||
| **In-Memory Database** | Current VFS State | Only files present on disk RIGHT NOW |
|
| **Redis Streams** | Event Log | Ordered record of all metadata changes (`meta:events`) |
|
||||||
| **Discovery** | Source of Truth | Validates file existence and cleans stale entries |
|
| **StreamingStateBuilder** | State Manager | Processes events, maintains VFS-relevant state in memory |
|
||||||
|
|
||||||
### Key Insight
|
### Key Insight
|
||||||
|
|
||||||
> **etcd stores the historical truth** (what files have existed)
|
> **Redis stores the current truth** (what file properties exist)
|
||||||
> **In-memory DB stores the current truth** (what files exist NOW)
|
> **Redis Streams store the change log** (what changed and when)
|
||||||
> **Discovery enforces reality** (makes memory match disk state)
|
> **StreamingStateBuilder filters and projects** (extracts only VFS-relevant properties)
|
||||||
|
|
||||||
This separation enables:
|
This separation enables:
|
||||||
- **Fast cold starts** (1-2 seconds for 10,000 files)
|
- **Fast cold starts** (1-2 seconds for 10,000 files via stream replay)
|
||||||
- **Distributed metadata sharing** via meta-orbit (etcd never deletes)
|
- **Memory efficiency** (~500 bytes/file, only VFS-relevant properties)
|
||||||
- **Accurate VFS state** (discovery continuously validates)
|
- **Real-time updates** (live stream consumption after bootstrap)
|
||||||
- **Graceful handling of offline storage** (network drives, removable media)
|
- **Graceful handling of offline storage** (state rebuilds on reconnection)
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Two-Phase Strategy
|
## Streaming Architecture
|
||||||
|
|
||||||
### Phase 1: Fast Bootstrap (Startup)
|
### Phase 1: Streaming Bootstrap (Startup)
|
||||||
|
|
||||||
**Goal**: Make VFS available as quickly as possible
|
**Goal**: Rebuild VFS state as quickly as possible from event stream
|
||||||
|
|
||||||
**Implementation**: `WatchedFileProcessor.rebuildVFSFromEtcd()`
|
**Implementation**: See `index.ts:158-174`
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
async rebuildVFSFromEtcd(): Promise<void> {
|
// Bootstrap: Replay meta:events stream from position 0
|
||||||
// 1. Fetch all hash IDs from etcd
|
const lastId = await redisClient.replayStream(
|
||||||
const hashIds = await etcdClient.getAllHashIds();
|
EVENTS_STREAM, // 'meta:events'
|
||||||
|
'0', // Start from beginning
|
||||||
|
async (message: StreamMessage) => {
|
||||||
|
await stateBuilder.processEvent(message);
|
||||||
|
},
|
||||||
|
100 // Batch size
|
||||||
|
);
|
||||||
|
|
||||||
// 2. Batch fetch ALL metadata in parallel (no disk I/O)
|
const stats = vfs.getStats();
|
||||||
const metadataList = await Promise.all(
|
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files`);
|
||||||
hashIds.map(hashId => etcdClient.getMetadataFlat(hashId))
|
|
||||||
);
|
|
||||||
|
|
||||||
// 3. Load into in-memory DB without validation
|
|
||||||
for (const metadata of metadataList) {
|
|
||||||
metadata._lastVerified = 0; // Mark as unverified
|
|
||||||
this.fileProcessor.getDatabase().set(metadata.filePath, metadata);
|
|
||||||
}
|
|
||||||
|
|
||||||
// 4. Build VFS immediately (1-2 seconds)
|
|
||||||
await this.finalize();
|
|
||||||
}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
|
**How It Works**:
|
||||||
|
1. Read stream entries in batches using `XREAD`
|
||||||
|
2. For each `set` event, check if property is VFS-relevant
|
||||||
|
3. If relevant, fetch property value with `GET file:{hashId}/{property}`
|
||||||
|
4. Update in-memory state and notify VFS when file has `filePath`
|
||||||
|
|
||||||
**Performance**:
|
**Performance**:
|
||||||
- 10,000 files: ~1-2 seconds (previously 60+ seconds)
|
- 10,000 files: ~1-2 seconds
|
||||||
- Zero disk I/O during startup
|
- Zero SCAN/HGETALL operations
|
||||||
- VFS immediately usable (may contain stale entries)
|
- Only fetches properties needed for VFS path computation
|
||||||
|
|
||||||
**Characteristics**:
|
**Characteristics**:
|
||||||
- ✅ Fast startup
|
- Stream replay from position 0
|
||||||
- ✅ No blocking on disk validation
|
- Incremental state building
|
||||||
- ✅ All files appear immediately
|
- VFS populated as files become "complete" (have `filePath`)
|
||||||
- ⚠️ May include deleted/offline files temporarily
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
### Phase 2: Continuous Validation (Discovery)
|
### Phase 2: Live Stream Consumption
|
||||||
|
|
||||||
**Goal**: Make in-memory DB match current disk reality
|
**Goal**: Keep VFS synchronized with metadata changes in real-time
|
||||||
|
|
||||||
**Implementation**: Discovery scans folders progressively and validates file existence
|
**Implementation**: See `index.ts:177-187`
|
||||||
|
|
||||||
```typescript
|
```typescript
|
||||||
// Progressive scan: Process folders one at a time
|
// Start live stream consumer from where replay left off
|
||||||
for (const folder of folderList) {
|
redisClient.startSimpleStreamConsumer(
|
||||||
const discoveredFilesInFolder = new Set<string>();
|
EVENTS_STREAM,
|
||||||
|
lastId, // Resume from bootstrap position
|
||||||
// Track discovered files for this folder
|
async (message: StreamMessage) => {
|
||||||
async function* trackDiscoveredFiles(stream: AsyncGenerator<string>) {
|
await stateBuilder.processEvent(message);
|
||||||
for await (const filePath of stream) {
|
|
||||||
discoveredFilesInFolder.add(filePath); // Track what exists
|
|
||||||
yield filePath;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
const discoveryStream = folderWatcher.discoverFiles([folder]);
|
|
||||||
await pipeline.start(trackDiscoveredFiles(discoveryStream));
|
|
||||||
|
|
||||||
// Progressive cleanup: Remove stale entries after each folder
|
|
||||||
await fileProcessor.cleanupStaleEntries([folder], discoveredFilesInFolder);
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Cleanup Logic**: `WatchedFileProcessor.cleanupStaleEntries()`
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
async cleanupStaleEntries(scannedFolders: string[], discoveredFiles: Set<string>) {
|
|
||||||
for (const folder of scannedFolders) {
|
|
||||||
// Find all in-memory files for this folder
|
|
||||||
const filesToCheck = db.getAllFilesStartingWith(folder);
|
|
||||||
|
|
||||||
// Remove files not seen during scan
|
|
||||||
for (const filePath of filesToCheck) {
|
|
||||||
if (!discoveredFiles.has(filePath)) {
|
|
||||||
// File no longer exists → remove from memory
|
|
||||||
db.delete(filePath);
|
|
||||||
vfs.removeFileByRealPath(filePath);
|
|
||||||
stateManager.removeFile(filePath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rebuild VFS if anything changed
|
|
||||||
await this.finalize();
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**Validation Rules** (applied progressively per folder):
|
|
||||||
|
|
||||||
| Scenario | Action | VFS Update |
|
|
||||||
|----------|--------|------------|
|
|
||||||
| File found during scan | Update `_lastVerified` timestamp, keep in memory | No change |
|
|
||||||
| File not found in scanned folder | Remove from in-memory DB, keep in etcd | VFS rebuilt incrementally |
|
|
||||||
| Empty folder scanned | Remove ALL child paths from memory immediately | VFS rebuilt (files disappear) |
|
|
||||||
| Folder missing/inaccessible | Remove ALL child paths from memory immediately | VFS rebuilt (files disappear) |
|
|
||||||
|
|
||||||
**Key Behavior**: When a NAS/network drive is disconnected (folder becomes empty/inaccessible), all files from that folder are **progressively removed from the VFS as discovery proceeds** through each folder.
|
|
||||||
|
|
||||||
**Characteristics**:
|
|
||||||
- ✅ Discovery is authoritative
|
|
||||||
- ✅ Progressive cleanup (files disappear as folders are scanned)
|
|
||||||
- ✅ Handles deleted files automatically
|
|
||||||
- ✅ Handles offline network drives (files removed immediately when folder is scanned)
|
|
||||||
- ✅ No manual TTL/expiry needed
|
|
||||||
- ✅ Real-time VFS updates during discovery
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Architecture Comparison
|
|
||||||
|
|
||||||
### Old Approach (Disk Validation at Startup)
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────────────────────┐
|
|
||||||
│ Startup: rebuildVFSFromEtcd() │
|
|
||||||
│ │
|
|
||||||
│ For each file in etcd (sequential): │
|
|
||||||
│ 1. Fetch metadata from etcd (1-5ms) │
|
|
||||||
│ 2. Check if file exists (fs.access) │
|
|
||||||
│ - Local disk: 1-10ms │
|
|
||||||
│ - Network drive: 10-100ms │
|
|
||||||
│ 3. Add to memory if exists │
|
|
||||||
│ │
|
|
||||||
│ Result: 60-120 seconds for 10k files │
|
|
||||||
└─────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
**Problems**:
|
|
||||||
- ❌ Slow startup (blocks on disk I/O)
|
|
||||||
- ❌ Sequential processing (no parallelization)
|
|
||||||
- ❌ Network latency multiplies wait time
|
|
||||||
- ❌ VFS unavailable during validation
|
|
||||||
|
|
||||||
### New Approach (Fast Load + Progressive Discovery)
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────────────────────┐
|
|
||||||
│ Startup: rebuildVFSFromEtcd() │
|
|
||||||
│ │
|
|
||||||
│ 1. Batch fetch ALL metadata (parallel) │
|
|
||||||
│ → 10k files in ~500ms │
|
|
||||||
│ │
|
|
||||||
│ 2. Load into memory (no disk checks) │
|
|
||||||
│ → 10k files in ~200ms │
|
|
||||||
│ │
|
|
||||||
│ 3. Build VFS │
|
|
||||||
│ → Ready in 1-2 seconds total │
|
|
||||||
│ │
|
|
||||||
│ Result: VFS immediately usable │
|
|
||||||
└─────────────────────────────────────────────┘
|
|
||||||
|
|
||||||
┌─────────────────────────────────────────────┐
|
|
||||||
│ Background: Progressive Discovery Scan │
|
|
||||||
│ │
|
|
||||||
│ For each folder in watch list: │
|
|
||||||
│ 1. Walk folder and track discovered files│
|
|
||||||
│ 2. Update _lastVerified for found files │
|
|
||||||
│ 3. Remove stale entries for this folder │
|
|
||||||
│ 4. Rebuild VFS (files disappear) │
|
|
||||||
│ 5. Move to next folder │
|
|
||||||
│ │
|
|
||||||
│ Result: Files progressively disappear as │
|
|
||||||
│ folders are scanned (real-time) │
|
|
||||||
└─────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
**Benefits**:
|
|
||||||
- ✅ 10-50x faster startup
|
|
||||||
- ✅ Parallel etcd fetching
|
|
||||||
- ✅ No disk I/O blocking
|
|
||||||
- ✅ VFS immediately available
|
|
||||||
- ✅ Progressive cleanup (real-time feedback)
|
|
||||||
- ✅ Files disappear as folders are scanned
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Implementation Details
|
|
||||||
|
|
||||||
### File Metadata Structure
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
interface FileMetadata {
|
|
||||||
filePath: string;
|
|
||||||
cid_midhash256: string; // Permanent file ID
|
|
||||||
title: string;
|
|
||||||
// ... other metadata fields
|
|
||||||
|
|
||||||
// Validation tracking (optional)
|
|
||||||
_lastVerified?: number; // Timestamp of last disk validation
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
The `_lastVerified` field:
|
|
||||||
- Set to `0` during fast load (unverified)
|
|
||||||
- Updated to `Date.now()` when discovery sees the file
|
|
||||||
- Used for debugging and monitoring (not enforced)
|
|
||||||
|
|
||||||
### State Management
|
|
||||||
|
|
||||||
**In-Memory Database** (`FileProcessor.database`):
|
|
||||||
- `Map<filePath, metadata>`
|
|
||||||
- Cleared on restart
|
|
||||||
- Reflects only currently accessible files
|
|
||||||
|
|
||||||
**etcd Storage**:
|
|
||||||
- `/file/midhash256:{hash}/property` - Permanent metadata
|
|
||||||
- Never deleted (even if file removed from disk)
|
|
||||||
- Shared across meta-orbit network
|
|
||||||
|
|
||||||
**Virtual Filesystem** (`VirtualFileSystem`):
|
|
||||||
- Generated from in-memory database
|
|
||||||
- Rebuilt after cleanup operations
|
|
||||||
- Accessible via FUSE/WebDAV
|
|
||||||
|
|
||||||
### Discovery Integration
|
|
||||||
|
|
||||||
Discovery runs in two modes:
|
|
||||||
|
|
||||||
**1. Manual Scan** (startup or triggered via API) - Progressive:
|
|
||||||
```typescript
|
|
||||||
// Process folders one at a time
|
|
||||||
for (const folder of folderList) {
|
|
||||||
const discoveredFiles = new Set<string>();
|
|
||||||
const discoveryStream = folderWatcher.discoverFiles([folder]);
|
|
||||||
|
|
||||||
// Track and process files for this folder
|
|
||||||
await pipeline.start(trackFiles(discoveryStream, discoveredFiles));
|
|
||||||
|
|
||||||
// Progressive cleanup: Remove stale entries immediately
|
|
||||||
await fileProcessor.cleanupStaleEntries([folder], discoveredFiles);
|
|
||||||
// → VFS rebuilt, files from disconnected NAS disappear
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
**2. Watch Mode** (continuous monitoring):
|
|
||||||
```typescript
|
|
||||||
folderWatcher.watch(folderList, {
|
|
||||||
onAdd: (filePath) => {
|
|
||||||
// File added → process and add to memory
|
|
||||||
pipeline.processFile(filePath);
|
|
||||||
},
|
},
|
||||||
onUnlink: (filePath) => {
|
5000 // 5 second block timeout
|
||||||
// File removed → remove from memory immediately
|
);
|
||||||
fileProcessor.deleteFile(filePath);
|
```
|
||||||
|
|
||||||
|
**Event Processing** (see `StreamingStateBuilder.ts:123-136`):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
async processEvent(message: StreamMessage): Promise<void> {
|
||||||
|
// Handle new meta:events format (set/del with key field)
|
||||||
|
if ((message.type === 'set' || message.type === 'del') && message.key) {
|
||||||
|
await this.processMetaEvent(message);
|
||||||
|
return;
|
||||||
}
|
}
|
||||||
});
|
// Legacy events ignored
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
**Property Filtering** (see `RulesPropertyExtractor.ts`):
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// Only fetch VFS-relevant properties
|
||||||
|
if (!isVfsRelevantProperty(property, this.vfsRelevantProps)) {
|
||||||
|
this.stats.propertiesSkipped++;
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fetch and update state
|
||||||
|
const value = await this.redisClient.getProperty(hashId, property);
|
||||||
|
this.updateProperty(hashId, property, value);
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Architecture Diagrams
|
||||||
|
|
||||||
|
### Startup Sequence
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Streaming Bootstrap │
|
||||||
|
├─────────────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ 1. Initialize KVManager (leader discovery) │
|
||||||
|
│ ├── Read /meta-core/locks/kv-leader.info │
|
||||||
|
│ ├── Call /urls API for Redis URL │
|
||||||
|
│ └── Connect to Redis │
|
||||||
|
│ │
|
||||||
|
│ 2. Create StreamingStateBuilder │
|
||||||
|
│ ├── Extract VFS-relevant properties from rules │
|
||||||
|
│ └── Wire up VFS callback │
|
||||||
|
│ │
|
||||||
|
│ 3. Replay meta:events stream from position 0 │
|
||||||
|
│ ┌────────────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ For each event in stream: │ │
|
||||||
|
│ │ ├── Parse key: file:{hashId}/{property} │ │
|
||||||
|
│ │ ├── Check if property is VFS-relevant │ │
|
||||||
|
│ │ ├── If yes: GET property value │ │
|
||||||
|
│ │ ├── Update in-memory state │ │
|
||||||
|
│ │ └── If file has filePath: notify VFS (file appears) │ │
|
||||||
|
│ └────────────────────────────────────────────────────────────┘ │
|
||||||
|
│ │
|
||||||
|
│ 4. Start live consumer from last processed ID │
|
||||||
|
│ └── Continue processing new events in real-time │
|
||||||
|
│ │
|
||||||
|
│ 5. Start API server │
|
||||||
|
│ └── VFS ready for FUSE/WebDAV operations │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
### Event Processing Pipeline
|
||||||
|
|
||||||
|
```
|
||||||
|
┌─────────────────────────────────────────────────────────────────────────┐
|
||||||
|
│ Event Processing Pipeline │
|
||||||
|
├─────────────────────────────────────────────────────────────────────────┤
|
||||||
|
│ │
|
||||||
|
│ Redis Stream (meta:events) │
|
||||||
|
│ ┌──────────────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ {id: "1234-0", type: "set", key: "file:abc123/title"} │ │
|
||||||
|
│ │ {id: "1234-1", type: "set", key: "file:abc123/filePath"} │ │
|
||||||
|
│ │ {id: "1234-2", type: "del", key: "file:xyz789/filePath"} │ │
|
||||||
|
│ └───────────────────────────────┬──────────────────────────────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ ▼ │
|
||||||
|
│ ┌─────────────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ StreamingStateBuilder.processEvent() │ │
|
||||||
|
│ │ 1. Parse key → {hashId: "abc123", property: "title"} │ │
|
||||||
|
│ │ 2. Check VFS relevance (from RulesPropertyExtractor) │ │
|
||||||
|
│ │ 3. If relevant: GET file:abc123/title → "Inception" │ │
|
||||||
|
│ │ 4. Update state: state.get("abc123").set("title", ...) │ │
|
||||||
|
│ └─────────────────────────────────────────────────────────────┘ │
|
||||||
|
│ │ │
|
||||||
|
│ ▼ │
|
||||||
|
│ ┌─────────────────────────────────────────────────────────────┐ │
|
||||||
|
│ │ VFSUpdateCallback │ │
|
||||||
|
│ │ - onPropertyChange(hashId, property, value) │ │
|
||||||
|
│ │ - onFileComplete(hashId, metadata) → file appears in VFS │ │
|
||||||
|
│ │ - onFileDelete(hashId) → file removed from VFS │ │
|
||||||
|
│ └─────────────────────────────────────────────────────────────┘ │
|
||||||
|
│ │
|
||||||
|
└─────────────────────────────────────────────────────────────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## State Management
|
||||||
|
|
||||||
|
### StreamingStateBuilder State
|
||||||
|
|
||||||
|
```typescript
|
||||||
|
// FilesState: Map of hashId to property map
|
||||||
|
type FileState = Map<string, string>; // property → value
|
||||||
|
type FilesState = Map<string, FileState>; // hashId → properties
|
||||||
|
|
||||||
|
// Example state
|
||||||
|
{
|
||||||
|
"abc123": {
|
||||||
|
"filePath": "watch/Movies/Inception.mkv",
|
||||||
|
"title": "Inception",
|
||||||
|
"year": "2010",
|
||||||
|
"size": "4831838208"
|
||||||
|
},
|
||||||
|
"xyz789": {
|
||||||
|
"filePath": "watch/TV/Breaking Bad/S01E01.mkv",
|
||||||
|
"title": "Breaking Bad",
|
||||||
|
"season": "1",
|
||||||
|
"episode": "1"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
### VFS-Relevant Properties
|
||||||
|
|
||||||
|
Properties are extracted from renaming rules (see `RulesPropertyExtractor.ts`):
|
||||||
|
|
||||||
|
**Core Properties** (always required):
|
||||||
|
- `filePath` - Source file location (required for file to appear)
|
||||||
|
- `size`, `fileSize` - File size
|
||||||
|
- `mtime`, `ctime` - Timestamps
|
||||||
|
- `fileName`, `extension` - Filename components
|
||||||
|
|
||||||
|
**Rule-Derived Properties**:
|
||||||
|
- From templates: `{title}`, `{season:pad2}`, `{episode:pad2}`
|
||||||
|
- From conditions: `{ field: 'fileType', value: 'video' }`
|
||||||
|
|
||||||
|
### Redis Data Structure
|
||||||
|
|
||||||
|
```
|
||||||
|
# Flat key format
|
||||||
|
file:{hashId}/{property} → value
|
||||||
|
|
||||||
|
# Examples
|
||||||
|
file:abc123/filePath → "watch/Movies/Inception.mkv"
|
||||||
|
file:abc123/title → "Inception"
|
||||||
|
file:abc123/titles/eng → "Inception" (nested property)
|
||||||
|
file:abc123/year → "2010"
|
||||||
|
|
||||||
|
# Index
|
||||||
|
file:__index__ → SET["abc123", "xyz789", ...]
|
||||||
|
|
||||||
|
# Event stream
|
||||||
|
meta:events → Stream of {type, key, ts} events
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
---
|
||||||
@@ -288,74 +254,92 @@ folderWatcher.watch(folderList, {
|
|||||||
|
|
||||||
### Startup Performance
|
### Startup Performance
|
||||||
|
|
||||||
| Dataset Size | Old Approach | New Approach | Speedup |
|
| Dataset Size | Streaming Bootstrap | Notes |
|
||||||
|--------------|-------------|--------------|---------|
|
|--------------|---------------------|-------|
|
||||||
| 1,000 files | 5-10s | 0.5-1s | 10x |
|
| 1,000 files | 0.5-1s | Stream replay + selective GET |
|
||||||
| 10,000 files | 60-120s | 1-2s | 60x |
|
| 10,000 files | 1-2s | Batch processing (100 events) |
|
||||||
| 100,000 files | 10-20m | 10-15s | 60-80x |
|
| 100,000 files | 10-15s | Memory-efficient incremental build |
|
||||||
|
|
||||||
### Storage Type Impact
|
|
||||||
|
|
||||||
| Storage Type | Old Approach | New Approach | Notes |
|
|
||||||
|--------------|-------------|--------------|-------|
|
|
||||||
| Local SSD | 5-10s | 1-2s | Validation was fast, but sequential |
|
|
||||||
| Local HDD | 20-40s | 1-2s | Seek time penalty eliminated |
|
|
||||||
| Network (NFS/SMB) | 60-300s | 1-2s | Latency eliminated from startup |
|
|
||||||
| Offline/Slow Network | Timeout/Fail | 1-2s | Gracefully handles offline storage |
|
|
||||||
|
|
||||||
### Memory Usage
|
### Memory Usage
|
||||||
|
|
||||||
- **In-Memory DB**: ~500 bytes/file (metadata only)
|
- **Per-file overhead**: ~500 bytes (only VFS-relevant properties)
|
||||||
- **10,000 files**: ~5 MB
|
- **10,000 files**: ~5 MB
|
||||||
- **100,000 files**: ~50 MB
|
- **100,000 files**: ~50 MB
|
||||||
- **Negligible overhead** compared to old approach
|
- **Comparison**: Previous HGETALL approach loaded all properties (~2-5KB/file)
|
||||||
|
|
||||||
|
### Why Streaming is Fast
|
||||||
|
|
||||||
|
| Old Approach | New Approach |
|
||||||
|
|--------------|--------------|
|
||||||
|
| `SCAN file:*` to find all keys | `XREAD meta:events` from position 0 |
|
||||||
|
| `HGETALL file:{id}` for each file | `GET file:{id}/{prop}` only for relevant props |
|
||||||
|
| All properties loaded | Only VFS-relevant properties |
|
||||||
|
| O(n × m) where m = all properties | O(n × k) where k = relevant properties |
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Handling Edge Cases
|
## Handling Edge Cases
|
||||||
|
|
||||||
|
### New Files
|
||||||
|
|
||||||
|
1. meta-sort processes file, writes properties to Redis
|
||||||
|
2. meta-sort publishes `set` events to `meta:events` stream
|
||||||
|
3. meta-fuse consumes events, fetches relevant properties
|
||||||
|
4. When `filePath` is set, file appears in VFS
|
||||||
|
|
||||||
### Deleted Files
|
### Deleted Files
|
||||||
|
|
||||||
**Scenario**: User deletes a file after startup but before discovery scans
|
1. meta-sort publishes `del file:{hashId}/filePath` event
|
||||||
|
2. meta-fuse processes event, removes file from state
|
||||||
|
3. VFS updated, file disappears
|
||||||
|
|
||||||
**Old Behavior**: File appears in VFS until next full rebuild
|
### Redis Reconnection
|
||||||
|
|
||||||
**New Behavior**:
|
1. Connection lost, stream consumer stops
|
||||||
1. Fast load: File appears in VFS (loaded from etcd)
|
2. KVManager detects disconnect, attempts reconnect
|
||||||
2. Discovery scan: File not found → removed from memory
|
3. On reconnect, can replay stream from last position
|
||||||
3. VFS rebuilt without the file
|
4. State incrementally updated with missed events
|
||||||
4. etcd still has metadata (for meta-orbit sharing)
|
|
||||||
|
|
||||||
### Offline Network Drives
|
### VFS Refresh
|
||||||
|
|
||||||
**Scenario**: NFS mount becomes unavailable
|
Triggered via `POST /api/fuse/refresh`:
|
||||||
|
|
||||||
**Old Behavior**: Startup hangs or fails with timeout
|
```typescript
|
||||||
|
// Fallback for when stream is unavailable
|
||||||
|
async refresh(): Promise<void> {
|
||||||
|
// Use flat key scanning fallback
|
||||||
|
const files = await redisClient.scanFlatKeysForFiles(
|
||||||
|
stateBuilder.getVfsRelevantProperties()
|
||||||
|
);
|
||||||
|
// Rebuild VFS from scanned data
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
**New Behavior**:
|
---
|
||||||
1. Fast load: All files appear in VFS (1-2s)
|
|
||||||
2. Discovery scan: Folder inaccessible → removes all child paths
|
|
||||||
3. When drive comes back online: Next discovery re-adds files
|
|
||||||
|
|
||||||
### Moved Files
|
## VFSUpdateCallback Interface
|
||||||
|
|
||||||
**Scenario**: User moves file outside watch folders
|
The VFS implements this callback interface to receive updates from StreamingStateBuilder:
|
||||||
|
|
||||||
**Old Behavior**: File appears in VFS until manual cleanup
|
```typescript
|
||||||
|
interface VFSUpdateCallback {
|
||||||
|
// Property changed for a file
|
||||||
|
onPropertyChange(hashId: string, property: string, value: string): void;
|
||||||
|
|
||||||
**New Behavior**:
|
// Property deleted from a file
|
||||||
1. Discovery scans old location → file not found → removed from memory
|
onPropertyDelete(hashId: string, property: string): void;
|
||||||
2. If moved within watch folders: Discovered as new file
|
|
||||||
3. etcd has both old and new paths (deduplication via midhash256)
|
|
||||||
|
|
||||||
### Partial Scans
|
// File completely removed (filePath deleted)
|
||||||
|
onFileDelete(hashId: string): void;
|
||||||
|
|
||||||
**Scenario**: Discovery interrupted mid-scan (crash, kill signal)
|
// File has all required properties, ready to appear in VFS
|
||||||
|
onFileComplete(hashId: string, metadata: Record<string, string>): void;
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
**Safety**: Cleanup only runs after scan completes successfully
|
The `onFileComplete` callback is called when:
|
||||||
- Incomplete scan → no cleanup
|
1. A file first receives a `filePath` property
|
||||||
- Memory state unchanged
|
2. Subsequent property changes after file is already complete
|
||||||
- Next scan will complete and clean up
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
@@ -364,164 +348,64 @@ folderWatcher.watch(folderList, {
|
|||||||
### Startup Logs
|
### Startup Logs
|
||||||
|
|
||||||
```
|
```
|
||||||
[VFS Rebuild] Starting fast-load from etcd...
|
[meta-fuse] Starting meta-fuse...
|
||||||
[VFS Rebuild] Found 10000 files in etcd
|
[KVManager] Waiting for leader...
|
||||||
[VFS Rebuild] Fetching metadata in parallel...
|
[LeaderClient] Leader found: meta-core-dev at redis://meta-core:6379
|
||||||
[VFS Rebuild] Loaded 9856/10000 files into memory (unverified)
|
[meta-fuse] KV Manager ready, initializing VFS with streaming bootstrap...
|
||||||
[VFS Rebuild] VFS rebuilt in 1523ms
|
[StreamingStateBuilder] Initialized with 12 VFS-relevant properties
|
||||||
[VFS Rebuild] Discovery will validate file existence in background
|
[meta-fuse] Replaying meta:events stream from position 0...
|
||||||
|
[RedisClient] Stream replay complete: 15234 entries in 1823ms
|
||||||
|
[meta-fuse] Streaming bootstrap complete: 5123 files in 1893ms
|
||||||
|
[meta-fuse] Starting live stream consumer from 1703808000000-0...
|
||||||
|
[meta-fuse] meta-fuse is ready!
|
||||||
|
[meta-fuse] VFS: 5123 files, 847 directories
|
||||||
|
[meta-fuse] State builder: 15234 events, 8934 properties fetched, 6300 skipped
|
||||||
```
|
```
|
||||||
|
|
||||||
### Discovery Logs (Progressive Mode)
|
### Stats API
|
||||||
|
|
||||||
```
|
|
||||||
[Discovery] Progressive scan mode: 3 folders
|
|
||||||
[Discovery] Scanning folder: /data/watch/local
|
|
||||||
[Discovery] Folder scan complete: /data/watch/local. Found 128 files. Running cleanup...
|
|
||||||
[Cleanup] Starting cleanup for 1 scanned folders...
|
|
||||||
[Cleanup] Removed 0 stale entries in 2ms
|
|
||||||
[Cleanup] Cleanup complete for folder: /data/watch/local
|
|
||||||
|
|
||||||
[Discovery] Scanning folder: /data/watch/smb-nas
|
|
||||||
[Discovery] Folder scan complete: /data/watch/smb-nas. Found 0 files. Running cleanup...
|
|
||||||
[Cleanup] Starting cleanup for 1 scanned folders...
|
|
||||||
[Cleanup] Removed stale entry: /data/watch/smb-nas/Movies/deleted_movie.mkv
|
|
||||||
[Cleanup] Removed stale entry: /data/watch/smb-nas/Movies/another_movie.mkv
|
|
||||||
[Cleanup] Removed 13621 stale entries in 87ms
|
|
||||||
[Cleanup] Rebuilding VFS after cleanup...
|
|
||||||
Virtual filesystem update start
|
|
||||||
Duplicate find took 1ms - found 0 hash groups, 21 title groups
|
|
||||||
Generating virtual structure for 128 files
|
|
||||||
Virtual filesystem update took 167ms
|
|
||||||
[Cleanup] Cleanup complete for folder: /data/watch/smb-nas
|
|
||||||
|
|
||||||
[Discovery] Scanning folder: /data/watch/downloads
|
|
||||||
[Discovery] Folder scan complete: /data/watch/downloads. Found 45 files. Running cleanup...
|
|
||||||
[Cleanup] Starting cleanup for 1 scanned folders...
|
|
||||||
[Cleanup] Removed 0 stale entries in 1ms
|
|
||||||
[Cleanup] Cleanup complete for folder: /data/watch/downloads
|
|
||||||
|
|
||||||
[Discovery] All folders scanned and cleaned up
|
|
||||||
```
|
|
||||||
|
|
||||||
**Progressive Behavior**: Notice how the NAS folder (`/data/watch/smb-nas`) with 0 discovered files immediately triggers removal of 13,621 stale entries and VFS rebuild. Users see files disappear in real-time as each folder completes scanning.
|
|
||||||
|
|
||||||
### API Endpoints
|
|
||||||
|
|
||||||
Monitor VFS state via Unified API:
|
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# VFS statistics
|
|
||||||
curl http://localhost:3000/api/fuse/stats
|
curl http://localhost:3000/api/fuse/stats
|
||||||
|
|
||||||
# Processing status
|
|
||||||
curl http://localhost:3000/api/processing/status
|
|
||||||
|
|
||||||
# Trigger manual scan + cleanup
|
|
||||||
curl -X POST http://localhost:3000/api/scan/trigger
|
|
||||||
```
|
```
|
||||||
|
|
||||||
---
|
```json
|
||||||
|
{
|
||||||
## Best Practices
|
"fileCount": 5123,
|
||||||
|
"directoryCount": 847,
|
||||||
### For Operators
|
"totalSize": 15847392847362,
|
||||||
|
"lastRefresh": "2024-01-15T10:30:00Z",
|
||||||
1. **Initial Startup**: First boot may show all historical files (even deleted ones)
|
"stateBuilder": {
|
||||||
- Wait for first discovery scan to clean up
|
"eventsProcessed": 15234,
|
||||||
- Or run manual scan: `POST /api/scan/trigger`
|
"propertiesFetched": 8934,
|
||||||
|
"propertiesSkipped": 6300,
|
||||||
2. **Network Drives**: If drives are slow or offline at startup
|
"filesComplete": 5123,
|
||||||
- VFS still loads in 1-2 seconds
|
"lastEventId": "1703808000000-0"
|
||||||
- Files appear/disappear as drives come online/offline
|
}
|
||||||
- No manual intervention needed
|
}
|
||||||
|
```
|
||||||
3. **Monitoring**: Watch for cleanup logs to understand disk changes
|
|
||||||
```
|
|
||||||
[Cleanup] Removed 622 stale entries
|
|
||||||
```
|
|
||||||
Large numbers indicate significant file deletions
|
|
||||||
|
|
||||||
### For Developers
|
|
||||||
|
|
||||||
1. **Adding New Metadata Sources**: Always update both:
|
|
||||||
- etcd (permanent record)
|
|
||||||
- In-memory DB (current state)
|
|
||||||
|
|
||||||
2. **Testing Cleanup**: Simulate file deletions
|
|
||||||
```bash
|
|
||||||
# Delete files from watch folder
|
|
||||||
rm -rf /data/watch/Movies/test_movie.mkv
|
|
||||||
|
|
||||||
# Trigger scan
|
|
||||||
curl -X POST http://localhost:3000/api/scan/trigger
|
|
||||||
|
|
||||||
# Verify cleanup in logs
|
|
||||||
docker logs meta-mesh-dev-container | grep Cleanup
|
|
||||||
```
|
|
||||||
|
|
||||||
3. **Performance Testing**: Benchmark with large datasets
|
|
||||||
```bash
|
|
||||||
# Time startup
|
|
||||||
docker restart meta-mesh-dev-container
|
|
||||||
docker logs -f meta-mesh-dev-container | grep "VFS rebuilt"
|
|
||||||
|
|
||||||
# Time discovery + cleanup
|
|
||||||
time curl -X POST http://localhost:3000/api/scan/trigger
|
|
||||||
```
|
|
||||||
|
|
||||||
---
|
|
||||||
|
|
||||||
## Future Optimizations
|
|
||||||
|
|
||||||
### Potential Improvements
|
|
||||||
|
|
||||||
1. **Incremental Cleanup**: Instead of scanning all folders, track modified folders
|
|
||||||
- Use filesystem events (inotify) to detect folder changes
|
|
||||||
- Only clean up modified folders
|
|
||||||
|
|
||||||
2. **Background Validation Queue**: Validate files lazily over time
|
|
||||||
- Prioritize recently accessed files
|
|
||||||
- Validate cold files during idle periods
|
|
||||||
|
|
||||||
3. **Smart Caching**: Cache validation results
|
|
||||||
- LRU cache with TTL (5-10 minutes)
|
|
||||||
- Reduces redundant disk checks for frequently accessed files
|
|
||||||
|
|
||||||
4. **Partial VFS Rebuild**: Only rebuild changed directories
|
|
||||||
- Track which folders had files removed
|
|
||||||
- Rebuild only affected VFS subtrees
|
|
||||||
|
|
||||||
### Not Recommended
|
|
||||||
|
|
||||||
❌ **TTL-based expiry**: Requires guessing timeout values, fails with slow scans
|
|
||||||
|
|
||||||
❌ **Storing validation state in etcd**: Wrong tool for transient state, creates write amplification
|
|
||||||
|
|
||||||
❌ **Immediate removal on unlink events**: Race conditions with discovery, inconsistent state
|
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Related Documentation
|
## Related Documentation
|
||||||
|
|
||||||
- [Streaming Pipeline Architecture](streaming-pipeline-architecture.md) - File processing flow
|
- [Streaming Architecture](streaming-architecture.md) - Detailed streaming event processing
|
||||||
- [etcd Architecture](../../packages/meta-mesh/doc/architecture/etcd-architecture.md) - Storage design
|
- [API Reference](api-reference.md) - Complete API documentation
|
||||||
- [FUSE API](../../packages/meta-mesh/doc/architecture/fuse-api.md) - Virtual filesystem API
|
- [meta-core Architecture](../../../meta-core/docs/architecture.md) - Leader election and Redis management
|
||||||
|
|
||||||
---
|
---
|
||||||
|
|
||||||
## Summary
|
## Summary
|
||||||
|
|
||||||
The VFS rebuild architecture achieves **10-50x faster startup** by separating concerns:
|
The VFS rebuild architecture achieves **fast startup and real-time updates** through:
|
||||||
|
|
||||||
- **etcd** = Permanent distributed metadata archive
|
- **Redis Streams** = Ordered event log with replay capability
|
||||||
- **In-memory DB** = Current filesystem state
|
- **StreamingStateBuilder** = Memory-efficient state projection
|
||||||
- **Discovery** = Authoritative source of truth
|
- **Property Filtering** = Only VFS-relevant properties fetched
|
||||||
|
|
||||||
This design enables:
|
This design enables:
|
||||||
- ⚡ Sub-2-second cold starts (10k files)
|
- Sub-2-second cold starts (10k files)
|
||||||
- 🔄 Continuous accuracy through discovery validation
|
- Real-time VFS updates via stream consumption
|
||||||
- 🌐 Distributed metadata sharing via meta-orbit
|
- Memory-efficient state (~500 bytes/file)
|
||||||
- 📦 Graceful handling of offline/slow storage
|
- Graceful reconnection and recovery
|
||||||
|
|
||||||
The key insight: **Fast bootstrap + continuous validation > Blocking validation at startup**
|
The key insight: **Stream replay + selective property fetching > Full scan + full data loading**
|
||||||
|
|||||||
@@ -151,6 +151,10 @@ async function main(): Promise<void> {
|
|||||||
filesPath: config.filesPath,
|
filesPath: config.filesPath,
|
||||||
});
|
});
|
||||||
|
|
||||||
|
// Wire up StateBuilder to VFS for refresh operations
|
||||||
|
// This enables vfs.refresh() to replay the event stream
|
||||||
|
vfs.setStreamingStateBuilder(stateBuilder);
|
||||||
|
|
||||||
// Bootstrap: Replay meta:events stream from position 0
|
// Bootstrap: Replay meta:events stream from position 0
|
||||||
// This rebuilds state from all historical events
|
// This rebuilds state from all historical events
|
||||||
logger.info(`Replaying ${EVENTS_STREAM} stream from position 0 (streaming bootstrap)...`);
|
logger.info(`Replaying ${EVENTS_STREAM} stream from position 0 (streaming bootstrap)...`);
|
||||||
@@ -166,21 +170,9 @@ async function main(): Promise<void> {
|
|||||||
);
|
);
|
||||||
|
|
||||||
const bootstrapTime = Date.now() - bootstrapStart;
|
const bootstrapTime = Date.now() - bootstrapStart;
|
||||||
let stats = vfs.getStats();
|
const stats = vfs.getStats();
|
||||||
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
|
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
|
||||||
|
|
||||||
// Fallback: If stream was empty but Redis has files, bootstrap from Redis keys
|
|
||||||
// This handles the case where meta-core restarted and cleared the stream
|
|
||||||
if (stats.fileCount === 0) {
|
|
||||||
const indexCount = await redisClient.getIndexCount();
|
|
||||||
if (indexCount > 0) {
|
|
||||||
logger.info(`Stream was empty but Redis has ${indexCount} files - bootstrapping from Redis keys...`);
|
|
||||||
await vfs.refresh();
|
|
||||||
stats = vfs.getStats();
|
|
||||||
logger.info(`Redis key bootstrap complete: ${stats.fileCount} files`);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Start live stream consumer from where replay left off
|
// Start live stream consumer from where replay left off
|
||||||
logger.info(`Starting live stream consumer from ${lastId}...`);
|
logger.info(`Starting live stream consumer from ${lastId}...`);
|
||||||
redisClient.startSimpleStreamConsumer(
|
redisClient.startSimpleStreamConsumer(
|
||||||
|
|||||||
@@ -74,9 +74,12 @@ export interface LeaderLockInfo {
|
|||||||
/** Redis connection URL (e.g., redis://10.0.1.50:6379) */
|
/** Redis connection URL (e.g., redis://10.0.1.50:6379) */
|
||||||
redisUrl: string;
|
redisUrl: string;
|
||||||
|
|
||||||
/** WebDAV URL for file access */
|
/** WebDAV URL for file access (external, via nginx/HTTPS) */
|
||||||
webdavUrl: string;
|
webdavUrl: string;
|
||||||
|
|
||||||
|
/** WebDAV URL for internal container-to-container access (direct to port 9000) */
|
||||||
|
webdavUrlInternal: string;
|
||||||
|
|
||||||
/** Timestamp when leadership was acquired */
|
/** Timestamp when leadership was acquired */
|
||||||
timestamp: number;
|
timestamp: number;
|
||||||
|
|
||||||
|
|||||||
@@ -32,6 +32,7 @@ export interface URLsResponse {
|
|||||||
apiUrl: string;
|
apiUrl: string;
|
||||||
redisUrl: string;
|
redisUrl: string;
|
||||||
webdavUrl: string;
|
webdavUrl: string;
|
||||||
|
webdavUrlInternal: string;
|
||||||
isLeader: boolean;
|
isLeader: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -120,6 +121,7 @@ export class LeaderClient {
|
|||||||
apiUrl: urls.apiUrl,
|
apiUrl: urls.apiUrl,
|
||||||
redisUrl: urls.redisUrl,
|
redisUrl: urls.redisUrl,
|
||||||
webdavUrl: urls.webdavUrl,
|
webdavUrl: urls.webdavUrl,
|
||||||
|
webdavUrlInternal: urls.webdavUrlInternal,
|
||||||
timestamp: Date.now(),
|
timestamp: Date.now(),
|
||||||
pid: 0 // Unknown for remote leader
|
pid: 0 // Unknown for remote leader
|
||||||
};
|
};
|
||||||
@@ -139,13 +141,22 @@ export class LeaderClient {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get WebDAV URL from leader info
|
* Get WebDAV URL from leader info (external, via nginx/HTTPS)
|
||||||
*/
|
*/
|
||||||
async getWebdavUrl(): Promise<string | null> {
|
async getWebdavUrl(): Promise<string | null> {
|
||||||
const info = await this.getLeaderInfo();
|
const info = await this.getLeaderInfo();
|
||||||
return info?.webdavUrl ?? null;
|
return info?.webdavUrl ?? null;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get internal WebDAV URL from leader info (direct to port 9000)
|
||||||
|
* Use this for container-to-container communication
|
||||||
|
*/
|
||||||
|
async getWebdavUrlInternal(): Promise<string | null> {
|
||||||
|
const info = await this.getLeaderInfo();
|
||||||
|
return info?.webdavUrlInternal ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Get meta-core API URL from leader info
|
* Get meta-core API URL from leader info
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -463,6 +463,76 @@ export class RedisClient implements Partial<IKVClient> {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Scan flat keys to build file metadata for all indexed files
|
||||||
|
* This is used as a fallback when the meta:events stream is empty or unavailable.
|
||||||
|
*
|
||||||
|
* Algorithm:
|
||||||
|
* 1. Get all hash IDs from the index
|
||||||
|
* 2. For each hash ID, fetch VFS-relevant properties using MGET
|
||||||
|
* 3. Return a Map of hashId to flat properties
|
||||||
|
*
|
||||||
|
* This replaces the broken getAllFiles() which used HGETALL on flat keys.
|
||||||
|
*
|
||||||
|
* @param vfsRelevantProperties - Set of property names that are VFS-relevant
|
||||||
|
* @returns Map of hashId to properties (only files with filePath)
|
||||||
|
*/
|
||||||
|
async scanFlatKeysForFiles(vfsRelevantProperties: Set<string>): Promise<Map<string, Record<string, string>>> {
|
||||||
|
const result = new Map<string, Record<string, string>>();
|
||||||
|
|
||||||
|
if (!this.client || !this.isConnected) {
|
||||||
|
logger.warn('Redis not connected, returning empty result');
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
// Get all hash IDs from the index
|
||||||
|
const hashIds = await this.getAllHashIds();
|
||||||
|
logger.info(`Scanning flat keys for ${hashIds.length} files...`);
|
||||||
|
|
||||||
|
if (hashIds.length === 0) {
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Batch process hashIds to avoid overwhelming Redis
|
||||||
|
const batchSize = 100;
|
||||||
|
const propertiesArray = Array.from(vfsRelevantProperties);
|
||||||
|
|
||||||
|
for (let i = 0; i < hashIds.length; i += batchSize) {
|
||||||
|
const batch = hashIds.slice(i, i + batchSize);
|
||||||
|
|
||||||
|
// Process each hashId in the batch
|
||||||
|
const batchPromises = batch.map(async (hashId) => {
|
||||||
|
const props = await this.getProperties(hashId, propertiesArray);
|
||||||
|
|
||||||
|
// Only include files that have a filePath
|
||||||
|
if (props.has('filePath')) {
|
||||||
|
const propsObj: Record<string, string> = {};
|
||||||
|
for (const [key, value] of props) {
|
||||||
|
propsObj[key] = value;
|
||||||
|
}
|
||||||
|
return { hashId, props: propsObj };
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
});
|
||||||
|
|
||||||
|
const batchResults = await Promise.all(batchPromises);
|
||||||
|
|
||||||
|
for (const item of batchResults) {
|
||||||
|
if (item) {
|
||||||
|
result.set(item.hashId, item.props);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Flat key scan complete: ${result.size} files with filePath`);
|
||||||
|
return result;
|
||||||
|
} catch (error: any) {
|
||||||
|
logger.error('Failed to scan flat keys:', error.message);
|
||||||
|
return result;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Scan Redis keys matching pattern
|
* Scan Redis keys matching pattern
|
||||||
*/
|
*/
|
||||||
|
|||||||
@@ -101,6 +101,21 @@ export class StreamingStateBuilder {
|
|||||||
logger.info(`Updated VFS-relevant properties: ${this.vfsRelevantProps.size}`);
|
logger.info(`Updated VFS-relevant properties: ${this.vfsRelevantProps.size}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the set of VFS-relevant properties
|
||||||
|
* Used by VFS refresh for flat key scanning fallback
|
||||||
|
*/
|
||||||
|
getVfsRelevantProperties(): Set<string> {
|
||||||
|
return this.vfsRelevantProps;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Get the current VFS callback
|
||||||
|
*/
|
||||||
|
getVFSCallback(): VFSUpdateCallback | undefined {
|
||||||
|
return this.vfsCallback;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Process a stream message event
|
* Process a stream message event
|
||||||
* This is the main entry point for event processing
|
* This is the main entry point for event processing
|
||||||
|
|||||||
@@ -15,7 +15,7 @@ import { TemplateEngine } from './template/TemplateEngine.js';
|
|||||||
import { ConditionEvaluator } from './template/ConditionEvaluator.js';
|
import { ConditionEvaluator } from './template/ConditionEvaluator.js';
|
||||||
import { ConfigStorage } from '../config/ConfigStorage.js';
|
import { ConfigStorage } from '../config/ConfigStorage.js';
|
||||||
import type { RenamingConfig, RenamingRule, PreviewItem, PreviewResponse } from './types/RenamingRuleTypes.js';
|
import type { RenamingConfig, RenamingRule, PreviewItem, PreviewResponse } from './types/RenamingRuleTypes.js';
|
||||||
import type { VFSUpdateCallback } from './StreamingStateBuilder.js';
|
import type { VFSUpdateCallback, StreamingStateBuilder } from './StreamingStateBuilder.js';
|
||||||
|
|
||||||
const logger = new Logger({ name: 'VirtualFileSystem' });
|
const logger = new Logger({ name: 'VirtualFileSystem' });
|
||||||
|
|
||||||
@@ -72,6 +72,7 @@ export class VirtualFileSystem implements VFSUpdateCallback {
|
|||||||
private config: Required<VFSConfig>;
|
private config: Required<VFSConfig>;
|
||||||
private lastRefresh: Date | null = null;
|
private lastRefresh: Date | null = null;
|
||||||
private useStreamingMode: boolean = false; // Flag to indicate streaming mode
|
private useStreamingMode: boolean = false; // Flag to indicate streaming mode
|
||||||
|
private stateBuilder: StreamingStateBuilder | null = null; // For stream replay during refresh
|
||||||
|
|
||||||
private cachedStats = {
|
private cachedStats = {
|
||||||
fileCount: 0,
|
fileCount: 0,
|
||||||
@@ -128,6 +129,15 @@ export class VirtualFileSystem implements VFSUpdateCallback {
|
|||||||
logger.info('VFS streaming mode enabled');
|
logger.info('VFS streaming mode enabled');
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Set the streaming state builder for refresh operations
|
||||||
|
* This enables refresh() to replay the event stream instead of using broken HGETALL
|
||||||
|
*/
|
||||||
|
setStreamingStateBuilder(builder: StreamingStateBuilder): void {
|
||||||
|
this.stateBuilder = builder;
|
||||||
|
logger.debug('StreamingStateBuilder set for VFS refresh');
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Convert flat properties map to FileMetadata
|
* Convert flat properties map to FileMetadata
|
||||||
* Used when processing streaming events
|
* Used when processing streaming events
|
||||||
@@ -281,46 +291,144 @@ export class VirtualFileSystem implements VFSUpdateCallback {
|
|||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Refresh the VFS from Redis
|
* Refresh the VFS from Redis by replaying the event stream
|
||||||
* Called once on startup for initial bootstrap.
|
*
|
||||||
* Subsequent updates come via Redis Streams (onFileUpdate).
|
* This method replays the meta:events stream from position 0, applying updates
|
||||||
* Uses template-based rules if configured, otherwise falls back to MetaDataToFolderStruct
|
* incrementally without wiping the existing VFS state. Files not seen during
|
||||||
|
* the replay are considered deleted and removed.
|
||||||
|
*
|
||||||
|
* Design principles:
|
||||||
|
* - Do NOT wipe the internal model (no initRoot() call)
|
||||||
|
* - Track which files are seen during replay
|
||||||
|
* - Remove only files that are no longer in the stream (deleted)
|
||||||
|
* - Single code path - reuses StreamingStateBuilder event processing
|
||||||
*/
|
*/
|
||||||
async refresh(): Promise<void> {
|
async refresh(): Promise<void> {
|
||||||
logger.debug('Loading VFS from Redis (bootstrap)');
|
logger.info('Refreshing VFS from event stream...');
|
||||||
|
const startTime = Date.now();
|
||||||
|
|
||||||
|
// Load rules configuration
|
||||||
|
this.loadRulesConfig();
|
||||||
|
|
||||||
|
// Check if we have a streaming state builder configured
|
||||||
|
if (!this.stateBuilder) {
|
||||||
|
logger.warn('No StreamingStateBuilder set - cannot refresh from stream');
|
||||||
|
logger.warn('VFS refresh skipped. Set streaming state builder via setStreamingStateBuilder()');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
// Load rules configuration
|
// Track which hashIds we see during this refresh
|
||||||
this.loadRulesConfig();
|
const seenHashIds = new Set<string>();
|
||||||
|
|
||||||
// Get all files from Redis (keyed by sourcePath)
|
// Create a tracking callback that records seen hashIds
|
||||||
const files = await this.redisClient.getAllFiles();
|
const originalCallback = this.stateBuilder.getVFSCallback();
|
||||||
|
const trackingVFS: VFSUpdateCallback = {
|
||||||
|
onPropertyChange: (hashId, property, value) => {
|
||||||
|
this.onPropertyChange(hashId, property, value);
|
||||||
|
},
|
||||||
|
onPropertyDelete: (hashId, property) => {
|
||||||
|
this.onPropertyDelete(hashId, property);
|
||||||
|
},
|
||||||
|
onFileDelete: (hashId) => {
|
||||||
|
this.onFileDelete(hashId);
|
||||||
|
},
|
||||||
|
onFileComplete: (hashId, metadata) => {
|
||||||
|
seenHashIds.add(hashId);
|
||||||
|
this.onFileComplete(hashId, metadata);
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
if (files.size === 0) {
|
// Temporarily set tracking callback
|
||||||
logger.info('No files found in Redis');
|
this.stateBuilder.setVFSCallback(trackingVFS);
|
||||||
this.initRoot();
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Rebuild VFS with computed paths
|
// Clear state builder's internal state (but NOT VFS nodes)
|
||||||
this.initRoot();
|
this.stateBuilder.clear();
|
||||||
|
|
||||||
for (const [sourcePath, metadata] of files) {
|
// Replay meta:events stream from position 0
|
||||||
// Compute virtual path using rules or fallback
|
const lastId = await this.redisClient.replayStream(
|
||||||
const virtualPath = this.computeVirtualPath(metadata, sourcePath);
|
'meta:events',
|
||||||
if (virtualPath) {
|
'0',
|
||||||
await this.addFile(virtualPath, metadata);
|
async (message) => {
|
||||||
this.sourcePathToVirtualPath.set(sourcePath, virtualPath);
|
await this.stateBuilder!.processEvent(message);
|
||||||
|
},
|
||||||
|
100 // Batch size
|
||||||
|
);
|
||||||
|
|
||||||
|
// Fallback: If no files found from stream replay, scan flat keys directly
|
||||||
|
// This handles the case where meta:events stream isn't being populated
|
||||||
|
// but data exists in flat key format
|
||||||
|
if (seenHashIds.size === 0) {
|
||||||
|
const indexCount = await this.redisClient.getIndexCount();
|
||||||
|
if (indexCount > 0) {
|
||||||
|
logger.info(`Stream replay found no files but index has ${indexCount} - scanning flat keys...`);
|
||||||
|
|
||||||
|
// Get VFS-relevant properties from state builder
|
||||||
|
const vfsRelevantProps = this.stateBuilder.getVfsRelevantProperties();
|
||||||
|
|
||||||
|
// Scan flat keys
|
||||||
|
const flatKeyFiles = await this.redisClient.scanFlatKeysForFiles(vfsRelevantProps);
|
||||||
|
|
||||||
|
// Process each file through onFileComplete
|
||||||
|
for (const [hashId, props] of flatKeyFiles) {
|
||||||
|
seenHashIds.add(hashId);
|
||||||
|
this.onFileComplete(hashId, props);
|
||||||
|
}
|
||||||
|
|
||||||
|
logger.info(`Flat key scan populated ${seenHashIds.size} files`);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Restore original callback
|
||||||
|
this.stateBuilder.setVFSCallback(originalCallback || this);
|
||||||
|
|
||||||
|
// Remove files not seen during replay (these were deleted)
|
||||||
|
const toRemove: string[] = [];
|
||||||
|
for (const [hashId, virtualPath] of this.hashIdToVirtualPath) {
|
||||||
|
if (!seenHashIds.has(hashId)) {
|
||||||
|
logger.debug(`Removing deleted file: ${hashId} -> ${virtualPath}`);
|
||||||
|
toRemove.push(hashId);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const hashId of toRemove) {
|
||||||
|
const virtualPath = this.hashIdToVirtualPath.get(hashId);
|
||||||
|
if (virtualPath) {
|
||||||
|
this.removeFile(virtualPath);
|
||||||
|
}
|
||||||
|
this.hashIdToVirtualPath.delete(hashId);
|
||||||
|
}
|
||||||
|
|
||||||
this.lastRefresh = new Date();
|
this.lastRefresh = new Date();
|
||||||
logger.info(`VFS refreshed: ${this.cachedStats.fileCount} files, ${this.cachedStats.directoryCount} directories`);
|
this.updateStats();
|
||||||
|
const elapsed = Date.now() - startTime;
|
||||||
|
logger.info(`VFS refresh complete: ${this.cachedStats.fileCount} files, ${this.cachedStats.directoryCount} directories in ${elapsed}ms`);
|
||||||
|
logger.info(`Replay ended at stream position: ${lastId}, removed ${toRemove.length} deleted files`);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error('Failed to refresh VFS:', error);
|
logger.error('Failed to refresh VFS:', error);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Update cached stats from current nodes
|
||||||
|
*/
|
||||||
|
private updateStats(): void {
|
||||||
|
let fileCount = 0;
|
||||||
|
let directoryCount = 0;
|
||||||
|
let totalSize = 0;
|
||||||
|
|
||||||
|
for (const node of this.nodes.values()) {
|
||||||
|
if (node.type === 'file') {
|
||||||
|
fileCount++;
|
||||||
|
totalSize += node.size ?? 0;
|
||||||
|
} else {
|
||||||
|
directoryCount++;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
this.cachedStats = { fileCount, directoryCount, totalSize };
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Load rules configuration from file
|
* Load rules configuration from file
|
||||||
* ConfigStorage creates the default config file if it doesn't exist
|
* ConfigStorage creates the default config file if it doesn't exist
|
||||||
|
|||||||
Reference in New Issue
Block a user