6 Commits
Author SHA1 Message Date
Worph d35a64a744 align with metadata standard
Build and Push Docker Image / build (push) Failing after 15s
2026-09-13 15:57:12 +02:00
Worph 9061c7a37d gitattributes 2026-07-14 16:08:49 +02:00
Pierre b100b3a77c improve doc + bugfix 2026-05-25 23:47:48 +02:00
Pierre 8e50b85e08 bugfix 2026-05-24 17:43:31 +02:00
Pierre a03aa35169 adaptation related to meta-core api changes 2026-05-24 17:26:13 +02:00
Pierre 797cf98afd ignore file 2026-04-28 12:20:57 +02:00
16 changed files with 522 additions and 83 deletions
+6
View File
@@ -0,0 +1,6 @@
# FIX CRLF always when developer has not set
# Linux/Mac: git config --global core.autocrlf input
# Windows: git config --global core.autocrlf true
# Auto detect text files and perform LF normalization
* text=auto
* eol=lf
+1
View File
@@ -25,3 +25,4 @@ logs/
# Runtime data
data/
docker/__pycache__/
+11 -11
View File
@@ -84,7 +84,7 @@ Meta-Fuse is a read-only process in the MetaMesh ecosystem that:
│ [3] meta-stremio (PROCESS-READ) │
│ └─► Reads metadata from KV, streams media content │
│ │
│ [4] meta-orbit (SHARING-READ-WRITE) │
│ [4] meta-share (SHARING-READ-WRITE) │
│ └─► P2P metadata sync across network │
│ │
└────────────────────────────────────────────────────────────────────────────┘
@@ -104,7 +104,7 @@ http://meta-core:9000
/urls API Response (JSON):
{
"hostname": "meta-core-dev",
"hostname": "metacore-app",
"baseUrl": "http://localhost:8083",
"apiUrl": "http://meta-core:9000",
"redisUrl": "redis://meta-core:6379",
@@ -606,10 +606,10 @@ journalctl -u meta-fuse-driver -f
cat /meta-core/locks/kv-leader.info
# Check /urls API
curl -k https://dev.localhost:8083/api/urls
curl -k https://metacore-dev.localhost:8083/api/urls
# Verify redis is running (on leader)
docker exec meta-core-dev redis-cli ping
docker exec metacore-app redis-cli ping
# Check meta-fuse logs
docker logs meta-fuse | grep "KV"
@@ -632,10 +632,10 @@ docker logs meta-fuse | grep "wsgidav"
```bash
# Check Redis has file data (flat key format)
docker exec meta-core-dev redis-cli keys "file:*" | head -20
docker exec metacore-app redis-cli keys "file:*" | head -20
# Check file index
docker exec meta-core-dev redis-cli scard "file:__index__"
docker exec metacore-app redis-cli scard "file:__index__"
# Verify source files exist
ls -la /files/watch/
@@ -644,7 +644,7 @@ ls -la /files/watch/
curl http://localhost:3000/api/fuse/stats
# Check streaming state builder stats
docker logs meta-fuse-dev | grep "State builder"
docker logs metafuse-app | grep "State builder"
```
## Technology Stack
@@ -665,7 +665,7 @@ Meta-Fuse is designed to work seamlessly with other MetaMesh services:
- **meta-sort**: Provides metadata in KV database
- **meta-stremio**: Can stream files exposed by meta-fuse
- **meta-orbit**: Syncs metadata across P2P network
- **meta-share**: Syncs metadata across P2P network
### Connecting to meta-sort
@@ -673,13 +673,13 @@ Ensure meta-sort is running and has processed files:
```bash
# Check meta-sort processing status
curl -k https://dev.localhost:8180/api/processing/status
curl -k https://metasort-dev.localhost:8180/api/processing/status
# Verify Redis has metadata (via meta-core)
docker exec meta-core-dev redis-cli scard "file:__index__"
docker exec metacore-app redis-cli scard "file:__index__"
# Check meta:events stream has events
docker exec meta-core-dev redis-cli xlen "meta:events"
docker exec metacore-app redis-cli xlen "meta:events"
```
Meta-fuse will automatically discover the KV database through the shared lock file at `/meta-core/locks/kv-leader.info`.
+7 -7
View File
@@ -535,29 +535,29 @@ List all discovered MetaMesh services.
"services": [
{
"name": "meta-sort",
"url": "http://meta-sort-dev",
"api": "http://meta-sort-dev/api",
"url": "http://metasort-app",
"api": "http://metasort-app/api",
"status": "healthy",
"role": "writer"
},
{
"name": "meta-fuse",
"url": "http://meta-fuse-dev",
"api": "http://meta-fuse-dev/api",
"url": "http://metafuse-app",
"api": "http://metafuse-app/api",
"status": "healthy",
"role": "reader"
},
{
"name": "meta-stremio",
"url": "http://meta-stremio-dev",
"api": "http://meta-stremio-dev/api",
"url": "http://metastremio-app",
"api": "http://metastremio-app/api",
"status": "healthy",
"role": "reader"
}
],
"current": "meta-fuse",
"leader": {
"hostname": "meta-core-dev",
"hostname": "metacore-app",
"baseUrl": "http://localhost:8083",
"apiUrl": "http://meta-core:9000",
"redisUrl": "redis://meta-core:6379",
+1 -1
View File
@@ -350,7 +350,7 @@ The `onFileComplete` callback is called when:
```
[meta-fuse] Starting meta-fuse...
[KVManager] Waiting for leader...
[LeaderClient] Leader found: meta-core-dev at redis://meta-core:6379
[LeaderClient] Leader found: metacore-app at redis://meta-core:6379
[meta-fuse] KV Manager ready, initializing VFS with streaming bootstrap...
[StreamingStateBuilder] Initialized with 12 VFS-relevant properties
[meta-fuse] Replaying meta:events stream from position 0...
+48 -27
View File
@@ -11,6 +11,7 @@ import 'dotenv/config';
import { Logger } from 'tslog';
import { KVManager } from './kv/KVManager.js';
import { RedisClient, StreamMessage } from './kv/RedisClient.js';
import { SSEEventClient, type SSEEvent } from './kv/SSEEventClient.js';
import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
import { StreamingStateBuilder } from './vfs/StreamingStateBuilder.js';
import { APIServer } from './api/APIServer.js';
@@ -76,13 +77,16 @@ async function main(): Promise<void> {
let vfs: VirtualFileSystem | null = null;
let stateBuilder: StreamingStateBuilder | null = null;
let apiServer: APIServer | null = null;
let sseClient: SSEEventClient | null = null;
// Graceful shutdown
const shutdown = async (signal: string): Promise<void> => {
logger.info(`Received ${signal}, shutting down...`);
try {
// Stop stream consumer first
// Stop SSE consumer first
if (sseClient) await sseClient.stop();
// Stop legacy stream consumer if it ever ran
if (redisClient) {
redisClient.stopStreamConsumer();
}
@@ -155,35 +159,52 @@ async function main(): Promise<void> {
// This enables vfs.refresh() to replay the event stream
vfs.setStreamingStateBuilder(stateBuilder);
// Bootstrap: Replay meta:events stream from position 0
// This rebuilds state from all historical events
logger.info(`Replaying ${EVENTS_STREAM} stream from position 0 (streaming bootstrap)...`);
const bootstrapStart = Date.now();
const lastId = await redisClient.replayStream(
EVENTS_STREAM,
'0',
async (message: StreamMessage) => {
// Bootstrap + live stream via SSE on /api/events/meta.
//
// We persist the cursor; on a clean restart we resume from where
// we left off. On a `gap` event (cursor trimmed out of retention)
// we fall back to replaying from the oldest available entry,
// which is good enough to rebuild VFS state because every property
// change has been republished into the stream by meta-core.
const apiUrl = leaderInfo?.apiUrl;
if (!apiUrl) {
throw new Error('meta-core apiUrl not available from leader info; cannot start SSE consumer');
}
logger.info(`Starting SSE consumer for /api/events/meta (apiUrl=${apiUrl})...`);
sseClient = new SSEEventClient({
url: `${apiUrl.replace(/\/+$/, '')}/api/events/meta`,
// Deliberately NOT persisting the cursor. meta-fuse holds
// VFS state purely in memory, so each restart needs to
// rebuild from a full event replay. Setting cursorPath to
// null + initialCursor to "0-0" forces the gap-handling
// path which resumes from the oldest available stream
// entry — i.e. effectively a full bootstrap.
cursorPath: null,
logTag: '[meta-fuse SSE]',
initialCursor: '0-0',
onEvent: async (e: SSEEvent) => {
// Translate SSE event back into the StreamMessage shape
// StreamingStateBuilder expects.
const message: StreamMessage = {
id: e.id,
type: e.event as any,
key: e.data?.key,
path: e.data?.path,
size: e.data?.size,
midhash256: e.data?.midhash256,
oldPath: e.data?.oldPath,
watcherId: e.data?.watcherId,
timestamp: e.data?.timestamp ?? e.data?.ts,
} as StreamMessage;
await handleStreamingEvent(message, stateBuilder!);
},
100 // Batch size
);
const bootstrapTime = Date.now() - bootstrapStart;
const stats = vfs.getStats();
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
// Start live stream consumer from where replay left off
logger.info(`Starting live stream consumer from ${lastId}...`);
redisClient.startSimpleStreamConsumer(
EVENTS_STREAM,
lastId,
async (message: StreamMessage) => {
await handleStreamingEvent(message, stateBuilder!);
onGap: (payload) => {
logger.warn(`SSE cursor ${payload.requested} trimmed; resuming from ${payload.resumeFrom}`);
},
5000 // 5 second block timeout
).catch(error => {
logger.error('Stream consumer error:', error);
});
// Fire-and-forget; start() returns when stop() fires.
sseClient.start().catch((error) => {
logger.error('SSE consumer terminated:', error);
});
// Initialize API server (with kvManager for service discovery)
+14 -6
View File
@@ -113,7 +113,7 @@ export class KVManager {
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
logger.info(`Connecting to Redis at ${leaderInfo.redisUrl}...`);
await this.connectToRedis(leaderInfo.redisUrl);
await this.connectToRedis(leaderInfo.redisUrl, leaderInfo.apiUrl);
} catch (error: any) {
logger.error(`Failed to connect to leader: ${error.message}`);
throw error;
@@ -141,7 +141,7 @@ export class KVManager {
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
logger.info(`Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
await this.connectToRedis(leaderInfo.redisUrl);
await this.connectToRedis(leaderInfo.redisUrl, leaderInfo.apiUrl);
} catch (error: any) {
logger.error(`Failed to reconnect: ${error.message}`);
}
@@ -178,20 +178,28 @@ export class KVManager {
/**
* Connect to Redis
*/
private async connectToRedis(url: string): Promise<void> {
private async connectToRedis(url: string, apiUrl?: string): Promise<void> {
try {
// Disconnect existing client if any
await this.disconnectRedis();
// Create new client
// Create new client. When apiUrl is supplied, reads route
// through meta-core HTTP (api-mediated-access PR C).
// When url is also empty, the client is HTTP-only and never
// opens a Redis socket (post-PR D state).
this.redisClient = new RedisClient({
url,
url, // may be '' in PR D world
prefix: this.config.redisPrefix,
filesVolume: this.config.filesPath
filesVolume: this.config.filesPath,
metaCoreApiUrl: apiUrl,
});
await this.redisClient.connect();
if (url) {
logger.info('Connected to Redis');
} else {
logger.info('Storage client ready (HTTP-only mode, no Redis)');
}
// Notify ready
this.notifyReady();
+18 -1
View File
@@ -88,7 +88,24 @@ export class LeaderClient {
return null;
}
this.cachedUrls = await response.json() as URLsResponse;
const parsed = await response.json() as URLsResponse;
// api-mediated-access PR D: redisUrl is no longer published by
// meta-core. If it shows up, an older build snuck back in —
// fail loudly so the rollback can't go unnoticed. Set
// ALLOW_LEGACY_REDIS_URL=1 to downgrade to a warning during a
// deliberate temporary rollback.
if (parsed.redisUrl) {
const msg =
'meta-core still publishes redisUrl; direct Redis access ' +
'was retired by the api-mediated-access lockdown. Verify ' +
'meta-core version.';
if (process.env.ALLOW_LEGACY_REDIS_URL === '1') {
logger.warn('WARNING: ' + msg);
} else {
throw new Error(msg);
}
}
this.cachedUrls = parsed;
this.urlsCacheTime = now;
return this.cachedUrls;
} catch (error: any) {
@@ -0,0 +1,121 @@
/**
* MetaCoreApiClient — HTTP read client against meta-core for meta-fuse.
*
* meta-fuse needs to read file metadata (property GETs, full-record fetch,
* hashId enumeration) as it builds the VFS. After the api-mediated-access
* lockdown lands these go through meta-core's /meta/{hash} family instead
* of direct Redis reads.
*
* Only the methods meta-fuse actually calls are implemented. See
* meta-core docs/api-mediated-access.md for the full surface.
*/
export interface MetaCoreApiClientConfig {
apiUrl: string;
timeoutMs?: number;
}
export class MetaCoreApiClient {
private apiUrl: string;
private timeoutMs: number;
constructor(config: MetaCoreApiClientConfig) {
this.apiUrl = config.apiUrl.replace(/\/+$/, '');
this.timeoutMs = config.timeoutMs ?? 30000;
}
/**
* GET /meta/{hash}/{prop}. Returns the raw string value or null on 404.
* meta-core sends text/plain for property reads.
*/
async getProperty(hashId: string, property: string): Promise<string | null> {
const path = `/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(property)}`;
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method: 'GET',
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'text/plain' },
});
if (response.status === 404) return null;
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] GET ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
return text === '' ? null : text;
}
/**
* GET /meta/{hash}. Returns the flat property map or null on 404.
*/
async getMetadataFlat(hashId: string): Promise<Record<string, string> | null> {
const path = `/meta/${encodeURIComponent(hashId)}`;
const body = await this.fetchJsonAllowing404('GET', path);
if (body === null) return null;
const flat = (body as any)?.metadata;
if (!flat || typeof flat !== 'object') return null;
return flat as Record<string, string>;
}
/**
* GET /meta. Enumerates every known hashId.
*/
async getAllHashIds(): Promise<string[]> {
const body = await this.fetchJson('GET', '/meta');
const ids = (body as any)?.hashIds;
return Array.isArray(ids) ? ids : [];
}
/**
* GET /health on meta-core. Used as the connectivity probe.
*/
async health(): Promise<boolean> {
try {
const response = await fetch(`${this.apiUrl}/health`, {
method: 'GET',
signal: AbortSignal.timeout(Math.min(this.timeoutMs, 5000)),
headers: { Accept: 'application/json' },
});
return response.ok;
} catch {
return false;
}
}
private encodePropertyPath(property: string): string {
return property
.split('/')
.map(segment => encodeURIComponent(segment))
.join('/');
}
private async fetchJson(method: string, path: string): Promise<unknown> {
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method,
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'application/json' },
});
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] ${method} ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
if (!text) return null;
try { return JSON.parse(text); } catch { return text; }
}
private async fetchJsonAllowing404(method: string, path: string): Promise<unknown> {
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method,
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'application/json' },
});
if (response.status === 404) return null;
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] ${method} ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
if (!text) return null;
try { return JSON.parse(text); } catch { return text; }
}
}
+57 -13
View File
@@ -12,6 +12,7 @@ import { Redis } from 'ioredis';
import { Logger } from 'tslog';
import * as os from 'os';
import type { IKVClient, KeyValuePair } from './IKVClient.js';
import { MetaCoreApiClient } from './MetaCoreApiClient.js';
const logger = new Logger({ name: 'RedisClient' });
@@ -74,7 +75,6 @@ export interface FileMetadata {
// Title information (for folder organization)
title?: string;
titles?: { eng?: string; [key: string]: string | undefined };
originalTitle?: string;
fileName?: string;
@@ -104,15 +104,27 @@ export interface RedisClientConfig {
prefix?: string;
filesVolume?: string;
reconnectInterval?: number;
/**
* meta-core HTTP API base URL. When set, read methods (getProperty,
* getProperties, getMetadataFlat, getAllHashIds, health) route through
* the /meta/{hash} HTTP family instead of touching Redis directly.
*
* This is the meta-fuse half of the api-mediated-access lockdown (see
* meta-core docs/api-mediated-access.md). Reads stay on Redis when
* left unset so tests / one-off scripts still work.
*/
metaCoreApiUrl?: string;
}
export class RedisClient implements Partial<IKVClient> {
private client: Redis | null = null;
private subscriber: Redis | null = null;
private config: Required<RedisClientConfig>;
private config: Required<Omit<RedisClientConfig, 'metaCoreApiUrl'>> & Pick<RedisClientConfig, 'metaCoreApiUrl'>;
private isConnected = false;
private reconnectTimer: NodeJS.Timeout | null = null;
private subscriptions: Map<string, (message: string) => void> = new Map();
private apiClient: MetaCoreApiClient | null = null;
// Stream consumer state
private streamConsumerRunning = false;
@@ -125,20 +137,33 @@ export class RedisClient implements Partial<IKVClient> {
prefix: config.prefix ?? process.env.REDIS_PREFIX ?? '',
filesVolume: config.filesVolume ?? process.env.FILES_VOLUME ?? '/files',
reconnectInterval: config.reconnectInterval ?? 5000,
metaCoreApiUrl: config.metaCoreApiUrl,
};
if (this.config.metaCoreApiUrl) {
this.apiClient = new MetaCoreApiClient({ apiUrl: this.config.metaCoreApiUrl });
logger.info(`Reads routed through meta-core API at ${this.config.metaCoreApiUrl}`);
}
// Unique consumer name for this instance
this.consumerName = `${os.hostname()}-${process.pid}`;
}
/**
* Connect to Redis
* Connect to Redis. In HTTP-only mode (no URL + apiClient set) this is
* a no-op — reads route through MetaCoreApiClient and stream
* consumption is owned by SSEEventClient at the index.ts layer.
*/
async connect(): Promise<void> {
if (this.client && this.isConnected) {
return;
}
if (this.apiClient && !this.config.url) {
logger.info('HTTP-only mode — no direct Redis connection');
return;
}
try {
this.client = new Redis(this.config.url, {
retryStrategy: (times: number) => {
@@ -236,9 +261,18 @@ export class RedisClient implements Partial<IKVClient> {
}
/**
* Check if connected
* Check if the metadata backend is reachable.
*
* In HTTP-only mode (api-mediated-access PR D) there's no Redis socket;
* reachability is signalled by having an apiClient configured. The
* health badge is keyed off this; reporting `false` there would make
* the UI permanently show "Disconnected" even though the service is
* working fine over HTTP.
*/
get connected(): boolean {
if (this.apiClient && !this.config.url) {
return true;
}
return this.isConnected;
}
@@ -255,6 +289,15 @@ export class RedisClient implements Partial<IKVClient> {
* @returns The property value or null if not found
*/
async getProperty(hashId: string, property: string): Promise<string | null> {
if (this.apiClient) {
try {
return await this.apiClient.getProperty(hashId, property);
} catch (error: any) {
logger.error(`HTTP getProperty failed for ${hashId}/${property}:`, error.message);
return null;
}
}
if (!this.client || !this.isConnected) {
return null;
}
@@ -579,12 +622,6 @@ export class RedisClient implements Partial<IKVClient> {
mtime = isNaN(parsed) ? parseFloat(data.mtime) : parsed;
}
// Parse titles - can be nested object or simple string
let titles: { eng?: string; [key: string]: string | undefined } | undefined;
if (data['titles/eng']) {
titles = { eng: data['titles/eng'] };
}
// Parse season/episode (can be 0 for specials)
const season = data.season !== undefined ? parseInt(data.season, 10) : undefined;
const episode = data.episode !== undefined ? parseInt(data.episode, 10) : undefined;
@@ -603,12 +640,19 @@ export class RedisClient implements Partial<IKVClient> {
mtime,
ctime: data.ctime ? parseFloat(data.ctime) : mtime,
// Identification
hashId: hashId ?? data.cid_midhash256 ?? data.hashId,
// Identification.
//
// `hashId` is always supplied by the caller — every `parseMetadata`
// call site derives it from the Redis key (`file:<hashId>/…`), so the
// fallbacks below it were already unreachable. The two that used to sit
// here (`data.cid_midhash256`, then `data.hashId`) were also the wrong
// shape: `cid_*` fields no longer exist on records, and meta-core
// rejects them at the write boundary. Kept as a single defensive `??`
// rather than a chain of dead ones. See METADATA_KEYS.md §14.13.
hashId: hashId ?? data.hashId,
// Title information
title: data.title,
titles,
originalTitle: data.originalTitle,
fileName: data.fileName,
@@ -0,0 +1,229 @@
/**
* SSEEventClient — HTTP Server-Sent Events client for meta-core event streams.
*
* Replaces direct Redis Streams consumption. See the same-named module in
* meta-sort for the design rationale (meta-core docs/api-mediated-access.md).
*/
import { promises as fs } from 'fs';
import { dirname } from 'path';
export interface SSEEvent {
id: string;
event: string;
data: any;
}
export interface SSEClientOptions {
url: string;
cursorPath: string | null;
onEvent: (e: SSEEvent) => Promise<void> | void;
onGap?: (payload: { requested: string; resumeFrom: string; reason: string }) => Promise<void> | void;
logTag?: string;
reconnectBaseMs?: number;
reconnectMaxMs?: number;
flushIntervalMs?: number;
/** When set, sent as Last-Event-ID on the *first* connect (e.g. to
* force replay from a specific point). After the first event the
* persisted cursor takes over. */
initialCursor?: string;
}
export class SSEEventClient {
private readonly opts: Required<Omit<SSEClientOptions, 'onGap' | 'initialCursor'>> & Pick<SSEClientOptions, 'onGap' | 'initialCursor'>;
private cursor: string = '';
private cursorDirty = false;
private abort: AbortController | null = null;
private flushTimer: NodeJS.Timeout | null = null;
private stopped = false;
private backoff = 0;
private firstConnect = true;
constructor(options: SSEClientOptions) {
this.opts = {
url: options.url,
cursorPath: options.cursorPath,
onEvent: options.onEvent,
onGap: options.onGap,
logTag: options.logTag ?? '[SSE]',
reconnectBaseMs: options.reconnectBaseMs ?? 500,
reconnectMaxMs: options.reconnectMaxMs ?? 30_000,
flushIntervalMs: options.flushIntervalMs ?? 2000,
initialCursor: options.initialCursor,
};
this.backoff = this.opts.reconnectBaseMs;
}
async start(): Promise<void> {
await this.loadCursor();
if (!this.cursor && this.opts.initialCursor) {
this.cursor = this.opts.initialCursor;
}
this.startFlushTimer();
await this.runLoop();
}
async stop(): Promise<void> {
this.stopped = true;
if (this.abort) {
this.abort.abort();
this.abort = null;
}
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = null;
}
await this.flushCursor();
}
getCursor(): string {
return this.cursor;
}
private async loadCursor(): Promise<void> {
if (!this.opts.cursorPath) return;
try {
const data = await fs.readFile(this.opts.cursorPath, 'utf-8');
const trimmed = data.trim();
if (trimmed) {
this.cursor = trimmed;
console.log(`${this.opts.logTag} Resuming from cursor ${this.cursor}`);
}
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.warn(`${this.opts.logTag} Could not read cursor file ${this.opts.cursorPath}: ${err.message}`);
}
}
}
private startFlushTimer(): void {
this.flushTimer = setInterval(() => {
if (this.cursorDirty) {
this.flushCursor().catch(err =>
console.warn(`${this.opts.logTag} Cursor flush failed: ${err.message}`)
);
}
}, this.opts.flushIntervalMs);
}
private async flushCursor(): Promise<void> {
if (!this.opts.cursorPath || !this.cursorDirty) return;
this.cursorDirty = false;
try {
await fs.mkdir(dirname(this.opts.cursorPath), { recursive: true });
await fs.writeFile(this.opts.cursorPath, this.cursor, 'utf-8');
} catch (err: any) {
this.cursorDirty = true;
throw err;
}
}
private async runLoop(): Promise<void> {
while (!this.stopped) {
try {
await this.connect();
this.backoff = this.opts.reconnectBaseMs;
} catch (err: any) {
if (this.stopped) return;
console.warn(`${this.opts.logTag} Stream error, reconnecting in ${this.backoff}ms: ${err?.message ?? err}`);
await this.sleep(this.backoff);
this.backoff = Math.min(this.backoff * 2, this.opts.reconnectMaxMs);
}
}
}
private async connect(): Promise<void> {
this.abort = new AbortController();
const headers: Record<string, string> = {
Accept: 'text/event-stream',
'Cache-Control': 'no-cache',
};
if (this.cursor) {
headers['Last-Event-ID'] = this.cursor;
}
this.firstConnect = false;
const response = await fetch(this.opts.url, {
method: 'GET',
headers,
signal: this.abort.signal,
});
if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
if (!response.body) throw new Error('Response has no body');
const reader = response.body.getReader();
const decoder = new TextDecoder('utf-8');
let buffer = '';
try {
while (!this.stopped) {
const { value, done } = await reader.read();
if (done) return;
buffer += decoder.decode(value, { stream: true });
let idx: number;
while ((idx = buffer.indexOf('\n\n')) !== -1) {
const rawChunk = buffer.slice(0, idx);
buffer = buffer.slice(idx + 2);
await this.handleChunk(rawChunk);
}
}
} finally {
try { reader.releaseLock(); } catch { /* nothing useful */ }
}
}
private async handleChunk(chunk: string): Promise<void> {
let id = '';
let event = '';
const dataLines: string[] = [];
for (const rawLine of chunk.split('\n')) {
if (!rawLine || rawLine.startsWith(':')) continue;
const colon = rawLine.indexOf(':');
if (colon === -1) continue;
const field = rawLine.slice(0, colon);
let value = rawLine.slice(colon + 1);
if (value.startsWith(' ')) value = value.slice(1);
switch (field) {
case 'id': id = value; break;
case 'event': event = value; break;
case 'data': dataLines.push(value); break;
}
}
if (dataLines.length === 0) return;
let data: any;
try {
data = JSON.parse(dataLines.join('\n'));
} catch (err: any) {
console.warn(`${this.opts.logTag} Skipping unparseable event ${id || '(no id)'}: ${err.message}`);
return;
}
const ev: SSEEvent = { id, event: event || 'message', data };
if (ev.event === 'gap') {
if (this.opts.onGap) {
await this.opts.onGap(ev.data);
} else {
console.warn(`${this.opts.logTag} Gap event (cursor ${ev.data?.requested} trimmed): resumed from ${ev.data?.resumeFrom}`);
}
if (typeof ev.data?.resumeFrom === 'string' && ev.data.resumeFrom) {
this.cursor = ev.data.resumeFrom;
this.cursorDirty = true;
}
return;
}
try {
await this.opts.onEvent(ev);
} catch (err: any) {
console.error(`${this.opts.logTag} onEvent handler failed for ${ev.id}: ${err?.stack ?? err}`);
}
if (ev.id) {
this.cursor = ev.id;
this.cursorDirty = true;
}
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
@@ -13,7 +13,6 @@
*/
export interface MediaMeta {
// Title information
titles?: { eng?: string; [key: string]: string | undefined };
originalTitle?: string;
title?: string;
@@ -62,7 +61,8 @@ export function renamingRule(metadata: Partial<MediaMeta>, filepath?: string): s
}
// Get title - try various fields
const title = metadata.titles?.eng || metadata.originalTitle;
// The display name — never the `titles/*` name set (METADATA_KEYS.md §3).
const title = metadata.title || metadata.originalTitle;
if (!title) {
if (metadata.fileType !== 'torrent') {
// Don't warn for torrent files because they don't always have a title
@@ -174,12 +174,6 @@ export class VirtualFileSystem implements VFSUpdateCallback {
mtime = isNaN(parsed) ? parseFloat(props.mtime) : parsed;
}
// Parse titles - can be from nested paths like 'titles/eng'
let titles: { eng?: string; [key: string]: string | undefined } | undefined;
if (props['titles/eng']) {
titles = { eng: props['titles/eng'] };
}
// Parse season/episode
const season = props.season !== undefined ? parseInt(props.season, 10) : undefined;
const episode = props.episode !== undefined ? parseInt(props.episode, 10) : undefined;
@@ -212,7 +206,6 @@ export class VirtualFileSystem implements VFSUpdateCallback {
ctime: props.ctime ? parseFloat(props.ctime) : mtime,
hashId,
title: props.title,
titles,
originalTitle: props.originalTitle,
fileName,
season,
@@ -7,7 +7,7 @@
* Old behavior:
* - Only processes video, subtitle, torrent files
* - Requires extension
* - Title from titles.eng || originalTitle
* - Title from title || originalTitle
* - Files without title/extension are hidden (not in Unsorted)
*/
@@ -21,7 +21,7 @@ export type ConditionOperator =
*/
export interface Condition {
type: ConditionOperator;
field: string; // Metadata field path, e.g., "titles.eng", "season", "fileType"
field: string; // Metadata field path, e.g., "title", "season", "fileType"
value?: string | number | boolean; // Required for EQUALS, NOT_EQUALS, CONTAINS, MATCHES
}
@@ -83,7 +83,7 @@ export interface PreviewResponse {
*/
export interface TemplateVariable {
name: string; // Display name
path: string; // Field path (e.g., "titles.eng")
path: string; // Field path (e.g., "title")
type: 'string' | 'number' | 'boolean';
description: string;
example?: string;
@@ -93,7 +93,6 @@ export interface TemplateVariable {
* List of all available template variables
*/
export const TEMPLATE_VARIABLES: TemplateVariable[] = [
{ name: 'English Title', path: 'titles.eng', type: 'string', description: 'English title from metadata', example: 'Breaking Bad' },
{ name: 'Original Title', path: 'originalTitle', type: 'string', description: 'Original/native title', example: 'Breaking Bad' },
{ name: 'Title', path: 'title', type: 'string', description: 'Simple title field', example: 'Breaking Bad' },
{ name: 'Season', path: 'season', type: 'number', description: 'Season number', example: '1' },
@@ -16,7 +16,7 @@ interface RulesEditorProps {
// Available metadata fields for conditions
const AVAILABLE_FIELDS = [
{ value: 'fileType', label: 'File Type', type: 'string' },
{ value: 'titles.eng', label: 'English Title', type: 'string' },
{ value: 'title', label: 'Title', type: 'string' },
{ value: 'originalTitle', label: 'Original Title', type: 'string' },
{ value: 'season', label: 'Season', type: 'number' },
{ value: 'episode', label: 'Episode', type: 'number' },
@@ -53,7 +53,7 @@ function createEmptyRule(): RenamingRule {
function createEmptyCondition(): Condition {
return {
type: 'EXISTS',
field: 'titles.eng',
field: 'title',
};
}
@@ -197,7 +197,7 @@ function TemplateEditor({ template, onChange, variables }: TemplateEditorProps)
<textarea
value={template}
onChange={e => onChange(e.target.value)}
placeholder="e.g., Movies/{titles.eng|originalTitle}{movieYear? ({movieYear})}/{fileName}.{extension}"
placeholder="e.g., Movies/{title|originalTitle}{movieYear? ({movieYear})}/{fileName}.{extension}"
rows={3}
className="template-input"
/>