9 Commits
Author SHA1 Message Date
Worph af495c4ad4 bugfix meshdisco
Build and Push Docker Image / build (push) Failing after 14s
2026-09-14 11:55:17 +02:00
Worph daf464261d replace file discovery by udp discovery
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 11:25:03 +02:00
Worph 2c485d5616 change service discovery 2026-09-13 19:49:11 +02:00
Worph d35a64a744 align with metadata standard
Build and Push Docker Image / build (push) Failing after 15s
2026-09-13 15:57:12 +02:00
Worph 9061c7a37d gitattributes 2026-07-14 16:08:49 +02:00
Pierre b100b3a77c improve doc + bugfix 2026-05-25 23:47:48 +02:00
Pierre 8e50b85e08 bugfix 2026-05-24 17:43:31 +02:00
Pierre a03aa35169 adaptation related to meta-core api changes 2026-05-24 17:26:13 +02:00
Pierre 797cf98afd ignore file 2026-04-28 12:20:57 +02:00
23 changed files with 1463 additions and 650 deletions
+6
View File
@@ -0,0 +1,6 @@
# FIX CRLF always when developer has not set
# Linux/Mac: git config --global core.autocrlf input
# Windows: git config --global core.autocrlf true
# Auto detect text files and perform LF normalization
* text=auto
* eol=lf
+1
View File
@@ -25,3 +25,4 @@ logs/
# Runtime data
data/
docker/__pycache__/
+1 -1
View File
@@ -39,7 +39,7 @@ COPY packages/meta-fuse-core/src/ ./src/
RUN npm run build
# Stage 3: Build FUSE Driver
FROM rust:1.86-bookworm AS fuse-builder
FROM rust:1.89-bookworm AS fuse-builder
WORKDIR /build
+11 -11
View File
@@ -84,7 +84,7 @@ Meta-Fuse is a read-only process in the MetaMesh ecosystem that:
│ [3] meta-stremio (PROCESS-READ) │
│ └─► Reads metadata from KV, streams media content │
│ │
│ [4] meta-orbit (SHARING-READ-WRITE) │
│ [4] meta-share (SHARING-READ-WRITE) │
│ └─► P2P metadata sync across network │
│ │
└────────────────────────────────────────────────────────────────────────────┘
@@ -104,7 +104,7 @@ http://meta-core:9000
/urls API Response (JSON):
{
"hostname": "meta-core-dev",
"hostname": "metacore-app",
"baseUrl": "http://localhost:8083",
"apiUrl": "http://meta-core:9000",
"redisUrl": "redis://meta-core:6379",
@@ -606,10 +606,10 @@ journalctl -u meta-fuse-driver -f
cat /meta-core/locks/kv-leader.info
# Check /urls API
curl -k https://dev.localhost:8083/api/urls
curl -k https://metacore-dev.localhost:8083/api/urls
# Verify redis is running (on leader)
docker exec meta-core-dev redis-cli ping
docker exec metacore-app redis-cli ping
# Check meta-fuse logs
docker logs meta-fuse | grep "KV"
@@ -632,10 +632,10 @@ docker logs meta-fuse | grep "wsgidav"
```bash
# Check Redis has file data (flat key format)
docker exec meta-core-dev redis-cli keys "file:*" | head -20
docker exec metacore-app redis-cli keys "file:*" | head -20
# Check file index
docker exec meta-core-dev redis-cli scard "file:__index__"
docker exec metacore-app redis-cli scard "file:__index__"
# Verify source files exist
ls -la /files/watch/
@@ -644,7 +644,7 @@ ls -la /files/watch/
curl http://localhost:3000/api/fuse/stats
# Check streaming state builder stats
docker logs meta-fuse-dev | grep "State builder"
docker logs metafuse-app | grep "State builder"
```
## Technology Stack
@@ -665,7 +665,7 @@ Meta-Fuse is designed to work seamlessly with other MetaMesh services:
- **meta-sort**: Provides metadata in KV database
- **meta-stremio**: Can stream files exposed by meta-fuse
- **meta-orbit**: Syncs metadata across P2P network
- **meta-share**: Syncs metadata across P2P network
### Connecting to meta-sort
@@ -673,13 +673,13 @@ Ensure meta-sort is running and has processed files:
```bash
# Check meta-sort processing status
curl -k https://dev.localhost:8180/api/processing/status
curl -k https://metasort-dev.localhost:8180/api/processing/status
# Verify Redis has metadata (via meta-core)
docker exec meta-core-dev redis-cli scard "file:__index__"
docker exec metacore-app redis-cli scard "file:__index__"
# Check meta:events stream has events
docker exec meta-core-dev redis-cli xlen "meta:events"
docker exec metacore-app redis-cli xlen "meta:events"
```
Meta-fuse will automatically discover the KV database through the shared lock file at `/meta-core/locks/kv-leader.info`.
+7 -7
View File
@@ -535,29 +535,29 @@ List all discovered MetaMesh services.
"services": [
{
"name": "meta-sort",
"url": "http://meta-sort-dev",
"api": "http://meta-sort-dev/api",
"url": "http://metasort-app",
"api": "http://metasort-app/api",
"status": "healthy",
"role": "writer"
},
{
"name": "meta-fuse",
"url": "http://meta-fuse-dev",
"api": "http://meta-fuse-dev/api",
"url": "http://metafuse-app",
"api": "http://metafuse-app/api",
"status": "healthy",
"role": "reader"
},
{
"name": "meta-stremio",
"url": "http://meta-stremio-dev",
"api": "http://meta-stremio-dev/api",
"url": "http://metastremio-app",
"api": "http://metastremio-app/api",
"status": "healthy",
"role": "reader"
}
],
"current": "meta-fuse",
"leader": {
"hostname": "meta-core-dev",
"hostname": "metacore-app",
"baseUrl": "http://localhost:8083",
"apiUrl": "http://meta-core:9000",
"redisUrl": "redis://meta-core:6379",
+1 -1
View File
@@ -350,7 +350,7 @@ The `onFileComplete` callback is called when:
```
[meta-fuse] Starting meta-fuse...
[KVManager] Waiting for leader...
[LeaderClient] Leader found: meta-core-dev at redis://meta-core:6379
[LeaderClient] Leader found: metacore-app at redis://meta-core:6379
[meta-fuse] KV Manager ready, initializing VFS with streaming bootstrap...
[StreamingStateBuilder] Initialized with 12 VFS-relevant properties
[meta-fuse] Replaying meta:events stream from position 0...
+20 -29
View File
@@ -91,9 +91,9 @@ export class APIServer {
this.app.post('/api/fuse/rules/validate', this.handleValidateRule.bind(this));
this.app.get('/api/fuse/rules/variables', this.handleGetVariables.bind(this));
// Service discovery — proxies to meta-core's /api/services, resolving the
// current leader's URL via LeaderClient (no hardcoded hostname).
this.app.get('/api/services', this.handleServices.bind(this));
// meta-discovery v1: neighbours heard over UDP, from this service's own
// map — no meta-core needed, so the nav renders even when the core is down.
this.app.get('/api/neighbors', this.handleNeighbors.bind(this));
// WebDAV access tokens (per-device basic-auth passwords for /webdav).
this.app.get('/api/webdav-tokens', this.handleListTokens.bind(this));
@@ -134,36 +134,27 @@ export class APIServer {
}
/**
* Proxy handler that forwards /api/services to meta-core's canonical
* registry endpoint using LeaderClient for discovery.
* meta-discovery v1 neighbours, served from this service's own UDP map.
* `services` is an alias kept while older dashboard builds are around.
*/
private async handleServices(_req: FastifyRequest, reply: FastifyReply): Promise<void> {
try {
const leaderClient = this.kvManager?.getLeaderClient();
if (!leaderClient) {
reply.status(503).send({ error: 'KVManager not initialized' });
return;
}
const apiUrl = await leaderClient.getApiUrl();
if (!apiUrl) {
reply.status(503).send({ error: 'Leader not available yet' });
return;
}
const response = await fetch(`${apiUrl}/api/services`, {
signal: AbortSignal.timeout(5000),
});
if (!response.ok) {
reply.status(response.status).send({
error: `Upstream ${apiUrl} returned ${response.status}`,
});
return;
}
reply.send(await response.json());
} catch (err: any) {
reply.status(502).send({ error: err?.message ?? String(err) });
private async handleNeighbors(_request: FastifyRequest, reply: FastifyReply): Promise<void> {
const leaderClient = this.kvManager?.getLeaderClient();
if (!leaderClient) {
void reply.send({ current: 'meta-fuse', enabled: false, count: 0, neighbors: [] });
return;
}
const neighbors = leaderClient.getNeighbors();
void reply.send({
current: 'meta-fuse',
enabled: true,
count: neighbors.length,
neighbors,
services: neighbors,
self: leaderClient.self(),
});
}
/**
* Start the server
*/
@@ -0,0 +1,441 @@
/**
* meta-discovery v1 — UDP multicast service discovery.
*
* Replaces the two file-based mechanisms that required a shared /meta-core
* volume: `locks/kv-leader.info` (locating meta-core at boot) and
* `services/<name>-<host>.json` (the dashboard nav registry). Both answered the
* same question — where is meta-core, and who else is on this network — so one
* announce packet answers both, and discovery scope becomes the docker network
* rather than the mounted volume.
*
* The normative spec is docs/project-architecture/service-discovery.md.
*
* ⚠ MIRRORED FILE. Byte-identical copies live in meta-sort-core, meta-fuse-core
* and meta-dup-core. They cannot share a package: meta-fuse-core and
* meta-dup-core do not depend on @metazla/meta-interface at all, and each
* submodule is its own Docker build context, so a workspace dep resolves
* outside the build. Edit one copy → edit all three, and
* `scripts/check-mirrors.sh` will tell you if you forgot.
*/
import dgram from 'dgram';
import { hostname, networkInterfaces } from 'os';
export const PROTOCOL_VERSION = 1;
export const DEFAULT_GROUP = '239.255.77.1';
export const DEFAULT_PORT = 9399;
export const DEFAULT_INTERVAL_MS = 10_000;
/** A neighbour unseen for interval × this is dropped. Replaces the reaper. */
export const LIVENESS_FACTOR = 3;
export const TYPE_DISCOVERY = 'discovery';
export const TYPE_ANNOUNCE = 'announce';
export const ROLE_CORE = 'core';
export const ROLE_SERVICE = 'service';
/** Mirrors meta-core's GET /urls response minus redisUrl (retired). */
export interface MeshUrls {
hostname: string;
baseUrl: string;
apiUrl: string;
webdavUrl: string;
webdavUrlInternal: string;
}
export interface MeshMessage {
v: number;
type: string;
name?: string;
instance?: string;
role?: string;
version?: string;
status?: string;
baseUrl?: string;
/** Present only when role === 'core'. */
urls?: MeshUrls;
/** Reserved for a future HMAC; v1 ignores it. */
token?: string;
}
export interface MeshNeighbor extends MeshMessage {
/** Source address taken from the packet, never from the payload. */
addr: string;
lastSeen: number;
}
export interface MeshNodeConfig {
name: string;
instance?: string;
role?: string;
version?: string;
group?: string;
port?: number;
intervalMs?: number;
enabled?: boolean;
/** Rebuilt on every announce so the payload never goes stale. */
payload?: () => { baseUrl?: string; status?: string; urls?: MeshUrls };
}
/** IPv4 addresses of every up, non-internal interface. */
function localInterfaceAddresses(): string[] {
const out: string[] = [];
const nets = networkInterfaces();
for (const name of Object.keys(nets)) {
for (const ni of nets[name] ?? []) {
// Node has flip-flopped between 'IPv4' and 4 for `family`.
const isV4 = (ni as { family: string | number }).family === 'IPv4' ||
(ni as { family: string | number }).family === 4;
if (isV4 && !ni.internal && ni.address) out.push(ni.address);
}
}
return out;
}
/** First non-internal IPv4 address, or the hostname. Matches the Go helper. */
export function localIPv4(): string {
return localInterfaceAddresses()[0] ?? hostname();
}
/**
* One participant in the mesh: announces itself, answers probes, and keeps a
* TTL map of everyone else.
*/
export class MeshNode {
private cfg: Required<Omit<MeshNodeConfig, 'payload'>> & Pick<MeshNodeConfig, 'payload'>;
private socket: dgram.Socket | null = null;
private timer: NodeJS.Timeout | null = null;
private neighbors = new Map<string, MeshNeighbor>();
private ifaceAddrs: string[] = [];
private listeners: ((n: MeshNeighbor) => void)[] = [];
private started = false;
constructor(config: MeshNodeConfig) {
this.cfg = {
name: config.name,
instance: config.instance ?? hostname(),
role: config.role ?? ROLE_SERVICE,
version: config.version ?? '',
group: config.group ?? DEFAULT_GROUP,
port: config.port ?? DEFAULT_PORT,
intervalMs: config.intervalMs ?? DEFAULT_INTERVAL_MS,
enabled: config.enabled ?? true,
payload: config.payload,
};
}
async start(): Promise<void> {
if (this.started || !this.cfg.enabled) {
if (!this.cfg.enabled) console.log('[meshdisco] Disabled by configuration');
return;
}
this.started = true;
await new Promise<void>((resolve) => {
const socket = dgram.createSocket({ type: 'udp4', reuseAddr: true });
this.socket = socket;
socket.on('message', (buf, rinfo) => this.onMessage(buf, rinfo));
socket.on('error', (err) => {
// Never fatal: a host that blocks multicast must degrade, not
// take the service down.
console.error('[meshdisco] Socket error:', err.message);
});
socket.bind(this.cfg.port, '0.0.0.0', () => {
try {
socket.setMulticastTTL(1); // link-local only
} catch (e) {
console.warn('[meshdisco] Could not set multicast TTL:', e);
}
// Join on EVERY interface. A container on two docker networks
// that joins only the default route silently sees one network.
this.ifaceAddrs = localInterfaceAddresses();
let joined = 0;
for (const addr of this.ifaceAddrs) {
try {
socket.addMembership(this.cfg.group, addr);
joined++;
} catch (e) {
console.warn(`[meshdisco] Join ${this.cfg.group} on ${addr} failed:`, e);
}
}
console.log(
`[meshdisco] Listening on ${this.cfg.group}:${this.cfg.port} as ` +
`${this.cfg.name}/${this.cfg.instance} (joined ${joined} of ${this.ifaceAddrs.length} interfaces)`
);
resolve();
});
});
this.announce();
this.probe();
this.timer = setInterval(() => this.announce(), this.cfg.intervalMs);
// Don't hold the event loop open on shutdown.
this.timer.unref?.();
}
async stop(): Promise<void> {
if (!this.started) return;
this.started = false;
if (this.timer) { clearInterval(this.timer); this.timer = null; }
// Final "stopping" announce so neighbours drop us immediately rather
// than waiting out the staleness window.
this.announce('stopping');
await new Promise<void>((resolve) => {
if (!this.socket) return resolve();
try { this.socket.close(() => resolve()); } catch { resolve(); }
});
this.socket = null;
this.listeners = [];
}
/** Multicast a probe; every listener replies immediately. */
probe(): void {
this.send({ v: PROTOCOL_VERSION, type: TYPE_DISCOVERY });
}
/** Everyone heard from inside the staleness window, sorted by name. */
getNeighbors(): MeshNeighbor[] {
const cutoff = Date.now() - this.cfg.intervalMs * LIVENESS_FACTOR;
const out: MeshNeighbor[] = [];
for (const [key, nb] of this.neighbors) {
if (nb.lastSeen < cutoff) { this.neighbors.delete(key); continue; }
out.push(nb);
}
return out.sort((a, b) => (a.name ?? '').localeCompare(b.name ?? ''));
}
/** One row per service name, most recently seen instance wins. */
getNeighborsByName(): MeshNeighbor[] {
const best = new Map<string, MeshNeighbor>();
for (const nb of this.getNeighbors()) {
const cur = best.get(nb.name ?? '');
if (!cur || nb.lastSeen > cur.lastSeen) best.set(nb.name ?? '', nb);
}
return [...best.values()].sort((a, b) => (a.name ?? '').localeCompare(b.name ?? ''));
}
/** Cores only — what a locator filters on. */
getCores(): MeshNeighbor[] {
return this.getNeighbors().filter((n) => n.role === ROLE_CORE && n.urls);
}
/** This node's own announce, so a UI can render itself without waiting. */
self(): MeshNeighbor {
return { ...this.buildAnnounce('running'), addr: '', lastSeen: Date.now() };
}
/** Fires for every announce received (after the self-echo filter). */
onAnnounce(cb: (n: MeshNeighbor) => void): void {
this.listeners.push(cb);
}
private onMessage(buf: Buffer, rinfo: dgram.RemoteInfo): void {
let msg: MeshMessage;
try {
msg = JSON.parse(buf.toString('utf8')) as MeshMessage;
} catch {
return; // foreign traffic on the group
}
if (msg.v !== PROTOCOL_VERSION) return;
// Drop our own multicast echo.
if (msg.instance && msg.instance === this.cfg.instance && msg.name === this.cfg.name) return;
if (msg.type === TYPE_DISCOVERY) {
this.replyTo(rinfo);
return;
}
if (msg.type !== TYPE_ANNOUNCE || !msg.name) return;
const key = msg.instance ? `${msg.name}|${msg.instance}` : msg.name;
if (msg.status === 'stopping') { this.neighbors.delete(key); return; }
const nb: MeshNeighbor = { ...msg, addr: rinfo.address, lastSeen: Date.now() };
this.neighbors.set(key, nb);
for (const cb of this.listeners) {
try { cb(nb); } catch (e) { console.error('[meshdisco] Listener threw:', e); }
}
}
private buildAnnounce(status: string): MeshMessage {
const msg: MeshMessage = {
v: PROTOCOL_VERSION,
type: TYPE_ANNOUNCE,
name: this.cfg.name,
instance: this.cfg.instance,
role: this.cfg.role,
version: this.cfg.version,
status,
};
if (this.cfg.payload) {
const p = this.cfg.payload();
if (p.baseUrl) msg.baseUrl = p.baseUrl;
if (p.status && status === 'running') msg.status = p.status;
// Only a core may carry a URLs block — a service announcing one
// would let any container impersonate meta-core.
if (this.cfg.role === ROLE_CORE && p.urls) msg.urls = p.urls;
}
return msg;
}
private announce(status = 'running'): void {
this.send(this.buildAnnounce(status));
}
private replyTo(rinfo: dgram.RemoteInfo): void {
const body = Buffer.from(JSON.stringify(this.buildAnnounce('running')));
try {
this.socket?.send(body, rinfo.port, rinfo.address);
} catch (e) {
console.warn('[meshdisco] Reply failed:', e);
}
}
/** Writes once per interface — the default route alone reaches one network. */
private send(msg: MeshMessage): void {
const socket = this.socket;
if (!socket) return;
const body = Buffer.from(JSON.stringify(msg));
for (const addr of this.ifaceAddrs) {
try {
socket.setMulticastInterface(addr);
socket.send(body, this.cfg.port, this.cfg.group);
} catch (e) {
console.warn(`[meshdisco] Send on ${addr} failed:`, e);
}
}
}
}
export interface MetaCoreLocatorConfig {
/** This service's name, for its own announce. */
serviceName: string;
/** Browser-facing URL for the nav menu. */
baseUrl?: string;
version?: string;
/**
* Explicit meta-core API URL. When set this ALWAYS wins and the wire is
* never consulted for core selection — see the spec's "the pin always
* wins". This is what stops a client box from latching onto a gateway
* box's core when both are reachable on a shared network.
*/
metaCoreUrl?: string;
group?: string;
port?: number;
intervalMs?: number;
enabled?: boolean;
}
/**
* Locates meta-core over UDP, replacing the read of
* /meta-core/locks/kv-leader.info.
*/
export class MetaCoreLocator {
private node: MeshNode;
private cfg: MetaCoreLocatorConfig;
private current: MeshUrls | null = null;
private changeCallbacks: (() => void)[] = [];
constructor(config: MetaCoreLocatorConfig) {
this.cfg = config;
this.node = new MeshNode({
name: config.serviceName,
role: ROLE_SERVICE,
version: config.version,
group: config.group,
port: config.port,
intervalMs: config.intervalMs,
enabled: config.enabled,
payload: () => ({ baseUrl: config.baseUrl ?? `http://${localIPv4()}` }),
});
this.node.onAnnounce((nb) => {
if (nb.role !== ROLE_CORE || !nb.urls) return;
if (this.cfg.metaCoreUrl) return; // pinned: the wire cannot move us
const prev = this.current;
if (!prev) {
this.current = nb.urls;
console.log(`[meshdisco] meta-core discovered at ${nb.urls.apiUrl} (${nb.addr})`);
this.notifyChange();
return;
}
if (prev.apiUrl === nb.urls.apiUrl) {
this.current = nb.urls; // refresh the rest of the fields
return;
}
// A second, different core is announcing. Do not flap between
// them — keep the first and say so loudly, because on a PCS box
// this means a client container can see the gateway's core.
console.warn(
`[meshdisco] Ignoring a second meta-core at ${nb.urls.apiUrl} (${nb.addr}); ` +
`staying with ${prev.apiUrl}. Set META_CORE_URL to pin this explicitly.`
);
});
}
async start(): Promise<void> {
if (this.cfg.metaCoreUrl) {
console.log(`[meshdisco] meta-core pinned to ${this.cfg.metaCoreUrl}; discovery is advisory`);
}
await this.node.start();
}
async stop(): Promise<void> {
await this.node.stop();
this.changeCallbacks = [];
}
/** The pin if set, else whatever discovery elected. */
getApiUrl(): string | null {
if (this.cfg.metaCoreUrl) return this.cfg.metaCoreUrl;
return this.current?.apiUrl ?? null;
}
/** Full URLs block. null when pinned and /urls has not been fetched yet. */
getUrls(): MeshUrls | null {
return this.current;
}
/**
* Block until meta-core is located. Probes immediately and re-probes every
* 500ms — the same loop shape as the old LeaderClient.waitForLeader, so the
* 30s boot timeout semantics are preserved exactly.
*/
async waitForCore(timeoutMs = 30_000): Promise<string> {
const deadline = Date.now() + timeoutMs;
let logged = false;
while (Date.now() < deadline) {
const url = this.getApiUrl();
if (url) return url;
if (!logged) { console.log('[meshdisco] Waiting for meta-core...'); logged = true; }
this.node.probe();
await new Promise((r) => setTimeout(r, 500));
}
throw new Error(`[meshdisco] No meta-core found within ${timeoutMs}ms`);
}
/** Fires when the elected core's apiUrl changes. Replaces the fs.watch. */
onChange(cb: () => void): void {
this.changeCallbacks.push(cb);
}
getNeighbors(): MeshNeighbor[] {
return this.node.getNeighborsByName();
}
self(): MeshNeighbor {
return this.node.self();
}
probe(): void {
this.node.probe();
}
private notifyChange(): void {
for (const cb of this.changeCallbacks) {
try { cb(); } catch (e) { console.error('[meshdisco] Change callback threw:', e); }
}
}
}
+51 -27
View File
@@ -11,6 +11,7 @@ import 'dotenv/config';
import { Logger } from 'tslog';
import { KVManager } from './kv/KVManager.js';
import { RedisClient, StreamMessage } from './kv/RedisClient.js';
import { SSEEventClient, type SSEEvent } from './kv/SSEEventClient.js';
import { VirtualFileSystem } from './vfs/VirtualFileSystem.js';
import { StreamingStateBuilder } from './vfs/StreamingStateBuilder.js';
import { APIServer } from './api/APIServer.js';
@@ -69,6 +70,9 @@ async function main(): Promise<void> {
serviceName: 'meta-fuse',
apiPort: config.apiPort,
baseUrl: config.baseUrl,
// Pin: when META_CORE_URL is set, UDP discovery never overrides it.
// See docs/project-architecture/service-discovery.md.
metaCoreUrl: process.env.META_CORE_URL,
redisPrefix: config.redisPrefix,
});
@@ -76,13 +80,16 @@ async function main(): Promise<void> {
let vfs: VirtualFileSystem | null = null;
let stateBuilder: StreamingStateBuilder | null = null;
let apiServer: APIServer | null = null;
let sseClient: SSEEventClient | null = null;
// Graceful shutdown
const shutdown = async (signal: string): Promise<void> => {
logger.info(`Received ${signal}, shutting down...`);
try {
// Stop stream consumer first
// Stop SSE consumer first
if (sseClient) await sseClient.stop();
// Stop legacy stream consumer if it ever ran
if (redisClient) {
redisClient.stopStreamConsumer();
}
@@ -155,35 +162,52 @@ async function main(): Promise<void> {
// This enables vfs.refresh() to replay the event stream
vfs.setStreamingStateBuilder(stateBuilder);
// Bootstrap: Replay meta:events stream from position 0
// This rebuilds state from all historical events
logger.info(`Replaying ${EVENTS_STREAM} stream from position 0 (streaming bootstrap)...`);
const bootstrapStart = Date.now();
const lastId = await redisClient.replayStream(
EVENTS_STREAM,
'0',
async (message: StreamMessage) => {
// Bootstrap + live stream via SSE on /api/events/meta.
//
// We persist the cursor; on a clean restart we resume from where
// we left off. On a `gap` event (cursor trimmed out of retention)
// we fall back to replaying from the oldest available entry,
// which is good enough to rebuild VFS state because every property
// change has been republished into the stream by meta-core.
const apiUrl = leaderInfo?.apiUrl;
if (!apiUrl) {
throw new Error('meta-core apiUrl not available from leader info; cannot start SSE consumer');
}
logger.info(`Starting SSE consumer for /api/events/meta (apiUrl=${apiUrl})...`);
sseClient = new SSEEventClient({
url: `${apiUrl.replace(/\/+$/, '')}/api/events/meta`,
// Deliberately NOT persisting the cursor. meta-fuse holds
// VFS state purely in memory, so each restart needs to
// rebuild from a full event replay. Setting cursorPath to
// null + initialCursor to "0-0" forces the gap-handling
// path which resumes from the oldest available stream
// entry — i.e. effectively a full bootstrap.
cursorPath: null,
logTag: '[meta-fuse SSE]',
initialCursor: '0-0',
onEvent: async (e: SSEEvent) => {
// Translate SSE event back into the StreamMessage shape
// StreamingStateBuilder expects.
const message: StreamMessage = {
id: e.id,
type: e.event as any,
key: e.data?.key,
path: e.data?.path,
size: e.data?.size,
midhash256: e.data?.midhash256,
oldPath: e.data?.oldPath,
watcherId: e.data?.watcherId,
timestamp: e.data?.timestamp ?? e.data?.ts,
} as StreamMessage;
await handleStreamingEvent(message, stateBuilder!);
},
100 // Batch size
);
const bootstrapTime = Date.now() - bootstrapStart;
const stats = vfs.getStats();
logger.info(`Streaming bootstrap complete: ${stats.fileCount} files in ${bootstrapTime}ms`);
// Start live stream consumer from where replay left off
logger.info(`Starting live stream consumer from ${lastId}...`);
redisClient.startSimpleStreamConsumer(
EVENTS_STREAM,
lastId,
async (message: StreamMessage) => {
await handleStreamingEvent(message, stateBuilder!);
onGap: (payload) => {
logger.warn(`SSE cursor ${payload.requested} trimmed; resuming from ${payload.resumeFrom}`);
},
5000 // 5 second block timeout
).catch(error => {
logger.error('Stream consumer error:', error);
});
// Fire-and-forget; start() returns when stop() fires.
sseClient.start().catch((error) => {
logger.error('SSE consumer terminated:', error);
});
// Initialize API server (with kvManager for service discovery)
+1 -1
View File
@@ -58,7 +58,7 @@ export interface IKVClient {
}
/**
* Leader lock file content format (kv-leader.info)
* meta-core's identity and URL set (from its UDP announce, or /urls)
* Written by meta-core leader, read by other services
*/
export interface LeaderLockInfo {
+28 -27
View File
@@ -2,7 +2,7 @@
* KV Manager for meta-fuse (FOLLOWER-only)
*
* Simplified KV manager that:
* 1. Discovers the leader via LeaderClient (reads kv-leader.info)
* 1. Locates meta-core via LeaderClient (UDP announce, meta-discovery v1)
* 2. Creates Redis client connection
* 3. Handles reconnection on leader failure
*
@@ -13,7 +13,6 @@
import { hostname } from 'os';
import { Logger } from 'tslog';
import { LeaderClient } from './LeaderClient.js';
import { ServiceRegistration } from './ServiceRegistration.js';
import { RedisClient } from './RedisClient.js';
import type { LeaderLockInfo } from './IKVClient.js';
@@ -48,7 +47,6 @@ interface KVManagerConfig {
export class KVManager {
private config: Required<KVManagerConfig>;
private leaderClient: LeaderClient | null = null;
private serviceRegistration: ServiceRegistration | null = null;
private redisClient: RedisClient | null = null;
private isStarted = false;
private isShuttingDown = false;
@@ -80,14 +78,15 @@ export class KVManager {
logger.info(`Starting KVManager for ${this.config.serviceName}...`);
// Initialize and start service registration for heartbeat
const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
this.serviceRegistration = new ServiceRegistration({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
apiUrl
});
await this.serviceRegistration.start();
// No separate service registration step any more: the LeaderClient's
// discovery node announces this service on the same UDP group it
// listens on, so being discoverable and discovering are one thing.
// Browser-facing URL for the nav menu, in the order the spec requires:
// PUBLIC_URL (reachable when there is no Caddy perimeter in front — a
// debug-direct port) -> BASE_URL (the Caddy URL) -> container host.
// Announcing a Caddy URL on a stack with no Caddy running is what puts
// dead links in every neighbour's menu.
const apiUrl = process.env.PUBLIC_URL || this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
// If direct Redis URL provided, skip leader discovery
if (this.config.redisUrl) {
@@ -99,8 +98,9 @@ export class KVManager {
// Use leader client
this.leaderClient = new LeaderClient({
metaCorePath: this.config.metaCorePath,
metaCoreUrl: this.config.metaCoreUrl || undefined
metaCoreUrl: this.config.metaCoreUrl || undefined,
serviceName: this.config.serviceName,
baseUrl: apiUrl,
});
// Set up leader change callback
@@ -113,7 +113,7 @@ export class KVManager {
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
logger.info(`Connecting to Redis at ${leaderInfo.redisUrl}...`);
await this.connectToRedis(leaderInfo.redisUrl);
await this.connectToRedis(leaderInfo.redisUrl, leaderInfo.apiUrl);
} catch (error: any) {
logger.error(`Failed to connect to leader: ${error.message}`);
throw error;
@@ -141,7 +141,7 @@ export class KVManager {
try {
const leaderInfo = await this.leaderClient.waitForLeader(30000);
logger.info(`Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
await this.connectToRedis(leaderInfo.redisUrl);
await this.connectToRedis(leaderInfo.redisUrl, leaderInfo.apiUrl);
} catch (error: any) {
logger.error(`Failed to reconnect: ${error.message}`);
}
@@ -163,10 +163,6 @@ export class KVManager {
}
// Stop service registration (if it was started)
if (this.serviceRegistration) {
await this.serviceRegistration.stop();
this.serviceRegistration = null;
}
// Disconnect Redis
await this.disconnectRedis();
@@ -178,20 +174,28 @@ export class KVManager {
/**
* Connect to Redis
*/
private async connectToRedis(url: string): Promise<void> {
private async connectToRedis(url: string, apiUrl?: string): Promise<void> {
try {
// Disconnect existing client if any
await this.disconnectRedis();
// Create new client
// Create new client. When apiUrl is supplied, reads route
// through meta-core HTTP (api-mediated-access PR C).
// When url is also empty, the client is HTTP-only and never
// opens a Redis socket (post-PR D state).
this.redisClient = new RedisClient({
url,
url, // may be '' in PR D world
prefix: this.config.redisPrefix,
filesVolume: this.config.filesPath
filesVolume: this.config.filesPath,
metaCoreApiUrl: apiUrl,
});
await this.redisClient.connect();
logger.info('Connected to Redis');
if (url) {
logger.info('Connected to Redis');
} else {
logger.info('Storage client ready (HTTP-only mode, no Redis)');
}
// Notify ready
this.notifyReady();
@@ -343,9 +347,6 @@ export class KVManager {
/**
* Get service registration instance
*/
getServiceRegistration(): ServiceRegistration | null {
return this.serviceRegistration;
}
/**
* Get configured paths
+133 -151
View File
@@ -1,29 +1,48 @@
/**
* LeaderClient - Client for reading leader info from meta-core
* LeaderClient - locates meta-core and exposes its URLs.
*
* This replaces the old LeaderDiscovery.ts by delegating leader election
* to meta-core (Go sidecar) and just reading the results.
* Since meta-discovery v1 this is a thin adapter over MetaCoreLocator
* (../discovery/meshdisco.js): meta-core is found by UDP announce instead of by
* reading /meta-core/locks/kv-leader.info, which means this service no longer
* needs the /meta-core volume mounted at all.
*
* Features:
* - Reads kv-leader.info file for leader URLs
* - Calls meta-core /urls API for current leader URLs
* - Watches for leader changes via file system watcher
* The public surface is deliberately unchanged so KVManager and its callers did
* not have to move: getLeaderInfo / getUrls / getApiUrl / getWebdavUrl*,
* waitForLeader (same 30s default), startWatching / onChange (the fs.watch on
* the lock dir became "an announce arrived carrying a different apiUrl").
*
* Precedence is the pin: when metaCoreUrl (META_CORE_URL) is set it always
* wins and the wire is never consulted for core selection. See
* docs/project-architecture/service-discovery.md.
*
* The name "leader" is legacy: meta-core's flock election is vestigial, and
* redisUrl has been empty since the api-mediated-access lockdown.
*/
import { promises as fs } from 'fs';
import { watch, FSWatcher } from 'fs';
import { dirname } from 'path';
import { Logger } from 'tslog';
import { MetaCoreLocator, type MeshNeighbor } from '../discovery/meshdisco.js';
import type { LeaderLockInfo } from './IKVClient.js';
const logger = new Logger({ name: 'LeaderClient' });
export interface LeaderClientConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/**
* @deprecated Unused since meta-discovery v1. Kept so existing callers
* compile unchanged; nothing reads the volume any more.
*/
metaCorePath?: string;
/** meta-core API URL (e.g., http://meta-core:9000 or http://localhost:9000) */
/** meta-core API URL. When set, discovery never overrides it. */
metaCoreUrl?: string;
/** This service's name, used for its own announce. */
serviceName?: string;
/** Browser-facing URL announced for the nav menu. */
baseUrl?: string;
/** Service version, display only. */
version?: string;
}
export interface URLsResponse {
@@ -38,57 +57,75 @@ export interface URLsResponse {
export class LeaderClient {
private config: LeaderClientConfig;
private infoFilePath: string;
private locator: MetaCoreLocator;
private leaderInfo: LeaderLockInfo | null = null;
private watcher: FSWatcher | null = null;
private onChangeCallbacks: (() => void)[] = [];
private started = false;
// URL caching
// Only used on the pinned path, where URLs still come from GET /urls.
// A discovered core carries them in the announce itself.
private cachedUrls: URLsResponse | null = null;
private urlsCacheTime: number = 0;
private readonly urlsCacheTTL: number = 5000; // 5 seconds
private readonly urlsCacheTTL: number = 5000;
constructor(config: LeaderClientConfig) {
this.config = config;
this.infoFilePath = `${config.metaCorePath}/locks/kv-leader.info`;
this.locator = new MetaCoreLocator({
serviceName: config.serviceName ?? 'meta-fuse',
baseUrl: config.baseUrl,
version: config.version,
metaCoreUrl: config.metaCoreUrl,
enabled: process.env.ENABLE_UDP_DISCOVERY !== 'false' &&
process.env.ENABLE_UDP_DISCOVERY !== '0',
});
this.locator.onChange(() => {
this.cachedUrls = null;
this.urlsCacheTime = 0;
this.notifyChange();
});
}
private async ensureStarted(): Promise<void> {
if (this.started) return;
this.started = true;
await this.locator.start();
}
/**
* Read API URL from file (plain text format)
* Guard against a rollback reintroducing direct Redis exposure.
* api-mediated-access PR D removed redisUrl; ALLOW_LEGACY_REDIS_URL=1
* downgrades this to a warning during a deliberate temporary rollback.
*/
private async getApiUrlFromFile(): Promise<string | null> {
try {
const content = await fs.readFile(this.infoFilePath, 'utf-8');
return content.trim() || null;
} catch (error: any) {
logger.error(`Failed to read API URL from file: ${error.message}`);
return null;
private assertNoRedisUrl(redisUrl: string | undefined): void {
if (!redisUrl) return;
const msg =
'meta-core still publishes redisUrl; direct Redis access was retired ' +
'by the api-mediated-access lockdown. Verify meta-core version.';
if (process.env.ALLOW_LEGACY_REDIS_URL === '1') {
logger.warn('WARNING: ' + msg);
} else {
throw new Error(msg);
}
}
/**
* Fetch URLs from meta-core /urls API with caching
*/
private async fetchUrls(apiUrl: string): Promise<URLsResponse | null> {
// Check cache
const now = Date.now();
if (this.cachedUrls && (now - this.urlsCacheTime) < this.urlsCacheTTL) {
return this.cachedUrls;
}
try {
const response = await fetch(`${apiUrl}/urls`, {
method: 'GET',
headers: { 'Accept': 'application/json' },
signal: AbortSignal.timeout(5000)
});
if (!response.ok) {
logger.error(`Failed to fetch URLs: ${response.status} ${response.statusText}`);
return null;
}
this.cachedUrls = await response.json() as URLsResponse;
const parsed = await response.json() as URLsResponse;
this.assertNoRedisUrl(parsed.redisUrl);
this.cachedUrls = parsed;
this.urlsCacheTime = now;
return this.cachedUrls;
} catch (error: any) {
@@ -98,23 +135,37 @@ export class LeaderClient {
}
/**
* Read leader info from file and /urls API
* Resolve meta-core's URLs — straight from the announce when discovered
* (no HTTP hop needed), or over HTTP when pinned.
*/
async getUrls(): Promise<URLsResponse | null> {
await this.ensureStarted();
const wire = this.locator.getUrls();
if (wire && !this.config.metaCoreUrl) {
return {
hostname: wire.hostname,
baseUrl: wire.baseUrl,
apiUrl: wire.apiUrl,
redisUrl: '',
webdavUrl: wire.webdavUrl,
webdavUrlInternal: wire.webdavUrlInternal,
isLeader: true,
};
}
const apiUrl = this.locator.getApiUrl();
if (!apiUrl) {
logger.error('No meta-core API URL available');
return null;
}
return this.fetchUrls(apiUrl);
}
async getLeaderInfo(): Promise<LeaderLockInfo | null> {
try {
// Read API URL from file (plain text)
const apiUrl = await this.getApiUrlFromFile();
if (!apiUrl) {
return null;
}
// Fetch full info from /urls API
const urls = await this.fetchUrls(apiUrl);
if (!urls) {
return null;
}
// Convert URLsResponse to LeaderLockInfo
const urls = await this.getUrls();
if (!urls) return null;
this.leaderInfo = {
hostname: urls.hostname,
baseUrl: urls.baseUrl,
@@ -123,7 +174,7 @@ export class LeaderClient {
webdavUrl: urls.webdavUrl,
webdavUrlInternal: urls.webdavUrlInternal,
timestamp: Date.now(),
pid: 0 // Unknown for remote leader
pid: 0
};
return this.leaderInfo;
} catch (error: any) {
@@ -132,138 +183,64 @@ export class LeaderClient {
}
}
/**
* Get Redis URL from leader info
*/
async getRedisUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.redisUrl ?? null;
return (await this.getLeaderInfo())?.redisUrl ?? null;
}
/**
* Get WebDAV URL from leader info (external, via nginx/HTTPS)
*/
async getWebdavUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.webdavUrl ?? null;
return (await this.getLeaderInfo())?.webdavUrl ?? null;
}
/**
* Get internal WebDAV URL from leader info (direct to port 9000)
* Use this for container-to-container communication
*/
/** Internal WebDAV URL — use for container-to-container access. */
async getWebdavUrlInternal(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.webdavUrlInternal ?? null;
return (await this.getLeaderInfo())?.webdavUrlInternal ?? null;
}
/**
* Get meta-core API URL from leader info
*/
async getApiUrl(): Promise<string | null> {
const info = await this.getLeaderInfo();
return info?.apiUrl ?? null;
return (await this.getLeaderInfo())?.apiUrl ?? null;
}
/**
* Call meta-core /urls API to get current URLs
* Useful for getting URLs when you don't want to read the file directly
*/
async getUrls(): Promise<URLsResponse | null> {
// First try using configured metaCoreUrl
let apiUrl: string | null = this.config.metaCoreUrl ?? null;
// Fall back to reading from file
if (!apiUrl) {
apiUrl = await this.getApiUrlFromFile();
}
if (!apiUrl) {
logger.error('No meta-core API URL available');
return null;
}
return this.fetchUrls(apiUrl);
}
/**
* Wait for leader info to be available
* Block until meta-core is reachable. Probes immediately then every 500ms,
* matching the previous file-polling loop's timing exactly.
*/
async waitForLeader(timeoutMs: number = 30000): Promise<LeaderLockInfo> {
const startTime = Date.now();
const pollInterval = 500;
await this.ensureStarted();
const deadline = Date.now() + timeoutMs;
while (Date.now() - startTime < timeoutMs) {
await this.locator.waitForCore(timeoutMs);
while (Date.now() < deadline) {
const info = await this.getLeaderInfo();
if (info) {
logger.info(`Leader found: ${info.hostname} at ${info.redisUrl}`);
logger.info(`meta-core found: ${info.hostname} at ${info.apiUrl}`);
return info;
}
logger.debug('Waiting for leader...');
await new Promise(resolve => setTimeout(resolve, pollInterval));
logger.debug('Waiting for meta-core...');
await new Promise(resolve => setTimeout(resolve, 500));
}
throw new Error(`No leader found within ${timeoutMs}ms`);
throw new Error(`No meta-core found within ${timeoutMs}ms`);
}
/**
* Start watching for leader changes
* Previously an fs.watch on the lock directory. Discovery is always
* listening, so this only has to make sure the node is running.
*/
startWatching(): void {
if (this.watcher) {
return; // Already watching
}
const lockDir = dirname(this.infoFilePath);
try {
this.watcher = watch(lockDir, (eventType, filename) => {
if (filename === 'kv-leader.info') {
logger.debug(`Lock file ${eventType}, invalidating cache...`);
// Invalidate cache to force fresh API call
this.cachedUrls = null;
this.urlsCacheTime = 0;
this.getLeaderInfo().then(() => {
this.notifyChange();
}).catch((err) => logger.error(`Error reloading leader info: ${err.message}`));
}
});
this.watcher.on('error', (error: any) => {
logger.warn(`File watcher error: ${error.message}`);
});
logger.info(`Watching for leader changes in ${lockDir}`);
} catch (error: any) {
logger.error(`Failed to start watching: ${error.message}`);
}
void this.ensureStarted();
}
/**
* Stop watching for leader changes
*/
stopWatching(): void {
if (this.watcher) {
this.watcher.close();
this.watcher = null;
logger.debug('Stopped watching for leader changes');
}
// No-op: the locator keeps listening until close().
}
/**
* Register callback for leader changes
*/
onChange(callback: () => void): this {
this.onChangeCallbacks.push(callback);
return this;
}
/**
* Notify all change callbacks
*/
private notifyChange(): void {
logger.debug('meta-core changed, invalidating cache...');
for (const callback of this.onChangeCallbacks) {
try {
callback();
@@ -273,18 +250,23 @@ export class LeaderClient {
}
}
/**
* Get cached leader info (without re-reading file)
*/
getCachedLeaderInfo(): LeaderLockInfo | null {
return this.leaderInfo;
}
/**
* Clean up resources
*/
/** Neighbours for the nav menu (one row per service name). */
getNeighbors(): MeshNeighbor[] {
return this.locator.getNeighbors();
}
/** This service's own announce, so the menu can show itself. */
self(): MeshNeighbor {
return this.locator.self();
}
close(): void {
this.stopWatching();
this.onChangeCallbacks = [];
void this.locator.stop();
this.started = false;
}
}
@@ -0,0 +1,121 @@
/**
* MetaCoreApiClient — HTTP read client against meta-core for meta-fuse.
*
* meta-fuse needs to read file metadata (property GETs, full-record fetch,
* hashId enumeration) as it builds the VFS. After the api-mediated-access
* lockdown lands these go through meta-core's /meta/{hash} family instead
* of direct Redis reads.
*
* Only the methods meta-fuse actually calls are implemented. See
* meta-core docs/api-mediated-access.md for the full surface.
*/
export interface MetaCoreApiClientConfig {
apiUrl: string;
timeoutMs?: number;
}
export class MetaCoreApiClient {
private apiUrl: string;
private timeoutMs: number;
constructor(config: MetaCoreApiClientConfig) {
this.apiUrl = config.apiUrl.replace(/\/+$/, '');
this.timeoutMs = config.timeoutMs ?? 30000;
}
/**
* GET /meta/{hash}/{prop}. Returns the raw string value or null on 404.
* meta-core sends text/plain for property reads.
*/
async getProperty(hashId: string, property: string): Promise<string | null> {
const path = `/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(property)}`;
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method: 'GET',
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'text/plain' },
});
if (response.status === 404) return null;
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] GET ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
return text === '' ? null : text;
}
/**
* GET /meta/{hash}. Returns the flat property map or null on 404.
*/
async getMetadataFlat(hashId: string): Promise<Record<string, string> | null> {
const path = `/meta/${encodeURIComponent(hashId)}`;
const body = await this.fetchJsonAllowing404('GET', path);
if (body === null) return null;
const flat = (body as any)?.metadata;
if (!flat || typeof flat !== 'object') return null;
return flat as Record<string, string>;
}
/**
* GET /meta. Enumerates every known hashId.
*/
async getAllHashIds(): Promise<string[]> {
const body = await this.fetchJson('GET', '/meta');
const ids = (body as any)?.hashIds;
return Array.isArray(ids) ? ids : [];
}
/**
* GET /health on meta-core. Used as the connectivity probe.
*/
async health(): Promise<boolean> {
try {
const response = await fetch(`${this.apiUrl}/health`, {
method: 'GET',
signal: AbortSignal.timeout(Math.min(this.timeoutMs, 5000)),
headers: { Accept: 'application/json' },
});
return response.ok;
} catch {
return false;
}
}
private encodePropertyPath(property: string): string {
return property
.split('/')
.map(segment => encodeURIComponent(segment))
.join('/');
}
private async fetchJson(method: string, path: string): Promise<unknown> {
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method,
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'application/json' },
});
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] ${method} ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
if (!text) return null;
try { return JSON.parse(text); } catch { return text; }
}
private async fetchJsonAllowing404(method: string, path: string): Promise<unknown> {
const url = `${this.apiUrl}${path}`;
const response = await fetch(url, {
method,
signal: AbortSignal.timeout(this.timeoutMs),
headers: { Accept: 'application/json' },
});
if (response.status === 404) return null;
if (!response.ok) {
throw new Error(`[MetaCoreApiClient] ${method} ${path} → ${response.status} ${response.statusText}`);
}
const text = await response.text();
if (!text) return null;
try { return JSON.parse(text); } catch { return text; }
}
}
+57 -13
View File
@@ -12,6 +12,7 @@ import { Redis } from 'ioredis';
import { Logger } from 'tslog';
import * as os from 'os';
import type { IKVClient, KeyValuePair } from './IKVClient.js';
import { MetaCoreApiClient } from './MetaCoreApiClient.js';
const logger = new Logger({ name: 'RedisClient' });
@@ -74,7 +75,6 @@ export interface FileMetadata {
// Title information (for folder organization)
title?: string;
titles?: { eng?: string; [key: string]: string | undefined };
originalTitle?: string;
fileName?: string;
@@ -104,15 +104,27 @@ export interface RedisClientConfig {
prefix?: string;
filesVolume?: string;
reconnectInterval?: number;
/**
* meta-core HTTP API base URL. When set, read methods (getProperty,
* getProperties, getMetadataFlat, getAllHashIds, health) route through
* the /meta/{hash} HTTP family instead of touching Redis directly.
*
* This is the meta-fuse half of the api-mediated-access lockdown (see
* meta-core docs/api-mediated-access.md). Reads stay on Redis when
* left unset so tests / one-off scripts still work.
*/
metaCoreApiUrl?: string;
}
export class RedisClient implements Partial<IKVClient> {
private client: Redis | null = null;
private subscriber: Redis | null = null;
private config: Required<RedisClientConfig>;
private config: Required<Omit<RedisClientConfig, 'metaCoreApiUrl'>> & Pick<RedisClientConfig, 'metaCoreApiUrl'>;
private isConnected = false;
private reconnectTimer: NodeJS.Timeout | null = null;
private subscriptions: Map<string, (message: string) => void> = new Map();
private apiClient: MetaCoreApiClient | null = null;
// Stream consumer state
private streamConsumerRunning = false;
@@ -125,20 +137,33 @@ export class RedisClient implements Partial<IKVClient> {
prefix: config.prefix ?? process.env.REDIS_PREFIX ?? '',
filesVolume: config.filesVolume ?? process.env.FILES_VOLUME ?? '/files',
reconnectInterval: config.reconnectInterval ?? 5000,
metaCoreApiUrl: config.metaCoreApiUrl,
};
if (this.config.metaCoreApiUrl) {
this.apiClient = new MetaCoreApiClient({ apiUrl: this.config.metaCoreApiUrl });
logger.info(`Reads routed through meta-core API at ${this.config.metaCoreApiUrl}`);
}
// Unique consumer name for this instance
this.consumerName = `${os.hostname()}-${process.pid}`;
}
/**
* Connect to Redis
* Connect to Redis. In HTTP-only mode (no URL + apiClient set) this is
* a no-op — reads route through MetaCoreApiClient and stream
* consumption is owned by SSEEventClient at the index.ts layer.
*/
async connect(): Promise<void> {
if (this.client && this.isConnected) {
return;
}
if (this.apiClient && !this.config.url) {
logger.info('HTTP-only mode — no direct Redis connection');
return;
}
try {
this.client = new Redis(this.config.url, {
retryStrategy: (times: number) => {
@@ -236,9 +261,18 @@ export class RedisClient implements Partial<IKVClient> {
}
/**
* Check if connected
* Check if the metadata backend is reachable.
*
* In HTTP-only mode (api-mediated-access PR D) there's no Redis socket;
* reachability is signalled by having an apiClient configured. The
* health badge is keyed off this; reporting `false` there would make
* the UI permanently show "Disconnected" even though the service is
* working fine over HTTP.
*/
get connected(): boolean {
if (this.apiClient && !this.config.url) {
return true;
}
return this.isConnected;
}
@@ -255,6 +289,15 @@ export class RedisClient implements Partial<IKVClient> {
* @returns The property value or null if not found
*/
async getProperty(hashId: string, property: string): Promise<string | null> {
if (this.apiClient) {
try {
return await this.apiClient.getProperty(hashId, property);
} catch (error: any) {
logger.error(`HTTP getProperty failed for ${hashId}/${property}:`, error.message);
return null;
}
}
if (!this.client || !this.isConnected) {
return null;
}
@@ -579,12 +622,6 @@ export class RedisClient implements Partial<IKVClient> {
mtime = isNaN(parsed) ? parseFloat(data.mtime) : parsed;
}
// Parse titles - can be nested object or simple string
let titles: { eng?: string; [key: string]: string | undefined } | undefined;
if (data['titles/eng']) {
titles = { eng: data['titles/eng'] };
}
// Parse season/episode (can be 0 for specials)
const season = data.season !== undefined ? parseInt(data.season, 10) : undefined;
const episode = data.episode !== undefined ? parseInt(data.episode, 10) : undefined;
@@ -603,12 +640,19 @@ export class RedisClient implements Partial<IKVClient> {
mtime,
ctime: data.ctime ? parseFloat(data.ctime) : mtime,
// Identification
hashId: hashId ?? data.cid_midhash256 ?? data.hashId,
// Identification.
//
// `hashId` is always supplied by the caller — every `parseMetadata`
// call site derives it from the Redis key (`file:<hashId>/…`), so the
// fallbacks below it were already unreachable. The two that used to sit
// here (`data.cid_midhash256`, then `data.hashId`) were also the wrong
// shape: `cid_*` fields no longer exist on records, and meta-core
// rejects them at the write boundary. Kept as a single defensive `??`
// rather than a chain of dead ones. See METADATA_KEYS.md §14.13.
hashId: hashId ?? data.hashId,
// Title information
title: data.title,
titles,
originalTitle: data.originalTitle,
fileName: data.fileName,
@@ -0,0 +1,229 @@
/**
* SSEEventClient — HTTP Server-Sent Events client for meta-core event streams.
*
* Replaces direct Redis Streams consumption. See the same-named module in
* meta-sort for the design rationale (meta-core docs/api-mediated-access.md).
*/
import { promises as fs } from 'fs';
import { dirname } from 'path';
export interface SSEEvent {
id: string;
event: string;
data: any;
}
export interface SSEClientOptions {
url: string;
cursorPath: string | null;
onEvent: (e: SSEEvent) => Promise<void> | void;
onGap?: (payload: { requested: string; resumeFrom: string; reason: string }) => Promise<void> | void;
logTag?: string;
reconnectBaseMs?: number;
reconnectMaxMs?: number;
flushIntervalMs?: number;
/** When set, sent as Last-Event-ID on the *first* connect (e.g. to
* force replay from a specific point). After the first event the
* persisted cursor takes over. */
initialCursor?: string;
}
export class SSEEventClient {
private readonly opts: Required<Omit<SSEClientOptions, 'onGap' | 'initialCursor'>> & Pick<SSEClientOptions, 'onGap' | 'initialCursor'>;
private cursor: string = '';
private cursorDirty = false;
private abort: AbortController | null = null;
private flushTimer: NodeJS.Timeout | null = null;
private stopped = false;
private backoff = 0;
private firstConnect = true;
constructor(options: SSEClientOptions) {
this.opts = {
url: options.url,
cursorPath: options.cursorPath,
onEvent: options.onEvent,
onGap: options.onGap,
logTag: options.logTag ?? '[SSE]',
reconnectBaseMs: options.reconnectBaseMs ?? 500,
reconnectMaxMs: options.reconnectMaxMs ?? 30_000,
flushIntervalMs: options.flushIntervalMs ?? 2000,
initialCursor: options.initialCursor,
};
this.backoff = this.opts.reconnectBaseMs;
}
async start(): Promise<void> {
await this.loadCursor();
if (!this.cursor && this.opts.initialCursor) {
this.cursor = this.opts.initialCursor;
}
this.startFlushTimer();
await this.runLoop();
}
async stop(): Promise<void> {
this.stopped = true;
if (this.abort) {
this.abort.abort();
this.abort = null;
}
if (this.flushTimer) {
clearInterval(this.flushTimer);
this.flushTimer = null;
}
await this.flushCursor();
}
getCursor(): string {
return this.cursor;
}
private async loadCursor(): Promise<void> {
if (!this.opts.cursorPath) return;
try {
const data = await fs.readFile(this.opts.cursorPath, 'utf-8');
const trimmed = data.trim();
if (trimmed) {
this.cursor = trimmed;
console.log(`${this.opts.logTag} Resuming from cursor ${this.cursor}`);
}
} catch (err: any) {
if (err.code !== 'ENOENT') {
console.warn(`${this.opts.logTag} Could not read cursor file ${this.opts.cursorPath}: ${err.message}`);
}
}
}
private startFlushTimer(): void {
this.flushTimer = setInterval(() => {
if (this.cursorDirty) {
this.flushCursor().catch(err =>
console.warn(`${this.opts.logTag} Cursor flush failed: ${err.message}`)
);
}
}, this.opts.flushIntervalMs);
}
private async flushCursor(): Promise<void> {
if (!this.opts.cursorPath || !this.cursorDirty) return;
this.cursorDirty = false;
try {
await fs.mkdir(dirname(this.opts.cursorPath), { recursive: true });
await fs.writeFile(this.opts.cursorPath, this.cursor, 'utf-8');
} catch (err: any) {
this.cursorDirty = true;
throw err;
}
}
private async runLoop(): Promise<void> {
while (!this.stopped) {
try {
await this.connect();
this.backoff = this.opts.reconnectBaseMs;
} catch (err: any) {
if (this.stopped) return;
console.warn(`${this.opts.logTag} Stream error, reconnecting in ${this.backoff}ms: ${err?.message ?? err}`);
await this.sleep(this.backoff);
this.backoff = Math.min(this.backoff * 2, this.opts.reconnectMaxMs);
}
}
}
private async connect(): Promise<void> {
this.abort = new AbortController();
const headers: Record<string, string> = {
Accept: 'text/event-stream',
'Cache-Control': 'no-cache',
};
if (this.cursor) {
headers['Last-Event-ID'] = this.cursor;
}
this.firstConnect = false;
const response = await fetch(this.opts.url, {
method: 'GET',
headers,
signal: this.abort.signal,
});
if (!response.ok) throw new Error(`HTTP ${response.status} ${response.statusText}`);
if (!response.body) throw new Error('Response has no body');
const reader = response.body.getReader();
const decoder = new TextDecoder('utf-8');
let buffer = '';
try {
while (!this.stopped) {
const { value, done } = await reader.read();
if (done) return;
buffer += decoder.decode(value, { stream: true });
let idx: number;
while ((idx = buffer.indexOf('\n\n')) !== -1) {
const rawChunk = buffer.slice(0, idx);
buffer = buffer.slice(idx + 2);
await this.handleChunk(rawChunk);
}
}
} finally {
try { reader.releaseLock(); } catch { /* nothing useful */ }
}
}
private async handleChunk(chunk: string): Promise<void> {
let id = '';
let event = '';
const dataLines: string[] = [];
for (const rawLine of chunk.split('\n')) {
if (!rawLine || rawLine.startsWith(':')) continue;
const colon = rawLine.indexOf(':');
if (colon === -1) continue;
const field = rawLine.slice(0, colon);
let value = rawLine.slice(colon + 1);
if (value.startsWith(' ')) value = value.slice(1);
switch (field) {
case 'id': id = value; break;
case 'event': event = value; break;
case 'data': dataLines.push(value); break;
}
}
if (dataLines.length === 0) return;
let data: any;
try {
data = JSON.parse(dataLines.join('\n'));
} catch (err: any) {
console.warn(`${this.opts.logTag} Skipping unparseable event ${id || '(no id)'}: ${err.message}`);
return;
}
const ev: SSEEvent = { id, event: event || 'message', data };
if (ev.event === 'gap') {
if (this.opts.onGap) {
await this.opts.onGap(ev.data);
} else {
console.warn(`${this.opts.logTag} Gap event (cursor ${ev.data?.requested} trimmed): resumed from ${ev.data?.resumeFrom}`);
}
if (typeof ev.data?.resumeFrom === 'string' && ev.data.resumeFrom) {
this.cursor = ev.data.resumeFrom;
this.cursorDirty = true;
}
return;
}
try {
await this.opts.onEvent(ev);
} catch (err: any) {
console.error(`${this.opts.logTag} onEvent handler failed for ${ev.id}: ${err?.stack ?? err}`);
}
if (ev.id) {
this.cursor = ev.id;
this.cursorDirty = true;
}
}
private sleep(ms: number): Promise<void> {
return new Promise(resolve => setTimeout(resolve, ms));
}
}
@@ -1,211 +0,0 @@
/**
* Service Registration using shared filesystem
*
* Each service registers itself in /meta-core/services/{service-name}-{hostname}.json
* with its base URL, status, and heartbeat timestamp.
*
* Service discovery is centralized in meta-core - this class only handles registration.
* For service discovery, use the /api/services endpoint (proxied to meta-core via nginx).
*
* Each service must call start() to register itself and begin heartbeat updates.
* meta-core runs a cleanup process that removes services with stale heartbeats.
*/
import { promises as fs } from 'fs';
import { hostname } from 'os';
import { Logger } from 'tslog';
import type { ServiceInfo } from './IKVClient.js';
const logger = new Logger({ name: 'ServiceRegistration' });
interface ServiceRegistrationConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** Service name */
serviceName: string;
/** Base URL for the service */
apiUrl: string;
/** Heartbeat interval in ms (default: 30000) */
heartbeatInterval?: number;
/** Stale threshold in ms (default: 60000) */
staleThreshold?: number;
}
export class ServiceRegistration {
private config: ServiceRegistrationConfig;
private servicesDir: string;
private serviceFilePath: string;
private heartbeatTimer: NodeJS.Timeout | null = null;
private isShuttingDown = false;
private isStarted = false;
private currentHostname: string;
constructor(config: ServiceRegistrationConfig) {
this.config = {
heartbeatInterval: 30000,
staleThreshold: 60000,
...config
};
this.currentHostname = hostname();
this.servicesDir = `${this.config.metaCorePath}/services`;
// Use hostname-based file naming
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`;
}
/**
* Ensure services directory exists
*/
private async ensureServicesDir(): Promise<void> {
await fs.mkdir(this.servicesDir, { recursive: true });
}
/**
* Build service info for this service
*/
private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
return {
name: this.config.serviceName,
hostname: this.currentHostname,
baseUrl: this.config.apiUrl,
status,
lastHeartbeat: new Date().toISOString()
};
}
/**
* Register this service by writing service info to the shared filesystem.
* meta-core's cleanup process monitors these files and removes stale entries.
*/
async register(): Promise<void> {
await this.ensureServicesDir();
const info = this.buildServiceInfo('running');
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
logger.info(`Registered ${this.config.serviceName}-${this.currentHostname}`);
}
/**
* Update service status
*/
async updateStatus(status: ServiceInfo['status']): Promise<void> {
try {
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
const info = JSON.parse(content) as ServiceInfo;
info.status = status;
info.lastHeartbeat = new Date().toISOString();
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
} catch (error) {
logger.error('Failed to update status:', error);
}
}
/**
* Send heartbeat (update lastHeartbeat timestamp)
*/
async heartbeat(): Promise<void> {
try {
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
const info = JSON.parse(content) as ServiceInfo;
info.lastHeartbeat = new Date().toISOString();
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
} catch (error: any) {
// If file was deleted, re-register
if (error.code === 'ENOENT') {
await this.register();
} else {
logger.error('Heartbeat failed:', error);
}
}
}
/**
* Start heartbeat loop
*/
startHeartbeat(): void {
this.heartbeatTimer = setInterval(async () => {
if (this.isShuttingDown) return;
await this.heartbeat();
}, this.config.heartbeatInterval);
}
/**
* Stop heartbeat loop
*/
stopHeartbeat(): void {
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
}
/**
* Unregister this service (on shutdown)
*/
async unregister(): Promise<void> {
this.isShuttingDown = true;
this.stopHeartbeat();
try {
// Remove the service file
await fs.unlink(this.serviceFilePath);
logger.info(`Unregistered ${this.config.serviceName}-${this.currentHostname}`);
} catch (error: any) {
if (error.code !== 'ENOENT') {
logger.error('Failed to unregister:', error);
}
}
}
// ========================================================================
// Lifecycle
// ========================================================================
/**
* Full startup sequence: register service and start heartbeat loop.
*/
async start(): Promise<void> {
if (this.isStarted) return;
await this.register();
this.startHeartbeat();
this.isStarted = true;
logger.info(`Started ${this.config.serviceName} at ${this.config.apiUrl}`);
}
/**
* Full shutdown sequence
*/
async stop(): Promise<void> {
if (!this.isStarted) return;
await this.unregister();
this.isStarted = false;
}
// ========================================================================
// Getters
// ========================================================================
getServiceFilePath(): string {
return this.serviceFilePath;
}
getServicesDir(): string {
return this.servicesDir;
}
getIsStarted(): boolean {
return this.isStarted;
}
}
@@ -13,7 +13,6 @@
*/
export interface MediaMeta {
// Title information
titles?: { eng?: string; [key: string]: string | undefined };
originalTitle?: string;
title?: string;
@@ -62,7 +61,8 @@ export function renamingRule(metadata: Partial<MediaMeta>, filepath?: string): s
}
// Get title - try various fields
const title = metadata.titles?.eng || metadata.originalTitle;
// The display name — never the `titles/*` name set (METADATA_KEYS.md §3).
const title = metadata.title || metadata.originalTitle;
if (!title) {
if (metadata.fileType !== 'torrent') {
// Don't warn for torrent files because they don't always have a title
@@ -174,12 +174,6 @@ export class VirtualFileSystem implements VFSUpdateCallback {
mtime = isNaN(parsed) ? parseFloat(props.mtime) : parsed;
}
// Parse titles - can be from nested paths like 'titles/eng'
let titles: { eng?: string; [key: string]: string | undefined } | undefined;
if (props['titles/eng']) {
titles = { eng: props['titles/eng'] };
}
// Parse season/episode
const season = props.season !== undefined ? parseInt(props.season, 10) : undefined;
const episode = props.episode !== undefined ? parseInt(props.episode, 10) : undefined;
@@ -212,7 +206,6 @@ export class VirtualFileSystem implements VFSUpdateCallback {
ctime: props.ctime ? parseFloat(props.ctime) : mtime,
hashId,
title: props.title,
titles,
originalTitle: props.originalTitle,
fileName,
season,
@@ -7,7 +7,7 @@
* Old behavior:
* - Only processes video, subtitle, torrent files
* - Requires extension
* - Title from titles.eng || originalTitle
* - Title from title || originalTitle
* - Files without title/extension are hidden (not in Unsorted)
*/
@@ -21,7 +21,7 @@ export type ConditionOperator =
*/
export interface Condition {
type: ConditionOperator;
field: string; // Metadata field path, e.g., "titles.eng", "season", "fileType"
field: string; // Metadata field path, e.g., "title", "season", "fileType"
value?: string | number | boolean; // Required for EQUALS, NOT_EQUALS, CONTAINS, MATCHES
}
@@ -83,7 +83,7 @@ export interface PreviewResponse {
*/
export interface TemplateVariable {
name: string; // Display name
path: string; // Field path (e.g., "titles.eng")
path: string; // Field path (e.g., "title")
type: 'string' | 'number' | 'boolean';
description: string;
example?: string;
@@ -93,7 +93,6 @@ export interface TemplateVariable {
* List of all available template variables
*/
export const TEMPLATE_VARIABLES: TemplateVariable[] = [
{ name: 'English Title', path: 'titles.eng', type: 'string', description: 'English title from metadata', example: 'Breaking Bad' },
{ name: 'Original Title', path: 'originalTitle', type: 'string', description: 'Original/native title', example: 'Breaking Bad' },
{ name: 'Title', path: 'title', type: 'string', description: 'Simple title field', example: 'Breaking Bad' },
{ name: 'Season', path: 'season', type: 'number', description: 'Season number', example: '1' },
@@ -16,7 +16,7 @@ interface RulesEditorProps {
// Available metadata fields for conditions
const AVAILABLE_FIELDS = [
{ value: 'fileType', label: 'File Type', type: 'string' },
{ value: 'titles.eng', label: 'English Title', type: 'string' },
{ value: 'title', label: 'Title', type: 'string' },
{ value: 'originalTitle', label: 'Original Title', type: 'string' },
{ value: 'season', label: 'Season', type: 'number' },
{ value: 'episode', label: 'Episode', type: 'number' },
@@ -53,7 +53,7 @@ function createEmptyRule(): RenamingRule {
function createEmptyCondition(): Condition {
return {
type: 'EXISTS',
field: 'titles.eng',
field: 'title',
};
}
@@ -197,7 +197,7 @@ function TemplateEditor({ template, onChange, variables }: TemplateEditorProps)
<textarea
value={template}
onChange={e => onChange(e.target.value)}
placeholder="e.g., Movies/{titles.eng|originalTitle}{movieYear? ({movieYear})}/{fileName}.{extension}"
placeholder="e.g., Movies/{title|originalTitle}{movieYear? ({movieYear})}/{fileName}.{extension}"
rows={3}
className="template-input"
/>
@@ -1,165 +1,54 @@
import { useState, useEffect } from 'react';
import { useEffect } from 'react';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
// of defining the custom element.
import '../meta-service-menu.js';
interface ServiceInfo {
name: string;
hostname: string;
baseUrl: string;
status: string;
lastHeartbeat: string;
}
/**
* ServiceNav — the neighbouring meta-* services, as a burger menu.
*
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
* a framework-free custom element shared by every meta-* UI (see
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
* that existed in four near-identical copies and polled meta-core's
* /api/services; the element polls this service's own /api/neighbors instead,
* so the nav still renders when meta-core is down.
*
* The element lives in shadow DOM and cannot see this app's CSS variables, so
* the four --mm-nav-* tokens below map our palette onto it.
*/
interface ServicesResponse {
services: ServiceInfo[];
current: string;
}
const serviceIcons: Record<string, string> = {
'meta-sort': '📁',
'meta-fuse': '🗂️',
'meta-stremio': '🎬',
'meta-core': '⚙️',
'meta-dup': '🔍',
'meta-orbit': '🌐',
'default': '📦'
};
function formatServiceName(name: string): string {
return name.split('-').map(word =>
word.charAt(0).toUpperCase() + word.slice(1)
).join(' ');
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace JSX {
interface IntrinsicElements {
'meta-service-menu': React.DetailedHTMLProps<
React.HTMLAttributes<HTMLElement>,
HTMLElement
> & { current?: string; endpoint?: string; label?: string };
}
}
}
function ServiceNav() {
const [services, setServices] = useState<ServiceInfo[]>([]);
const [loading, setLoading] = useState(true);
const currentService = 'meta-fuse';
useEffect(() => {
// Import above is for the side effect of defining the element; this
// keeps bundlers from tree-shaking it away in production builds.
}, []);
useEffect(() => {
const fetchServices = async () => {
try {
const response = await fetch('/api/services');
if (response.ok) {
const data: ServicesResponse = await response.json();
setServices(data.services || []);
}
} catch (error) {
console.error('Failed to fetch services:', error);
} finally {
setLoading(false);
}
};
fetchServices();
const interval = setInterval(fetchServices, 30000);
return () => clearInterval(interval);
}, []);
if (loading) return null;
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
if (sortedServices.length === 0) return null;
return (
<nav className="services-nav">
<span className="services-nav-label">Services:</span>
<div className="services-nav-items">
{sortedServices.map(service => {
const icon = serviceIcons[service.name] || serviceIcons.default;
const isActive = service.name === currentService;
return (
<a
key={service.name}
href={isActive ? '#' : service.baseUrl}
className={`service-link${isActive ? ' active' : ''}`}
onClick={(e) => {
e.preventDefault();
if (!isActive) {
window.location.href = service.baseUrl;
return (
<>
<meta-service-menu current="meta-fuse" />
<style>{`
meta-service-menu {
--mm-nav-fg: var(--text-primary, #e0e0e0);
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
--mm-nav-accent: var(--accent-primary, #4ecdc4);
}
}}
>
<span className="service-icon">{icon}</span>
<span>{formatServiceName(service.name)}</span>
<span className="service-status"></span>
</a>
);
})}
</div>
<style>{`
.services-nav {
background: var(--bg-secondary, #12121a);
border-radius: 12px;
padding: 0.5rem;
margin-bottom: 1.5rem;
display: flex;
align-items: center;
gap: 0.5rem;
flex-wrap: wrap;
border: 1px solid var(--border-color, #2a2a3a);
}
.services-nav-label {
color: var(--text-secondary, #a0a0a0);
font-size: 0.85rem;
padding: 0.5rem 0.75rem;
white-space: nowrap;
}
.services-nav-items {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
}
.service-link {
display: inline-flex;
align-items: center;
gap: 0.4rem;
padding: 0.5rem 1rem;
border-radius: 8px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: all 0.2s;
color: var(--text-primary, #e0e0e0);
background: var(--bg-tertiary, #1a1a2a);
border: 1px solid var(--border-color, #2a2a3a);
}
.service-link:hover {
background: var(--border-color, #2a2a3a);
border-color: rgba(78, 205, 196, 0.5);
text-decoration: none;
}
.service-link.active {
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
border-color: rgba(78, 205, 196, 0.5);
}
.service-link .service-icon {
font-size: 1.1rem;
}
.service-link .service-status {
width: 6px;
height: 6px;
border-radius: 50%;
background: var(--accent-primary, #4ecdc4);
margin-left: 0.25rem;
}
@media (max-width: 600px) {
.services-nav {
flex-direction: column;
align-items: flex-start;
}
}
`}</style>
</nav>
);
`}</style>
</>
);
}
export default ServiceNav;
@@ -0,0 +1,303 @@
/**
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
*
* Replaces the eight divergent "services bar" implementations (four copies of
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
* ones) with a single framework-free custom element. It drops into the React
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
* as an ES module.
*
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
* make a fetch to a neighbour a dead end. Links are plain navigations.
*
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
* `scripts/check-mirrors.sh`. Edit one → edit all.
*
* Shadow DOM is not optional here. The eight UIs use four incompatible token
* vocabularies, and two of them define *different colours under the same
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
* #161b22 in meta-dup). meta-search and meta-share are also the only
* light-mode-capable UIs, so a hardcoded dark palette would break them.
* The element therefore styles itself from four tokens with dark defaults:
*
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
*
* Each host maps its own variables onto those, e.g.
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
*
* Attributes:
* current name of the host service, so it can be marked and not linked
* endpoint override the default "/api/neighbors"
* label button tooltip (default "Services")
*
* Property:
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
* translate internal hostnames into browser-reachable ones
* (meta-gateway does this for its dev/prod split).
*/
const ICONS = {
"meta-core": "⚙️",
"meta-sort": "📁",
"meta-fuse": "🗂️",
"meta-stremio": "🎬",
"meta-dup": "🔍",
"meta-search": "🔎",
"meta-share": "🔗",
"meta-gateway": "🌉",
"meta-watch": "📺",
"meta-listen": "🎧",
"meta-read": "📖",
};
const DEFAULT_ICON = "📦";
const POLL_MS = 30000;
function titleCase(name) {
return String(name || "")
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ");
}
const TEMPLATE = `
<style>
:host {
--_fg: var(--mm-nav-fg, #e8eaed);
--_bg: var(--mm-nav-bg, #1a1a2e);
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
--_accent: var(--mm-nav-accent, #4ecdc4);
position: relative;
display: inline-block;
font-family: inherit;
font-size: 14px;
line-height: 1.4;
}
button.trigger {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.4rem;
width: 2.1rem;
height: 2.1rem;
padding: 0;
cursor: pointer;
color: var(--_fg);
background: transparent;
border: 1px solid var(--_border);
border-radius: 8px;
font: inherit;
transition: border-color 0.15s, background 0.15s;
}
button.trigger:hover { border-color: var(--_accent); }
button.trigger:focus-visible {
outline: 2px solid var(--_accent);
outline-offset: 2px;
}
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
.bars { display: block; width: 15px; }
.bars span {
display: block;
height: 2px;
margin: 3px 0;
border-radius: 2px;
background: currentColor;
}
.panel {
position: absolute;
top: calc(100% + 6px);
right: 0;
z-index: 1000;
min-width: 13rem;
max-height: 70vh;
overflow-y: auto;
padding: 0.3rem;
background: var(--_bg);
border: 1px solid var(--_border);
border-radius: 10px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
}
.panel[hidden] { display: none; }
a.item, .item {
display: flex;
align-items: center;
gap: 0.55rem;
padding: 0.45rem 0.6rem;
border-radius: 6px;
color: var(--_fg);
text-decoration: none;
white-space: nowrap;
}
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
.item.dead { opacity: 0.55; cursor: not-allowed; }
.icon { font-size: 1rem; line-height: 1; }
.dot {
width: 6px;
height: 6px;
margin-left: auto;
border-radius: 50%;
background: var(--_accent);
}
.empty {
padding: 0.6rem;
opacity: 0.7;
font-size: 0.85em;
white-space: nowrap;
}
/* Respect a host that opts out of motion. */
@media (prefers-reduced-motion: reduce) {
button.trigger { transition: none; }
}
</style>
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
</button>
<div class="panel" role="menu" hidden></div>
`;
class MetaServiceMenu extends HTMLElement {
constructor() {
super();
this.attachShadow({ mode: "open" });
this.shadowRoot.innerHTML = TEMPLATE;
this._btn = this.shadowRoot.querySelector("button.trigger");
this._panel = this.shadowRoot.querySelector(".panel");
this._neighbors = [];
this._timer = null;
this._open = false;
this._onDocClick = (e) => {
if (!this._open) return;
if (e.composedPath().includes(this)) return;
this._setOpen(false);
};
this._onKey = (e) => {
if (e.key === "Escape" && this._open) {
this._setOpen(false);
this._btn.focus();
}
};
}
connectedCallback() {
this._btn.title = this.getAttribute("label") || "Services";
this._btn.setAttribute("aria-label", this._btn.title);
this._btn.addEventListener("click", () => this._setOpen(!this._open));
document.addEventListener("click", this._onDocClick);
document.addEventListener("keydown", this._onKey);
this.refresh();
this._timer = setInterval(() => this.refresh(), POLL_MS);
}
disconnectedCallback() {
if (this._timer) clearInterval(this._timer);
this._timer = null;
document.removeEventListener("click", this._onDocClick);
document.removeEventListener("keydown", this._onKey);
}
get endpoint() {
return this.getAttribute("endpoint") || "/api/neighbors";
}
get current() {
return this.getAttribute("current") || "";
}
async refresh() {
try {
const res = await fetch(this.endpoint, {
headers: { Accept: "application/json" },
});
if (!res.ok) throw new Error(String(res.status));
const data = await res.json();
// `services` is the legacy alias some backends still emit.
this._neighbors = data.neighbors || data.services || [];
} catch {
// Silent: the menu is navigation, not a health indicator. Keep whatever
// we last had rather than blanking the list on one failed poll.
}
this._render();
}
_setOpen(open) {
this._open = open;
this._panel.hidden = !open;
this._btn.setAttribute("aria-expanded", String(open));
if (open) this.refresh();
}
_render() {
const panel = this._panel;
panel.textContent = "";
const seen = new Set();
const rows = [];
for (const n of this._neighbors) {
const name = n.name || "";
if (!name || seen.has(name)) continue;
seen.add(name);
rows.push(n);
}
// Always show the host itself, even before its own echo comes back.
if (this.current && !seen.has(this.current)) {
rows.push({ name: this.current, baseUrl: "" });
}
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
if (rows.length === 0) {
const empty = document.createElement("div");
empty.className = "empty";
empty.textContent = "No services found";
panel.appendChild(empty);
return;
}
for (const n of rows) {
const isActive = n.name === this.current;
let url = n.baseUrl || "";
if (typeof this.rewriteUrl === "function") {
try {
url = this.rewriteUrl(n.name, url, n) || "";
} catch {
/* a bad hook must not blank the menu */
}
}
// A neighbour with no reachable URL is shown but not linked — better
// than a link that goes nowhere.
const el = document.createElement(isActive || !url ? "span" : "a");
el.className =
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
if (el.tagName === "A") {
el.href = url;
el.setAttribute("role", "menuitem");
}
if (!isActive && !url) el.title = "No reachable URL announced";
const icon = document.createElement("span");
icon.className = "icon";
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
const label = document.createElement("span");
label.textContent = titleCase(n.name);
el.append(icon, label);
if (n.status === "running" || isActive) {
const dot = document.createElement("span");
dot.className = "dot";
dot.title = n.status || "running";
el.appendChild(dot);
}
panel.appendChild(el);
}
}
}
if (!customElements.get("meta-service-menu")) {
customElements.define("meta-service-menu", MetaServiceMenu);
}
export default MetaServiceMenu;