metastar/meta-sort
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
Pierre
2026-01-11 17:26:49 +01:00
commit 888790fd11
240 changed files with 48691 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.dockerignore
+6
View File
@@ -0,0 +1,6 @@
node_modules
**/node_modules
**/.pnpm-store
packages/*/dist
*.log
.git
.github/workflows/docker-publish.yml
+68
View File
@@ -0,0 +1,68 @@
name: Build and Push Docker Image
on:
push:
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Tag for the image (optional, defaults to sha)'
required: false
type: string
meta_core_version:
description: 'meta-core version to use (default: latest)'
required: false
type: string
default: 'latest'
env:
REGISTRY: ghcr.io
IMAGE_NAME: ${{ github.repository }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
with:
submodules: recursive # For plugins
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Log in to Container Registry
uses: docker/login-action@v3
with:
registry: ${{ env.REGISTRY }}
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
META_CORE_IMAGE=ghcr.io/worph/meta-core:${{ inputs.meta_core_version || 'latest' }}
cache-from: type=gha
cache-to: type=gha,mode=max
.gitignore
+29
View File
@@ -0,0 +1,29 @@
# Build output
dist/
# Dependencies
node_modules/
# Environment
.env
.env.local
# IDE
.idea/
.vscode/
*.swp
*.swo
# OS
.DS_Store
Thumbs.db
# Test data
test-data/
# Logs
*.log
npm-debug.log*
# Cache
.cache/
Dockerfile
+205
View File
@@ -0,0 +1,205 @@
# meta-sort standalone Docker image
# Includes: Redis, nginx, rclone, FFmpeg, Node.js backend, React UI, and Editor
#
# Build standalone:
# docker build -t meta-sort .
#
# Build with custom meta-core (for development):
# docker build --build-arg META_CORE_IMAGE=meta-core:local -t meta-sort .
# Stage 0: Get meta-core binary from published image
ARG META_CORE_IMAGE=ghcr.io/worph/meta-core:latest
FROM ${META_CORE_IMAGE} AS meta-core
# Stage 1: Build UI
FROM node:21-alpine AS ui-builder
WORKDIR /build
# Copy UI package and install
COPY packages/meta-sort-ui/package.json ./
RUN npm install
# Copy UI source and build
COPY packages/meta-sort-ui/ ./
RUN npm run build
# Stage 2: Build Editor
FROM node:21-alpine AS editor-builder
WORKDIR /build
# Copy editor package and install
COPY packages/meta-sort-editor/package.json ./
RUN npm install
# Copy editor source and build
COPY packages/meta-sort-editor/ ./
RUN npm run build
# Stage 3: Build Backend using pnpm workspace
FROM node:21-alpine AS backend-builder
# Install pnpm
RUN npm install -g pnpm
WORKDIR /build
# Copy workspace configuration
COPY package.json pnpm-lock.yaml pnpm-workspace.yaml ./
# Copy all packages for the workspace
COPY packages/shared/ ./packages/shared/
COPY packages/async-utils/ ./packages/async-utils/
COPY packages/meta-sort-core/ ./packages/meta-sort-core/
# Install all dependencies
RUN pnpm install --frozen-lockfile || pnpm install
# Build shared dependencies first
RUN pnpm --filter="@metazla/meta-interface" build || true
RUN pnpm --filter="@metazla/meta-hash" build || true
RUN pnpm --filter="@metazla/filename-tools" build || true
# Build async-utils
RUN pnpm --filter="@worph/async-utils" build || true
# Build the core package
RUN pnpm --filter="@meta-sort/core" build
# Stage 4: Runtime
FROM ubuntu:22.04
# Container registry metadata
LABEL org.opencontainers.image.source=https://github.com/worph/meta-sort
LABEL org.opencontainers.image.description="MetaMesh file processing and metadata extraction service"
LABEL org.opencontainers.image.licenses=MIT
# Avoid prompts during install
ENV DEBIAN_FRONTEND=noninteractive
# Install system dependencies
RUN apt-get update && apt-get install -y \
curl \
gnupg \
ca-certificates \
nginx \
redis-server \
supervisor \
ffmpeg \
apache2-utils \
fuse3 \
unzip \
cifs-utils \
nfs-common \
&& rm -rf /var/lib/apt/lists/*
# Install Node.js 21
RUN curl -fsSL https://deb.nodesource.com/setup_21.x | bash - \
&& apt-get install -y nodejs
# Install rclone
RUN curl https://rclone.org/install.sh | bash
# Create directories
RUN mkdir -p \
/app/backend \
/app/ui \
/app/editor \
/data/watch \
/data/cache \
/data/redis \
/data/mounts \
/meta-core/db/redis \
/meta-core/locks \
/meta-core/services \
/files \
/var/log/supervisor \
/var/log/nginx \
/root/.config/rclone
# Copy built UI
COPY --from=ui-builder /build/dist /app/ui
RUN chmod -R 755 /app/ui
# Copy built Editor
COPY --from=editor-builder /build/dist /app/editor
RUN chmod -R 755 /app/editor
# Copy built backend with all dependencies
COPY --from=backend-builder /build/packages/meta-sort-core/dist /app/backend/dist
COPY --from=backend-builder /build/packages/meta-sort-core/package.json /app/backend/
# Copy plugins directory
COPY --from=backend-builder /build/packages/meta-sort-core/plugins /app/backend/plugins
# Copy node_modules (has all deps including workspace packages)
COPY --from=backend-builder /build/node_modules /app/node_modules
# Recreate ALL pnpm symlinks (pnpm stores in .pnpm, symlinks at top level)
RUN for pkg_path in /app/node_modules/.pnpm/*/node_modules/*; do \
pkg_name=$(basename "$pkg_path"); \
if [ ! -L "/app/node_modules/$pkg_name" ] && [ ! -d "/app/node_modules/$pkg_name" ]; then \
ln -sf "$pkg_path" /app/node_modules/$pkg_name 2>/dev/null || true; \
fi; \
done
# Also handle scoped packages (@scope/name)
RUN for scope_path in /app/node_modules/.pnpm/*/node_modules/@*; do \
if [ -d "$scope_path" ]; then \
scope_name=$(basename "$scope_path"); \
mkdir -p "/app/node_modules/$scope_name" 2>/dev/null; \
for pkg_path in "$scope_path"/*; do \
pkg_name=$(basename "$pkg_path"); \
if [ ! -L "/app/node_modules/$scope_name/$pkg_name" ] && [ ! -d "/app/node_modules/$scope_name/$pkg_name" ]; then \
ln -sf "$pkg_path" /app/node_modules/$scope_name/$pkg_name 2>/dev/null || true; \
fi; \
done; \
fi; \
done
# Copy built workspace packages (overwrite symlinks with actual dist)
COPY --from=backend-builder /build/packages/shared/meta-hash/dist /app/node_modules/@metazla/meta-hash/dist
COPY --from=backend-builder /build/packages/shared/meta-hash/package.json /app/node_modules/@metazla/meta-hash/
COPY --from=backend-builder /build/packages/shared/meta-interface/dist /app/node_modules/@metazla/meta-interface/dist
COPY --from=backend-builder /build/packages/shared/meta-interface/package.json /app/node_modules/@metazla/meta-interface/
COPY --from=backend-builder /build/packages/shared/filename-tool/dist /app/node_modules/@metazla/filename-tools/dist
COPY --from=backend-builder /build/packages/shared/filename-tool/package.json /app/node_modules/@metazla/filename-tools/
COPY --from=backend-builder /build/packages/async-utils/dist /app/node_modules/@worph/async-utils/dist
COPY --from=backend-builder /build/packages/async-utils/package.json /app/node_modules/@worph/async-utils/
# Copy meta-core sidecar binary
COPY --from=meta-core /usr/local/bin/meta-core /usr/local/bin/meta-core
RUN chmod +x /usr/local/bin/meta-core
# Copy configuration files
COPY docker/nginx.conf /etc/nginx/nginx.conf
COPY docker/redis.conf /etc/redis/redis.conf
COPY docker/supervisord.conf /etc/supervisor/conf.d/supervisord.conf
# Create nginx htpasswd file (default: admin/admin)
RUN htpasswd -bc /etc/nginx/.htpasswd admin admin
# Environment variables
ENV NODE_ENV=production \
NODE_PATH=/app/node_modules \
WATCH_PATHS=/data/watch \
META_CORE_PATH=/meta-core \
FILES_PATH=/files \
REDIS_URL=redis://127.0.0.1:6379 \
REDIS_PREFIX=meta-sort: \
API_HOST=0.0.0.0 \
API_PORT=3000
# Set working directory for backend
WORKDIR /app/backend
# Expose port 80 (nginx)
EXPOSE 80
# Health check
HEALTHCHECK --interval=30s --timeout=10s --start-period=30s --retries=3 \
CMD curl -f http://localhost/health || exit 1
# Start supervisord (manages redis, backend, nginx)
CMD ["/usr/bin/supervisord", "-c", "/etc/supervisor/conf.d/supervisord.conf"]
README.md
+841
View File
@@ -0,0 +1,841 @@
# meta-sort
A standalone file sorting and metadata extraction service for the MetaMesh ecosystem. meta-sort monitors watch folders, extracts metadata from media files, stores it in a distributed key-value store, and manages remote drive mounting for shared access.
## Overview
meta-sort is a **process-write** service that serves as the primary data ingestion point for the MetaMesh platform. It:
1. **Watches folders** for new media files (video, audio, subtitles)
2. **Extracts metadata** using FFmpeg and filename parsing
3. **Computes content identifiers** using midhash256 (fast) and SHA-256 (background)
4. **Stores metadata** in a distributed KV store (Redis or etcd)
5. **Manages remote mounts** (NFS, SMB, cloud storage) for shared access
---
## Meta-Core Architecture
MetaMesh uses **two shared volumes** for operation:
### 1. META_CORE_VOLUME - Infrastructure
The infrastructure volume stores all service coordination data:
| Context | Path | Description |
|---------|------|-------------|
| **Inside Container** | `/meta-core` | Mounted shared volume |
| **Production (Host)** | `/DATA/Apps/meta-core` | External shared drive mount point |
| **Development** | `./data/meta-core` | Local development directory |
```
META_CORE_VOLUME (/meta-core or /DATA/Apps/meta-core)
├── db/ # KV Database storage
│ ├── redis/ # Redis persistence (RDB + AOF)
│ │ ├── dump.rdb # Redis snapshot
│ │ └── appendonly.aof # Redis append-only log
│ └── etcd/ # etcd data (alternative)
│ └── member/
│
├── locks/ # Distributed locking
│ └── kv-leader.lock # Leader election lock file
│ # Contains: {"host":"service-name","api":"http://host:port"}
│
├── services/ # Service discovery registry
│ ├── meta-sort.json # {"name":"meta-sort","api":"http://...","status":"running","pid":123}
│ ├── meta-fuse.json # {"name":"meta-fuse","api":"http://...","status":"running","pid":456}
│ ├── meta-stremio.json # {"name":"meta-stremio","api":"http://...","status":"running","pid":789}
│ └── meta-orbit.json # {"name":"meta-orbit","api":"http://...","status":"running","pid":321}
│
└── config/ # Shared configuration
├── remote-storage.conf # Remote mount definitions
└── smb-credentials.txt # SMB credentials (chmod 600)
```
### 2. FILES_VOLUME - Shared Media Drive
The **FILES_VOLUME** (canonical name: `/files`) is the shared drive for all media files. This is where all actual media content resides, and file paths in the KV database reference this volume.
| Context | Path | Description |
|---------|------|-------------|
| **Inside Container** | `/files` | Mounted shared media volume |
| **Production (Host)** | Varies (NFS/SMB/local) | External shared drive |
| **Development** | `./data/files` | Local development directory |
```
FILES_VOLUME (/files)
├── media1/ # Local folder mounted from host
│ ├── Movies/
│ │ └── Inception (2010)/
│ │ └── Inception.mkv
│ └── TV Shows/
│ └── Breaking Bad/
│ └── Season 01/
│
├── media2/ # Another local media folder
│ └── Anime/
│ └── Steins Gate/
│
├── smb/ # SMB/CIFS remote mount
│ └── NAS-Media/
│ └── Movies/
│
├── nfs/ # NFS remote mount
│ └── server-media/
│
└── gdrive/ # rclone remote mount (Google Drive)
└── Cloud-Media/
```
### File Path References
All file paths stored in the KV database are **relative to FILES_VOLUME**:
```
KV Key: /file/{midhash256}/filePath
KV Value: "media1/Movies/Inception (2010)/Inception.mkv"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Relative to FILES_VOLUME (/files)
Full path construction:
Container: /files + /media1/Movies/... = /files/media1/Movies/Inception (2010)/Inception.mkv
Any host: $FILES_VOLUME + /media1/Movies/... = $FILES_VOLUME/media1/Movies/Inception (2010)/Inception.mkv
```
This allows services to be deployed on different hosts while sharing the same volume. The FILES_VOLUME can be:
- A local folder bind mount
- An NFS share mounted on the host
- An SMB/CIFS share
- A FUSE mount (rclone, sshfs, etc.)
### Mount Management
meta-sort manages remote mounts **inside** the FILES_VOLUME for shared access:
```bash
# meta-sort mounts remote storage INTO /files
/files/smb/nas-share ← SMB mount managed by meta-sort
/files/nfs/server ← NFS mount managed by meta-sort
/files/gdrive/ ← rclone mount managed by meta-sort
# Other folders are bind mounts from host
/files/media1 ← Host folder mounted into container
/files/media2 ← Another host folder
```
All services access files through `/files/*`, regardless of whether the underlying storage is local, NFS, SMB, or cloud.
---
## Leader Election and Lock Mechanism
### How It Works
MetaMesh uses a **filesystem-based distributed consensus** mechanism using `flock(2)`. The shared filesystem acts as the consensus layer.
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Leader Election via flock │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ Step 1: Service Startup │
│ ┌─────────────┐ │
│ │ Service │ ──▶ Try to acquire flock on /meta-core/locks/kv-leader.lock│
│ └─────────────┘ │
│ │ │
│ ▼ │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ flock() Result │ │
│ │ ┌─────────────────────┐ ┌──────────────────────────────────┐ │ │
│ │ │ Lock Acquired │ │ Lock Blocked │ │ │
│ │ │ ────────────── │ │ ───────────── │ │ │
│ │ │ │ │ │ │ │
│ │ │ BECOME LEADER │ │ BECOME FOLLOWER │ │ │
│ │ │ 1. Spawn Redis │ │ 1. Read lock file │ │ │
│ │ │ 2. Write API │ │ 2. Extract leader API │ │ │
│ │ │ endpoint to │ │ 3. Connect to leader's │ │ │
│ │ │ lock file │ │ Redis │ │ │
│ │ │ 3. Accept clients │ │ 4. Start reconnect loop │ │ │
│ │ └─────────────────────┘ └──────────────────────────────────┘ │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Lock File Format
The lock file at `/meta-core/locks/kv-leader.lock` contains JSON:
```json
{
"host": "meta-sort-instance-1",
"api": "redis://10.0.1.50:6379",
"http": "http://10.0.1.50:3000",
"timestamp": 1703808000000,
"pid": 12345
}
```
### flock Implementation
```typescript
import { open, flock } from 'fs';
import { promisify } from 'util';
const LOCK_FILE = '/meta-core/locks/kv-leader.lock';
async function tryBecomeLeader(): Promise<boolean> {
const fd = await promisify(open)(LOCK_FILE, 'w');
try {
// LOCK_EX = exclusive lock, LOCK_NB = non-blocking
await promisify(flock)(fd, 'exnb'); // Throws if cannot acquire
// We are the leader - spawn KV database and write our endpoint
await spawnRedis();
await fs.writeFile(LOCK_FILE, JSON.stringify({
host: hostname(),
api: `redis://${myIP}:6379`,
http: `http://${myIP}:3000`,
timestamp: Date.now(),
pid: process.pid
}));
return true; // We are leader
} catch (err) {
if (err.code === 'EAGAIN' || err.code === 'EWOULDBLOCK') {
// Lock held by another process - become follower
return false;
}
throw err;
}
}
```
### Failure Detection and Recovery
```
┌──────────────────────────────────────────────────────────────────────────┐
│ Failure Detection Loop │
├──────────────────────────────────────────────────────────────────────────┤
│ │
│ Follower Service: │
│ ┌─────────────────────────────────────────────────────────────────────┐ │
│ │ while (true) { │ │
│ │ try { │ │
│ │ await redis.ping(); // Health check │ │
│ │ await sleep(5000); // Wait 5 seconds │ │
│ │ } catch (connectionError) { │ │
│ │ // Leader might be down │ │
│ │ await tryBecomeLeader(); // Attempt to take over │ │
│ │ if (isLeader) break; │ │
│ │ await reconnectToNewLeader(); // Or connect to new leader │ │
│ │ } │ │
│ │ } │ │
│ └─────────────────────────────────────────────────────────────────────┘ │
│ │
│ When Leader Dies: │
│ 1. flock is automatically released by OS │
│ 2. First follower to call flock() wins │
│ 3. New leader spawns Redis, writes new endpoint │
│ 4. Other followers detect connection failure, read new endpoint │
│ │
└──────────────────────────────────────────────────────────────────────────┘
```
---
## Service Discovery
### Registration Mechanism
Each service registers itself in `/meta-core/services/{service-name}.json`:
```json
{
"name": "meta-sort",
"version": "1.0.0",
"api": "http://10.0.1.50:3000",
"status": "running",
"pid": 12345,
"hostname": "meta-sort-instance-1",
"startedAt": "2024-12-28T10:00:00Z",
"lastHeartbeat": "2024-12-28T12:30:00Z",
"capabilities": ["write", "mount", "monitor"],
"endpoints": {
"health": "/health",
"api": "/api",
"metrics": "/api/metrics"
}
}
```
### Service Lifecycle
```
┌──────────────────────────────────────────────────────────────────────────┐
│ Service Lifecycle │
├──────────────────────────────────────────────────────────────────────────┤
│ │
│ Startup: │
│ 1. Create/update /meta-core/services/{name}.json │
│ 2. Set status = "starting" │
│ 3. Initialize (connect to KV, etc.) │
│ 4. Set status = "running" │
│ 5. Start heartbeat loop (update lastHeartbeat every 30s) │
│ │
│ Running: │
│ - Heartbeat updates lastHeartbeat timestamp │
│ - Other services read this file to discover endpoints │
│ - stale detection: if (now - lastHeartbeat > 60s) → service dead │
│ │
│ Shutdown: │
│ 1. Set status = "stopping" │
│ 2. Graceful shutdown (finish current work) │
│ 3. Delete /meta-core/services/{name}.json OR set status = "stopped" │
│ │
└──────────────────────────────────────────────────────────────────────────┘
```
### Discovering Other Services
```typescript
async function discoverService(name: string): Promise<ServiceInfo | null> {
const servicePath = `/meta-core/services/${name}.json`;
try {
const content = await fs.readFile(servicePath, 'utf-8');
const service = JSON.parse(content);
// Check if service is alive (heartbeat within last 60 seconds)
const lastHeartbeat = new Date(service.lastHeartbeat).getTime();
const isAlive = Date.now() - lastHeartbeat < 60_000;
if (!isAlive) {
console.warn(`Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`);
return null;
}
return service;
} catch (err) {
if (err.code === 'ENOENT') {
return null; // Service not registered
}
throw err;
}
}
```
---
## MetaMesh Ecosystem Architecture
### Service Hierarchy
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ MetaMesh Ecosystem │
│ │
│ Shared Volumes: │
│ ┌──────────────────────────────────┐ ┌──────────────────────────────────┐ │
│ │ META_CORE_VOLUME │ │ FILES_VOLUME │ │
│ │ (/meta-core) │ │ (/files) │ │
│ │ ┌────────┐ ┌────────┐ ┌──────┐ │ │ ┌────────┐ ┌────────┐ ┌──────┐ │ │
│ │ │ db/ │ │ locks/ │ │svc/ │ │ │ │media1/ │ │ smb/ │ │ nfs/ │ │ │
│ │ │(Redis) │ │(flock) │ │(JSON)│ │ │ │(local) │ │(mount) │ │(mnt) │ │ │
│ │ └────┬───┘ └───┬────┘ └──┬───┘ │ │ └───┬────┘ └───┬────┘ └──┬───┘ │ │
│ └───────┼─────────┼─────────┼─────┘ └──────┼──────────┼─────────┼─────┘ │
│ │ │ │ │ │ │ │
│ ▼ ▼ ▼ ▼ ▼ ▼ │
│ ┌───────────────────────────────────────────────────────────────────────┐│
│ │ ││
│ │ [1] meta-sort PROCESS-WRITE ││
│ │ ├── Watches /files/* for new media files ││
│ │ ├── Extracts metadata → writes to KV (Redis in /meta-core/db) ││
│ │ ├── Manages remote mounts INTO /files (NFS, SMB, rclone) ││
│ │ ├── Leader election (spawns Redis if leader) ││
│ │ └── Serves monitoring UI (meta-sort-editor) ││
│ │ ││
│ │ [2] meta-fuse PROCESS-READ ││
│ │ ├── Reads metadata from KV (Redis) ││
│ │ ├── Builds virtual filesystem from metadata ││
│ │ ├── Serves FUSE mount + WebDAV ││
│ │ └── File paths resolve to /files/* ││
│ │ ││
│ │ [3] meta-stremio PROCESS-READ ││
│ │ ├── Reads metadata from KV (Redis) ││
│ │ ├── Stremio addon server (manifest, catalog, streams) ││
│ │ ├── HLS transcoding via FFmpeg ││
│ │ └── Video files from /files/* ││
│ │ ││
│ │ [4] meta-orbit SHARING-READ-WRITE ││
│ │ ├── P2P metadata synchronization (libp2p + OrbitDB) ││
│ │ ├── Decentralized search across network ││
│ │ ├── Reads from local KV, shares to network ││
│ │ └── Receives from network, writes to local KV ││
│ │ ││
│ └───────────────────────────────────────────────────────────────────────┘│
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
### Data Flow
```
┌─────────────────────────────────────────────────────────────────────────────┐
│ Data Flow │
├─────────────────────────────────────────────────────────────────────────────┤
│ │
│ File Discovery: │
│ │
│ /files/media1/Movies/Inception (2010)/Inception.mkv │
│ │ │
│ ▼ │
│ [meta-sort] ──▶ Extract metadata ──▶ Write to Redis (in /meta-core/db) │
│ │ │ │
│ │ Key: file:{midhash256} │ │
│ │ filePath: "media1/Movies/Inception (2010)/Inception.mkv" │
│ │ title: "Inception" │
│ │ year: "2010" │
│ │ duration: 8880 │
│ │ │ │
│ ▼ ▼ │
│ │
│ File Access: │
│ │
│ [meta-fuse] ◀── Read from Redis │
│ │ │
│ ▼ │
│ Virtual FS: /mnt/virtual/Movies/Inception (2010)/Inception.mkv │
│ │ ↓ │
│ │ symlink to /files/media1/Movies/Inception (2010)/... │
│ │ │
│ [meta-stremio] ◀── Read from Redis │
│ │ │
│ ▼ │
│ Stream: GET /stremio/stream/{midhash256}/playlist.m3u8 │
│ │ ↓ │
│ │ transcode /files/media1/Movies/Inception (2010)/... │
│ │ │
│ [meta-orbit] ◀──▶ Sync with P2P network │
│ │ │
│ ▼ │
│ Share metadata to peers / Receive metadata from peers │
│ │
└─────────────────────────────────────────────────────────────────────────────┘
```
---
## Deployment
### Docker Compose (Production)
```yaml
version: '3.8'
services:
meta-sort:
image: metazla/meta-sort:latest
volumes:
# Infrastructure volume (KV DB, locks, services)
- /DATA/Apps/meta-core:/meta-core:rw
# Shared media volume
- /DATA/Apps/meta-core/files:/files:rw,shared
# OR mount multiple sources into /files
# - /mnt/nas/media:/files/nas:ro
# - /mnt/local/media:/files/local:ro
environment:
- META_CORE_PATH=/meta-core
- FILES_PATH=/files
- WATCH_PATHS=/files
- REDIS_DATA_DIR=/meta-core/db/redis
privileged: true # Required for rclone FUSE mounts
meta-fuse:
image: metazla/meta-fuse:latest
volumes:
# Infrastructure (read-only for followers)
- /DATA/Apps/meta-core:/meta-core:ro
# Shared media volume (read-only)
- /DATA/Apps/meta-core/files:/files:ro
# FUSE mount output
- /mnt/metamesh:/mnt/metamesh:rw,shared
environment:
- META_CORE_PATH=/meta-core
- FILES_PATH=/files
privileged: true # Required for FUSE
cap_add:
- SYS_ADMIN
devices:
- /dev/fuse
meta-stremio:
image: metazla/meta-stremio:latest
volumes:
# Infrastructure (read-only)
- /DATA/Apps/meta-core:/meta-core:ro
# Shared media volume (read-only)
- /DATA/Apps/meta-core/files:/files:ro
environment:
- META_CORE_PATH=/meta-core
- FILES_PATH=/files
ports:
- "7000:7000"
meta-orbit:
image: metazla/meta-orbit:latest
volumes:
# Infrastructure (read-write for P2P sync)
- /DATA/Apps/meta-core:/meta-core:rw
environment:
- META_CORE_PATH=/meta-core
ports:
- "4001:4001" # libp2p
```
### Environment Variables
| Variable | Default | Description |
|----------|---------|-------------|
| `META_CORE_PATH` | `/meta-core` | Path to META_CORE_VOLUME (infrastructure) |
| `FILES_PATH` | `/files` | Path to FILES_VOLUME (shared media) |
| `REDIS_URL` | Auto-discover | Redis connection (from leader lock file) |
| `WATCH_PATHS` | `$FILES_PATH` | Folders to watch for media |
| `SERVICE_NAME` | Container hostname | Unique service identifier |
---
## File Processing Pipeline
meta-sort uses a **plugin-based architecture** with a two-queue task scheduler:
```
┌──────────────┐ ┌────────────────────────────────────────────────────┐
│ Discovery │ │ Task Scheduler │
│ (Stream) │───▶│ ┌─────────────────┐ ┌──────────────────────────┐ │
└──────────────┘ │ │ Fast Queue │ │ Background Queue │ │
Watch folder │ │ (16 workers) │ │ (4 workers) │ │
+ manual scan │ │ <1s tasks │ │ >=1s tasks │ │
│ └─────────────────┘ └──────────────────────────┘ │
└────────────────────────────────────────────────────┘
```
Tasks are automatically classified and scheduled based on plugin dependencies:
```
Discovery → file-info → ffmpeg → filename-parser → jellyfin-nfo → tmdb
↓
anime-detector ← language ← subtitle ← torrent ← full-hash
```
---
## Plugin System
Plugins are the core processing units that extract and enrich metadata. Each plugin:
- Has a `manifest.yml` defining metadata schema, config, and dependencies
- Runs in either **fast** (high concurrency) or **background** (low concurrency) queue
- Can depend on other plugins to ensure correct execution order
### Built-in Plugins
| Plugin | Queue | Priority | Description |
|--------|-------|----------|-------------|
| **file-info** | fast | 10 | Basic file info (type, MIME, size, midhash256) |
| **ffmpeg** | fast | 15 | Video/audio/subtitle stream metadata via FFprobe |
| **filename-parser** | fast | 20 | Parse title, season, episode, year, quality from filename |
| **jellyfin-nfo** | fast | 25 | Parse Jellyfin/Kodi NFO metadata files |
| **tmdb** | background | 30 | Fetch metadata from The Movie Database API |
| **anime-detector** | fast | 35 | Detect anime content from keywords and audio tracks |
| **language** | fast | 40 | Aggregate languages from all streams (ISO 639-3) |
| **subtitle** | fast | 45 | Find sibling subtitle files and detect language |
| **torrent** | fast | 50 | Parse .torrent file metadata |
| **full-hash** | background | 100 | Compute SHA-256 hash of entire file |
### Plugin Manifest Example
```yaml
id: ffmpeg
name: FFmpeg Metadata
version: 1.0.0
description: Extracts video/audio/subtitle stream metadata using FFprobe
priority: 15
color: "#4CAF50"
defaultQueue: fast
dependencies:
- file-info
config: {}
schema:
fileinfo/duration:
label: Duration
type: string
readonly: true
fileinfo/streamdetails/video:
label: Video Streams
type: json
readonly: true
pattern: "fileinfo/streamdetails/video/{n}/*"
```
### Task Queue Architecture
- **Fast Queue** (16 workers): High-throughput for quick operations (<1s)
- File info extraction, FFprobe, filename parsing, NFO reading
- **Background Queue** (4 workers): Low-concurrency for slow operations (>=1s)
- TMDB API calls, full SHA-256 hashing
- Automatically pauses when fast queue has work, resumes when idle
### Plugin Management
Plugins can be enabled/disabled and configured via:
- **Dashboard**: http://localhost:8180/ → Plugins tab
- **Config file**: `plugins.json` in cache directory
---
## Processing States
1. **Pending** - File discovered in watch folder
2. **Processing** - Plugins running (fast queue first, then background)
3. **Done** - All plugins completed successfully
---
## Package Structure
```
meta-sort/
├── packages/
│ ├── meta-sort-core/ # Core processing service (@meta-sort/core)
│ │ ├── plugins/ # Built-in plugins
│ │ │ ├── file-info/ # Basic file information
│ │ │ ├── ffmpeg/ # FFprobe metadata extraction
│ │ │ ├── filename-parser/# Filename parsing
│ │ │ ├── jellyfin-nfo/ # NFO file parsing
│ │ │ ├── tmdb/ # TMDB API integration
│ │ │ ├── anime-detector/ # Anime content detection
│ │ │ ├── language/ # Language aggregation
│ │ │ ├── subtitle/ # Subtitle file finder
│ │ │ ├── torrent/ # Torrent file parsing
│ │ │ └── full-hash/ # SHA-256 full file hashing
│ │ ├── src/
│ │ │ ├── api/ # Fastify REST API (UnifiedAPIServer)
│ │ │ ├── config/ # Environment configuration
│ │ │ ├── jellyfin/ # Jellyfin integration
│ │ │ ├── kv/ # KV manager with leader election (Redis)
│ │ │ ├── logic/ # File processing logic
│ │ │ ├── metrics/ # Performance tracking
│ │ │ ├── plugin-engine/ # Plugin loader and task scheduler
│ │ │ ├── types/ # TypeScript types
│ │ │ ├── utils/ # Utility functions
│ │ │ └── index.ts # Entry point
│ │ └── package.json
│ │
│ ├── meta-sort-editor/ # Metadata editor UI (@meta-sort/editor)
│ │ └── ... # React/Vite app
│ │
│ ├── meta-sort-ui/ # Monitoring dashboard (@meta-sort/ui)
│ │ └── ... # React/Vite app
│ │
│ └── async-utils/ # Async utility library
│ └── ... # MultiQueue, PromiseQueue, etc.
│
├── docker/
│ ├── nginx.conf # Reverse proxy config
│ ├── redis.conf # Redis configuration
│ └── supervisord.conf # Process management
│
├── Dockerfile
├── docker-compose.yml
├── package.json # Workspace root
├── pnpm-workspace.yaml
└── README.md
```
### Key Components (meta-sort-core)
| Component | File | Purpose |
|-----------|------|---------|
| Entry Point | `src/index.ts` | Initializes KV manager, plugin engine, API server, file watching |
| File Processor | `src/logic/WatchedFileProcessor.ts` | File discovery and plugin task scheduling |
| Plugin Loader | `src/plugin-engine/PluginLoader.ts` | Loads plugins from manifest.yml, manages plugin lifecycle |
| Task Scheduler | `src/plugin-engine/TaskScheduler.ts` | Two-queue scheduler (fast/background) with dependency resolution |
| KV Manager | `src/kv/KVManager.ts` | Leader election via flock, Redis spawning, service discovery |
| Unified API | `src/api/UnifiedAPIServer.ts` | Fastify REST API for monitoring and control |
| Performance Metrics | `src/metrics/PerformanceMetrics.ts` | Tracks processing throughput and timing |
| Config | `src/config/EnvConfig.ts` | Environment variable parsing |
---
## Core Features
### 1. Folder Watching
- **Real-time monitoring** using chokidar for file system events
- **Streaming discovery** via async generators for efficient memory usage
- **Configurable watch paths** supporting local and remote mounted directories
- **File type filtering** (video, audio, subtitle, torrent files)
### 2. Plugin-Based Metadata Extraction
Metadata extraction is handled by plugins that can be enabled/disabled and configured:
- **file-info** - Basic file info, MIME type, midhash256 content identifier
- **ffmpeg** - Duration, codec, resolution, bitrate, audio tracks, subtitles via FFprobe
- **filename-parser** - Extract title, year, season, episode, quality from filenames
- **jellyfin-nfo** - Parse NFO sidecar files (Jellyfin/Kodi format)
- **tmdb** - Fetch rich metadata from The Movie Database API
- **anime-detector** - Detect anime content from keywords and audio tracks
- **language** - Aggregate languages from all streams (ISO 639-3)
- **subtitle** - Find sibling subtitle files and detect language
- **torrent** - Parse .torrent file metadata
- **full-hash** - Compute SHA-256 hash of entire file (background)
### 3. KV Storage
- **Nested key architecture** for property-level granularity
- **Leader election** via flock for high availability
- **Service discovery** via shared filesystem
- **CRDT-compatible** for distributed synchronization
```
Key Structure:
/file/{midhash256}/title → "Inception"
/file/{midhash256}/year → "2010"
/file/{midhash256}/video/codec → "h265"
/file/{midhash256}/audio/0/language → "en"
/file/{midhash256}/filePath → "media1/Movies/Inception (2010)/Inception.mkv"
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
Relative to FILES_VOLUME (/files)
```
### 4. Remote Mount Management
- **NFS shares** - Network file systems
- **SMB/CIFS** - Windows network shares with credentials
- **rclone** - Cloud storage (Google Drive, Dropbox, S3, etc.)
- **Automatic mounting** on startup
- **Mounts created inside FILES_VOLUME** for shared access by all services
Mounts are created inside `/files/` (FILES_VOLUME) so all services can access them:
```bash
# Example mount structure
/files/ # FILES_VOLUME
├── media1/ # Local folder bind mount from host
├── media2/ # Another local folder
├── smb/ # SMB/CIFS remote mounts
│ └── nas-share/ # SMB: //server/share
├── nfs/ # NFS remote mounts
│ └── server-media/ # NFS: nas.local:/volume1/media
└── gdrive/ # rclone: gdrive:Media
```
---
## Configuration
### Environment Variables
```bash
# Volume paths
META_CORE_PATH=/meta-core # Infrastructure volume
FILES_PATH=/files # Shared media volume
# Watch paths (comma-separated, relative to FILES_PATH or absolute)
WATCH_PATHS=/files
# Processing configuration
MAX_WORKER_THREADS=16 # Defaults to CPU count
LIGHT_PROCESSING_TIMEOUT_MS=120000
HASH_PROCESSING_TIMEOUT_MS=7200000
# Remote mount configuration
REMOTE_STORAGE_CONFIG=/meta-core/config/remote-storage.conf
```
### Remote Storage Configuration
Create `/meta-core/config/remote-storage.conf`:
```conf
# NFS mount (into FILES_VOLUME)
nfs://nas.local:/volume1/media /files/nfs/nas-media nfs defaults,rw 0 0
# SMB mount (credentials in separate file)
//server/share /files/smb/nas-share cifs credentials=/meta-core/config/smb-creds.txt,uid=1000 0 0
# rclone mount
gdrive: /files/gdrive rclone vfs-cache-mode=full 0 0
```
---
## API
### REST Endpoints
| Endpoint | Method | Description |
|----------|--------|-------------|
| `/api/health` | GET | Service health check |
| `/api/scan/trigger` | POST | Trigger manual folder scan |
| `/api/processing/status` | GET | Current pipeline state |
| `/api/metrics` | GET | Performance metrics |
| `/api/mounts` | GET | List mounted remote storage |
| `/api/mounts` | POST | Add new remote mount |
| `/api/mounts/{id}` | DELETE | Remove remote mount |
### Processing Status Response
```json
{
"totalPending": 150,
"totalLightProcessing": 16,
"totalHashProcessing": 8,
"totalDone": 45000,
"totalFilesInVFS": 45150,
"throughput": {
"filesPerSecond": 34,
"avgLightProcessingMs": 470,
"avgHashProcessingMs": 15000
}
}
```
---
## Usage
### Development
```bash
# Install dependencies
pnpm install
# Build all packages
pnpm run build
# Run in development mode
pnpm run dev
```
### Docker
```bash
# Build and run with Docker Compose
docker-compose up -d
# Check health
curl http://localhost/health
curl http://localhost/api/processing/status
```
---
## License
MIT License - see LICENSE file for details.
docker/nginx.conf
+91
View File
@@ -0,0 +1,91 @@
worker_processes auto;
error_log /var/log/nginx/error.log warn;
pid /var/run/nginx.pid;
events {
worker_connections 1024;
}
http {
include /etc/nginx/mime.types;
default_type application/octet-stream;
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /var/log/nginx/access.log main;
sendfile on;
tcp_nopush on;
keepalive_timeout 65;
gzip on;
gzip_types text/plain text/css application/json application/javascript text/xml application/xml;
# Upstream servers
upstream api {
server 127.0.0.1:3000;
}
upstream rclone {
server 127.0.0.1:5572;
}
server {
listen 80;
server_name _;
# Static files (UI)
root /app/ui;
index index.html;
# SPA routing - serve index.html for any non-file request
location / {
try_files $uri $uri/ /index.html;
}
# Editor UI - SPA with base path /editor/
location /editor/ {
alias /app/editor/;
index index.html;
try_files $uri $uri/ /editor/index.html;
}
# Redirect /editor to /editor/
location = /editor {
return 301 /editor/;
}
# API routes
location /api/ {
proxy_pass http://api;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
}
# Health check (direct to API)
location /health {
proxy_pass http://api/health;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
# rclone web UI
location /rclone/ {
proxy_pass http://rclone/;
proxy_http_version 1.1;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Upgrade $http_upgrade;
proxy_set_header Connection "upgrade";
# Basic auth for rclone
auth_basic "rclone";
auth_basic_user_file /etc/nginx/.htpasswd;
}
}
}
docker/redis.conf
+30
View File
@@ -0,0 +1,30 @@
# Redis configuration for meta-sort
# Optimized for metadata storage
# Basic settings
bind 0.0.0.0
port 6379
protected-mode no
daemonize no
# Persistence
appendonly yes
appendfsync everysec
dir /data/redis
# Memory management
maxmemory 256mb
maxmemory-policy allkeys-lru
# Logging
loglevel notice
logfile ""
# Performance
tcp-keepalive 300
timeout 0
# Disable dangerous commands in production
rename-command FLUSHDB ""
rename-command FLUSHALL ""
rename-command DEBUG ""
docker/supervisord.conf
+69
View File
@@ -0,0 +1,69 @@
[supervisord]
nodaemon=true
user=root
logfile=/var/log/supervisor/supervisord.log
pidfile=/var/run/supervisord.pid
loglevel=info
[unix_http_server]
file=/var/run/supervisor.sock
chmod=0700
[rpcinterface:supervisor]
supervisor.rpc_interface_factory = supervisor.rpcinterface:make_main_rpcinterface
[supervisorctl]
serverurl=unix:///var/run/supervisor.sock
[program:meta-core]
command=/usr/local/bin/meta-core
priority=5
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
startsecs=3
environment=
META_CORE_PATH="/meta-core",
FILES_PATH="/files",
SERVICE_NAME="meta-sort",
SERVICE_VERSION="1.0.0",
META_CORE_HTTP_PORT="9000",
REDIS_PORT="6379"
[program:nginx]
command=/usr/sbin/nginx -g "daemon off;"
priority=20
autostart=true
autorestart=true
stdout_logfile=/var/log/supervisor/nginx.log
stderr_logfile=/var/log/supervisor/nginx_error.log
startsecs=2
[program:rclone]
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
priority=30
autostart=true
autorestart=true
stdout_logfile=/var/log/supervisor/rclone.log
stderr_logfile=/var/log/supervisor/rclone_error.log
startsecs=3
[program:meta-sort]
command=/usr/bin/node /app/backend/dist/index.js
directory=/app/backend
priority=40
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
startsecs=5
environment=
REDIS_URL="redis://127.0.0.1:6379",
REDIS_PREFIX="",
API_HOST="0.0.0.0",
API_PORT="3000"
package.json
+16
View File
@@ -0,0 +1,16 @@
{
"name": "meta-sort",
"version": "1.0.0",
"private": true,
"description": "Standalone file sorting and metadata extraction service",
"scripts": {
"build": "pnpm -r build",
"dev": "pnpm -r --parallel dev",
"start": "pnpm --filter @meta-sort/core start",
"start:core": "pnpm --filter @meta-sort/core start",
"start:ui": "pnpm --filter @meta-sort/ui dev",
"test": "pnpm --filter @meta-sort/core test",
"lint": "pnpm -r lint"
},
"license": "MIT"
}
packages/async-utils/LICENSE
+21
View File
@@ -0,0 +1,21 @@
MIT License
Copyright (c) 2024 Pierre-Henri
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.
packages/async-utils/README.md
+96
View File
@@ -0,0 +1,96 @@
# Async Utils
A TypeScript library providing utility classes and functions to simplify working with asynchronous operations and promises.
## Installation
```bash
yarn add github:worph/async-utils#main
```
## Features
- **Lazy<T>**: A utility type for lazy initialization of asynchronous operations.
- **ListenerCleaner**: Manages cleanup functions, allowing for easy resource management.
- **PromiseQueue**: A queue system for promises, allowing for sequential execution and cancellation.
- **MultiQueue**: Manages multiple `PromiseQueue` instances, distributing tasks to optimize concurrency.
- **Id Generation**: Utility function for generating random IDs.
## Usage
### Lazy Initialization
```typescript
import { Lazy } from "@worph/async-utils";
const lazyValue: Lazy<number> = async () => {
// some asynchronous operation
return 42;
};
// Usage
lazyValue().then(value => console.log(value));
```
### Listener Cleaner
```typescript
import { ListenerCleaner } from "worph/async-utils";
const cleaner = new ListenerCleaner();
cleaner.add(() => console.log("Cleanup action"));
// Trigger all cleanup actions
cleaner.cleanUp();
```
### Promise Queue
```typescript
import { PromiseQueue } from "@worph/async-utils";
const queue = new PromiseQueue<number>();
queue.add(async () => {
// some asynchronous task
return 1;
});
// Wait for the queue to be empty
queue.awaitQueueEmpty().then(() => console.log("Queue empty"));
```
### Multi Queue
```typescript
import { MultiQueue } from "@worph/async-utils";
const multiQueue = new MultiQueue<number>(2); // 2 concurrent tasks
multiQueue.add(async () => {
// some asynchronous task
return 1;
});
// Add more tasks...
// Wait for all queues to be empty
multiQueue.awaitQueueEmpty().then(() => console.log("All queues empty"));
```
### Generating an ID
```typescript
import { makeid } from "@worph/async-utils";
const id = makeid(10); // Generates a random 10 character string
console.log(id);
```
## Contributing
Contributions are welcome! Please submit a pull request or open an issue for discussion.
## License
This project is licensed under the MIT License - see the LICENSE file for details.
packages/async-utils/package.json
+37
View File
@@ -0,0 +1,37 @@
{
"name": "@worph/async-utils",
"version": "1.0.0",
"description": "",
"repository": {
"type": "git",
"url": "git+https://github.com/ApteroSAS/async-utils.git"
},
"author": "",
"license": "ISC",
"bugs": {
"url": "https://github.com/ApteroSAS/async-utils/issues"
},
"homepage": "https://github.com/ApteroSAS/async-utils#readme",
"type": "module",
"main": "./dist/index.cjs",
"module": "./dist/index.js",
"types": "./dist/index.d.ts",
"exports": {
".": {
"import": "./dist/index.js",
"require": "./dist/index.cjs"
}
},
"scripts": {
"build": "tsup src/index.ts",
"test": "echo \"Error: no test specified\" && exit 1"
},
"dependencies": {
},
"devDependencies": {
"@types/jest": "^29.5.1",
"jest": "^29.5.0",
"tsup": "^6.7.0",
"typescript": "^5.0.4"
}
}
packages/async-utils/src/Id.ts
+10
View File
@@ -0,0 +1,10 @@
export function makeid(length) {
let result = '';
let characters = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789';
let charactersLength = characters.length;
for ( let i = 0; i < length; i++ ) {
result += characters.charAt(Math.floor(Math.random() *
charactersLength));
}
return result;
}
packages/async-utils/src/Lazy.ts
+2
View File
@@ -0,0 +1,2 @@
export type Lazy<T> = () => Promise<T>;
packages/async-utils/src/ListenerCleaner.ts
+23
View File
@@ -0,0 +1,23 @@
export class ListenerCleaner {
cleaners: (() => void)[] = [];
add(cleanerCallback: () => void) {
this.cleaners.push(cleanerCallback);
}
cleaner(): () => void {
return () => {
this.cleanUp();
}
}
/**
* Call all the cleaner callbacks and reset this cleaner to be reused.
*/
cleanUp() {
for (const cleaner of this.cleaners) {
cleaner();
}
this.cleaners = [];
}
}
packages/async-utils/src/MultiQueue.ts
+66
View File
@@ -0,0 +1,66 @@
import {PromiseQueue} from "./PromiseQueue";
export class MultiQueue<T> {
private readonly queues:PromiseQueue<T>[];
/**
*
* @param concurrentTask
*/
constructor(
concurrentTask:number=1,
) {
if (concurrentTask<1){
throw new Error("concurrentTask must be >=1");
}
this.queues = [];
for(let i=0;i<concurrentTask;i++){
this.queues.push(new PromiseQueue<T>());
}
}
public getQueueSize(): number {
let ret = 0;
for(let queue of this.queues){
ret+=queue.getQueueSize();
}
return ret;
}
public async awaitQueueEmpty(){
for(let queue of this.queues){
await queue.awaitQueueEmpty();
}
}
/**
* Add a task to the queue. The queue with the smallest task size is used
* @param task
* @param id
**/
public add(task: () => Promise<T>, id: string = null): { id: string, promise: Promise<T> } {
let queue = this.queues[0];
for(let i=1;i<this.queues.length;i++){
if(this.queues[i].getQueueSize()<queue.getQueueSize()){
queue = this.queues[i];
}
}
return queue.add(task,id);
}
public cancel(id: string) {
for(let queue of this.queues){
queue.cancel(id);
}
}
/**
* The last task in progress is not canceled
*/
public async cancelAll(){
for(let queue of this.queues){
await queue.cancelAll();
}
}
}
packages/async-utils/src/PromiseQueue.ts
+80
View File
@@ -0,0 +1,80 @@
import {makeid} from "./Id";
export class PromiseQueue<T> {
private nextPromise: Promise<T> = Promise.resolve(null);
private canceled = {};
private queueSize = 0;
private queueID = 0;
/**
*
* @param onlyLast if true all task are cancel and only the last one is keep alive (a task started still finishes)
*/
constructor(
private onlyLast: boolean = false
) {
this.cancelAll();
}
getQueueSize(): number {
return this.queueSize;
}
async awaitQueueEmpty(){
await this.nextPromise;//await the current promise
while (this.queueSize!=0){//await all the fowllofing promise
await this.nextPromise;
}
}
add(task: () => Promise<T>, id: string = null): { id: string, promise: Promise<T> } {
if (!id) {
id = "promise-" + makeid(8);
}
let lastPromise = this.nextPromise;
this.queueSize++;
let currentID = this.queueID;
let currentPromise=null;
currentPromise = (async () => {
this.queueSize--;
try {
await lastPromise;
}catch (e) {
/* ignore error of previous promise*/
}
if(currentID!==this.queueID){
throw new Error("canceled caused by new queue");
}
if (this.canceled[id]) {
delete this.canceled[id];
throw new Error("canceled by user");
}
if(this.onlyLast && currentPromise!=this.nextPromise){
throw new Error("canceled caused by not last");
}
//execute task and return result
return task();
})();
currentPromise.catch(e=>{
/* we catch the exception this way to avoid an uncaught exception in the console*/
});
this.nextPromise = currentPromise
return {id, promise: this.nextPromise};
}
cancel(id: string) {
this.canceled[id] = true;
}
/**
* The last task in progress is not canceled
*/
async cancelAll(){
this.queueID++;//create a new queue will cancel all previous task except the one in progress
await this.awaitQueueEmpty();
this.nextPromise = Promise.resolve(null);
this.canceled = {};
this.queueSize = 0;
}
}
packages/async-utils/src/index.ts
+5
View File
@@ -0,0 +1,5 @@
export * from "./ListenerCleaner";
export * from "./PromiseQueue";
export * from "./MultiQueue";
export * from "./Id";
export * from "./Lazy";
packages/async-utils/tsconfig.json
+25
View File
@@ -0,0 +1,25 @@
{
"compilerOptions": {
"experimentalDecorators": true,//needed for inversify
"module": "ESNext",
"target": "ESNext",
"moduleResolution": "node",
"declaration": true,
"declarationMap": true,
"esModuleInterop": true,
"noImplicitAny": false,
"noLib": false,
"skipLibCheck": true,
"removeComments": true,
"sourceMap": true,
"outDir": "./dist"
},
"include": [
"./src"
],
"exclude": [
"node_modules",
"test/*",
"examples/*"
]
}
packages/async-utils/tsup.config.js
+11
View File
@@ -0,0 +1,11 @@
import { defineConfig } from 'tsup'
export default defineConfig({
entry: ['src/index.ts'],
splitting: false,
sourcemap: true,
clean: true,
dts:true,
minify:true,
format: ['cjs', 'esm'],
})
packages/meta-sort-core/.env.example
+102
View File
@@ -0,0 +1,102 @@
# meta-sort-core Environment Configuration
# Copy this file to .env and adjust values as needed
# =============================================================================
# Required Settings
# =============================================================================
# Comma-separated list of folders to watch for media files (REQUIRED)
WATCH_FOLDER_LIST=/files
# =============================================================================
# Volume Paths (New Architecture)
# =============================================================================
# Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services
# This is where Redis data, leader election locks, and service discovery files are stored
META_CORE_PATH=/meta-core
# Path to FILES_VOLUME - shared media volume
# All file paths in the KV database are relative to this path
FILES_PATH=/files
# =============================================================================
# KV Storage Configuration
# =============================================================================
# Use legacy etcd mode instead of Redis (default: false)
# Set to 'true' to use etcd instead of Redis
USE_ETCD=false
# etcd API URL (only used if USE_ETCD=true)
# ETCD_URL=http://localhost:2379
# Direct Redis URL (skip leader election if provided)
# If set, the service will connect directly to this Redis instance
# instead of participating in leader election
# REDIS_URL=redis://localhost:6379
# Redis port for leader to listen on (default: 6379)
REDIS_PORT=6379
# =============================================================================
# Service Identity
# =============================================================================
# Service name for leader election and service discovery
SERVICE_NAME=meta-sort
# Service version
SERVICE_VERSION=1.0.0
# =============================================================================
# API Server
# =============================================================================
# FUSE API server port (default: 3000)
FUSE_API_PORT=3000
# FUSE API server host (default: localhost)
FUSE_API_HOST=0.0.0.0
# =============================================================================
# Processing Configuration
# =============================================================================
# Maximum number of worker threads for Piscina worker pool
# Default: CPU count
# MAX_WORKER_THREADS=16
# Interval in milliseconds for polling watch folders (0 to disable)
# WATCH_FOLDER_POOLING_INTERVAL_MS=0
# Metadata formats to generate: 'meta' (.meta YAML), 'jellyfin' (.nfo XML)
METADATA_FORMATS=meta
# =============================================================================
# Cache Configuration
# =============================================================================
# Path to hash index cache
INDEX_FOLDER_PATH=/data/cache/hash
# Path to plugin cache folder
CACHE_FOLDER_PATH=/data/cache/plugins
# =============================================================================
# Jellyfin Integration (Optional)
# =============================================================================
# Jellyfin server endpoint
# JELLYFIN_ENDPOINT=http://localhost:8096
# Jellyfin API key
# JELLYFIN_API_KEY=your-api-key-here
# =============================================================================
# Polling Paths (Optional)
# =============================================================================
# JSON array of polling path configurations for paths that don't support inotify
# Example: '[{"path": "/files/smb/downloads", "intervalMs": 60000}]'
# POLLING_PATHS=[]
packages/meta-sort-core/package.json
+53
View File
@@ -0,0 +1,53 @@
{
"name": "@meta-sort/core",
"version": "1.0.0",
"description": "Core file sorting and metadata extraction service",
"main": "dist/index.js",
"type": "module",
"scripts": {
"build": "tsc",
"dev": "tsc --watch",
"start": "node dist/index.js",
"exec": "node dist/index.js",
"test": "mocha",
"lint": "eslint src --ext ts"
},
"license": "MIT",
"dependencies": {
"@fastify/cors": "^11.1.0",
"@fastify/static": "^8.3.0",
"@metazla/filename-tools": "github:worph/filename-tool",
"@metazla/meta-hash": "workspace:*",
"@metazla/meta-interface": "workspace:*",
"@worph/async-utils": "workspace:*",
"axios": "^1.6.7",
"bencode": "^4.0.0",
"chokidar": "^3.5.3",
"dockerode": "^4.0.2",
"dotenv": "^16.4.1",
"fastify": "^5.6.1",
"fluent-ffmpeg": "^2.1.2",
"franc-min": "^6.2.0",
"ioredis": "^5.4.1",
"iso-639-1": "^3.1.2",
"lodash": "^4.17.21",
"multiformats": "^13.1.0",
"p-queue": "^8.0.1",
"piscina": "^4.4.0",
"tslog": "^4.9.2",
"wanakana": "^5.3.1",
"xml2js": "^0.6.2",
"yaml": "^2.4.3"
},
"devDependencies": {
"@types/chai": "^4.3.11",
"@types/dockerode": "^3.3.31",
"@types/fluent-ffmpeg": "^2.1.24",
"@types/lodash": "^4.17.0",
"@types/mocha": "^10.0.6",
"@types/node": "^20.11.16",
"chai": "^5.0.3",
"mocha": "^10.2.0",
"typescript": "^5.3.3"
}
}
packages/meta-sort-core/src/api/DuplicatesAPI.ts
+166
View File
@@ -0,0 +1,166 @@
/**
* HTTP REST API Server for Duplicate Files
*
* Provides endpoints for accessing duplicate file detection results.
* Separate from FUSE API to maintain proper separation of concerns.
*/
import * as http from 'http';
export interface DuplicatesAPIConfig {
/** Port to listen on (default: 3003) */
port?: number;
/** Host to bind to (default: 'localhost') */
host?: string;
/** Enable CORS (default: true) */
enableCors?: boolean;
/** Mount path for duplicates endpoint (default: '/duplicates') */
mountPath?: string;
}
export class DuplicatesAPI {
private server: http.Server | null = null;
private config: Required<DuplicatesAPIConfig>;
constructor(config: DuplicatesAPIConfig = {}) {
this.config = {
port: config.port ?? 3003,
host: config.host ?? 'localhost',
enableCors: config.enableCors ?? true,
mountPath: config.mountPath ?? '/duplicates'
};
}
/**
* Start the HTTP server
*/
public async start(): Promise<void> {
return new Promise((resolve, reject) => {
this.server = http.createServer(this.handleRequest.bind(this));
this.server.on('error', reject);
this.server.listen(this.config.port, this.config.host, () => {
console.log(`Duplicates API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`);
resolve();
});
});
}
/**
* Stop the HTTP server
*/
public async stop(): Promise<void> {
if (!this.server) {
return;
}
return new Promise((resolve, reject) => {
this.server!.close((err) => {
if (err) {
reject(err);
} else {
console.log('Duplicates API stopped');
this.server = null;
resolve();
}
});
});
}
/**
* Handle incoming HTTP requests
*/
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
if (this.config.enableCors) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
}
try {
const url = new URL(req.url!, `http://${req.headers.host}`);
const pathname = url.pathname;
const method = req.method!;
// Route the request
if (method === 'GET' && pathname === this.config.mountPath) {
await this.handleDuplicates(res);
} else if (method === 'GET' && pathname === '/health') {
this.handleHealth(res);
} else {
this.sendError(res, 404, 'Not Found');
}
} catch (error) {
console.error('Error handling request:', error);
this.sendError(res, 500, 'Internal Server Error', error);
}
}
/**
* Send JSON response
*/
private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
res.writeHead(statusCode, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(data, null, 2));
}
/**
* Send error response
*/
private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
this.sendJson(res, {
error: message,
details: error?.message ?? error
}, statusCode);
}
/**
* GET /duplicates
* Get duplicate files
* NOTE: This API is deprecated. Use UnifiedAPIServer's /api/duplicates endpoint instead.
* Disk-based duplicate logging has been removed for performance.
*/
private async handleDuplicates(res: http.ServerResponse): Promise<void> {
try {
// Return empty results - duplicate detection now happens in-memory via UnifiedAPIServer
const hashDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
const titleDuplicates: Array<{ key: string; files: string[]; virtualPath?: string }> = [];
this.sendJson(res, {
hashDuplicates,
titleDuplicates,
stats: {
hashGroupCount: 0,
hashFileCount: 0,
titleGroupCount: 0,
titleFileCount: 0
},
deprecated: true,
message: 'This API is deprecated. Use UnifiedAPIServer /api/duplicates endpoint instead.'
});
} catch (error) {
console.error('Error in duplicates endpoint:', error);
this.sendError(res, 500, 'Failed to get duplicate data', error);
}
}
/**
* GET /health
* Health check endpoint
*/
private handleHealth(res: http.ServerResponse): void {
this.sendJson(res, {
status: 'ok',
service: 'duplicates-api',
timestamp: new Date().toISOString()
});
}
}
packages/meta-sort-core/src/api/FuseAPI.ts
+173
View File
@@ -0,0 +1,173 @@
/**
* FUSE API for meta-mesh
*
* This API provides a virtual filesystem interface that can be consumed by a FUSE driver.
* It exposes the computed folder structure without creating actual hardlinks.
*/
import { HashMeta } from '@metazla/meta-interface';
import { Stats } from 'fs';
/**
* Represents a file or directory in the virtual filesystem
*/
export interface VirtualNode {
/** Virtual path in the output filesystem */
path: string;
/** Node type */
type: 'file' | 'directory';
/** File name */
name: string;
/** Parent directory path */
parent: string;
}
/**
* Represents a regular file in the virtual filesystem
*/
export interface VirtualFile extends VirtualNode {
type: 'file';
/** Source file path on the real filesystem (null for virtual files like .meta) */
sourcePath: string | null;
/** File size in bytes */
size: number;
/** Last modified time */
mtime: Date;
/** Creation time */
ctime: Date;
/** File permissions (octal) */
mode: number;
/** Associated metadata */
metadata?: HashMeta;
}
/**
* Represents a directory in the virtual filesystem
*/
export interface VirtualDirectory extends VirtualNode {
type: 'directory';
/** Child nodes (file/directory names) */
children: string[];
/** Directory permissions (octal) */
mode: number;
/** Last modified time */
mtime: Date;
}
/**
* File attributes returned by getattr() - compatible with FUSE stat structure
*/
export interface FileAttributes {
/** File size in bytes */
size: number;
/** File mode (permissions + type) */
mode: number;
/** Last modified time (Unix timestamp in seconds) */
mtime: number;
/** Last access time (Unix timestamp in seconds) */
atime: number;
/** Creation/change time (Unix timestamp in seconds) */
ctime: number;
/** Number of hard links */
nlink: number;
/** User ID */
uid: number;
/** Group ID */
gid: number;
}
/**
* Read result for file content
*/
export interface ReadResult {
/** For regular files: source path to read from. For virtual files: null */
sourcePath: string | null;
/** For virtual files: generated content. For regular files: null */
content: Buffer | null;
/** Total file size */
size: number;
}
/**
* FUSE API operations
*/
export interface IFuseAPI {
/**
* List directory contents
* @param path Virtual directory path (e.g., "/Anime/Naruto/Season 1")
* @returns Array of file/directory names in the directory, or null if not found
*/
readdir(path: string): string[] | null;
/**
* Get file or directory attributes
* @param path Virtual file or directory path
* @returns File attributes, or null if not found
*/
getattr(path: string): FileAttributes | null;
/**
* Check if a path exists
* @param path Virtual path
* @returns true if path exists, false otherwise
*/
exists(path: string): boolean;
/**
* Get read information for a file
* @param path Virtual file path
* @returns Read result with source path or content, or null if not found
*/
read(path: string): ReadResult | null;
/**
* Get metadata for a file
* @param path Virtual file path
* @returns HashMeta if available, null otherwise
*/
getMetadata(path: string): HashMeta | null;
/**
* Get the complete virtual filesystem tree
* @returns Root directory node
*/
getTree(): VirtualDirectory;
/**
* Refresh the virtual filesystem from current state
* Called after processing completes to update the VFS
*/
refresh(): Promise<void>;
}
/**
* Events emitted by the FUSE API
*/
export interface FuseAPIEvents {
/** Emitted when the virtual filesystem is updated */
'vfs-updated': () => void;
/** Emitted when a file is added */
'file-added': (path: string) => void;
/** Emitted when a file is updated (e.g., metadata changed) */
'file-updated': (path: string) => void;
/** Emitted when a file is removed */
'file-removed': (path: string) => void;
/** Emitted when a directory is added */
'directory-added': (path: string) => void;
}
/**
* Configuration for FUSE API
*/
export interface FuseAPIConfig {
/** Base path for the virtual filesystem (default: "/") */
basePath?: string;
/** Default file permissions (default: 0o644) */
fileMode?: number;
/** Default directory permissions (default: 0o755) */
directoryMode?: number;
/** User ID for files (default: process.getuid()) */
uid?: number;
/** Group ID for files (default: process.getgid()) */
gid?: number;
}
packages/meta-sort-core/src/api/FuseAPIServer.ts
+345
View File
@@ -0,0 +1,345 @@
/**
* HTTP REST API Server for FUSE operations
*
* Exposes the VirtualFileSystem over HTTP for consumption by a FUSE driver.
*/
import * as http from 'http';
import { VirtualFileSystem } from './VirtualFileSystem.js';
import { FileAttributes, ReadResult } from './FuseAPI.js';
export interface FuseAPIServerConfig {
/** Port to listen on (default: 3000) */
port?: number;
/** Host to bind to (default: 'localhost') */
host?: string;
/** Enable CORS (default: false) */
enableCors?: boolean;
}
export class FuseAPIServer {
private server: http.Server | null = null;
private vfs: VirtualFileSystem;
private config: Required<FuseAPIServerConfig>;
constructor(vfs: VirtualFileSystem, config: FuseAPIServerConfig = {}) {
this.vfs = vfs;
this.config = {
port: config.port ?? 3000,
host: config.host ?? 'localhost',
enableCors: config.enableCors ?? false
};
}
/**
* Start the HTTP server
*/
public async start(): Promise<void> {
return new Promise((resolve, reject) => {
this.server = http.createServer(this.handleRequest.bind(this));
this.server.on('error', reject);
this.server.listen(this.config.port, this.config.host, () => {
console.log(`FUSE API Server listening on http://${this.config.host}:${this.config.port}`);
resolve();
});
});
}
/**
* Stop the HTTP server
*/
public async stop(): Promise<void> {
if (!this.server) {
return;
}
return new Promise((resolve, reject) => {
this.server!.close((err) => {
if (err) {
reject(err);
} else {
console.log('FUSE API Server stopped');
this.server = null;
resolve();
}
});
});
}
/**
* Handle incoming HTTP requests
*/
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
if (this.config.enableCors) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, POST, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
}
try {
const url = new URL(req.url!, `http://${req.headers.host}`);
const pathname = url.pathname;
const method = req.method!;
// Route the request
if (method === 'GET' && pathname === '/api/fuse/health') {
this.handleHealth(res);
} else if (method === 'GET' && pathname === '/api/fuse/stats') {
this.handleStats(res);
} else if (method === 'POST' && pathname === '/api/fuse/readdir') {
await this.handleReaddir(req, res);
} else if (method === 'POST' && pathname === '/api/fuse/getattr') {
await this.handleGetattr(req, res);
} else if (method === 'POST' && pathname === '/api/fuse/exists') {
await this.handleExists(req, res);
} else if (method === 'POST' && pathname === '/api/fuse/read') {
await this.handleRead(req, res);
} else if (method === 'POST' && pathname === '/api/fuse/metadata') {
await this.handleMetadata(req, res);
} else if (method === 'GET' && pathname === '/api/fuse/tree') {
this.handleTree(res);
} else if (method === 'GET' && pathname === '/api/fuse/files') {
this.handleFiles(res);
} else if (method === 'GET' && pathname === '/api/fuse/directories') {
this.handleDirectories(res);
} else if (method === 'POST' && pathname === '/api/fuse/refresh') {
await this.handleRefresh(req, res);
} else {
this.sendError(res, 404, 'Not Found');
}
} catch (error) {
console.error('Error handling request:', error);
this.sendError(res, 500, 'Internal Server Error', error);
}
}
/**
* Parse JSON body from request
*/
private async parseBody(req: http.IncomingMessage): Promise<any> {
return new Promise((resolve, reject) => {
let body = '';
req.on('data', (chunk) => {
body += chunk.toString();
});
req.on('end', () => {
try {
resolve(JSON.parse(body));
} catch (error) {
reject(new Error('Invalid JSON'));
}
});
req.on('error', reject);
});
}
/**
* Send JSON response
*/
private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
res.writeHead(statusCode, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(data));
}
/**
* Send error response
*/
private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
this.sendJson(res, {
error: message,
details: error?.message ?? error
}, statusCode);
}
// API Handlers
/**
* GET /api/health
* Health check endpoint
*/
private handleHealth(res: http.ServerResponse): void {
this.sendJson(res, {
status: 'ok',
timestamp: new Date().toISOString()
});
}
/**
* GET /api/stats
* Get VFS statistics
*/
private handleStats(res: http.ServerResponse): void {
const stats = this.vfs.getStats();
this.sendJson(res, stats);
}
/**
* POST /api/readdir
* List directory contents
* Body: { path: string }
* Response: { entries: string[] } | { error: string }
*/
private async handleReaddir(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
const { path } = await this.parseBody(req);
if (typeof path !== 'string') {
this.sendError(res, 400, 'Missing or invalid "path" parameter');
return;
}
const entries = this.vfs.readdir(path);
if (entries === null) {
this.sendError(res, 404, 'Directory not found');
return;
}
this.sendJson(res, { entries });
}
/**
* POST /api/getattr
* Get file attributes
* Body: { path: string }
* Response: FileAttributes | { error: string }
*/
private async handleGetattr(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
const { path } = await this.parseBody(req);
if (typeof path !== 'string') {
this.sendError(res, 400, 'Missing or invalid "path" parameter');
return;
}
const attrs = this.vfs.getattr(path);
if (attrs === null) {
this.sendError(res, 404, 'Path not found');
return;
}
this.sendJson(res, attrs);
}
/**
* POST /api/exists
* Check if path exists
* Body: { path: string }
* Response: { exists: boolean }
*/
private async handleExists(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
const { path } = await this.parseBody(req);
if (typeof path !== 'string') {
this.sendError(res, 400, 'Missing or invalid "path" parameter');
return;
}
const exists = this.vfs.exists(path);
this.sendJson(res, { exists });
}
/**
* POST /api/read
* Read file
* Body: { path: string }
* Response: ReadResult | { error: string }
*/
private async handleRead(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
const { path } = await this.parseBody(req);
if (typeof path !== 'string') {
this.sendError(res, 400, 'Missing or invalid "path" parameter');
return;
}
const result = this.vfs.read(path);
if (result === null) {
this.sendError(res, 404, 'File not found');
return;
}
// Convert Buffer to base64 for JSON serialization
const response: any = {
sourcePath: result.sourcePath,
size: result.size
};
if (result.content !== null) {
response.content = result.content.toString('base64');
response.contentEncoding = 'base64';
}
this.sendJson(res, response);
}
/**
* POST /api/metadata
* Get file metadata
* Body: { path: string }
* Response: HashMeta | { error: string }
*/
private async handleMetadata(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
const { path } = await this.parseBody(req);
if (typeof path !== 'string') {
this.sendError(res, 400, 'Missing or invalid "path" parameter');
return;
}
const metadata = this.vfs.getMetadata(path);
if (metadata === null) {
this.sendError(res, 404, 'Metadata not found');
return;
}
this.sendJson(res, metadata);
}
/**
* GET /api/tree
* Get complete VFS tree
*/
private handleTree(res: http.ServerResponse): void {
const tree = this.vfs.getTree();
this.sendJson(res, tree);
}
/**
* GET /api/files
* Get all file paths
*/
private handleFiles(res: http.ServerResponse): void {
const files = this.vfs.getAllFiles();
this.sendJson(res, { files });
}
/**
* GET /api/directories
* Get all directory paths
*/
private handleDirectories(res: http.ServerResponse): void {
const directories = this.vfs.getAllDirectories();
this.sendJson(res, { directories });
}
/**
* POST /api/refresh
* Refresh the VFS (no-op, VFS is updated by processing pipeline)
*/
private async handleRefresh(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
await this.vfs.refresh();
this.sendJson(res, { status: 'ok' });
}
}
packages/meta-sort-core/src/api/MetricsAPI.ts
+160
View File
@@ -0,0 +1,160 @@
/**
* HTTP REST API Server for Performance Metrics
*
* Provides a dedicated endpoint for exposing application performance metrics.
* Separate from FUSE API to maintain proper separation of concerns.
*/
import * as http from 'http';
import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
export interface MetricsAPIConfig {
/** Port to listen on (default: 3001) */
port?: number;
/** Host to bind to (default: 'localhost') */
host?: string;
/** Enable CORS (default: true for metrics) */
enableCors?: boolean;
/** Mount path for metrics endpoint (default: '/metrics') */
mountPath?: string;
}
export class MetricsAPI {
private server: http.Server | null = null;
private config: Required<MetricsAPIConfig>;
constructor(config: MetricsAPIConfig = {}) {
this.config = {
port: config.port ?? 3001,
host: config.host ?? 'localhost',
enableCors: config.enableCors ?? true,
mountPath: config.mountPath ?? '/metrics'
};
}
/**
* Start the HTTP server
*/
public async start(): Promise<void> {
return new Promise((resolve, reject) => {
this.server = http.createServer(this.handleRequest.bind(this));
this.server.on('error', reject);
this.server.listen(this.config.port, this.config.host, () => {
console.log(`Metrics API listening on http://${this.config.host}:${this.config.port}${this.config.mountPath}`);
resolve();
});
});
}
/**
* Stop the HTTP server
*/
public async stop(): Promise<void> {
if (!this.server) {
return;
}
return new Promise((resolve, reject) => {
this.server!.close((err) => {
if (err) {
reject(err);
} else {
console.log('Metrics API stopped');
this.server = null;
resolve();
}
});
});
}
/**
* Handle incoming HTTP requests
*/
private async handleRequest(req: http.IncomingMessage, res: http.ServerResponse): Promise<void> {
// CORS headers
if (this.config.enableCors) {
res.setHeader('Access-Control-Allow-Origin', '*');
res.setHeader('Access-Control-Allow-Methods', 'GET, OPTIONS');
res.setHeader('Access-Control-Allow-Headers', 'Content-Type');
if (req.method === 'OPTIONS') {
res.writeHead(204);
res.end();
return;
}
}
try {
const url = new URL(req.url!, `http://${req.headers.host}`);
const pathname = url.pathname;
const method = req.method!;
// Route the request
if (method === 'GET' && pathname === this.config.mountPath) {
this.handleMetrics(res);
} else if (method === 'GET' && pathname === '/health') {
this.handleHealth(res);
} else if (method === 'POST' && pathname === '/reset') {
this.handleReset(res);
} else {
this.sendError(res, 404, 'Not Found');
}
} catch (error) {
console.error('Error handling request:', error);
this.sendError(res, 500, 'Internal Server Error', error);
}
}
/**
* Send JSON response
*/
private sendJson(res: http.ServerResponse, data: any, statusCode: number = 200): void {
res.writeHead(statusCode, { 'Content-Type': 'application/json' });
res.end(JSON.stringify(data, null, 2));
}
/**
* Send error response
*/
private sendError(res: http.ServerResponse, statusCode: number, message: string, error?: any): void {
this.sendJson(res, {
error: message,
details: error?.message ?? error
}, statusCode);
}
/**
* GET /metrics
* Get performance metrics
*/
private handleMetrics(res: http.ServerResponse): void {
const metrics = performanceMetrics.getMetrics();
this.sendJson(res, metrics);
}
/**
* GET /health
* Health check endpoint
*/
private handleHealth(res: http.ServerResponse): void {
this.sendJson(res, {
status: 'ok',
service: 'metrics-api',
timestamp: new Date().toISOString()
});
}
/**
* POST /reset
* Reset all metrics
*/
private handleReset(res: http.ServerResponse): void {
performanceMetrics.reset();
this.sendJson(res, {
status: 'ok',
message: 'Metrics reset successfully'
});
}
}
packages/meta-sort-core/src/api/README.md
+299
View File
@@ -0,0 +1,299 @@
# FUSE API for Meta-Mesh
This module provides a FUSE-compatible HTTP API that runs alongside meta-mesh's hardlink system. The API exposes a virtual filesystem that can be consumed by a future FUSE driver.
## Overview
The FUSE API runs **always-on alongside hardlinks**, providing:
- **Dual output**: Both hardlinks (for local access) and virtual filesystem (for FUSE driver)
- **Dynamic updates**: VirtualFileSystem rebuilds on every processing cycle
- **Virtual .meta files**: Metadata files are generated on-the-fly without disk I/O
- **HTTP REST API**: Can be consumed by local or remote FUSE drivers
- **Zero configuration**: Always enabled, no opt-in required
## Architecture
### Components
1. **VirtualFileSystem** - In-memory representation of the output directory structure
2. **FuseAPIServer** - HTTP REST API server exposing FUSE operations
3. **FuseAPI** - TypeScript interfaces defining the API contract
### Data Flow
```
Input Files → File Processing → Metadata Extraction → Duplicate Removal
↓
Virtual Folder Structure Computation (diff generation)
↓
├─→ VirtualFileSystem Build (in-memory) → FUSE API Server → Future FUSE Driver
│
└─→ Hardlink Creation (on disk) → Direct File Access
```
Both outputs are generated in parallel from the same computed folder structure.
## Configuration
The FUSE API is **always enabled** and runs alongside hardlinks. You can optionally configure the server port and host:
```bash
# FUSE API Server configuration (optional)
FUSE_API_PORT=3000 # Default: 3000
FUSE_API_HOST=localhost # Default: localhost
```
The API server starts automatically when meta-mesh starts.
## API Endpoints
The FUSE API Server exposes the following HTTP REST endpoints:
### Health & Info
- **GET /api/health** - Health check
```json
{ "status": "ok", "timestamp": "2025-01-15T10:30:00.000Z" }
```
- **GET /api/stats** - VFS statistics
```json
{
"fileCount": 1250,
"directoryCount": 45,
"totalSize": 52428800,
"metaFileCount": 1250
}
```
### Directory Operations
- **POST /api/readdir** - List directory contents
```json
// Request
{ "path": "/Anime/Naruto/Season 1" }
// Response
{ "entries": ["Naruto S01E01.mkv", "Naruto S01E01.mkv.meta", ...] }
```
### File Operations
- **POST /api/getattr** - Get file attributes
```json
// Request
{ "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
// Response
{
"size": 524288000,
"mode": 33188, // 0o100644 (regular file, rw-r--r--)
"mtime": 1705315200,
"atime": 1705315200,
"ctime": 1705315200,
"nlink": 1,
"uid": 1000,
"gid": 1000
}
```
- **POST /api/exists** - Check if path exists
```json
// Request
{ "path": "/Anime/Naruto" }
// Response
{ "exists": true }
```
- **POST /api/read** - Read file
```json
// Request
{ "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
// Response (regular file)
{
"sourcePath": "/data/input/naruto_01.mkv",
"size": 524288000
}
// Response (virtual .meta file)
{
"sourcePath": null,
"content": "..base64-encoded content..",
"contentEncoding": "base64",
"size": 1024
}
```
- **POST /api/metadata** - Get file metadata
```json
// Request
{ "path": "/Anime/Naruto/Season 1/Naruto S01E01.mkv" }
// Response
{
"title": "Naruto",
"season": "01",
"episode": "01",
"anime": true,
"sha256": "abc123...",
// ... full HashMeta object
}
```
### Tree Operations
- **GET /api/tree** - Get complete VFS tree structure
- **GET /api/files** - Get all file paths
- **GET /api/directories** - Get all directory paths
- **POST /api/refresh** - Refresh VFS (no-op, VFS updates automatically)
## Usage Example
### Starting meta-mesh with FUSE API
The FUSE API starts automatically with meta-mesh:
```bash
# Run meta-mesh (FUSE API starts automatically on port 3000)
pnpm run start:mesh
# Or customize the port
export FUSE_API_PORT=8080
pnpm run start:mesh
```
You'll see output like:
```
FUSE API Server started on http://localhost:3000
VirtualFileSystem build took 15ms
VirtualFileSystem: 1250 files (1250 .meta), 45 directories, 50.00 MB
```
### Consuming the API
```typescript
import { VirtualFileSystem, FuseAPIServer } from '@metazla/meta-mesh/api';
// Create VFS instance
const vfs = new VirtualFileSystem({
fileMode: 0o644,
directoryMode: 0o755,
uid: process.getuid(),
gid: process.getgid()
});
// Build from computed structure
vfs.buildFromComputed(sourceToDestMap, metadataMap);
// Start API server
const server = new FuseAPIServer(vfs, {
port: 3000,
host: 'localhost'
});
await server.start();
```
### FUSE Driver Integration
A FUSE driver would make HTTP requests to these endpoints:
```typescript
// Example pseudo-code for FUSE driver
async function readdir(path: string): Promise<string[]> {
const response = await fetch('http://localhost:3000/api/readdir', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ path })
});
const { entries } = await response.json();
return entries;
}
async function read(path: string): Promise<Buffer> {
const response = await fetch('http://localhost:3000/api/read', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ path })
});
const result = await response.json();
if (result.sourcePath) {
// Regular file - read from source
return fs.readFileSync(result.sourcePath);
} else {
// Virtual file - use provided content
return Buffer.from(result.content, 'base64');
}
}
```
## Virtual .meta Files
`.meta` files are entirely virtual and generated on-the-fly:
- **No disk I/O**: Content is serialized from in-memory metadata
- **Always up-to-date**: Reflects current metadata state
- **YAML format**: Uses `MetaMeshFormat.serialize()` for consistency
- **No source path**: `sourcePath` is `null`, `content` contains the YAML
## Events
The VirtualFileSystem emits events for monitoring:
```typescript
vfs.on('vfs-updated', () => {
console.log('Virtual filesystem updated');
});
vfs.on('file-added', (path: string) => {
console.log(`File added: ${path}`);
});
vfs.on('file-removed', (path: string) => {
console.log(`File removed: ${path}`);
});
vfs.on('directory-added', (path: string) => {
console.log(`Directory added: ${path}`);
});
```
## Performance
The VirtualFileSystem is highly performant:
- **In-memory**: All operations are O(1) or O(n) in-memory
- **No disk I/O**: Except for reading source files when FUSE driver requests content
- **Lazy .meta generation**: Content is only serialized when requested
- **Fast updates**: Rebuilding the VFS takes milliseconds
## Dual Output: Hardlinks + FUSE API
Meta-mesh now provides **both** output methods simultaneously:
| Feature | Hardlinks | FUSE API | Best For |
|---------|-----------|----------|----------|
| Disk usage | 1 inode per file | 0 (virtual) | Hardlinks: local use |
| Update speed | Slow (file I/O) | Fast (in-memory) | FUSE: frequent updates |
| .meta files | Written to disk | Virtual (on-demand) | FUSE: dynamic metadata |
| Remote access | No | Yes (HTTP) | FUSE: network drives |
| Compatibility | Any filesystem | Requires FUSE driver | Hardlinks: broad compat |
| Access method | Direct file access | HTTP + FUSE driver | Hardlinks: native apps |
**Use hardlinks** for local applications like Jellyfin, Plex, Kodi that read files directly.
**Use FUSE API** for future FUSE drivers that want dynamic, virtual filesystem access.
## Future Enhancements
- **WebSocket support**: Real-time updates for FUSE drivers
- **Authentication**: API key or JWT-based auth
- **Compression**: Gzip/Brotli for API responses
- **Caching**: ETag/Last-Modified headers
- **Batch operations**: Multi-file operations in single request
packages/meta-sort-core/src/api/UnifiedAPIServer.ts
+2145
View File
File diff suppressed because it is too large Load Diff
packages/meta-sort-core/src/api/VirtualFileSystem.ts
+627
View File
@@ -0,0 +1,627 @@
/**
* VirtualFileSystem - In-memory representation of the virtual output filesystem
*
* This class maintains the virtual directory structure and provides FUSE-compatible operations.
*/
import { EventEmitter } from 'events';
import * as path from 'path';
import * as fs from 'fs';
import { HashMeta } from '@metazla/meta-interface';
import {
IFuseAPI,
VirtualDirectory,
VirtualFile,
FileAttributes,
ReadResult,
FuseAPIConfig,
FuseAPIEvents
} from './FuseAPI.js';
import { MetaMeshFormat } from '../logic/MetaMeshFormat.js';
import { MetadataFormatRegistry, IMetadataFormatHandler } from '../logic/MetadataFormatRegistry.js';
import { config } from '../config/EnvConfig.js';
interface VFSNode {
type: 'file' | 'directory';
name: string;
parent: string | null;
// For files
sourcePath?: string | null;
size?: number;
mtime?: Date;
ctime?: Date;
metadata?: HashMeta;
// For virtual metadata files
formatHandler?: IMetadataFormatHandler;
// For directories
children?: Set<string>;
}
export class VirtualFileSystem extends EventEmitter implements IFuseAPI {
private nodes: Map<string, VFSNode> = new Map();
private config: Required<FuseAPIConfig>;
private metaFormat: MetaMeshFormat = new MetaMeshFormat();
private formatRegistry: MetadataFormatRegistry = new MetadataFormatRegistry();
private activeFormatHandlers: IMetadataFormatHandler[] = [];
// Cached stats - updated incrementally on each modification
private cachedStats = {
fileCount: 0,
directoryCount: 0,
totalSize: 0,
metaFileCount: 0,
metadataFileCount: 0
};
constructor(fuseConfig: FuseAPIConfig = {}) {
super();
this.config = {
basePath: fuseConfig.basePath ?? '/',
fileMode: fuseConfig.fileMode ?? 0o644,
directoryMode: fuseConfig.directoryMode ?? 0o755,
uid: fuseConfig.uid ?? process.getuid(),
gid: fuseConfig.gid ?? process.getgid()
};
// Initialize active format handlers from configuration
this.activeFormatHandlers = this.formatRegistry.getHandlersForFormats(config.METADATA_FORMATS);
if (this.activeFormatHandlers.length > 0) {
console.log(`VirtualFileSystem: Active metadata formats: ${this.activeFormatHandlers.map(h => h.formatType).join(', ')}`);
} else {
console.log('VirtualFileSystem: No metadata formats configured');
}
// Initialize root directory
this.nodes.set('/', {
type: 'directory',
name: '',
parent: null,
children: new Set()
});
// Initialize cached stats with root directory
this.cachedStats.directoryCount = 1;
}
/**
* Build the virtual filesystem from a computed folder structure
* @param computed Map of source paths to destination paths
* @param metadata Map of source paths to HashMeta
* @param basePath Base path to strip from destination paths (e.g., OUTPUT_FOLDER)
*/
public buildFromComputed(
computed: Map<string, string>,
metadata: Map<string, HashMeta>,
basePath?: string
): void {
// Clear existing structure (except root)
this.nodes.clear();
this.nodes.set('/', {
type: 'directory',
name: '',
parent: null,
children: new Set()
});
// Reset cached stats (root directory will be counted)
this.cachedStats = {
fileCount: 0,
directoryCount: 1, // Root directory
totalSize: 0,
metaFileCount: 0,
metadataFileCount: 0
};
// Normalize basePath if provided
const normalizedBasePath = basePath ? path.normalize(basePath) : null;
// Add all files and their parent directories
for (const [sourcePath, destPath] of computed.entries()) {
// Strip basePath from destPath to create relative virtual path
let virtualPath = destPath;
if (normalizedBasePath && destPath.startsWith(normalizedBasePath)) {
virtualPath = destPath.slice(normalizedBasePath.length);
// Ensure path starts with /
if (!virtualPath.startsWith('/')) {
virtualPath = '/' + virtualPath;
}
}
this.addFile(virtualPath, sourcePath, metadata.get(sourcePath));
}
this.emit('vfs-updated');
}
/**
* Add a file to the virtual filesystem, creating parent directories as needed
*/
public addFile(virtualPath: string, sourcePath: string, metadata?: HashMeta): void {
// Normalize path
virtualPath = this.normalizePath(virtualPath);
// Create parent directories
const dirname = path.dirname(virtualPath);
this.ensureDirectory(dirname);
// Try to use metadata for file stats (avoid disk I/O during fast load)
let size: number;
let mtime: Date;
let ctime: Date;
if (metadata && (metadata as any).sizeByte) {
// Use cached metadata (fast path - no disk I/O)
size = parseInt((metadata as any).sizeByte, 10);
// Use current time as placeholder for mtime/ctime (will be corrected on next scan)
mtime = new Date();
ctime = new Date();
} else {
// Fallback: stat the file (slow path - disk I/O)
try {
const stats = fs.statSync(sourcePath);
size = stats.size;
mtime = stats.mtime;
ctime = stats.ctime;
} catch (error) {
console.warn(`Cannot stat source file: ${sourcePath}`, error);
return;
}
}
// Add the file node
const filename = path.basename(virtualPath);
this.nodes.set(virtualPath, {
type: 'file',
name: filename,
parent: dirname,
sourcePath,
size,
mtime,
ctime,
metadata
});
// Add to parent's children
const parentNode = this.nodes.get(dirname);
if (parentNode && parentNode.type === 'directory') {
parentNode.children!.add(filename);
}
// Update cached stats
this.cachedStats.fileCount++;
this.cachedStats.totalSize += size;
// Also add corresponding metadata files based on configured formats
this.addMetadataFiles(virtualPath, metadata, mtime);
this.emit('file-added', virtualPath);
}
/**
* Add virtual metadata files for all configured formats
*/
private addMetadataFiles(mediaPath: string, metadata: HashMeta | undefined, mtime: Date): void {
if (!metadata) return;
// Create a metadata file for each configured format
for (const handler of this.activeFormatHandlers) {
const metaPath = `${mediaPath}${handler.extension}`;
const dirname = path.dirname(metaPath);
const filename = path.basename(metaPath);
// Pre-generate metadata content to calculate size
const metaContent = handler.serialize(metadata);
const metaBuffer = Buffer.from(metaContent, 'utf-8');
// Add the virtual metadata file node
this.nodes.set(metaPath, {
type: 'file',
name: filename,
parent: dirname,
sourcePath: null, // Virtual file, no source
size: metaBuffer.length,
mtime: mtime,
ctime: mtime,
metadata,
formatHandler: handler
});
// Add to parent's children
const parentNode = this.nodes.get(dirname);
if (parentNode && parentNode.type === 'directory') {
parentNode.children!.add(filename);
}
// Update cached stats
this.cachedStats.fileCount++;
this.cachedStats.totalSize += metaBuffer.length;
this.cachedStats.metadataFileCount++;
// Legacy: Count .meta files separately for backward compatibility
if (handler.extension === '.meta') {
this.cachedStats.metaFileCount++;
}
}
}
/**
* Ensure a directory exists in the VFS, creating parent directories as needed
*/
private ensureDirectory(dirPath: string): void {
dirPath = this.normalizePath(dirPath);
if (dirPath === '/' || this.nodes.has(dirPath)) {
return;
}
// Recursively create parent
const parentPath = path.dirname(dirPath);
this.ensureDirectory(parentPath);
// Create this directory
const dirName = path.basename(dirPath);
this.nodes.set(dirPath, {
type: 'directory',
name: dirName,
parent: parentPath,
children: new Set()
});
// Add to parent's children
const parentNode = this.nodes.get(parentPath);
if (parentNode && parentNode.type === 'directory') {
parentNode.children!.add(dirName);
}
// Update cached stats
this.cachedStats.directoryCount++;
this.emit('directory-added', dirPath);
}
/**
* Normalize path to use forward slashes and ensure leading slash
*/
private normalizePath(p: string): string {
// Convert to forward slashes
p = p.split(path.sep).join('/');
// Ensure leading slash
if (!p.startsWith('/')) {
p = '/' + p;
}
// Remove trailing slash (except for root)
if (p !== '/' && p.endsWith('/')) {
p = p.slice(0, -1);
}
return p;
}
// IFuseAPI implementation
public readdir(dirPath: string): string[] | null {
dirPath = this.normalizePath(dirPath);
const node = this.nodes.get(dirPath);
if (!node || node.type !== 'directory') {
return null;
}
return Array.from(node.children!);
}
public getattr(filepath: string): FileAttributes | null {
filepath = this.normalizePath(filepath);
const node = this.nodes.get(filepath);
if (!node) {
return null;
}
if (node.type === 'directory') {
return {
size: 4096, // Standard directory size
mode: 0o040000 | this.config.directoryMode, // S_IFDIR | permissions
mtime: Date.now() / 1000,
atime: Date.now() / 1000,
ctime: Date.now() / 1000,
nlink: 2, // . and ..
uid: this.config.uid,
gid: this.config.gid
};
} else {
return {
size: node.size!,
mode: 0o100000 | this.config.fileMode, // S_IFREG | permissions
mtime: node.mtime!.getTime() / 1000,
atime: node.mtime!.getTime() / 1000,
ctime: node.ctime!.getTime() / 1000,
nlink: 1,
uid: this.config.uid,
gid: this.config.gid
};
}
}
public exists(filepath: string): boolean {
filepath = this.normalizePath(filepath);
return this.nodes.has(filepath);
}
public read(filepath: string): ReadResult | null {
filepath = this.normalizePath(filepath);
const node = this.nodes.get(filepath);
if (!node || node.type !== 'file') {
return null;
}
// Virtual metadata file (any format)
if (node.sourcePath === null && node.metadata && node.formatHandler) {
const metaContent = node.formatHandler.serialize(node.metadata);
const metaBuffer = Buffer.from(metaContent, 'utf-8');
return {
sourcePath: null,
content: metaBuffer,
size: metaBuffer.length
};
}
// Regular file - return source path for reading
return {
sourcePath: node.sourcePath!,
content: null,
size: node.size!
};
}
public getMetadata(filepath: string): HashMeta | null {
filepath = this.normalizePath(filepath);
const node = this.nodes.get(filepath);
if (!node || node.type !== 'file') {
return null;
}
return node.metadata ?? null;
}
public getTree(): VirtualDirectory {
const rootNode = this.nodes.get('/')!;
return {
path: '/',
type: 'directory',
name: '',
parent: '',
children: Array.from(rootNode.children!),
mode: this.config.directoryMode,
mtime: new Date()
};
}
public async refresh(): Promise<void> {
// Refresh is triggered by external calls after processing
// The buildFromComputed method handles the actual refresh
this.emit('vfs-updated');
}
/**
* Get all file paths in the VFS
*/
public getAllFiles(): string[] {
const files: string[] = [];
for (const [filepath, node] of this.nodes.entries()) {
if (node.type === 'file') {
files.push(filepath);
}
}
return files;
}
/**
* Get all directory paths in the VFS
*/
public getAllDirectories(): string[] {
const dirs: string[] = [];
for (const [dirpath, node] of this.nodes.entries()) {
if (node.type === 'directory') {
dirs.push(dirpath);
}
}
return dirs;
}
/**
* Get statistics about the VFS (returns cached values, updated incrementally)
*/
public getStats(): {
fileCount: number;
directoryCount: number;
totalSize: number;
metaFileCount: number;
metadataFileCount: number;
} {
// Return cached stats - no iteration needed!
return { ...this.cachedStats };
}
/**
* Count files in VFS that have completed hash computation
* A file is considered "done" if hashStatus === 'complete' (all hashes computed without errors)
*/
public countFilesWithHash(): number {
let count = 0;
for (const [filepath, node] of this.nodes.entries()) {
// Only count actual media files (not directories, not metadata files)
if (node.type === 'file' && node.sourcePath && node.metadata) {
// Check if file has completed processing (both light and hash phases)
if ((node.metadata as any).processingStatus === 'complete') {
count++;
}
}
}
return count;
}
/**
* Update metadata for an existing file in the VFS (e.g., when hash computation completes)
* This also regenerates the metadata files with updated content
*/
public updateFileMetadata(virtualPath: string, metadata: HashMeta): void {
virtualPath = this.normalizePath(virtualPath);
const node = this.nodes.get(virtualPath);
if (!node || node.type !== 'directory') {
// Update file metadata
if (node) {
node.metadata = metadata;
// Regenerate metadata files with updated content
if (node.mtime) {
// Get old metadata file sizes before regenerating
const oldSizes = new Map<string, number>();
for (const handler of this.activeFormatHandlers) {
const metaPath = `${virtualPath}${handler.extension}`;
const oldMetaNode = this.nodes.get(metaPath);
if (oldMetaNode) {
oldSizes.set(metaPath, oldMetaNode.size ?? 0);
}
}
// Regenerate metadata files (this will overwrite the nodes)
this.addMetadataFilesUpdate(virtualPath, metadata, node.mtime, oldSizes);
}
this.emit('file-updated', virtualPath);
}
}
}
/**
* Add/update virtual metadata files for all configured formats (used during updates)
* This version adjusts cached stats based on size differences
*/
private addMetadataFilesUpdate(
mediaPath: string,
metadata: HashMeta,
mtime: Date,
oldSizes: Map<string, number>
): void {
// Create a metadata file for each configured format
for (const handler of this.activeFormatHandlers) {
const metaPath = `${mediaPath}${handler.extension}`;
const dirname = path.dirname(metaPath);
const filename = path.basename(metaPath);
// Pre-generate metadata content to calculate size
const metaContent = handler.serialize(metadata);
const metaBuffer = Buffer.from(metaContent, 'utf-8');
const newSize = metaBuffer.length;
const oldSize = oldSizes.get(metaPath);
const isNew = oldSize === undefined;
// Update the virtual metadata file node
this.nodes.set(metaPath, {
type: 'file',
name: filename,
parent: dirname,
sourcePath: null, // Virtual file, no source
size: newSize,
mtime: mtime,
ctime: mtime,
metadata,
formatHandler: handler
});
// Add to parent's children (no-op if already exists)
const parentNode = this.nodes.get(dirname);
if (parentNode && parentNode.type === 'directory') {
parentNode.children!.add(filename);
}
// Update cached stats
if (isNew) {
// New metadata file
this.cachedStats.fileCount++;
this.cachedStats.totalSize += newSize;
this.cachedStats.metadataFileCount++;
if (handler.extension === '.meta') {
this.cachedStats.metaFileCount++;
}
} else {
// Update existing - adjust size difference
this.cachedStats.totalSize += (newSize - oldSize);
}
}
}
/**
* Remove a file and its associated metadata files from the VFS
*/
public removeFile(virtualPath: string): void {
virtualPath = this.normalizePath(virtualPath);
const node = this.nodes.get(virtualPath);
if (!node || node.type !== 'file') {
return;
}
// Update cached stats for main file
this.cachedStats.fileCount--;
this.cachedStats.totalSize -= node.size ?? 0;
// Remove the main file
this.nodes.delete(virtualPath);
// Remove from parent's children
const parentNode = this.nodes.get(node.parent!);
if (parentNode && parentNode.type === 'directory') {
parentNode.children!.delete(node.name);
}
// Remove associated metadata files (.nfo, .meta, etc.)
for (const handler of this.activeFormatHandlers) {
const metaPath = `${virtualPath}${handler.extension}`;
const metaNode = this.nodes.get(metaPath);
if (metaNode) {
// Update cached stats for metadata file
this.cachedStats.fileCount--;
this.cachedStats.totalSize -= metaNode.size ?? 0;
this.cachedStats.metadataFileCount--;
if (handler.extension === '.meta') {
this.cachedStats.metaFileCount--;
}
this.nodes.delete(metaPath);
// Remove from parent's children
const metaParent = this.nodes.get(metaNode.parent!);
if (metaParent && metaParent.type === 'directory') {
metaParent.children!.delete(metaNode.name);
}
}
}
this.emit('file-removed', virtualPath);
}
/**
* Check if a file exists in the VFS and return its metadata
*/
public getFileMetadata(virtualPath: string): HashMeta | undefined {
virtualPath = this.normalizePath(virtualPath);
const node = this.nodes.get(virtualPath);
return node?.metadata;
}
}
// Type-safe event emitter
export interface VirtualFileSystem {
on<K extends keyof FuseAPIEvents>(event: K, listener: FuseAPIEvents[K]): this;
emit<K extends keyof FuseAPIEvents>(event: K, ...args: Parameters<FuseAPIEvents[K]>): boolean;
}
packages/meta-sort-core/src/api/index.ts
+11
View File
@@ -0,0 +1,11 @@
/**
* API exports
*
* This module exports all API components including FUSE API, Metrics API, and Unified API Server.
*/
export { IFuseAPI, VirtualNode, VirtualFile, VirtualDirectory, FileAttributes, ReadResult, FuseAPIConfig, FuseAPIEvents } from './FuseAPI.js';
export { VirtualFileSystem } from './VirtualFileSystem.js';
export { FuseAPIServer, FuseAPIServerConfig } from './FuseAPIServer.js';
export { MetricsAPI, MetricsAPIConfig } from './MetricsAPI.js';
export { UnifiedAPIServer, UnifiedAPIServerConfig } from './UnifiedAPIServer.js';
packages/meta-sort-core/src/config/EnvConfig.ts
+216
View File
@@ -0,0 +1,216 @@
import dotenv from 'dotenv';
dotenv.config();
export interface PollingPathConfig {
path: string;
intervalMs: number;
}
interface EnvConfig {
/** Comma-separated list of folders to watch for changes.(mandatory) */
WATCH_FOLDER_LIST: string;
/** Path to the index file within the NFO CID library. */
INDEX_FOLDER_PATH: string;
/** Interval in milliseconds to update/check for changes. (default 30000) */
UPDATE_INTERVAL_MS: number;
/** Interval in milliseconds for polling watch folders. Set to 0 to disable polling. (default 0) */
WATCH_FOLDER_POOLING_INTERVAL_MS: number;
/** CHMOD for new folder format (default 0o777) */
CHMOD_FOR_NEW_FOLDER: number;
/** Path to the cache folder. */
CACHE_FOLDER_PATH: string;
/** jellyfin */
JELLYFIN_ENDPOINT: string;
JELLYFIN_API_KEY: string;
/** FUSE API server port (default 3000) */
FUSE_API_PORT: number;
/** FUSE API server host (default 'localhost') */
FUSE_API_HOST: string;
/** Comma-separated list of metadata formats to generate in virtual filesystem (default 'meta'). Options: 'meta' (.meta YAML files), 'jellyfin' (.nfo XML files). Example: 'meta,jellyfin' or 'jellyfin' or '' (no metadata files) */
METADATA_FORMATS: string[];
/** Maximum number of worker threads for Piscina worker pool AND max concurrent file processing (default: CPU count) */
MAX_WORKER_THREADS?: number;
/** JSON array of polling path configurations. Example: '[{"path": "/data/watch/smb/downloads", "intervalMs": 60000}]' (optional) */
POLLING_PATHS?: PollingPathConfig[];
// ========================================================================
// KV Storage Configuration (NEW)
// ========================================================================
/** Path to META_CORE_VOLUME - infrastructure volume for KV DB, locks, services (default '/meta-core') */
META_CORE_PATH: string;
/** Path to FILES_VOLUME - shared media volume (default '/files') */
FILES_PATH: string;
/** Service name for leader election and service discovery (default 'meta-sort') */
SERVICE_NAME: string;
/** Service version (default '1.0.0') */
SERVICE_VERSION: string;
/** Redis port (default 6379) */
REDIS_PORT: number;
/** Direct Redis URL (skip leader election if provided) */
REDIS_URL?: string;
/** Base URL for service discovery (e.g., 'http://localhost:3000' or 'http://meta-sort:80') */
BASE_URL?: string;
/**
* Hostname to advertise in lock file for other services to connect.
* Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution.
* If not set, auto-detects local IP (may be unstable across container restarts).
*/
ADVERTISE_HOST?: string;
// ========================================================================
// TaskScheduler Configuration (Plugin Queue System)
// ========================================================================
/** Concurrency for fast plugin queue (plugins with <1s avg execution time). Default: 32 */
FAST_QUEUE_CONCURRENCY: number;
/** Concurrency for background plugin queue (plugins with >=1s avg execution time). Default: 8 */
BACKGROUND_QUEUE_CONCURRENCY: number;
/** Threshold in ms to classify plugin as fast vs background. Default: 1000 */
FAST_THRESHOLD_MS: number;
/** Minimum samples before using measured timing for classification. Default: 10 */
MIN_SAMPLES_FOR_MEASUREMENT: number;
// ========================================================================
// Container Plugins Configuration
// All plugins run as Docker containers for isolation and flexibility
// ========================================================================
/** Path to container plugins config file (default: '/app/plugins.yml') */
CONTAINER_PLUGINS_CONFIG: string;
/** Docker socket path (default: '/var/run/docker.sock') */
DOCKER_SOCKET_PATH: string;
/** Callback URL for container plugins (default: 'http://meta-sort:8180') */
CONTAINER_CALLBACK_URL: string;
/** meta-core URL for container plugins (default: 'http://meta-sort:9000') */
CONTAINER_META_CORE_URL: string;
/** Docker network for plugin containers (default: 'meta-network') */
CONTAINER_NETWORK: string;
/**
* Plugin file mounts - maps subpaths to host paths for /files in plugin containers.
* Format: "subpath:host_path,subpath2:host_path2"
* Example: "watch:/data/media,test:/app/test/media"
* Results in /files/watch and /files/test mounted in plugin containers (read-only).
*/
PLUGIN_FILE_MOUNTS?: string;
/** Host path for plugin cache folder - each plugin gets its own subdirectory at /cache (optional, dev only) */
PLUGIN_CACHE_FOLDER?: string;
/**
* Host path for plugin output folders - each plugin gets /files/plugin/<id> mount (READ-WRITE)
* Plugins decide whether to use this folder - we always mount it if this env is set
*/
PLUGIN_OUTPUT_FOLDER?: string;
/** Docker Compose project name for grouping containers in Docker Desktop (optional). When set, plugin containers appear grouped as a stack. */
PLUGIN_STACK_NAME?: string;
}
// Parse POLLING_PATHS from JSON string
function parsePollingPaths(): PollingPathConfig[] | undefined {
if (!process.env.POLLING_PATHS) {
return undefined;
}
try {
const parsed = JSON.parse(process.env.POLLING_PATHS);
if (!Array.isArray(parsed)) {
console.error('POLLING_PATHS must be a JSON array');
return undefined;
}
// Validate structure
const valid = parsed.every(item =>
item && typeof item === 'object' &&
typeof item.path === 'string' &&
typeof item.intervalMs === 'number'
);
if (!valid) {
console.error('POLLING_PATHS entries must have {path: string, intervalMs: number}');
return undefined;
}
return parsed as PollingPathConfig[];
} catch (error) {
console.error('Failed to parse POLLING_PATHS:', error);
return undefined;
}
}
export const config: EnvConfig = {
WATCH_FOLDER_LIST: process.env.WATCH_FOLDER_LIST,
INDEX_FOLDER_PATH: process.env.INDEX_FOLDER_PATH || '/data/cache/hash',//default /data/cache/hash
UPDATE_INTERVAL_MS: parseInt(process.env.UPDATE_INTERVAL_MS || "30000", 10),//default 30000
WATCH_FOLDER_POOLING_INTERVAL_MS: parseInt(process.env.WATCH_FOLDER_POOLING_INTERVAL_MS || "0", 10),//default 0
CHMOD_FOR_NEW_FOLDER: process.env.CHMOD_FOR_NEW_FOLDER && parseInt(process.env.CHMOD_FOR_NEW_FOLDER, 8),//default 0o777 (this is the default for mkdir)
CACHE_FOLDER_PATH: process.env.CACHE_FOLDER_PATH || '/data/cache/plugins',//default /data/cache/plugins
JELLYFIN_ENDPOINT: process.env.JELLYFIN_ENDPOINT!,
JELLYFIN_API_KEY: process.env.JELLYFIN_API_KEY!,
FUSE_API_PORT: parseInt(process.env.FUSE_API_PORT || "3000", 10),//default 3000
FUSE_API_HOST: process.env.FUSE_API_HOST || 'localhost',//default localhost
METADATA_FORMATS: (process.env.METADATA_FORMATS || 'meta').split(',').map(f => f.trim()).filter(f => f.length > 0),//default 'meta'
MAX_WORKER_THREADS: process.env.MAX_WORKER_THREADS ? parseInt(process.env.MAX_WORKER_THREADS, 10) : undefined,
POLLING_PATHS: parsePollingPaths(),
// KV Storage Configuration
META_CORE_PATH: process.env.META_CORE_PATH || '/meta-core',
FILES_PATH: process.env.FILES_PATH || '/files',
SERVICE_NAME: process.env.SERVICE_NAME || 'meta-sort',
SERVICE_VERSION: process.env.SERVICE_VERSION || '1.0.0',
REDIS_PORT: parseInt(process.env.REDIS_PORT || "6379", 10),
REDIS_URL: process.env.REDIS_URL,
BASE_URL: process.env.BASE_URL,
ADVERTISE_HOST: process.env.ADVERTISE_HOST,
// TaskScheduler Configuration
FAST_QUEUE_CONCURRENCY: parseInt(process.env.FAST_QUEUE_CONCURRENCY || "32", 10),
BACKGROUND_QUEUE_CONCURRENCY: parseInt(process.env.BACKGROUND_QUEUE_CONCURRENCY || "8", 10),
FAST_THRESHOLD_MS: parseInt(process.env.FAST_THRESHOLD_MS || "1000", 10),
MIN_SAMPLES_FOR_MEASUREMENT: parseInt(process.env.MIN_SAMPLES_FOR_MEASUREMENT || "10", 10),
// Container Plugins Configuration
CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml',
DOCKER_SOCKET_PATH: process.env.DOCKER_SOCKET_PATH || '/var/run/docker.sock',
CONTAINER_CALLBACK_URL: process.env.CONTAINER_CALLBACK_URL || 'http://meta-sort:8180',
CONTAINER_META_CORE_URL: process.env.CONTAINER_META_CORE_URL || 'http://meta-sort:9000',
CONTAINER_NETWORK: process.env.CONTAINER_NETWORK || 'meta-network',
PLUGIN_FILE_MOUNTS: process.env.PLUGIN_FILE_MOUNTS,
PLUGIN_CACHE_FOLDER: process.env.PLUGIN_CACHE_FOLDER,
PLUGIN_OUTPUT_FOLDER: process.env.PLUGIN_OUTPUT_FOLDER,
PLUGIN_STACK_NAME: process.env.PLUGIN_STACK_NAME,
};
if(process.env.TEST!=='true') {
if (!config.WATCH_FOLDER_LIST) {
console.error('Invalid configuration: WATCH_FOLDER_LIST is required');
}
}
packages/meta-sort-core/src/config/RenamingRule.ts
+62
View File
@@ -0,0 +1,62 @@
import {
AnimeMeta, FileNameMeta,
FileStatMeta,
HashMeta,
LanguagesMeta, SubtitleFileMeta,
VersionMeta,
VideoMeta
} from "@metazla/meta-interface";
type Local = VideoMeta & HashMeta & FileStatMeta & AnimeMeta & VersionMeta & LanguagesMeta & SubtitleFileMeta & FileNameMeta;
/**
* return the path relative to the root folder
*/
export function renamingRule(metamesh: Local, filepath?: string): string | null {
let supported = ['video', 'subtitle', 'torrent'];
//check if filetype is one of the suported ones
if (!supported.includes(metamesh.fileType)) {
return null;
}
// Extract the file extension from the original file path
const extension = metamesh.extension;
if (!extension) {
throw new Error(`No extension found for file: ${filepath}`);
}
const title = metamesh?.titles?.eng || metamesh.originalTitle;
if (!title) {
if (metamesh.fileType !== "torrent") {
//don't warn for torrent files because they don't always have a title (eg representing a folder of files)
throw new Error(`No title found for file: ${filepath}`);
}
return null;
}
let newPath = "";
const season = metamesh.season ? ("S" + String(metamesh.season).padStart(2, '0')) : "";
const seasonSpace = metamesh.season ? ` ${season}` : "";
const episode = metamesh.episode ? ("E" + String(metamesh.episode).padStart(2, '0')) : "";
const version = metamesh.version ? ` ${metamesh.version}` : "";
const year = metamesh.movieYear ? ` (${metamesh.movieYear})` : "";
const subtitle = metamesh.fileType === "subtitle" && metamesh.subtitleLanguage ? `.${metamesh.subtitleLanguage}` : "";
const fileName = `${title}${seasonSpace}${episode}${year}${subtitle}${version}.${extension}`;
// Simplified structure: TV Shows for series, Movies for standalone content
// Note: season can be 0 (special episodes), so check for null/undefined explicitly
const hasSeason = metamesh.season !== null && metamesh.season !== undefined;
const hasEpisode = metamesh.episode !== null && metamesh.episode !== undefined;
if (metamesh.extra) {
newPath = `TV Shows/${title}/extra/${fileName}`;
} else if (hasSeason && hasEpisode) {
newPath = `TV Shows/${title}/${season}/${fileName}`;
} else {
newPath = `Movies/${title}${year}/${fileName}`;
}
return newPath;
}
packages/meta-sort-core/src/config/SupportedFileTypes.ts
+68
View File
@@ -0,0 +1,68 @@
/**
* Supported file extensions for Meta Mesh
* Application-specific configuration
*/
export const SUPPORTED_VIDEO_EXTENSIONS = new Set([
'.mkv',
'.mp4',
'.avi',
'.mov',
'.wmv',
'.flv',
'.webm',
'.m4v',
'.mpg',
'.mpeg',
'.m2ts',
'.ts',
'.vob',
'.3gp',
'.ogv',
'.divx',
'.xvid',
'.rm',
'.rmvb'
]);
export const SUPPORTED_SUBTITLE_EXTENSIONS = new Set([
'.srt',
'.ass',
'.ssa',
'.sub',
'.vtt',
'.idx'
]);
export const SUPPORTED_METADATA_EXTENSIONS = new Set([
'.torrent'
]);
export const SUPPORTED_IMAGE_EXTENSIONS = new Set([
'.jpg',
'.jpeg',
'.png',
'.webp',
'.gif',
'.bmp',
'.tiff',
'.tif'
]);
/**
* All supported file extensions
*/
export const SUPPORTED_EXTENSIONS = new Set([
...SUPPORTED_VIDEO_EXTENSIONS,
...SUPPORTED_SUBTITLE_EXTENSIONS,
...SUPPORTED_METADATA_EXTENSIONS,
...SUPPORTED_IMAGE_EXTENSIONS
]);
/**
* Check if file extension is supported
*/
export function isSupportedExtension(filePath: string): boolean {
const ext = filePath.substring(filePath.lastIndexOf('.')).toLowerCase();
return SUPPORTED_EXTENSIONS.has(ext);
}
packages/meta-sort-core/src/config/TargetHash.ts
+21
View File
@@ -0,0 +1,21 @@
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
// Target hashes for Piscina worker (full-file hash computation)
// Note: midhash256 is NOT included here because it's computed in the main thread
// during light processing, not by the Piscina worker
export const targetHash = [
CID_ALGORITHM_NAMES.sha1, // For torrent verification
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
CID_ALGORITHM_NAMES.md5, // For compatibility
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
CID_ALGORITHM_NAMES.crc32, // Fast checksum
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
]
// Target hashes for index-cid cache (includes all hashes we want to cache)
// This includes midhash256 which is computed separately from the worker hashes
export const targetHashForIndex = [
...targetHash,
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
]
packages/meta-sort-core/src/container-plugins/ConfigParser.ts
+372
View File
@@ -0,0 +1,372 @@
/**
* Container Plugin Configuration Parser
*
* Parses plugins.yml configuration files for container plugins.
*/
import * as fs from 'fs/promises';
import * as path from 'path';
import YAML from 'yaml';
import type {
ContainerPluginsConfig,
ContainerPluginConfig,
ContainerResourceLimits,
ContainerMount,
ContainerHealthCheck,
} from './types.js';
import type { TaskQueueType } from '../plugin-engine/types.js';
/**
* Parse error with context
*/
export class ConfigParseError extends Error {
constructor(
message: string,
public readonly filePath: string,
public readonly cause?: Error
) {
super(`${message} (file: ${filePath})`);
this.name = 'ConfigParseError';
}
}
/**
* Default configuration values
*/
const DEFAULTS = {
version: '1.0',
instances: 1,
enabled: true,
network: false,
healthCheck: {
interval: '30s',
timeout: '5s',
retries: 3,
},
} as const;
/**
* Substitute environment variables in a string
* Supports: ${VAR}, ${VAR:-default}, ${VAR:=default}
*/
function substituteEnvVars(value: string): string {
// Match ${VAR}, ${VAR:-default}, or ${VAR:=default}
const envPattern = /\$\{([A-Z_][A-Z0-9_]*)(?:(:?[-=])([^}]*))?\}/gi;
return value.replace(envPattern, (match, varName, operator, defaultValue) => {
const envValue = process.env[varName];
if (envValue !== undefined && envValue !== '') {
return envValue;
}
// Handle default values
if (operator === ':-' || operator === ':=') {
return defaultValue || '';
}
if (operator === '-' || operator === '=') {
// Only use default if variable is unset (not if empty)
if (envValue === undefined) {
return defaultValue || '';
}
return '';
}
// No default, return empty string
return '';
});
}
/**
* Recursively substitute env vars in an object
*/
function substituteEnvVarsInObject(obj: unknown): unknown {
if (typeof obj === 'string') {
return substituteEnvVars(obj);
}
if (Array.isArray(obj)) {
return obj.map(item => substituteEnvVarsInObject(item));
}
if (obj !== null && typeof obj === 'object') {
const result: Record<string, unknown> = {};
for (const [key, value] of Object.entries(obj)) {
result[key] = substituteEnvVarsInObject(value);
}
return result;
}
return obj;
}
/**
* Validate resource limits
*/
function validateResourceLimits(
resources: unknown,
pluginId: string
): ContainerResourceLimits | undefined {
if (!resources || typeof resources !== 'object') {
return undefined;
}
const r = resources as Record<string, unknown>;
const result: ContainerResourceLimits = {};
if (r.memory !== undefined) {
if (typeof r.memory !== 'string') {
throw new Error(`Plugin '${pluginId}': resources.memory must be a string (e.g., '512m', '2g')`);
}
// Validate format
if (!/^\d+[kmgKMG]?$/.test(r.memory)) {
throw new Error(`Plugin '${pluginId}': invalid memory format '${r.memory}' (use e.g., '512m', '2g')`);
}
result.memory = r.memory;
}
if (r.cpus !== undefined) {
const cpus = Number(r.cpus);
if (isNaN(cpus) || cpus <= 0) {
throw new Error(`Plugin '${pluginId}': resources.cpus must be a positive number`);
}
result.cpus = cpus;
}
return Object.keys(result).length > 0 ? result : undefined;
}
/**
* Validate volume mounts
*/
function validateMounts(mounts: unknown, pluginId: string): ContainerMount[] | undefined {
if (!mounts) {
return undefined;
}
if (!Array.isArray(mounts)) {
throw new Error(`Plugin '${pluginId}': mounts must be an array`);
}
return mounts.map((mount, index) => {
if (!mount || typeof mount !== 'object') {
throw new Error(`Plugin '${pluginId}': mount[${index}] must be an object`);
}
const m = mount as Record<string, unknown>;
if (typeof m.source !== 'string' || !m.source) {
throw new Error(`Plugin '${pluginId}': mount[${index}].source is required`);
}
if (typeof m.target !== 'string' || !m.target) {
throw new Error(`Plugin '${pluginId}': mount[${index}].target is required`);
}
return {
source: m.source,
target: m.target,
readonly: m.readonly !== false, // Default to true
};
});
}
/**
* Validate health check config
*/
function validateHealthCheck(
healthCheck: unknown,
pluginId: string
): ContainerHealthCheck | undefined {
if (!healthCheck || typeof healthCheck !== 'object') {
return undefined;
}
const h = healthCheck as Record<string, unknown>;
const result: ContainerHealthCheck = {};
if (h.interval !== undefined) {
if (typeof h.interval !== 'string') {
throw new Error(`Plugin '${pluginId}': healthCheck.interval must be a string (e.g., '30s')`);
}
result.interval = h.interval;
}
if (h.timeout !== undefined) {
if (typeof h.timeout !== 'string') {
throw new Error(`Plugin '${pluginId}': healthCheck.timeout must be a string (e.g., '5s')`);
}
result.timeout = h.timeout;
}
if (h.retries !== undefined) {
const retries = Number(h.retries);
if (isNaN(retries) || retries < 0 || !Number.isInteger(retries)) {
throw new Error(`Plugin '${pluginId}': healthCheck.retries must be a non-negative integer`);
}
result.retries = retries;
}
return Object.keys(result).length > 0 ? result : undefined;
}
/**
* Validate a single plugin configuration
*/
function validatePluginConfig(
pluginId: string,
config: unknown
): ContainerPluginConfig {
if (!config || typeof config !== 'object') {
throw new Error(`Plugin '${pluginId}': configuration must be an object`);
}
const c = config as Record<string, unknown>;
// Required: image
if (typeof c.image !== 'string' || !c.image) {
throw new Error(`Plugin '${pluginId}': 'image' is required and must be a non-empty string`);
}
// Optional: enabled (default: true)
const enabled = c.enabled !== false;
// Optional: instances (default: 1)
let instances: number = DEFAULTS.instances;
if (c.instances !== undefined) {
instances = Number(c.instances);
if (isNaN(instances) || instances < 1 || !Number.isInteger(instances)) {
throw new Error(`Plugin '${pluginId}': 'instances' must be a positive integer`);
}
}
// Optional: defaultQueue
let defaultQueue: TaskQueueType | undefined;
if (c.defaultQueue !== undefined) {
if (c.defaultQueue !== 'fast' && c.defaultQueue !== 'background') {
throw new Error(`Plugin '${pluginId}': 'defaultQueue' must be 'fast' or 'background'`);
}
defaultQueue = c.defaultQueue;
}
// Optional: network (default: false)
const network = c.network === true;
return {
enabled,
image: c.image,
instances,
resources: validateResourceLimits(c.resources, pluginId),
config: c.config && typeof c.config === 'object'
? c.config as Record<string, unknown>
: undefined,
network,
mounts: validateMounts(c.mounts, pluginId),
healthCheck: validateHealthCheck(c.healthCheck, pluginId),
defaultQueue,
};
}
/**
* Parse and validate container plugins configuration
*/
export function parseConfig(rawConfig: unknown, filePath: string): ContainerPluginsConfig {
if (!rawConfig || typeof rawConfig !== 'object') {
throw new ConfigParseError('Configuration must be an object', filePath);
}
const config = rawConfig as Record<string, unknown>;
// Optional: version
const version = typeof config.version === 'string'
? config.version
: DEFAULTS.version;
// Required: plugins
if (!config.plugins || typeof config.plugins !== 'object') {
throw new ConfigParseError("'plugins' section is required and must be an object", filePath);
}
const plugins: Record<string, ContainerPluginConfig> = {};
for (const [pluginId, pluginConfig] of Object.entries(config.plugins)) {
// Validate plugin ID format
if (!/^[a-z][a-z0-9-]*$/.test(pluginId)) {
throw new ConfigParseError(
`Invalid plugin ID '${pluginId}': must be lowercase alphanumeric with hyphens, starting with a letter`,
filePath
);
}
try {
plugins[pluginId] = validatePluginConfig(pluginId, pluginConfig);
} catch (error) {
throw new ConfigParseError(
error instanceof Error ? error.message : String(error),
filePath
);
}
}
return { version, plugins };
}
/**
* Load and parse plugins.yml configuration file
*/
export async function loadConfig(configPath: string): Promise<ContainerPluginsConfig> {
// Check if file exists
try {
await fs.access(configPath, fs.constants.R_OK);
} catch {
throw new ConfigParseError(`Configuration file not found or not readable`, configPath);
}
// Read file
let content: string;
try {
content = await fs.readFile(configPath, 'utf-8');
} catch (error) {
throw new ConfigParseError(
'Failed to read configuration file',
configPath,
error instanceof Error ? error : undefined
);
}
// Parse YAML
let rawConfig: unknown;
try {
rawConfig = YAML.parse(content);
} catch (error) {
throw new ConfigParseError(
'Invalid YAML syntax',
configPath,
error instanceof Error ? error : undefined
);
}
// Substitute environment variables
const substituted = substituteEnvVarsInObject(rawConfig);
// Validate and return
return parseConfig(substituted, configPath);
}
/**
* Get enabled plugins from config
*/
export function getEnabledPlugins(
config: ContainerPluginsConfig
): Array<{ id: string; config: ContainerPluginConfig }> {
return Object.entries(config.plugins)
.filter(([, pluginConfig]) => (pluginConfig as ContainerPluginConfig).enabled !== false)
.map(([id, pluginConfig]) => ({ id, config: pluginConfig as ContainerPluginConfig }));
}
/**
* Get default config path
*/
export function getDefaultConfigPath(): string {
return process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml';
}
packages/meta-sort-core/src/container-plugins/ContainerManager.ts
+771
View File
@@ -0,0 +1,771 @@
/**
* Container Manager
*
* Manages the lifecycle of plugin containers including spawning,
* health checking, configuration, and cleanup.
*/
import { EventEmitter } from 'events';
import { DockerClient } from './DockerClient.js';
import {
loadConfig,
getEnabledPlugins,
} from './ConfigParser.js';
import type {
ContainerPluginsConfig,
ContainerPluginConfig,
ContainerPluginManifest,
ContainerPluginInstance,
ContainerInstanceStatus,
ContainerManagerStatus,
PluginHealthResponse,
PluginConfigureResponse,
} from './types.js';
import { config } from '../config/EnvConfig.js';
/**
* Container manager events
*/
export interface ContainerManagerEvents {
'plugin:starting': { pluginId: string; instanceIndex: number };
'plugin:started': { pluginId: string; instance: ContainerPluginInstance };
'plugin:healthy': { pluginId: string; instance: ContainerPluginInstance };
'plugin:unhealthy': { pluginId: string; instance: ContainerPluginInstance; error: string };
'plugin:stopped': { pluginId: string; instanceIndex: number };
'plugin:error': { pluginId: string; error: Error };
'manager:initialized': { pluginCount: number };
'manager:shutdown': {};
}
/**
* Container naming helper
*/
function getContainerName(pluginId: string, instanceIndex: number): string {
return `meta-plugin-${pluginId}-${instanceIndex}`;
}
/**
* Container Manager
*/
/**
* ============================================================================
* PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS
* ============================================================================
*
* Each plugin container has exactly 3 mount points (hardcoded):
*
* /files (READ-ONLY) - Files to process, same structure as meta-sort
* /output (READ-WRITE) - Plugin-generated files
* /cache (READ-WRITE) - Plugin-specific cache
*
* Host paths are configured via environment variables:
* - PLUGIN_FILE_MOUNTS: Maps subpaths under /files to host paths
* Format: "subpath:host_path,subpath2:host_path2"
* Example: "watch:/data/media,test:/app/test/media"
* Results in: /files/watch, /files/test mounted in container
*
* - PLUGIN_CACHE_FOLDER: Base path for /cache mounts
* - PLUGIN_OUTPUT_FOLDER: Base path for /output mounts
*
* SECURITY: Plugins have read-only access to /files.
* Only /cache and /output are writable.
*
* ============================================================================
*/
/**
* File mount configuration - maps container path to host path
*/
interface FileMountConfig {
containerPath: string;
hostPath: string;
}
/**
* Parse PLUGIN_FILE_MOUNTS env var into mount configs
* Format: "container_path:host_path,container_path2:host_path2"
*/
function parseFileMounts(mountsStr?: string): FileMountConfig[] {
if (!mountsStr) {
return [];
}
const mounts: FileMountConfig[] = [];
const pairs = mountsStr.split(',').map(s => s.trim()).filter(s => s);
for (const pair of pairs) {
const [containerPath, hostPath] = pair.split(':').map(s => s.trim());
if (containerPath && hostPath) {
mounts.push({ containerPath, hostPath });
} else {
console.warn(`[ContainerManager] Invalid mount format: "${pair}" - expected "container_path:host_path"`);
}
}
return mounts;
}
export class ContainerManager extends EventEmitter {
private dockerClient: DockerClient;
private pluginsConfig: ContainerPluginsConfig | null = null;
private instances: Map<string, ContainerPluginInstance[]> = new Map();
private healthCheckIntervals: Map<string, NodeJS.Timeout> = new Map();
private roundRobinCounters: Map<string, number> = new Map();
private initialized: boolean = false;
private shutdownInProgress: boolean = false;
private fileMounts: FileMountConfig[] = [];
private pluginCacheFolder?: string;
private pluginOutputFolder?: string;
private stackName?: string;
constructor(
private configPath: string = config.CONTAINER_PLUGINS_CONFIG,
private network: string = config.CONTAINER_NETWORK,
private filesPath: string = config.FILES_PATH,
private callbackUrl: string = config.CONTAINER_CALLBACK_URL,
private metaCoreUrl: string = config.CONTAINER_META_CORE_URL,
dockerClient?: DockerClient
) {
super();
this.dockerClient = dockerClient || new DockerClient(config.DOCKER_SOCKET_PATH);
// Parse file mounts from PLUGIN_FILE_MOUNTS env var
this.fileMounts = parseFileMounts(config.PLUGIN_FILE_MOUNTS);
// Configure plugin cache folder (READ-WRITE)
this.pluginCacheFolder = config.PLUGIN_CACHE_FOLDER;
// Configure plugin output folder (READ-WRITE for /plugin-files)
this.pluginOutputFolder = config.PLUGIN_OUTPUT_FOLDER;
// Configure stack name for Docker Desktop grouping
this.stackName = config.PLUGIN_STACK_NAME;
}
/**
* Initialize the container manager
*/
async initialize(): Promise<void> {
if (this.initialized) {
return;
}
console.log('[ContainerManager] Initializing...');
// Log configured host mounts
console.log('[ContainerManager] Plugin mount configuration (hardcoded paths):');
console.log(' /files (RO) - Files to process:');
if (this.fileMounts.length > 0) {
for (const mount of this.fileMounts) {
const targetPath = mount.containerPath.startsWith('/files')
? mount.containerPath
: `/files/${mount.containerPath}`;
console.log(` ${targetPath} -> ${mount.hostPath}`);
}
} else {
console.warn(' (not configured) - Set PLUGIN_FILE_MOUNTS');
}
console.log(` /cache (RW) - Plugin cache: ${this.pluginCacheFolder || '(not configured)'}`);
console.log(` /output (RW) - Plugin output: ${this.pluginOutputFolder || '(not configured)'}`)
// Initialize Docker client
await this.dockerClient.initialize();
// Ensure network exists
await this.dockerClient.ensureNetwork(this.network);
// Load configuration
this.pluginsConfig = await loadConfig(this.configPath);
// Get enabled plugins
const enabledPlugins = getEnabledPlugins(this.pluginsConfig);
console.log(`[ContainerManager] Found ${enabledPlugins.length} enabled plugins`);
// Clean up any stale containers from previous runs
await this.cleanupStaleContainers();
// Spawn containers for each enabled plugin
for (const { id, config: pluginConfig } of enabledPlugins) {
try {
await this.spawnPlugin(id, pluginConfig);
} catch (error) {
console.error(`[ContainerManager] Failed to spawn plugin '${id}':`, error);
this.emit('plugin:error', {
pluginId: id,
error: error instanceof Error ? error : new Error(String(error)),
});
}
}
this.initialized = true;
this.emit('manager:initialized', { pluginCount: enabledPlugins.length });
console.log('[ContainerManager] Initialization complete');
}
/**
* Shutdown the container manager
*/
async shutdown(): Promise<void> {
if (this.shutdownInProgress) {
return;
}
this.shutdownInProgress = true;
console.log('[ContainerManager] Shutting down...');
// Stop health checks
for (const interval of this.healthCheckIntervals.values()) {
clearInterval(interval);
}
this.healthCheckIntervals.clear();
// Stop all plugin containers
for (const [pluginId, instances] of this.instances) {
for (const instance of instances) {
try {
await this.stopInstance(instance);
} catch (error) {
console.error(
`[ContainerManager] Error stopping ${instance.containerName}:`,
error
);
}
}
}
this.instances.clear();
this.initialized = false;
this.shutdownInProgress = false;
this.emit('manager:shutdown', {});
console.log('[ContainerManager] Shutdown complete');
}
/**
* Spawn all instances for a plugin
*/
async spawnPlugin(pluginId: string, pluginConfig: ContainerPluginConfig): Promise<void> {
const instanceCount = pluginConfig.instances || 1;
const instances: ContainerPluginInstance[] = [];
console.log(
`[ContainerManager] Spawning ${instanceCount} instance(s) for plugin '${pluginId}'`
);
// Pull image if needed
const imageExists = await this.dockerClient.imageExists(pluginConfig.image);
if (!imageExists) {
console.log(`[ContainerManager] Pulling image ${pluginConfig.image}...`);
await this.dockerClient.pullImage(pluginConfig.image, (event) => {
if (event.status) {
console.log(`[ContainerManager] Pull: ${event.status}`);
}
});
}
// Spawn each instance
for (let i = 0; i < instanceCount; i++) {
this.emit('plugin:starting', { pluginId, instanceIndex: i });
const instance = await this.spawnInstance(pluginId, pluginConfig, i);
instances.push(instance);
this.emit('plugin:started', { pluginId, instance });
}
this.instances.set(pluginId, instances);
// Start health checking
this.startHealthCheck(pluginId);
// Wait for at least one instance to be healthy
await this.waitForHealthy(pluginId, 30000);
// Fetch manifest from first healthy instance
const healthyInstance = this.getHealthyInstance(pluginId);
if (healthyInstance) {
try {
const manifest = await this.fetchManifest(healthyInstance);
// Update all instances with manifest
for (const inst of instances) {
inst.manifest = manifest;
}
} catch (error) {
console.warn(
`[ContainerManager] Failed to fetch manifest for '${pluginId}':`,
error
);
}
// Send configuration
if (pluginConfig.config) {
try {
await this.sendConfig(healthyInstance, pluginConfig.config);
} catch (error) {
console.warn(
`[ContainerManager] Failed to configure '${pluginId}':`,
error
);
}
}
}
}
/**
* Spawn a single container instance
*/
private async spawnInstance(
pluginId: string,
pluginConfig: ContainerPluginConfig,
instanceIndex: number
): Promise<ContainerPluginInstance> {
const containerName = getContainerName(pluginId, instanceIndex);
// Check if container already exists
const existingContainers = await this.dockerClient.listContainers({
name: [containerName],
});
if (existingContainers.length > 0) {
const existing = existingContainers[0];
// Remove existing container
console.log(`[ContainerManager] Removing existing container ${containerName}`);
await this.dockerClient.removeContainer(existing.Id, true);
}
/**
* PLUGIN MOUNT ARCHITECTURE - HARDCODED PATHS:
* /files (READ-ONLY) - Files to process
* /output (READ-WRITE) - Plugin-generated files
* /cache (READ-WRITE) - Plugin-specific cache
*/
const mounts: Array<{ source: string; target: string; readonly: boolean }> = [];
// /files/* (READ-ONLY) - Mount each configured subpath under /files
for (const fileMount of this.fileMounts) {
// fileMount.containerPath is the subpath (e.g., "watch", "test")
// Mount at /files/{subpath}
const targetPath = fileMount.containerPath.startsWith('/files')
? fileMount.containerPath
: `/files/${fileMount.containerPath}`;
mounts.push({
source: fileMount.hostPath,
target: targetPath,
readonly: true,
});
}
if (this.fileMounts.length === 0) {
console.warn(`[ContainerManager] WARNING: No file mounts configured. Plugin '${pluginId}' won't have access to files.`);
}
// /cache (READ-WRITE) - Plugin-specific cache
if (this.pluginCacheFolder) {
mounts.push({
source: `${this.pluginCacheFolder}/${pluginId}`,
target: '/cache',
readonly: false,
});
}
// /output (READ-WRITE) - Plugin-generated files
if (this.pluginOutputFolder) {
mounts.push({
source: `${this.pluginOutputFolder}/${pluginId}`,
target: '/output',
readonly: false,
});
}
// Create container
const containerId = await this.dockerClient.createContainer({
image: pluginConfig.image,
name: containerName,
network: this.network,
resources: pluginConfig.resources,
mounts,
labels: {
'meta-mesh.plugin': 'true',
'meta-mesh.plugin.id': pluginId,
'meta-mesh.plugin.instance': String(instanceIndex),
},
env: {
PLUGIN_ID: pluginId,
META_CORE_URL: this.metaCoreUrl,
CALLBACK_URL: this.callbackUrl,
},
// Docker Desktop grouping
stackName: this.stackName,
serviceName: pluginId,
});
// Start container
await this.dockerClient.startContainer(containerId);
// Create instance record
const instance: ContainerPluginInstance = {
pluginId,
containerId,
containerName,
baseUrl: `http://${containerName}:8080`,
instanceIndex,
status: 'starting',
tasksProcessed: 0,
tasksFailed: 0,
};
console.log(`[ContainerManager] Started container ${containerName}`);
return instance;
}
/**
* Stop a container instance
*/
private async stopInstance(instance: ContainerPluginInstance): Promise<void> {
console.log(`[ContainerManager] Stopping ${instance.containerName}...`);
try {
await this.dockerClient.stopContainer(instance.containerId, 10);
} catch (error) {
// Container might already be stopped
}
try {
await this.dockerClient.removeContainer(instance.containerId, true);
} catch (error) {
// Container might already be removed
}
instance.status = 'stopped';
this.emit('plugin:stopped', {
pluginId: instance.pluginId,
instanceIndex: instance.instanceIndex,
});
}
/**
* Start health check loop for a plugin
*/
private startHealthCheck(pluginId: string, intervalMs: number = 30000): void {
// Clear existing interval if any
const existing = this.healthCheckIntervals.get(pluginId);
if (existing) {
clearInterval(existing);
}
const checkHealth = async () => {
const instances = this.instances.get(pluginId);
if (!instances) return;
for (const instance of instances) {
if (instance.status === 'stopped') continue;
try {
const healthy = await this.checkHealth(instance);
const previousStatus = instance.status;
if (healthy) {
instance.status = 'healthy';
instance.lastHealthCheck = Date.now();
if (previousStatus !== 'healthy') {
this.emit('plugin:healthy', { pluginId, instance });
}
} else {
instance.status = 'unhealthy';
instance.lastError = 'Health check failed';
if (previousStatus === 'healthy') {
this.emit('plugin:unhealthy', {
pluginId,
instance,
error: 'Health check failed',
});
}
}
} catch (error) {
instance.status = 'unhealthy';
instance.lastError = error instanceof Error ? error.message : String(error);
this.emit('plugin:unhealthy', {
pluginId,
instance,
error: instance.lastError,
});
}
}
};
// Initial check
checkHealth();
// Schedule periodic checks
const interval = setInterval(checkHealth, intervalMs);
this.healthCheckIntervals.set(pluginId, interval);
}
/**
* Check health of a container instance
*/
async checkHealth(instance: ContainerPluginInstance): Promise<boolean> {
try {
const response = await fetch(`${instance.baseUrl}/health`, {
method: 'GET',
signal: AbortSignal.timeout(5000),
});
if (!response.ok) {
return false;
}
const data = (await response.json()) as PluginHealthResponse;
return data.status === 'healthy' && data.ready === true;
} catch {
return false;
}
}
/**
* Wait for at least one healthy instance
*/
private async waitForHealthy(pluginId: string, timeoutMs: number): Promise<void> {
const startTime = Date.now();
const checkInterval = 1000;
while (Date.now() - startTime < timeoutMs) {
const instances = this.instances.get(pluginId);
if (!instances) return;
for (const instance of instances) {
const healthy = await this.checkHealth(instance);
if (healthy) {
instance.status = 'healthy';
instance.lastHealthCheck = Date.now();
this.emit('plugin:healthy', { pluginId, instance });
return;
}
}
await new Promise((resolve) => setTimeout(resolve, checkInterval));
}
console.warn(`[ContainerManager] Timeout waiting for healthy instance of '${pluginId}'`);
}
/**
* Fetch manifest from a container instance
*/
async fetchManifest(instance: ContainerPluginInstance): Promise<ContainerPluginManifest> {
const response = await fetch(`${instance.baseUrl}/manifest`, {
method: 'GET',
signal: AbortSignal.timeout(10000),
});
if (!response.ok) {
throw new Error(`Failed to fetch manifest: ${response.status}`);
}
return response.json() as Promise<ContainerPluginManifest>;
}
/**
* Send configuration to a container instance
*/
async sendConfig(
instance: ContainerPluginInstance,
pluginConfig: Record<string, unknown>
): Promise<void> {
const response = await fetch(`${instance.baseUrl}/configure`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ config: pluginConfig }),
signal: AbortSignal.timeout(10000),
});
if (!response.ok) {
throw new Error(`Failed to configure plugin: ${response.status}`);
}
const data = (await response.json()) as PluginConfigureResponse;
if (!data.success) {
throw new Error(`Plugin configuration failed: ${data.error}`);
}
}
/**
* Get a healthy instance for a plugin (round-robin)
*/
getHealthyInstance(pluginId: string): ContainerPluginInstance | null {
const instances = this.instances.get(pluginId);
if (!instances || instances.length === 0) {
return null;
}
const healthyInstances = instances.filter((i) => i.status === 'healthy');
if (healthyInstances.length === 0) {
return null;
}
// Round-robin selection
const counter = this.roundRobinCounters.get(pluginId) || 0;
const selected = healthyInstances[counter % healthyInstances.length];
this.roundRobinCounters.set(pluginId, counter + 1);
return selected;
}
/**
* Get all instances for a plugin
*/
getInstances(pluginId: string): ContainerPluginInstance[] {
return this.instances.get(pluginId) || [];
}
/**
* Get all plugins
*/
getPluginIds(): string[] {
return Array.from(this.instances.keys());
}
/**
* Check if a plugin is a container plugin
*/
isContainerPlugin(pluginId: string): boolean {
return this.instances.has(pluginId);
}
/**
* Get plugin manifest
*/
getPluginManifest(pluginId: string): ContainerPluginManifest | undefined {
const instances = this.instances.get(pluginId);
if (!instances || instances.length === 0) {
return undefined;
}
return instances[0].manifest;
}
/**
* Get manager status
*/
getStatus(): ContainerManagerStatus {
const plugins: ContainerManagerStatus['plugins'] = [];
for (const [pluginId, instances] of this.instances) {
const pluginConfig = this.pluginsConfig?.plugins[pluginId];
const healthyCount = instances.filter((i) => i.status === 'healthy').length;
plugins.push({
pluginId,
enabled: pluginConfig?.enabled !== false,
image: pluginConfig?.image || 'unknown',
instances: instances.length,
healthyInstances: healthyCount,
manifest: instances[0]?.manifest,
});
}
const totalInstances = Array.from(this.instances.values()).reduce(
(sum, instances) => sum + instances.length,
0
);
const healthyInstances = Array.from(this.instances.values()).reduce(
(sum, instances) => sum + instances.filter((i) => i.status === 'healthy').length,
0
);
return {
initialized: this.initialized,
pluginCount: this.instances.size,
runningContainers: totalInstances,
healthyContainers: healthyInstances,
plugins,
};
}
/**
* Restart a plugin (stop and respawn all instances)
*/
async restartPlugin(pluginId: string): Promise<void> {
console.log(`[ContainerManager] Restarting plugin '${pluginId}'...`);
const instances = this.instances.get(pluginId);
const pluginConfig = this.pluginsConfig?.plugins[pluginId];
if (!instances || !pluginConfig) {
throw new Error(`Plugin '${pluginId}' not found`);
}
// Stop health check
const interval = this.healthCheckIntervals.get(pluginId);
if (interval) {
clearInterval(interval);
this.healthCheckIntervals.delete(pluginId);
}
// Stop all instances
for (const instance of instances) {
await this.stopInstance(instance);
}
this.instances.delete(pluginId);
// Respawn
await this.spawnPlugin(pluginId, pluginConfig);
}
/**
* Clean up stale containers from previous runs
*/
private async cleanupStaleContainers(): Promise<void> {
console.log('[ContainerManager] Cleaning up stale containers...');
const staleContainers = await this.dockerClient.listPluginContainers();
for (const container of staleContainers) {
console.log(
`[ContainerManager] Removing stale container ${container.Names?.[0] || container.Id}`
);
try {
await this.dockerClient.removeContainer(container.Id, true);
} catch (error) {
console.warn(
`[ContainerManager] Failed to remove stale container:`,
error
);
}
}
}
/**
* Record task completion for statistics
*/
recordTaskCompletion(pluginId: string, instanceName: string, success: boolean): void {
const instances = this.instances.get(pluginId);
if (!instances) return;
const instance = instances.find((i) => i.containerName === instanceName);
if (!instance) return;
if (success) {
instance.tasksProcessed++;
} else {
instance.tasksFailed++;
}
}
}
/**
* Create a container manager instance
*/
export function createContainerManager(
configPath?: string,
network?: string
): ContainerManager {
return new ContainerManager(configPath, network);
}
packages/meta-sort-core/src/container-plugins/ContainerPluginScheduler.ts
+651
View File
@@ -0,0 +1,651 @@
/**
* Container Plugin Scheduler
*
* Handles dispatching tasks to containerized plugins and
* processing completion callbacks.
*/
import { EventEmitter } from 'events';
import PQueue from 'p-queue';
import { randomUUID } from 'crypto';
import type { ContainerManager } from './ContainerManager.js';
import type {
ContainerTask,
ContainerTaskStatus,
PluginProcessRequest,
PluginProcessResponse,
PluginCallbackPayload,
CallbackStatus,
} from './types.js';
import type { TaskQueueType } from '../plugin-engine/types.js';
import type { IKVClient } from '../kv/IKVClient.js';
import { config } from '../config/EnvConfig.js';
/**
* Scheduler events
*/
export interface ContainerSchedulerEvents {
'task:created': { task: ContainerTask };
'task:dispatched': { task: ContainerTask };
'task:completed': { task: ContainerTask };
'task:failed': { task: ContainerTask; error: string };
'task:timeout': { task: ContainerTask };
'file:complete': { fileHash: string; filePath: string };
}
/**
* Pending callback info
*/
interface PendingCallback {
task: ContainerTask;
resolve: (result: PluginCallbackPayload) => void;
reject: (error: Error) => void;
timeoutHandle: NodeJS.Timeout;
}
/**
* Container Plugin Scheduler
*/
export class ContainerPluginScheduler extends EventEmitter {
private containerManager: ContainerManager;
private fastQueue: PQueue;
private backgroundQueue: PQueue;
private pendingTasks: Map<string, ContainerTask> = new Map();
private pendingCallbacks: Map<string, PendingCallback> = new Map();
private fileTasks: Map<string, Set<string>> = new Map(); // fileHash -> taskIds
private fileCompletedPlugins: Map<string, Set<string>> = new Map(); // fileHash -> completed pluginIds
private dependencyWaiters: Map<string, Array<() => void>> = new Map(); // "fileHash:pluginId" -> resolve callbacks
private callbackUrl: string;
private metaCoreUrl: string;
private defaultTimeout: number = 60000; // 60 seconds
private kvClient: IKVClient | null = null;
constructor(
containerManager: ContainerManager,
options?: {
fastConcurrency?: number;
backgroundConcurrency?: number;
callbackUrl?: string;
metaCoreUrl?: string;
defaultTimeout?: number;
kvClient?: IKVClient;
}
) {
super();
this.containerManager = containerManager;
this.fastQueue = new PQueue({
concurrency: options?.fastConcurrency ?? config.FAST_QUEUE_CONCURRENCY,
});
this.backgroundQueue = new PQueue({
concurrency: options?.backgroundConcurrency ?? config.BACKGROUND_QUEUE_CONCURRENCY,
});
this.callbackUrl = options?.callbackUrl ?? `${config.CONTAINER_CALLBACK_URL}/api/plugins/callback`;
this.metaCoreUrl = options?.metaCoreUrl ?? config.CONTAINER_META_CORE_URL;
this.defaultTimeout = options?.defaultTimeout ?? 60000;
this.kvClient = options?.kvClient ?? null;
}
/**
* Wait for dependency plugins to complete for a file
*/
private async waitForDependencies(
fileHash: string,
dependencies: string[],
timeoutMs: number = 30000
): Promise<void> {
if (!dependencies || dependencies.length === 0) {
return;
}
const completedPlugins = this.fileCompletedPlugins.get(fileHash) || new Set();
const pendingDeps = dependencies.filter(dep => !completedPlugins.has(dep));
if (pendingDeps.length === 0) {
return; // All dependencies already completed
}
// Wait for each pending dependency
const waitPromises = pendingDeps.map(depPluginId => {
return new Promise<void>((resolve, reject) => {
const key = `${fileHash}:${depPluginId}`;
// Check again in case it completed while we were setting up
const currentCompleted = this.fileCompletedPlugins.get(fileHash);
if (currentCompleted?.has(depPluginId)) {
resolve();
return;
}
// Set up timeout
const timeoutHandle = setTimeout(() => {
// Remove waiter on timeout
const waiters = this.dependencyWaiters.get(key);
if (waiters) {
const idx = waiters.indexOf(resolve);
if (idx >= 0) waiters.splice(idx, 1);
}
// Resolve anyway to not block - plugin will get stale metadata but won't hang
console.warn(`[ContainerScheduler] Timeout waiting for dependency '${depPluginId}' for file ${fileHash}`);
resolve();
}, timeoutMs);
// Register waiter
if (!this.dependencyWaiters.has(key)) {
this.dependencyWaiters.set(key, []);
}
this.dependencyWaiters.get(key)!.push(() => {
clearTimeout(timeoutHandle);
resolve();
});
});
});
await Promise.all(waitPromises);
}
/**
* Mark a plugin as completed for a file and notify waiters
*/
private markPluginCompleted(fileHash: string, pluginId: string, filePath?: string): void {
// Track completion
if (!this.fileCompletedPlugins.has(fileHash)) {
this.fileCompletedPlugins.set(fileHash, new Set());
}
this.fileCompletedPlugins.get(fileHash)!.add(pluginId);
// Notify waiters
const key = `${fileHash}:${pluginId}`;
const waiters = this.dependencyWaiters.get(key);
if (waiters) {
for (const resolve of waiters) {
resolve();
}
this.dependencyWaiters.delete(key);
}
// Publish to Redis for external subscribers (e.g., meta-stremio)
this.publishPluginComplete(fileHash, pluginId, filePath || '');
}
/**
* Publish plugin completion event to Redis
* Channel: meta-sort:plugin:complete
*
* External services (like meta-stremio) can subscribe to this channel
* to be notified when specific plugins complete for a file.
*/
private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise<void> {
if (!this.kvClient) {
return;
}
try {
const message = JSON.stringify({
fileHash,
pluginId,
filePath,
timestamp: Date.now()
});
await (this.kvClient as any).publish('meta-sort:plugin:complete', message);
} catch (error) {
// Silent fail - pub/sub is optional
}
}
/**
* Create a task for a container plugin
*/
createTask(
pluginId: string,
fileHash: string,
filePath: string,
dependencies: string[] = [],
queue: TaskQueueType = 'fast'
): ContainerTask {
const task: ContainerTask = {
id: randomUUID(),
fileHash,
filePath,
pluginId,
dependencies,
queue,
status: 'pending',
createdAt: Date.now(),
};
this.pendingTasks.set(task.id, task);
// Track tasks by file
if (!this.fileTasks.has(fileHash)) {
this.fileTasks.set(fileHash, new Set());
}
this.fileTasks.get(fileHash)!.add(task.id);
this.emit('task:created', { task });
return task;
}
/**
* Enqueue a task for execution
*/
enqueueTask(task: ContainerTask): void {
const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
queue.add(async () => {
await this.executeTask(task);
});
}
/**
* Execute a task (dispatch to container)
*/
private async executeTask(task: ContainerTask): Promise<void> {
// Wait for dependencies to complete first
if (task.dependencies && task.dependencies.length > 0) {
task.status = 'waiting';
await this.waitForDependencies(task.fileHash, task.dependencies);
}
const instance = this.containerManager.getHealthyInstance(task.pluginId);
if (!instance) {
task.status = 'failed';
task.error = `No healthy instance available for plugin '${task.pluginId}'`;
this.emit('task:failed', { task, error: task.error });
this.checkFileCompletion(task.fileHash);
return;
}
task.status = 'dispatched';
task.dispatchedAt = Date.now();
task.instanceName = instance.containerName;
this.emit('task:dispatched', { task });
try {
// Fetch fresh metadata after dependencies complete
// This ensures we get metadata written by dependency plugins
const existingMeta = await this.fetchExistingMetadata(task.fileHash);
// Build process request
const request: PluginProcessRequest = {
taskId: task.id,
cid: task.fileHash,
filePath: task.filePath,
callbackUrl: this.callbackUrl,
metaCoreUrl: this.metaCoreUrl,
existingMeta,
};
// Dispatch to container
const response = await this.dispatchToContainer(instance.baseUrl, request);
if (response.status !== 'accepted') {
throw new Error(response.error || 'Task rejected by plugin');
}
// Wait for callback with timeout
const manifest = instance.manifest;
const timeout = manifest?.timeout ?? this.defaultTimeout;
await this.waitForCallback(task, timeout);
} catch (error) {
task.status = 'failed';
task.error = error instanceof Error ? error.message : String(error);
task.completedAt = Date.now();
if (task.dispatchedAt) {
task.duration = task.completedAt - task.dispatchedAt;
}
this.containerManager.recordTaskCompletion(
task.pluginId,
task.instanceName!,
false
);
this.emit('task:failed', { task, error: task.error });
}
this.checkFileCompletion(task.fileHash);
}
/**
* Dispatch task to container plugin
*/
private async dispatchToContainer(
baseUrl: string,
request: PluginProcessRequest
): Promise<PluginProcessResponse> {
const response = await fetch(`${baseUrl}/process`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(request),
signal: AbortSignal.timeout(10000), // 10s timeout for dispatch
});
if (!response.ok) {
throw new Error(`Plugin returned ${response.status}: ${await response.text()}`);
}
return response.json() as Promise<PluginProcessResponse>;
}
/**
* Wait for callback from container
*/
private waitForCallback(task: ContainerTask, timeoutMs: number): Promise<PluginCallbackPayload> {
return new Promise((resolve, reject) => {
const timeoutHandle = setTimeout(() => {
this.pendingCallbacks.delete(task.id);
task.status = 'timeout';
task.completedAt = Date.now();
if (task.dispatchedAt) {
task.duration = task.completedAt - task.dispatchedAt;
}
this.emit('task:timeout', { task });
reject(new Error(`Task timeout after ${timeoutMs}ms`));
}, timeoutMs);
this.pendingCallbacks.set(task.id, {
task,
resolve,
reject,
timeoutHandle,
});
});
}
/**
* Handle callback from container plugin
*/
handleCallback(payload: PluginCallbackPayload): boolean {
const pending = this.pendingCallbacks.get(payload.taskId);
if (!pending) {
console.warn(
`[ContainerScheduler] Received callback for unknown task: ${payload.taskId}`
);
return false;
}
// Clear timeout
clearTimeout(pending.timeoutHandle);
this.pendingCallbacks.delete(payload.taskId);
const task = pending.task;
task.completedAt = Date.now();
task.duration = payload.duration;
if (payload.status === 'completed') {
task.status = 'completed';
this.containerManager.recordTaskCompletion(
task.pluginId,
task.instanceName!,
true
);
// Mark plugin as completed for dependency tracking
this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
this.emit('task:completed', { task });
pending.resolve(payload);
} else if (payload.status === 'failed') {
task.status = 'failed';
task.error = payload.error;
this.containerManager.recordTaskCompletion(
task.pluginId,
task.instanceName!,
false
);
// Still mark as completed for dependency purposes (don't block other plugins)
this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
this.emit('task:failed', { task, error: payload.error || 'Unknown error' });
pending.reject(new Error(payload.error || 'Task failed'));
} else if (payload.status === 'skipped') {
task.status = 'completed'; // Treat skipped as completed
task.error = payload.reason;
this.containerManager.recordTaskCompletion(
task.pluginId,
task.instanceName!,
true
);
// Mark plugin as completed for dependency tracking
this.markPluginCompleted(task.fileHash, task.pluginId, task.filePath);
this.emit('task:completed', { task });
pending.resolve(payload);
}
return true;
}
/**
* Check if all tasks for a file are complete
*/
private checkFileCompletion(fileHash: string): void {
const taskIds = this.fileTasks.get(fileHash);
if (!taskIds) return;
let allComplete = true;
let filePath: string | undefined;
for (const taskId of taskIds) {
const task = this.pendingTasks.get(taskId);
if (!task) continue;
filePath = task.filePath;
if (
task.status !== 'completed' &&
task.status !== 'failed' &&
task.status !== 'timeout'
) {
allComplete = false;
break;
}
}
if (allComplete && filePath) {
// Clean up task tracking
for (const taskId of taskIds) {
this.pendingTasks.delete(taskId);
}
this.fileTasks.delete(fileHash);
// Clean up dependency tracking
this.fileCompletedPlugins.delete(fileHash);
// Clean up any stale dependency waiters for this file
for (const key of this.dependencyWaiters.keys()) {
if (key.startsWith(`${fileHash}:`)) {
this.dependencyWaiters.delete(key);
}
}
this.emit('file:complete', { fileHash, filePath });
}
}
/**
* Fetch existing metadata for a file
*/
private async fetchExistingMetadata(fileHash: string): Promise<Record<string, string>> {
try {
const response = await fetch(`${this.metaCoreUrl}/meta/${fileHash}`, {
method: 'GET',
signal: AbortSignal.timeout(5000),
});
if (!response.ok) {
return {};
}
const data = await response.json() as { metadata?: Record<string, string> };
return data.metadata || {};
} catch {
return {};
}
}
/**
* Get queue status
*/
getQueueStatus(): {
fast: { pending: number; running: number };
background: { pending: number; running: number };
pendingTasks: number;
pendingCallbacks: number;
} {
return {
fast: {
pending: this.fastQueue.pending,
running: this.fastQueue.pending - this.fastQueue.size,
},
background: {
pending: this.backgroundQueue.pending,
running: this.backgroundQueue.pending - this.backgroundQueue.size,
},
pendingTasks: this.pendingTasks.size,
pendingCallbacks: this.pendingCallbacks.size,
};
}
/**
* Get pending tasks for a file
*/
getPendingTasksForFile(fileHash: string): ContainerTask[] {
const taskIds = this.fileTasks.get(fileHash);
if (!taskIds) return [];
return Array.from(taskIds)
.map(id => this.pendingTasks.get(id))
.filter((t): t is ContainerTask => t !== undefined);
}
/**
* Cancel all tasks for a file
*/
cancelFile(fileHash: string): void {
const taskIds = this.fileTasks.get(fileHash);
if (!taskIds) return;
for (const taskId of taskIds) {
const pending = this.pendingCallbacks.get(taskId);
if (pending) {
clearTimeout(pending.timeoutHandle);
pending.reject(new Error('Task cancelled'));
this.pendingCallbacks.delete(taskId);
}
this.pendingTasks.delete(taskId);
}
this.fileTasks.delete(fileHash);
}
/**
* Clear all queues
*/
clear(): void {
this.fastQueue.clear();
this.backgroundQueue.clear();
for (const pending of this.pendingCallbacks.values()) {
clearTimeout(pending.timeoutHandle);
pending.reject(new Error('Scheduler cleared'));
}
this.pendingCallbacks.clear();
this.pendingTasks.clear();
this.fileTasks.clear();
this.fileCompletedPlugins.clear();
this.dependencyWaiters.clear();
}
/**
* Pause queues
*/
pause(): void {
this.fastQueue.pause();
this.backgroundQueue.pause();
}
/**
* Resume queues
*/
resume(): void {
this.fastQueue.start();
this.backgroundQueue.start();
}
/**
* Dispatch tasks to all enabled plugins for a file
* This creates and enqueues tasks for all plugins that should process this file
*/
async dispatchAllPlugins(
fileHash: string,
filePath: string,
existingMeta: Record<string, string>
): Promise<void> {
const status = this.containerManager.getStatus();
if (!status.initialized || status.pluginCount === 0) {
return; // No plugins available
}
console.log(`[ContainerScheduler] Dispatching tasks for ${fileHash} to ${status.pluginCount} plugins`);
// Get plugin info and dispatch tasks
for (const pluginStatus of status.plugins) {
if (!pluginStatus.enabled || pluginStatus.healthyInstances === 0) {
continue; // Skip disabled or unhealthy plugins
}
const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId);
const queue: TaskQueueType = manifest?.defaultQueue || 'fast';
const dependencies = manifest?.dependencies || [];
// Create and enqueue task
const task = this.createTask(
pluginStatus.pluginId,
fileHash,
filePath,
dependencies,
queue
);
// Store existing metadata in task for dispatch
(task as any).existingMeta = existingMeta;
this.enqueueTask(task);
}
}
}
/**
* Create a container plugin scheduler
*/
export function createContainerPluginScheduler(
containerManager: ContainerManager,
options?: {
fastConcurrency?: number;
backgroundConcurrency?: number;
callbackUrl?: string;
metaCoreUrl?: string;
kvClient?: IKVClient;
}
): ContainerPluginScheduler {
return new ContainerPluginScheduler(containerManager, options);
}
packages/meta-sort-core/src/container-plugins/DockerClient.ts
+417
View File
@@ -0,0 +1,417 @@
/**
* Docker Client
*
* Wrapper around the Docker Engine API for container management.
* Requires: npm install dockerode @types/dockerode
*/
import Docker from 'dockerode';
import type { Container, ContainerInfo, ContainerCreateOptions } from 'dockerode';
import type { Readable } from 'stream';
import type { ContainerSpawnOptions, ContainerResourceLimits } from './types.js';
/**
* Docker client for container operations
*/
export class DockerClient {
private docker: Docker;
private initialized: boolean = false;
constructor(socketPath: string = '/var/run/docker.sock') {
this.docker = new Docker({ socketPath });
}
/**
* Initialize and verify Docker connection
*/
async initialize(): Promise<void> {
try {
await this.docker.ping();
this.initialized = true;
} catch (error) {
throw new Error(`Failed to connect to Docker: ${error instanceof Error ? error.message : error}`);
}
}
/**
* Check if Docker is connected
*/
isInitialized(): boolean {
return this.initialized;
}
/**
* Get Docker info
*/
async getInfo(): Promise<any> {
return this.docker.info();
}
/**
* Pull a Docker image
*/
async pullImage(imageName: string, onProgress?: (event: any) => void): Promise<void> {
return new Promise((resolve, reject) => {
this.docker.pull(imageName, (err: Error | null, stream: NodeJS.ReadableStream) => {
if (err) {
reject(err);
return;
}
this.docker.modem.followProgress(
stream,
(err: Error | null) => {
if (err) {
reject(err);
} else {
resolve();
}
},
onProgress
);
});
});
}
/**
* Check if an image exists locally
*/
async imageExists(imageName: string): Promise<boolean> {
try {
await this.docker.getImage(imageName).inspect();
return true;
} catch {
return false;
}
}
/**
* Create a container
*/
async createContainer(options: ContainerSpawnOptions): Promise<string> {
// Build labels, adding Docker Compose labels if stackName is provided
const labels: Record<string, string> = {
'meta-mesh.plugin': 'true',
...options.labels,
};
// Add compose labels for Docker Desktop grouping
if (options.stackName) {
labels['com.docker.compose.project'] = options.stackName;
if (options.serviceName) {
labels['com.docker.compose.service'] = options.serviceName;
}
}
const createOptions: ContainerCreateOptions = {
Image: options.image,
name: options.name,
Env: options.env
? Object.entries(options.env).map(([k, v]) => `${k}=${v}`)
: undefined,
Labels: labels,
HostConfig: {
Binds: options.mounts?.map(m =>
`${m.source}:${m.target}:${m.readonly ? 'ro' : 'rw'}`
),
NetworkMode: options.network,
Memory: options.resources?.memory
? parseMemoryLimit(options.resources.memory)
: undefined,
NanoCpus: options.resources?.cpus
? Math.floor(options.resources.cpus * 1e9)
: undefined,
RestartPolicy: {
Name: 'unless-stopped',
},
},
ExposedPorts: {
'8080/tcp': {},
},
};
const container = await this.docker.createContainer(createOptions);
return container.id;
}
/**
* Start a container
*/
async startContainer(containerId: string): Promise<void> {
const container = this.docker.getContainer(containerId);
await container.start();
}
/**
* Stop a container
*/
async stopContainer(containerId: string, timeout: number = 10): Promise<void> {
const container = this.docker.getContainer(containerId);
await container.stop({ t: timeout });
}
/**
* Remove a container
*/
async removeContainer(containerId: string, force: boolean = false): Promise<void> {
const container = this.docker.getContainer(containerId);
await container.remove({ force });
}
/**
* Get container info
*/
async inspectContainer(containerId: string): Promise<Docker.ContainerInspectInfo> {
const container = this.docker.getContainer(containerId);
return container.inspect();
}
/**
* Check if a container exists
*/
async containerExists(containerId: string): Promise<boolean> {
try {
await this.inspectContainer(containerId);
return true;
} catch {
return false;
}
}
/**
* Check if a container is running
*/
async isContainerRunning(containerId: string): Promise<boolean> {
try {
const info = await this.inspectContainer(containerId);
return info.State?.Running === true;
} catch {
return false;
}
}
/**
* Get container IP address on a specific network
*/
async getContainerIP(containerId: string, network: string): Promise<string | null> {
try {
const info = await this.inspectContainer(containerId);
const networkSettings = info.NetworkSettings?.Networks?.[network];
return networkSettings?.IPAddress || null;
} catch {
return null;
}
}
/**
* List containers with optional filters
*/
async listContainers(filters?: {
label?: string[];
name?: string[];
status?: string[];
}): Promise<ContainerInfo[]> {
const dockerFilters: Record<string, string[]> = {};
if (filters?.label) {
dockerFilters.label = filters.label;
}
if (filters?.name) {
dockerFilters.name = filters.name;
}
if (filters?.status) {
dockerFilters.status = filters.status;
}
return this.docker.listContainers({
all: true,
filters: Object.keys(dockerFilters).length > 0 ? dockerFilters : undefined,
});
}
/**
* List plugin containers (created by meta-sort)
*/
async listPluginContainers(): Promise<ContainerInfo[]> {
return this.listContainers({
label: ['meta-mesh.plugin=true'],
});
}
/**
* Get container logs
*/
async getContainerLogs(
containerId: string,
options?: {
tail?: number;
since?: number;
stdout?: boolean;
stderr?: boolean;
}
): Promise<string> {
const container = this.docker.getContainer(containerId);
const logs = await container.logs({
stdout: options?.stdout ?? true,
stderr: options?.stderr ?? true,
tail: options?.tail ?? 100,
since: options?.since,
});
// Convert buffer to string
if (Buffer.isBuffer(logs)) {
return logs.toString('utf-8');
}
return logs as unknown as string;
}
/**
* Stream container logs
*/
async streamContainerLogs(
containerId: string,
onLog: (log: string) => void,
options?: {
stdout?: boolean;
stderr?: boolean;
}
): Promise<() => void> {
const container = this.docker.getContainer(containerId);
const stream = await container.logs({
stdout: options?.stdout ?? true,
stderr: options?.stderr ?? true,
follow: true,
tail: 0,
});
const readable = stream as Readable;
readable.on('data', (chunk: Buffer) => {
onLog(chunk.toString('utf-8'));
});
return () => {
readable.destroy();
};
}
/**
* Check if a network exists
*/
async networkExists(networkName: string): Promise<boolean> {
try {
const networks = await this.docker.listNetworks({
filters: { name: [networkName] },
});
return networks.some(n => n.Name === networkName);
} catch {
return false;
}
}
/**
* Create a network if it doesn't exist
*/
async ensureNetwork(networkName: string): Promise<void> {
if (await this.networkExists(networkName)) {
return;
}
await this.docker.createNetwork({
Name: networkName,
Driver: 'bridge',
Labels: {
'meta-mesh': 'true',
},
});
}
/**
* Connect container to network
*/
async connectToNetwork(containerId: string, networkName: string): Promise<void> {
const network = this.docker.getNetwork(networkName);
await network.connect({ Container: containerId });
}
/**
* Disconnect container from network
*/
async disconnectFromNetwork(containerId: string, networkName: string): Promise<void> {
const network = this.docker.getNetwork(networkName);
await network.disconnect({ Container: containerId });
}
/**
* Execute a command in a container
*/
async exec(
containerId: string,
cmd: string[],
options?: { workingDir?: string }
): Promise<{ exitCode: number; output: string }> {
const container = this.docker.getContainer(containerId);
const exec = await container.exec({
Cmd: cmd,
AttachStdout: true,
AttachStderr: true,
WorkingDir: options?.workingDir,
});
const stream = await exec.start({ Detach: false });
return new Promise((resolve, reject) => {
let output = '';
stream.on('data', (chunk: Buffer) => {
output += chunk.toString('utf-8');
});
stream.on('end', async () => {
try {
const info = await exec.inspect();
resolve({
exitCode: info.ExitCode ?? -1,
output,
});
} catch (err) {
reject(err);
}
});
stream.on('error', reject);
});
}
}
/**
* Parse memory limit string to bytes
* Supports: '512m', '1g', '1024k'
*/
function parseMemoryLimit(limit: string): number {
const match = limit.toLowerCase().match(/^(\d+)([kmg]?)$/);
if (!match) {
throw new Error(`Invalid memory limit format: ${limit}`);
}
const value = parseInt(match[1], 10);
const unit = match[2] || '';
switch (unit) {
case 'k':
return value * 1024;
case 'm':
return value * 1024 * 1024;
case 'g':
return value * 1024 * 1024 * 1024;
default:
return value;
}
}
/**
* Create a new Docker client instance
*/
export function createDockerClient(socketPath?: string): DockerClient {
return new DockerClient(socketPath);
}
packages/meta-sort-core/src/container-plugins/index.ts
+34
View File
@@ -0,0 +1,34 @@
/**
* Container Plugins Module
*
* Provides support for running plugins as Docker containers.
*/
// Types
export * from './types.js';
// Configuration
export {
loadConfig,
parseConfig,
getEnabledPlugins,
getDefaultConfigPath,
ConfigParseError,
} from './ConfigParser.js';
// Docker Client
export { DockerClient, createDockerClient } from './DockerClient.js';
// Container Manager
export {
ContainerManager,
createContainerManager,
type ContainerManagerEvents,
} from './ContainerManager.js';
// Container Plugin Scheduler
export {
ContainerPluginScheduler,
createContainerPluginScheduler,
type ContainerSchedulerEvents,
} from './ContainerPluginScheduler.js';
packages/meta-sort-core/src/container-plugins/types.ts
+364
View File
@@ -0,0 +1,364 @@
/**
* Container Plugin System Type Definitions
*
* Types for containerized plugins that run in Docker containers
* and communicate via HTTP APIs.
*/
import type { ConfigField, SchemaField, TaskQueueType } from '../plugin-engine/types.js';
// =============================================================================
// Plugin Configuration (stored in plugins.yml)
// =============================================================================
/**
* Resource limits for a container plugin
*/
export interface ContainerResourceLimits {
/** Memory limit (e.g., '512m', '2g') */
memory?: string;
/** CPU limit (e.g., 1.0, 2.5) */
cpus?: number;
}
/**
* Volume mount configuration
*/
export interface ContainerMount {
/** Source path (relative to FILES_PATH) */
source: string;
/** Target path inside container */
target: string;
/** Mount as read-only (default: true) */
readonly?: boolean;
}
/**
* Health check configuration
*/
export interface ContainerHealthCheck {
/** Health check interval (e.g., '30s', '1m') */
interval?: string;
/** Health check timeout (e.g., '5s') */
timeout?: string;
/** Number of retries before marking unhealthy */
retries?: number;
}
/**
* Single container plugin configuration (from plugins.yml)
*/
export interface ContainerPluginConfig {
/** Whether this plugin is enabled */
enabled?: boolean;
/** Docker image name and tag */
image: string;
/** Number of container instances to spawn (default: 1) */
instances?: number;
/** Resource limits */
resources?: ContainerResourceLimits;
/** Plugin configuration values (sent to /configure endpoint) */
config?: Record<string, unknown>;
/** Whether plugin needs external network access (default: false) */
network?: boolean;
/** Additional volume mounts */
mounts?: ContainerMount[];
/** Health check settings */
healthCheck?: ContainerHealthCheck;
/** Default queue override (normally from manifest) */
defaultQueue?: TaskQueueType;
}
/**
* Root configuration file structure (plugins.yml)
*/
export interface ContainerPluginsConfig {
/** Config version */
version?: string;
/** Plugin configurations keyed by plugin ID */
plugins: Record<string, ContainerPluginConfig>;
}
// =============================================================================
// Plugin Manifest (returned by plugin container via GET /manifest)
// =============================================================================
/**
* File filter criteria for plugin
*/
export interface PluginFileFilter {
/** MIME type patterns (e.g., 'video/*', 'image/jpeg') */
mimeTypes?: string[];
/** File extension patterns (e.g., 'mkv', 'mp4') */
extensions?: string[];
/** Minimum file size in bytes */
minSize?: number;
/** Maximum file size in bytes */
maxSize?: number;
}
/**
* UI display hints
*/
export interface PluginUIConfig {
/** Color for UI display (e.g., '#4CAF50') */
color?: string;
/** Icon name (optional) */
icon?: string;
}
/**
* Container plugin manifest (from GET /manifest)
*/
export interface ContainerPluginManifest {
/** Unique plugin identifier */
id: string;
/** Human-readable name */
name: string;
/** Semantic version */
version: string;
/** Description */
description?: string;
/** Author or organization */
author?: string;
/** Source repository URL */
repository?: string;
/** Execution priority (lower = runs earlier, 10-100 range) */
priority?: number;
/** Default queue assignment */
defaultQueue?: TaskQueueType;
/** Max processing time in milliseconds */
timeout?: number;
/** Plugin IDs that must complete before this one */
dependencies?: string[];
/** File filter criteria */
filter?: PluginFileFilter;
/** Configuration schema (aliased as config for compatibility) */
config?: Record<string, ConfigField>;
/** Metadata output schema (aliased as schema for compatibility) */
schema?: Record<string, SchemaField>;
/** UI display color */
color?: string;
/** UI display hints */
ui?: PluginUIConfig;
}
// =============================================================================
// Container Plugin Instance (runtime state)
// =============================================================================
/**
* Container instance status
*/
export type ContainerInstanceStatus = 'starting' | 'healthy' | 'unhealthy' | 'stopped' | 'error';
/**
* Runtime state of a container plugin instance
*/
export interface ContainerPluginInstance {
/** Plugin ID */
pluginId: string;
/** Docker container ID */
containerId: string;
/** Container name (e.g., 'meta-plugin-ffmpeg-0') */
containerName: string;
/** HTTP base URL (e.g., 'http://meta-plugin-ffmpeg-0:8080') */
baseUrl: string;
/** Instance index (0, 1, 2, ...) */
instanceIndex: number;
/** Current status */
status: ContainerInstanceStatus;
/** Cached manifest from container */
manifest?: ContainerPluginManifest;
/** Last successful health check timestamp */
lastHealthCheck?: number;
/** Last error message */
lastError?: string;
/** Number of tasks processed */
tasksProcessed: number;
/** Number of failed tasks */
tasksFailed: number;
}
// =============================================================================
// Plugin HTTP API Types
// =============================================================================
/**
* Health check response from GET /health
*/
export interface PluginHealthResponse {
status: 'healthy' | 'unhealthy';
ready: boolean;
version: string;
message?: string;
}
/**
* Configure request body for POST /configure
*/
export interface PluginConfigureRequest {
config: Record<string, unknown>;
}
/**
* Configure response from POST /configure
*/
export interface PluginConfigureResponse {
success: boolean;
error?: string;
}
/**
* Process request body for POST /process
*/
export interface PluginProcessRequest {
/** Unique task identifier */
taskId: string;
/** Content identifier (midhash256) */
cid: string;
/** Absolute path to file in container */
filePath: string;
/** Callback URL for completion notification */
callbackUrl: string;
/** meta-core API base URL */
metaCoreUrl: string;
/** Current metadata (flat key-value map) */
existingMeta: Record<string, string>;
}
/**
* Process response from POST /process (immediate)
*/
export interface PluginProcessResponse {
status: 'accepted' | 'rejected';
taskId: string;
error?: string;
}
/**
* Callback status
*/
export type CallbackStatus = 'completed' | 'failed' | 'skipped';
/**
* Callback payload sent by plugin to meta-sort
*/
export interface PluginCallbackPayload {
/** Task identifier */
taskId: string;
/** Plugin ID */
pluginId: string;
/** Content identifier */
cid: string;
/** Task status */
status: CallbackStatus;
/** Processing duration in milliseconds */
duration: number;
/** Error message (if status is 'failed') */
error?: string;
/** Skip reason (if status is 'skipped') */
reason?: string;
}
// =============================================================================
// Container Manager Types
// =============================================================================
/**
* Options for spawning a container
*/
export interface ContainerSpawnOptions {
/** Docker image */
image: string;
/** Container name */
name: string;
/** Environment variables */
env?: Record<string, string>;
/** Resource limits */
resources?: ContainerResourceLimits;
/** Volume mounts */
mounts?: Array<{
source: string;
target: string;
readonly?: boolean;
}>;
/** Docker network to join */
network?: string;
/** Labels for the container */
labels?: Record<string, string>;
/** Stack name for grouping containers in Docker Desktop (adds compose labels) */
stackName?: string;
/** Service name for compose grouping (usually the plugin ID) */
serviceName?: string;
}
/**
* Container manager status
*/
export interface ContainerManagerStatus {
/** Whether manager is initialized */
initialized: boolean;
/** Total number of plugin types */
pluginCount: number;
/** Total running containers */
runningContainers: number;
/** Healthy containers */
healthyContainers: number;
/** Per-plugin status */
plugins: Array<{
pluginId: string;
enabled: boolean;
image: string;
instances: number;
healthyInstances: number;
manifest?: ContainerPluginManifest;
}>;
}
// =============================================================================
// Container Task Types
// =============================================================================
/**
* Container task status
*/
export type ContainerTaskStatus = 'pending' | 'waiting' | 'dispatched' | 'processing' | 'completed' | 'failed' | 'timeout';
/**
* Container plugin task
*/
export interface ContainerTask {
/** Unique task ID (UUID) */
id: string;
/** File hash (midhash256) */
fileHash: string;
/** File path */
filePath: string;
/** Target plugin ID */
pluginId: string;
/** Plugin dependencies */
dependencies: string[];
/** Task queue */
queue: TaskQueueType;
/** Task status */
status: ContainerTaskStatus;
/** Timestamp when created */
createdAt: number;
/** Timestamp when dispatched to container */
dispatchedAt?: number;
/** Timestamp when completed */
completedAt?: number;
/** Container instance that processed this task */
instanceName?: string;
/** Processing duration in ms */
duration?: number;
/** Error message if failed */
error?: string;
}
packages/meta-sort-core/src/index.ts
+308
View File
@@ -0,0 +1,308 @@
import {FolderWatcher, PollingWatcher} from "@metazla/meta-hash";
import {config} from "./config/EnvConfig.js";
import os from 'os';
import {existsSync} from 'fs';
import {WatchedFileProcessor} from "./logic/WatchedFileProcessor.js";
import {UnifiedAPIServer} from "./api/UnifiedAPIServer.js";
import {StreamingPipeline} from "./logic/pipeline/StreamingPipeline.js";
import {SUPPORTED_EXTENSIONS} from "./config/SupportedFileTypes.js";
import {performanceMetrics} from "./metrics/PerformanceMetrics.js";
import {KVManager} from "./kv/KVManager.js";
import {getPluginManagerInstance, initializePluginManager} from "./logic/fileProcessor/FileProcessorPiscina.js";
import {ContainerManager} from "./container-plugins/ContainerManager.js";
import {ContainerPluginScheduler} from "./container-plugins/ContainerPluginScheduler.js";
try {
// Get current user information
const userInfo = os.userInfo();
// Display the current user and UID
console.log(`[node] Current User: ${userInfo.username}`);
console.log(`[node] User ID (UID): ${userInfo.uid}`);
}catch (error) {
/* ignore */
}
console.log(`BUILD_VERSION: ${process.env.BUILD_VERSION || 'dev'}`);
//display the config
console.log(config);
process.umask(0);//This unable the process to create files with 777 permissions
let fileProcessor = new WatchedFileProcessor();
let folders = config.WATCH_FOLDER_LIST;
// KV Manager for Redis-based storage with leader election (optional)
let kvManager: KVManager | null = null;
// Container Manager for containerized plugins
let containerManager: ContainerManager | null = null;
let containerPluginScheduler: ContainerPluginScheduler | null = null;
// Calculate concurrency for pipeline stages
const cpuCount = os.cpus().length;
// Smart defaults optimized for network storage + HDD scenarios:
// - Fast queue: CPU (fast operations: filename parsing, stats)
// - Background queue: CPU / 2 (slow operations: SHA-256, FFmpeg)
const defaultBackgroundWorkers = Math.max(1, Math.floor(cpuCount / 2));
const validationConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS * 2 : cpuCount * 2;
const fastQueueConcurrency = config.MAX_WORKER_THREADS ? config.MAX_WORKER_THREADS : cpuCount;
const backgroundQueueConcurrency = config.MAX_WORKER_THREADS || defaultBackgroundWorkers;
console.log(`Pipeline concurrency: validation=${validationConcurrency}, fastQueue=${fastQueueConcurrency}, backgroundQueue=${backgroundQueueConcurrency} (cpuCount=${cpuCount}, defaultBackgroundWorkers=${defaultBackgroundWorkers})`);
// Create streaming pipeline
const pipeline = new StreamingPipeline({
supportedExtensions: SUPPORTED_EXTENSIONS,
validationConcurrency,
fastQueueConcurrency,
backgroundQueueConcurrency,
strictMimeValidation: false, // Skip MIME for speed
fileProcessor: fileProcessor.fileProcessor,
stateManager: fileProcessor.unifiedStateManager,
kvClient: fileProcessor.getKVClient(),
virtualFileSystem: fileProcessor.getVirtualFileSystem(),
metaDataToFolderStruct: fileProcessor.metaDataToFolderStruct
});
// Connect pipeline to fileProcessor for queue status monitoring
fileProcessor.setPipeline(pipeline);
// Create generic file discovery
const folderWatcher = new FolderWatcher();
// Parse folder list
const folderList = folders.split(',').map(f => f.trim()).filter(f => f.length > 0);
console.log(`Watching folders: ${folderList.join(', ')}`);
// VFS and state manager are available immediately
const vfs = fileProcessor.getVirtualFileSystem();
const unifiedStateManager = fileProcessor.getUnifiedStateManager();
const getDuplicateResult = () => fileProcessor.getDuplicateResult();
// API Server will be initialized after KV is ready (deferred to async init)
let apiServer: UnifiedAPIServer | null = null;
// Shared progressive discovery and cleanup logic (used by both startup and manual scan)
const runProgressiveDiscovery = async () => {
console.log(`[Discovery] Progressive scan mode: ${folderList.length} folders`);
// Process folders one at a time for progressive cleanup
for (const folder of folderList) {
console.log(`[Discovery] Scanning folder: ${folder}`);
// Track discovered files for this folder
const discoveredFilesInFolder = new Set<string>();
// Wrap discovery stream to track files
async function* trackDiscoveredFiles(stream: AsyncGenerator<string>) {
for await (const filePath of stream) {
discoveredFilesInFolder.add(filePath);
yield filePath;
}
}
const discoveryStream = folderWatcher.discoverFiles([folder]);
const trackedStream = trackDiscoveredFiles(discoveryStream);
await pipeline.start(trackedStream);
// Progressive cleanup: clean up after each folder completes
console.log(`[Discovery] Folder scan complete: ${folder}. Found ${discoveredFilesInFolder.size} files. Running cleanup...`);
await fileProcessor.cleanupStaleEntries([folder], discoveredFilesInFolder);
console.log(`[Discovery] Cleanup complete for folder: ${folder}`);
}
console.log(`[Discovery] All folders scanned and cleaned up`);
};
// Manual scan trigger (called from API)
// Note: Stremio updates happen automatically via incremental add/remove notifications
// during file processing (like FUSE), so no explicit refresh needed here
const triggerScan = async () => {
console.log('Triggering manual scan from API...');
await runProgressiveDiscovery();
};
// Global error handlers - prevent crashes from unforeseen errors
process.on('uncaughtException', (error: Error) => {
console.error('❌ [CRITICAL] Uncaught Exception:', error);
console.error('Stack:', error.stack);
// Log but don't exit - allow process to continue
// Monitor for worker health issues
});
process.on('unhandledRejection', (reason: any, promise: Promise<any>) => {
console.error('❌ [CRITICAL] Unhandled Rejection at:', promise);
console.error('Reason:', reason);
// Log but don't exit - allow process to continue
});
// Graceful shutdown
process.on('SIGINT', async () => {
console.log('\nShutting down...');
if (apiServer) {
await apiServer.stop();
}
// Stop Container Manager if running
if (containerManager) {
console.log('[Shutdown] Stopping Container Manager...');
await containerManager.shutdown();
}
// Stop KV Manager if running
if (kvManager) {
console.log('[Shutdown] Stopping KV Manager...');
await kvManager.stop();
}
process.exit(0);
});
// Start initial discovery and processing (streaming pipeline)
(async () => {
try {
// Initialize KV Manager if configured (Redis + leader election)
if (config.META_CORE_PATH) {
console.log('[Startup] Initializing KV Manager (Redis + leader election)...');
kvManager = new KVManager({
metaCorePath: config.META_CORE_PATH,
filesPath: config.FILES_PATH,
serviceName: config.SERVICE_NAME,
version: config.SERVICE_VERSION,
apiPort: config.FUSE_API_PORT,
redisPort: config.REDIS_PORT,
redisUrl: config.REDIS_URL,
baseUrl: config.BASE_URL,
capabilities: ['write', 'mount', 'monitor']
});
// Wait for KV to be ready
await kvManager.start();
await kvManager.waitForReady();
// Pass KV client to file processor
const client = kvManager.getClient();
if (client) {
fileProcessor.setKVClient(client);
console.log(`[Startup] KV Manager ready (role: ${kvManager.isLeader() ? 'LEADER' : 'FOLLOWER'})`);
}
}
// Start Unified API Server AFTER KV is ready (so kvClient is available)
const kvClient = fileProcessor.getKVClient();
apiServer = new UnifiedAPIServer(vfs, {
port: config.FUSE_API_PORT,
host: config.FUSE_API_HOST,
enableCors: true,
}, unifiedStateManager, getDuplicateResult, kvClient || undefined, triggerScan, backgroundQueueConcurrency, fastQueueConcurrency, () => fileProcessor.getQueueStatus(), kvManager || undefined, getPluginManagerInstance, () => fileProcessor.getTaskScheduler());
await apiServer.start();
console.log('Unified API Server started successfully');
// Initialize plugin manager early so it's available for API
console.log('[Startup] Initializing plugin manager...');
const pluginManager = await initializePluginManager();
console.log('[Startup] Plugin manager initialized');
// Initialize container plugins if config file exists
if (existsSync(config.CONTAINER_PLUGINS_CONFIG)) {
console.log('[Startup] Initializing container plugins...');
try {
containerManager = new ContainerManager();
await containerManager.initialize();
// Connect container manager to plugin manager
pluginManager.setContainerManager(containerManager);
await pluginManager.loadContainerPlugins();
// Create container plugin scheduler
containerPluginScheduler = new ContainerPluginScheduler(containerManager, {
fastConcurrency: config.FAST_QUEUE_CONCURRENCY,
backgroundConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY,
kvClient: kvClient || undefined,
});
// Connect to API server (for callback handling)
apiServer.setContainerPluginManager(containerManager, containerPluginScheduler);
console.log('[Startup] ContainerPluginScheduler connected to API server');
const status = containerManager.getStatus();
console.log(`[Startup] Container plugins initialized: ${status.pluginCount} plugins, ${status.healthyContainers}/${status.runningContainers} healthy`);
} catch (error) {
console.error('[Startup] Failed to initialize container plugins:', error);
console.log('[Startup] Continuing without container plugins...');
}
} else {
console.log(`[Startup] Container plugins config not found at ${config.CONTAINER_PLUGINS_CONFIG}, skipping`);
}
// Initialize hash index cache (loads existing hashes from disk)
// This also initializes the TaskScheduler
console.log('[Startup] Initializing hash index cache...');
await fileProcessor.initialize();
console.log('[Startup] Hash index cache initialized');
// Connect ContainerPluginScheduler to TaskScheduler and Pipeline
if (containerPluginScheduler) {
const taskScheduler = fileProcessor.getTaskScheduler();
if (taskScheduler) {
taskScheduler.setContainerPluginScheduler(containerPluginScheduler);
console.log('[Startup] ContainerPluginScheduler connected to TaskScheduler');
}
// Connect to StreamingPipeline for file processing dispatch
pipeline.setContainerPluginScheduler(containerPluginScheduler);
console.log('[Startup] ContainerPluginScheduler connected to StreamingPipeline');
}
// First, rebuild VFS from KV to restore state from previous runs
console.log('[Startup] Rebuilding VirtualFileSystem from KV...');
await fileProcessor.rebuildVFSFromKV();
console.log('[Startup] VFS rebuild completed');
console.log('[Startup] Starting initial file discovery with progressive cleanup...');
await runProgressiveDiscovery();
console.log('[Startup] Initial discovery completed, processing continues in background');
// Log cache performance summary
const metrics = performanceMetrics.getMetrics();
if (metrics.cacheHits.midhash256 || metrics.cacheMisses.midhash256) {
const hits = metrics.cacheHits.midhash256 || 0;
const misses = metrics.cacheMisses.midhash256 || 0;
const total = hits + misses;
const hitRate = total > 0 ? ((hits / total) * 100).toFixed(1) : '0.0';
console.log(`[Cache] midhash256 cache: ${hits} hits, ${misses} misses (${hitRate}% hit rate)`);
}
// Setup file watching for changes
console.log('[Startup] Setting up file watching...');
folderWatcher.watch(folderList, {
onAdd: (filePath) => pipeline.handleFileAdded(filePath),
onChange: (filePath) => pipeline.handleFileChanged(filePath),
onUnlink: (filePath) => pipeline.handleFileDeleted(filePath)
}, {
interval: config.WATCH_FOLDER_POOLING_INTERVAL_MS || 30000,
stabilityThreshold: 30000,
pollInterval: 5000
});
console.log('[Startup] File watching active');
} catch (error) {
console.error('[Startup] Error during initial discovery:', error);
}
})();
// Start PollingWatcher for path-specific polling (if configured)
if (config.POLLING_PATHS && config.POLLING_PATHS.length > 0) {
console.log(`PollingWatcher: ${config.POLLING_PATHS.length} paths configured`);
const pollingWatcher = new PollingWatcher(fileProcessor, config.POLLING_PATHS);
pollingWatcher.start().then(() => {
console.log('PollingWatcher started successfully');
}).catch((error) => {
console.error('Error starting PollingWatcher:', error);
});
} else {
console.log('PollingWatcher: No polling paths configured (skipped)');
}
packages/meta-sort-core/src/jellyfin/JellyfinAPI.ts
+97
View File
@@ -0,0 +1,97 @@
import {config} from "../config/EnvConfig.js";
import axios from "axios";
export class JellyfinAPI {
constructor() {
}
async getMetaData(filepath: string): Promise<any> {
console.log("geteMetaData");
}
private async sendAuthedRequest(url: string, method: string, data?: any, params?: any) {
return (await axios({
method: method,
url: url,
headers: {
'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
},
data: data,
params: params
})).data;
}
private async getRootUserId(): Promise<string> {
const users = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Users`, 'GET');
for (const user of users) {
if (user.Policy.IsAdministrator) {
return user.Id;
}
}
return users[0].Id;
}
private async getItemIdByFilePath(filePath: string): Promise<string | null> {
try {
let user = await this.getRootUserId();
const items = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Items`, 'GET', null, {
Path: encodeURIComponent(filePath),
userId: user
});
if (items.Items && items.Items.length > 0) {
return items.Items[0].Id;
} else {
console.log(`Item with file path "${filePath}" not found.`);
return null;
}
} catch (error) {
console.error('Error fetching item ID by file path:', error.message);
return null;
}
}
jellyApiAvailable(): boolean {
return !!config.JELLYFIN_API_KEY && !!config.JELLYFIN_ENDPOINT;
}
/** takes around 30s to complete (if light update) */
async refreshLibrary(): Promise<void>{
try {
const response = await this.sendAuthedRequest(`${config.JELLYFIN_ENDPOINT}/Library/Refresh`, "POST",null, {
headers: {
'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
}
});
} catch (error) {
console.error('Error refreshing library:', error.message);
}
}
async notifyJellyfin(filePath): Promise<void> {
const itemId = await this.getItemIdByFilePath(filePath);
if(!itemId) {
return;
}
// Fetch the public system info
const response = await axios.post(`${config.JELLYFIN_ENDPOINT}/Items/${itemId}/Refresh`, {
},{
headers: {
'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
}
});
console.log(response.data);
}
async systemEndpoint() {
const response = await axios.get(`${config.JELLYFIN_ENDPOINT}/System/Endpoint`, {
headers: {
'X-MediaBrowser-Token': config.JELLYFIN_API_KEY
}
});
console.log(response.data);
}
}
packages/meta-sort-core/src/jellyfin/JellyfinNfo.ts
+278
View File
@@ -0,0 +1,278 @@
import {readFile, writeFile} from 'fs/promises';
import {Builder, Parser} from 'xml2js';
import {mergeMetadata} from "../logic/tool/MetadataMerger.js";
import {existsAsync} from "@metazla/meta-hash";
import {VideoMeta, JellyfinMeta, FileStatMeta} from "@metazla/meta-interface";
import lodash from 'lodash';
import * as xml2js from "xml2js";
export class JellyfinNfo {
options = {explicitArray: true, ignoreAttrs: true};
/**
* Convert a JSON XML object back to a regular JSON object where arrays containing `set` are converted back to Set objects
* @param input - The input JSON XML object
*/
private convertFromJsonXml(input: any): any {
const output: any = {};
for (const [key, value] of Object.entries(input)) {
if (!Array.isArray(value) || value.length === 0) {
throw new Error("Unexpected format: each property should be an array.");
}
const firstElement = value[0];
if (firstElement && typeof firstElement === 'object') {
if ('set' in firstElement) {
// Convert array from 'set' back to Set object
output[key] = new Set(firstElement.set);
} else {
// Recursively convert from JsonXml for nested objects
output[key] = this.convertFromJsonXml(firstElement);
}
} else {
// Primitive value, simply assign it as it is the only element in the array
output[key] = firstElement;
}
}
return output;
}
/**
* Filter only known jellyfin supported data
* @param data
*/
filterData(data: any): any {
const supportedFields = ["originalTitle", "season", "episode", "videoType", "fileType"]
const result = {};
for (const key of supportedFields) {
if (data[key]) {
result[key] = data[key];
}
}
return result;
}
async read<T>(nfoPath: string): Promise<T> {
if (!nfoPath.endsWith('.nfo')) {
throw new Error(`NFO file path must end with '.nfo'`);
}
let metadata: T = {} as T;
if (await existsAsync(nfoPath)) {
try {
const nfoContent = await readFile(nfoPath, {encoding: 'utf8'});
const parser = new xml2js.Parser(this.options);
const result = await parser.parseStringPromise(nfoContent);
metadata = result.root || result.episodedetails || result.movie || {};
metadata = this.convertFromJsonXml(metadata);
} catch (error) {
console.error(`Error parsing NFO file ${nfoPath}`);
}
}
// we always read the file for meta data but ignore its content except for cid which are long to compute
return metadata;
}
async update<T extends FileStatMeta>(nfoPath: string, data: T): Promise<void> {
data = this.filterData(data);
if (!nfoPath.endsWith('.nfo')) {
throw new Error(`NFO file path must end with '.nfo'`);
}
if (data.fileType !== "video") {
return;//only process video files for writing jellyfin nfo
}
if (await existsAsync(nfoPath)) {
await this.updateNfoFile(nfoPath, data as any);
} else {
throw new Error(`NFO file does not exist at path: ${nfoPath}`);
}
}
public async createOrUpdate<T extends FileStatMeta>(nfoPath: string, data: T): Promise<void> {
data = this.filterData(data);
if (!nfoPath.endsWith('.nfo')) {
throw new Error(`NFO file path must end with '.nfo'`);
}
if (data.fileType !== "video") {
return;//only process video files for writing jellyfin nfo
}
if (await existsAsync(nfoPath)) {
await this.updateNfoFile(nfoPath, data as any);
} else {
// If the NFO file doesn't exist, just generate a new one
await this.generateNfoFile(nfoPath, data as any);
}
}
/**
* Convert a regular JSON object to an XML JSON object (add arrays)
* @param input
*/
private convertToJsonXml(input: any): any {
const output: any = {};
// Use Object.entries to loop through each key-value pair in the input object
for (const [key, value] of Object.entries(input)) {
if (!this.assertLegalName(key)) {
throw new Error(`Illegal character in key: ${key}`);
}
if (value instanceof Set) {
// If the value is a Set, convert it to an array and then to XmlJson format
const array = Array.from(value as any);
if (array.length > 0) {
output[key] = [{set: array}];
}
} else if (value instanceof Array) {
throw new Error("unsupported");
} else if (typeof value === 'object' && value !== null) {
// For nested objects, recursively convert
let jsonXml = this.convertToJsonXml(value);
if (jsonXml instanceof Array) {
output[key] = jsonXml;
} else {
output[key] = [jsonXml];
}
} else {
output[key] = [value];
}
}
return output;
}
private async updateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
try {
const existingContent = await readFile(nfoFilePath, {encoding: "utf-8"});
const parser = new Parser(this.options);
let existingData = await parser.parseStringPromise(existingContent);
let hasChanged = false;
// 1st try to update the type of the metadata (root can be "root" or "movie" or "episodedetails") and
// according to the 'jelly' parameter and the isMovieOrTvShow function
const targetRoot = (this.isMovieOrTvShow(data) === "tvshow" ? 'episodedetails' : "movie");
if (existingData.root) {
existingData[targetRoot] = existingData.root;
delete existingData.root;
hasChanged = true;
} else if (existingData.movie && "movie" !== targetRoot) {
existingData[targetRoot] = existingData.movie;
delete existingData.movie;
hasChanged = true;
} else if (existingData.episodedetails && "episodedetails" !== targetRoot) {
existingData[targetRoot] = existingData.episodedetails;
delete existingData.episodedetails;
hasChanged = true;
}
//2nd update the data
let root = existingData[targetRoot];
if (typeof root !== 'object') {
root = {};//don't allow primitive type in root
hasChanged = true;
}
const convertedData = this.convertToJsonXml(data);
hasChanged = mergeMetadata(root, convertedData, nfoFilePath) || hasChanged;
root = this.sanitizeData(root);
existingData[targetRoot] = root;
if (hasChanged) {
//const builder = new Builder({ headless: true, renderOpts: { 'pretty': true } });
const builder = new Builder();
// Convert the updated object back to XML
try {
const updatedContent = builder.buildObject(existingData);
await writeFile(nfoFilePath, updatedContent, {encoding: "utf-8"});
} catch (error) {
console.error(`Failed to write updated NFO content to ${nfoFilePath} : ${error}`);
}
}
} catch (error) {
console.error(`Failed to parse existing NFO content ${nfoFilePath} : ${error}`);
return;
}
}
private isMovieOrTvShow(data: VideoMeta): "tvshow" | "movie" {
if (data.videoType) {
return data.videoType;
} else if (data.episode) {
return "tvshow";
} else {
return "movie";
}
}
private assertLegalChar(str): boolean {
return str.match(/[\0\uFFFE\uFFFF]|[\uD800-\uDBFF](?![\uDC00-\uDFFF])|(?:[^\uD800-\uDBFF]|^)[\uDC00-\uDFFF]/);
};
private assertLegalName(str: string) {
if (this.assertLegalChar(str)) {
return false;
}
const regex = /^([:A-Z_a-z\xC0-\xD6\xD8-\xF6\xF8-\u02FF\u0370-\u037D\u037F-\u1FFF\u200C\u200D\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])([\x2D\.0-:A-Z_a-z\xB7\xC0-\xD6\xD8-\xF6\xF8-\u037D\u037F-\u1FFF\u200C\u200D\u203F\u2040\u2070-\u218F\u2C00-\u2FEF\u3001-\uD7FF\uF900-\uFDCF\uFDF0-\uFFFD]|[\uD800-\uDB7F][\uDC00-\uDFFF])*$/;
if (!str.match(regex)) {
return false;
}
return true;
};
private sanitizeData(data: any): any {
const sanitize = (value: any): any => {
if (typeof value === 'string') {
// Escaping the string values
return lodash.escape(value);
} else if (Array.isArray(value)) {
throw new Error(`Arrays are not supported in NFO files`);
} else if (typeof value === 'object' && value !== null) {
// Recursively sanitize each property in the object
let result = {};
for (const key of Object.keys(value)) {
if (this.assertLegalName(key)) {
result[key] = value[key];
}
}
return result;
}
return value;
};
return sanitize(data);
}
private async generateNfoFile(nfoFilePath: string, data: JellyfinMeta) {
if (!nfoFilePath.endsWith('.nfo')) {
throw new Error(`NFO file path must end with '.nfo'`);
}
if (await existsAsync(nfoFilePath)) {
throw new Error(`NFO file already exists at path: ${nfoFilePath}`);
}
const rootElementName = data.videoType === "tvshow" ? 'episodedetails' : "movie";
/*const schemaAttributes = jelly ? {} : {
'xmlns:xsi': 'http://www.w3.org/2001/XMLSchema-instance',
'xsi:schemaLocation': 'http://www.example.com/schema http://www.example.com/schema/mediaFileSchema.xsd'
};*/
const schemaAttributes = {};
const convertedData = this.convertToJsonXml(data);
const sanitizedData = this.sanitizeData(convertedData);
// Prepare the data object with the correct root element and attributes
const obj = {
[rootElementName]: {
...({$: schemaAttributes}),
...sanitizedData
}
};
const builder = new Builder({xmldec: {version: '1.0', encoding: 'UTF-8', standalone: true}});
const nfoContent = builder.buildObject(obj);
await writeFile(nfoFilePath, nfoContent, {encoding: "utf-8"});
}
}
packages/meta-sort-core/src/jellyfin/README
+10
View File
@@ -0,0 +1,10 @@
TODO:
Jellyfin nfo format (only for video)
- nfo to metadata (this is used as lib)
- metadata to nfo
- create or update nfo
meta yml to nfo
- nfo to metadata (this is used as lib)
- metadata to nfo
- create or update nfo
packages/meta-sort-core/src/jellyfin/nfoTool.ts
+22
View File
@@ -0,0 +1,22 @@
import { parse, sep } from "path";
/**
* Get the nfo file path for a given file path (replace the extension with .nfo)
* @param filePath
* @param ext - The extension to replace with, default is "nfo"
* @returns The nfo file path
*/
export function filePathReplace(filePath: string, ext: string = "nfo"): string {
const { dir, name } = parse(filePath);
return `${dir}${sep}${name}.${ext}`;
}
/**
* Get the nfo file path for a given folder path (add .nfo to the folder path)
* @param folderPath
* @param ext
* @returns The nfo file path
*/
export function filePathAdd(folderPath: string, ext: string = "nfo"): string {
return `${folderPath}.${ext}`;
}
packages/meta-sort-core/src/kv/IKVClient.ts
+192
View File
@@ -0,0 +1,192 @@
/**
* KV Client Interface - Abstraction layer for key-value storage
*
* Redis-based key-value storage with nested key architecture.
* Uses property-level granularity for efficient updates.
*
* Key Structure:
* /file/{hashId}/{property/path} → value
*
* Example:
* /file/midhash256:abc123/title → "Inception"
* /file/midhash256:abc123/video/codec → "h265"
*/
export interface KeyValuePair {
key: string;
value: string;
}
export interface IKVClient {
// ========================================================================
// Basic Operations
// ========================================================================
/**
* Set a key-value pair (value is JSON serialized)
*/
set(key: string, value: any): Promise<void>;
/**
* Get a value by key (value is JSON deserialized)
*/
get(key: string): Promise<any | null>;
/**
* Delete a key
*/
delete(key: string): Promise<void>;
/**
* Count keys with a given prefix
*/
countKeysWithPrefix(prefix: string): Promise<number>;
/**
* Health check - verifies KV store is accessible
*/
health(): Promise<boolean>;
// ========================================================================
// Nested Key Architecture Methods
// ========================================================================
/**
* Set a single property value (raw string, not JSON)
*/
setProperty(key: string, value: string): Promise<void>;
/**
* Get a single property value (raw string)
*/
getProperty(key: string): Promise<string | null>;
/**
* Get all key-value pairs with a given prefix
*/
getRange(prefix: string): Promise<KeyValuePair[]>;
/**
* Set multiple key-value pairs atomically (or as close as possible)
*/
setMultiple(pairs: KeyValuePair[]): Promise<void>;
/**
* Delete all keys with a given prefix
*/
deleteRange(prefix: string): Promise<number>;
// ========================================================================
// High-Level Metadata Operations
// ========================================================================
/**
* Store file metadata using nested key architecture
* Flattens the metadata object and stores each property as a separate key
*/
setMetadataFlat(hashId: string, metadata: any, excludeFields?: string[]): Promise<void>;
/**
* Retrieve file metadata - reconstructs from individual property keys
*/
getMetadataFlat(hashId: string): Promise<any | null>;
/**
* Get a specific property from a file's metadata
*/
getMetadata(hashId: string, propertyPath: string): Promise<any | null>;
/**
* Delete all metadata for a file
*/
deleteMetadataFlat(hashId: string): Promise<number>;
/**
* Get all unique hash IDs (files) stored in KV
*/
getAllHashIds(): Promise<string[]>;
// ========================================================================
// Lifecycle
// ========================================================================
/**
* Close connections and cleanup resources
*/
close(): Promise<void>;
}
/**
* KV Client configuration
*/
export interface KVClientConfig {
/** Backend type */
type: 'redis';
/** Connection URL (e.g., redis://localhost:6379) */
url: string;
/** Operation timeout in ms (default: 30000) */
timeout?: number;
/** Key prefix for all operations (default: '' for none) */
prefix?: string;
}
/**
* Leader lock file content format
*/
export interface LeaderLockInfo {
/** Hostname of the leader */
host: string;
/** Redis connection URL (e.g., redis://10.0.1.50:6379) */
api: string;
/** HTTP API URL for the leader service */
http: string;
/** Configured BASE_URL for stable service discovery (optional, falls back to http) */
baseUrl?: string;
/** Timestamp when leadership was acquired */
timestamp: number;
/** Process ID of the leader */
pid: number;
}
/**
* Service registration info
*/
export interface ServiceInfo {
/** Service name (e.g., 'meta-sort', 'meta-fuse') */
name: string;
/** Service version */
version: string;
/** HTTP API URL */
api: string;
/** Current status */
status: 'starting' | 'running' | 'stopping' | 'stopped';
/** Process ID */
pid: number;
/** Hostname */
hostname: string;
/** Start timestamp (ISO string) */
startedAt: string;
/** Last heartbeat timestamp (ISO string) */
lastHeartbeat: string;
/** Service capabilities */
capabilities: string[];
/** Named endpoints */
endpoints: Record<string, string>;
}
packages/meta-sort-core/src/kv/KVManager.ts
+388
View File
@@ -0,0 +1,388 @@
/**
* KV Manager - Unified manager for KV storage, leader election, and service discovery
*
* This is the main entry point for the KV subsystem. It:
* 1. Participates in leader election (spawns Redis if leader)
* 2. Creates and manages the KV client (Redis)
* 3. Handles service discovery registration
* 4. Manages reconnection on leader failure
*
* Usage:
* ```typescript
* const kvManager = new KVManager({
* metaCorePath: '/meta-core',
* filesPath: '/files',
* serviceName: 'meta-sort',
* apiPort: 3000
* });
*
* await kvManager.start();
* const client = kvManager.getClient();
* // Use client...
* await kvManager.stop();
* ```
*/
import { networkInterfaces } from 'os';
import { LeaderElection } from './LeaderElection.js';
import { ServiceDiscovery } from './ServiceDiscovery.js';
import { RedisKVClient } from './RedisClient.js';
import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
interface KVManagerConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** Path to FILES_VOLUME (e.g., /files) */
filesPath: string;
/** Service name (e.g., 'meta-sort', 'meta-fuse') */
serviceName: string;
/** Service version */
version?: string;
/** HTTP API port */
apiPort: number;
/** Redis port (default: 6379) */
redisPort?: number;
/** Service capabilities */
capabilities?: string[];
/** Whether to skip leader election and use provided Redis URL */
redisUrl?: string;
/** Base URL for service discovery (overrides auto-detected URL) */
baseUrl?: string;
/**
* Hostname to advertise in lock file for other services to connect.
* Use Docker service name (e.g., 'meta-sort-dev') for stable DNS resolution.
*/
advertiseHost?: string;
}
export class KVManager {
private config: KVManagerConfig;
private leaderElection: LeaderElection | null = null;
private serviceDiscovery: ServiceDiscovery | null = null;
private kvClient: IKVClient | null = null;
private isStarted = false;
private isShuttingDown = false;
// Event callbacks
private onReadyCallbacks: (() => void)[] = [];
private onReconnectCallbacks: (() => void)[] = [];
constructor(config: KVManagerConfig) {
this.config = {
version: '1.0.0',
redisPort: 6379,
capabilities: [],
...config
};
}
/**
* Get the local IP for API URL construction
*/
private getLocalIP(): string {
const interfaces = networkInterfaces();
for (const name of Object.keys(interfaces)) {
const addrs = interfaces[name];
if (!addrs) continue;
for (const addr of addrs) {
if (addr.family === 'IPv4' && !addr.internal) {
return addr.address;
}
}
}
return 'localhost';
}
/**
* Create Redis client from URL
*/
private async createRedisClient(redisUrl: string): Promise<RedisKVClient> {
const client = new RedisKVClient({
url: redisUrl,
timeout: 30000,
prefix: '' // No prefix for shared access
});
await client.connect();
return client;
}
/**
* Start the KV manager
*/
async start(): Promise<void> {
if (this.isStarted) {
console.warn('[KVManager] Already started');
return;
}
console.log(`[KVManager] Starting for ${this.config.serviceName}...`);
// Check for direct Redis URL (skip leader election)
if (this.config.redisUrl) {
console.log(`[KVManager] Using direct Redis URL: ${this.config.redisUrl}`);
this.kvClient = await this.createRedisClient(this.config.redisUrl);
// Initialize service discovery even with direct Redis URL
const apiUrl = this.config.baseUrl || `http://localhost:${this.config.apiPort}`;
this.serviceDiscovery = new ServiceDiscovery({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
version: this.config.version!,
apiUrl,
capabilities: this.config.capabilities,
endpoints: {
health: '/health',
dashboard: '/',
api: '/api',
metrics: '/api/metrics',
processing: '/api/processing/status'
}
});
await this.serviceDiscovery.start();
this.isStarted = true;
this.notifyReady();
return;
}
// Initialize leader election
const localIP = this.getLocalIP();
this.leaderElection = new LeaderElection({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
apiPort: this.config.apiPort,
redisPort: this.config.redisPort,
baseUrl: this.config.baseUrl
});
// Setup leader election callbacks
this.leaderElection.onLeader(async () => {
console.log('[KVManager] We are the leader, connecting to local Redis...');
// Connect to local Redis (spawned by leader election)
const redisUrl = `redis://127.0.0.1:${this.config.redisPort}`;
this.kvClient = await this.createRedisClient(redisUrl);
this.notifyReady();
});
this.leaderElection.onFollower(async (leaderInfo: LeaderLockInfo) => {
console.log(`[KVManager] We are a follower, connecting to leader at ${leaderInfo.api}...`);
// Connect to leader's Redis
this.kvClient = await this.createRedisClient(leaderInfo.api);
this.notifyReady();
});
this.leaderElection.onLostLeader(async () => {
console.log('[KVManager] Leader lost, disconnecting...');
if (this.kvClient) {
await this.kvClient.close();
this.kvClient = null;
}
// Leader election will handle re-election and reconnection
});
// Initialize service discovery
// Use BASE_URL if provided, otherwise fall back to auto-detected local IP
const apiUrl = this.config.baseUrl || `http://${localIP}:${this.config.apiPort}`;
this.serviceDiscovery = new ServiceDiscovery({
metaCorePath: this.config.metaCorePath,
serviceName: this.config.serviceName,
version: this.config.version!,
apiUrl,
capabilities: this.config.capabilities,
endpoints: {
health: '/health',
dashboard: '/',
api: '/api',
metrics: '/api/metrics',
processing: '/api/processing/status'
}
});
// Start leader election first
await this.leaderElection.start();
// Start service discovery
await this.serviceDiscovery.start();
this.isStarted = true;
}
/**
* Stop the KV manager
*/
async stop(): Promise<void> {
if (!this.isStarted) return;
console.log('[KVManager] Stopping...');
this.isShuttingDown = true;
// Stop service discovery
if (this.serviceDiscovery) {
await this.serviceDiscovery.stop();
this.serviceDiscovery = null;
}
// Close KV client
if (this.kvClient) {
await this.kvClient.close();
this.kvClient = null;
}
// Stop leader election (releases lock, stops Redis if leader)
if (this.leaderElection) {
await this.leaderElection.stop();
this.leaderElection = null;
}
this.isStarted = false;
console.log('[KVManager] Stopped');
}
/**
* Notify ready callbacks
*/
private notifyReady(): void {
for (const callback of this.onReadyCallbacks) {
try {
callback();
} catch (error) {
console.error('[KVManager] Error in ready callback:', error);
}
}
}
/**
* Register a callback for when the KV client is ready
*/
onReady(callback: () => void): this {
this.onReadyCallbacks.push(callback);
// If already ready, call immediately
if (this.kvClient) {
callback();
}
return this;
}
/**
* Register a callback for reconnection events
*/
onReconnect(callback: () => void): this {
this.onReconnectCallbacks.push(callback);
return this;
}
/**
* Wait for the KV client to be ready
*/
async waitForReady(timeoutMs: number = 30000): Promise<void> {
if (this.kvClient) return;
return new Promise((resolve, reject) => {
const timeout = setTimeout(() => {
reject(new Error(`KV client not ready after ${timeoutMs}ms`));
}, timeoutMs);
this.onReady(() => {
clearTimeout(timeout);
resolve();
});
});
}
// ========================================================================
// Getters
// ========================================================================
/**
* Get the KV client (may be null if not yet connected)
*/
getClient(): IKVClient | null {
return this.kvClient;
}
/**
* Get the KV client (throws if not connected)
*/
requireClient(): IKVClient {
if (!this.kvClient) {
throw new Error('KV client not available. Call start() and wait for ready.');
}
return this.kvClient;
}
/**
* Check if this service is the leader
*/
isLeader(): boolean {
return this.leaderElection?.isLeader() ?? false;
}
/**
* Check if this service is a follower
*/
isFollower(): boolean {
return this.leaderElection?.isFollower() ?? false;
}
/**
* Get the current leader info
*/
getLeaderInfo(): LeaderLockInfo | null {
return this.leaderElection?.getLeaderInfo() ?? null;
}
/**
* Get service discovery instance
*/
getServiceDiscovery(): ServiceDiscovery | null {
return this.serviceDiscovery;
}
/**
* Discover another service
*/
async discoverService(name: string) {
return this.serviceDiscovery?.discoverService(name) ?? null;
}
/**
* Check if the KV store is healthy
*/
async isHealthy(): Promise<boolean> {
if (!this.kvClient) return false;
return this.kvClient.health();
}
/**
* Get paths configuration
*/
getPaths() {
return {
metaCorePath: this.config.metaCorePath,
filesPath: this.config.filesPath
};
}
}
packages/meta-sort-core/src/kv/LeaderElection.ts
+615
View File
@@ -0,0 +1,615 @@
/**
* Leader Election using flock on shared filesystem
*
* This implements a distributed leader election mechanism using:
* 1. flock (advisory file locking) for mutual exclusion
* 2. Shared filesystem as the consensus layer
* 3. Separate info file for service discovery
*
* Architecture:
* - Service tries to acquire exclusive flock on /meta-core/locks/kv-leader.lock
* - If acquired → becomes LEADER, spawns Redis, writes endpoint to info file
* - If blocked → becomes FOLLOWER, reads leader endpoint from info file
* - flock automatically releases when process dies (no stale lock issues!)
*
* Key advantage over O_EXCL approach:
* - Lock is automatically released when process exits/crashes
* - No need for timestamp-based stale detection
* - No race conditions during takeover
*/
import { promises as fs } from 'fs';
import { dirname } from 'path';
import { hostname, networkInterfaces } from 'os';
import { spawn, ChildProcess, execSync } from 'child_process';
import * as IORedis from 'ioredis';
import type { LeaderLockInfo } from './IKVClient.js';
// ESM/CJS interop for ioredis
const Redis = (IORedis as any).default ?? IORedis;
interface LeaderElectionConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** Service name for identification */
serviceName: string;
/** HTTP API port */
apiPort: number;
/** Redis port (default: 6379) */
redisPort?: number;
/** Redis data directory (relative to META_CORE_VOLUME/db/redis) */
redisDataDir?: string;
/** Health check interval in ms (default: 5000) */
healthCheckInterval?: number;
/** Base URL for stable service discovery (e.g., http://localhost:8180) */
baseUrl?: string;
}
type LeaderRole = 'leader' | 'follower' | 'unknown';
export class LeaderElection {
private config: LeaderElectionConfig;
private lockFilePath: string;
private infoFilePath: string;
private flockProcess: ChildProcess | null = null;
private role: LeaderRole = 'unknown';
private leaderInfo: LeaderLockInfo | null = null;
private redisProcess: ChildProcess | null = null;
private healthCheckTimer: NodeJS.Timeout | null = null;
private isShuttingDown = false;
// Event callbacks
private onBecomeLeader?: () => void;
private onBecomeFollower?: (leaderInfo: LeaderLockInfo) => void;
private onLeaderLost?: () => void;
constructor(config: LeaderElectionConfig) {
this.config = {
redisPort: 6379,
healthCheckInterval: 5000,
...config
};
this.lockFilePath = `${this.config.metaCorePath}/locks/kv-leader.lock`;
this.infoFilePath = `${this.config.metaCorePath}/locks/kv-leader.info`;
}
/**
* Get the local machine's IP address
*/
private getLocalIP(): string {
const interfaces = networkInterfaces();
// Prefer non-internal IPv4 addresses
for (const name of Object.keys(interfaces)) {
const addrs = interfaces[name];
if (!addrs) continue;
for (const addr of addrs) {
if (addr.family === 'IPv4' && !addr.internal) {
return addr.address;
}
}
}
// Fallback to hostname or localhost
return hostname() || 'localhost';
}
/**
* Build leader info for this service
*/
private buildLeaderInfo(): LeaderLockInfo {
const ip = this.getLocalIP();
return {
host: hostname(),
api: `redis://${ip}:${this.config.redisPort}`,
http: `http://${ip}:${this.config.apiPort}`,
baseUrl: this.config.baseUrl,
timestamp: Date.now(),
pid: process.pid
};
}
/**
* Ensure lock directory exists
*/
private async ensureLockDir(): Promise<void> {
const lockDir = dirname(this.lockFilePath);
await fs.mkdir(lockDir, { recursive: true });
}
/**
* Try to acquire the leader lock using flock
* Returns true if lock acquired (we are leader), false otherwise
*
* Uses `flock -n -x` for non-blocking exclusive lock attempt.
* The lock is held by keeping a child process alive with the lock fd open.
*/
private async tryAcquireLock(): Promise<boolean> {
await this.ensureLockDir();
// Ensure lock file exists (flock needs an existing file)
try {
await fs.writeFile(this.lockFilePath, '', { flag: 'a' });
} catch {
// Ignore if file already exists
}
return new Promise((resolve) => {
// Spawn a process that tries to acquire the lock and holds it
// Using flock with file descriptor: open fd 200 on lock file, then sleep forever
// The lock is held as long as fd 200 is open (process alive)
this.flockProcess = spawn('bash', [
'-c',
`exec 200>"${this.lockFilePath}" && flock -n -x 200 && echo LOCKED && while true; do sleep 86400; done`
], {
stdio: ['ignore', 'pipe', 'pipe']
});
let resolved = false;
// Check stdout for "LOCKED" message indicating we got the lock
this.flockProcess.stdout?.on('data', (data) => {
const output = data.toString().trim();
if (output === 'LOCKED' && !resolved) {
resolved = true;
console.log('[LeaderElection] Acquired flock on', this.lockFilePath);
resolve(true);
}
});
this.flockProcess.stderr?.on('data', (data) => {
const msg = data.toString().trim();
if (msg) {
console.error('[LeaderElection] flock stderr:', msg);
}
});
// If process exits quickly, we didn't get the lock
this.flockProcess.on('exit', (code) => {
if (!resolved) {
resolved = true;
console.log('[LeaderElection] Could not acquire flock (lock held by another process)');
this.flockProcess = null;
resolve(false);
} else if (!this.isShuttingDown && this.role === 'leader') {
// Lock process died unexpectedly while we were leader
console.error('[LeaderElection] flock process died unexpectedly!');
this.flockProcess = null;
this.handleLockLost();
}
});
// Timeout: if we don't get the lock within 2 seconds, assume it's held
setTimeout(() => {
if (!resolved) {
resolved = true;
if (this.flockProcess) {
this.flockProcess.kill();
this.flockProcess = null;
}
console.log('[LeaderElection] Timeout acquiring flock (lock held by another process)');
resolve(false);
}
}, 2000);
});
}
/**
* Handle losing the lock unexpectedly
*/
private async handleLockLost(): Promise<void> {
if (this.isShuttingDown) return;
console.log('[LeaderElection] Lost leadership, cleaning up...');
this.role = 'unknown';
// Stop Redis
await this.stopRedis();
// Notify callback
this.onLeaderLost?.();
// Try to re-elect
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
await this.transitionToFollower();
}
}
/**
* Check if the lock is currently held by someone
* Uses flock -n which fails immediately if lock is held
*/
private isLockHeld(): boolean {
try {
// Try to acquire lock non-blocking - if it succeeds, no one holds it
execSync(`flock -n -x "${this.lockFilePath}" -c "exit 0"`, {
timeout: 1000,
stdio: 'ignore'
});
// Lock was free (we briefly held it and released)
return false;
} catch {
// Lock is held by someone else
return true;
}
}
/**
* Read leader info from info file
*/
private async readLeaderInfo(): Promise<LeaderLockInfo | null> {
try {
const content = await fs.readFile(this.infoFilePath, 'utf-8');
return JSON.parse(content) as LeaderLockInfo;
} catch {
return null;
}
}
/**
* Write leader info to info file (atomic write)
*/
private async writeLeaderInfo(): Promise<void> {
this.leaderInfo = this.buildLeaderInfo();
// Atomic write: write to temp file, then rename
const tempPath = `${this.infoFilePath}.tmp`;
await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
await fs.rename(tempPath, this.infoFilePath);
console.log('[LeaderElection] Wrote leader info to', this.infoFilePath);
}
/**
* Update leader timestamp (heartbeat) using atomic write
*/
private async updateLeaderTimestamp(): Promise<void> {
if (this.role !== 'leader' || !this.leaderInfo) return;
this.leaderInfo.timestamp = Date.now();
try {
// Atomic write: write to temp file, then rename
const tempPath = `${this.infoFilePath}.tmp`;
await fs.writeFile(tempPath, JSON.stringify(this.leaderInfo, null, 2));
await fs.rename(tempPath, this.infoFilePath);
} catch (error) {
console.error('[LeaderElection] Failed to update timestamp:', error);
}
}
/**
* Spawn Redis server as child process
*/
private async spawnRedis(): Promise<void> {
const redisDataDir = `${this.config.metaCorePath}/db/redis`;
// Ensure Redis data directory exists
await fs.mkdir(redisDataDir, { recursive: true });
console.log(`[LeaderElection] Spawning Redis on port ${this.config.redisPort}...`);
// Spawn Redis server
this.redisProcess = spawn('redis-server', [
'--port', String(this.config.redisPort),
'--bind', '0.0.0.0',
'--dir', redisDataDir,
'--appendonly', 'yes',
'--appendfilename', 'appendonly.aof',
'--dbfilename', 'dump.rdb',
'--save', '60', '1', // Save after 60 seconds if at least 1 key changed
'--loglevel', 'warning'
], {
stdio: ['ignore', 'pipe', 'pipe'],
detached: false
});
this.redisProcess.stdout?.on('data', (data) => {
console.log(`[Redis] ${data.toString().trim()}`);
});
this.redisProcess.stderr?.on('data', (data) => {
console.error(`[Redis] ${data.toString().trim()}`);
});
this.redisProcess.on('exit', (code, signal) => {
console.log(`[Redis] Process exited with code ${code}, signal ${signal}`);
this.redisProcess = null;
if (!this.isShuttingDown && this.role === 'leader') {
// Redis crashed - try to restart
console.log('[LeaderElection] Redis crashed, attempting restart...');
this.spawnRedis().catch(console.error);
}
});
// Wait for Redis to be ready
await this.waitForRedisReady();
}
/**
* Wait for Redis to be ready to accept connections
*/
private async waitForRedisReady(): Promise<void> {
const maxAttempts = 30;
const ip = this.getLocalIP();
for (let i = 0; i < maxAttempts; i++) {
try {
const client = new Redis({
host: ip,
port: this.config.redisPort,
connectTimeout: 1000,
maxRetriesPerRequest: 1
});
const pong = await client.ping();
await client.quit();
if (pong === 'PONG') {
console.log('[LeaderElection] Redis is ready');
return;
}
} catch {
// Redis not ready yet
}
await new Promise(resolve => setTimeout(resolve, 1000));
}
throw new Error('Redis failed to start within timeout');
}
/**
* Stop Redis server
*/
private async stopRedis(): Promise<void> {
if (this.redisProcess) {
console.log('[LeaderElection] Stopping Redis...');
this.redisProcess.kill('SIGTERM');
// Wait for graceful shutdown
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
if (this.redisProcess) {
this.redisProcess.kill('SIGKILL');
}
resolve();
}, 5000);
this.redisProcess!.on('exit', () => {
clearTimeout(timeout);
resolve();
});
});
this.redisProcess = null;
}
}
/**
* Release the leader lock by killing the flock process
*/
private async releaseLock(): Promise<void> {
if (this.flockProcess) {
console.log('[LeaderElection] Releasing flock...');
// Kill the flock holder process - this releases the lock
this.flockProcess.kill('SIGTERM');
// Wait briefly for graceful exit
await new Promise<void>((resolve) => {
const timeout = setTimeout(() => {
if (this.flockProcess) {
this.flockProcess.kill('SIGKILL');
}
resolve();
}, 1000);
if (this.flockProcess) {
this.flockProcess.on('exit', () => {
clearTimeout(timeout);
resolve();
});
} else {
clearTimeout(timeout);
resolve();
}
});
this.flockProcess = null;
}
// Clean up info file (optional, but good hygiene)
try {
await fs.unlink(this.infoFilePath);
} catch {
// Ignore if file doesn't exist
}
}
/**
* Start health check loop
*/
private startHealthCheck(): void {
this.healthCheckTimer = setInterval(async () => {
if (this.isShuttingDown) return;
try {
if (this.role === 'leader') {
// Update timestamp as heartbeat
await this.updateLeaderTimestamp();
// Verify we still hold the lock
if (!this.flockProcess || this.flockProcess.exitCode !== null) {
console.error('[LeaderElection] Lost flock process!');
await this.handleLockLost();
}
} else if (this.role === 'follower') {
// Check if lock is still held (leader still alive)
if (!this.isLockHeld()) {
console.log('[LeaderElection] Lock is free, attempting to become leader...');
this.onLeaderLost?.();
// Try to become leader
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
// Another service became leader - reconnect
await this.transitionToFollower();
}
}
}
} catch (error) {
console.error('[LeaderElection] Health check error:', error);
}
}, this.config.healthCheckInterval);
}
/**
* Transition to leader role
*/
private async transitionToLeader(): Promise<void> {
console.log('[LeaderElection] Becoming LEADER');
this.role = 'leader';
// Write leader info BEFORE spawning Redis so followers can find us
await this.writeLeaderInfo();
// Spawn Redis
await this.spawnRedis();
// Notify callback
this.onBecomeLeader?.();
}
/**
* Transition to follower role
*/
private async transitionToFollower(): Promise<void> {
console.log('[LeaderElection] Becoming FOLLOWER');
this.role = 'follower';
// Wait a bit for leader to write info file
await new Promise(resolve => setTimeout(resolve, 500));
// Read leader info
this.leaderInfo = await this.readLeaderInfo();
if (this.leaderInfo) {
console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
this.onBecomeFollower?.(this.leaderInfo);
} else {
console.warn('[LeaderElection] Could not read leader info, will retry...');
// Retry reading after a delay
setTimeout(async () => {
this.leaderInfo = await this.readLeaderInfo();
if (this.leaderInfo) {
console.log(`[LeaderElection] Leader is ${this.leaderInfo.host} at ${this.leaderInfo.api}`);
this.onBecomeFollower?.(this.leaderInfo);
}
}, 2000);
}
}
/**
* Start the leader election process
*/
async start(): Promise<void> {
console.log(`[LeaderElection] Starting election for ${this.config.serviceName}...`);
console.log(`[LeaderElection] Lock file: ${this.lockFilePath}`);
console.log(`[LeaderElection] Info file: ${this.infoFilePath}`);
// Try to become leader
const acquired = await this.tryAcquireLock();
if (acquired) {
await this.transitionToLeader();
} else {
await this.transitionToFollower();
}
// Start health check
this.startHealthCheck();
}
/**
* Stop and cleanup
*/
async stop(): Promise<void> {
console.log('[LeaderElection] Stopping...');
this.isShuttingDown = true;
// Stop health check
if (this.healthCheckTimer) {
clearInterval(this.healthCheckTimer);
this.healthCheckTimer = null;
}
// If we were leader, cleanup
if (this.role === 'leader') {
await this.stopRedis();
await this.releaseLock();
}
this.role = 'unknown';
console.log('[LeaderElection] Stopped');
}
// ========================================================================
// Event Registration
// ========================================================================
onLeader(callback: () => void): this {
this.onBecomeLeader = callback;
return this;
}
onFollower(callback: (leaderInfo: LeaderLockInfo) => void): this {
this.onBecomeFollower = callback;
return this;
}
onLostLeader(callback: () => void): this {
this.onLeaderLost = callback;
return this;
}
// ========================================================================
// Getters
// ========================================================================
getRole(): LeaderRole {
return this.role;
}
isLeader(): boolean {
return this.role === 'leader';
}
isFollower(): boolean {
return this.role === 'follower';
}
getLeaderInfo(): LeaderLockInfo | null {
return this.leaderInfo;
}
getRedisUrl(): string | null {
if (this.leaderInfo) {
return this.leaderInfo.api;
}
return null;
}
}
packages/meta-sort-core/src/kv/MetadataUtils.ts
+274
View File
@@ -0,0 +1,274 @@
/**
* Utility functions for converting between nested JSON metadata and flat key-value pairs.
* Implements the hierarchical nested key architecture for KV storage (Redis, etc.)
*/
import type { KeyValuePair } from './IKVClient.js';
/**
* Flattens a nested metadata object into an array of key-value pairs.
* Each property becomes a separate key with a hierarchical path.
*
* @param metadata - The nested metadata object to flatten
* @param prefix - The key prefix (e.g., "/file/{hashId}")
* @param excludeFields - Fields to exclude from flattening (e.g., internal tracking fields)
* @returns Array of key-value pairs suitable for KV storage
*
* @example
* flattenMetadata({ title: "Inception", video: { codec: "h265" } }, "/file/abc123")
* // Returns:
* // [
* // { key: "/file/abc123/title", value: "Inception" },
* // { key: "/file/abc123/video/codec", value: "h265" }
* // ]
*/
export function flattenMetadata(
metadata: any,
prefix: string,
excludeFields: string[] = []
): KeyValuePair[] {
const pairs: KeyValuePair[] = [];
function flatten(obj: any, path: string) {
if (obj === null || obj === undefined) {
// Store null/undefined as empty string
pairs.push({ key: path, value: '' });
return;
}
if (typeof obj !== 'object') {
// Primitive value - store as string
pairs.push({ key: path, value: String(obj) });
return;
}
if (Array.isArray(obj)) {
// Handle arrays with numeric indices
obj.forEach((item, index) => {
flatten(item, `${path}/${index}`);
});
return;
}
// Handle objects recursively
Object.keys(obj).forEach((key) => {
// Skip excluded fields
if (excludeFields.includes(key)) {
return;
}
const value = obj[key];
const newPath = path ? `${path}/${key}` : key;
flatten(value, newPath);
});
}
flatten(metadata, prefix);
return pairs;
}
/**
* Reconstructs a nested metadata object from flat key-value pairs.
* Reverses the flattening process to restore the original structure.
*
* @param pairs - Array of key-value pairs from KV storage
* @param prefix - The key prefix to strip (e.g., "/file/{hashId}")
* @returns Reconstructed nested metadata object
*
* @example
* reconstructMetadata([
* { key: "/file/abc123/title", value: "Inception" },
* { key: "/file/abc123/video/codec", value: "h265" }
* ], "/file/abc123")
* // Returns: { title: "Inception", video: { codec: "h265" } }
*/
export function reconstructMetadata(
pairs: KeyValuePair[],
prefix: string
): any {
const result: any = {};
pairs.forEach(({ key, value }) => {
// Strip the prefix to get the property path
if (!key.startsWith(prefix)) {
return; // Skip keys that don't match the prefix
}
const propertyPath = key.slice(prefix.length + 1); // +1 to skip the leading slash
if (!propertyPath) {
return; // Skip if no property path after prefix
}
const parts = propertyPath.split('/');
let current = result;
// Navigate/create the nested structure
for (let i = 0; i < parts.length; i++) {
const part = parts[i];
const isLast = i === parts.length - 1;
if (isLast) {
// Last part - set the value
current[part] = parseValue(value);
} else {
// Intermediate part - create object or array as needed
const nextPart = parts[i + 1];
const isNextPartNumeric = /^\d+$/.test(nextPart);
if (!(part in current)) {
// Create array if next part is numeric, otherwise create object
current[part] = isNextPartNumeric ? [] : {};
}
current = current[part];
}
}
});
// Post-process: Convert objects with only numeric keys to arrays
return convertNumericObjectsToArrays(result);
}
/**
* Recursively converts objects with only numeric sequential keys to arrays.
* This handles cases where arrays were stored with numeric indices.
*/
function convertNumericObjectsToArrays(obj: any): any {
if (obj === null || typeof obj !== 'object') {
return obj;
}
if (Array.isArray(obj)) {
// Already an array, recursively process elements
return obj.map(convertNumericObjectsToArrays);
}
const keys = Object.keys(obj);
// Check if all keys are numeric and sequential (0, 1, 2, ...)
const allNumeric = keys.every(key => /^\d+$/.test(key));
if (allNumeric && keys.length > 0) {
// Sort keys numerically
const sortedKeys = keys.map(Number).sort((a, b) => a - b);
// Check if keys are sequential starting from 0
const isSequential = sortedKeys.every((key, index) => key === index);
if (isSequential) {
// Convert to array
const arr = sortedKeys.map(index =>
convertNumericObjectsToArrays(obj[index.toString()])
);
return arr;
}
}
// Regular object - recursively process values
const result: any = {};
for (const key of keys) {
result[key] = convertNumericObjectsToArrays(obj[key]);
}
return result;
}
/**
* Parses a string value from KV storage into the appropriate JavaScript type.
* Handles numbers, booleans, null, and strings.
*
* @param value - The string value from KV storage
* @returns Parsed value with appropriate type
*/
function parseValue(value: string): any {
// Handle empty string as null
if (value === '') {
return null;
}
// Handle booleans
if (value === 'true') return true;
if (value === 'false') return false;
// Handle null
if (value === 'null') return null;
// Try to parse as number, but be conservative
// Only parse as number if it's a valid integer or floating point without leading zeros
// This preserves strings like "5.1" (channel layout), "23.976" (frame rate)
// Note: In actual storage, these will be stored as strings, so this only matters
// when the original value was numeric. For metadata consistency, we convert
// what looks like numbers to numbers.
const num = Number(value);
if (!isNaN(num) && value.trim() !== '' && /^-?\d+(\.\d+)?$/.test(value)) {
return num;
}
// Return as string
return value;
}
/**
* Extracts the hash ID from a file key.
*
* @param key - Full key like "/file/{hashId}/property/path"
* @returns The hash ID or null if not a valid file key
*
* @example
* extractHashId("/file/sha256:abc123/title") // Returns "sha256:abc123"
*/
export function extractHashId(key: string): string | null {
const match = key.match(/^\/file\/([^/]+)/);
return match ? match[1] : null;
}
/**
* Builds a property key path.
*
* @param hashId - The file's hash ID
* @param propertyPath - The property path (e.g., "video/codec")
* @returns Full KV key path
*
* @example
* buildPropertyKey("sha256:abc123", "video/codec")
* // Returns "/file/sha256:abc123/video/codec"
*/
export function buildPropertyKey(hashId: string, propertyPath: string): string {
return `/file/${hashId}/${propertyPath}`;
}
/**
* Builds a file prefix for range queries.
*
* @param hashId - The file's hash ID
* @returns Prefix for querying all properties of a file
*
* @example
* buildFilePrefix("sha256:abc123") // Returns "/file/sha256:abc123"
*/
export function buildFilePrefix(hashId: string): string {
return `/file/${hashId}`;
}
/**
* Groups key-value pairs by hash ID.
* Useful for batch processing multiple files' metadata.
*
* @param pairs - Array of key-value pairs
* @returns Map of hash ID to array of key-value pairs
*/
export function groupByHashId(pairs: KeyValuePair[]): Map<string, KeyValuePair[]> {
const grouped = new Map<string, KeyValuePair[]>();
pairs.forEach((pair) => {
const hashId = extractHashId(pair.key);
if (hashId) {
if (!grouped.has(hashId)) {
grouped.set(hashId, []);
}
grouped.get(hashId)!.push(pair);
}
});
return grouped;
}
packages/meta-sort-core/src/kv/RedisClient.ts
+351
View File
@@ -0,0 +1,351 @@
/**
* Redis KV Client - Implements IKVClient interface using Redis
*
* Uses Redis hashes for efficient nested key storage:
* - Each file's metadata is stored as a Redis hash
* - Hash key: file:{hashId}
* - Hash field: property/path (e.g., "video/codec")
*
* Benefits of Redis hashes:
* - Single command to get all file metadata (HGETALL)
* - Single command to set multiple properties (HMSET)
* - Atomic operations on file metadata
*/
import * as IORedis from 'ioredis';
import type { IKVClient, KeyValuePair } from './IKVClient.js';
import {
flattenMetadata,
reconstructMetadata,
buildFilePrefix,
buildPropertyKey,
} from './MetadataUtils.js';
// ESM/CJS interop for ioredis
const Redis = (IORedis as any).default ?? IORedis;
type RedisInstance = InstanceType<typeof Redis>;
interface RedisClientConfig {
url: string;
timeout?: number;
prefix?: string;
}
export class RedisKVClient implements IKVClient {
private redis: RedisInstance;
private timeout: number;
private prefix: string;
private isConnected: boolean = false;
constructor(config: RedisClientConfig) {
this.timeout = config.timeout || 30000;
this.prefix = config.prefix || '';
// Parse Redis URL and create client
this.redis = new Redis(config.url, {
commandTimeout: this.timeout,
connectTimeout: this.timeout,
lazyConnect: true,
maxRetriesPerRequest: 3,
enableOfflineQueue: true,
retryStrategy: (times) => {
const delay = Math.min(times * 200, 2000);
console.log(`[Redis] Retry attempt ${times}, waiting ${delay}ms`);
return delay;
}
});
// Event handlers
this.redis.on('connect', () => {
this.isConnected = true;
console.log('[Redis] Connected');
});
this.redis.on('error', (err) => {
console.error('[Redis] Error:', err.message);
});
this.redis.on('close', () => {
this.isConnected = false;
console.log('[Redis] Connection closed');
});
}
/**
* Connect to Redis (call before using other methods)
*/
async connect(): Promise<void> {
if (this.isConnected) return;
await this.redis.connect();
}
/**
* Build the full Redis key with optional prefix
*/
private buildKey(key: string): string {
return this.prefix ? `${this.prefix}${key}` : key;
}
// ========================================================================
// Basic Operations
// ========================================================================
async set(key: string, value: any): Promise<void> {
const fullKey = this.buildKey(key);
await this.redis.set(fullKey, JSON.stringify(value));
}
async get(key: string): Promise<any | null> {
const fullKey = this.buildKey(key);
const value = await this.redis.get(fullKey);
if (value === null) return null;
try {
return JSON.parse(value);
} catch {
return value;
}
}
async delete(key: string): Promise<void> {
const fullKey = this.buildKey(key);
await this.redis.del(fullKey);
}
async countKeysWithPrefix(prefix: string): Promise<number> {
const fullPrefix = this.buildKey(prefix);
// Use SCAN to count keys (don't use KEYS in production)
let count = 0;
let cursor = '0';
do {
const [nextCursor, keys] = await this.redis.scan(
cursor,
'MATCH',
`${fullPrefix}*`,
'COUNT',
1000
);
cursor = nextCursor;
count += keys.length;
} while (cursor !== '0');
return count;
}
async health(): Promise<boolean> {
try {
const result = await this.redis.ping();
return result === 'PONG';
} catch {
return false;
}
}
// ========================================================================
// Nested Key Architecture Methods
// ========================================================================
async setProperty(key: string, value: string): Promise<void> {
const fullKey = this.buildKey(key);
await this.redis.set(fullKey, value);
}
async getProperty(key: string): Promise<string | null> {
const fullKey = this.buildKey(key);
return await this.redis.get(fullKey);
}
async getRange(prefix: string): Promise<KeyValuePair[]> {
const fullPrefix = this.buildKey(prefix);
const pairs: KeyValuePair[] = [];
let cursor = '0';
do {
const [nextCursor, keys] = await this.redis.scan(
cursor,
'MATCH',
`${fullPrefix}*`,
'COUNT',
1000
);
cursor = nextCursor;
if (keys.length > 0) {
// Get values for all found keys
const values = await this.redis.mget(...keys);
keys.forEach((key, index) => {
const value = values[index];
if (value !== null) {
// Strip prefix to get original key
const originalKey = this.prefix
? key.slice(this.prefix.length)
: key;
pairs.push({ key: originalKey, value });
}
});
}
} while (cursor !== '0');
return pairs;
}
async setMultiple(pairs: KeyValuePair[]): Promise<void> {
if (pairs.length === 0) return;
// Use pipeline for efficient batch operations
const pipeline = this.redis.pipeline();
for (const pair of pairs) {
const fullKey = this.buildKey(pair.key);
pipeline.set(fullKey, pair.value);
}
await pipeline.exec();
}
async deleteRange(prefix: string): Promise<number> {
const fullPrefix = this.buildKey(prefix);
let deletedCount = 0;
let cursor = '0';
do {
const [nextCursor, keys] = await this.redis.scan(
cursor,
'MATCH',
`${fullPrefix}*`,
'COUNT',
1000
);
cursor = nextCursor;
if (keys.length > 0) {
const deleted = await this.redis.del(...keys);
deletedCount += deleted;
}
} while (cursor !== '0');
return deletedCount;
}
// ========================================================================
// High-Level Metadata Operations (Using Redis Hashes)
// ========================================================================
/**
* Store file metadata using Redis hash
* More efficient than individual keys - single HMSET command
*/
async setMetadataFlat(
hashId: string,
metadata: any,
excludeFields: string[] = []
): Promise<void> {
const prefix = buildFilePrefix(hashId);
const pairs = flattenMetadata(metadata, prefix, excludeFields);
if (pairs.length === 0) return;
// Convert to Redis hash format
// Hash key: file:{hashId}
// Hash fields: property paths (without /file/{hashId} prefix)
const hashKey = this.buildKey(`file:${hashId}`);
const hashData: Record<string, string> = {};
for (const pair of pairs) {
// Extract field name by removing the prefix
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
if (field) {
hashData[field] = pair.value;
}
}
if (Object.keys(hashData).length > 0) {
await this.redis.hmset(hashKey, hashData);
}
// Also store in key index for getAllHashIds
await this.redis.sadd(this.buildKey('file:__index__'), hashId);
}
async getMetadataFlat(hashId: string): Promise<any | null> {
const hashKey = this.buildKey(`file:${hashId}`);
const hashData = await this.redis.hgetall(hashKey);
if (!hashData || Object.keys(hashData).length === 0) {
return null;
}
// Convert Redis hash back to KeyValuePair format for reconstruction
const prefix = buildFilePrefix(hashId);
const pairs: KeyValuePair[] = Object.entries(hashData).map(([field, value]) => ({
key: `${prefix}/${field}`,
value: value as string
}));
return reconstructMetadata(pairs, prefix);
}
async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
const hashKey = this.buildKey(`file:${hashId}`);
const value = await this.redis.hget(hashKey, propertyPath);
if (value === null) return null;
try {
return JSON.parse(value);
} catch {
return value;
}
}
async deleteMetadataFlat(hashId: string): Promise<number> {
const hashKey = this.buildKey(`file:${hashId}`);
const fieldCount = await this.redis.hlen(hashKey);
await this.redis.del(hashKey);
await this.redis.srem(this.buildKey('file:__index__'), hashId);
return fieldCount;
}
async getAllHashIds(): Promise<string[]> {
const indexKey = this.buildKey('file:__index__');
return await this.redis.smembers(indexKey);
}
// ========================================================================
// Pub/Sub Operations
// ========================================================================
/**
* Publish a message to a Redis channel
* Used for notifying other services (meta-fuse) about file changes
* Note: Channel names are NOT prefixed - they use their own namespace
*/
async publish(channel: string, message: string): Promise<number> {
return await this.redis.publish(channel, message);
}
// ========================================================================
// Lifecycle
// ========================================================================
async close(): Promise<void> {
await this.redis.quit();
this.isConnected = false;
}
/**
* Check if client is connected
*/
isHealthy(): boolean {
return this.isConnected;
}
/**
* Get the underlying Redis client (for advanced operations)
*/
getRedisClient(): RedisInstance {
return this.redis;
}
}
packages/meta-sort-core/src/kv/ServiceDiscovery.ts
+312
View File
@@ -0,0 +1,312 @@
/**
* Service Discovery using shared filesystem
*
* Each service registers itself in /meta-core/services/{service-name}.json
* with its API endpoint, status, and heartbeat timestamp.
*
* Services can discover each other by reading these JSON files.
* Stale detection: if lastHeartbeat > 60 seconds ago, service is considered dead.
*/
import { promises as fs } from 'fs';
import { hostname } from 'os';
import { join } from 'path';
import type { ServiceInfo } from './IKVClient.js';
interface ServiceDiscoveryConfig {
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
metaCorePath: string;
/** Service name */
serviceName: string;
/** Service version */
version: string;
/** HTTP API URL */
apiUrl: string;
/** Service capabilities */
capabilities?: string[];
/** Named endpoints */
endpoints?: Record<string, string>;
/** Heartbeat interval in ms (default: 30000) */
heartbeatInterval?: number;
/** Stale threshold in ms (default: 60000) */
staleThreshold?: number;
}
export class ServiceDiscovery {
private config: ServiceDiscoveryConfig;
private servicesDir: string;
private serviceFilePath: string;
private heartbeatTimer: NodeJS.Timeout | null = null;
private isShuttingDown = false;
constructor(config: ServiceDiscoveryConfig) {
this.config = {
heartbeatInterval: 30000,
staleThreshold: 60000,
capabilities: [],
endpoints: {},
...config
};
this.servicesDir = `${this.config.metaCorePath}/services`;
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}.json`;
}
/**
* Ensure services directory exists
*/
private async ensureServicesDir(): Promise<void> {
await fs.mkdir(this.servicesDir, { recursive: true });
}
/**
* Build service info for this service
*/
private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
return {
name: this.config.serviceName,
version: this.config.version,
api: this.config.apiUrl,
status,
pid: process.pid,
hostname: hostname(),
startedAt: new Date().toISOString(),
lastHeartbeat: new Date().toISOString(),
capabilities: this.config.capabilities || [],
endpoints: this.config.endpoints || {}
};
}
/**
* Register this service
*/
async register(): Promise<void> {
await this.ensureServicesDir();
const info = this.buildServiceInfo('starting');
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
console.log(`[ServiceDiscovery] Registered ${this.config.serviceName}`);
}
/**
* Update service status
*/
async updateStatus(status: ServiceInfo['status']): Promise<void> {
try {
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
const info = JSON.parse(content) as ServiceInfo;
info.status = status;
info.lastHeartbeat = new Date().toISOString();
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
} catch (error) {
console.error('[ServiceDiscovery] Failed to update status:', error);
}
}
/**
* Send heartbeat (update lastHeartbeat timestamp)
*/
async heartbeat(): Promise<void> {
try {
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
const info = JSON.parse(content) as ServiceInfo;
info.lastHeartbeat = new Date().toISOString();
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
} catch (error) {
// If file was deleted, re-register
if ((error as any).code === 'ENOENT') {
await this.register();
await this.updateStatus('running');
} else {
console.error('[ServiceDiscovery] Heartbeat failed:', error);
}
}
}
/**
* Start heartbeat loop
*/
startHeartbeat(): void {
this.heartbeatTimer = setInterval(async () => {
if (this.isShuttingDown) return;
await this.heartbeat();
}, this.config.heartbeatInterval);
}
/**
* Stop heartbeat loop
*/
stopHeartbeat(): void {
if (this.heartbeatTimer) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
}
/**
* Unregister this service (on shutdown)
*/
async unregister(): Promise<void> {
this.isShuttingDown = true;
this.stopHeartbeat();
try {
// Update status to stopped before removing
await this.updateStatus('stopped');
// Optionally remove the file (or keep for debugging)
// await fs.unlink(this.serviceFilePath);
console.log(`[ServiceDiscovery] Unregistered ${this.config.serviceName}`);
} catch (error) {
console.error('[ServiceDiscovery] Failed to unregister:', error);
}
}
// ========================================================================
// Discovery Methods
// ========================================================================
/**
* Discover a service by name
*/
async discoverService(name: string): Promise<ServiceInfo | null> {
const servicePath = join(this.servicesDir, `${name}.json`);
try {
const content = await fs.readFile(servicePath, 'utf-8');
const service = JSON.parse(content) as ServiceInfo;
// Check if service is alive (heartbeat within threshold)
const lastHeartbeat = new Date(service.lastHeartbeat).getTime();
const isAlive = Date.now() - lastHeartbeat < this.config.staleThreshold!;
if (!isAlive) {
console.warn(`[ServiceDiscovery] Service ${name} appears stale (last heartbeat: ${service.lastHeartbeat})`);
return null;
}
return service;
} catch (error) {
if ((error as any).code === 'ENOENT') {
return null; // Service not registered
}
throw error;
}
}
/**
* Discover all registered services
*/
async discoverAllServices(): Promise<ServiceInfo[]> {
try {
const files = await fs.readdir(this.servicesDir);
const services: ServiceInfo[] = [];
for (const file of files) {
if (!file.endsWith('.json')) continue;
const name = file.replace('.json', '');
const service = await this.discoverService(name);
if (service && service.status === 'running') {
services.push(service);
}
}
return services;
} catch (error) {
if ((error as any).code === 'ENOENT') {
return []; // Services directory doesn't exist yet
}
throw error;
}
}
/**
* Wait for a service to become available
*/
async waitForService(
name: string,
timeoutMs: number = 30000,
pollInterval: number = 1000
): Promise<ServiceInfo> {
const startTime = Date.now();
while (Date.now() - startTime < timeoutMs) {
const service = await this.discoverService(name);
if (service && service.status === 'running') {
return service;
}
await new Promise(resolve => setTimeout(resolve, pollInterval));
}
throw new Error(`Service ${name} not available after ${timeoutMs}ms`);
}
/**
* Check if a service is healthy
*/
async isServiceHealthy(name: string): Promise<boolean> {
const service = await this.discoverService(name);
if (!service) return false;
// Try to ping the service's health endpoint
const healthEndpoint = service.endpoints.health || '/health';
const url = `${service.api}${healthEndpoint}`;
try {
const response = await fetch(url, {
signal: AbortSignal.timeout(5000)
});
return response.ok;
} catch {
return false;
}
}
// ========================================================================
// Lifecycle
// ========================================================================
/**
* Full startup sequence
*/
async start(): Promise<void> {
await this.register();
await this.updateStatus('running');
this.startHeartbeat();
}
/**
* Full shutdown sequence
*/
async stop(): Promise<void> {
await this.unregister();
}
// ========================================================================
// Getters
// ========================================================================
getServiceFilePath(): string {
return this.serviceFilePath;
}
getServicesDir(): string {
return this.servicesDir;
}
}
packages/meta-sort-core/src/kv/index.ts
+18
View File
@@ -0,0 +1,18 @@
/**
* KV Module - Unified key-value storage with leader election
*
* This module provides:
* - IKVClient interface for KV storage abstraction
* - RedisKVClient for Redis-based storage
* - LeaderElection for distributed leader election via flock
* - ServiceDiscovery for service registration and discovery
* - KVManager for unified management of all components
* - MetadataUtils for flattening/reconstructing nested metadata
*/
export * from './IKVClient.js';
export * from './RedisClient.js';
export * from './LeaderElection.js';
export * from './ServiceDiscovery.js';
export * from './KVManager.js';
export * from './MetadataUtils.js';
packages/meta-sort-core/src/logic/DuplicateFinder.ts
+113
View File
@@ -0,0 +1,113 @@
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
import {VideoMeta, HashMeta, VersionMeta, FileNameMeta, FileStatMeta} from "@metazla/meta-interface";
import {renamingRule} from "../config/RenamingRule.js";
type Local = VideoMeta & HashMeta & VersionMeta & FileNameMeta & FileStatMeta;
export interface DuplicateGroup {
key: string;
files: string[];
virtualPath?: string;
}
export interface DuplicateResult {
hashDuplicates: DuplicateGroup[];
titleDuplicates: DuplicateGroup[];
}
export class DuplicateFinder {
async findDuplicates(database: Map<string, Local>): Promise<DuplicateResult> {
let [hashDuplicates, titleDuplicates] = await Promise.all([
this.findDuplicatesGeneric(database, this.hashKeyExtractor, 'remove'),
this.findDuplicatesGeneric(database, this.pathExtractor, 'version')
]);
return {
hashDuplicates,
titleDuplicates
};
}
private hashKeyExtractor(metadata: Local): string | null {
return metadata[CID_ALGORITHM_NAMES.sha256] ?? null;
}
private pathExtractor(metadata: Local): string | null {
//case insensitive
try {
return renamingRule(metadata, '')?.toLowerCase() ?? null;
} catch (e) {
return null;
}
}
private async findDuplicatesGeneric(
database: Map<string, Local>,
keyExtractor: (metadata: Local) => string | null,
action?: 'remove' | 'version'
): Promise<DuplicateGroup[]> {
const duplicates = new Map<string, string[]>();
const duplicateGroups: DuplicateGroup[] = [];
for (const [filePath, metadata] of database) {
const key = keyExtractor(metadata);
if (key) {
const files = duplicates.get(key) ?? [];
files.push(filePath);
duplicates.set(key, files);
}
}
for (const [key, files] of duplicates) {
if (files.length > 1) {
// Store duplicate group before modifying database
const sortedFiles = [...files].sort();
// Generate virtualPath for both hash and title duplicates
let virtualPath: string | undefined;
if (action === 'version') {
// For title duplicates, use the key (normalized title path)
virtualPath = key;
} else if (action === 'remove') {
// For hash duplicates, use renamingRule on the first file's metadata
const firstFile = sortedFiles[0];
const metadata = database.get(firstFile);
if (metadata) {
try {
virtualPath = renamingRule(metadata, '');
} catch (e) {
// If renamingRule fails, virtualPath remains undefined
}
}
}
duplicateGroups.push({
key,
files: sortedFiles,
virtualPath
});
//alphabetical order
const remainingFiles = files.slice(1).sort();
//rename all problematic entries V2, V3, V4, V5
let i = 2;
for (const file of remainingFiles) {
if (action === 'remove') {
database.delete(file);
} else if (action === 'version') {
let definition = database.get(file);
definition.version = "V" + i;
if (definition.fileType !== "subtitle") {//allow multiple version of subtitle in the same folder
definition.extra = "true";//mark as extra so it don't pollute the main list
}
i++;
} else {
throw new Error();
}
}
}
}
return duplicateGroups;
}
}
packages/meta-sort-core/src/logic/HashProcessingStateManager.ts
+143
View File
@@ -0,0 +1,143 @@
/**
* HashProcessingStateManager
*
* Tracks the state of background hash computation operations:
* - Pending: Files queued for hash computation
* - Computing: Files currently having hashes computed
* - Complete: Recently completed hash computations
*/
export type HashStatus = 'pending' | 'computing' | 'complete' | 'error';
export interface HashProcessingState {
filePath: string;
tempId?: string;
hashStatus: HashStatus;
queuedAt?: number;
startedAt?: number;
completedAt?: number;
processingTime?: number;
hash?: string;
error?: string;
virtualPath?: string;
}
export interface HashProcessingStatusSnapshot {
pending: HashProcessingState[];
computing: HashProcessingState[];
complete: HashProcessingState[];
totalPending: number;
totalComputing: number;
totalComplete: number;
}
export class HashProcessingStateManager {
private pending: Map<string, HashProcessingState> = new Map();
private computing: Map<string, HashProcessingState> = new Map();
private complete: HashProcessingState[] = [];
private maxCompleteHistory = 100; // Keep last 100 completed hashes
/**
* Add a file to the pending hash queue
*/
addPending(filePath: string, tempId?: string, virtualPath?: string): void {
this.pending.set(filePath, {
filePath,
tempId,
virtualPath,
hashStatus: 'pending',
queuedAt: Date.now()
});
}
/**
* Move a file from pending to computing
*/
startComputing(filePath: string): void {
const pendingState = this.pending.get(filePath);
this.pending.delete(filePath);
this.computing.set(filePath, {
filePath,
tempId: pendingState?.tempId,
virtualPath: pendingState?.virtualPath,
hashStatus: 'computing',
queuedAt: pendingState?.queuedAt,
startedAt: Date.now()
});
}
/**
* Move a file from computing to complete
*/
completeComputing(
filePath: string,
processingTime: number,
hash?: string,
error?: string,
virtualPath?: string
): void {
const computingState = this.computing.get(filePath);
this.computing.delete(filePath);
const completedState: HashProcessingState = {
filePath,
tempId: computingState?.tempId,
virtualPath: virtualPath || computingState?.virtualPath,
hashStatus: error ? 'error' : 'complete',
queuedAt: computingState?.queuedAt,
startedAt: computingState?.startedAt,
completedAt: Date.now(),
processingTime,
hash,
error
};
// Add to complete list (most recent first)
this.complete.unshift(completedState);
// Trim history if needed
if (this.complete.length > this.maxCompleteHistory) {
this.complete = this.complete.slice(0, this.maxCompleteHistory);
}
}
/**
* Remove a file from all states (e.g., when deleted)
*/
removeFile(filePath: string): void {
this.pending.delete(filePath);
this.computing.delete(filePath);
this.complete = this.complete.filter(f => f.filePath !== filePath);
}
/**
* Get a snapshot of the current hash processing state
*/
getSnapshot(): HashProcessingStatusSnapshot {
return {
pending: Array.from(this.pending.values()),
computing: Array.from(this.computing.values()),
complete: this.complete.slice(0, 50), // Return only last 50 for API
totalPending: this.pending.size,
totalComputing: this.computing.size,
totalComplete: this.complete.length
};
}
/**
* Clear all states (useful for testing or reset)
*/
clear(): void {
this.pending.clear();
this.computing.clear();
this.complete = [];
}
/**
* Get the total number of files in all states
*/
getTotalFileCount(): number {
return this.pending.size + this.computing.size + this.complete.length;
}
}
packages/meta-sort-core/src/logic/MetaDataToFolderStruct.ts
+75
View File
@@ -0,0 +1,75 @@
import * as path from 'path';
import {
HashMeta,
} from "@metazla/meta-interface";
type Local = HashMeta;
export class MetaDataToFolderStruct {
public renamingRule: (metamesh: Local, filepath: string) => string | null;
constructor(renamingRule: (metamesh: Local, filepath: string) => string | null) {
this.renamingRule = renamingRule;
}
/**
* Compute virtual filesystem structure for FUSE/WebDAV
* Generates organized paths based on metadata without creating physical files
* @param data Map of file paths to metadata
* @private
*/
private computeVirtualStructure(data: Map<string, Local>): Map<string, string> {
const virtualMap = new Map<string, string>();
for (const [filepath, metamesh] of data) {
// Compute the virtual path for the file
let virtualPath: string;
try {
virtualPath = this.renamingRule(metamesh, filepath);
}catch (e){
console.warn(e.message || e);
}
if (!virtualPath) {
continue;
}
virtualPath = this.sanitizePath(virtualPath);
virtualMap.set(filepath, virtualPath);
}
console.log(`Computed virtual structure for ${virtualMap.size} files`);
return virtualMap;
}
sanitizePath(newPath: string): string {
// First, isolate any potential Windows drive letter at the start of the path
const driveMatch = newPath.match(/^[a-zA-Z]:/);
const drive = driveMatch ? driveMatch[0] : '';
// Define illegal characters for Windows and potentially problematic ones for Unix
const illegalChars = /[<>:"|?*]/g; // Null character added for Unix-like systems
// Replace illegal characters with an underscore in the path, excluding the drive part
const pathWithoutDrive = newPath.slice(drive.length);
const sanitizedPath = pathWithoutDrive.replace(illegalChars, '');
// Combine the drive (if any) and the sanitized path
const cleanPath = drive + sanitizedPath;
// Normalize the clean path to handle slashes appropriately
return path.normalize(cleanPath);
}
/**
* Generate virtual filesystem structure for FUSE/WebDAV
* Returns only the computed virtual paths without checking physical filesystem
* @param data Map of file paths to metadata
*/
public generateVirtualStructure(data: Map<string, Local>): Map<string, string> {
const start = performance.now();
const computed = this.computeVirtualStructure(data);
console.log(`Compute virtual structure took ${performance.now() - start}ms`);
return computed;
}
}
packages/meta-sort-core/src/logic/MetaMeshFormat.ts
+95
View File
@@ -0,0 +1,95 @@
import { promises as fs } from 'fs';
import yaml from 'yaml';
const { parse, stringify } = yaml;
/**
* File type:
* filename.<ext>.meta
* filename.meta (contains multiple metadata entry "mkv", "str" etc...)
* folder.meta
*/
export class MetaMeshFormat {
async createOrUpdate<T>(metaDataFilePath: string, data: T): Promise<void> {
if(!metaDataFilePath.endsWith('.meta')){
throw new Error(`File path must end with '.meta'`);
}
try {
// Check if the file exists
const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
if (fileExists) {
// If the file exists, update it
await this.update(metaDataFilePath, data);
} else {
// If the file does not exist, create it
await fs.writeFile(metaDataFilePath, stringify(data), 'utf8');
}
} catch (error) {
console.error(`Error in createOrUpdate: ${error}`);
throw error;
}
}
async read<T>(metaDataFilePath: string): Promise<T> {
if(!metaDataFilePath.endsWith('.meta')){
throw new Error(`File path must end with '.meta'`);
}
try {
const fileExists = await fs.access(metaDataFilePath).then(() => true).catch(() => false);
if (!fileExists) {
return null;
}
// Read the YAML file
const fileContent = await fs.readFile(metaDataFilePath, 'utf8');
// Parse the YAML content to a JavaScript object
return parse(fileContent) as T;
} catch (error) {
console.error(`Error in read: ${error}`);
throw error;
}
}
async update<T>(metaDataFilePath: string, data: T): Promise<void> {
if(!metaDataFilePath.endsWith('.meta')){
throw new Error(`File path must end with '.meta'`);
}
try {
// Read the existing file content
const existingData = await this.read<any>(metaDataFilePath) || {};
// Merge the existing data with the new data
const updatedData = { ...existingData, ...data };
// Write the updated data back to the file
await fs.writeFile(metaDataFilePath, stringify(updatedData), 'utf8');
} catch (error) {
console.error(`Error in update: ${error}`);
throw error;
}
}
async write<T>(metaDataFilePath: string, data: T): Promise<void> {
if(!metaDataFilePath.endsWith('.meta')){
throw new Error(`File path must end with '.meta'`);
}
try {
// Write the updated data back to the file
await fs.writeFile(metaDataFilePath, stringify(data), {
encoding:'utf8',
flag:'w' // Write mode : Overwrite
});
} catch (error) {
console.error(`Error in update: ${error}`);
throw error;
}
}
/**
* Serialize data to YAML string (for virtual .meta files)
*/
serialize<T>(data: T): string {
return stringify(data);
}
}
packages/meta-sort-core/src/logic/MetadataFormatRegistry.ts
+217
View File
@@ -0,0 +1,217 @@
import yaml from 'yaml';
const { stringify: yamlStringify } = yaml;
import { Builder } from 'xml2js';
/**
* Supported metadata format types
*/
export enum MetadataFormatType {
META = 'meta',
JELLYFIN = 'jellyfin'
}
/**
* Interface for metadata format handlers
* Note: metadata parameter uses `any` type because it combines multiple metadata interfaces
* (HashMeta, JellyfinMeta, VideoMeta, etc.) and TypeScript doesn't have a single type for this union
*/
export interface IMetadataFormatHandler {
/**
* The file extension for this format (e.g., '.meta', '.nfo')
*/
readonly extension: string;
/**
* The format type identifier
*/
readonly formatType: MetadataFormatType;
/**
* Serialize metadata to string format
* @param metadata The metadata to serialize (combination of various metadata interfaces)
* @returns Serialized string content
*/
serialize(metadata: any): string;
}
/**
* Handler for .meta YAML format
*/
export class MetaFormatHandler implements IMetadataFormatHandler {
readonly extension = '.meta';
readonly formatType = MetadataFormatType.META;
serialize(metadata: any): string {
return yamlStringify(metadata);
}
}
/**
* Handler for Jellyfin .nfo XML format
*/
export class JellyfinFormatHandler implements IMetadataFormatHandler {
readonly extension = '.nfo';
readonly formatType = MetadataFormatType.JELLYFIN;
serialize(metadata: any): string {
// Only include Jellyfin-compatible fields
const jellyfinData = this.filterJellyfinFields(metadata);
// Determine root element based on video type
const rootElementName = this.isMovieOrTvShow(jellyfinData) === "tvshow" ? 'episodedetails' : "movie";
// Convert to XML-friendly format
const xmlData = this.convertToJsonXml(jellyfinData);
// Build XML
const obj = {
[rootElementName]: xmlData
};
const builder = new Builder({
xmldec: { version: '1.0', encoding: 'UTF-8', standalone: true }
});
return builder.buildObject(obj);
}
private filterJellyfinFields(metadata: any): any {
const jellyfinFields: any = {};
// Video metadata
if (metadata.videoType) jellyfinFields.videoType = metadata.videoType;
if (metadata.season) jellyfinFields.season = metadata.season;
if (metadata.episode) jellyfinFields.episode = metadata.episode;
if (metadata.originalTitle) jellyfinFields.originalTitle = metadata.originalTitle;
if (metadata.movieYear) jellyfinFields.year = metadata.movieYear;
// Titles (first title or original title)
if (metadata.titles) {
const titleEntries = Object.entries(metadata.titles);
if (titleEntries.length > 0) {
jellyfinFields.title = titleEntries[0][1];
}
}
// IDs
if (metadata.imdbid) jellyfinFields.imdbid = metadata.imdbid;
if (metadata.tmdbid) jellyfinFields.tmdbid = metadata.tmdbid;
if (metadata.anidbid) jellyfinFields.anidbid = metadata.anidbid;
// Additional fields
if (metadata.rating) jellyfinFields.rating = metadata.rating;
if (metadata.plot) jellyfinFields.plot = metadata.plot;
if (metadata.studio) jellyfinFields.studio = metadata.studio;
if (metadata.releasedate) jellyfinFields.releasedate = metadata.releasedate;
if (metadata.mpaa) jellyfinFields.mpaa = metadata.mpaa;
// Sets (genres, tags, languages)
if (metadata.genres) jellyfinFields.genre = metadata.genres;
if (metadata.tags) jellyfinFields.tag = metadata.tags;
if (metadata.languages) jellyfinFields.language = metadata.languages;
// Art
if (metadata.art) {
if (metadata.art.poster) jellyfinFields.poster = metadata.art.poster;
if (metadata.art.fanart) jellyfinFields.fanart = metadata.art.fanart;
}
return jellyfinFields;
}
private isMovieOrTvShow(metadata: any): "tvshow" | "movie" {
if (metadata.videoType) {
return metadata.videoType;
} else if (metadata.episode) {
return "tvshow";
} else {
return "movie";
}
}
private convertToJsonXml(input: any): any {
const output: any = {};
for (const [key, value] of Object.entries(input)) {
if (value instanceof Set) {
// Convert Set to array of items for XML
const array = Array.from(value as any);
if (array.length > 0) {
output[key] = array;
}
} else if (value instanceof Array) {
output[key] = value;
} else if (typeof value === 'object' && value !== null) {
// For nested objects, recursively convert
output[key] = [this.convertToJsonXml(value)];
} else {
output[key] = [value];
}
}
return output;
}
}
/**
* Registry for metadata format handlers
*/
export class MetadataFormatRegistry {
private handlers: Map<MetadataFormatType, IMetadataFormatHandler> = new Map();
constructor() {
// Register default handlers
this.registerHandler(new MetaFormatHandler());
this.registerHandler(new JellyfinFormatHandler());
}
/**
* Register a new format handler
*/
registerHandler(handler: IMetadataFormatHandler): void {
this.handlers.set(handler.formatType, handler);
}
/**
* Get handler by format type
*/
getHandler(formatType: MetadataFormatType): IMetadataFormatHandler | undefined {
return this.handlers.get(formatType);
}
/**
* Get handler by format name (case-insensitive)
*/
getHandlerByName(formatName: string): IMetadataFormatHandler | undefined {
const normalizedName = formatName.toLowerCase();
for (const [type, handler] of this.handlers.entries()) {
if (type.toLowerCase() === normalizedName) {
return handler;
}
}
return undefined;
}
/**
* Get all registered handlers
*/
getAllHandlers(): IMetadataFormatHandler[] {
return Array.from(this.handlers.values());
}
/**
* Get handlers for specified format names
*/
getHandlersForFormats(formatNames: string[]): IMetadataFormatHandler[] {
const handlers: IMetadataFormatHandler[] = [];
for (const name of formatNames) {
const handler = this.getHandlerByName(name);
if (handler) {
handlers.push(handler);
} else {
console.warn(`Unknown metadata format: ${name}. Supported formats: ${Array.from(this.handlers.keys()).join(', ')}`);
}
}
return handlers;
}
}
packages/meta-sort-core/src/logic/ProcessingStateManager.ts
+127
View File
@@ -0,0 +1,127 @@
/**
* ProcessingStateManager
*
* Tracks the state of file processing operations:
* - Pending: Files queued for processing
* - Processing: Files currently being processed
* - Processed: Recently completed files
*/
export interface FileProcessingState {
filePath: string;
state: 'pending' | 'processing' | 'processed';
queuedAt?: number;
startedAt?: number;
completedAt?: number;
processingTime?: number;
error?: string;
virtualPath?: string;
}
export interface ProcessingStatusSnapshot {
pending: FileProcessingState[];
processing: FileProcessingState[];
processed: FileProcessingState[];
totalPending: number;
totalProcessing: number;
totalProcessed: number;
}
export class ProcessingStateManager {
private pending: Map<string, FileProcessingState> = new Map();
private processing: Map<string, FileProcessingState> = new Map();
private processed: FileProcessingState[] = [];
private maxProcessedHistory = 100; // Keep last 100 processed files
/**
* Add a file to the pending queue
*/
addPending(filePath: string): void {
this.pending.set(filePath, {
filePath,
state: 'pending',
queuedAt: Date.now()
});
}
/**
* Move a file from pending to processing
*/
startProcessing(filePath: string): void {
const pendingState = this.pending.get(filePath);
this.pending.delete(filePath);
this.processing.set(filePath, {
filePath,
state: 'processing',
queuedAt: pendingState?.queuedAt,
startedAt: Date.now()
});
}
/**
* Move a file from processing to processed
*/
completeProcessing(filePath: string, processingTime: number, error?: string, virtualPath?: string): void {
const processingState = this.processing.get(filePath);
this.processing.delete(filePath);
const completedState: FileProcessingState = {
filePath,
state: 'processed',
queuedAt: processingState?.queuedAt,
startedAt: processingState?.startedAt,
completedAt: Date.now(),
processingTime,
error,
virtualPath
};
// Add to processed list (most recent first)
this.processed.unshift(completedState);
// Trim history if needed
if (this.processed.length > this.maxProcessedHistory) {
this.processed = this.processed.slice(0, this.maxProcessedHistory);
}
}
/**
* Remove a file from all states (e.g., when deleted)
*/
removeFile(filePath: string): void {
this.pending.delete(filePath);
this.processing.delete(filePath);
this.processed = this.processed.filter(f => f.filePath !== filePath);
}
/**
* Get a snapshot of the current processing state
*/
getSnapshot(): ProcessingStatusSnapshot {
return {
pending: Array.from(this.pending.values()),
processing: Array.from(this.processing.values()),
processed: this.processed.slice(0, 50), // Return only last 50 for API
totalPending: this.pending.size,
totalProcessing: this.processing.size,
totalProcessed: this.processed.length
};
}
/**
* Clear all states (useful for testing or reset)
*/
clear(): void {
this.pending.clear();
this.processing.clear();
this.processed = [];
}
/**
* Get the total number of files in all states
*/
getTotalFileCount(): number {
return this.pending.size + this.processing.size + this.processed.length;
}
}
packages/meta-sort-core/src/logic/UnifiedProcessingStateManager.ts
+354
View File
@@ -0,0 +1,354 @@
/**
* UnifiedProcessingStateManager
*
* Tracks files through the complete processing pipeline with 4 states:
* 1. Pending: Files discovered in filesystem (not yet processed)
* 2. Light Processing: Quick metadata extraction + midhash256 computation (~20ms)
* 3. Hash Processing: SHA-256 computation for compatibility (slow, background)
* 4. Done: Fully processed files
*
* Key change from old system:
* - midhash256 (fast) is computed in lightProcessing and becomes the permanent file ID
* - SHA-256 (slow) is computed in hashProcessing as additional metadata
* - Files become accessible in VFS immediately after lightProcessing completes
*/
import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
export type FileState = 'pending' | 'lightProcessing' | 'hashProcessing' | 'done';
export interface UnifiedFileState {
filePath: string;
state: FileState;
hash?: string; // midhash256 (set after lightProcessing)
virtualPath?: string;
error?: string;
// Retry tracking
retryCount?: number;
lastRetryAt?: number;
// Timestamps
discoveredAt?: number;
lightProcessingStartedAt?: number;
lightProcessingCompletedAt?: number;
hashProcessingStartedAt?: number;
hashProcessingCompletedAt?: number;
// Processing times
lightProcessingTime?: number;
hashProcessingTime?: number;
totalProcessingTime?: number;
}
export interface UnifiedProcessingSnapshot {
pending: UnifiedFileState[];
lightProcessing: UnifiedFileState[];
hashProcessing: UnifiedFileState[];
done: UnifiedFileState[];
totalPending: number;
totalLightProcessing: number;
totalHashProcessing: number;
totalDone: number;
totalFilesInVFS: number; // Total number of files accessible in the Virtual File System
// Configuration
fastQueueConcurrency?: number; // Number of fast queue workers
backgroundQueueConcurrency?: number; // Number of background queue workers
}
export class UnifiedProcessingStateManager {
private pending: Map<string, UnifiedFileState> = new Map();
private lightProcessing: Map<string, UnifiedFileState> = new Map();
private hashProcessing: Map<string, UnifiedFileState> = new Map();
private done: UnifiedFileState[] = [];
private maxDoneHistory = 100; // Keep last 100 completed files
private totalProcessedCount = 0; // Track total number of completed files (not limited)
/**
* Add a file to pending state (file discovered)
*/
addPending(filePath: string): void {
this.pending.set(filePath, {
filePath,
state: 'pending',
discoveredAt: Date.now()
});
}
/**
* Move file from pending to light processing
*/
startLightProcessing(filePath: string): void {
const pendingState = this.pending.get(filePath);
const existingState = this.lightProcessing.get(filePath);
this.pending.delete(filePath);
this.lightProcessing.set(filePath, {
filePath,
state: 'lightProcessing',
discoveredAt: pendingState?.discoveredAt || existingState?.discoveredAt,
lightProcessingStartedAt: Date.now(),
retryCount: existingState?.retryCount || 0
});
}
/**
* Complete light processing and move to hash processing queue
*/
completeLightProcessing(
filePath: string,
hash: string, // midhash256
virtualPath?: string,
error?: string
): void {
const lightState = this.lightProcessing.get(filePath);
this.lightProcessing.delete(filePath);
const now = Date.now();
const lightProcessingTime = lightState?.lightProcessingStartedAt
? now - lightState.lightProcessingStartedAt
: undefined;
// Record light processing time metric
if (lightProcessingTime !== undefined && !error) {
performanceMetrics.recordMetadataProcessing(lightProcessingTime);
}
if (error) {
// If error, mark as done with error
this.completeHashProcessing(filePath, hash, virtualPath, error);
return;
}
// Move to hash processing queue
this.hashProcessing.set(filePath, {
filePath,
state: 'hashProcessing',
hash,
virtualPath,
discoveredAt: lightState?.discoveredAt,
lightProcessingStartedAt: lightState?.lightProcessingStartedAt,
lightProcessingCompletedAt: now,
lightProcessingTime,
retryCount: lightState?.retryCount || 0
});
}
/**
* Start hash processing (SHA-256 computation)
*/
startHashProcessing(filePath: string): void {
const hashState = this.hashProcessing.get(filePath);
if (hashState) {
hashState.hashProcessingStartedAt = Date.now();
}
}
/**
* Complete hash processing and move to done
*/
completeHashProcessing(
filePath: string,
hash?: string,
virtualPath?: string,
error?: string
): void {
const hashState = this.hashProcessing.get(filePath);
this.hashProcessing.delete(filePath);
const now = Date.now();
const hashProcessingTime = hashState?.hashProcessingStartedAt
? now - hashState.hashProcessingStartedAt
: undefined;
const totalProcessingTime = hashState?.discoveredAt
? now - hashState.discoveredAt
: undefined;
// Record hash processing time metric
if (hashProcessingTime !== undefined && !error) {
performanceMetrics.recordHashProcessing(hashProcessingTime);
}
const doneState: UnifiedFileState = {
filePath,
state: 'done',
hash: hash || hashState?.hash,
virtualPath: virtualPath || hashState?.virtualPath,
error,
discoveredAt: hashState?.discoveredAt,
lightProcessingStartedAt: hashState?.lightProcessingStartedAt,
lightProcessingCompletedAt: hashState?.lightProcessingCompletedAt,
hashProcessingStartedAt: hashState?.hashProcessingStartedAt,
hashProcessingCompletedAt: now,
lightProcessingTime: hashState?.lightProcessingTime,
hashProcessingTime,
totalProcessingTime
};
// Add to done list (most recent first)
this.done.unshift(doneState);
// Increment total processed counter (tracks all files, not limited)
this.totalProcessedCount++;
// Trim history if needed
if (this.done.length > this.maxDoneHistory) {
this.done = this.done.slice(0, this.maxDoneHistory);
}
}
/**
* Remove a file from all states (e.g., when deleted)
*/
removeFile(filePath: string): void {
this.pending.delete(filePath);
this.lightProcessing.delete(filePath);
this.hashProcessing.delete(filePath);
this.done = this.done.filter(f => f.filePath !== filePath);
}
/**
* Get a snapshot of the current processing state
* @param vfsFileCount - Optional total number of files in the Virtual File System
* @param fastQueueConcurrency - Number of fast queue workers
* @param backgroundQueueConcurrency - Number of background queue workers
*/
getSnapshot(vfsFileCount: number = 0, fastQueueConcurrency?: number, backgroundQueueConcurrency?: number): UnifiedProcessingSnapshot {
// Limit returned arrays to prevent slow API responses with large queues
const MAX_ITEMS_PER_STATE = 100;
// Efficiently extract only the needed items without creating full arrays
const getPendingSample = (): UnifiedFileState[] => {
const result: UnifiedFileState[] = [];
let count = 0;
for (const item of this.pending.values()) {
if (count >= MAX_ITEMS_PER_STATE) break;
result.push(item);
count++;
}
return result;
};
const getLightProcessingSample = (): UnifiedFileState[] => {
const result: UnifiedFileState[] = [];
let count = 0;
for (const item of this.lightProcessing.values()) {
if (count >= MAX_ITEMS_PER_STATE) break;
result.push(item);
count++;
}
return result;
};
const getHashProcessingSample = (): UnifiedFileState[] => {
const result: UnifiedFileState[] = [];
let count = 0;
for (const item of this.hashProcessing.values()) {
if (count >= MAX_ITEMS_PER_STATE) break;
result.push(item);
count++;
}
return result;
};
return {
pending: getPendingSample(),
lightProcessing: getLightProcessingSample(),
hashProcessing: getHashProcessingSample(),
done: this.done.slice(0, 50), // Return only last 50 for API
totalPending: this.pending.size,
totalLightProcessing: this.lightProcessing.size,
totalHashProcessing: this.hashProcessing.size,
totalDone: this.totalProcessedCount, // Total count of all processed files (not limited to history)
totalFilesInVFS: vfsFileCount, // Total number of files accessible in the Virtual File System
fastQueueConcurrency,
backgroundQueueConcurrency
};
}
/**
* Get the state of a file in any processing queue
*/
getFileState(filePath: string): UnifiedFileState | undefined {
return this.pending.get(filePath) ||
this.lightProcessing.get(filePath) ||
this.hashProcessing.get(filePath);
}
/**
* Clear all states (useful for testing or reset)
*/
clear(): void {
this.pending.clear();
this.lightProcessing.clear();
this.hashProcessing.clear();
this.done = [];
this.totalProcessedCount = 0;
}
/**
* Get the total number of files in all states
*/
getTotalFileCount(): number {
return this.pending.size + this.lightProcessing.size + this.hashProcessing.size + this.done.length;
}
/**
* Retry processing for a file (move back to pending with incremented retry count)
*/
retryProcessing(filePath: string): boolean {
const lightState = this.lightProcessing.get(filePath);
const hashState = this.hashProcessing.get(filePath);
const state = lightState || hashState;
if (!state) {
return false;
}
const retryCount = (state.retryCount || 0) + 1;
this.lightProcessing.delete(filePath);
this.hashProcessing.delete(filePath);
this.pending.set(filePath, {
filePath,
state: 'pending',
discoveredAt: state.discoveredAt,
retryCount,
lastRetryAt: Date.now()
});
return true;
}
/**
* Get the retry count for a file
*/
getRetryCount(filePath: string): number {
const lightState = this.lightProcessing.get(filePath);
const hashState = this.hashProcessing.get(filePath);
const pendingState = this.pending.get(filePath);
return lightState?.retryCount || hashState?.retryCount || pendingState?.retryCount || 0;
}
/**
* Get all files currently in light processing
* Returns the internal Map for direct iteration
*/
getLightProcessingFiles(): Map<string, UnifiedFileState> {
return this.lightProcessing;
}
/**
* Get all files currently in hash processing
* Returns the internal Map for direct iteration
*/
getHashProcessingFiles(): Map<string, UnifiedFileState> {
return this.hashProcessing;
}
}
packages/meta-sort-core/src/logic/WatchedFileProcessor.ts
+620
View File
@@ -0,0 +1,620 @@
import {promises as fs} from 'fs';
import {dirname, join, parse} from 'path';
import {config} from "../config/EnvConfig.js";
import {FileProcessorInterface} from "@metazla/meta-hash";
import {MetaDataToFolderStruct} from "./MetaDataToFolderStruct.js";
import {DuplicateFinder, DuplicateResult} from "./DuplicateFinder.js";
import {FileType} from "@metazla/filename-tools";
import {existsAsync} from "@metazla/meta-hash";
import {FileAnalyzerInterface} from "./fileProcessor/FileAnalyzerInterface.js";
import {FileProcessorPiscina} from "./fileProcessor/FileProcessorPiscina.js";
import {renamingRule} from "../config/RenamingRule.js";
import {MetaMeshFormat} from "./MetaMeshFormat.js";
import {JellyfinAPI} from "../jellyfin/JellyfinAPI.js";
import {VirtualFileSystem} from "../api/VirtualFileSystem.js";
import {performanceMetrics} from "../metrics/PerformanceMetrics.js";
import {UnifiedProcessingStateManager} from "./UnifiedProcessingStateManager.js";
import type {IKVClient} from "../kv/IKVClient.js";
import PQueue from "p-queue";
import os from 'os';
import { TaskScheduler, createTaskScheduler, MetadataNodeKVStore } from "../plugin-engine/index.js";
import { initializePluginManager } from "./fileProcessor/FileProcessorPiscina.js";
import { from } from "@metazla/meta-interface";
import { stat } from "fs/promises";
import type { StreamingPipeline } from "./pipeline/StreamingPipeline.js";
export class WatchedFileProcessor implements FileProcessorInterface {
jellyfinAPI = new JellyfinAPI();
metaMeshFormat = new MetaMeshFormat();
metaDataToFolderStruct = new MetaDataToFolderStruct(renamingRule);
virtualFileSystem: VirtualFileSystem;
unifiedStateManager = new UnifiedProcessingStateManager();
//fileProcessor:FileAnalyzerInterface = new FileProcessor();//TODO only use for debug
fileProcessor:FileAnalyzerInterface = new FileProcessorPiscina();
duplicateFinder = new DuplicateFinder();
duplicateResult: DuplicateResult | null = null;
fileType = new FileType();
supported = ['video','subtitle','torrent'];
// Task-based plugin scheduler (replaces two-queue system)
private taskScheduler: TaskScheduler | null = null;
// Pre-processing queue for midhash256 computation (before plugins)
private preProcessQueue: PQueue;
// Reference to StreamingPipeline for queue status
private pipeline: StreamingPipeline | null = null;
constructor() {
// Always initialize VirtualFileSystem for FUSE API
this.virtualFileSystem = new VirtualFileSystem({
fileMode: 0o644,
directoryMode: config.CHMOD_FOR_NEW_FOLDER || 0o755
});
// Pass the unified state manager to the fileProcessor
if (this.fileProcessor instanceof FileProcessorPiscina) {
this.fileProcessor.setUnifiedStateManager(this.unifiedStateManager);
}
// Pre-processing queue for midhash256 computation
const cpuCount = os.cpus().length;
const preProcessConcurrency = config.MAX_WORKER_THREADS || cpuCount * 2;
this.preProcessQueue = new PQueue({concurrency: preProcessConcurrency, autoStart: true});
console.log(`WatchedFileProcessor: TaskScheduler-based system initialized`);
console.log(` - Pre-process queue: ${preProcessConcurrency} workers (midhash256)`);
console.log(` - TaskScheduler handles plugin execution (fast/background queues)`);
}
getVirtualFileSystem(): VirtualFileSystem {
return this.virtualFileSystem;
}
getUnifiedStateManager(): UnifiedProcessingStateManager {
return this.unifiedStateManager;
}
getDuplicateResult(): DuplicateResult | null {
return this.duplicateResult;
}
/**
* Get the KV client
*/
getKVClient(): IKVClient | null {
if (this.fileProcessor instanceof FileProcessorPiscina) {
return this.fileProcessor.getKVClient();
}
return null;
}
/**
* Set the KV client
* This is called by KVManager after successful connection
*/
setKVClient(client: IKVClient): void {
if (this.fileProcessor instanceof FileProcessorPiscina) {
this.fileProcessor.setKVClient(client);
}
}
/**
* Set the StreamingPipeline reference for queue status
* This is called by index.ts after creating the pipeline
*/
setPipeline(pipeline: StreamingPipeline): void {
this.pipeline = pipeline;
console.log('[WatchedFileProcessor] Pipeline connected for queue status');
}
/**
* Get current queue status for monitoring UI
* Uses StreamingPipeline queues which do the actual processing:
* Pipeline flow: validation → lightProcessing → hashProcessing
*
* Mapping to UI:
* - Pre-Process = validation (quick extension checks, high concurrency)
* - Fast Queue = lightProcessing (midhash256 + metadata extraction)
* - Background Queue = hashProcessing (SHA-256 full file hash)
*/
getQueueStatus() {
// Use StreamingPipeline stats if available (this is the actual processing engine)
if (this.pipeline) {
const pipelineStats = this.pipeline.getStats();
const queues = pipelineStats.queues;
// Map StreamingPipeline queues to the 3-queue architecture UI expects
// PQueue: .pending = currently running, .size = waiting in queue
// API expects: .pending = running, .size = waiting
const result = {
preProcessQueue: {
// validation = pre-process (quick extension checks)
size: queues.validation.size, // waiting in queue
pending: queues.validation.pending, // running jobs
isPaused: false
},
fastQueue: {
// fastQueue = fast queue (midhash256 + metadata)
running: queues.fastQueue.pending, // running jobs
pending: queues.fastQueue.size, // waiting in queue
size: queues.fastQueue.size,
isPaused: false
},
backgroundQueue: {
// backgroundQueue = background (SHA-256 computation)
running: queues.backgroundQueue.pending, // running jobs
pending: queues.backgroundQueue.size, // waiting in queue
size: queues.backgroundQueue.size,
isPaused: false
}
};
return result;
}
// Fallback: use TaskScheduler if available (for future plugin-based architecture)
if (this.taskScheduler) {
const status = this.taskScheduler.getQueueStatus();
return {
preProcessQueue: {
size: this.preProcessQueue.pending,
pending: this.preProcessQueue.size,
isPaused: this.preProcessQueue.isPaused
},
fastQueue: {
size: status.fast.ready,
pending: status.fast.pending,
running: status.fast.running,
isPaused: status.fast.isPaused
},
backgroundQueue: {
size: status.background.ready,
pending: status.background.pending,
running: status.background.running,
isPaused: status.background.isPaused
}
};
}
// Fallback: empty status
return {
preProcessQueue: { size: 0, pending: 0, isPaused: false },
fastQueue: { size: 0, pending: 0, running: 0, isPaused: false },
backgroundQueue: { size: 0, pending: 0, running: 0, isPaused: false }
};
}
/**
* Get the TaskScheduler instance
*/
getTaskScheduler(): TaskScheduler | null {
return this.taskScheduler;
}
async initialize() {
await this.fileProcessor.init();
// Initialize TaskScheduler with PluginManager using config values
const pluginManager = await initializePluginManager();
this.taskScheduler = createTaskScheduler(pluginManager, performanceMetrics, {
fastQueueConcurrency: config.FAST_QUEUE_CONCURRENCY,
backgroundQueueConcurrency: config.BACKGROUND_QUEUE_CONCURRENCY,
fastThresholdMs: config.FAST_THRESHOLD_MS,
minSamplesForMeasurement: config.MIN_SAMPLES_FOR_MEASUREMENT,
});
// Set up event listeners for VFS updates
this.setupTaskSchedulerEvents();
console.log(`[WatchedFileProcessor] TaskScheduler initialized with:`);
console.log(` - Fast queue concurrency: ${config.FAST_QUEUE_CONCURRENCY}`);
console.log(` - Background queue concurrency: ${config.BACKGROUND_QUEUE_CONCURRENCY}`);
console.log(` - Fast threshold: ${config.FAST_THRESHOLD_MS}ms`);
console.log(` - Min samples for measurement: ${config.MIN_SAMPLES_FOR_MEASUREMENT}`);
}
/**
* Set up TaskScheduler event listeners for VFS integration and Redis pub/sub
*/
private setupTaskSchedulerEvents(): void {
if (!this.taskScheduler) return;
// When a task completes, check if we should update VFS
this.taskScheduler.on('task:complete', (event: any) => {
const { task } = event;
// Update VFS when filename-parser completes (has basic metadata)
if (task.pluginId === 'filename-parser') {
this.updateVFSForFile(task.fileHash, task.filePath);
}
// Publish plugin completion event to Redis
this.publishPluginComplete(task.fileHash, task.pluginId, task.filePath);
});
// When all tasks for a file complete
this.taskScheduler.on('file:complete', (event: any) => {
const { fileHash, filePath } = event;
console.log(`[TaskScheduler] All plugins complete for ${filePath}`);
// Final VFS update with complete metadata
this.updateVFSForFile(fileHash, filePath);
// Update unified state manager
if (this.unifiedStateManager) {
this.unifiedStateManager.completeHashProcessing(filePath, fileHash);
}
});
}
/**
* Publish plugin completion event to Redis
* Channel: meta-sort:plugin:complete
*/
private async publishPluginComplete(fileHash: string, pluginId: string, filePath: string): Promise<void> {
const kvClient = this.getKVClient();
if (!kvClient || typeof (kvClient as any).publish !== 'function') {
return;
}
try {
const message = JSON.stringify({
fileHash,
pluginId,
filePath,
timestamp: Date.now()
});
await (kvClient as any).publish('meta-sort:plugin:complete', message);
} catch (error) {
// Silent fail - pub/sub is optional
}
}
/**
* Update VFS for a file with current metadata
*/
private updateVFSForFile(fileHash: string, filePath: string): void {
const metadata = this.fileProcessor.getDatabase().get(filePath);
if (metadata && metadata.cid_midhash256) {
try {
const virtualPath = this.metaDataToFolderStruct.renamingRule(metadata as any, '');
this.virtualFileSystem.addFile(virtualPath, filePath, metadata);
} catch (e) {
// Renaming rule may fail for incomplete metadata
}
}
}
markPending(filePath: string): void {
// STATE 1: Mark as pending when file is discovered (before queue)
this.unifiedStateManager.addPending(filePath);
}
async queueFile(filePath: string): Promise<void> {
// TaskScheduler-based processing
// Phase 1: Pre-process (midhash256 computation)
// Phase 2: Plugin execution via TaskScheduler (fast/background queues)
// Debug: log queue state before adding
console.log(`[DEBUG queueFile] Before add: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`);
await this.preProcessQueue.add(async () => {
console.log(`[DEBUG queueFile] Task started: size=${this.preProcessQueue.size}, pending=${this.preProcessQueue.pending}`);
try {
// STATE: Start LIGHT processing
if (this.unifiedStateManager) {
this.unifiedStateManager.startLightProcessing(filePath);
}
// STEP 1: Compute midhash256 (file identifier)
const fileStats = await stat(filePath);
const { computeMidHash256 } = await import('@metazla/meta-hash');
// Check cache first
const indexLine = (this.fileProcessor as any).indexManager?.getCidForFile(
filePath,
fileStats.size,
fileStats.mtime.toISOString()
);
let midHash256: string;
if (indexLine?.cid_midhash256) {
midHash256 = indexLine.cid_midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
const computeStart = performance.now();
midHash256 = await computeMidHash256(filePath);
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
performanceMetrics.recordCacheMiss('midhash256');
}
// STEP 2: Initialize metadata with midhash256
const metadata: any = {
cid_midhash256: midHash256,
filePath,
processingStatus: 'processing'
};
// Store in local database
this.fileProcessor.getDatabase().set(filePath, metadata);
// STEP 3: Create KV store for plugins
const metaDataNode = from(metadata);
const kv = new MetadataNodeKVStore(metaDataNode);
// STEP 4: Create and enqueue plugin tasks
if (this.taskScheduler) {
const tasks = this.taskScheduler.createTasksForFile(filePath, midHash256, kv);
this.taskScheduler.enqueueTasks(tasks);
}
// STATE: Light processing setup complete
if (this.unifiedStateManager) {
this.unifiedStateManager.completeLightProcessing(
filePath,
midHash256,
undefined // Virtual path will be computed by plugins
);
}
} catch (error) {
console.error(`Failed pre-processing for ${filePath}:`, error);
// Mark processing as failed
if (this.unifiedStateManager) {
this.unifiedStateManager.completeLightProcessing(
filePath,
'',
undefined,
error instanceof Error ? error.message : String(error)
);
}
}
});
}
async processFile(current: number, queueSize: number, filePath: string): Promise<void> {
// This method is now a wrapper for queueFile for backwards compatibility
// In the new single-phase model, queueFile handles everything
await this.queueFile(filePath);
}
async finalize(): Promise<void> {
try {
console.log(`Virtual filesystem update start`);
const start = performance.now();
if (this.fileProcessor.getDatabase().size === 0) {
console.log(`No files to process`);
return;
}
let startTime = performance.now();
const newDuplicateResult = await this.duplicateFinder.findDuplicates(this.fileProcessor.getDatabase());//find duplicates and removes them to avoid conflict
const duplicateTime = Math.ceil(performance.now() - startTime);
const totalDuplicates = newDuplicateResult.hashDuplicates.length + newDuplicateResult.titleDuplicates.length;
console.log(`Duplicate find took ${duplicateTime}ms - found ${newDuplicateResult.hashDuplicates.length} hash groups, ${newDuplicateResult.titleDuplicates.length} title groups`);
// Only update duplicateResult if we found new duplicates, or if it's the first run
if (totalDuplicates > 0 || this.duplicateResult === null) {
this.duplicateResult = newDuplicateResult;
}
performanceMetrics.recordDuplicateDetection(duplicateTime, totalDuplicates);
startTime = performance.now();
console.log(`Generating virtual structure for ${this.fileProcessor.getDatabase().size} files`);
const virtualStructure = this.metaDataToFolderStruct.generateVirtualStructure(this.fileProcessor.getDatabase());
const virtualStructureTime = Math.ceil(performance.now() - startTime);
console.log(`Virtual structure generation took ${virtualStructureTime}ms`);
performanceMetrics.recordVirtualStructureGeneration(virtualStructureTime);
// Build VirtualFileSystem for FUSE/WebDAV API
startTime = performance.now();
this.virtualFileSystem.buildFromComputed(virtualStructure, this.fileProcessor.getDatabase());
const vfsBuildTime = Math.ceil(performance.now() - startTime);
console.log(`VirtualFileSystem build took ${vfsBuildTime}ms`);
performanceMetrics.recordVFSUpdate(vfsBuildTime);
const stats = this.virtualFileSystem.getStats();
console.log(`VirtualFileSystem: ${stats.fileCount} files (${stats.metaFileCount} .meta), ${stats.directoryCount} directories, ${(stats.totalSize / 1024 / 1024).toFixed(2)} MB`);
if(this.jellyfinAPI.jellyApiAvailable()) {
await this.jellyfinAPI.refreshLibrary();
console.log(`Jellyfin refreshed`);
}
console.log(`Virtual filesystem update took ${Math.ceil(performance.now() - start)}ms`);
} catch (error) {
console.error(error);
}
}
async canProcessFile(filePath: string) {
const fileTypeResult = await this.fileType.getFileType(filePath);
//check if filetype is one of the suported ones
return this.supported.includes(fileTypeResult);
}
async deleteFile(filePath: string) {
this.fileProcessor.deleteFile(filePath);
this.unifiedStateManager.removeFile(filePath);
}
/**
* Clean up stale entries from in-memory database for scanned directories
*
* This is called after discovery scans a folder. It removes files from
* in-memory DB that belong to the scanned folder but were not seen during
* the scan (meaning they no longer exist on disk).
*
* Design Philosophy:
* - Discovery is the source of truth for file existence
* - Empty/missing folders → remove all child paths from memory
* - KV keeps everything (distributed metadata archive)
*
* @param scannedFolders - Array of folder paths that were completely scanned
* @param discoveredFiles - Set of file paths that were found during scan
*/
async cleanupStaleEntries(scannedFolders: string[], discoveredFiles: Set<string>): Promise<void> {
console.log(`[Cleanup] Starting cleanup for ${scannedFolders.length} scanned folders...`);
const startTime = performance.now();
let removedCount = 0;
const db = this.fileProcessor.getDatabase();
const removedHashIds: string[] = []; // Collect hashIds for Stremio notification
for (const folder of scannedFolders) {
const normalizedFolder = folder.endsWith('/') ? folder : folder + '/';
// Find all in-memory files that belong to this folder
const filesToCheck: string[] = [];
for (const [filePath] of db.entries()) {
if (filePath.startsWith(normalizedFolder) || filePath === folder) {
filesToCheck.push(filePath);
}
}
// Remove files that weren't discovered (no longer exist)
for (const filePath of filesToCheck) {
if (!discoveredFiles.has(filePath)) {
// File is in memory but wasn't seen during scan → removed/offline
const metadata = db.get(filePath);
// Collect hashId before deletion (for Stremio notification)
if (metadata && metadata.cid_midhash256) {
removedHashIds.push(metadata.cid_midhash256);
}
db.delete(filePath);
this.unifiedStateManager.removeFile(filePath);
removedCount++;
}
}
}
const elapsed = Math.ceil(performance.now() - startTime);
console.log(`[Cleanup] Removed ${removedCount} stale entries in ${elapsed}ms`);
// If we removed files, rebuild VFS to update structure
if (removedCount > 0) {
console.log('[Cleanup] Rebuilding VFS after cleanup...');
await this.finalize();
// Notify Stremio addon to remove videos (incremental updates like FUSE)
if (removedHashIds.length > 0) {
console.log(`[Cleanup] Notifying Stremio to remove ${removedHashIds.length} videos...`);
for (const hashId of removedHashIds) {
this.notifyStremioRemove(hashId).catch(() => {
// Silent fail - Stremio addon may not be running
});
}
}
}
}
/**
* Rebuild VirtualFileSystem from KV data
* This is called on startup to restore the VFS from persistent storage
*
* Fast-load strategy: Load all metadata from KV WITHOUT disk validation.
* Discovery will later validate file existence and clean up stale entries.
*
* Design Philosophy:
* - KV = distributed metadata archive (permanent record)
* - In-memory DB = current VFS state (only present files)
* - Discovery = source of truth (validates and cleans)
*/
async rebuildVFSFromKV(): Promise<void> {
const kvClient = this.getKVClient();
if (!kvClient) {
console.warn('[VFS Rebuild] KV client not available, skipping VFS rebuild');
return;
}
try {
console.log('[VFS Rebuild] Starting fast-load from KV...');
const startTime = performance.now();
// Get all hash IDs from KV
const hashIds = await kvClient.getAllHashIds();
console.log(`[VFS Rebuild] Found ${hashIds.length} files in KV`);
if (hashIds.length === 0) {
console.log('[VFS Rebuild] No files in KV, VFS remains empty');
return;
}
// Batch fetch all metadata in parallel (no disk validation)
console.log('[VFS Rebuild] Fetching metadata in parallel...');
const metadataList = await Promise.all(
hashIds.map(async (hashId) => {
try {
return await kvClient.getMetadataFlat(hashId);
} catch (error) {
console.error(`[VFS Rebuild] Failed to load metadata for ${hashId}:`, error);
return null;
}
})
);
// Populate in-memory database without disk validation
let loadedCount = 0;
for (const metadata of metadataList) {
if (metadata && metadata.filePath) {
// Mark as unverified - discovery will validate later
(metadata as any)._lastVerified = 0;
this.fileProcessor.getDatabase().set(metadata.filePath, metadata);
loadedCount++;
}
}
console.log(`[VFS Rebuild] Loaded ${loadedCount}/${hashIds.length} files into memory (unverified)`);
// Build VFS immediately - may contain stale entries until discovery validates
if (this.fileProcessor.getDatabase().size > 0) {
await this.finalize();
console.log(`[VFS Rebuild] VFS rebuilt in ${Math.ceil(performance.now() - startTime)}ms`);
console.log('[VFS Rebuild] Discovery will validate file existence in background');
} else {
console.log('[VFS Rebuild] No files loaded, VFS remains empty');
}
} catch (error) {
console.error('[VFS Rebuild] Error rebuilding VFS from KV:', error);
}
}
/**
* Notify Stremio addon to remove video (incremental update like FUSE)
*/
private async notifyStremioRemove(hashId: string): Promise<void> {
try {
const response = await fetch('http://localhost:7000/remove', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ hashId }),
signal: AbortSignal.timeout(5000) // 5 second timeout
});
if (!response.ok) {
// Silent fail - Stremio may not be running
return;
}
} catch (error: any) {
// Silent fail - Stremio addon may not be running or network issue
}
}
/**
* Shutdown: clean up resources
*/
async shutdown(): Promise<void> {
console.log('[WatchedFileProcessor] Shutting down...');
// Clear pre-process queue
this.preProcessQueue.clear();
// Shutdown TaskScheduler
if (this.taskScheduler) {
await this.taskScheduler.shutdown();
}
console.log('[WatchedFileProcessor] Shutdown complete');
}
}
packages/meta-sort-core/src/logic/fileProcessor/FileAnalyzerInterface.ts
+37
View File
@@ -0,0 +1,37 @@
import {HashMeta} from "@metazla/meta-interface";
import {FileMetadata} from "../../types/FileMetadata.js";
export interface FileAnalyzerInterface {
/**
* Full file processing - computes midhash256 and extracts all metadata
* Since midhash256 is instant (< 1s), files appear in VFS immediately with permanent ID
*/
processFile(filePath: string, current: number, queueSize: number):Promise<void>;
/**
* Light processing phase - fast metadata extraction + midhash256
* File becomes accessible in VFS with permanent ID after this phase
*/
processLightPhase(filePath: string, current: number, queueSize: number):Promise<void>;
/**
* Hash processing phase - background hash computation
* Computes full hashes (SHA-256, SHA-1, MD5, CRC32) and updates KV
*/
processHashPhase(filePath: string, current: number, queueSize: number):Promise<void>;
/**
* Get the metadata database
*/
getDatabase<T extends FileMetadata>(): Map<string, T>;
/**
* Delete a file from the database
*/
deleteFile(filePath: string): Promise<void> | void;
/**
* Initialize the file analyzer
*/
init():Promise<void>;
}
packages/meta-sort-core/src/logic/fileProcessor/FileProcessor.ts
+92
View File
@@ -0,0 +1,92 @@
import {HashIndexManager} from "@metazla/meta-hash";
import {config} from "../../config/EnvConfig.js";
import {stat} from "fs/promises";
import {targetHash} from "../../config/TargetHash.js";
import {HashMeta} from "@metazla/meta-interface";
import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js";
import {FileProcessorInternal} from "./FileProcessorInternal.js";
import {globalHashIndex} from "./SimpleFileToCid.js";
import {FileMetadata} from "../../types/FileMetadata.js";
export class FileProcessor implements FileAnalyzerInterface{
database = new Map<string, FileMetadata>();//Map<filePath,FileMetadata>
indexManager = globalHashIndex;
fileProcessor = new FileProcessorInternal();
constructor() {
}
async init() {
await this.indexManager.init(true);
console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry');
}
/**
* Quick metadata processing - extracts metadata without computing content hash
* Note: Plugin processing is now handled by container plugins via TaskScheduler
*/
async processFileQuick(filePath: string): Promise<FileMetadata> {
// Initialize metadata with processing status
const metadata: FileMetadata = {
processingStatus: 'processing'
};
// Add to database immediately
// Plugin processing will be done asynchronously by container plugins
this.database.set(filePath, metadata);
return metadata;
}
getDatabase<T>(): Map<string, T> {
return this.database as Map<string, T>;
}
async processFile(filePath: string, current: number, queueSize: number) {
try {
if (this.database.has(filePath)) {
return;
}
const stats = await stat(filePath);
let indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
let hash: HashMeta = {};
if(indexLine) {
for (const hashKey of targetHash) {
if (indexLine[hashKey]) {
hash[hashKey] = indexLine[hashKey];
}
}
}
// --------------------------------------------
// Hash logic compute and get the hash - use cache etc...
// --------------------------------------------
const result = await this.fileProcessor.processFile(filePath,hash);
//const result = await this.fileProcessor.processFile(filePath,params.hash);
//save the metadata to the cache or disk
this.database.set(filePath, result.metadata);
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), result.metadata);
//console.log(`File [${current}/${queueSize}] ${filePath} took ${result.time}ms`);
} catch (e) {
console.error(`Error processing file [No Thread] ${filePath}:`, e);
}
}
async processLightPhase(filePath: string, current: number, queueSize: number) {
// Legacy FileProcessor doesn't support split phases - just use processFile
await this.processFile(filePath, current, queueSize);
}
async processHashPhase(filePath: string, current: number, queueSize: number) {
// Legacy FileProcessor doesn't support split phases - no-op
// (processFile already did everything)
}
deleteFile(filePath: string) {
this.database.delete(filePath);
}
}
packages/meta-sort-core/src/logic/fileProcessor/FileProcessorInternal.ts
+84
View File
@@ -0,0 +1,84 @@
import { HashComputerFile } from "@metazla/meta-hash";
import { targetHash } from "../../config/TargetHash.js";
import { DebugMeta, HashMeta } from "@metazla/meta-interface";
import { MetaMeshFormat } from "../MetaMeshFormat.js";
type Local = HashMeta & DebugMeta;
export class FileProcessorInternal {
hashComputer = new HashComputerFile(targetHash);
metaMeshFormat = new MetaMeshFormat();
async processFile(filePath: string, hash: HashMeta): Promise<{
time: number,
metadata: HashMeta,
hashTimings?: Record<string, number>,
pluginTimings?: Record<string, number>
}> {
const start = performance.now();
const hashTimings: Record<string, number> = {};
const pluginTimings: Record<string, number> = {};
// --------------------------------------------
// Hash logic compute and get the hash - use cache etc...
// --------------------------------------------
const metaDataPath = `${filePath}.meta`;
let metadata: HashMeta = await this.metaMeshFormat.read(metaDataPath) || {}; // Read the reference file
// try to update the sha values from the index
if (hash) {
for (let key in hash) {
if (!metadata[key] && hash[key]) {
metadata[key] = hash[key];
}
}
}
// Time consuming - compute hashes and track timing
try {
// Determine which hashes need to be computed
const neededHashes = this.hashComputer['targetHash'].filter(hashName => !metadata[hashName]);
if (neededHashes.length > 0) {
// Measure total time for hash computation
const hashStart = performance.now();
// Compute all needed hashes at once (more efficient than individual)
await this.hashComputer.computeMissingHash(filePath, metadata);
const totalHashTime = Math.ceil(performance.now() - hashStart);
// Distribute the time evenly among computed hashes
// This is an approximation - actual individual hash times may vary
const avgTimePerHash = totalHashTime / neededHashes.length;
for (const hashAlgorithm of neededHashes) {
hashTimings[hashAlgorithm] = Math.ceil(avgTimePerHash);
}
}
} catch (hashError) {
console.error(`[Worker] Hash computation failed for ${filePath}:`, hashError);
throw new Error(`Hash computation failed: ${hashError instanceof Error ? hashError.message : String(hashError)}`);
}
// Plugin processing is now handled by container plugins via TaskScheduler
// This internal processor only handles hash computation
// Validate all required hashes are present before returning
const missingHashesAfterCompute: string[] = [];
for (const hashKey of this.hashComputer['targetHash']) {
if (!metadata[hashKey]) {
missingHashesAfterCompute.push(hashKey);
}
}
if (missingHashesAfterCompute.length > 0) {
throw new Error(`[Worker] Hash validation failed - missing hashes: ${missingHashesAfterCompute.join(', ')}`);
}
return {
time: Math.ceil(performance.now() - start),
metadata,
hashTimings,
pluginTimings
};
}
}
packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts
+594
View File
@@ -0,0 +1,594 @@
import {Piscina} from "piscina";
import {Input, Output} from "./worker.js";
import {HashIndexManager} from "@metazla/meta-hash";
import {config} from "../../config/EnvConfig.js";
import {stat} from "fs/promises";
// Note: Full hashes are now computed by the full-hash container plugin
import {HashMeta} from "@metazla/meta-interface";
import {FileAnalyzerInterface} from "./FileAnalyzerInterface.js";
import {globalHashIndex} from "./SimpleFileToCid.js";
import {performanceMetrics} from "../../metrics/PerformanceMetrics.js";
import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js";
import {renamingRule} from "../../config/RenamingRule.js";
import {FileMetadata, ProcessingStatus} from "../../types/FileMetadata.js";
import { PluginManager } from "../../plugin-engine/index.js";
import type {IKVClient} from "../../kv/IKVClient.js";
import * as path from 'path';
import { fileURLToPath } from 'url';
import os from 'os';
// Get the directory of this file to resolve plugin paths
const __filename = fileURLToPath(import.meta.url);
const __dirname = path.dirname(__filename);
// Singleton PluginManager instance
let pluginManagerInstance: PluginManager | null = null;
let pluginManagerInitPromise: Promise<void> | null = null;
/**
* Get the plugin manager instance (for API access)
* Returns null if not yet initialized
*/
export function getPluginManagerInstance(): PluginManager | null {
return pluginManagerInstance;
}
/**
* Set the plugin manager instance (called by container initialization)
*/
export function setPluginManagerInstance(manager: PluginManager): void {
pluginManagerInstance = manager;
}
/**
* Initialize the plugin manager on startup
* Note: With container-based plugins, the PluginManager is initialized
* by the ContainerManager. This function creates a basic instance
* for state management if container plugins are not yet ready.
*/
export async function initializePluginManager(): Promise<PluginManager> {
if (pluginManagerInstance) {
return pluginManagerInstance;
}
// Create basic plugin manager for state management
const statePath = path.join(config.CACHE_FOLDER_PATH, 'plugins.json');
const cacheDir = path.join(config.CACHE_FOLDER_PATH, 'plugin-cache');
console.log(`[PluginManager] Initializing with:`);
console.log(` - State path: ${statePath}`);
console.log(` - Cache dir: ${cacheDir}`);
pluginManagerInstance = new PluginManager({
statePath,
cacheDir,
});
console.log(`[PluginManager] Basic instance created (waiting for container plugins)`);
return pluginManagerInstance;
}
export class FileProcessorPiscina implements FileAnalyzerInterface{
private piscina: Piscina;
database = new Map<string, FileMetadata>();//Map<filePath,FileMetadata>
indexManager = globalHashIndex;
private unifiedStateManager: UnifiedProcessingStateManager | null = null;
// Unified KV client (Redis)
private kvClient: IKVClient | null = null;
// Track running hash workers for abort capability
private runningHashTasks = new Map<string, AbortController>();
constructor() {
// Construct the URL for the current module
let distFolder = import.meta.dirname;
distFolder = distFolder.replace('src', 'dist');
distFolder = distFolder + "/worker.js";
const workerUrl = new URL(distFolder,"file://").href;
// Set maxThreads: Smart default uses half CPU cores to avoid starving the system
// (especially important for network storage + HDD scenarios)
const cpuCount = os.cpus().length;
const defaultWorkers = Math.max(1, Math.floor(cpuCount / 2));
const maxThreads = config.MAX_WORKER_THREADS || defaultWorkers;
this.piscina = new Piscina({
maxThreads,
//maxThreads: 1,//TODO note for DEBUG must be set to 1 to avoid memory issues
filename:workerUrl
});
console.log(`Piscina worker pool initialized with ${maxThreads} threads (cpuCount=${cpuCount}, defaultWorkers=${defaultWorkers})`);
}
async init() {
await this.indexManager.init(true);
console.log('Index loaded:', this.indexManager.getCacheSize(), 'entry');
}
/**
* Single-phase metadata processing - computes midhash256 immediately and extracts metadata
* Files appear in VFS instantly with their permanent midhash256 ID
*
* @param filePath - Path to the file to process
* @param current - Current file number (for progress reporting)
* @param queueSize - Total queue size (for progress reporting)
* @returns FileMetadata with midhash256 and all metadata
*/
setUnifiedStateManager(manager: UnifiedProcessingStateManager): void {
this.unifiedStateManager = manager;
}
getDatabase<T>(): Map<string, T> {
return this.database as Map<string, T>;
}
/**
* Abort all running hash workers immediately (for priority enforcement)
* @returns Number of workers aborted
*/
abortAllHashWorkers(): number {
const abortedCount = this.runningHashTasks.size;
for (const [filePath, controller] of this.runningHashTasks.entries()) {
controller.abort();
console.log(`[Abort] Hash worker aborted: ${filePath}`);
}
this.runningHashTasks.clear();
return abortedCount;
}
/**
* Gracefully shutdown - close KV client
*/
async shutdown(): Promise<void> {
if (this.kvClient) {
console.log('[FileProcessor] Closing KV client...');
await this.kvClient.close();
console.log('[FileProcessor] KV client closed');
}
}
async processFile(filePath: string, current: number, queueSize: number) {
const startTime = Date.now();
try {
// STATE: Start LIGHT processing (fast: midhash256 + metadata extraction)
if (this.unifiedStateManager) {
this.unifiedStateManager.startLightProcessing(filePath);
}
const stats = await stat(filePath);
// STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files)
let midHash256: string | undefined;
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if (indexLine && indexLine.cid_midhash256) {
// Cache hit! Use cached midhash256 (no network I/O needed)
midHash256 = indexLine.cid_midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
const computeStart = performance.now();
const { computeMidHash256 } = await import('@metazla/meta-hash');
midHash256 = await computeMidHash256(filePath);
const computeTime = Math.ceil(performance.now() - computeStart);
// Record midhash256 computation time (excludes cache hits)
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
performanceMetrics.recordCacheMiss('midhash256');
}
// Initialize metadata with midhash256
const metadata: FileMetadata = {
cid_midhash256: midHash256,
processingStatus: 'processing'
};
// STEP 2: Plugin metadata extraction is now handled by container plugins
// The TaskScheduler will dispatch tasks to container plugins asynchronously
// Basic metadata processing continues below
// Generate virtualPath from metadata (needed for VFS access)
let virtualPath: string | undefined;
try {
virtualPath = renamingRule(metadata as any, '');
} catch (e) {
// If renamingRule fails, virtualPath remains undefined
}
// STEP 3: Collision detection and store preliminary metadata in KV
let collisionDetected = false;
if (this.kvClient) {
try {
const hashId = midHash256; // Use midhash256 as primary ID
if (hashId) {
// COLLISION DETECTION: Check if this midhash256 already exists
const existingFilePath = await this.kvClient.getMetadata(hashId, 'filePath');
if (existingFilePath && existingFilePath !== filePath) {
// COLLISION DETECTED: Different file with same midhash256!
collisionDetected = true;
console.warn(`⚠️ MIDHASH256 COLLISION DETECTED!`);
console.warn(` Hash: ${hashId}`);
console.warn(` Existing file: ${existingFilePath}`);
console.warn(` New file: ${filePath}`);
console.warn(` Action: Skipping VFS addition, forcing full hash computation`);
// Do NOT store in KV with midhash256
// File will be processed with full SHA-256 instead (see below)
} else {
// No collision or same file - safe to store
const finalMetadata = {
...metadata,
filePath
};
// Store using nested key architecture
await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
}
}
} catch (metaError) {
console.error(`Failed to store metadata in KV:`, metaError);
}
}
// STATE: Light processing complete, move to HASH processing queue
// If collision detected, skip VFS addition (virtualPath = undefined)
if (this.unifiedStateManager) {
if (collisionDetected) {
// Collision: Do NOT complete light processing with virtualPath
// This prevents file from being added to VFS
this.unifiedStateManager.completeLightProcessing(
filePath,
midHash256,
undefined // No virtualPath = not added to VFS
);
} else {
// No collision: File becomes accessible in VFS
this.unifiedStateManager.completeLightProcessing(
filePath,
midHash256,
virtualPath
);
}
// Start hash processing phase immediately
this.unifiedStateManager.startHashProcessing(filePath);
}
// STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the
// container plugin "full-hash" in background queue - not here.
// Only midhash256 is computed in-process for fast file identification.
// Mark processing as complete
metadata.processingStatus = 'complete';
this.database.set(filePath, metadata);
// STEP 5: Update metadata in KV with full hashes
if (this.kvClient) {
try {
// Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
let hashId: string;
if (collisionDetected && metadata['cid_sha2-256']) {
// Collision: Use full SHA-256 as primary ID
hashId = metadata['cid_sha2-256'];
console.log(` Using SHA-256 as primary ID for collision: ${hashId}`);
} else {
// Normal: Use midhash256 as primary ID
hashId = midHash256;
}
if (hashId) {
// Prepare metadata with filePath added
const finalMetadata = {
...metadata,
filePath
};
// Store using nested key architecture
await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
}
} catch (metaError) {
console.error(`Failed to store metadata in KV:`, metaError);
}
}
// STATE: Hash processing complete, move to done
if (this.unifiedStateManager) {
this.unifiedStateManager.completeHashProcessing(
filePath,
midHash256,
virtualPath
);
}
} catch (e) {
console.error(`Error processing file ${filePath}:`, e);
// Mark processing as failed (complete hash processing with error)
if (this.unifiedStateManager) {
this.unifiedStateManager.completeHashProcessing(
filePath,
undefined,
undefined,
e instanceof Error ? e.message : String(e)
);
}
}
}
/**
* Light Processing Phase - Fast processing to make file accessible in VFS
* - Compute midhash256 (< 1s)
* - Extract metadata (filename, FFmpeg analysis)
* - Store in KV
* - File becomes accessible in VFS with midhash256 ID
*
* @deprecated Use TaskScheduler-based processing via WatchedFileProcessor.queueFile() instead.
* This method is kept for backward compatibility with StreamingPipeline.
*/
async processLightPhase(filePath: string, current: number, queueSize: number) {
try {
// STATE: Start LIGHT processing
if (this.unifiedStateManager) {
this.unifiedStateManager.startLightProcessing(filePath);
}
const stats = await stat(filePath);
// STEP 1: Check cache for midhash256 FIRST (avoids network I/O on unchanged files)
let midHash256: string | undefined;
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if (indexLine && indexLine.cid_midhash256) {
// Cache hit! Use cached midhash256 (no network I/O needed)
midHash256 = indexLine.cid_midhash256;
performanceMetrics.recordCacheHit('midhash256');
} else {
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
const computeStart = performance.now();
const { computeMidHash256 } = await import('@metazla/meta-hash');
midHash256 = await computeMidHash256(filePath);
const computeTime = Math.ceil(performance.now() - computeStart);
// Record midhash256 computation time (excludes cache hits)
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
performanceMetrics.recordCacheMiss('midhash256');
}
// Initialize metadata with midhash256
const metadata: FileMetadata = {
cid_midhash256: midHash256,
processingStatus: 'processing'
};
// STEP 2: Plugin metadata extraction is now handled by container plugins
// The TaskScheduler will dispatch tasks to container plugins asynchronously
// Generate virtualPath from metadata (needed for VFS access)
let virtualPath: string | undefined;
try {
virtualPath = renamingRule(metadata as any, '');
} catch (e) {
// If renamingRule fails, virtualPath remains undefined
}
// STEP 3: Store metadata in KV (file becomes accessible in VFS)
if (this.kvClient) {
try {
const hashId = midHash256; // Use midhash256 as primary ID
if (hashId) {
// Prepare metadata with filePath added
const finalMetadata = {
...metadata,
filePath
};
// Store using nested key architecture
await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
}
} catch (metaError) {
console.error(`Failed to store metadata in KV:`, metaError);
}
}
// Store in local database
this.database.set(filePath, metadata);
// STATE: Light processing complete, move to HASH processing queue
// File is now accessible in VFS with midhash256 ID
if (this.unifiedStateManager) {
this.unifiedStateManager.completeLightProcessing(
filePath,
midHash256,
virtualPath
);
}
} catch (e) {
console.error(`Error in light processing for ${filePath}:`, e);
// Mark processing as failed
if (this.unifiedStateManager) {
this.unifiedStateManager.completeLightProcessing(
filePath,
'',
undefined,
e instanceof Error ? e.message : String(e)
);
}
throw e;
}
}
/**
* Hash Processing Phase - Background hash computation
* - Compute full hashes (SHA-256, SHA-1, MD5, CRC32)
* - Update KV with additional hash metadata
* - Mark processing as complete
* - Supports abortion for priority enforcement
*
* @deprecated Use the full-hash plugin via TaskScheduler instead.
* This method is kept for backward compatibility with StreamingPipeline.
*/
async processHashPhase(filePath: string, current: number, queueSize: number) {
// Create abort controller for this task
const abortController = new AbortController();
this.runningHashTasks.set(filePath, abortController);
try {
// STATE: Start HASH processing
if (this.unifiedStateManager) {
this.unifiedStateManager.startHashProcessing(filePath);
}
const stats = await stat(filePath);
const metadata = this.database.get(filePath);
if (!metadata) {
throw new Error('Metadata not found - light processing may have failed');
}
const midHash256 = metadata.cid_midhash256;
if (!midHash256) {
throw new Error('midhash256 not found in metadata');
}
// Generate virtualPath (should already be done but ensure it's available)
let virtualPath: string | undefined;
try {
virtualPath = renamingRule(metadata as any, '');
} catch (e) {
// If renamingRule fails, virtualPath remains undefined
}
// Check if collision was detected during light processing
// If the midhash256 key exists in KV with a different file path, collision occurred
let collisionDetected = false;
if (this.kvClient) {
try {
const existingFilePath = await this.kvClient.getMetadata(midHash256, 'filePath');
if (existingFilePath && existingFilePath !== filePath) {
collisionDetected = true;
console.log(` Collision detected in hash phase for ${filePath}`);
}
} catch (e) {
// Ignore errors - if key doesn't exist, no collision
}
}
// Check if aborted before processing
if (abortController.signal.aborted) {
throw new Error('AbortError');
}
// STEP 4: Full hashes (SHA-256, SHA-1, MD5, CRC32) are computed by the
// container plugin "full-hash" in background queue - not here.
// Only midhash256 is computed in-process for fast file identification.
// Mark processing as complete
metadata.processingStatus = 'complete';
this.database.set(filePath, metadata);
// STEP 5: Update metadata in KV (full hashes will be added later by container plugin)
if (this.kvClient) {
try {
// Use midhash256 as primary ID
// Note: SHA-256 is computed later by the full-hash container plugin
const hashId = midHash256;
if (hashId) {
// Prepare metadata with filePath added
const finalMetadata = {
...metadata,
filePath
};
// Store using nested key architecture
await this.kvClient.setMetadataFlat(hashId, finalMetadata, ['processingStatus']);
}
} catch (metaError) {
console.error(`Failed to store metadata in KV:`, metaError);
}
}
// STATE: Hash processing complete, move to done
if (this.unifiedStateManager) {
this.unifiedStateManager.completeHashProcessing(
filePath,
midHash256,
virtualPath
);
}
} catch (e) {
// Handle abort errors gracefully
if (e instanceof Error && (e.message === 'AbortError' || e.name === 'AbortError')) {
// Task was aborted for priority - re-queue it
console.log(`[Abort] Hash processing aborted for ${filePath} - will retry when idle`);
// Don't mark as failed, just remove from running tasks
// The task will remain in hash queue and retry later
if (this.unifiedStateManager) {
// Move back to hash processing start (will retry)
this.unifiedStateManager.startHashProcessing(filePath);
}
} else {
console.error(`Error in hash processing for ${filePath}:`, e);
// Mark hash processing as failed for real errors
if (this.unifiedStateManager) {
this.unifiedStateManager.completeHashProcessing(
filePath,
undefined,
undefined,
e instanceof Error ? e.message : String(e)
);
}
}
throw e;
} finally {
// Always remove from running tasks
this.runningHashTasks.delete(filePath);
}
}
async deleteFile(filePath: string) {
const metadata = this.database.get(filePath);
this.database.delete(filePath);
// Delete from KV if configured (using nested key architecture)
if (this.kvClient && metadata) {
try {
const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
// Delete all properties for this file using nested key architecture
if (hashId) {
const deletedCount = await this.kvClient.deleteMetadataFlat(hashId);
console.log(`Metadata deleted from KV: /file/${hashId} (${deletedCount} keys)`);
}
} catch (metaError) {
// Log error but don't fail the entire process
console.error(`Failed to delete metadata from KV:`, metaError);
}
}
}
/**
* Set the KV client
* This is called by KVManager after successful connection
*/
setKVClient(client: IKVClient): void {
this.kvClient = client;
console.log('[FileProcessor] KV client set');
}
/**
* Get the KV client
*/
getKVClient(): IKVClient | null {
return this.kvClient;
}
}
packages/meta-sort-core/src/logic/fileProcessor/SimpleFileToCid.ts
+31
View File
@@ -0,0 +1,31 @@
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
import {HashIndexManager} from "@metazla/meta-hash";
import {config} from "../../config/EnvConfig.js";
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
import {stat} from "fs/promises";
export const globalHashIndex = new HashIndexManager(config.INDEX_FOLDER_PATH, targetHashForIndex);
export const globalHashComputer = new HashComputerFile(targetHash);
class SimpleFileToCid{
async getcid(filePath: string): Promise<string | null> {
try {
const stats = await stat(filePath);
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
if(indexLine) {
return indexLine[CID_ALGORITHM_NAMES.sha256];
}else {
let hashs = {};
await globalHashComputer.computeMissingHash(filePath, hashs);
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
return hashs[CID_ALGORITHM_NAMES.sha256];
}
} catch (e) {
return null;
}
}
}
export const simpleFileToCid = new SimpleFileToCid();
packages/meta-sort-core/src/logic/fileProcessor/worker.ts
+20
View File
@@ -0,0 +1,20 @@
import {FileProcessorInternal} from "./FileProcessorInternal.js";
import {HashMeta} from "@metazla/meta-interface";
const fileProcessor = new FileProcessorInternal();
export interface Input {
filePath: string;
hash:HashMeta;
}
export interface Output {
time: number;
metadata: HashMeta;
hashTimings?: Record<string, number>; // Per-hash computation times in ms
pluginTimings?: Record<string, number>; // Per-plugin processing times in ms
}
export default async function processFile({ filePath,hash}: Input): Promise<Output> {
return fileProcessor.processFile(filePath,hash);
}
packages/meta-sort-core/src/logic/pipeline/PipelineConfig.ts
+80
View File
@@ -0,0 +1,80 @@
import {FileAnalyzerInterface} from "../fileProcessor/FileAnalyzerInterface.js";
import {UnifiedProcessingStateManager} from "../UnifiedProcessingStateManager.js";
import type {IKVClient} from "../../kv/IKVClient.js";
import {VirtualFileSystem} from "../../api/VirtualFileSystem.js";
import {MetaDataToFolderStruct} from "../MetaDataToFolderStruct.js";
import type {ContainerPluginScheduler} from "../../container-plugins/ContainerPluginScheduler.js";
/**
* Configuration for StreamingPipeline
*/
export interface PipelineConfig {
/**
* Supported file extensions (application-specific)
*/
supportedExtensions: Set<string>;
/**
* Concurrency for validation stage (extension + optional MIME check)
* Default: 32 (I/O bound - fast checks)
*/
validationConcurrency: number;
/**
* Concurrency for fast queue stage (filename parsing, stats)
* Default: 16 (CPU/I/O balanced)
*/
fastQueueConcurrency: number;
/**
* Concurrency for background queue stage (hash computation, FFmpeg)
* Default: 16 (CPU bound)
*/
backgroundQueueConcurrency: number;
/**
* Enable strict MIME validation (reads file header)
* Default: false (skip for performance)
*/
strictMimeValidation: boolean;
/**
* File processor (handles light and heavy processing)
*/
fileProcessor: FileAnalyzerInterface;
/**
* State manager (tracks processing states)
*/
stateManager: UnifiedProcessingStateManager;
/**
* Optional KV client for metadata storage
*/
kvClient?: IKVClient;
/**
* Virtual filesystem (for instant file visibility)
*/
virtualFileSystem: VirtualFileSystem;
/**
* Metadata to folder structure converter (for virtual path generation)
*/
metaDataToFolderStruct: MetaDataToFolderStruct;
/**
* Optional container plugin scheduler for dispatching tasks to containerized plugins
*/
containerPluginScheduler?: ContainerPluginScheduler;
}
/**
* Default pipeline configuration
*/
export const DEFAULT_PIPELINE_CONFIG: Partial<PipelineConfig> = {
validationConcurrency: 32,
fastQueueConcurrency: 16,
backgroundQueueConcurrency: 16,
strictMimeValidation: false
};
packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts
+512
View File
@@ -0,0 +1,512 @@
import * as path from 'path';
import PQueue from 'p-queue';
import {PipelineConfig} from './PipelineConfig.js';
import {withTimeoutAndRetry, TimeoutConfig} from '../../utils/TimeoutWrapper.js';
import {performanceMetrics} from '../../metrics/PerformanceMetrics.js';
// Pub/sub channel for batched updates to meta-fuse
const UPDATE_CHANNEL = 'meta-sort:file:batch';
// Batch update interval (5 seconds)
const BATCH_INTERVAL_MS = 5000;
interface BatchChange {
action: 'add' | 'update' | 'remove';
hashId: string;
}
/**
* Streaming Pipeline for file processing
* Files flow through two stages as they're discovered:
* Discovery → Validation → Processing (midhash256 + metadata)
*
* With midhash256 being instant (< 1s), files appear in VFS immediately
*
* Publishes batched updates to meta-fuse via Redis pub/sub every 5 seconds.
*/
export class StreamingPipeline {
private validationQueue: PQueue;
private fastQueue: PQueue;
private backgroundQueue: PQueue;
private config: PipelineConfig;
private discoveredCount = 0;
private validatedCount = 0;
private fastProcessedCount = 0;
private backgroundProcessedCount = 0;
// Batch buffer for pub/sub notifications
private batchBuffer: BatchChange[] = [];
private batchTimer: NodeJS.Timeout | null = null;
// Retry configuration
private readonly MAX_PROCESSING_RETRIES = 10;
// Fast queue timeout: Fast metadata extraction + midhash256
private readonly fastQueueTimeoutConfig: TimeoutConfig = {
baseTimeout: 120000, // 2 minutes base timeout
timeoutMultiplier: 1.5, // Increase by 1.5x each retry
maxTimeout: 600000 // Max 10 minutes
};
// Background queue timeout: Full file hashing over NAS (can be very slow)
private readonly backgroundQueueTimeoutConfig: TimeoutConfig = {
baseTimeout: 7200000, // 2 hours base timeout (for NAS)
timeoutMultiplier: 1.5, // Increase by 1.5x each retry
maxTimeout: 14400000 // Max 4 hours
};
constructor(config: PipelineConfig) {
this.config = config;
// Stage 1: Validation (high concurrency - fast I/O checks)
this.validationQueue = new PQueue({
concurrency: config.validationConcurrency,
autoStart: true
});
// Stage 2: Fast Queue - metadata + midhash256 (high concurrency, fast)
this.fastQueue = new PQueue({
concurrency: config.fastQueueConcurrency,
autoStart: true
});
// Stage 3: Background Queue - SHA-256 computation (lower concurrency, CPU-intensive)
this.backgroundQueue = new PQueue({
concurrency: config.backgroundQueueConcurrency,
autoStart: true
});
console.log(`[Pipeline] Initialized with concurrency: validation=${config.validationConcurrency}, fastQueue=${config.fastQueueConcurrency}, backgroundQueue=${config.backgroundQueueConcurrency}`);
// Start batch publish timer
this.startBatchTimer();
}
/**
* Start the batch publish timer
* Publishes accumulated changes to meta-fuse every 5 seconds
*/
private startBatchTimer(): void {
this.batchTimer = setInterval(() => {
this.flushBatch().catch(err => {
console.error('[Pipeline] Failed to flush batch:', err.message);
});
}, BATCH_INTERVAL_MS);
}
/**
* Stop the batch publish timer
*/
stopBatchTimer(): void {
if (this.batchTimer) {
clearInterval(this.batchTimer);
this.batchTimer = null;
}
}
/**
* Queue a change for batched notification
*/
private queueChange(action: 'add' | 'update' | 'remove', hashId: string): void {
// Deduplicate: remove existing entries for same hashId
this.batchBuffer = this.batchBuffer.filter(c => c.hashId !== hashId);
this.batchBuffer.push({ action, hashId });
}
/**
* Flush the batch buffer and publish to Redis
*/
private async flushBatch(): Promise<void> {
if (this.batchBuffer.length === 0) {
return;
}
const changes = [...this.batchBuffer];
this.batchBuffer = [];
try {
const kvClient = this.config.kvClient;
if (kvClient && typeof (kvClient as any).publish === 'function') {
const message = JSON.stringify({
timestamp: Date.now(),
changes
});
await (kvClient as any).publish(UPDATE_CHANNEL, message);
console.log(`[Pipeline] Published batch update: ${changes.length} changes`);
}
} catch (error: any) {
// Put changes back in buffer on failure
this.batchBuffer = [...changes, ...this.batchBuffer];
console.warn(`[Pipeline] Failed to publish batch: ${error.message}`);
}
}
/**
* Start processing files from discovery stream
* Processing begins immediately as files are discovered
*
* @param discoveryStream - Async generator yielding file paths
*/
async start(discoveryStream: AsyncGenerator<string>): Promise<void> {
console.log('[Pipeline] Starting file processing stream...');
const startTime = Date.now();
try {
// Consume discovery stream
for await (const filePath of discoveryStream) {
this.discoveredCount++;
// Stage 1: Validation (fire and forget)
this.validationQueue.add(() => this.validateFile(filePath))
.catch(err => console.error(`[Pipeline] Validation error for ${filePath}:`, err.message));
// Log progress every 1000 files
if (this.discoveredCount % 1000 === 0) {
this.logProgress();
}
}
const elapsed = Date.now() - startTime;
console.log(`[Pipeline] Discovery completed: ${this.discoveredCount} files found in ${elapsed}ms`);
console.log(`[Pipeline] Processing continues in background...`);
this.logProgress();
} catch (error: any) {
console.error(`[Pipeline] Error in discovery stream:`, error.message);
throw error;
}
}
/**
* Stage 1: Validation
* - Quick extension check (no file I/O)
* - Optional MIME validation (reads file header)
* - Mark file as pending
*/
private async validateFile(filePath: string): Promise<void> {
try {
// Extension check (instant, no I/O)
const ext = path.extname(filePath).toLowerCase();
if (!this.config.supportedExtensions.has(ext)) {
// Skip unsupported file types
return;
}
// Optional strict MIME validation
// Note: Skipped for performance - extension check is sufficient
// If needed, could add MIME validation via external library
// Mark as pending
this.config.stateManager.addPending(filePath);
this.validatedCount++;
// Stage 2: Fast Queue - metadata + midhash256 (fire and forget)
this.fastQueue.add(() => this.processLightPhase(filePath))
.catch(err => console.error(`[Pipeline] Fast queue error for ${filePath}:`, err.message));
} catch (error: any) {
console.error(`[Pipeline] Validation failed for ${filePath}:`, error.message);
}
}
/**
* Stage 2: Fast Queue Phase
* - Extract metadata (filename parsing, FFmpeg analysis)
* - Compute midhash256 (< 1s)
* - Store to KV (file becomes accessible in VFS)
* - Queue for background processing
* - Includes timeout and retry logic
*/
private async processLightPhase(filePath: string): Promise<void> {
try {
// Get current retry count
const retryCount = this.config.stateManager.getRetryCount(filePath);
// Call light processing with timeout
const current = this.fastProcessedCount + 1;
const queueSize = this.discoveredCount;
await withTimeoutAndRetry(
() => this.config.fileProcessor.processLightPhase(filePath, current, queueSize),
retryCount,
this.fastQueueTimeoutConfig,
`Fast queue processing file ${filePath}`
);
// After light processing, add to VFS
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
if (metadata && metadata.cid_midhash256) {
// Generate virtual path from metadata
const virtualPath = this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath);
if (virtualPath) {
// Add to VFS immediately (file appears with permanent midhash256 ID)
this.config.virtualFileSystem.addFile(virtualPath, filePath, metadata);
//console.log(`✓ File added to VFS: ${filePath} → ${virtualPath} (midhash256: ${metadata.cid_midhash256})`);
// Queue for pub/sub notification to meta-fuse
if (metadata.cid_midhash256) {
this.queueChange('add', metadata.cid_midhash256);
}
// Notify Stremio addon to add video (incremental update like FUSE)
if (metadata.cid_midhash256) {
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
// Silent fail - Stremio addon may not be running
});
}
} else {
// File cannot be added to VFS (e.g., unsupported type, missing title)
console.log(`⚠ File skipped from VFS (no virtual path): ${filePath}`);
}
// Dispatch tasks to container plugins (fire and forget)
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
// Stage 3: Queue for background processing (SHA-256 computation)
this.backgroundQueue.add(() => this.processHashPhase(filePath))
.catch(err => console.error(`[Pipeline] Background queue error for ${filePath}:`, err.message));
}
this.fastProcessedCount++;
} catch (error: any) {
// Handle timeout or processing error
const retryCount = this.config.stateManager.getRetryCount(filePath);
if (retryCount < this.MAX_PROCESSING_RETRIES) {
// Retry: move back to pending queue with incremented retry count
const success = this.config.stateManager.retryProcessing(filePath);
if (success) {
console.warn(`[Pipeline] Fast queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying with longer timeout. Error: ${error.message}`);
// Re-queue for fast processing
this.fastQueue.add(() => this.processLightPhase(filePath))
.catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message));
return;
}
}
// Max retries exceeded or retry failed - mark as permanently failed
console.error(`[Pipeline] Fast queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`);
// Record failed file in metrics
performanceMetrics.recordFailedFile(
filePath,
error.message || 'Unknown error',
retryCount + 1,
'processing'
);
// Remove from processing
this.config.stateManager.removeFile(filePath);
}
}
/**
* Stage 3: Background Queue Phase
* - Compute full hashes (SHA-256, SHA-1, MD5, CRC32)
* - Update KV with additional hash metadata
* - Mark processing as complete
* - Includes timeout and retry logic
*/
private async processHashPhase(filePath: string): Promise<void> {
try {
// Get current retry count
const retryCount = this.config.stateManager.getRetryCount(filePath);
// Call hash processing with timeout
const current = this.backgroundProcessedCount + 1;
const queueSize = this.discoveredCount;
await withTimeoutAndRetry(
() => this.config.fileProcessor.processHashPhase(filePath, current, queueSize),
retryCount,
this.backgroundQueueTimeoutConfig,
`Background queue processing file ${filePath}`
);
this.backgroundProcessedCount++;
} catch (error: any) {
// Handle timeout or processing error
const retryCount = this.config.stateManager.getRetryCount(filePath);
if (retryCount < this.MAX_PROCESSING_RETRIES) {
// Retry: move back to background queue with incremented retry count
console.warn(`[Pipeline] Background queue processing failed for ${filePath} (attempt ${retryCount + 1}/${this.MAX_PROCESSING_RETRIES}), retrying. Error: ${error.message}`);
// Re-queue for background processing
this.backgroundQueue.add(() => this.processHashPhase(filePath))
.catch(err => console.error(`[Pipeline] Retry failed for ${filePath}:`, err.message));
return;
}
// Max retries exceeded - mark as permanently failed
console.error(`[Pipeline] Background queue processing permanently failed for ${filePath} after ${retryCount + 1} attempts: ${error.message}`);
// Record failed file in metrics
performanceMetrics.recordFailedFile(
filePath,
error.message || 'Unknown error',
retryCount + 1,
'hash'
);
// Complete with error
this.config.stateManager.completeHashProcessing(filePath, undefined, undefined, error.message);
}
}
/**
* Handle file added event (from chokidar)
*/
async handleFileAdded(filePath: string): Promise<void> {
// Process through validation stage
this.validationQueue.add(() => this.validateFile(filePath))
.catch(err => console.error(`[Pipeline] Error processing added file ${filePath}:`, err.message));
}
/**
* Handle file changed event (from chokidar)
*/
async handleFileChanged(filePath: string): Promise<void> {
// Reprocess through validation stage
this.config.stateManager.removeFile(filePath);
this.validationQueue.add(() => this.validateFile(filePath))
.catch(err => console.error(`[Pipeline] Error processing changed file ${filePath}:`, err.message));
}
/**
* Handle file deleted event (from chokidar)
*/
async handleFileDeleted(filePath: string): Promise<void> {
// Get hashId before removal for pub/sub notification
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
const hashId = (metadata as any)?.cid_midhash256;
// Remove from state and call cleanup
if (this.config.fileProcessor.deleteFile) {
this.config.fileProcessor.deleteFile(filePath);
}
this.config.stateManager.removeFile(filePath);
// Queue for pub/sub notification to meta-fuse
if (hashId) {
this.queueChange('remove', hashId);
}
}
/**
* Get pipeline statistics
*/
getStats() {
return {
discovered: this.discoveredCount,
validated: this.validatedCount,
fastProcessed: this.fastProcessedCount,
backgroundProcessed: this.backgroundProcessedCount,
queues: {
validation: {
pending: this.validationQueue.pending,
size: this.validationQueue.size
},
fastQueue: {
pending: this.fastQueue.pending,
size: this.fastQueue.size
},
backgroundQueue: {
pending: this.backgroundQueue.pending,
size: this.backgroundQueue.size
}
},
state: this.config.stateManager.getSnapshot()
};
}
/**
* Log current pipeline progress
*/
private logProgress(): void {
const stats = this.getStats();
}
/**
* Notify Stremio addon to add video (incremental update like FUSE)
*/
private async notifyStremioAdd(hashId: string): Promise<void> {
try {
const response = await fetch('http://localhost:7000/add', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ hashId }),
signal: AbortSignal.timeout(5000) // 5 second timeout
});
if (!response.ok) {
// Silent fail - Stremio may not be running
return;
}
} catch (error: any) {
// Silent fail - Stremio addon may not be running or network issue
// This is expected during development when Stremio isn't started
}
}
/**
* Notify Stremio addon to remove video (incremental update like FUSE)
*/
private async notifyStremioRemove(hashId: string): Promise<void> {
try {
const response = await fetch('http://localhost:7000/remove', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ hashId }),
signal: AbortSignal.timeout(5000) // 5 second timeout
});
if (!response.ok) {
// Silent fail - Stremio may not be running
return;
}
} catch (error: any) {
// Silent fail - Stremio addon may not be running or network issue
}
}
/**
* Set the container plugin scheduler (called after initialization)
*/
setContainerPluginScheduler(scheduler: import('../../container-plugins/ContainerPluginScheduler.js').ContainerPluginScheduler): void {
(this.config as any).containerPluginScheduler = scheduler;
console.log('[Pipeline] Container plugin scheduler connected');
}
/**
* Dispatch tasks to container plugins for metadata extraction
* This is fire-and-forget - container plugins write results directly to meta-core
*/
private dispatchContainerPluginTasks(filePath: string, hashId: string, existingMeta: Record<string, any>): void {
const scheduler = this.config.containerPluginScheduler;
if (!scheduler) {
return; // Container plugins not configured
}
try {
// Convert metadata to flat string map for plugin API
const metaFlat: Record<string, string> = {};
for (const [key, value] of Object.entries(existingMeta)) {
if (value != null) {
metaFlat[key] = String(value);
}
}
// Dispatch tasks to all enabled plugins
scheduler.dispatchAllPlugins(hashId, filePath, metaFlat)
.catch(err => {
console.error(`[Pipeline] Failed to dispatch container plugin tasks for ${filePath}:`, err.message);
});
} catch (error: any) {
console.error(`[Pipeline] Error dispatching container plugin tasks for ${filePath}:`, error.message);
}
}
}
packages/meta-sort-core/src/logic/pipeline/index.ts
+2
View File
@@ -0,0 +1,2 @@
export { StreamingPipeline } from './StreamingPipeline.js';
export { PipelineConfig, DEFAULT_PIPELINE_CONFIG } from './PipelineConfig.js';
packages/meta-sort-core/src/logic/tool/MetadataMerger.ts
+36
View File
@@ -0,0 +1,36 @@
import lodash from 'lodash';
import {HashMeta} from "@metazla/meta-interface";
/**
* Merge the new metadata into the old metadata
* @param existingMetadata
* @param newMetadata
* @param id
* @returns {boolean} true if the metadata has changed
*/
export function mergeMetadata(existingMetadata: any, newMetadata: any, id?:string): boolean{
let hasChanged = false;
const mergeLogic = (existingVal:any, newVal:any, key:string) => {
if ((lodash.isObject(existingVal) || lodash.isObject(newVal)) ||
(lodash.isArray(existingVal) || lodash.isArray(newVal))) {
// Recursively merge objects and arrays
return lodash.mergeWith(existingVal, newVal, mergeLogic);
}
if (existingVal) {
if (!lodash.isEqual(""+existingVal, ""+newVal)){
//conflict on primitive types
console.warn(`Conflict found for '${id}'. Existing value: '${existingVal}', New value: '${newVal}'.`);
}
// Keep the original value (especially in case of conflict)
return existingVal;
}else {
if(newVal !== undefined) {
hasChanged = true;
}
return newVal;
}
};
lodash.mergeWith(existingMetadata, newMetadata, mergeLogic);
return hasChanged;
}
packages/meta-sort-core/src/metrics/PerformanceMetrics.ts
+582
View File
@@ -0,0 +1,582 @@
/**
* Performance metrics tracker for meta-mesh
* Tracks detailed performance statistics and timing information
*/
export interface MetricEntry {
timestamp: number;
value: number;
label?: string;
}
export interface FailedFileEntry {
filePath: string;
reason: string;
timestamp: number;
retryCount: number;
stage: 'metadata' | 'hash' | 'processing';
}
export interface HashTimingStats {
total: number;
count: number;
average: number;
min: number;
max: number;
}
export interface PluginTimingStats {
total: number;
count: number;
average: number;
min: number;
max: number;
}
export interface PerformanceMetricsData {
// File processing metrics
totalFilesProcessed: number;
averageFileProcessingTime: number;
minFileProcessingTime: number;
maxFileProcessingTime: number;
recentFileProcessingTimes: MetricEntry[];
// Metadata (light processing) metrics
totalMetadataProcessed: number;
averageMetadataProcessingTime: number;
minMetadataProcessingTime: number;
maxMetadataProcessingTime: number;
// Hash processing metrics
totalHashProcessed: number;
averageHashProcessingTime: number;
minHashProcessingTime: number;
maxHashProcessingTime: number;
// Per-hash-algorithm timing metrics (excludes cache hits)
hashComputationTimes: Record<string, HashTimingStats>;
// Per-plugin timing metrics
pluginProcessingTimes: Record<string, PluginTimingStats>;
// VFS update metrics
totalVFSUpdates: number;
averageVFSUpdateTime: number;
lastVFSUpdateTime: number;
recentVFSUpdateTimes: MetricEntry[];
// Duplicate detection metrics
averageDuplicateDetectionTime: number;
lastDuplicateDetectionTime: number;
totalDuplicatesFound: number;
// Virtual structure generation metrics
averageVirtualStructureTime: number;
lastVirtualStructureTime: number;
// Failed files tracking
failedFiles: FailedFileEntry[];
totalFailedFiles: number;
totalMetadataFailures: number;
totalHashFailures: number;
// System metrics
uptime: number;
startTime: number;
lastActivityTime: number;
// Memory and performance
memoryUsage?: NodeJS.MemoryUsage;
// Processing rate
filesPerSecond: number;
totalProcessingTime: number;
// Cache metrics
cacheHits: Record<string, number>;
cacheMisses: Record<string, number>;
cacheHitRate: Record<string, number>;
}
export class PerformanceMetrics {
private startTime: number;
private lastActivityTime: number;
// File processing tracking - running statistics
private recentFileProcessing: MetricEntry[] = [];
private totalFilesProcessed: number = 0;
private fileProcessingSum: number = 0;
private fileProcessingMin: number = Infinity;
private fileProcessingMax: number = 0;
// Metadata (light processing) tracking - running statistics
private totalMetadataProcessed: number = 0;
private metadataProcessingSum: number = 0;
private metadataProcessingMin: number = Infinity;
private metadataProcessingMax: number = 0;
// Hash processing tracking - running statistics
private totalHashProcessed: number = 0;
private hashProcessingSum: number = 0;
private hashProcessingMin: number = Infinity;
private hashProcessingMax: number = 0;
// VFS update tracking - running statistics
private recentVFSUpdates: MetricEntry[] = [];
private totalVFSUpdates: number = 0;
private vfsUpdateSum: number = 0;
private lastVFSUpdateTime: number = 0;
// Duplicate detection tracking - running statistics
private totalDuplicatesFound: number = 0;
private duplicateDetectionSum: number = 0;
private duplicateDetectionCount: number = 0;
private lastDuplicateDetectionTime: number = 0;
// Virtual structure generation tracking - running statistics
private virtualStructureSum: number = 0;
private virtualStructureCount: number = 0;
private lastVirtualStructureTime: number = 0;
// Failed files tracking
private failedFiles: FailedFileEntry[] = [];
private totalMetadataFailures: number = 0;
private totalHashFailures: number = 0;
// Cache metrics tracking
private cacheHits: Record<string, number> = {};
private cacheMisses: Record<string, number> = {};
// Per-hash-algorithm timing tracking (excludes cache hits)
private hashTimings: Record<string, {
sum: number;
count: number;
min: number;
max: number;
}> = {};
// Per-plugin timing tracking
private pluginTimings: Record<string, {
sum: number;
count: number;
min: number;
max: number;
}> = {};
// Configuration
private maxRecentEntries: number = 100;
private maxFailedFilesHistory: number = 50;
constructor() {
this.startTime = Date.now();
this.lastActivityTime = this.startTime;
}
/**
* Record a file processing event
*/
recordFileProcessing(filePath: string, timeMs: number): void {
// Update running statistics
this.fileProcessingSum += timeMs;
this.fileProcessingMin = Math.min(this.fileProcessingMin, timeMs);
this.fileProcessingMax = Math.max(this.fileProcessingMax, timeMs);
this.totalFilesProcessed++;
// Keep recent list for display only
this.recentFileProcessing.push({
timestamp: Date.now(),
value: timeMs,
label: filePath
});
if (this.recentFileProcessing.length > this.maxRecentEntries) {
this.recentFileProcessing.shift();
}
this.lastActivityTime = Date.now();
}
/**
* Record a metadata (light processing) event
*/
recordMetadataProcessing(timeMs: number): void {
// Update running statistics
this.metadataProcessingSum += timeMs;
this.metadataProcessingMin = Math.min(this.metadataProcessingMin, timeMs);
this.metadataProcessingMax = Math.max(this.metadataProcessingMax, timeMs);
this.totalMetadataProcessed++;
this.lastActivityTime = Date.now();
}
/**
* Record a hash processing event
*/
recordHashProcessing(timeMs: number): void {
// Update running statistics
this.hashProcessingSum += timeMs;
this.hashProcessingMin = Math.min(this.hashProcessingMin, timeMs);
this.hashProcessingMax = Math.max(this.hashProcessingMax, timeMs);
this.totalHashProcessed++;
this.lastActivityTime = Date.now();
}
/**
* Record a VFS update event
*/
recordVFSUpdate(timeMs: number): void {
// Update running statistics
this.vfsUpdateSum += timeMs;
this.lastVFSUpdateTime = timeMs;
this.totalVFSUpdates++;
// Keep recent list for display only
this.recentVFSUpdates.push({
timestamp: Date.now(),
value: timeMs
});
if (this.recentVFSUpdates.length > this.maxRecentEntries) {
this.recentVFSUpdates.shift();
}
this.lastActivityTime = Date.now();
}
/**
* Record a duplicate detection event
*/
recordDuplicateDetection(timeMs: number, duplicatesFound: number): void {
// Update running statistics
this.duplicateDetectionSum += timeMs;
this.duplicateDetectionCount++;
this.lastDuplicateDetectionTime = timeMs;
this.totalDuplicatesFound += duplicatesFound;
this.lastActivityTime = Date.now();
}
/**
* Record a virtual structure generation event
*/
recordVirtualStructureGeneration(timeMs: number): void {
// Update running statistics
this.virtualStructureSum += timeMs;
this.virtualStructureCount++;
this.lastVirtualStructureTime = timeMs;
this.lastActivityTime = Date.now();
}
/**
* Record a cache hit event
*/
recordCacheHit(cacheType: string): void {
if (!this.cacheHits[cacheType]) {
this.cacheHits[cacheType] = 0;
}
this.cacheHits[cacheType]++;
this.lastActivityTime = Date.now();
}
/**
* Record a cache miss event
*/
recordCacheMiss(cacheType: string): void {
if (!this.cacheMisses[cacheType]) {
this.cacheMisses[cacheType] = 0;
}
this.cacheMisses[cacheType]++;
this.lastActivityTime = Date.now();
}
/**
* Record hash computation time for a specific algorithm (excludes cache hits)
* @param hashAlgorithm - Hash algorithm name (e.g., 'cid_sha2-256', 'cid_midhash256')
* @param timeMs - Time taken to compute the hash in milliseconds
*/
recordHashComputation(hashAlgorithm: string, timeMs: number): void {
if (!this.hashTimings[hashAlgorithm]) {
this.hashTimings[hashAlgorithm] = {
sum: 0,
count: 0,
min: Infinity,
max: 0
};
}
const stats = this.hashTimings[hashAlgorithm];
stats.sum += timeMs;
stats.count++;
stats.min = Math.min(stats.min, timeMs);
stats.max = Math.max(stats.max, timeMs);
this.lastActivityTime = Date.now();
}
/**
* Record plugin processing time
* @param pluginName - Name of the plugin
* @param timeMs - Time taken by the plugin in milliseconds
*/
recordPluginProcessing(pluginName: string, timeMs: number): void {
if (!this.pluginTimings[pluginName]) {
this.pluginTimings[pluginName] = {
sum: 0,
count: 0,
min: Infinity,
max: 0
};
}
const stats = this.pluginTimings[pluginName];
stats.sum += timeMs;
stats.count++;
stats.min = Math.min(stats.min, timeMs);
stats.max = Math.max(stats.max, timeMs);
this.lastActivityTime = Date.now();
}
/**
* Record a failed file processing event
*/
recordFailedFile(filePath: string, reason: string, retryCount: number, stage: 'metadata' | 'hash' | 'processing'): void {
const failedEntry: FailedFileEntry = {
filePath,
reason,
timestamp: Date.now(),
retryCount,
stage
};
// Add to failed files list (most recent first)
this.failedFiles.unshift(failedEntry);
// Trim history if needed
if (this.failedFiles.length > this.maxFailedFilesHistory) {
this.failedFiles = this.failedFiles.slice(0, this.maxFailedFilesHistory);
}
// Update failure counters
if (stage === 'metadata') {
this.totalMetadataFailures++;
} else {
this.totalHashFailures++;
}
this.lastActivityTime = Date.now();
}
/**
* Get all metrics data - O(1) complexity using cached running statistics
*/
getMetrics(): PerformanceMetricsData & {
// UI compatibility aliases
totalFilesDiscovered: number;
totalFilesFailed: number;
averageLightProcessingMs: number;
averageHashProcessingMs: number;
} {
const now = Date.now();
const uptime = now - this.startTime;
const totalFailedFiles = this.totalMetadataFailures + this.totalHashFailures;
const avgMetadata = this.totalMetadataProcessed > 0
? this.metadataProcessingSum / this.totalMetadataProcessed
: 0;
const avgHash = this.totalHashProcessed > 0
? this.hashProcessingSum / this.totalHashProcessed
: 0;
return {
// File processing metrics - O(1) from running statistics
totalFilesProcessed: this.totalFilesProcessed,
// UI compatibility aliases
totalFilesDiscovered: this.totalFilesProcessed + totalFailedFiles,
totalFilesFailed: totalFailedFiles,
averageLightProcessingMs: avgMetadata,
averageHashProcessingMs: avgHash,
averageFileProcessingTime: this.totalFilesProcessed > 0
? this.fileProcessingSum / this.totalFilesProcessed
: 0,
minFileProcessingTime: this.fileProcessingMin === Infinity ? 0 : this.fileProcessingMin,
maxFileProcessingTime: this.fileProcessingMax,
recentFileProcessingTimes: this.recentFileProcessing.slice(-20), // Last 20 files
// Metadata (light processing) metrics - O(1) from running statistics
totalMetadataProcessed: this.totalMetadataProcessed,
averageMetadataProcessingTime: this.totalMetadataProcessed > 0
? this.metadataProcessingSum / this.totalMetadataProcessed
: 0,
minMetadataProcessingTime: this.metadataProcessingMin === Infinity ? 0 : this.metadataProcessingMin,
maxMetadataProcessingTime: this.metadataProcessingMax,
// Hash processing metrics - O(1) from running statistics
totalHashProcessed: this.totalHashProcessed,
averageHashProcessingTime: this.totalHashProcessed > 0
? this.hashProcessingSum / this.totalHashProcessed
: 0,
minHashProcessingTime: this.hashProcessingMin === Infinity ? 0 : this.hashProcessingMin,
maxHashProcessingTime: this.hashProcessingMax,
// Per-hash-algorithm timing metrics (excludes cache hits)
hashComputationTimes: Object.keys(this.hashTimings).reduce((acc, hashAlgorithm) => {
const stats = this.hashTimings[hashAlgorithm];
acc[hashAlgorithm] = {
total: stats.sum,
count: stats.count,
average: stats.count > 0 ? stats.sum / stats.count : 0,
min: stats.min === Infinity ? 0 : stats.min,
max: stats.max
};
return acc;
}, {} as Record<string, HashTimingStats>),
// Per-plugin timing metrics
pluginProcessingTimes: Object.keys(this.pluginTimings).reduce((acc, pluginName) => {
const stats = this.pluginTimings[pluginName];
acc[pluginName] = {
total: stats.sum,
count: stats.count,
average: stats.count > 0 ? stats.sum / stats.count : 0,
min: stats.min === Infinity ? 0 : stats.min,
max: stats.max
};
return acc;
}, {} as Record<string, PluginTimingStats>),
// VFS update metrics - O(1) from running statistics
totalVFSUpdates: this.totalVFSUpdates,
averageVFSUpdateTime: this.totalVFSUpdates > 0
? this.vfsUpdateSum / this.totalVFSUpdates
: 0,
lastVFSUpdateTime: this.lastVFSUpdateTime,
recentVFSUpdateTimes: this.recentVFSUpdates.slice(-10), // Last 10 updates
// Duplicate detection metrics - O(1) from running statistics
averageDuplicateDetectionTime: this.duplicateDetectionCount > 0
? this.duplicateDetectionSum / this.duplicateDetectionCount
: 0,
lastDuplicateDetectionTime: this.lastDuplicateDetectionTime,
totalDuplicatesFound: this.totalDuplicatesFound,
// Virtual structure generation metrics - O(1) from running statistics
averageVirtualStructureTime: this.virtualStructureCount > 0
? this.virtualStructureSum / this.virtualStructureCount
: 0,
lastVirtualStructureTime: this.lastVirtualStructureTime,
// Failed files tracking
failedFiles: this.failedFiles,
totalFailedFiles: this.totalMetadataFailures + this.totalHashFailures,
totalMetadataFailures: this.totalMetadataFailures,
totalHashFailures: this.totalHashFailures,
// System metrics
uptime: uptime,
startTime: this.startTime,
lastActivityTime: this.lastActivityTime,
// Memory usage
memoryUsage: process.memoryUsage(),
// Processing rate
filesPerSecond: uptime > 0 ? (this.totalFilesProcessed / (uptime / 1000)) : 0,
totalProcessingTime: this.fileProcessingSum,
// Cache metrics
cacheHits: { ...this.cacheHits },
cacheMisses: { ...this.cacheMisses },
cacheHitRate: Object.keys(this.cacheHits).reduce((acc, key) => {
const hits = this.cacheHits[key] || 0;
const misses = this.cacheMisses[key] || 0;
const total = hits + misses;
acc[key] = total > 0 ? (hits / total) * 100 : 0;
return acc;
}, {} as Record<string, number>)
};
}
/**
* Get timing stats for a specific plugin
* @param pluginId - Plugin identifier
* @returns Plugin timing stats or null if no data
*/
getPluginTiming(pluginId: string): PluginTimingStats | null {
const stats = this.pluginTimings[pluginId];
if (!stats || stats.count === 0) {
return null;
}
return {
total: stats.sum,
count: stats.count,
average: stats.sum / stats.count,
min: stats.min === Infinity ? 0 : stats.min,
max: stats.max
};
}
/**
* Reset all metrics
*/
reset(): void {
// Reset file processing metrics
this.recentFileProcessing = [];
this.totalFilesProcessed = 0;
this.fileProcessingSum = 0;
this.fileProcessingMin = Infinity;
this.fileProcessingMax = 0;
// Reset metadata processing metrics
this.totalMetadataProcessed = 0;
this.metadataProcessingSum = 0;
this.metadataProcessingMin = Infinity;
this.metadataProcessingMax = 0;
// Reset hash processing metrics
this.totalHashProcessed = 0;
this.hashProcessingSum = 0;
this.hashProcessingMin = Infinity;
this.hashProcessingMax = 0;
// Reset VFS update metrics
this.recentVFSUpdates = [];
this.totalVFSUpdates = 0;
this.vfsUpdateSum = 0;
this.lastVFSUpdateTime = 0;
// Reset duplicate detection metrics
this.totalDuplicatesFound = 0;
this.duplicateDetectionSum = 0;
this.duplicateDetectionCount = 0;
this.lastDuplicateDetectionTime = 0;
// Reset virtual structure generation metrics
this.virtualStructureSum = 0;
this.virtualStructureCount = 0;
this.lastVirtualStructureTime = 0;
// Reset failed files tracking
this.failedFiles = [];
this.totalMetadataFailures = 0;
this.totalHashFailures = 0;
// Reset cache metrics
this.cacheHits = {};
this.cacheMisses = {};
// Reset hash computation timing
this.hashTimings = {};
// Reset plugin processing timing
this.pluginTimings = {};
this.startTime = Date.now();
this.lastActivityTime = this.startTime;
}
}
// Global singleton instance
export const performanceMetrics = new PerformanceMetrics();
packages/meta-sort-core/src/plugin-engine/PluginContext.ts
+392
View File
@@ -0,0 +1,392 @@
/**
* Plugin Context Implementation
*
* Provides KV store, cache, and logger implementations for plugins.
*/
import * as fs from 'fs/promises';
import * as path from 'path';
import { existsSync, mkdirSync } from 'fs';
import type { MetadataNode } from '@metazla/meta-interface';
import type {
PluginKVStore,
PluginCache,
PluginLogger,
PluginContext,
PluginLifecycleContext,
} from './types.js';
// =============================================================================
// KV Store Implementation
// =============================================================================
/**
* In-memory KV store implementation with path-style keys
*/
export class KVStore implements PluginKVStore {
private data: Map<string, string> = new Map();
constructor(initialData?: Map<string, string> | Record<string, string>) {
if (initialData) {
if (initialData instanceof Map) {
this.data = new Map(initialData);
} else {
for (const [key, value] of Object.entries(initialData)) {
if (typeof value === 'string') {
this.data.set(key, value);
}
}
}
}
}
get(key: string): string | undefined {
return this.data.get(key);
}
set(key: string, value: string): void {
if (key === undefined || key === null || key === '') {
return;
}
if (value === undefined || value === null) {
return;
}
this.data.set(key, String(value));
}
delete(key: string): void {
this.data.delete(key);
}
keys(prefix?: string): string[] {
const allKeys = Array.from(this.data.keys());
if (!prefix) {
return allKeys;
}
return allKeys.filter(k => k.startsWith(prefix));
}
entries(prefix?: string): Array<[string, string]> {
const allEntries = Array.from(this.data.entries());
if (!prefix) {
return allEntries;
}
return allEntries.filter(([k]) => k.startsWith(prefix));
}
/**
* Get all data as a plain object (for serialization)
*/
toObject(): Record<string, string> {
const obj: Record<string, string> = {};
this.data.forEach((value, key) => {
obj[key] = value;
});
return obj;
}
/**
* Get the underlying Map
*/
toMap(): Map<string, string> {
return new Map(this.data);
}
/**
* Clear all data
*/
clear(): void {
this.data.clear();
}
}
// =============================================================================
// MetadataNode KV Store Adapter
// =============================================================================
/**
* KV store that wraps MetadataNode from @metazla/meta-interface.
* This ensures plugins output the same nested object format as the old processors.
*
* Path conversion:
* - 'fileType' -> metadata.at('fileType')
* - 'fileinfo/duration' -> metadata.at('fileinfo').at('duration')
* - 'fileinfo/streamdetails/video/0/codec' -> metadata.at('fileinfo').at('streamdetails').at('video').at('0').at('codec')
*/
export class MetadataNodeKVStore implements PluginKVStore {
private metadata: MetadataNode<any>;
constructor(metadata: MetadataNode<any>) {
this.metadata = metadata;
}
/**
* Navigate to a nested node using path segments
*/
private navigate(key: string): MetadataNode<any> {
const parts = key.split('/');
let node = this.metadata;
for (const part of parts) {
node = node.at(part);
}
return node;
}
get(key: string): string | undefined {
try {
const value = this.navigate(key).get();
if (value === undefined || value === null) {
return undefined;
}
if (typeof value === 'string') {
return value;
}
if (typeof value === 'object') {
// Return undefined for objects (indicates path to nested data)
return undefined;
}
return String(value);
} catch {
return undefined;
}
}
set(key: string, value: string): void {
if (key === undefined || key === null || key === '') {
return;
}
if (value === undefined || value === null) {
return;
}
try {
this.navigate(key).set(String(value));
} catch (e) {
console.error(`[MetadataNodeKVStore] Error setting ${key}:`, e);
}
}
delete(key: string): void {
// MetadataNode doesn't have a delete method, so we set to undefined
try {
this.navigate(key).set(undefined as any);
} catch {
// Ignore errors
}
}
/**
* Get keys is not easily supported by MetadataNode.
* This returns an empty array - plugins should track their own keys if needed.
*/
keys(prefix?: string): string[] {
// MetadataNode doesn't expose iteration over keys
// Plugins that need to iterate should use the metadata object directly
console.warn('[MetadataNodeKVStore] keys() not supported - use metadata.get() for object inspection');
return [];
}
entries(prefix?: string): Array<[string, string]> {
console.warn('[MetadataNodeKVStore] entries() not supported - use metadata.get() for object inspection');
return [];
}
/**
* Get the underlying MetadataNode
*/
getMetadataNode(): MetadataNode<any> {
return this.metadata;
}
/**
* Add a value to a set (for RecordSet fields like genres, tags)
*/
add(key: string, value: string): void {
if (key === undefined || key === null || key === '') {
return;
}
if (value === undefined || value === null) {
return;
}
try {
this.navigate(key).add(value);
} catch (e) {
console.error(`[MetadataNodeKVStore] Error adding to ${key}:`, e);
}
}
/**
* Get full object at a path (for inspecting nested structures)
*/
getObject(key: string): any {
try {
return this.navigate(key).get();
} catch {
return undefined;
}
}
}
// =============================================================================
// Cache Implementation
// =============================================================================
/**
* File-based cache implementation for plugins
*/
export class PluginCacheImpl implements PluginCache {
private cacheDir: string;
private initialized: boolean = false;
constructor(cacheDir: string) {
this.cacheDir = cacheDir;
}
private ensureDir(): void {
if (!this.initialized) {
if (!existsSync(this.cacheDir)) {
mkdirSync(this.cacheDir, { recursive: true });
}
this.initialized = true;
}
}
getPath(filename: string): string {
this.ensureDir();
return path.join(this.cacheDir, filename);
}
async readJson<T>(filename: string): Promise<T | null> {
try {
const filePath = this.getPath(filename);
const data = await fs.readFile(filePath, 'utf8');
return JSON.parse(data) as T;
} catch (error) {
if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
return null;
}
throw error;
}
}
async writeJson(filename: string, data: unknown): Promise<void> {
this.ensureDir();
const filePath = this.getPath(filename);
const dir = path.dirname(filePath);
if (!existsSync(dir)) {
await fs.mkdir(dir, { recursive: true });
}
await fs.writeFile(filePath, JSON.stringify(data, null, 2), 'utf8');
}
async exists(filename: string): Promise<boolean> {
try {
await fs.access(this.getPath(filename));
return true;
} catch {
return false;
}
}
async delete(filename: string): Promise<void> {
try {
await fs.unlink(this.getPath(filename));
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw error;
}
}
}
async clear(): Promise<void> {
try {
const files = await fs.readdir(this.cacheDir);
await Promise.all(
files.map(file => fs.unlink(path.join(this.cacheDir, file)).catch(() => {}))
);
} catch (error) {
if ((error as NodeJS.ErrnoException).code !== 'ENOENT') {
throw error;
}
}
}
}
// =============================================================================
// Logger Implementation
// =============================================================================
/**
* Scoped logger implementation for plugins
*/
export class PluginLoggerImpl implements PluginLogger {
private prefix: string;
constructor(pluginId: string) {
this.prefix = `[plugin:${pluginId}]`;
}
debug(message: string, ...args: unknown[]): void {
console.debug(this.prefix, message, ...args);
}
info(message: string, ...args: unknown[]): void {
console.info(this.prefix, message, ...args);
}
warn(message: string, ...args: unknown[]): void {
console.warn(this.prefix, message, ...args);
}
error(message: string, ...args: unknown[]): void {
console.error(this.prefix, message, ...args);
}
}
// =============================================================================
// Context Factory
// =============================================================================
/**
* Create a lifecycle context for a plugin
*/
export function createLifecycleContext(
pluginId: string,
config: Record<string, unknown>,
cacheDir: string
): PluginLifecycleContext {
return {
config,
cacheDir,
log: new PluginLoggerImpl(pluginId),
};
}
/**
* Create a full plugin context for file processing
*/
export function createPluginContext(
pluginId: string,
filePath: string,
kv: PluginKVStore,
config: Record<string, unknown>,
cacheDir: string,
metadata?: any
): PluginContext {
const ctx: PluginContext = {
kv,
filePath,
config,
cacheDir,
cache: new PluginCacheImpl(cacheDir),
log: new PluginLoggerImpl(pluginId),
};
// If kv is a MetadataNodeKVStore, extract the MetadataNode
if (metadata) {
ctx.metadata = metadata;
} else if (kv instanceof MetadataNodeKVStore) {
ctx.metadata = kv.getMetadataNode();
}
return ctx;
}
packages/meta-sort-core/src/plugin-engine/PluginExecutor.ts
+146
View File
@@ -0,0 +1,146 @@
/**
* Plugin Executor
*
* Handles dependency resolution for the container-based plugin system.
*/
import type { PluginManifest } from './types.js';
// =============================================================================
// Dependency Resolution
// =============================================================================
/**
* Result of dependency resolution
*/
export interface DependencyResolution {
order: string[];
errors: string[];
}
/**
* Perform topological sort on plugins based on dependencies.
* Returns execution order (plugins that must run first come first).
*
* @param plugins Map of plugin id to manifest
* @param activePluginIds Set of active plugin ids to include in the order
*/
export function resolveDependencyOrder(
plugins: Map<string, PluginManifest>,
activePluginIds: Set<string>
): DependencyResolution {
const errors: string[] = [];
const order: string[] = [];
const visited = new Set<string>();
const visiting = new Set<string>(); // For cycle detection
// Filter to only active plugins
const activePlugins = new Map<string, PluginManifest>();
activePluginIds.forEach(pluginId => {
const manifest = plugins.get(pluginId);
if (manifest) {
activePlugins.set(pluginId, manifest);
}
});
// DFS visit function
function visit(pluginId: string): boolean {
if (visited.has(pluginId)) {
return true;
}
if (visiting.has(pluginId)) {
errors.push(`Circular dependency detected involving plugin '${pluginId}'`);
return false;
}
const manifest = activePlugins.get(pluginId);
if (!manifest) {
// Plugin not found or not active
return true;
}
visiting.add(pluginId);
// Visit dependencies first
for (const depId of manifest.dependencies || []) {
if (!activePlugins.has(depId)) {
// Dependency is not active
if (plugins.has(depId)) {
errors.push(`Plugin '${pluginId}' depends on '${depId}' which is not active`);
} else {
errors.push(`Plugin '${pluginId}' depends on '${depId}' which does not exist`);
}
continue;
}
if (!visit(depId)) {
return false;
}
}
visiting.delete(pluginId);
visited.add(pluginId);
order.push(pluginId);
return true;
}
// Visit all active plugins
Array.from(activePluginIds).forEach(pluginId => {
if (!visited.has(pluginId)) {
visit(pluginId);
}
});
return { order, errors };
}
/**
* Check if a plugin can be activated (all dependencies are active)
*/
export function canActivate(
pluginId: string,
plugins: Map<string, PluginManifest>,
activePluginIds: Set<string>
): { canActivate: boolean; missingDeps: string[] } {
const manifest = plugins.get(pluginId);
if (!manifest) {
return { canActivate: false, missingDeps: [] };
}
const missingDeps: string[] = [];
for (const depId of manifest.dependencies || []) {
if (!activePluginIds.has(depId)) {
missingDeps.push(depId);
}
}
return {
canActivate: missingDeps.length === 0,
missingDeps,
};
}
/**
* Check if a plugin can be deactivated (no active plugins depend on it)
*/
export function canDeactivate(
pluginId: string,
plugins: Map<string, PluginManifest>,
activePluginIds: Set<string>
): { canDeactivate: boolean; dependents: string[] } {
const dependents: string[] = [];
activePluginIds.forEach(activeId => {
if (activeId === pluginId) return;
const manifest = plugins.get(activeId);
if (manifest?.dependencies?.includes(pluginId)) {
dependents.push(activeId);
}
});
return {
canDeactivate: dependents.length === 0,
dependents,
};
}
packages/meta-sort-core/src/plugin-engine/PluginManager.ts
+406
View File
@@ -0,0 +1,406 @@
/**
* Plugin Manager
*
* Main orchestrator for the container-based plugin system.
* Handles plugin manifests, lifecycle management, and state persistence.
* All plugins run as Docker containers - there is no local plugin execution.
*/
import * as fs from 'fs/promises';
import * as path from 'path';
import { existsSync, mkdirSync } from 'fs';
import { EventEmitter } from 'events';
import type {
PluginManifest,
PluginState,
PluginStateEntry,
PluginInfo,
LoadedPlugin,
PluginManagerEvent,
PluginKVStore,
} from './types.js';
import { KVStore } from './PluginContext.js';
import {
resolveDependencyOrder,
canActivate,
canDeactivate,
} from './PluginExecutor.js';
import type { ContainerManager, ContainerPluginManifest } from '../container-plugins/index.js';
// =============================================================================
// Plugin Manager
// =============================================================================
export interface PluginManagerOptions {
/** Path to plugins.json state file */
statePath: string;
/** Base directory for plugin caches */
cacheDir: string;
/** Container manager for containerized plugins (required) */
containerManager?: ContainerManager;
}
export class PluginManager extends EventEmitter {
private statePath: string;
private cacheDir: string;
// Runtime state
private manifests: Map<string, PluginManifest> = new Map();
private loadedPlugins: Map<string, LoadedPlugin> = new Map();
private activePluginIds: Set<string> = new Set();
private configs: Map<string, Record<string, unknown>> = new Map();
private executionOrder: string[] = [];
// Container plugins (all plugins are container-based)
private containerManager: ContainerManager | null = null;
constructor(options: PluginManagerOptions) {
super();
this.statePath = options.statePath;
this.cacheDir = options.cacheDir;
this.containerManager = options.containerManager || null;
// Ensure cache directory exists
if (!existsSync(this.cacheDir)) {
mkdirSync(this.cacheDir, { recursive: true });
}
}
// =========================================================================
// Initialization
// =========================================================================
/**
* Initialize the plugin manager with container plugins.
* This loads plugin manifests from the ContainerManager and sets up the execution order.
*/
async initialize(): Promise<void> {
if (!this.containerManager) {
console.warn('[PluginManager] No container manager set, no plugins will be available');
return;
}
await this.loadContainerPlugins();
console.log(`[PluginManager] Initialized with ${this.manifests.size} container plugins`);
}
// =========================================================================
// Lifecycle Management
// =========================================================================
/**
* Activate a plugin
*/
async activate(pluginId: string): Promise<void> {
const manifest = this.manifests.get(pluginId);
if (!manifest) {
throw new Error(`Plugin '${pluginId}' not found`);
}
if (this.activePluginIds.has(pluginId)) {
return; // Already active
}
// Check dependencies
const { canActivate: can, missingDeps } = canActivate(
pluginId,
this.manifests,
this.activePluginIds
);
if (!can) {
throw new Error(
`Cannot activate '${pluginId}': missing dependencies: ${missingDeps.join(', ')}`
);
}
// Mark as active
this.activePluginIds.add(pluginId);
this.rebuildExecutionOrder();
// Save state
await this.saveState();
// Emit event
this.emit('plugin:activated', { pluginId });
}
/**
* Deactivate a plugin
*/
async deactivate(pluginId: string): Promise<void> {
if (!this.activePluginIds.has(pluginId)) {
return; // Already inactive
}
// Check if other plugins depend on this one
const { canDeactivate: can, dependents } = canDeactivate(
pluginId,
this.manifests,
this.activePluginIds
);
if (!can) {
throw new Error(
`Cannot deactivate '${pluginId}': other plugins depend on it: ${dependents.join(', ')}`
);
}
// Mark as inactive
this.activePluginIds.delete(pluginId);
this.rebuildExecutionOrder();
// Save state
await this.saveState();
// Emit event
this.emit('plugin:deactivated', { pluginId });
}
/**
* Update plugin configuration
*/
async updateConfig(pluginId: string, config: Record<string, unknown>): Promise<void> {
const manifest = this.manifests.get(pluginId);
if (!manifest) {
throw new Error(`Plugin '${pluginId}' not found`);
}
// Store the config
this.configs.set(pluginId, config);
// Save state
await this.saveState();
// Emit event
this.emit('plugin:config-changed', { pluginId, config });
}
// =========================================================================
// Container Plugin Support
// =========================================================================
/**
* Set the container manager (can be set after construction)
*/
setContainerManager(containerManager: ContainerManager): void {
this.containerManager = containerManager;
console.log('[PluginManager] Container manager set');
}
/**
* Check if a plugin is a containerized plugin (all plugins are container-based)
*/
isContainerPlugin(pluginId: string): boolean {
return this.manifests.has(pluginId);
}
/**
* Load plugin manifests from ContainerManager
* Should be called after container plugins are initialized
*/
async loadContainerPlugins(): Promise<void> {
if (!this.containerManager) {
return;
}
const status = this.containerManager.getStatus();
if (!status.initialized) {
console.warn('[PluginManager] Container manager not initialized, skipping plugins');
return;
}
// Load state for plugin active status
const state = await this.loadState();
for (const pluginStatus of status.plugins) {
const manifest = this.containerManager.getPluginManifest(pluginStatus.pluginId);
if (!manifest) continue;
const pluginId = pluginStatus.pluginId;
// Convert container manifest to local manifest format
const localManifest: PluginManifest = {
id: manifest.id,
name: manifest.name,
version: manifest.version,
description: manifest.description || '',
author: manifest.author,
dependencies: manifest.dependencies || [],
schema: manifest.schema || {},
config: manifest.config,
priority: manifest.priority || 100,
color: manifest.color,
defaultQueue: manifest.defaultQueue || 'background',
};
this.manifests.set(pluginId, localManifest);
// Container plugins are always "loaded" but execution is remote
this.loadedPlugins.set(pluginId, {
manifest: localManifest,
instance: null as any, // No local instance for container plugins
status: 'loaded',
pluginDir: '', // No local dir
});
// Check if should be active
const stateEntry = state.plugins[pluginId];
if (stateEntry?.active !== false) {
// Default to active for plugins
this.activePluginIds.add(pluginId);
}
// Store config
const config = stateEntry?.config || {};
this.configs.set(pluginId, config);
console.log(`[PluginManager] Loaded plugin: ${pluginId}`);
}
// Rebuild execution order
this.rebuildExecutionOrder();
await this.saveState();
}
/**
* Get the container manager instance
*/
getContainerManager(): ContainerManager | null {
return this.containerManager;
}
// =========================================================================
// State Management
// =========================================================================
/**
* Get all discovered plugins
*/
getPlugins(): PluginInfo[] {
const plugins: PluginInfo[] = [];
this.manifests.forEach((manifest, pluginId) => {
const loaded = this.loadedPlugins.get(pluginId);
const config = this.configs.get(pluginId) || {};
plugins.push({
id: pluginId,
name: manifest.name,
version: manifest.version,
description: manifest.description,
priority: manifest.priority,
color: manifest.color,
dependencies: manifest.dependencies || [],
active: this.activePluginIds.has(pluginId),
config,
configSchema: manifest.config,
metadataSchema: manifest.schema,
status: loaded?.status || 'unloaded',
error: loaded?.error,
isContainer: true, // All plugins are container-based
});
});
return plugins;
}
/**
* Get only active plugins
*/
getActivePlugins(): PluginInfo[] {
return this.getPlugins().filter(p => p.active);
}
/**
* Get execution order (dependency-sorted active plugin IDs)
*/
getExecutionOrder(): string[] {
return [...this.executionOrder];
}
/**
* Create a new KV store for file processing
*/
createKVStore(initialData?: Record<string, string>): KVStore {
return new KVStore(initialData);
}
// =========================================================================
// Shutdown
// =========================================================================
/**
* Shutdown: clear plugin state
*/
async shutdown(): Promise<void> {
this.manifests.clear();
this.loadedPlugins.clear();
this.activePluginIds.clear();
this.executionOrder = [];
console.log('[PluginManager] Shutdown complete');
}
// =========================================================================
// Private Helpers
// =========================================================================
private rebuildExecutionOrder(): void {
const { order, errors } = resolveDependencyOrder(
this.manifests,
this.activePluginIds
);
if (errors.length > 0) {
for (const error of errors) {
console.warn(`Dependency resolution warning: ${error}`);
}
}
this.executionOrder = order;
}
private async loadState(): Promise<PluginState> {
try {
if (existsSync(this.statePath)) {
const content = await fs.readFile(this.statePath, 'utf8');
return JSON.parse(content) as PluginState;
}
} catch (error) {
console.error('Failed to load plugin state:', error);
}
return { plugins: {} };
}
private async saveState(): Promise<void> {
const state: PluginState = { plugins: {} };
this.manifests.forEach((_, pluginId) => {
state.plugins[pluginId] = {
active: this.activePluginIds.has(pluginId),
config: this.configs.get(pluginId) || {},
};
});
const dir = path.dirname(this.statePath);
if (!existsSync(dir)) {
await fs.mkdir(dir, { recursive: true });
}
await fs.writeFile(this.statePath, JSON.stringify(state, null, 2), 'utf8');
}
}
// =============================================================================
// Factory Function
// =============================================================================
/**
* Create a plugin manager with default paths based on config
*/
export function createPluginManager(
statePath: string,
cacheDir: string
): PluginManager {
return new PluginManager({
statePath,
cacheDir,
});
}
packages/meta-sort-core/src/plugin-engine/TaskScheduler.ts
+634
View File
@@ -0,0 +1,634 @@
/**
* Task Scheduler
*
* Manages plugin task execution across fast and background queues.
* All plugins run as Docker containers - task execution is delegated to ContainerPluginScheduler.
*/
import { EventEmitter } from 'events';
import PQueue from 'p-queue';
import type {
PluginTask,
PluginTaskStatus,
PluginTaskResult,
TaskQueueType,
QueueStatus,
TaskSchedulerEvent,
PluginManifest,
PluginKVStore,
} from './types.js';
import type { PluginManager } from './PluginManager.js';
import type { PerformanceMetrics, PluginTimingStats } from '../metrics/PerformanceMetrics.js';
import type { ContainerPluginScheduler } from '../container-plugins/index.js';
import { KVStore } from './PluginContext.js';
// =============================================================================
// Configuration
// =============================================================================
export interface TaskSchedulerConfig {
/** Concurrency for fast queue (default: 32) */
fastQueueConcurrency?: number;
/** Concurrency for background queue (default: 8) */
backgroundQueueConcurrency?: number;
/** Threshold in ms for fast classification (default: 1000) */
fastThresholdMs?: number;
/** Minimum samples before using measured timing (default: 10) */
minSamplesForMeasurement?: number;
}
const DEFAULT_CONFIG: Required<TaskSchedulerConfig> = {
fastQueueConcurrency: 32,
backgroundQueueConcurrency: 8,
fastThresholdMs: 1000,
minSamplesForMeasurement: 10,
};
// =============================================================================
// Task Scheduler
// =============================================================================
export class TaskScheduler extends EventEmitter {
private pluginManager: PluginManager;
private performanceMetrics: PerformanceMetrics;
private config: Required<TaskSchedulerConfig>;
// Queues
private fastQueue: PQueue;
private backgroundQueue: PQueue;
// Task tracking
private pendingTasks: Map<string, PluginTask> = new Map();
private completedTasks: Set<string> = new Set();
private fileTasks: Map<string, Set<string>> = new Map(); // fileHash -> taskIds
private fileKVStores: Map<string, PluginKVStore> = new Map(); // fileHash -> KV store
// Container plugin support
private containerPluginScheduler: ContainerPluginScheduler | null = null;
// Statistics
private totalTasksCreated: number = 0;
private totalTasksCompleted: number = 0;
private totalTasksFailed: number = 0;
constructor(
pluginManager: PluginManager,
performanceMetrics: PerformanceMetrics,
config: TaskSchedulerConfig = {}
) {
super();
this.pluginManager = pluginManager;
this.performanceMetrics = performanceMetrics;
this.config = { ...DEFAULT_CONFIG, ...config };
// Initialize fast queue
this.fastQueue = new PQueue({
concurrency: this.config.fastQueueConcurrency,
});
// Initialize background queue (starts paused)
this.backgroundQueue = new PQueue({
concurrency: this.config.backgroundQueueConcurrency,
autoStart: false,
});
// When fast queue becomes idle, start background queue
this.fastQueue.on('idle', () => {
this.onFastQueueIdle();
});
}
/**
* Set the container plugin scheduler (called after initialization)
*/
setContainerPluginScheduler(scheduler: ContainerPluginScheduler): void {
this.containerPluginScheduler = scheduler;
// Forward container scheduler events to this scheduler
scheduler.on('task:completed', ({ task }) => {
this.onContainerTaskComplete(task.id, task.fileHash, true);
});
scheduler.on('task:failed', ({ task }) => {
this.onContainerTaskComplete(task.id, task.fileHash, false);
});
scheduler.on('file:complete', ({ fileHash, filePath }) => {
// Container file completion - check if local tasks are also done
this.checkFileComplete(fileHash);
});
console.log('[TaskScheduler] Container plugin scheduler set');
}
/**
* Handle completion of a container plugin task
*/
private onContainerTaskComplete(taskId: string, fileHash: string, success: boolean): void {
// Move from pending to completed
this.pendingTasks.delete(taskId);
this.completedTasks.add(taskId);
if (success) {
this.totalTasksCompleted++;
} else {
this.totalTasksFailed++;
}
// Schedule dependent tasks
const task = { id: taskId, fileHash, pluginId: taskId.split(':')[1] } as PluginTask;
this.scheduleDependentTasks(task);
// Check if file is complete
this.checkFileComplete(fileHash);
}
// =========================================================================
// Task Creation
// =========================================================================
/**
* Create tasks for all active plugins for a file
* @param filePath - Absolute path to the file
* @param fileHash - File identity hash (midhash256)
* @param kv - KV store for the file (will be shared across all plugins)
*/
createTasksForFile(filePath: string, fileHash: string, kv?: PluginKVStore): PluginTask[] {
const tasks: PluginTask[] = [];
const executionOrder = this.pluginManager.getExecutionOrder();
const plugins = this.pluginManager.getPlugins();
const pluginMap = new Map(plugins.map(p => [p.id, p]));
// Initialize file task tracking
if (!this.fileTasks.has(fileHash)) {
this.fileTasks.set(fileHash, new Set());
}
const fileTaskIds = this.fileTasks.get(fileHash)!;
// Store KV for this file (create new if not provided)
if (!this.fileKVStores.has(fileHash)) {
this.fileKVStores.set(fileHash, kv || new KVStore());
}
for (const pluginId of executionOrder) {
const plugin = pluginMap.get(pluginId);
if (!plugin || !plugin.active) continue;
const taskId = `${fileHash}:${pluginId}`;
// Skip if task already exists
if (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId)) {
continue;
}
const timing = this.performanceMetrics.getPluginTiming(pluginId);
const queue = this.classifyPlugin(pluginId);
const task: PluginTask = {
id: taskId,
fileHash,
filePath,
pluginId,
dependencies: plugin.dependencies || [],
queue,
priority: timing?.average || 0,
estimatedTimeMs: timing?.average || 0,
status: 'pending',
createdAt: Date.now(),
};
tasks.push(task);
fileTaskIds.add(taskId);
this.totalTasksCreated++;
this.emitEvent({ type: 'task:created', task });
}
return tasks;
}
/**
* Create a task for a specific plugin on a single file
* Used for recompute scenarios where we want to run just one plugin
* @param pluginId - Plugin to execute
* @param filePath - Absolute path to the file
* @param fileHash - File identity hash (midhash256)
* @param options - Runtime options merged into plugin config (e.g., { forceRecompute: true })
* @param kvData - Optional initial KV data (flattened metadata from Redis)
* @returns The created task, or null if plugin not found/not active
*/
createTaskForPlugin(
pluginId: string,
filePath: string,
fileHash: string,
options?: Record<string, unknown>,
kvData?: Record<string, string>
): PluginTask | null {
const plugins = this.pluginManager.getPlugins();
const plugin = plugins.find(p => p.id === pluginId);
if (!plugin || !plugin.active) {
console.warn(`[TaskScheduler] Plugin '${pluginId}' not found or not active`);
return null;
}
// Use a unique task ID for recompute tasks to avoid conflicts
const taskId = options?.forceRecompute
? `${fileHash}:${pluginId}:recompute:${Date.now()}`
: `${fileHash}:${pluginId}`;
// Skip if non-recompute task already exists
if (!options?.forceRecompute && (this.pendingTasks.has(taskId) || this.completedTasks.has(taskId))) {
return null;
}
// Initialize file task tracking if needed
if (!this.fileTasks.has(fileHash)) {
this.fileTasks.set(fileHash, new Set());
}
const fileTaskIds = this.fileTasks.get(fileHash)!;
// Initialize or update KV store for this file
// For recompute, we want to load existing metadata so plugin can see current state
if (!this.fileKVStores.has(fileHash)) {
this.fileKVStores.set(fileHash, new KVStore(kvData));
} else if (kvData) {
// Update existing KV store with provided data
const existingKv = this.fileKVStores.get(fileHash)!;
for (const [key, value] of Object.entries(kvData)) {
existingKv.set(key, value);
}
}
const timing = this.performanceMetrics.getPluginTiming(pluginId);
const queue = this.classifyPlugin(pluginId);
const task: PluginTask = {
id: taskId,
fileHash,
filePath,
pluginId,
dependencies: plugin.dependencies || [],
queue,
priority: timing?.average || 0,
estimatedTimeMs: timing?.average || 0,
status: 'pending',
createdAt: Date.now(),
options,
};
fileTaskIds.add(taskId);
this.totalTasksCreated++;
this.emitEvent({ type: 'task:created', task });
return task;
}
/**
* Create tasks for a specific plugin on multiple files
* Used for "recompute plugin on all files" feature
* @param pluginId - Plugin to execute
* @param files - Array of { filePath, fileHash, kvData? } objects
* @param options - Runtime options merged into plugin config
* @returns Array of created tasks
*/
createTasksForPluginOnFiles(
pluginId: string,
files: Array<{ filePath: string; fileHash: string; kvData?: Record<string, string> }>,
options?: Record<string, unknown>
): PluginTask[] {
const tasks: PluginTask[] = [];
for (const { filePath, fileHash, kvData } of files) {
const task = this.createTaskForPlugin(pluginId, filePath, fileHash, options, kvData);
if (task) {
tasks.push(task);
}
}
console.log(`[TaskScheduler] Created ${tasks.length} tasks for plugin '${pluginId}'`);
return tasks;
}
/**
* Enqueue a task for execution
* All plugins are container-based, so all tasks go to the container scheduler
*/
enqueueTask(task: PluginTask): void {
this.pendingTasks.set(task.id, task);
this.enqueueContainerTask(task);
}
/**
* Enqueue multiple tasks
*/
enqueueTasks(tasks: PluginTask[]): void {
for (const task of tasks) {
this.enqueueTask(task);
}
}
/**
* Enqueue a task to the container plugin scheduler
*/
private enqueueContainerTask(task: PluginTask): void {
if (!this.containerPluginScheduler) {
console.warn(
`[TaskScheduler] Container plugin task ${task.id} cannot be scheduled: no container scheduler`
);
task.status = 'failed';
task.result = {
success: false,
timeMs: 0,
error: 'Container plugin scheduler not available',
};
this.pendingTasks.delete(task.id);
this.completedTasks.add(task.id);
this.totalTasksFailed++;
return;
}
// Create container task and enqueue
const containerTask = this.containerPluginScheduler.createTask(
task.pluginId,
task.fileHash,
task.filePath,
task.dependencies,
task.queue
);
// Map the local task ID to the container task ID
// Update our pending tasks map with the container task ID
this.pendingTasks.delete(task.id);
this.pendingTasks.set(containerTask.id, {
...task,
id: containerTask.id, // Use container task ID
});
// Update file task tracking
const fileTaskIds = this.fileTasks.get(task.fileHash);
if (fileTaskIds) {
fileTaskIds.delete(task.id);
fileTaskIds.add(containerTask.id);
}
// Enqueue to container scheduler
this.containerPluginScheduler.enqueueTask(containerTask);
}
// =========================================================================
// Classification
// =========================================================================
/**
* Classify a plugin as fast or background based on timing data and manifest
*/
classifyPlugin(pluginId: string): TaskQueueType {
const plugins = this.pluginManager.getPlugins();
const plugin = plugins.find(p => p.id === pluginId);
const timing = this.performanceMetrics.getPluginTiming(pluginId);
// Get manifest default queue (need to access via plugin manager internals)
// For now, we'll use a workaround - check if plugin has defaultQueue in manifest
const manifest = this.getPluginManifest(pluginId);
const defaultQueue = manifest?.defaultQueue || 'fast';
// If we have enough samples, use measured timing
if (timing && timing.count >= this.config.minSamplesForMeasurement) {
return timing.average < this.config.fastThresholdMs ? 'fast' : 'background';
}
// Otherwise use manifest default
return defaultQueue;
}
// =========================================================================
// Task Completion
// =========================================================================
private scheduleDependentTasks(completedTask: PluginTask): void {
// Container scheduler handles dependency resolution
// This method is kept for compatibility but doesn't need to do anything
}
private checkFileComplete(fileHash: string): void {
const fileTaskIds = this.fileTasks.get(fileHash);
if (!fileTaskIds) return;
// Check if all tasks for this file are completed
let filePath = '';
for (const taskId of fileTaskIds) {
if (!this.completedTasks.has(taskId)) {
return; // Still have pending/running tasks
}
// Try to get filePath from any task
if (!filePath) {
const task = this.pendingTasks.get(taskId);
if (task) filePath = task.filePath;
}
}
// All tasks complete
this.emitEvent({
type: 'file:complete',
fileHash,
filePath,
});
// Clean up file tracking
this.fileTasks.delete(fileHash);
this.fileKVStores.delete(fileHash);
// Clean up completed task IDs for this file to prevent memory leak
for (const taskId of fileTaskIds) {
this.completedTasks.delete(taskId);
}
}
// =========================================================================
// Queue Control
// =========================================================================
private onFastQueueIdle(): void {
this.emitEvent({ type: 'queue:fast:idle' });
// Start background queue if it has pending work
if (this.backgroundQueue.size > 0 || this.backgroundQueue.pending > 0) {
console.log('[TaskScheduler] Fast queue idle, starting background queue');
this.backgroundQueue.start();
this.emitEvent({ type: 'queue:background:started' });
}
}
// =========================================================================
// Status & Metrics
// =========================================================================
/**
* Get status of both queues
*/
getQueueStatus(): { fast: QueueStatus; background: QueueStatus } {
return {
fast: this.buildQueueStatus('fast'),
background: this.buildQueueStatus('background'),
};
}
private buildQueueStatus(queueType: TaskQueueType): QueueStatus {
const queue = queueType === 'fast' ? this.fastQueue : this.backgroundQueue;
let pending = 0;
let ready = 0;
let running = 0;
let completed = 0;
let failed = 0;
// Count pending tasks for this queue type
for (const [, task] of this.pendingTasks) {
if (task.queue === queueType) {
if (task.status === 'pending') pending++;
else if (task.status === 'ready') ready++;
else if (task.status === 'running') running++;
}
}
// Count completed tasks
for (const taskId of this.completedTasks) {
const [, pluginId] = taskId.split(':');
if (this.classifyPlugin(pluginId) === queueType) {
completed++;
}
}
return {
pending,
ready: queue.size, // Tasks in queue waiting to run
running: queue.pending, // Tasks currently executing
completed,
failed, // Would need separate tracking
isPaused: queue.isPaused,
};
}
/**
* Get pending tasks for a specific file
*/
getPendingTasksForFile(fileHash: string): PluginTask[] {
const tasks: PluginTask[] = [];
const fileTaskIds = this.fileTasks.get(fileHash);
if (fileTaskIds) {
for (const taskId of fileTaskIds) {
const task = this.pendingTasks.get(taskId);
if (task) {
tasks.push(task);
}
}
}
return tasks;
}
/**
* Get the KV store for a file
*/
getFileKVStore(fileHash: string): PluginKVStore | undefined {
return this.fileKVStores.get(fileHash);
}
/**
* Cancel processing for a file (removes all pending tasks)
*/
cancelFile(fileHash: string): void {
const fileTaskIds = this.fileTasks.get(fileHash);
if (!fileTaskIds) return;
// Remove pending tasks
for (const taskId of fileTaskIds) {
this.pendingTasks.delete(taskId);
}
// Clean up file tracking
this.fileTasks.delete(fileHash);
this.fileKVStores.delete(fileHash);
console.log(`[TaskScheduler] Cancelled processing for file ${fileHash}`);
}
/**
* Get scheduler statistics
*/
getStats(): {
totalTasksCreated: number;
totalTasksCompleted: number;
totalTasksFailed: number;
pendingTaskCount: number;
completedTaskCount: number;
filesInProgress: number;
} {
return {
totalTasksCreated: this.totalTasksCreated,
totalTasksCompleted: this.totalTasksCompleted,
totalTasksFailed: this.totalTasksFailed,
pendingTaskCount: this.pendingTasks.size,
completedTaskCount: this.completedTasks.size,
filesInProgress: this.fileTasks.size,
};
}
// =========================================================================
// Shutdown
// =========================================================================
/**
* Gracefully shutdown the scheduler
*/
async shutdown(): Promise<void> {
console.log('[TaskScheduler] Shutting down...');
// Clear queues
this.fastQueue.clear();
this.backgroundQueue.clear();
// Wait for in-progress tasks to complete
await Promise.all([
this.fastQueue.onIdle(),
this.backgroundQueue.onIdle(),
]);
// Clear tracking
this.pendingTasks.clear();
this.completedTasks.clear();
this.fileTasks.clear();
this.fileKVStores.clear();
console.log('[TaskScheduler] Shutdown complete');
}
// =========================================================================
// Helpers
// =========================================================================
private getPluginManifest(pluginId: string): PluginManifest | undefined {
// Access manifests from PluginManager
const manifests = (this.pluginManager as any).manifests as Map<string, PluginManifest>;
return manifests.get(pluginId);
}
private emitEvent(event: TaskSchedulerEvent): void {
this.emit(event.type, event);
}
}
// =============================================================================
// Factory Function
// =============================================================================
/**
* Create a task scheduler instance
*/
export function createTaskScheduler(
pluginManager: PluginManager,
performanceMetrics: PerformanceMetrics,
config?: TaskSchedulerConfig
): TaskScheduler {
return new TaskScheduler(pluginManager, performanceMetrics, config);
}
packages/meta-sort-core/src/plugin-engine/index.ts
+59
View File
@@ -0,0 +1,59 @@
/**
* Plugin System
*
* Main entry point for the meta-sort container-based plugin system.
* All plugins run as Docker containers - there is no local plugin execution.
*/
// Types
export type {
// Manifest types
ConfigFieldType,
SchemaFieldType,
SelectOption,
ConfigField,
SchemaField,
PluginManifest,
TaskQueueType,
// State types
PluginStateEntry,
PluginState,
PluginStatus,
PluginInfo,
// Context types
PluginKVStore,
PluginCache,
PluginLogger,
PluginLifecycleContext,
PluginContext,
// Plugin interface
Plugin,
// Internal types
LoadedPlugin,
PluginManagerEvent,
// Task types
PluginTaskStatus,
PluginTaskResult,
PluginTask,
QueueStatus,
TaskSchedulerEvent,
} from './types.js';
// Context implementations (KVStore is still used for local state)
export { KVStore, MetadataNodeKVStore } from './PluginContext.js';
// Dependency resolution utilities
export {
resolveDependencyOrder,
canActivate,
canDeactivate,
} from './PluginExecutor.js';
export type { DependencyResolution } from './PluginExecutor.js';
// Plugin Manager
export { PluginManager, createPluginManager } from './PluginManager.js';
export type { PluginManagerOptions } from './PluginManager.js';
// Task Scheduler
export { TaskScheduler, createTaskScheduler } from './TaskScheduler.js';
export type { TaskSchedulerConfig } from './TaskScheduler.js';
packages/meta-sort-core/src/plugin-engine/plugin.test.ts
+667
View File
@@ -0,0 +1,667 @@
/**
* Plugin System Unit Tests
*/
import { expect } from 'chai';
import * as fs from 'fs/promises';
import * as path from 'path';
import { existsSync, mkdirSync, rmSync } from 'fs';
import { fileURLToPath } from 'url';
import {
KVStore,
PluginCacheImpl,
PluginLoggerImpl,
parseManifest,
validateManifest,
resolveDependencyOrder,
canActivate,
canDeactivate,
PluginManager,
} from './index.js';
import type { PluginManifest, Plugin, PluginContext } from './types.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const TEST_DIR = path.join(__dirname, '../../test-plugins');
const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache');
const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins.json');
// =============================================================================
// Test Helpers
// =============================================================================
async function setupTestDir() {
// Clean up
if (existsSync(TEST_DIR)) {
rmSync(TEST_DIR, { recursive: true });
}
if (existsSync(TEST_CACHE_DIR)) {
rmSync(TEST_CACHE_DIR, { recursive: true });
}
if (existsSync(TEST_STATE_PATH)) {
rmSync(TEST_STATE_PATH);
}
mkdirSync(TEST_DIR, { recursive: true });
mkdirSync(TEST_CACHE_DIR, { recursive: true });
}
async function cleanupTestDir() {
if (existsSync(TEST_DIR)) {
rmSync(TEST_DIR, { recursive: true });
}
if (existsSync(TEST_CACHE_DIR)) {
rmSync(TEST_CACHE_DIR, { recursive: true });
}
if (existsSync(TEST_STATE_PATH)) {
rmSync(TEST_STATE_PATH);
}
}
async function createTestPlugin(
id: string,
manifest: Partial<PluginManifest>,
processCode: string = 'ctx.kv.set("test/processed", "true");'
): Promise<string> {
const pluginDir = path.join(TEST_DIR, id);
mkdirSync(pluginDir, { recursive: true });
// Write manifest
const fullManifest = {
id,
name: manifest.name || id,
version: manifest.version || '1.0.0',
...manifest,
};
const manifestYaml = `id: ${fullManifest.id}
name: ${fullManifest.name}
version: ${fullManifest.version}
${fullManifest.description ? `description: ${fullManifest.description}` : ''}
${fullManifest.dependencies?.length ? `dependencies:\n${fullManifest.dependencies.map(d => ` - ${d}`).join('\n')}` : ''}
`;
await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml);
// Write plugin code
const pluginCode = `
export default {
async process(ctx) {
${processCode}
}
};
`;
await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode);
return pluginDir;
}
// =============================================================================
// KVStore Tests
// =============================================================================
describe('KVStore', () => {
it('should set and get values', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
expect(kv.get('file/name')).to.equal('test.mkv');
});
it('should return undefined for missing keys', () => {
const kv = new KVStore();
expect(kv.get('missing/key')).to.be.undefined;
});
it('should delete keys', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.delete('file/name');
expect(kv.get('file/name')).to.be.undefined;
});
it('should list all keys', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.set('video/codec', 'h264');
kv.set('audio/0/codec', 'aac');
const keys = kv.keys();
expect(keys).to.have.lengthOf(3);
expect(keys).to.include('file/name');
expect(keys).to.include('video/codec');
expect(keys).to.include('audio/0/codec');
});
it('should filter keys by prefix', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.set('audio/0/codec', 'aac');
kv.set('audio/0/language', 'eng');
kv.set('audio/1/codec', 'dts');
const audioKeys = kv.keys('audio/');
expect(audioKeys).to.have.lengthOf(3);
expect(audioKeys).to.include('audio/0/codec');
expect(audioKeys).to.include('audio/0/language');
expect(audioKeys).to.include('audio/1/codec');
});
it('should return entries', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.set('video/codec', 'h264');
const entries = kv.entries();
expect(entries).to.have.lengthOf(2);
expect(entries).to.deep.include(['file/name', 'test.mkv']);
expect(entries).to.deep.include(['video/codec', 'h264']);
});
it('should filter entries by prefix', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.set('audio/0/codec', 'aac');
kv.set('audio/1/codec', 'dts');
const audioEntries = kv.entries('audio/');
expect(audioEntries).to.have.lengthOf(2);
});
it('should initialize with data', () => {
const kv = new KVStore({ 'file/name': 'test.mkv', 'video/codec': 'h264' });
expect(kv.get('file/name')).to.equal('test.mkv');
expect(kv.get('video/codec')).to.equal('h264');
});
it('should convert to object', () => {
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
kv.set('video/codec', 'h264');
const obj = kv.toObject();
expect(obj).to.deep.equal({
'file/name': 'test.mkv',
'video/codec': 'h264',
});
});
it('should ignore null/undefined values', () => {
const kv = new KVStore();
kv.set('test', null as any);
kv.set('test2', undefined as any);
expect(kv.keys()).to.have.lengthOf(0);
});
it('should ignore empty keys', () => {
const kv = new KVStore();
kv.set('', 'value');
expect(kv.keys()).to.have.lengthOf(0);
});
});
// =============================================================================
// PluginCache Tests
// =============================================================================
describe('PluginCache', () => {
const cacheDir = path.join(TEST_CACHE_DIR, 'test-plugin');
beforeEach(async () => {
if (existsSync(cacheDir)) {
rmSync(cacheDir, { recursive: true });
}
});
afterEach(async () => {
if (existsSync(cacheDir)) {
rmSync(cacheDir, { recursive: true });
}
});
it('should write and read JSON', async () => {
const cache = new PluginCacheImpl(cacheDir);
const data = { foo: 'bar', num: 42 };
await cache.writeJson('test.json', data);
const result = await cache.readJson<typeof data>('test.json');
expect(result).to.deep.equal(data);
});
it('should return null for missing files', async () => {
const cache = new PluginCacheImpl(cacheDir);
const result = await cache.readJson('missing.json');
expect(result).to.be.null;
});
it('should check file existence', async () => {
const cache = new PluginCacheImpl(cacheDir);
expect(await cache.exists('test.json')).to.be.false;
await cache.writeJson('test.json', { foo: 'bar' });
expect(await cache.exists('test.json')).to.be.true;
});
it('should delete files', async () => {
const cache = new PluginCacheImpl(cacheDir);
await cache.writeJson('test.json', { foo: 'bar' });
await cache.delete('test.json');
expect(await cache.exists('test.json')).to.be.false;
});
it('should clear all files', async () => {
const cache = new PluginCacheImpl(cacheDir);
await cache.writeJson('test1.json', { a: 1 });
await cache.writeJson('test2.json', { b: 2 });
await cache.clear();
expect(await cache.exists('test1.json')).to.be.false;
expect(await cache.exists('test2.json')).to.be.false;
});
it('should get full path', () => {
const cache = new PluginCacheImpl(cacheDir);
const fullPath = cache.getPath('test.json');
expect(fullPath).to.equal(path.join(cacheDir, 'test.json'));
});
});
// =============================================================================
// Manifest Parsing Tests
// =============================================================================
describe('Manifest Parsing', () => {
beforeEach(setupTestDir);
afterEach(cleanupTestDir);
it('should parse valid manifest', async () => {
const manifestPath = path.join(TEST_DIR, 'manifest.yml');
await fs.writeFile(
manifestPath,
`id: test-plugin
name: Test Plugin
version: 1.0.0
description: A test plugin
dependencies:
- other-plugin
config:
apiKey:
type: string
required: true
secret: true
language:
type: string
default: en-US
`
);
const manifest = await parseManifest(manifestPath);
expect(manifest.id).to.equal('test-plugin');
expect(manifest.name).to.equal('Test Plugin');
expect(manifest.version).to.equal('1.0.0');
expect(manifest.description).to.equal('A test plugin');
expect(manifest.dependencies).to.deep.equal(['other-plugin']);
expect(manifest.config?.apiKey?.type).to.equal('string');
expect(manifest.config?.apiKey?.secret).to.be.true;
expect(manifest.config?.language?.default).to.equal('en-US');
});
it('should validate manifest - valid', () => {
const manifest: PluginManifest = {
id: 'test-plugin',
name: 'Test Plugin',
version: '1.0.0',
};
const errors = validateManifest(manifest);
expect(errors).to.have.lengthOf(0);
});
it('should validate manifest - invalid id', () => {
const manifest: PluginManifest = {
id: 'Test_Plugin',
name: 'Test Plugin',
version: '1.0.0',
};
const errors = validateManifest(manifest);
expect(errors).to.have.lengthOf(1);
expect(errors[0]).to.include('Invalid plugin id');
});
it('should validate manifest - invalid version', () => {
const manifest: PluginManifest = {
id: 'test-plugin',
name: 'Test Plugin',
version: 'invalid',
};
const errors = validateManifest(manifest);
expect(errors).to.have.lengthOf(1);
expect(errors[0]).to.include('Invalid version');
});
});
// =============================================================================
// Dependency Resolution Tests
// =============================================================================
describe('Dependency Resolution', () => {
it('should resolve simple dependency chain', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
]);
const active = new Set(['a', 'b', 'c']);
const { order, errors } = resolveDependencyOrder(plugins, active);
expect(errors).to.have.lengthOf(0);
expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b'));
expect(order.indexOf('b')).to.be.lessThan(order.indexOf('c'));
});
it('should handle diamond dependency', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['a'] }],
['d', { id: 'd', name: 'D', version: '1.0.0', dependencies: ['b', 'c'] }],
]);
const active = new Set(['a', 'b', 'c', 'd']);
const { order, errors } = resolveDependencyOrder(plugins, active);
expect(errors).to.have.lengthOf(0);
expect(order.indexOf('a')).to.be.lessThan(order.indexOf('b'));
expect(order.indexOf('a')).to.be.lessThan(order.indexOf('c'));
expect(order.indexOf('b')).to.be.lessThan(order.indexOf('d'));
expect(order.indexOf('c')).to.be.lessThan(order.indexOf('d'));
});
it('should detect circular dependencies', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['c'] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
]);
const active = new Set(['a', 'b', 'c']);
const { errors } = resolveDependencyOrder(plugins, active);
expect(errors.length).to.be.greaterThan(0);
expect(errors[0]).to.include('Circular dependency');
});
it('should report missing dependencies', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: ['missing'] }],
]);
const active = new Set(['a']);
const { errors } = resolveDependencyOrder(plugins, active);
expect(errors.length).to.be.greaterThan(0);
expect(errors[0]).to.include('does not exist');
});
it('should report inactive dependencies', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
]);
const active = new Set(['b']); // 'a' is not active
const { errors } = resolveDependencyOrder(plugins, active);
expect(errors.length).to.be.greaterThan(0);
expect(errors[0]).to.include('not active');
});
});
// =============================================================================
// Activation/Deactivation Tests
// =============================================================================
describe('Activation Checks', () => {
it('should allow activation when dependencies are met', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
]);
const active = new Set(['a']);
const { canActivate: can, missingDeps } = canActivate('b', plugins, active);
expect(can).to.be.true;
expect(missingDeps).to.have.lengthOf(0);
});
it('should prevent activation when dependencies are missing', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
]);
const active = new Set<string>(); // 'a' is not active
const { canActivate: can, missingDeps } = canActivate('b', plugins, active);
expect(can).to.be.false;
expect(missingDeps).to.deep.equal(['a']);
});
it('should allow deactivation when no dependents', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
]);
const active = new Set(['a', 'b']);
const { canDeactivate: can, dependents } = canDeactivate('b', plugins, active);
expect(can).to.be.true;
expect(dependents).to.have.lengthOf(0);
});
it('should prevent deactivation when has dependents', () => {
const plugins = new Map<string, PluginManifest>([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
]);
const active = new Set(['a', 'b']);
const { canDeactivate: can, dependents } = canDeactivate('a', plugins, active);
expect(can).to.be.false;
expect(dependents).to.deep.equal(['b']);
});
});
// =============================================================================
// Plugin Manager Integration Tests
// =============================================================================
describe('PluginManager', () => {
beforeEach(setupTestDir);
afterEach(async () => {
await cleanupTestDir();
});
it('should scan and discover plugins', async () => {
await createTestPlugin('test-plugin', {
name: 'Test Plugin',
version: '1.0.0',
});
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
const plugins = await manager.scanPlugins();
expect(plugins).to.have.lengthOf(1);
expect(plugins[0].id).to.equal('test-plugin');
expect(plugins[0].name).to.equal('Test Plugin');
});
it('should load and activate plugins', async () => {
await createTestPlugin('plugin-a', { name: 'Plugin A' });
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager.scanPlugins();
await manager.loadAll();
// Activate
await manager.activate('plugin-a');
const plugins = manager.getPlugins();
expect(plugins[0].active).to.be.true;
const order = manager.getExecutionOrder();
expect(order).to.include('plugin-a');
});
it('should respect dependency order in execution', async () => {
await createTestPlugin('plugin-a', { name: 'Plugin A' }, 'ctx.kv.set("order/a", Date.now().toString());');
await createTestPlugin(
'plugin-b',
{ name: 'Plugin B', dependencies: ['plugin-a'] },
'ctx.kv.set("order/b", Date.now().toString());'
);
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager.scanPlugins();
await manager.loadAll();
await manager.activate('plugin-a');
await manager.activate('plugin-b');
const order = manager.getExecutionOrder();
expect(order.indexOf('plugin-a')).to.be.lessThan(order.indexOf('plugin-b'));
});
it('should process files through plugins', async () => {
await createTestPlugin(
'test-plugin',
{ name: 'Test Plugin' },
`
ctx.kv.set('file/processed', 'true');
ctx.kv.set('file/path', ctx.filePath);
`
);
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager.scanPlugins();
await manager.loadAll();
await manager.activate('test-plugin');
const kv = manager.createKVStore();
const result = await manager.processFile('/test/file.mkv', kv);
expect(result.success).to.be.true;
expect(kv.get('file/processed')).to.equal('true');
expect(kv.get('file/path')).to.equal('/test/file.mkv');
});
it('should persist state across restarts', async () => {
await createTestPlugin('test-plugin', { name: 'Test Plugin' });
// First run
const manager1 = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager1.scanPlugins();
await manager1.loadAll();
await manager1.activate('test-plugin');
await manager1.updateConfig('test-plugin', { setting: 'value' });
await manager1.shutdown();
// Second run
const manager2 = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager2.scanPlugins();
await manager2.loadAll();
const plugins = manager2.getPlugins();
expect(plugins[0].active).to.be.true;
expect(plugins[0].config.setting).to.equal('value');
await manager2.shutdown();
});
it('should prevent deactivation of plugin with dependents', async () => {
await createTestPlugin('plugin-a', { name: 'Plugin A' });
await createTestPlugin('plugin-b', { name: 'Plugin B', dependencies: ['plugin-a'] });
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
await manager.scanPlugins();
await manager.loadAll();
await manager.activate('plugin-a');
await manager.activate('plugin-b');
try {
await manager.deactivate('plugin-a');
expect.fail('Should have thrown');
} catch (error) {
expect((error as Error).message).to.include('other plugins depend on it');
}
await manager.shutdown();
});
it('should emit events on activation/deactivation', async () => {
await createTestPlugin('test-plugin', { name: 'Test Plugin' });
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
const events: string[] = [];
manager.on('plugin:activated', (e) => events.push(`activated:${e.pluginId}`));
manager.on('plugin:deactivated', (e) => events.push(`deactivated:${e.pluginId}`));
await manager.scanPlugins();
await manager.loadAll();
await manager.activate('test-plugin');
await manager.deactivate('test-plugin');
expect(events).to.deep.equal(['activated:test-plugin', 'deactivated:test-plugin']);
await manager.shutdown();
});
});
packages/meta-sort-core/src/plugin-engine/types.ts
+429
View File
@@ -0,0 +1,429 @@
/**
* Plugin System Type Definitions
*
* This module defines all types for the meta-sort plugin system.
*/
// =============================================================================
// Plugin Manifest Types
// =============================================================================
/**
* Configuration field type in manifest
*/
export type ConfigFieldType = 'string' | 'number' | 'boolean' | 'select';
/**
* Schema field type for editor UI
*/
export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
/**
* Select option for config/schema fields
*/
export interface SelectOption {
value: string;
label: string;
}
/**
* Configuration field definition in manifest
*/
export interface ConfigField {
type: ConfigFieldType;
label?: string;
required?: boolean;
default?: unknown;
secret?: boolean;
options?: SelectOption[];
}
/**
* Metadata schema field definition (for editor UI)
*/
export interface SchemaField {
/** Display label */
label: string;
/** Field type for form control */
type?: SchemaFieldType;
/** Read-only field (not editable) */
readonly?: boolean;
/** Whether field is indexed (has {n} pattern) */
indexed?: boolean;
/** Options for select type */
options?: SelectOption[];
/** Hint text shown below field */
hint?: string;
/** URL template for clickable values (use {value} as placeholder) */
link?: string;
/** Pattern for JSON type to aggregate matching keys */
pattern?: string;
/** Minimum value for number type */
min?: number;
/** Maximum value for number type */
max?: number;
/** Placeholder text */
placeholder?: string;
}
/**
* Task queue type for plugin classification
*/
export type TaskQueueType = 'fast' | 'background';
/**
* Plugin manifest (parsed from manifest.yml)
*/
export interface PluginManifest {
id: string;
name: string;
version: string;
description?: string;
/** Plugin author */
author?: string;
/** Display priority (lower = higher priority) */
priority?: number;
/** Color for UI display */
color?: string;
dependencies?: string[];
config?: Record<string, ConfigField>;
schema?: Record<string, SchemaField>;
/**
* Default queue for this plugin.
* - 'fast': High concurrency queue for quick operations (<1s)
* - 'background': Low concurrency queue for slow operations (>=1s)
* Default: 'fast'
*/
defaultQueue?: TaskQueueType;
}
// =============================================================================
// Plugin State Types
// =============================================================================
/**
* Plugin state entry (stored in plugins.json)
*/
export interface PluginStateEntry {
active: boolean;
config: Record<string, unknown>;
}
/**
* Complete plugin state file structure
*/
export interface PluginState {
plugins: Record<string, PluginStateEntry>;
}
/**
* Plugin status
*/
export type PluginStatus = 'loaded' | 'error' | 'unloaded';
/**
* Plugin info (runtime representation)
*/
export interface PluginInfo {
id: string;
name: string;
version: string;
description?: string;
/** Display priority (lower = higher priority) */
priority?: number;
/** Color for UI display */
color?: string;
dependencies: string[];
active: boolean;
config: Record<string, unknown>;
configSchema?: Record<string, ConfigField>;
metadataSchema?: Record<string, SchemaField>;
status: PluginStatus;
error?: string;
/** True if this is a containerized plugin */
isContainer?: boolean;
}
// =============================================================================
// Plugin Context Types (injected into plugins)
// =============================================================================
/**
* Key-value store for file metadata
*/
export interface PluginKVStore {
/**
* Get a value by key
* @param key Path-style key, e.g., "video/codec" or "audio/0/language"
*/
get(key: string): string | undefined;
/**
* Set a value. Only string values are supported.
* @param key Path-style key
* @param value String value
*/
set(key: string, value: string): void;
/**
* Delete a key
*/
delete(key: string): void;
/**
* Get all keys, optionally filtered by prefix
* @param prefix Optional prefix filter, e.g., "audio/"
*/
keys(prefix?: string): string[];
/**
* Get all key-value pairs, optionally filtered by prefix
*/
entries(prefix?: string): Array<[string, string]>;
/**
* Add a value to a set (for RecordSet fields like genres, tags).
* Optional - only available with MetadataNodeKVStore.
*/
add?(key: string, value: string): void;
/**
* Get the full object at a path (for inspecting nested structures).
* Optional - only available with MetadataNodeKVStore.
*/
getObject?(key: string): any;
}
/**
* Cache utilities for persistent plugin storage
*/
export interface PluginCache {
/**
* Get full path for a cache file
* @param filename Relative filename within cache directory
*/
getPath(filename: string): string;
/**
* Read and parse a JSON cache file
* @returns Parsed data or null if not found
*/
readJson<T>(filename: string): Promise<T | null>;
/**
* Write data as JSON to cache file
*/
writeJson(filename: string, data: unknown): Promise<void>;
/**
* Check if a cache file exists
*/
exists(filename: string): Promise<boolean>;
/**
* Delete a cache file
*/
delete(filename: string): Promise<void>;
/**
* Clear all files in the plugin's cache directory
*/
clear(): Promise<void>;
}
/**
* Logger interface for plugins
*/
export interface PluginLogger {
debug(message: string, ...args: unknown[]): void;
info(message: string, ...args: unknown[]): void;
warn(message: string, ...args: unknown[]): void;
error(message: string, ...args: unknown[]): void;
}
/**
* Context passed to plugin lifecycle hooks
*/
export interface PluginLifecycleContext {
/** Plugin configuration from plugins.json */
config: Record<string, unknown>;
/** Absolute path to plugin's cache directory */
cacheDir: string;
/** Scoped logger */
log: PluginLogger;
}
/**
* Context passed to plugin process function
*/
export interface PluginContext extends PluginLifecycleContext {
/** Key-value storage for the current file's metadata */
kv: PluginKVStore;
/** Absolute path to the file being processed */
filePath: string;
/** Cache utilities for persistent storage */
cache: PluginCache;
/**
* Direct access to MetadataNode for plugins that need it.
* Use this for compatibility with existing libraries that expect MetadataNode.
* Only available when using MetadataNodeKVStore.
*/
metadata?: any;
}
// =============================================================================
// Plugin Interface
// =============================================================================
/**
* Plugin interface that all plugins must implement
*/
export interface Plugin {
/**
* Called once when the plugin is loaded at startup.
* Use for one-time initialization.
*/
onLoad?(ctx: PluginLifecycleContext): Promise<void>;
/**
* Called when the plugin is activated.
* Triggered at startup (if previously active) or from dashboard.
*/
onActivate?(ctx: PluginLifecycleContext): Promise<void>;
/**
* Called when the plugin is deactivated from the dashboard.
* Use for cleanup, releasing resources.
*/
onDeactivate?(ctx: PluginLifecycleContext): Promise<void>;
/**
* Called when the plugin is unloaded at shutdown.
*/
onUnload?(ctx: PluginLifecycleContext): Promise<void>;
/**
* Process a single file. Called for each file in the library.
* Must be implemented by all plugins.
*/
process(ctx: PluginContext): Promise<void>;
}
// =============================================================================
// Plugin Manager Types
// =============================================================================
/**
* Loaded plugin internal representation
*/
export interface LoadedPlugin {
manifest: PluginManifest;
instance: Plugin;
status: PluginStatus;
error?: string;
pluginDir: string;
}
/**
* Processing result for a single file
*/
export interface ProcessingResult {
success: boolean;
timings: Record<string, number>;
errors: Array<{ pluginId: string; error: string }>;
}
/**
* Plugin manager events
*/
export type PluginManagerEvent =
| { type: 'plugin:activated'; pluginId: string }
| { type: 'plugin:deactivated'; pluginId: string }
| { type: 'plugin:config-changed'; pluginId: string; config: Record<string, unknown> }
| { type: 'plugin:error'; pluginId: string; error: Error };
// =============================================================================
// Plugin Task Types (for TaskScheduler)
// =============================================================================
/**
* Status of a plugin task
*/
export type PluginTaskStatus = 'pending' | 'ready' | 'running' | 'completed' | 'failed';
/**
* Result of a plugin task execution
*/
export interface PluginTaskResult {
success: boolean;
timeMs: number;
/** KV updates to apply (key-value pairs) */
kvUpdates?: Record<string, string>;
error?: string;
}
/**
* Plugin task representing a single plugin execution for a file
*/
export interface PluginTask {
/** Unique task ID: `${fileHash}:${pluginId}` */
id: string;
/** File identity hash (midhash256) */
fileHash: string;
/** Absolute path to the file */
filePath: string;
/** Plugin to execute */
pluginId: string;
/** Plugin IDs this task depends on */
dependencies: string[];
/** Target queue (from classification) */
queue: TaskQueueType;
/** Priority based on avg execution time (lower = higher priority) */
priority: number;
/** Estimated execution time in ms (from performanceMetrics) */
estimatedTimeMs: number;
/** Current task status */
status: PluginTaskStatus;
/** Timestamp when task was created */
createdAt: number;
/** Timestamp when task started executing */
startedAt?: number;
/** Timestamp when task completed */
completedAt?: number;
/** Execution result */
result?: PluginTaskResult;
/** Runtime options merged into plugin config (e.g., forceRecompute) */
options?: Record<string, unknown>;
}
/**
* Queue status information
*/
export interface QueueStatus {
/** Number of tasks pending (waiting for dependencies) */
pending: number;
/** Number of tasks ready (in queue, waiting for worker) */
ready: number;
/** Number of tasks currently running */
running: number;
/** Number of tasks completed */
completed: number;
/** Number of tasks failed */
failed: number;
/** Whether the queue is currently paused */
isPaused: boolean;
}
/**
* Task scheduler events
*/
export type TaskSchedulerEvent =
| { type: 'task:created'; task: PluginTask }
| { type: 'task:ready'; task: PluginTask }
| { type: 'task:started'; task: PluginTask }
| { type: 'task:complete'; task: PluginTask }
| { type: 'task:failed'; task: PluginTask; error: string }
| { type: 'file:complete'; fileHash: string; filePath: string }
| { type: 'queue:fast:idle' }
| { type: 'queue:background:started' }
| { type: 'queue:background:paused' };
packages/meta-sort-core/src/plugin-engine/validate.js
+210
View File
@@ -0,0 +1,210 @@
/**
* Plugin System Validation Script
*
* Simple validation to ensure the plugin system works correctly.
* Run with: node dist/plugin2/validate.js
*/
import * as fs from 'fs/promises';
import * as path from 'path';
import { existsSync, mkdirSync, rmSync } from 'fs';
import { fileURLToPath } from 'url';
import {
KVStore,
PluginCacheImpl,
resolveDependencyOrder,
canActivate,
canDeactivate,
PluginManager,
} from './index.js';
const __dirname = path.dirname(fileURLToPath(import.meta.url));
const TEST_DIR = path.join(__dirname, '../../test-plugins-validation');
const TEST_CACHE_DIR = path.join(__dirname, '../../test-cache-validation');
const TEST_STATE_PATH = path.join(__dirname, '../../test-plugins-validation.json');
let passed = 0;
let failed = 0;
function assert(condition, message) {
if (condition) {
passed++;
console.log(` ✓ ${message}`);
} else {
failed++;
console.log(` ✗ ${message}`);
}
}
async function cleanup() {
if (existsSync(TEST_DIR)) rmSync(TEST_DIR, { recursive: true });
if (existsSync(TEST_CACHE_DIR)) rmSync(TEST_CACHE_DIR, { recursive: true });
if (existsSync(TEST_STATE_PATH)) rmSync(TEST_STATE_PATH);
}
async function setup() {
await cleanup();
mkdirSync(TEST_DIR, { recursive: true });
mkdirSync(TEST_CACHE_DIR, { recursive: true });
}
async function createTestPlugin(id, deps = []) {
const pluginDir = path.join(TEST_DIR, id);
mkdirSync(pluginDir, { recursive: true });
const manifestYaml = `id: ${id}
name: ${id}
version: 1.0.0
${deps.length ? `dependencies:\n${deps.map(d => ` - ${d}`).join('\n')}` : ''}
`;
await fs.writeFile(path.join(pluginDir, 'manifest.yml'), manifestYaml);
const pluginCode = `
export default {
async process(ctx) {
ctx.kv.set("${id}/processed", "true");
}
};
`;
await fs.writeFile(path.join(pluginDir, 'index.js'), pluginCode);
}
async function testKVStore() {
console.log('\nTesting KVStore...');
const kv = new KVStore();
kv.set('file/name', 'test.mkv');
assert(kv.get('file/name') === 'test.mkv', 'set and get value');
assert(kv.get('missing/key') === undefined, 'missing key returns undefined');
kv.set('audio/0/codec', 'aac');
kv.set('audio/0/language', 'eng');
kv.set('audio/1/codec', 'dts');
const audioKeys = kv.keys('audio/');
assert(audioKeys.length === 3, 'keys with prefix filters correctly');
kv.delete('file/name');
assert(kv.get('file/name') === undefined, 'delete removes key');
const obj = kv.toObject();
assert(typeof obj === 'object', 'toObject returns object');
}
async function testPluginCache() {
console.log('\nTesting PluginCache...');
const cacheDir = path.join(TEST_CACHE_DIR, 'test-cache');
const cache = new PluginCacheImpl(cacheDir);
await cache.writeJson('test.json', { foo: 'bar' });
const data = await cache.readJson('test.json');
assert(data?.foo === 'bar', 'writeJson and readJson work');
const exists = await cache.exists('test.json');
assert(exists === true, 'exists returns true for existing file');
const notExists = await cache.exists('missing.json');
assert(notExists === false, 'exists returns false for missing file');
await cache.delete('test.json');
const afterDelete = await cache.exists('test.json');
assert(afterDelete === false, 'delete removes file');
}
async function testDependencyResolution() {
console.log('\nTesting Dependency Resolution...');
const plugins = new Map([
['a', { id: 'a', name: 'A', version: '1.0.0', dependencies: [] }],
['b', { id: 'b', name: 'B', version: '1.0.0', dependencies: ['a'] }],
['c', { id: 'c', name: 'C', version: '1.0.0', dependencies: ['b'] }],
]);
const active = new Set(['a', 'b', 'c']);
const { order, errors } = resolveDependencyOrder(plugins, active);
assert(errors.length === 0, 'no errors for valid dependency chain');
assert(order.indexOf('a') < order.indexOf('b'), 'a comes before b');
assert(order.indexOf('b') < order.indexOf('c'), 'b comes before c');
// Test can activate
const { canActivate: can, missingDeps } = canActivate('b', plugins, new Set(['a']));
assert(can === true, 'can activate when dependencies met');
const { canActivate: cannot, missingDeps: missing } = canActivate('b', plugins, new Set());
assert(cannot === false, 'cannot activate when dependencies missing');
assert(missing.includes('a'), 'reports missing dependency');
// Test can deactivate
const { canDeactivate: canDeact } = canDeactivate('c', plugins, active);
assert(canDeact === true, 'can deactivate when no dependents');
const { canDeactivate: cannotDeact, dependents } = canDeactivate('a', plugins, active);
assert(cannotDeact === false, 'cannot deactivate when has dependents');
assert(dependents.includes('b'), 'reports dependent plugin');
}
async function testPluginManager() {
console.log('\nTesting PluginManager...');
await createTestPlugin('plugin-a');
await createTestPlugin('plugin-b', ['plugin-a']);
const manager = new PluginManager({
pluginsDir: TEST_DIR,
statePath: TEST_STATE_PATH,
cacheDir: TEST_CACHE_DIR,
});
const plugins = await manager.scanPlugins();
assert(plugins.length === 2, 'scans and discovers plugins');
await manager.loadAll();
assert(manager.getPlugins().length === 2, 'loads all plugins');
await manager.activate('plugin-a');
await manager.activate('plugin-b');
assert(manager.getActivePlugins().length === 2, 'activates plugins');
const order = manager.getExecutionOrder();
assert(order.indexOf('plugin-a') < order.indexOf('plugin-b'), 'respects dependency order');
const kv = manager.createKVStore();
const result = await manager.processFile('/test/file.mkv', kv);
assert(result.success === true, 'processes file successfully');
assert(kv.get('plugin-a/processed') === 'true', 'plugin-a executed');
assert(kv.get('plugin-b/processed') === 'true', 'plugin-b executed');
await manager.deactivate('plugin-b');
assert(manager.getActivePlugins().length === 1, 'deactivates plugin');
await manager.shutdown();
}
async function run() {
console.log('Plugin System Validation\n========================');
try {
await setup();
await testKVStore();
await testPluginCache();
await testDependencyResolution();
await testPluginManager();
await cleanup();
console.log(`\n========================`);
console.log(`Results: ${passed} passed, ${failed} failed`);
if (failed > 0) {
process.exit(1);
}
} catch (error) {
console.error('\nValidation error:', error);
await cleanup();
process.exit(1);
}
}
run();
packages/meta-sort-core/src/types/FileMetadata.ts
+21
View File
@@ -0,0 +1,21 @@
/**
* FileMetadata - Extended metadata for file processing
*
* Wraps HashMeta with application-specific fields for tracking
* file processing state.
*/
import { HashMeta } from '@metazla/meta-interface';
export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
/**
* Extended file metadata that includes processing state
* Extends the generic HashMeta with meta-mesh specific fields
*
* Note: No longer uses tempId - midhash256 is computed immediately and serves as permanent ID
*/
export interface FileMetadata extends HashMeta {
/** Current status of file processing */
processingStatus?: ProcessingStatus;
}
packages/meta-sort-core/src/utils/TempIdGenerator.ts
+29
View File
@@ -0,0 +1,29 @@
/**
* TempIdGenerator
*
* Generates temporary IDs for files before content hash is computed.
* Uses a deterministic approach based on file path, size, and mtime.
*/
import { createHash } from 'crypto';
import { Stats } from 'fs';
/**
* Generate a temporary ID for a file based on its path and stats
* This is deterministic - same file will always get same temp ID
*
* @param filePath - Absolute path to the file
* @param stats - File stats from fs.stat()
* @returns Temporary ID as MD5 hash (without prefix - prefix added by storage layer)
*/
export function generateTempId(filePath: string, stats: Stats): string {
// Create deterministic input from file metadata (NOT content)
const input = `${filePath}:${stats.size}:${stats.mtimeMs}`;
// Fast MD5 hash of the input
const hash = createHash('md5').update(input).digest('hex');
// Return hash without prefix - storage layer adds "tmp:" prefix
return hash;
}
packages/meta-sort-core/src/utils/TimeoutWrapper.ts
+76
View File
@@ -0,0 +1,76 @@
/**
* Timeout Wrapper Utility
*
* Provides timeout functionality for async operations with configurable retry logic
*/
export interface TimeoutConfig {
/** Base timeout in milliseconds (default: 30000ms = 30s) */
baseTimeout: number;
/** Timeout multiplier for each retry (default: 1.5x) */
timeoutMultiplier: number;
/** Maximum timeout in milliseconds (default: 600000ms = 10min) */
maxTimeout: number;
}
const DEFAULT_CONFIG: TimeoutConfig = {
baseTimeout: 30000, // 30 seconds
timeoutMultiplier: 1.5,
maxTimeout: 600000 // 10 minutes
};
/**
* Calculate timeout for a specific retry attempt
*/
export function calculateTimeout(retryCount: number, config: TimeoutConfig = DEFAULT_CONFIG): number {
const timeout = config.baseTimeout * Math.pow(config.timeoutMultiplier, retryCount);
return Math.min(timeout, config.maxTimeout);
}
/**
* Execute an async function with timeout
* @param fn - Function to execute
* @param timeoutMs - Timeout in milliseconds
* @param label - Label for error messages
* @returns Promise that rejects if timeout is exceeded
*/
export async function withTimeout<T>(
fn: () => Promise<T>,
timeoutMs: number,
label: string = 'Operation'
): Promise<T> {
let timeoutHandle: NodeJS.Timeout;
const timeoutPromise = new Promise<never>((_, reject) => {
timeoutHandle = setTimeout(() => {
reject(new Error(`${label} timed out after ${timeoutMs}ms`));
}, timeoutMs);
});
try {
const result = await Promise.race([fn(), timeoutPromise]);
clearTimeout(timeoutHandle!);
return result;
} catch (error) {
clearTimeout(timeoutHandle!);
throw error;
}
}
/**
* Execute an async function with timeout and retry logic
* @param fn - Function to execute
* @param retryCount - Current retry count (0-based)
* @param config - Timeout configuration
* @param label - Label for error messages
* @returns Promise that resolves with result or rejects with timeout error
*/
export async function withTimeoutAndRetry<T>(
fn: () => Promise<T>,
retryCount: number,
config: TimeoutConfig = DEFAULT_CONFIG,
label: string = 'Operation'
): Promise<T> {
const timeout = calculateTimeout(retryCount, config);
return withTimeout(fn, timeout, label);
}
packages/meta-sort-core/tsconfig.json
+16
View File
@@ -0,0 +1,16 @@
{
"compilerOptions": {
"target": "ESNext",
"module": "NodeNext",
"moduleResolution": "NodeNext",
"resolveJsonModule": true,
"sourceMap": true,
"outDir": "./dist",
"esModuleInterop": true,
"forceConsistentCasingInFileNames": true,
"strict": false,
"skipLibCheck": true
},
"include": ["src/**/*"],
"exclude": ["node_modules", "**/*.spec.ts", "**/*.test.ts"]
}
packages/meta-sort-editor/README.md
+140
View File
@@ -0,0 +1,140 @@
# @metazla/meta-editor
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface.
## Features
- **Search**: Find files by title, filename, or hash ID
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more
- **Batch Operations**: Update multiple files simultaneously
- **Real-time**: Direct integration with etcd via REST API
- **User-friendly**: Clean, intuitive interface with form validation
## Development
### Prerequisites
- Node.js 21.6.2+
- pnpm 10.19.0+
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
### Running Locally
```bash
# From the root directory
pnpm run start:editor
# Or from the package directory
cd packages/meta-editor
pnpm run dev
```
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
### Building
```bash
# From the root directory
pnpm run build:editor
# Or from the package directory
cd packages/meta-editor
pnpm run build
```
The built files will be in `dist/` directory.
## Docker Deployment
In the Docker development environment, the editor is automatically available at:
```
http://localhost/editor
```
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
## API Integration
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
- `POST /api/metadata/search` - Search for files
- `GET /api/metadata/:hashId` - Get file metadata
- `PUT /api/metadata/:hashId` - Update complete metadata
- `PUT /api/metadata/:hashId/property` - Update specific property
- `POST /api/metadata/batch` - Batch update multiple files
See `src/api/metadataApi.ts` for the API client implementation.
## Usage
### Searching for Files
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
2. Enter your search query
3. Click "Search"
4. Results will appear in the left sidebar
### Editing Metadata
1. Select a file from the search results
2. Modify any fields in the editor form
3. Click "Save Changes" to persist updates
4. Use "Reset" to discard changes
### Supported Metadata Fields
**Basic Information:**
- Title, Original Title, Show Title
- Video Type (Movie, TV, Anime, Other)
- Year, Season, Episode
**Jellyfin Metadata:**
- Plot, IMDb ID, TMDb ID, AniDB ID
- Director, Studio, Rating, Runtime
- Genres, Premiered Date, Release Date
**File Information:**
- Filename, File Size, MIME Type
- Hash values (SHA-256, MD5, etc.) - read-only
## Project Structure
```
packages/meta-editor/
├── src/
│ ├── api/ # API client for metadata operations
│ ├── components/ # React components
│ │ ├── SearchBar.tsx # Search interface
│ │ ├── FileList.tsx # Search results list
│ │ └── MetadataEditor.tsx # Main editing form
│ ├── types/ # TypeScript type definitions
│ ├── utils/ # Utility functions
│ ├── App.tsx # Main application component
│ ├── main.tsx # Entry point
│ └── index.css # Global styles
├── index.html
├── vite.config.ts
├── tsconfig.json
└── package.json
```
## Technology Stack
- **React 18** - UI framework
- **TypeScript 5.7** - Type safety
- **Vite 6** - Build tool and dev server
- **Native CSS** - Styling (no CSS framework)
## Contributing
When adding new features:
1. Add type definitions to `src/types/index.ts`
2. Update API client in `src/api/metadataApi.ts`
3. Create/update components as needed
4. Update this README with new features
## License
Part of the MetaMesh project.
packages/meta-sort-editor/eslint.config.js
+28
View File
@@ -0,0 +1,28 @@
import js from '@eslint/js'
import globals from 'globals'
import reactHooks from 'eslint-plugin-react-hooks'
import reactRefresh from 'eslint-plugin-react-refresh'
import tseslint from 'typescript-eslint'
export default tseslint.config(
{ ignores: ['dist'] },
{
extends: [js.configs.recommended, ...tseslint.configs.recommended],
files: ['**/*.{ts,tsx}'],
languageOptions: {
ecmaVersion: 2020,
globals: globals.browser,
},
plugins: {
'react-hooks': reactHooks,
'react-refresh': reactRefresh,
},
rules: {
...reactHooks.configs.recommended.rules,
'react-refresh/only-export-components': [
'warn',
{ allowConstantExport: true },
],
},
},
)
packages/meta-sort-editor/index.html
+13
View File
@@ -0,0 +1,13 @@
<!doctype html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<link rel="icon" type="image/svg+xml" href="/vite.svg" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>MetaMesh Metadata Editor</title>
</head>
<body>
<div id="root"></div>
<script type="module" src="/src/main.tsx"></script>
</body>
</html>
packages/meta-sort-editor/package-lock.json
Generated
+1570
View File
File diff suppressed because it is too large Load Diff
packages/meta-sort-editor/package.json
+23
View File
@@ -0,0 +1,23 @@
{
"name": "@meta-sort/editor",
"version": "0.1.0",
"private": true,
"type": "module",
"scripts": {
"dev": "vite",
"build": "tsc -b && vite build",
"preview": "vite preview"
},
"dependencies": {
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
"devDependencies": {
"@types/react": "^18.2.48",
"@types/react-dom": "^18.2.18",
"@vitejs/plugin-react": "^4.2.1",
"typescript": "^5.3.3",
"vite": "^5.0.12"
}
}
packages/meta-sort-editor/src/App.css
+185
View File
@@ -0,0 +1,185 @@
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.app-header {
padding: 2rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.app-header h1 {
margin: 0 0 0.5rem 0;
color: #646cff;
}
.app-description {
margin: 0;
color: rgba(255, 255, 255, 0.6);
}
.app-main {
flex: 1;
padding: 2rem;
max-width: 1400px;
width: 100%;
margin: 0 auto;
}
.app-error {
padding: 1rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
border-radius: 8px;
color: #ff6b6b;
margin-bottom: 1rem;
}
.app-loading {
text-align: center;
padding: 3rem 2rem;
color: rgba(255, 255, 255, 0.7);
font-size: 1.1rem;
}
.app-welcome {
text-align: center;
padding: 4rem 2rem;
color: rgba(255, 255, 255, 0.6);
}
.app-welcome h2 {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.8);
}
.app-welcome-features {
margin-top: 1rem;
margin-bottom: 1.5rem;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
}
.app-kv-browser {
margin-top: 1.5rem;
text-align: left;
max-width: 900px;
margin-left: auto;
margin-right: auto;
}
.app-content {
display: grid;
grid-template-columns: 350px 1fr;
gap: 1.5rem;
margin-top: 1rem;
}
.app-sidebar {
min-height: 400px;
}
.app-editor {
min-height: 400px;
}
.app-no-selection {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
min-height: 400px;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
text-align: center;
padding: 2rem;
}
.app-footer {
padding: 1.5rem 2rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-size: 0.9rem;
}
.app-footer a {
color: #646cff;
text-decoration: none;
}
.app-footer a:hover {
text-decoration: underline;
}
@media (max-width: 1024px) {
.app-content {
grid-template-columns: 1fr;
}
.app-sidebar {
order: 2;
}
.app-editor {
order: 1;
}
}
@media (max-width: 768px) {
.app-header,
.app-main,
.app-footer {
padding: 1rem;
}
.app-content {
gap: 1rem;
}
}
@media (prefers-color-scheme: light) {
.app-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.app-description {
color: rgba(0, 0, 0, 0.6);
}
.app-error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.app-loading {
color: rgba(0, 0, 0, 0.7);
}
.app-welcome {
color: rgba(0, 0, 0, 0.6);
}
.app-welcome h2 {
color: rgba(0, 0, 0, 0.8);
}
.app-welcome-features {
color: rgba(0, 0, 0, 0.5);
}
.app-no-selection {
background-color: rgba(0, 0, 0, 0.02);
color: rgba(0, 0, 0, 0.5);
}
.app-footer {
border-top-color: rgba(0, 0, 0, 0.1);
color: rgba(0, 0, 0, 0.5);
}
}
packages/meta-sort-editor/src/App.tsx
+246
View File
@@ -0,0 +1,246 @@
import { useState, useEffect } from 'react';
import { SearchBar } from './components/SearchBar';
import { FileList } from './components/FileList';
import { MetadataEditor } from './components/MetadataEditor';
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
import { KVTreeView } from './components/KVTreeView';
import { MetadataAPI } from './api/metadataApi';
import { SearchResult } from './types';
import './App.css';
function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
const [bulkEditMode, setBulkEditMode] = useState(false);
const [isSearching, setIsSearching] = useState(false);
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
const [error, setError] = useState<string | null>(null);
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
const getHashFromUrl = (): string | null => {
const path = window.location.pathname;
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
if (fileMatch) {
return fileMatch[1];
}
// Fallback to query param for backwards compatibility
const params = new URLSearchParams(window.location.search);
return params.get('hash');
};
// Update URL when file is selected
const updateUrlForFile = (hashId: string | undefined) => {
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
if (hashId) {
const newPath = `${basePath}/file/${hashId}`;
window.history.pushState({ hashId }, '', newPath);
} else {
window.history.pushState({}, '', basePath || '/editor/');
}
};
// Check for hash parameter in URL on component mount
useEffect(() => {
const loadHashFromUrl = async () => {
const hashParam = getHashFromUrl();
if (hashParam) {
try {
setIsLoadingDirectHash(true);
setError(null);
// Try to get metadata for this hash
const metadata = await MetadataAPI.getMetadata(hashParam);
// Create a search result from the metadata
const searchResult: SearchResult = {
hashId: hashParam,
metadata: metadata,
score: 1.0,
};
// Set the search results and select this file
setSearchResults([searchResult]);
setSelectedHashId(hashParam);
} catch (err: any) {
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsLoadingDirectHash(false);
}
}
};
loadHashFromUrl();
// Handle browser back/forward
const handlePopState = (event: PopStateEvent) => {
const hashId = event.state?.hashId || getHashFromUrl();
if (hashId) {
setSelectedHashId(hashId);
} else {
setSelectedHashId(undefined);
}
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []); // Empty dependency array means this runs once on mount
const handleSearch = async (query: string) => {
try {
setIsSearching(true);
setError(null);
// Unified search - searches across all fields
const response = await MetadataAPI.search({ query });
setSearchResults(response.results);
// Auto-select first result if available
if (response.results.length > 0) {
setSelectedHashId(response.results[0].hashId);
} else {
setSelectedHashId(undefined);
}
} catch (err: any) {
setError(err.message);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsSearching(false);
}
};
const handleSelectFile = (hashId: string) => {
if (!bulkEditMode) {
setSelectedHashId(hashId);
setSelectedHashIds(new Set());
updateUrlForFile(hashId);
}
};
const handleToggleSelection = (hashId: string) => {
const newSelection = new Set(selectedHashIds);
if (newSelection.has(hashId)) {
newSelection.delete(hashId);
} else {
newSelection.add(hashId);
}
setSelectedHashIds(newSelection);
};
const handleSelectAll = () => {
const allHashIds = new Set(searchResults.map(r => r.hashId));
setSelectedHashIds(allHashIds);
};
const handleClearSelection = () => {
setSelectedHashIds(new Set());
setBulkEditMode(false);
};
const handleBulkEditMode = (enabled: boolean) => {
setBulkEditMode(enabled);
if (enabled) {
setSelectedHashId(undefined);
} else {
setSelectedHashIds(new Set());
}
};
const handleMetadataSaved = () => {
// Optionally refresh the search results to reflect changes
console.log('Metadata saved successfully');
};
const handleBulkMetadataSaved = () => {
console.log('Bulk metadata saved successfully');
handleClearSelection();
};
return (
<div className="app">
<header className="app-header">
<h1>MetaMesh Metadata Editor</h1>
<p className="app-description">
Search and edit metadata for your media files
</p>
</header>
<main className="app-main">
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
{error && <div className="app-error">{error}</div>}
{isLoadingDirectHash && (
<div className="app-loading">Loading file metadata...</div>
)}
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
<div className="app-welcome">
<h2>Welcome to MetaMesh Metadata Editor</h2>
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
<p className="app-welcome-features">
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
</p>
<div className="app-kv-browser">
<KVTreeView />
</div>
</div>
)}
{searchResults.length > 0 && (
<div className="app-content">
<div className="app-sidebar">
<FileList
results={searchResults}
selectedHashId={selectedHashId}
selectedHashIds={selectedHashIds}
bulkEditMode={bulkEditMode}
onSelectFile={handleSelectFile}
onToggleSelection={handleToggleSelection}
onSelectAll={handleSelectAll}
onClearSelection={handleClearSelection}
onBulkEditMode={handleBulkEditMode}
/>
</div>
<div className="app-editor">
{bulkEditMode && selectedHashIds.size > 0 ? (
<BulkMetadataEditor
hashIds={Array.from(selectedHashIds)}
onSaved={handleBulkMetadataSaved}
onCancel={handleClearSelection}
/>
) : selectedHashId ? (
<MetadataEditor
key={selectedHashId}
hashId={selectedHashId}
onSaved={handleMetadataSaved}
/>
) : (
<div className="app-no-selection">
{bulkEditMode
? 'Select files from the list to bulk edit metadata'
: 'Select a file from the list to edit its metadata'}
</div>
)}
</div>
</div>
)}
</main>
<footer className="app-footer">
<p>
MetaMesh Metadata Editor | Part of{' '}
<a href="/" target="_blank">
MetaMesh
</a>
</p>
</footer>
</div>
);
}
export default App;
packages/meta-sort-editor/src/api/metadataApi.ts
+151
View File
@@ -0,0 +1,151 @@
import {
FileMetadata,
SearchResponse,
UpdateResult,
BatchUpdateResponse,
} from '../types';
const API_BASE = '/api/metadata';
export class MetadataAPI {
/**
* Search for files by query, hashId, or property
*/
static async search(params: {
query?: string;
hashId?: string;
property?: string;
propertyValue?: string;
limit?: number;
}): Promise<SearchResponse> {
const response = await fetch(`${API_BASE}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Search failed');
}
return response.json();
}
/**
* Get metadata for a specific file by hash ID
*/
static async getMetadata(hashId: string): Promise<FileMetadata> {
const response = await fetch(`${API_BASE}/${hashId}`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get metadata');
}
return response.json();
}
/**
* Update complete metadata for a file
*/
static async updateMetadata(
hashId: string,
metadata: FileMetadata
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(metadata),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update metadata');
}
return response.json();
}
/**
* Update a specific property for a file
*/
static async updateProperty(
hashId: string,
property: string,
value: string
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}/property`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ property, value }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update property');
}
return response.json();
}
/**
* Batch update multiple files
*/
static async batchUpdate(
updates: Array<{
hashId: string;
metadata?: FileMetadata;
properties?: Record<string, string>;
}>
): Promise<BatchUpdateResponse> {
const response = await fetch(`${API_BASE}/batch`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ updates }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Batch update failed');
}
return response.json();
}
/**
* Delete metadata for a file
*/
static async deleteMetadata(hashId: string): Promise<void> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'DELETE',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to delete metadata');
}
}
/**
* Get all hash IDs
*/
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
const response = await fetch(`${API_BASE}/hash-ids`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get hash IDs');
}
return response.json();
}
}
packages/meta-sort-editor/src/components/BulkMetadataEditor.css
+382
View File
@@ -0,0 +1,382 @@
.bulk-metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.bulk-metadata-editor-header {
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-editor-header h2 {
margin: 0 0 0.5rem 0;
font-size: 1.5rem;
}
.bulk-info {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.6);
font-size: 0.95rem;
}
.bulk-metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.bulk-metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.bulk-metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.bulk-metadata-editor-form {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
.bulk-metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.bulk-metadata-section-header {
margin-bottom: 1.5rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.section-hint {
margin: 0.25rem 0 0 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.section-hint-bulk {
margin: 0.5rem 0 0 0;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
font-weight: 500;
}
/* Plugin-based bulk fields */
.bulk-fields-container {
display: flex;
flex-direction: column;
gap: 1rem;
}
.bulk-field-wrapper {
border: 1px solid rgba(255, 255, 255, 0.05);
border-radius: 4px;
padding: 0.75rem;
background-color: rgba(0, 0, 0, 0.1);
transition: all 0.2s ease;
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.bulk-field-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.field-key {
font-size: 0.85rem;
font-weight: 600;
color: rgba(255, 255, 255, 0.6);
text-transform: lowercase;
font-family: 'Courier New', monospace;
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.bulk-field-input {
/* Plugin component will render inside this */
}
/* Legacy bulk fields (for compatibility) */
.bulk-metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.bulk-metadata-field:last-child {
margin-bottom: 0;
}
.field-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.field-header label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.clear-field-btn {
width: 20px;
height: 20px;
padding: 0;
border: 1px solid rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
border-radius: 50%;
cursor: pointer;
font-size: 16px;
line-height: 1;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
.bulk-metadata-field input,
.bulk-metadata-field textarea,
.bulk-metadata-field select {
width: 100%;
}
.bulk-metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.bulk-metadata-field-row:last-child {
margin-bottom: 0;
}
/* Modified fields summary */
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-fields h3 {
color: #646cff;
}
.modified-fields-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.modified-field-item {
display: flex;
gap: 0.75rem;
align-items: center;
padding: 0.75rem;
background-color: rgba(100, 108, 255, 0.05);
border: 1px solid rgba(100, 108, 255, 0.2);
border-radius: 4px;
}
.field-name {
font-weight: 600;
color: #646cff;
min-width: 150px;
font-family: 'Courier New', monospace;
font-size: 0.9rem;
}
.field-value {
flex: 1;
word-break: break-word;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
}
.remove-field-btn {
padding: 0.25rem 0.75rem;
font-size: 0.85rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
white-space: nowrap;
}
.remove-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
@media (max-width: 768px) {
.bulk-metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.bulk-metadata-editor-actions {
width: 100%;
flex-direction: column;
}
.bulk-metadata-editor-actions button {
width: 100%;
}
.bulk-metadata-field-row {
grid-template-columns: 1fr;
}
.modified-field-item {
flex-direction: column;
align-items: flex-start;
}
.field-name {
min-width: auto;
}
.remove-field-btn {
align-self: flex-end;
}
}
@media (prefers-color-scheme: light) {
.bulk-metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.bulk-info {
color: rgba(0, 0, 0, 0.6);
}
.bulk-metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.section-hint {
color: rgba(0, 0, 0, 0.5);
}
.section-hint-bulk {
color: rgba(0, 0, 0, 0.4);
}
.bulk-field-wrapper {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.field-key {
color: rgba(0, 0, 0, 0.6);
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.field-header label {
color: rgba(0, 0, 0, 0.7);
}
.field-value {
color: rgba(0, 0, 0, 0.8);
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-field-item {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.2);
}
.clear-field-btn {
border-color: rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #cc0000;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
}
packages/meta-sort-editor/src/components/BulkMetadataEditor.tsx
+280
View File
@@ -0,0 +1,280 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './BulkMetadataEditor.css';
interface PluginsResponse {
plugins: PluginSchema[];
}
interface BulkMetadataEditorProps {
hashIds: string[];
onSaved?: () => void;
onCancel?: () => void;
}
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
// Load plugins and metadata on mount
useEffect(() => {
const loadData = async () => {
try {
setIsLoading(true);
setError(null);
// Fetch plugins
const pluginsResponse = await fetch('/api/plugins');
if (pluginsResponse.ok) {
const data: PluginsResponse = await pluginsResponse.json();
// Sort by priority
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
// Load metadata for all selected files
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
const metadata = await Promise.all(metadataPromises);
// Calculate common values
const commons: Record<string, any> = {};
// Get all editable fields from all plugins
const allFields: string[] = [];
plugins.forEach((plugin) => {
if (plugin.metadataSchema) {
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
if (!field.readonly) {
allFields.push(key);
}
});
}
});
allFields.forEach((field) => {
const values = metadata.map((m) => m[field as keyof FileMetadata]);
// Filter out undefined and null values
const definedValues = values.filter((v) => v !== undefined && v !== null);
if (definedValues.length === 0) {
return;
}
// Safely stringify values
const uniqueValues = Array.from(
new Set(
definedValues.map((v) => {
try {
return JSON.stringify(v);
} catch {
return String(v);
}
})
)
);
if (uniqueValues.length === 1) {
try {
commons[field] = JSON.parse(uniqueValues[0]);
} catch {
commons[field] = uniqueValues[0];
}
} else {
commons[field] = '*';
}
});
setCommonValues(commons);
} catch (err: any) {
setError(`Failed to load data: ${err.message}`);
} finally {
setIsLoading(false);
}
};
loadData();
}, [hashIds]);
const handleFieldChange = (field: string, value: any) => {
setEditedFields({
...editedFields,
[field]: value,
});
};
const handleClearField = (field: string) => {
const newFields = { ...editedFields };
delete newFields[field];
setEditedFields(newFields);
};
const handleSave = async () => {
if (Object.keys(editedFields).length === 0) {
setError('No fields have been edited');
return;
}
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
// Build batch update payload
const updates = hashIds.map((hashId) => ({
hashId,
metadata: editedFields,
}));
const result = await MetadataAPI.batchUpdate(updates);
if (result.errors > 0) {
setError(
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
);
console.error('Batch update errors:', result.results.filter((r) => r.error));
} else {
setSuccessMessage(`Successfully updated ${result.success} files!`);
}
if (onSaved) {
onSaved();
}
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const hasChanges = Object.keys(editedFields).length > 0;
// Filter to only editable plugins (those with non-readonly fields)
const editablePlugins = plugins.filter((plugin) => {
if (!plugin.metadataSchema) return false;
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
});
if (isLoading) {
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-loading">
Loading metadata for {hashIds.length} files...
</div>
</div>
);
}
// Build display metadata (edited values or common values)
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-header">
<h2>Bulk Edit Metadata</h2>
<p className="bulk-info">
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
be updated. Fields showing "*" have different values across files.
</p>
<div className="bulk-metadata-editor-actions">
{onCancel && (
<button onClick={onCancel} disabled={isSaving}>
Cancel
</button>
)}
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
</button>
</div>
</div>
{successMessage && (
<div className="bulk-metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
<div className="bulk-metadata-editor-form">
{editablePlugins.map((plugin) => {
if (!plugin.metadataSchema) return null;
// Get editable fields for this plugin
const editableFields = Object.entries(plugin.metadataSchema).filter(
([, field]) => !field.readonly
);
if (editableFields.length === 0) return null;
return (
<section
key={plugin.id}
className="bulk-metadata-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="bulk-metadata-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="section-hint">{plugin.description}</p>
)}
<p className="section-hint-bulk">
Common values shown. "*" indicates different values. Edit to update all files.
</p>
</div>
<div className="plugin-fields">
{editableFields.map(([key, field]) => (
<DynamicFieldRenderer
key={key}
fieldKey={key}
schema={field}
metadata={displayMetadata as FileMetadata}
onChange={handleFieldChange}
readonly={false}
/>
))}
</div>
</section>
);
})}
{/* Modified Fields Summary */}
{hasChanges && (
<section className="bulk-metadata-section modified-fields">
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
<div className="modified-fields-list">
{Object.entries(editedFields).map(([key, value]) => (
<div key={key} className="modified-field-item">
<span className="field-name">{key}:</span>
<span className="field-value">
{Array.isArray(value)
? value.join(', ')
: typeof value === 'object'
? JSON.stringify(value)
: String(value || '(empty)')}
</span>
<button
className="remove-field-btn"
onClick={() => handleClearField(key)}
title="Remove this field"
>
Remove
</button>
</div>
))}
</div>
</section>
)}
</div>
</div>
);
}
packages/meta-sort-editor/src/components/DynamicFieldRenderer.css
+297
View File
@@ -0,0 +1,297 @@
.dynamic-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.dynamic-field:last-child {
margin-bottom: 0;
}
.dynamic-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
width: 100%;
padding: 0.5rem 0.75rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-size: 0.9rem;
background: rgba(0, 0, 0, 0.3);
color: #fff;
transition: border-color 0.2s;
}
.dynamic-field input:focus,
.dynamic-field select:focus,
.dynamic-field textarea:focus {
outline: none;
border-color: #646cff;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: rgba(0, 0, 0, 0.2);
color: rgba(255, 255, 255, 0.5);
cursor: not-allowed;
}
.dynamic-field textarea {
resize: vertical;
min-height: 80px;
font-family: inherit;
}
.field-hint {
display: block;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
cursor: pointer;
font-weight: normal;
color: rgba(255, 255, 255, 0.7);
}
.checkbox-label input[type="checkbox"] {
width: 1.1rem;
height: 1.1rem;
cursor: pointer;
accent-color: #646cff;
}
.checkbox-label input:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.readonly-value {
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.7);
font-family: 'Courier New', monospace;
font-size: 0.9rem;
word-break: break-all;
}
.field-link {
color: #646cff;
text-decoration: none;
}
.field-link:hover {
text-decoration: underline;
}
/* CID field styling */
.cid-field .cid-value-container {
display: flex;
align-items: center;
gap: 0.5rem;
}
.cid-field .cid-value {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: #a0d8ef;
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cid-field .cid-value:hover {
background: rgba(0, 0, 0, 0.4);
border-color: #646cff;
}
.cid-field .cid-open-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 0.75rem;
background: #646cff;
color: #fff;
border-radius: 4px;
text-decoration: none;
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
transition: background 0.2s;
}
.cid-field .cid-open-button:hover {
background: #535bf2;
}
/* JSON field styling */
.json-field .json-viewer {
background: rgba(0, 0, 0, 0.4);
color: #d4d4d4;
padding: 1rem;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
overflow-x: auto;
max-height: 300px;
overflow-y: auto;
margin: 0;
white-space: pre-wrap;
word-break: break-all;
border: 1px solid rgba(255, 255, 255, 0.1);
}
/* Language string field styling */
.language-string-field .language-entries {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.language-string-field .language-entry {
display: flex;
align-items: center;
gap: 0.5rem;
}
.language-string-field .lang-input {
flex: 1;
min-width: 0;
}
.language-string-field .lang-code {
display: inline-block;
width: 50px;
padding: 0.4rem 0.5rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.8rem;
color: #a0d8ef;
text-align: center;
flex-shrink: 0;
}
.language-string-field .lang-text {
flex: 1;
padding: 0.4rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.language-string-field .lang-remove-btn {
width: 32px;
height: 32px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 100, 100, 0.2);
border: 1px solid rgba(255, 100, 100, 0.3);
border-radius: 4px;
color: #ff6b6b;
font-size: 1.2rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-remove-btn:hover {
background: rgba(255, 100, 100, 0.3);
border-color: rgba(255, 100, 100, 0.5);
}
.language-string-field .lang-add-btn {
align-self: flex-start;
padding: 0.4rem 0.75rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
color: #646cff;
font-size: 0.85rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-add-btn:hover {
background: rgba(100, 108, 255, 0.3);
border-color: rgba(100, 108, 255, 0.5);
}
/* Light theme */
@media (prefers-color-scheme: light) {
.dynamic-field label {
color: rgba(0, 0, 0, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
border-color: rgba(0, 0, 0, 0.1);
background: #fff;
color: #333;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: #f5f5f5;
color: rgba(0, 0, 0, 0.5);
}
.field-hint {
color: rgba(0, 0, 0, 0.5);
}
.checkbox-label {
color: rgba(0, 0, 0, 0.7);
}
.readonly-value {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
.json-field .json-viewer {
background: #1e1e1e;
border-color: rgba(0, 0, 0, 0.2);
}
.language-string-field .lang-code {
background: rgba(100, 108, 255, 0.1);
color: #4a50c8;
}
.language-string-field .lang-text {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
}
packages/meta-sort-editor/src/components/DynamicFieldRenderer.tsx
+378
View File
@@ -0,0 +1,378 @@
import { SchemaField, FileMetadata } from '../types';
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
import './DynamicFieldRenderer.css';
interface DynamicFieldRendererProps {
fieldKey: string;
schema: SchemaField;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
readonly?: boolean;
}
/**
* Renders a single metadata field based on its schema definition
*/
export function DynamicFieldRenderer({
fieldKey,
schema,
metadata,
onChange,
readonly = false,
}: DynamicFieldRendererProps) {
const isReadonly = readonly || schema.readonly;
const fieldType = schema.type || 'string';
// Get value from metadata using path-style key
const getValue = (): any => {
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = fieldKey.split('/');
let value: any = metadata;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
};
// Set value in metadata using path-style key
const setValue = (newValue: any) => {
onChange(fieldKey, newValue);
};
const value = getValue();
// Render link if configured
const renderValueWithLink = (displayValue: string) => {
if (schema.link && displayValue) {
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
return (
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
{displayValue}
</a>
);
}
return displayValue;
};
// Render based on field type
switch (fieldType) {
case 'boolean':
return (
<div className="dynamic-field">
<label className="checkbox-label">
<input
type="checkbox"
checked={Boolean(value)}
onChange={(e) => setValue(e.target.checked)}
disabled={isReadonly}
/>
<span>{schema.label}</span>
</label>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'number':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="number"
value={value ?? ''}
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
disabled={isReadonly}
min={schema.min}
max={schema.max}
placeholder={schema.placeholder}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'select':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<select
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
>
<option value="">Select...</option>
{schema.options?.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'text':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<textarea
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
placeholder={schema.placeholder}
rows={4}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'array':
// Convert RecordSet to comma-separated string for display
const arrayValue = (() => {
if (!value) return '';
if (typeof value === 'string') return value;
if (typeof value === 'object') {
return Object.values(value).join(', ');
}
return '';
})();
// Convert comma-separated string back to RecordSet
const handleArrayChange = (str: string) => {
const items = str.split(',').map(s => s.trim()).filter(Boolean);
if (items.length === 0) {
setValue(undefined);
return;
}
const recordSet: Record<string, string> = {};
items.forEach((item, idx) => {
recordSet[idx.toString()] = item;
});
setValue(recordSet);
};
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="text"
value={arrayValue}
onChange={(e) => handleArrayChange(e.target.value)}
disabled={isReadonly}
placeholder={schema.placeholder || 'Comma-separated values'}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'json':
// Aggregate matching keys based on pattern
const jsonData = (() => {
if (!schema.pattern) {
return value;
}
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
const result: Record<string, any> = {};
// Recursively find all matching keys
const findMatching = (obj: any, currentPath: string = '') => {
if (!obj || typeof obj !== 'object') return;
for (const [key, val] of Object.entries(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
// Store in result with relative path
const relativePath = newPath.substring(patternPrefix.length - 1);
if (relativePath) {
setNestedValue(result, relativePath, val);
}
}
if (typeof val === 'object' && val !== null) {
findMatching(val, newPath);
}
}
};
findMatching(metadata);
return Object.keys(result).length > 0 ? result : undefined;
})();
if (!jsonData) {
return null;
}
return (
<div className="dynamic-field json-field">
<label>{schema.label}</label>
<pre className="json-viewer">
{JSON.stringify(jsonData, null, 2)}
</pre>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'cid':
// CID field type - displays CID with "Open in Editor" button
if (!value) {
return null;
}
const cidValue = String(value);
const editorUrl = `/editor/file/${cidValue}`;
return (
<div className="dynamic-field cid-field">
<label>{schema.label}</label>
<div className="cid-value-container">
<code
className="cid-value"
onClick={() => navigator.clipboard.writeText(cidValue)}
title="Click to copy"
>
{cidValue}
</code>
<a
href={editorUrl}
className="cid-open-button"
title="Open in Editor"
target="_blank"
rel="noopener noreferrer"
>
↗ Open
</a>
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'languageString':
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
const langEntries: Array<[string, string]> = value && typeof value === 'object'
? Object.entries(value).map(([k, v]) => [k, String(v)])
: [];
const handleLangCodeChange = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const newValue = { ...value };
const text = newValue[oldLang];
delete newValue[oldLang];
newValue[newLang] = text;
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleLangTextChange = (lang: string, text: string) => {
const newValue = { ...value, [lang]: text };
setValue(newValue);
};
const handleRemoveLang = (lang: string) => {
const newValue = { ...value };
delete newValue[lang];
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleAddLang = () => {
// Find a language code not already used, default to 'eng'
const usedLangs = new Set(langEntries.map(([k]) => k));
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
setValue({ ...value, [defaultLang]: '' });
};
if (isReadonly && langEntries.length === 0) {
return null;
}
return (
<div className="dynamic-field language-string-field">
<label>{schema.label}</label>
<div className="language-entries">
{langEntries.map(([lang, text]) => (
<div key={lang} className="language-entry">
{isReadonly ? (
<>
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
<span className="lang-text">{text}</span>
</>
) : (
<>
<LanguageCombobox
value={lang}
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
/>
<input
type="text"
className="lang-input"
value={text}
onChange={(e) => handleLangTextChange(lang, e.target.value)}
placeholder={schema.placeholder || 'Text...'}
/>
<button
type="button"
className="lang-remove-btn"
onClick={() => handleRemoveLang(lang)}
title="Remove"
>
×
</button>
</>
)}
</div>
))}
{!isReadonly && (
<button
type="button"
className="lang-add-btn"
onClick={handleAddLang}
>
+ Add Language
</button>
)}
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'string':
default:
// Show nothing if no value and readonly
if (isReadonly && !value) {
return null;
}
return (
<div className="dynamic-field">
<label>{schema.label}</label>
{isReadonly ? (
<div className="readonly-value">
{renderValueWithLink(String(value ?? ''))}
</div>
) : (
<input
type="text"
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
placeholder={schema.placeholder}
/>
)}
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
}
}
// Helper to set nested value in object
function setNestedValue(obj: any, path: string, value: any) {
const parts = path.split('/').filter(Boolean);
let current = obj;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
}
current = current[parts[i]];
}
if (parts.length > 0) {
current[parts[parts.length - 1]] = value;
}
}
packages/meta-sort-editor/src/components/DynamicPluginSection.css
+63
View File
@@ -0,0 +1,63 @@
.plugin-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.plugin-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-section-header .plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.plugin-fields {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Responsive grid for fields */
@media (min-width: 768px) {
.plugin-fields {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1rem;
}
/* Make text and json fields span full width */
.plugin-fields .dynamic-field:has(textarea),
.plugin-fields .json-field {
grid-column: 1 / -1;
}
}
/* Light theme */
@media (prefers-color-scheme: light) {
.plugin-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-section-header .plugin-description {
color: rgba(0, 0, 0, 0.5);
}
}
packages/meta-sort-editor/src/components/DynamicPluginSection.tsx
+126
View File
@@ -0,0 +1,126 @@
import { PluginSchema, FileMetadata, SchemaField } from '../types';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './DynamicPluginSection.css';
interface DynamicPluginSectionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
}
/**
* Renders a plugin section with all its fields based on schema
*/
export function DynamicPluginSection({
plugin,
metadata,
onChange,
}: DynamicPluginSectionProps) {
const schema = plugin.metadataSchema;
if (!schema || Object.keys(schema).length === 0) {
return null;
}
// Check if plugin has any data in metadata
const hasData = checkPluginHasData(schema, metadata);
if (!hasData) {
return null;
}
// Sort fields: non-readonly first, then by key
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
// Non-readonly fields first
if (a.readonly !== b.readonly) {
return a.readonly ? 1 : -1;
}
// Then alphabetically
return keyA.localeCompare(keyB);
});
return (
<section
className="plugin-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="plugin-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="plugin-description">{plugin.description}</p>
)}
</div>
<div className="plugin-fields">
{sortedFields.map(([fieldKey, fieldSchema]) => (
<DynamicFieldRenderer
key={fieldKey}
fieldKey={fieldKey}
schema={fieldSchema}
metadata={metadata}
onChange={onChange}
/>
))}
</div>
</section>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function checkPluginHasData(
schema: Record<string, SchemaField>,
metadata: FileMetadata
): boolean {
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
/**
* Get nested value from object using path-style key
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
/**
* Check if object has any key starting with prefix
*/
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
packages/meta-sort-editor/src/components/FileList.css
+174
View File
@@ -0,0 +1,174 @@
.file-list {
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
overflow: hidden;
}
.file-list-header {
padding: 0.75rem 1rem;
background-color: rgba(255, 255, 255, 0.05);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
justify-content: space-between;
align-items: center;
}
.file-count {
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
}
.bulk-edit-toggle {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(100, 108, 255, 0.1);
border: 1px solid rgba(100, 108, 255, 0.3);
color: #646cff;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.bulk-edit-toggle:hover {
background-color: rgba(100, 108, 255, 0.2);
border-color: rgba(100, 108, 255, 0.5);
}
.bulk-edit-toggle.active {
background-color: rgba(100, 108, 255, 0.3);
border-color: #646cff;
font-weight: 500;
}
.file-list-bulk-actions {
padding: 0.75rem 1rem;
background-color: rgba(100, 108, 255, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
gap: 1rem;
align-items: center;
}
.select-all-btn {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(255, 255, 255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.2);
color: rgba(255, 255, 255, 0.9);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.select-all-btn:hover {
background-color: rgba(255, 255, 255, 0.2);
border-color: rgba(255, 255, 255, 0.3);
}
.selection-count {
font-size: 0.9em;
color: #646cff;
font-weight: 500;
}
.file-list-items {
max-height: 500px;
overflow-y: auto;
}
.file-list-item {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
cursor: pointer;
transition: background-color 0.2s;
display: flex;
gap: 0.75rem;
align-items: flex-start;
}
.file-list-item-checkbox {
padding-top: 0.2rem;
}
.file-list-item-checkbox input[type='checkbox'] {
width: 18px;
height: 18px;
cursor: pointer;
}
.file-list-item-content {
flex: 1;
}
.file-list-item:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.2);
border-left: 3px solid #646cff;
}
.file-list-item-title {
font-weight: 500;
margin-bottom: 0.5rem;
}
.file-list-item-details {
display: flex;
gap: 1rem;
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
margin-bottom: 0.25rem;
}
.file-list-item-hash {
font-size: 0.8em;
color: rgba(255, 255, 255, 0.4);
font-family: 'Courier New', monospace;
}
.file-list-empty {
padding: 3rem 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
@media (prefers-color-scheme: light) {
.file-list {
border-color: rgba(0, 0, 0, 0.1);
}
.file-list-header {
background-color: rgba(0, 0, 0, 0.02);
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.file-count {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.file-list-item:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.1);
}
.file-list-item-details {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item-hash {
color: rgba(0, 0, 0, 0.4);
}
.file-list-empty {
color: rgba(0, 0, 0, 0.5);
}
}
packages/meta-sort-editor/src/components/FileList.tsx
+123
View File
@@ -0,0 +1,123 @@
import { SearchResult } from '../types';
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
import './FileList.css';
interface FileListProps {
results: SearchResult[];
selectedHashId?: string;
selectedHashIds: Set<string>;
bulkEditMode: boolean;
onSelectFile: (hashId: string) => void;
onToggleSelection: (hashId: string) => void;
onSelectAll: () => void;
onClearSelection: () => void;
onBulkEditMode: (enabled: boolean) => void;
}
export function FileList({
results,
selectedHashId,
selectedHashIds,
bulkEditMode,
onSelectFile,
onToggleSelection,
onSelectAll,
onClearSelection,
onBulkEditMode,
}: FileListProps) {
if (results.length === 0) {
return (
<div className="file-list-empty">
<p>No files found. Try searching for a title, filename, or hash ID.</p>
</div>
);
}
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
// Sort results by file path
const sortedResults = [...results].sort((a, b) => {
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
return pathA.localeCompare(pathB);
});
return (
<div className="file-list">
<div className="file-list-header">
<span className="file-count">{results.length} files found</span>
<button
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
onClick={() => onBulkEditMode(!bulkEditMode)}
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
>
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
</button>
</div>
{bulkEditMode && (
<div className="file-list-bulk-actions">
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
{allSelected ? 'Deselect All' : 'Select All'}
</button>
{selectedHashIds.size > 0 && (
<span className="selection-count">
{selectedHashIds.size} selected
</span>
)}
</div>
)}
<div className="file-list-items">
{sortedResults.map((result) => {
const isSelected = bulkEditMode
? selectedHashIds.has(result.hashId)
: selectedHashId === result.hashId;
return (
<div
key={result.hashId}
className={`file-list-item ${isSelected ? 'selected' : ''}`}
onClick={() => {
if (bulkEditMode) {
onToggleSelection(result.hashId);
} else {
onSelectFile(result.hashId);
}
}}
>
{bulkEditMode && (
<div className="file-list-item-checkbox">
<input
type="checkbox"
checked={selectedHashIds.has(result.hashId)}
onChange={(e) => {
e.stopPropagation();
onToggleSelection(result.hashId);
}}
/>
</div>
)}
<div className="file-list-item-content">
<div className="file-list-item-title">
{getDisplayName(result.metadata)}
</div>
<div className="file-list-item-details">
<span className="file-list-item-type">
{result.metadata.videoType || 'Unknown'}
</span>
<span className="file-list-item-size">
{formatBytes(result.metadata.byteSize)}
</span>
</div>
<div className="file-list-item-hash">
Hash: {truncateHash(result.hashId)}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}
packages/meta-sort-editor/src/components/KVTreeView.css
+247
View File
@@ -0,0 +1,247 @@
.kv-tree {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.1);
overflow: hidden;
}
.kv-tree-header {
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.kv-tree-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.8);
}
.kv-tree-info {
display: flex;
gap: 1rem;
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-refresh {
margin-left: auto;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
background-color: rgba(100, 108, 255, 0.2);
color: #646cff;
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.kv-tree-refresh:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.3);
}
.kv-tree-refresh:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-error {
padding: 0.5rem 1rem;
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
font-size: 0.8rem;
}
.kv-tree-content {
max-height: 400px;
overflow-y: auto;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 0.8rem;
}
.kv-tree-node {
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-node:last-child {
border-bottom: none;
}
.kv-tree-key {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-key:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-toggle {
width: 1rem;
color: rgba(255, 255, 255, 0.4);
font-size: 0.65rem;
}
.kv-tree-key-name {
flex: 1;
color: #88c0d0;
word-break: break-all;
}
.kv-tree-type {
padding: 0.1rem 0.4rem;
font-size: 0.65rem;
background-color: rgba(255, 255, 255, 0.1);
border-radius: 3px;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-value {
padding: 0.5rem 1rem 0.5rem 2.5rem;
background-color: rgba(0, 0, 0, 0.2);
border-top: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-fields {
width: 100%;
border-collapse: collapse;
}
.kv-tree-fields tr:hover {
background-color: rgba(255, 255, 255, 0.03);
}
.kv-field-name {
padding: 0.25rem 0.5rem 0.25rem 0;
color: #a3be8c;
white-space: nowrap;
vertical-align: top;
width: 1%;
}
.kv-field-value {
padding: 0.25rem 0;
color: rgba(255, 255, 255, 0.7);
word-break: break-all;
}
.kv-field-value pre {
margin: 0;
font-family: inherit;
white-space: pre-wrap;
font-size: 0.75rem;
}
.kv-tree-load-more {
display: block;
width: 100%;
padding: 0.75rem;
font-size: 0.8rem;
background-color: transparent;
color: #646cff;
border: none;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-empty {
padding: 2rem;
text-align: center;
color: rgba(255, 255, 255, 0.4);
}
/* Light mode */
@media (prefers-color-scheme: light) {
.kv-tree {
background-color: rgba(0, 0, 0, 0.02);
border-color: rgba(0, 0, 0, 0.1);
}
.kv-tree-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-header h3 {
color: rgba(0, 0, 0, 0.8);
}
.kv-tree-info {
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-error {
background-color: rgba(255, 0, 0, 0.08);
color: #cc0000;
}
.kv-tree-node {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-key:hover {
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-toggle {
color: rgba(0, 0, 0, 0.4);
}
.kv-tree-key-name {
color: #0077aa;
}
.kv-tree-type {
background-color: rgba(0, 0, 0, 0.08);
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-value {
background-color: rgba(0, 0, 0, 0.02);
border-top-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-fields tr:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.kv-field-name {
color: #228822;
}
.kv-field-value {
color: rgba(0, 0, 0, 0.7);
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-empty {
color: rgba(0, 0, 0, 0.4);
}
}

Some files were not shown because too many files have changed in this diff Show More