Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Worph
|
fa4826ee2e | ||
|
Worph
|
33f576c92a | ||
|
Worph
|
b4528a099a | ||
|
Worph
|
9280e23e83 | ||
|
Worph
|
f5cafa77b2 | ||
|
Worph
|
af51bbf943 | ||
|
Worph
|
985b000c24 | ||
|
Worph
|
021991b00d | ||
|
Worph
|
862f73d26d | ||
|
Worph
|
b855adb822 | ||
|
Worph
|
9bf2e623f9 | ||
|
Worph
|
7bdb54f9a7 | ||
|
Worph
|
e192632328 | ||
|
Worph
|
ba44daf7eb | ||
|
Worph
|
13fa36ffc7 | ||
|
Pierre
|
44b103c97e | ||
|
Pierre
|
31bb1245da | ||
|
Pierre
|
f77bb71b44 | ||
|
Pierre
|
b1d8a56c53 | ||
|
Pierre
|
f7315cbc3f | ||
|
Pierre
|
e3d26549d0 |
@@ -0,0 +1,6 @@
|
||||
# FIX CRLF always when developer has not set
|
||||
# Linux/Mac: git config --global core.autocrlf input
|
||||
# Windows: git config --global core.autocrlf true
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
* eol=lf
|
||||
@@ -1,386 +0,0 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Mount Watcher Script
|
||||
#
|
||||
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
|
||||
# Runs as a supervisord program, polling every 5 seconds.
|
||||
#
|
||||
# Uses findmnt to check actual mount status (source of truth).
|
||||
# Writes errors to /meta-core/mounts/errors/{id}.error
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
|
||||
CONFIG_DIR="/meta-core/mounts"
|
||||
CONFIG_FILE="$CONFIG_DIR/mounts.json"
|
||||
ERROR_DIR="$CONFIG_DIR/errors"
|
||||
FILES_PATH="${FILES_PATH:-/files}"
|
||||
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
||||
|
||||
# ANSI colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${GREEN}$1${NC}"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${YELLOW}$1${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${RED}$1${NC}"
|
||||
}
|
||||
|
||||
# Initialize directories
|
||||
init_dirs() {
|
||||
mkdir -p "$CONFIG_DIR" "$ERROR_DIR"
|
||||
|
||||
# Initialize empty config if not exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
echo '{"version":1,"mounts":[]}' > "$CONFIG_FILE"
|
||||
log "Initialized empty mounts config"
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if path is mounted
|
||||
is_mounted() {
|
||||
local mount_path="$1"
|
||||
findmnt -n "$mount_path" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# Clear error file for mount
|
||||
clear_error() {
|
||||
local id="$1"
|
||||
rm -f "$ERROR_DIR/$id.error"
|
||||
}
|
||||
|
||||
# Write error to file
|
||||
write_error() {
|
||||
local id="$1"
|
||||
local error="$2"
|
||||
echo "$(date -Iseconds)" > "$ERROR_DIR/$id.error"
|
||||
echo "$error" >> "$ERROR_DIR/$id.error"
|
||||
}
|
||||
|
||||
# Mount NFS share (always read-only)
|
||||
do_mount_nfs() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
|
||||
|
||||
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
|
||||
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
|
||||
local rc=$?
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "NFS mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "NFS mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Mount SMB/CIFS share (always read-only)
|
||||
do_mount_smb() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local share="$3"
|
||||
local mount_path="$4"
|
||||
local username="${5:-}"
|
||||
local password_obscured="${6:-}"
|
||||
local domain="${7:-}"
|
||||
local options="${8:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
|
||||
# If we have credentials, reveal password and build options
|
||||
if [ -n "$username" ]; then
|
||||
mount_opts="${mount_opts},username=${username}"
|
||||
|
||||
if [ -n "$password_obscured" ]; then
|
||||
# Reveal password using rclone (only exists in memory briefly)
|
||||
local password
|
||||
password=$(rclone reveal "$password_obscured" 2>/dev/null)
|
||||
if [ -n "$password" ]; then
|
||||
mount_opts="${mount_opts},password=${password}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$domain" ]; then
|
||||
mount_opts="${mount_opts},domain=${domain}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Add any additional options
|
||||
if [ -n "$options" ]; then
|
||||
if [ -n "$mount_opts" ]; then
|
||||
mount_opts="${mount_opts},${options}"
|
||||
else
|
||||
mount_opts="$options"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
|
||||
|
||||
# Execute mount directly (not via eval) to handle special chars in password
|
||||
local output
|
||||
local rc
|
||||
if [ -n "$mount_opts" ]; then
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
|
||||
rc=$?
|
||||
else
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
# Clear password from any variables (best effort)
|
||||
password=""
|
||||
mount_opts=""
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "SMB mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "SMB mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Mount rclone remote using RC API (always read-only)
|
||||
do_mount_rclone() {
|
||||
local id="$1"
|
||||
local remote="$2"
|
||||
local remote_path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Build the remote source string
|
||||
local fs="${remote}:${remote_path}"
|
||||
|
||||
# Use rclone RC API to mount (always read-only for safety)
|
||||
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
|
||||
|
||||
# Build JSON body for rclone mount with read-only option
|
||||
local json_body
|
||||
json_body=$(cat <<EOF
|
||||
{
|
||||
"fs": "${fs}",
|
||||
"mountPoint": "${mount_path}",
|
||||
"mountOpt": {
|
||||
"AllowOther": true,
|
||||
"ReadOnly": true
|
||||
},
|
||||
"vfsOpt": {
|
||||
"CacheMode": 2,
|
||||
"ReadOnly": true
|
||||
}
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
output=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-d "$json_body" \
|
||||
"http://127.0.0.1:5572/mount/mount" 2>&1)
|
||||
|
||||
# Wait a moment for mount to establish
|
||||
sleep 2
|
||||
|
||||
if is_mounted "$mount_path"; then
|
||||
clear_error "$id"
|
||||
log_success "rclone mount successful: $mount_path"
|
||||
return 0
|
||||
else
|
||||
local error_msg="rclone mount failed"
|
||||
# Try to extract error from response
|
||||
if echo "$output" | grep -q "error"; then
|
||||
error_msg="$output"
|
||||
fi
|
||||
write_error "$id" "$error_msg"
|
||||
log_error "rclone mount failed: $error_msg"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Unmount a path
|
||||
do_unmount() {
|
||||
local id="$1"
|
||||
local mount_path="$2"
|
||||
local type="$3"
|
||||
|
||||
if ! is_mounted "$mount_path"; then
|
||||
log "Already unmounted: $mount_path"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$type" = "rclone" ]; then
|
||||
# Use rclone RC API to unmount
|
||||
log "Unmounting rclone: $mount_path"
|
||||
curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
|
||||
else
|
||||
log "Unmounting: $mount_path"
|
||||
umount "$mount_path" 2>&1
|
||||
fi
|
||||
|
||||
# Wait a moment
|
||||
sleep 1
|
||||
|
||||
# Check if still mounted
|
||||
if is_mounted "$mount_path"; then
|
||||
# Force unmount with lazy option
|
||||
log_warn "Force unmounting (lazy): $mount_path"
|
||||
umount -l "$mount_path" 2>&1
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
if is_mounted "$mount_path"; then
|
||||
log_error "Failed to unmount: $mount_path"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "Unmounted: $mount_path"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Process all mounts from config
|
||||
process_mounts() {
|
||||
# Check if config file exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Parse JSON using python3
|
||||
local mounts
|
||||
mounts=$(python3 -c '
|
||||
import sys, json
|
||||
|
||||
try:
|
||||
with open("'"$CONFIG_FILE"'", "r") as f:
|
||||
data = json.load(f)
|
||||
except:
|
||||
sys.exit(0)
|
||||
|
||||
for m in data.get("mounts", []):
|
||||
# Output pipe-separated values
|
||||
fields = [
|
||||
m.get("id", ""),
|
||||
m.get("type", ""),
|
||||
str(m.get("enabled", True)),
|
||||
str(m.get("desiredMounted", False)),
|
||||
m.get("mountPath", ""),
|
||||
m.get("nfsServer", ""),
|
||||
m.get("nfsPath", ""),
|
||||
m.get("smbServer", ""),
|
||||
m.get("smbShare", ""),
|
||||
m.get("smbUsername", ""),
|
||||
m.get("smbPasswordObscured", ""),
|
||||
m.get("smbDomain", ""),
|
||||
m.get("rcloneRemote", ""),
|
||||
m.get("rclonePath", ""),
|
||||
m.get("options", "")
|
||||
]
|
||||
print("|".join(fields))
|
||||
' 2>/dev/null)
|
||||
|
||||
if [ -z "$mounts" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
|
||||
# Skip empty lines
|
||||
[ -z "$id" ] && continue
|
||||
|
||||
# Check actual mount status
|
||||
local is_mounted_now=false
|
||||
is_mounted "$mount_path" && is_mounted_now=true
|
||||
|
||||
# Handle disabled mounts - unmount if currently mounted
|
||||
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) disabled, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Handle desired state
|
||||
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
||||
# Should be mounted
|
||||
if [ "$is_mounted_now" = false ]; then
|
||||
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
||||
case "$type" in
|
||||
nfs)
|
||||
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
|
||||
;;
|
||||
smb)
|
||||
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
|
||||
;;
|
||||
rclone)
|
||||
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown mount type: $type"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
else
|
||||
# Should be unmounted
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) not desired, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
fi
|
||||
fi
|
||||
done <<< "$mounts"
|
||||
}
|
||||
|
||||
# Main loop
|
||||
main() {
|
||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
|
||||
|
||||
init_dirs
|
||||
|
||||
while true; do
|
||||
process_mounts
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
# Handle signals for graceful shutdown
|
||||
cleanup() {
|
||||
log "Shutting down mount watcher..."
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
main "$@"
|
||||
@@ -587,7 +587,7 @@ Deduplication logic remains identical. Fast-hash serves the same purpose as full
|
||||
|
||||
The Stremio addon retrieves files by metadata queries (title, season, episode), not by hash. The hash is used internally for metadata storage. Changing to fast-hash is transparent to the addon.
|
||||
|
||||
### Meta-Orbit (Distributed Synchronization)
|
||||
### meta-share (Distributed Synchronization)
|
||||
|
||||
**Impact:** Hash serves as content identifier for P2P synchronization.
|
||||
|
||||
@@ -716,7 +716,7 @@ MetaMesh implements a two-phase processing pipeline:
|
||||
**Phase 2: Hash Processing (Full Hashes - Background)**
|
||||
- Compute full-file hashes: SHA-256, SHA-1, MD5, CRC32 (slow, minutes)
|
||||
- Update etcd with additional hash metadata
|
||||
- Required for Meta-Orbit distributed synchronization
|
||||
- Required for meta-share distributed synchronization
|
||||
|
||||
### Why Both Are Needed
|
||||
|
||||
@@ -726,7 +726,7 @@ MetaMesh implements a two-phase processing pipeline:
|
||||
- **Fast deduplication**: Quickly identify potential duplicates
|
||||
|
||||
**Full Hash Purpose:**
|
||||
- **Meta-Orbit integration**: Distributed P2P synchronization requires cryptographically secure hashes
|
||||
- **meta-share integration**: Distributed P2P synchronization requires cryptographically secure hashes
|
||||
- **External system compatibility**: Torrent verification, IPFS CIDs
|
||||
- **Collision resolution**: If midhash256 collision detected, full SHA-256 provides definitive answer
|
||||
|
||||
@@ -744,7 +744,7 @@ Light Processing (32 workers, parallel)
|
||||
Hash Processing (8 workers, background)
|
||||
- Compute SHA-256, SHA-1, MD5, CRC32 (15+ minutes for 50GB)
|
||||
- Update etcd with full hashes
|
||||
- Enable Meta-Orbit synchronization ✓
|
||||
- Enable meta-share synchronization ✓
|
||||
```
|
||||
|
||||
### Collision Detection and Handling
|
||||
@@ -778,4 +778,4 @@ The trade-offs—weaker cryptographic guarantees, potential collisions for adver
|
||||
|
||||
This architecture aligns with Meta-Mesh's design philosophy: **optimize for the common case** (large media files, organic content, personal libraries) while maintaining **practicality and performance** over theoretical perfection.
|
||||
|
||||
Meta-Mesh leverages midhash256 for **immediate file availability** while computing full SHA-256 hashes **in the background** for Meta-Orbit distributed synchronization and external system compatibility. This dual-hash approach provides both speed and security.
|
||||
Meta-Mesh leverages midhash256 for **immediate file availability** while computing full SHA-256 hashes **in the background** for meta-share distributed synchronization and external system compatibility. This dual-hash approach provides both speed and security.
|
||||
|
||||
+27
-754
@@ -1,762 +1,35 @@
|
||||
# Meta-Sort Plugin System Architecture
|
||||
# Plugin System (Deprecated — See Container Plugins)
|
||||
|
||||
This document describes the plugin system architecture for meta-sort, enabling extensible metadata extraction through dynamically loaded plugins.
|
||||
> **This document is a historical pointer.** The in-process plugin system it
|
||||
> originally described — `plugins/<id>/{manifest.yml,index.ts}` directories
|
||||
> loaded via dynamic `import()` inside meta-sort-core — has been removed.
|
||||
> All plugins now run as Docker containers and communicate over HTTP.
|
||||
|
||||
## Overview
|
||||
For the current design, read:
|
||||
|
||||
The plugin system provides a modular architecture for metadata extraction. Plugins are self-contained units that:
|
||||
- **[containerized-plugin-architecture.md](./containerized-plugin-architecture.md)** — full container plugin design (HTTP contract, lifecycle, callbacks, KV access).
|
||||
- **[plugin-task-queue-architecture.md](./plugin-task-queue-architecture.md)** — how plugin work is dispatched across the fast / background queues.
|
||||
- Root **`CLAUDE.md`** (`Plugin Development` and `Runtime Plugin Configuration` sections) — operational quickstart.
|
||||
- Plugin sources at **`packages/plugins/metamesh-plugin-*`** (separate submodules) — `file-info`, `ffmpeg`, `filename-parser`, `jellyfin-nfo`, `tmdb`, `anime-detector`, `language`, `subtitle`, `subtitle-extractor`, `torrent`, `fullhash`.
|
||||
|
||||
- Are discovered and loaded from a plugins directory at startup
|
||||
- Can be activated/deactivated at runtime without restart
|
||||
- Process files sequentially in dependency order
|
||||
- Store metadata in a flat key-value format
|
||||
- Have access to their own persistent cache folder
|
||||
## What survived from the old design
|
||||
|
||||
## Directory Structure
|
||||
The `Plugin` / `PluginContext` / `PluginLifecycleContext` / `PluginKVStore`
|
||||
interfaces in `src/plugin-engine/types.ts` are still exported, but the
|
||||
container-only `PluginManager` (`src/plugin-engine/PluginManager.ts`) never
|
||||
constructs an in-process `Plugin` instance — manifests, lifecycle, and
|
||||
execution are owned by `ContainerManager` and `ContainerPluginScheduler`
|
||||
under `src/container-plugins/`. Treat the `Plugin`-interface types as
|
||||
vestigial and do not write new code against them.
|
||||
|
||||
```
|
||||
packages/meta-sort/packages/meta-sort-core/
|
||||
├── plugins/ # All plugins (shipped + custom)
|
||||
│ ├── file-info/
|
||||
│ │ ├── manifest.yml # Plugin metadata & config schema
|
||||
│ │ └── index.ts # Plugin implementation
|
||||
│ ├── ffmpeg/
|
||||
│ ├── filename-parser/
|
||||
│ ├── anime-detector/
|
||||
│ ├── jellyfin-nfo/
|
||||
│ ├── subtitle/
|
||||
│ ├── torrent/
|
||||
│ ├── language/
|
||||
│ ├── tmdb/
|
||||
│ └── debug/
|
||||
├── plugins.json # Runtime state (active/inactive + configs)
|
||||
└── src/
|
||||
└── plugin/
|
||||
├── types.ts # Type definitions
|
||||
├── PluginLoader.ts # Manifest parsing, dynamic import
|
||||
├── PluginContext.ts # Context factory
|
||||
├── PluginManager.ts # Lifecycle management
|
||||
└── PluginExecutor.ts # Dependency-ordered execution
|
||||
```
|
||||
## Why the change
|
||||
|
||||
Cache folders are stored separately under `CACHE_FOLDER_PATH` (default `/data/cache/plugins/`):
|
||||
| Limitation of in-process plugins | Container model |
|
||||
|----------------------------------|-----------------|
|
||||
| Heavy native deps (FFmpeg, Whisper, ML libs) inflated the meta-sort image | Each plugin ships its own image |
|
||||
| All plugins had to be TypeScript / Node | Any language with an HTTP server |
|
||||
| A crashing plugin took down meta-sort | Plugin crashes are isolated; meta-sort restarts the container |
|
||||
| Plugin updates required redeploying meta-sort | Plugins are versioned and pulled independently |
|
||||
|
||||
```
|
||||
/data/cache/plugins/
|
||||
├── ffmpeg/
|
||||
│ ├── {cid1}.json
|
||||
│ └── {cid2}.json
|
||||
├── tmdb/
|
||||
│ ├── {cid1}.json
|
||||
│ └── {cid2}.json
|
||||
└── ...
|
||||
```
|
||||
|
||||
## Plugin Manifest
|
||||
|
||||
Each plugin must have a `manifest.yml` file defining its metadata, dependencies, and configuration schema.
|
||||
|
||||
```yaml
|
||||
# manifest.yml
|
||||
id: tmdb # Unique identifier (matches folder name)
|
||||
name: TMDB Enricher # Human-readable name
|
||||
version: 1.0.0 # Semantic version
|
||||
description: Enriches metadata from The Movie Database API
|
||||
|
||||
# Plugins that must execute BEFORE this one
|
||||
dependencies:
|
||||
- filename-parser # Needs parsed title/year
|
||||
|
||||
# Configuration schema (rendered in dashboard)
|
||||
config:
|
||||
apiKey:
|
||||
type: string
|
||||
label: API Key
|
||||
required: true
|
||||
secret: true # Masked in UI
|
||||
language:
|
||||
type: string
|
||||
label: Language
|
||||
default: en-US
|
||||
includeAdult:
|
||||
type: boolean
|
||||
label: Include Adult Content
|
||||
default: false
|
||||
|
||||
# Metadata schema for editor UI (optional)
|
||||
# Describes keys this plugin writes
|
||||
schema:
|
||||
tmdb/id:
|
||||
label: TMDB ID
|
||||
type: string
|
||||
readonly: true
|
||||
tmdb/title:
|
||||
label: Title
|
||||
type: string
|
||||
tmdb/overview:
|
||||
label: Overview
|
||||
type: text
|
||||
tmdb/poster:
|
||||
label: Poster URL
|
||||
type: string
|
||||
format: url
|
||||
tmdb/genre/{n}:
|
||||
label: Genre
|
||||
type: string
|
||||
indexed: true # Indicates indexed keys (0, 1, 2...)
|
||||
```
|
||||
|
||||
### Manifest Fields
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `id` | Yes | Unique plugin identifier, must match folder name |
|
||||
| `name` | Yes | Display name for dashboard |
|
||||
| `version` | Yes | Semantic version string |
|
||||
| `description` | No | Brief description |
|
||||
| `dependencies` | No | Array of plugin IDs that must run before this one |
|
||||
| `config` | No | Configuration schema for dashboard UI |
|
||||
| `schema` | No | Metadata keys written by this plugin (for editor) |
|
||||
|
||||
### Config Field Types
|
||||
|
||||
```yaml
|
||||
config:
|
||||
fieldName:
|
||||
type: string | number | boolean | select
|
||||
label: Display Label
|
||||
required: true | false # Default: false
|
||||
default: <value> # Default value
|
||||
secret: true | false # Mask in UI (default: false)
|
||||
options: # For type: select
|
||||
- value: opt1
|
||||
label: Option 1
|
||||
- value: opt2
|
||||
label: Option 2
|
||||
```
|
||||
|
||||
## Plugin Interface
|
||||
|
||||
Plugins export a default object implementing the `Plugin` interface.
|
||||
|
||||
```typescript
|
||||
// types.ts
|
||||
|
||||
export interface Plugin {
|
||||
/**
|
||||
* Called once when the plugin is loaded at startup.
|
||||
* Use for one-time initialization.
|
||||
*/
|
||||
onLoad?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is activated.
|
||||
* Triggered at startup (if previously active) or from dashboard.
|
||||
*/
|
||||
onActivate?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is deactivated from the dashboard.
|
||||
* Use for cleanup, releasing resources.
|
||||
*/
|
||||
onDeactivate?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is unloaded at shutdown.
|
||||
*/
|
||||
onUnload?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Process a single file. Called for each file in the library.
|
||||
* Must be implemented by all plugins.
|
||||
*/
|
||||
process(ctx: PluginContext): Promise<void>;
|
||||
}
|
||||
|
||||
export interface PluginLifecycleContext {
|
||||
/** Plugin configuration from plugins.json */
|
||||
config: Record<string, unknown>;
|
||||
|
||||
/** Absolute path to plugin's cache directory */
|
||||
cacheDir: string;
|
||||
|
||||
/** Scoped logger */
|
||||
log: PluginLogger;
|
||||
}
|
||||
```
|
||||
|
||||
## Plugin Context
|
||||
|
||||
The `PluginContext` is injected into the `process()` function, providing access to file metadata and utilities.
|
||||
|
||||
```typescript
|
||||
export interface PluginContext {
|
||||
/**
|
||||
* Key-value storage for the current file's metadata.
|
||||
* Keys use path notation: "video/codec", "audio/0/language"
|
||||
*/
|
||||
kv: PluginKVStore;
|
||||
|
||||
/** Absolute path to the file being processed */
|
||||
filePath: string;
|
||||
|
||||
/** Plugin configuration values */
|
||||
config: Record<string, unknown>;
|
||||
|
||||
/** Absolute path to plugin's cache directory */
|
||||
cacheDir: string;
|
||||
|
||||
/** Cache utilities for persistent storage */
|
||||
cache: PluginCache;
|
||||
|
||||
/** Scoped logger (prefixed with plugin name) */
|
||||
log: PluginLogger;
|
||||
}
|
||||
```
|
||||
|
||||
### KV Store Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginKVStore {
|
||||
/**
|
||||
* Get a value by key.
|
||||
* @param key Path-style key, e.g., "video/codec" or "audio/0/language"
|
||||
*/
|
||||
get(key: string): string | undefined;
|
||||
|
||||
/**
|
||||
* Set a value. Only string values are supported.
|
||||
* @param key Path-style key
|
||||
* @param value String value
|
||||
*/
|
||||
set(key: string, value: string): void;
|
||||
|
||||
/**
|
||||
* Delete a key.
|
||||
*/
|
||||
delete(key: string): void;
|
||||
|
||||
/**
|
||||
* Get all keys, optionally filtered by prefix.
|
||||
* @param prefix Optional prefix filter, e.g., "audio/" returns ["audio/0/codec", "audio/0/language", ...]
|
||||
*/
|
||||
keys(prefix?: string): string[];
|
||||
|
||||
/**
|
||||
* Get all key-value pairs, optionally filtered by prefix.
|
||||
*/
|
||||
entries(prefix?: string): Array<[string, string]>;
|
||||
}
|
||||
```
|
||||
|
||||
### Cache Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginCache {
|
||||
/**
|
||||
* Get full path for a cache file.
|
||||
* @param filename Relative filename within cache directory
|
||||
*/
|
||||
getPath(filename: string): string;
|
||||
|
||||
/**
|
||||
* Read and parse a JSON cache file.
|
||||
* @returns Parsed data or null if not found
|
||||
*/
|
||||
readJson<T>(filename: string): Promise<T | null>;
|
||||
|
||||
/**
|
||||
* Write data as JSON to cache file.
|
||||
*/
|
||||
writeJson(filename: string, data: unknown): Promise<void>;
|
||||
|
||||
/**
|
||||
* Check if a cache file exists.
|
||||
*/
|
||||
exists(filename: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Delete a cache file.
|
||||
*/
|
||||
delete(filename: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Clear all files in the plugin's cache directory.
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### Logger Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginLogger {
|
||||
debug(message: string, ...args: unknown[]): void;
|
||||
info(message: string, ...args: unknown[]): void;
|
||||
warn(message: string, ...args: unknown[]): void;
|
||||
error(message: string, ...args: unknown[]): void;
|
||||
}
|
||||
```
|
||||
|
||||
## Plugin Implementation Example
|
||||
|
||||
```typescript
|
||||
// plugins/tmdb/index.ts
|
||||
import type { Plugin, PluginContext, PluginLifecycleContext } from '../../src/plugin/types';
|
||||
|
||||
interface TmdbConfig {
|
||||
apiKey: string;
|
||||
language: string;
|
||||
includeAdult: boolean;
|
||||
}
|
||||
|
||||
const plugin: Plugin = {
|
||||
async onLoad(ctx: PluginLifecycleContext) {
|
||||
ctx.log.info('TMDB plugin loaded');
|
||||
},
|
||||
|
||||
async onActivate(ctx: PluginLifecycleContext) {
|
||||
const config = ctx.config as TmdbConfig;
|
||||
if (!config.apiKey) {
|
||||
ctx.log.warn('No API key configured, plugin will skip processing');
|
||||
}
|
||||
},
|
||||
|
||||
async onDeactivate(ctx: PluginLifecycleContext) {
|
||||
ctx.log.info('TMDB plugin deactivated');
|
||||
},
|
||||
|
||||
async process(ctx: PluginContext) {
|
||||
const config = ctx.config as TmdbConfig;
|
||||
|
||||
// Skip if not configured
|
||||
if (!config.apiKey) return;
|
||||
|
||||
// Get data from previous plugins
|
||||
const title = ctx.kv.get('filename/title');
|
||||
const year = ctx.kv.get('filename/year');
|
||||
const videoType = ctx.kv.get('video/type');
|
||||
|
||||
if (!title) return;
|
||||
|
||||
// Check cache first (keyed by title+year)
|
||||
const cacheKey = `${sanitize(title)}-${year || 'unknown'}.json`;
|
||||
const cached = await ctx.cache.readJson<TmdbResult>(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
applyResult(ctx.kv, cached);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch from TMDB API
|
||||
try {
|
||||
const result = await fetchTmdb(title, year, videoType, config);
|
||||
if (result) {
|
||||
await ctx.cache.writeJson(cacheKey, result);
|
||||
applyResult(ctx.kv, result);
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.log.error('TMDB fetch failed', error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function applyResult(kv: PluginContext['kv'], result: TmdbResult) {
|
||||
kv.set('tmdb/id', String(result.id));
|
||||
kv.set('tmdb/title', result.title);
|
||||
kv.set('tmdb/overview', result.overview);
|
||||
kv.set('tmdb/poster', result.poster_path);
|
||||
kv.set('tmdb/backdrop', result.backdrop_path);
|
||||
kv.set('tmdb/rating', String(result.vote_average));
|
||||
kv.set('tmdb/year', result.release_date?.slice(0, 4) || '');
|
||||
|
||||
// Genres as indexed keys
|
||||
result.genres?.forEach((genre, i) => {
|
||||
kv.set(`tmdb/genre/${i}`, genre.name);
|
||||
});
|
||||
}
|
||||
|
||||
export default plugin;
|
||||
```
|
||||
|
||||
## Metadata Key Format
|
||||
|
||||
All metadata is stored as flat key-value pairs with string values only. Keys use path notation with `/` as separator.
|
||||
|
||||
### Key Naming Convention
|
||||
|
||||
```
|
||||
{plugin-id}/{field} # Simple field
|
||||
{plugin-id}/{group}/{field} # Grouped field
|
||||
{plugin-id}/{group}/{index}/{field} # Indexed field (for lists)
|
||||
```
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
# File info plugin
|
||||
file/name = "Movie.2024.1080p.mkv"
|
||||
file/size = "4294967296"
|
||||
file/type = "video"
|
||||
file/mime = "video/x-matroska"
|
||||
|
||||
# FFmpeg plugin
|
||||
video/codec = "h264"
|
||||
video/width = "1920"
|
||||
video/height = "1080"
|
||||
video/duration = "7200.5"
|
||||
video/bitrate = "8000000"
|
||||
audio/0/codec = "aac"
|
||||
audio/0/language = "eng"
|
||||
audio/0/channels = "6"
|
||||
audio/1/codec = "dts"
|
||||
audio/1/language = "fra"
|
||||
audio/1/channels = "6"
|
||||
subtitle/0/language = "eng"
|
||||
subtitle/0/format = "subrip"
|
||||
subtitle/1/language = "fra"
|
||||
subtitle/1/format = "subrip"
|
||||
|
||||
# Filename parser plugin
|
||||
filename/title = "Movie Title"
|
||||
filename/year = "2024"
|
||||
filename/quality = "1080p"
|
||||
filename/source = "bluray"
|
||||
|
||||
# TMDB plugin
|
||||
tmdb/id = "12345"
|
||||
tmdb/title = "Movie Title"
|
||||
tmdb/overview = "A movie about..."
|
||||
tmdb/poster = "/path/to/poster.jpg"
|
||||
tmdb/genre/0 = "Action"
|
||||
tmdb/genre/1 = "Adventure"
|
||||
```
|
||||
|
||||
### Key Constraints
|
||||
|
||||
- Keys must be non-empty strings
|
||||
- Keys use `/` as path separator
|
||||
- Values must be strings (numbers converted via `String()`)
|
||||
- No nested objects or arrays (use indexed keys instead)
|
||||
- Keys are case-sensitive
|
||||
- Recommended: use lowercase with hyphens for multi-word keys
|
||||
|
||||
## Plugin Manager
|
||||
|
||||
The `PluginManager` handles plugin lifecycle and orchestration.
|
||||
|
||||
```typescript
|
||||
export class PluginManager extends EventEmitter {
|
||||
/**
|
||||
* Scan plugins directory and load manifests.
|
||||
* Called once at startup.
|
||||
*/
|
||||
async scanPlugins(): Promise<PluginInfo[]>;
|
||||
|
||||
/**
|
||||
* Load all plugins and activate those marked active in state.
|
||||
* Called once at startup after scanning.
|
||||
*/
|
||||
async loadAll(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Activate a plugin (hot, no restart required).
|
||||
* Calls plugin.onActivate(), rebuilds execution order, saves state.
|
||||
* Emits 'plugin:activated' event.
|
||||
*/
|
||||
async activate(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Deactivate a plugin (hot, no restart required).
|
||||
* Calls plugin.onDeactivate(), rebuilds execution order, saves state.
|
||||
* Emits 'plugin:deactivated' event.
|
||||
*/
|
||||
async deactivate(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Update plugin configuration.
|
||||
* Saves to plugins.json.
|
||||
* Emits 'plugin:config-changed' event.
|
||||
*/
|
||||
async updateConfig(pluginId: string, config: Record<string, unknown>): Promise<void>;
|
||||
|
||||
/**
|
||||
* Clear a plugin's cache directory.
|
||||
*/
|
||||
async clearCache(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get all discovered plugins.
|
||||
*/
|
||||
getPlugins(): PluginInfo[];
|
||||
|
||||
/**
|
||||
* Get only active plugins.
|
||||
*/
|
||||
getActivePlugins(): PluginInfo[];
|
||||
|
||||
/**
|
||||
* Get execution order (dependency-sorted active plugin IDs).
|
||||
*/
|
||||
getExecutionOrder(): string[];
|
||||
|
||||
/**
|
||||
* Process a file through all active plugins.
|
||||
*/
|
||||
async processFile(filePath: string, kv: KVStore): Promise<ProcessingResult>;
|
||||
|
||||
/**
|
||||
* Shutdown: deactivate and unload all plugins.
|
||||
*/
|
||||
async shutdown(): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### Events
|
||||
|
||||
| Event | Payload | Description |
|
||||
|-------|---------|-------------|
|
||||
| `plugin:activated` | `{ pluginId: string }` | Plugin was activated |
|
||||
| `plugin:deactivated` | `{ pluginId: string }` | Plugin was deactivated |
|
||||
| `plugin:config-changed` | `{ pluginId: string, config: object }` | Plugin config updated |
|
||||
| `plugin:error` | `{ pluginId: string, error: Error }` | Plugin error occurred |
|
||||
|
||||
### Plugin Info
|
||||
|
||||
```typescript
|
||||
export interface PluginInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
version: string;
|
||||
description?: string;
|
||||
dependencies: string[];
|
||||
active: boolean;
|
||||
config: Record<string, unknown>;
|
||||
configSchema?: ConfigSchema;
|
||||
metadataSchema?: MetadataSchema;
|
||||
status: 'loaded' | 'error' | 'unloaded';
|
||||
error?: string;
|
||||
}
|
||||
```
|
||||
|
||||
## State Management
|
||||
|
||||
Plugin state is persisted in `plugins.json` located in the meta-sort-core package root.
|
||||
|
||||
```json
|
||||
{
|
||||
"plugins": {
|
||||
"file-info": {
|
||||
"active": true,
|
||||
"config": {}
|
||||
},
|
||||
"ffmpeg": {
|
||||
"active": true,
|
||||
"config": {}
|
||||
},
|
||||
"tmdb": {
|
||||
"active": true,
|
||||
"config": {
|
||||
"apiKey": "your-api-key",
|
||||
"language": "en-US",
|
||||
"includeAdult": false
|
||||
}
|
||||
},
|
||||
"custom-plugin": {
|
||||
"active": false,
|
||||
"config": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### State Behavior
|
||||
|
||||
- **New plugins**: Discovered plugins not in state file are added with `active: false`
|
||||
- **Removed plugins**: Plugins in state but not on disk are kept (preserves config for reinstall)
|
||||
- **Config merge**: Default values from manifest are merged with saved config
|
||||
- **Persistence**: State is saved immediately on any change
|
||||
|
||||
## Execution Pipeline
|
||||
|
||||
### Startup Sequence
|
||||
|
||||
```
|
||||
1. PluginManager.scanPlugins()
|
||||
└── Scan plugins/ directory
|
||||
└── Parse each manifest.yml
|
||||
└── Validate manifest structure
|
||||
|
||||
2. PluginManager.loadAll()
|
||||
└── Load plugins.json state
|
||||
└── For each discovered plugin:
|
||||
├── Dynamic import(plugin/index.js)
|
||||
├── Validate Plugin interface
|
||||
├── Call plugin.onLoad()
|
||||
└── If active in state:
|
||||
└── Call plugin.onActivate()
|
||||
└── Resolve execution order (topological sort)
|
||||
```
|
||||
|
||||
### File Processing Sequence
|
||||
|
||||
```
|
||||
1. Create KVStore for file
|
||||
|
||||
2. For each active plugin (in dependency order):
|
||||
├── Create PluginContext
|
||||
├── Start timing
|
||||
├── try:
|
||||
│ └── await plugin.process(ctx)
|
||||
└── catch:
|
||||
└── Log error, continue to next plugin
|
||||
|
||||
3. Persist KV to Redis
|
||||
```
|
||||
|
||||
### Activation Sequence (Hot)
|
||||
|
||||
```
|
||||
1. PluginManager.activate(pluginId)
|
||||
├── Validate plugin exists and is loaded
|
||||
├── Check dependencies are active
|
||||
├── Call plugin.onActivate()
|
||||
├── Mark as active in memory
|
||||
├── Rebuild execution order
|
||||
├── Save plugins.json
|
||||
└── Emit 'plugin:activated' event
|
||||
|
||||
2. Caller receives event
|
||||
└── Trigger library rescan
|
||||
```
|
||||
|
||||
### Deactivation Sequence (Hot)
|
||||
|
||||
```
|
||||
1. PluginManager.deactivate(pluginId)
|
||||
├── Check no active plugins depend on this one
|
||||
├── Call plugin.onDeactivate()
|
||||
├── Mark as inactive in memory
|
||||
├── Rebuild execution order
|
||||
├── Save plugins.json
|
||||
└── Emit 'plugin:deactivated' event
|
||||
|
||||
2. Caller receives event
|
||||
└── Trigger library rescan
|
||||
|
||||
Note: Metadata written by the plugin remains in Redis.
|
||||
A separate "clean metadata" feature can remove orphaned keys.
|
||||
```
|
||||
|
||||
## Dependency Resolution
|
||||
|
||||
Plugins declare dependencies in their manifest. The system performs topological sort to determine execution order.
|
||||
|
||||
### Rules
|
||||
|
||||
1. A plugin cannot be activated if any of its dependencies are inactive
|
||||
2. A plugin cannot be deactivated if any active plugin depends on it
|
||||
3. Circular dependencies are detected and rejected at load time
|
||||
4. Missing dependencies are logged as warnings; plugin can still load but may fail
|
||||
|
||||
### Example
|
||||
|
||||
```yaml
|
||||
# filename-parser depends on file-info
|
||||
# tmdb depends on filename-parser
|
||||
# anime-detector depends on filename-parser
|
||||
|
||||
Resolved order: file-info → filename-parser → [tmdb, anime-detector]
|
||||
```
|
||||
|
||||
Plugins at the same dependency level may execute in any order (or parallel in future).
|
||||
|
||||
## REST API
|
||||
|
||||
The plugin system exposes REST endpoints for dashboard integration.
|
||||
|
||||
### Endpoints
|
||||
|
||||
```
|
||||
GET /api/plugins # List all plugins
|
||||
GET /api/plugins/:id # Get plugin details
|
||||
POST /api/plugins/:id/activate # Activate plugin
|
||||
POST /api/plugins/:id/deactivate # Deactivate plugin
|
||||
PUT /api/plugins/:id/config # Update plugin config
|
||||
DELETE /api/plugins/:id/cache # Clear plugin cache
|
||||
GET /api/plugins/:id/schema # Get metadata schema (for editor)
|
||||
```
|
||||
|
||||
### Response Format
|
||||
|
||||
```typescript
|
||||
// GET /api/plugins
|
||||
interface PluginsResponse {
|
||||
plugins: PluginInfo[];
|
||||
executionOrder: string[];
|
||||
}
|
||||
|
||||
// POST /api/plugins/:id/activate
|
||||
interface ActivateResponse {
|
||||
success: boolean;
|
||||
error?: string; # e.g., "Dependency 'x' is not active"
|
||||
}
|
||||
|
||||
// PUT /api/plugins/:id/config
|
||||
interface ConfigUpdateRequest {
|
||||
config: Record<string, unknown>;
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Plugin Load Errors
|
||||
|
||||
- Invalid manifest: Plugin marked as `status: 'error'`, logged, skipped
|
||||
- Import failure: Plugin marked as `status: 'error'`, logged, skipped
|
||||
- Missing dependencies: Warning logged, plugin can load but activation may fail
|
||||
|
||||
### Plugin Execution Errors
|
||||
|
||||
- Errors in `process()` are caught, logged, and execution continues to next plugin
|
||||
- The file's metadata from previous plugins is preserved
|
||||
- Error count is tracked per plugin for monitoring
|
||||
|
||||
### Lifecycle Hook Errors
|
||||
|
||||
- Errors in `onActivate()` prevent activation, error returned to caller
|
||||
- Errors in `onDeactivate()` are logged but deactivation proceeds
|
||||
- Errors in `onLoad()`/`onUnload()` are logged, plugin marked as error state
|
||||
|
||||
## Future Considerations
|
||||
|
||||
### Metadata Cleanup
|
||||
|
||||
A future feature will allow cleaning metadata keys written by a specific plugin:
|
||||
|
||||
```typescript
|
||||
// Future API
|
||||
await pluginManager.cleanMetadata(pluginId);
|
||||
// Removes all keys matching "{pluginId}/*" from Redis
|
||||
```
|
||||
|
||||
### Parallel Execution
|
||||
|
||||
Plugins at the same dependency level could execute in parallel:
|
||||
|
||||
```typescript
|
||||
// Future: parallel execution of independent plugins
|
||||
const level1 = ['file-info'];
|
||||
const level2 = ['ffmpeg', 'filename-parser']; // parallel
|
||||
const level3 = ['tmdb', 'anime-detector']; // parallel
|
||||
```
|
||||
|
||||
### Plugin Hot Reload
|
||||
|
||||
Reloading a plugin's code without full restart (for development):
|
||||
|
||||
```typescript
|
||||
// Future API
|
||||
await pluginManager.reload(pluginId);
|
||||
// Re-imports plugin code, preserves state
|
||||
```
|
||||
See `containerized-plugin-architecture.md` for the migration rationale and
|
||||
the HTTP contract container plugins must implement.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+1
-1
@@ -13,5 +13,5 @@
|
||||
"lint": "pnpm -r lint"
|
||||
},
|
||||
"license": "MIT",
|
||||
"packageManager": "[email protected]+sha512.EFaLtKavtYyes2MNqQzJUWQXq+vT+rvmc58K55VyjaFJHp21pUTHatjrdXD1xLs9bGN7LLQb/c20f6gjyGSTGQ=="
|
||||
"packageManager": "[email protected]+sha512.10568bb4a6afb58c9eb3630da90cc9516417abebd3fabbe6739f0ae795728da1491e9db5a544c76ad8eb7570f5c4bb3d6c637b2cb41bfdcdb47fa823c8649319"
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ lazyValue().then(value => console.log(value));
|
||||
### Listener Cleaner
|
||||
|
||||
```typescript
|
||||
import { ListenerCleaner } from "worph/async-utils";
|
||||
import { ListenerCleaner } from "@worph/async-utils";
|
||||
|
||||
const cleaner = new ListenerCleaner();
|
||||
cleaner.add(() => console.log("Cleanup action"));
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
{
|
||||
"spec": "dist/api/api.test.js",
|
||||
"spec": [
|
||||
"dist/api/api.test.js",
|
||||
"dist/kv/*.test.js",
|
||||
"dist/container-plugins/*.test.js",
|
||||
"dist/discovery/*.test.js"
|
||||
],
|
||||
"timeout": 30000,
|
||||
"exit": true
|
||||
}
|
||||
|
||||
@@ -11,8 +11,9 @@ import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
|
||||
import { UnifiedProcessingStateManager } from '../logic/UnifiedProcessingStateManager.js';
|
||||
import type { IKVClient } from '../kv/IKVClient.js';
|
||||
import type { KVManager } from '../kv/KVManager.js';
|
||||
import type { ExtendedProcessingSnapshot, IKVClientWithPubSub } from '../types/ExtendedInterfaces.js';
|
||||
import { getErrorMessage } from '../types/ExtendedInterfaces.js';
|
||||
import type { ExtendedProcessingSnapshot } from '../types/ExtendedInterfaces.js';
|
||||
import { getErrorMessage, hasFileTuples } from '../types/ExtendedInterfaces.js';
|
||||
import type { IKVClientWithFileTuples } from '../types/ExtendedInterfaces.js';
|
||||
import { config } from '../config/EnvConfig.js';
|
||||
import * as webdav from '../webdav/WebdavClient.js';
|
||||
import type { PluginManager } from '../plugin-engine/PluginManager.js';
|
||||
@@ -29,6 +30,45 @@ export interface UnifiedAPIServerConfig {
|
||||
enableCors?: boolean;
|
||||
}
|
||||
|
||||
/** File totals served by /api/stats. Null when meta-core cannot supply them. */
|
||||
interface FileStats {
|
||||
fileCount: number | null;
|
||||
totalSize: number | null;
|
||||
source: 'meta-core' | 'unavailable';
|
||||
}
|
||||
|
||||
/** Metadata reads in flight at once while a recompute builds its task list. */
|
||||
const RECOMPUTE_READ_CONCURRENCY = 8;
|
||||
|
||||
/** Run `worker` over `items` with at most `limit` calls in flight. */
|
||||
async function forEachWithConcurrency<T>(
|
||||
items: readonly T[],
|
||||
limit: number,
|
||||
worker: (item: T) => Promise<void>
|
||||
): Promise<void> {
|
||||
let next = 0;
|
||||
const runners = Array.from({ length: Math.min(limit, items.length) }, async () => {
|
||||
while (next < items.length) {
|
||||
const item = items[next++];
|
||||
await worker(item);
|
||||
}
|
||||
});
|
||||
await Promise.all(runners);
|
||||
}
|
||||
|
||||
/** Flatten nested metadata into the `a/b/c` keys the task scheduler expects. */
|
||||
function flattenMetadata(obj: Record<string, unknown>, out: Record<string, string> = {}, prefix = ''): Record<string, string> {
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const fullKey = prefix ? `${prefix}/${key}` : key;
|
||||
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
||||
flattenMetadata(value as Record<string, unknown>, out, fullKey);
|
||||
} else if (value !== undefined && value !== null) {
|
||||
out[fullKey] = String(value);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export class UnifiedAPIServer {
|
||||
private app: FastifyInstance;
|
||||
private unifiedStateManager: UnifiedProcessingStateManager | null = null;
|
||||
@@ -44,9 +84,15 @@ export class UnifiedAPIServer {
|
||||
private containerPluginScheduler: ContainerPluginScheduler | null = null;
|
||||
private streamingPipeline: StreamingPipeline | null = null;
|
||||
|
||||
// Cache for total file size (avoid running du on every request)
|
||||
private totalSizeCache: { value: number; timestamp: number } | null = null;
|
||||
private static TOTAL_SIZE_CACHE_TTL_MS = 30000; // 30 seconds
|
||||
// /api/stats file totals: stale-while-revalidate, one refresh at a time.
|
||||
// Never walk hashIds for these — on a shared meta-core that is every gateway
|
||||
// record too (82k on watch.nsl.sh), and a sweep per poll stacked unbounded.
|
||||
private fileStats: { value: FileStats; fetchedAt: number } | null = null;
|
||||
private fileStatsInFlight: Promise<FileStats> | null = null;
|
||||
private static FILE_STATS_TTL_MS = 30000; // 30 seconds
|
||||
|
||||
// Plugins whose recompute is currently being queued (single-flight per plugin)
|
||||
private recomputeInFlight = new Set<string>();
|
||||
|
||||
constructor(
|
||||
config: UnifiedAPIServerConfig = {},
|
||||
@@ -97,48 +143,46 @@ export class UnifiedAPIServer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Get total size of processed files from Redis sizeByte field (cached)
|
||||
* This is "eventually correct" - accumulates as files are processed by the file-info plugin
|
||||
* File count + total size of the files meta-core holds on disk, for the
|
||||
* monitor's "Total Size" card: one summary call to meta-core's
|
||||
* GET /api/files/tuples (batched server-side), cached for FILE_STATS_TTL_MS.
|
||||
*
|
||||
* Stale-while-revalidate with a single in-flight refresh — a poll never waits
|
||||
* on meta-core once a value exists, and concurrent polls share one request.
|
||||
*/
|
||||
private async getTotalFileSizeFromRedis(): Promise<number> {
|
||||
// Check cache
|
||||
const now = Date.now();
|
||||
if (this.totalSizeCache && (now - this.totalSizeCache.timestamp) < UnifiedAPIServer.TOTAL_SIZE_CACHE_TTL_MS) {
|
||||
return this.totalSizeCache.value;
|
||||
private async getFileStats(): Promise<FileStats> {
|
||||
const cached = this.fileStats;
|
||||
if (cached && Date.now() - cached.fetchedAt < UnifiedAPIServer.FILE_STATS_TTL_MS) {
|
||||
return cached.value;
|
||||
}
|
||||
|
||||
// Sum sizeByte from all file entries in Redis
|
||||
let totalSize = 0;
|
||||
|
||||
if (!this.kvClient) {
|
||||
return 0;
|
||||
if (!this.fileStatsInFlight) {
|
||||
this.fileStatsInFlight = this.fetchFileStats()
|
||||
.catch((error: unknown): FileStats => {
|
||||
console.warn('[API] /api/stats: meta-core file totals unavailable:', getErrorMessage(error));
|
||||
return cached?.value ?? { fileCount: null, totalSize: null, source: 'unavailable' };
|
||||
})
|
||||
.then((value) => {
|
||||
this.fileStats = { value, fetchedAt: Date.now() };
|
||||
return value;
|
||||
})
|
||||
.finally(() => {
|
||||
this.fileStatsInFlight = null;
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
// Get all file hash IDs
|
||||
const hashIds = await this.kvClient.getAllHashIds();
|
||||
return cached ? cached.value : this.fileStatsInFlight;
|
||||
}
|
||||
|
||||
// Sum up sizeByte values (batch process for efficiency)
|
||||
for (const hashId of hashIds) {
|
||||
try {
|
||||
const sizeByte = await this.kvClient.getMetadata(hashId, 'sizeByte');
|
||||
if (sizeByte !== null && sizeByte !== undefined) {
|
||||
const size = typeof sizeByte === 'number' ? sizeByte : parseInt(String(sizeByte), 10);
|
||||
if (!isNaN(size) && size > 0) {
|
||||
totalSize += size;
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Skip entries without sizeByte
|
||||
}
|
||||
}
|
||||
} catch (error) {
|
||||
console.warn('[API] Failed to calculate total size from Redis:', error);
|
||||
private async fetchFileStats(): Promise<FileStats> {
|
||||
if (!this.kvClient || !hasFileTuples(this.kvClient)) {
|
||||
return { fileCount: null, totalSize: null, source: 'unavailable' };
|
||||
}
|
||||
|
||||
// Update cache
|
||||
this.totalSizeCache = { value: totalSize, timestamp: now };
|
||||
return totalSize;
|
||||
const summary = await this.kvClient.getFileTuples({ summary: true });
|
||||
if (!summary) {
|
||||
return { fileCount: null, totalSize: null, source: 'unavailable' };
|
||||
}
|
||||
return { fileCount: summary.count, totalSize: summary.totalSize, source: 'meta-core' };
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -193,7 +237,7 @@ export class UnifiedAPIServer {
|
||||
this.setupStatsRoutes();
|
||||
}
|
||||
|
||||
// Service discovery — proxies to meta-core's /api/services using the
|
||||
// Service discovery — beacon v2 neighbour list, served from the
|
||||
// LeaderClient to resolve meta-core's current URL dynamically.
|
||||
this.setupServiceDiscoveryRoutes();
|
||||
|
||||
@@ -524,53 +568,17 @@ export class UnifiedAPIServer {
|
||||
|
||||
|
||||
/**
|
||||
* Setup Stats API routes (Redis stats for monitor UI)
|
||||
* Setup Stats API routes (file totals for the monitor UI)
|
||||
*/
|
||||
private setupStatsRoutes(): void {
|
||||
if (!this.kvClient) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Get Redis/KV stats (for monitor UI)
|
||||
this.app.get('/api/stats', async (request, reply) => {
|
||||
try {
|
||||
const hashIds = await this.kvClient!.getAllHashIds();
|
||||
const redis = (this.kvClient as IKVClientWithPubSub).getRedisClient?.() as { info: (section: string) => Promise<string> } | undefined;
|
||||
|
||||
let memoryUsage = 'N/A';
|
||||
let memoryUsageBytes = 0;
|
||||
if (redis) {
|
||||
try {
|
||||
const info = await redis.info('memory');
|
||||
const match = info.match(/used_memory_human:(\S+)/);
|
||||
const bytesMatch = info.match(/used_memory:(\d+)/);
|
||||
if (match) {
|
||||
memoryUsage = match[1];
|
||||
}
|
||||
if (bytesMatch) {
|
||||
memoryUsageBytes = parseInt(bytesMatch[1], 10);
|
||||
}
|
||||
} catch {
|
||||
// Ignore memory info errors
|
||||
}
|
||||
}
|
||||
|
||||
// Get total size from Redis sizeByte (eventually correct as files are processed)
|
||||
const totalSize = await this.getTotalFileSizeFromRedis();
|
||||
|
||||
return {
|
||||
fileCount: hashIds.length,
|
||||
keyCount: hashIds.length,
|
||||
totalSize,
|
||||
memoryUsage,
|
||||
memoryUsageBytes
|
||||
};
|
||||
} catch (error: any) {
|
||||
return reply.status(500).send({
|
||||
error: 'Failed to get stats',
|
||||
details: error.message
|
||||
});
|
||||
}
|
||||
// At most one meta-core call per FILE_STATS_TTL_MS however often the
|
||||
// monitor polls — see getFileStats.
|
||||
this.app.get('/api/stats', async () => {
|
||||
return this.getFileStats();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -584,33 +592,31 @@ export class UnifiedAPIServer {
|
||||
}
|
||||
|
||||
/**
|
||||
* Service discovery — proxies to meta-core's /api/services, resolving the
|
||||
* current leader's URL via LeaderClient (no hardcoded hostname).
|
||||
* Service discovery — beacon v2 (docs/project-architecture/beacon-v2.md).
|
||||
* Served from this service's own beacon view; no meta-core round trip, so
|
||||
* the nav survives a core outage.
|
||||
*/
|
||||
private setupServiceDiscoveryRoutes(): void {
|
||||
this.app.get('/api/services', async (_request, reply) => {
|
||||
try {
|
||||
const leaderClient = this.kvManager?.getLeaderClient();
|
||||
if (!leaderClient) {
|
||||
return reply.status(503).send({ error: 'KVManager not initialized' });
|
||||
}
|
||||
const apiUrl = await leaderClient.getApiUrl();
|
||||
if (!apiUrl) {
|
||||
return reply.status(503).send({ error: 'Leader not available yet' });
|
||||
}
|
||||
const response = await fetch(`${apiUrl}/api/services`, {
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
if (!response.ok) {
|
||||
return reply.status(response.status).send({
|
||||
error: `Upstream ${apiUrl} returned ${response.status}`,
|
||||
});
|
||||
}
|
||||
const body = await response.json();
|
||||
return reply.send(body);
|
||||
} catch (err) {
|
||||
return reply.status(502).send({ error: getErrorMessage(err) });
|
||||
// beacon v2: neighbours heard over UDP, served from this service's own
|
||||
// view. Unlike /api/services below it needs no meta-core, so the nav still
|
||||
// renders when the core is down. `?all=1` returns every instance,
|
||||
// `?cap=<pattern>` filters by capability. `services` is kept as an alias
|
||||
// while any older dashboard build is still in circulation.
|
||||
this.app.get('/api/neighbors', async (request, reply) => {
|
||||
const leaderClient = this.kvManager?.getLeaderClient();
|
||||
if (!leaderClient) {
|
||||
return reply.send({ current: 'meta-sort', enabled: false, count: 0, neighbors: [] });
|
||||
}
|
||||
const q = (request.query ?? {}) as { all?: string; cap?: string };
|
||||
const neighbors = leaderClient.getNeighbors({ all: !!q.all, cap: q.cap || undefined });
|
||||
return reply.send({
|
||||
current: 'meta-sort',
|
||||
enabled: true,
|
||||
count: neighbors.length,
|
||||
neighbors,
|
||||
services: neighbors,
|
||||
self: leaderClient.self(),
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
@@ -886,38 +892,43 @@ export class UnifiedAPIServer {
|
||||
return reply.status(400).send({ error: `Plugin '${pluginId}' is not active` });
|
||||
}
|
||||
|
||||
if (!hasFileTuples(this.kvClient)) {
|
||||
return reply.status(501).send({ error: 'Recompute needs meta-core GET /api/files/tuples' });
|
||||
}
|
||||
const kvClient = this.kvClient as IKVClientWithFileTuples;
|
||||
|
||||
if (this.recomputeInFlight.has(pluginId)) {
|
||||
return reply.status(409).send({ error: `Recompute for plugin '${pluginId}' is already being queued` });
|
||||
}
|
||||
|
||||
this.recomputeInFlight.add(pluginId);
|
||||
try {
|
||||
// Get all files from KV
|
||||
const hashIds = await this.kvClient.getAllHashIds();
|
||||
console.log(`[Recompute] Starting recompute for plugin '${pluginId}' on ${hashIds.length} files`);
|
||||
// Only file-backed records can be recomputed, and meta-core lists exactly
|
||||
// those in one batched call. Walking getAllHashIds() instead reads every
|
||||
// gateway record too (82k on watch.nsl.sh) to keep a few thousand.
|
||||
const tuples = await kvClient.getFileTuples();
|
||||
if (!tuples || !tuples.files) {
|
||||
return reply.status(501).send({
|
||||
error: 'Recompute needs meta-core GET /api/files/tuples',
|
||||
details: 'meta-core did not return a file list'
|
||||
});
|
||||
}
|
||||
const candidates = tuples.files;
|
||||
console.log(`[Recompute] Starting recompute for plugin '${pluginId}' on ${candidates.length} files`);
|
||||
|
||||
// Prepare file data with metadata for TaskScheduler
|
||||
const files: Array<{ filePath: string; fileHash: string; kvData?: Record<string, string> }> = [];
|
||||
|
||||
for (const hashId of hashIds) {
|
||||
const metadata = await this.kvClient.getMetadataFlat(hashId);
|
||||
if (metadata && metadata.filePath) {
|
||||
// Convert nested metadata to flat key-value pairs for KVStore
|
||||
const kvData: Record<string, string> = {};
|
||||
const flattenObj = (obj: any, prefix = '') => {
|
||||
for (const [key, value] of Object.entries(obj)) {
|
||||
const fullKey = prefix ? `${prefix}/${key}` : key;
|
||||
if (value !== null && typeof value === 'object' && !Array.isArray(value)) {
|
||||
flattenObj(value, fullKey);
|
||||
} else if (value !== undefined && value !== null) {
|
||||
kvData[fullKey] = String(value);
|
||||
}
|
||||
}
|
||||
};
|
||||
flattenObj(metadata);
|
||||
|
||||
files.push({
|
||||
fileHash: hashId,
|
||||
filePath: metadata.filePath,
|
||||
kvData
|
||||
});
|
||||
await forEachWithConcurrency(candidates, RECOMPUTE_READ_CONCURRENCY, async (tuple) => {
|
||||
const metadata = await kvClient.getMetadataFlat(tuple.hashId);
|
||||
if (!metadata) {
|
||||
return;
|
||||
}
|
||||
}
|
||||
files.push({
|
||||
fileHash: tuple.hashId,
|
||||
filePath: metadata.filePath || tuple.filePath,
|
||||
kvData: flattenMetadata(metadata)
|
||||
});
|
||||
});
|
||||
|
||||
// Create and enqueue tasks with forceRecompute option
|
||||
const tasks = taskScheduler.createTasksForPluginOnFiles(
|
||||
@@ -935,7 +946,7 @@ export class UnifiedAPIServer {
|
||||
status: 'ok',
|
||||
message: `Recompute triggered for plugin '${pluginId}'`,
|
||||
filesQueued: tasks.length,
|
||||
totalFiles: hashIds.length
|
||||
totalFiles: candidates.length
|
||||
};
|
||||
} catch (error: any) {
|
||||
console.error(`[Recompute] Error triggering recompute for plugin '${pluginId}':`, error);
|
||||
@@ -943,6 +954,8 @@ export class UnifiedAPIServer {
|
||||
error: 'Failed to trigger recompute',
|
||||
details: error.message
|
||||
});
|
||||
} finally {
|
||||
this.recomputeInFlight.delete(pluginId);
|
||||
}
|
||||
});
|
||||
}
|
||||
@@ -1014,7 +1027,9 @@ export class UnifiedAPIServer {
|
||||
instanceCount: p.instances,
|
||||
healthyCount: p.healthyInstances,
|
||||
enabled: p.enabled,
|
||||
image: p.image
|
||||
image: p.image,
|
||||
kind: p.kind,
|
||||
url: p.url
|
||||
})),
|
||||
queue: queueStatus || null
|
||||
};
|
||||
@@ -1027,6 +1042,67 @@ export class UnifiedAPIServer {
|
||||
}
|
||||
});
|
||||
|
||||
// Beacon v2 Scan card: enrichment plugins advertising on the network
|
||||
// (metamesh.enrich/<id>), classified against what this meta-sort runs.
|
||||
this.app.post('/api/plugins/discover', async (_request, reply) => {
|
||||
const leaderClient = this.kvManager?.getLeaderClient();
|
||||
const empty = {
|
||||
cap: 'metamesh.enrich/*', disabled: true,
|
||||
summary: { nodes: 0, capable: 0, configured: 0, boundElsewhere: 0, addable: 0 },
|
||||
candidates: [],
|
||||
};
|
||||
if (!leaderClient) return reply.send(empty);
|
||||
const status = this.containerManager?.getStatus();
|
||||
const configured: Array<[string, string]> = (status?.plugins ?? [])
|
||||
.filter((p) => p.url)
|
||||
.map((p) => [p.pluginId, p.url as string]);
|
||||
const report = await leaderClient.scan('metamesh.enrich/*', configured, 1500);
|
||||
// Same plugin id already loaded under another URL (e.g. our own spawned
|
||||
// container of it): adding it would clash, so it counts as configured.
|
||||
const loaded = new Set((status?.plugins ?? []).map((p) => p.pluginId));
|
||||
for (const c of report.candidates) {
|
||||
if (c.state !== 'addable') continue;
|
||||
const id = c.caps.map((cap) => cap.match(/^metamesh\.enrich\/([^@]+)/)?.[1]).find((x) => x && loaded.has(x));
|
||||
if (id) {
|
||||
c.state = 'configured';
|
||||
c.configuredAs = id;
|
||||
report.summary.addable--;
|
||||
report.summary.configured++;
|
||||
}
|
||||
}
|
||||
return reply.send(report);
|
||||
});
|
||||
|
||||
// External plugin (one meta-sort did not spawn) — the Scan card's Add.
|
||||
this.app.post<{ Body: { url?: string; name?: string } }>('/api/plugins/external', async (request, reply) => {
|
||||
if (!this.containerManager) {
|
||||
return reply.status(503).send({ error: 'Container manager not initialized' });
|
||||
}
|
||||
const url = request.body?.url;
|
||||
if (!url) return reply.status(400).send({ error: 'url is required' });
|
||||
try {
|
||||
const entry = await this.containerManager.addExternalPlugin(url, request.body?.name);
|
||||
// Bring it into the processing pipeline (same as /api/plugins/rescan).
|
||||
await this.getPluginManager?.()?.loadContainerPlugins();
|
||||
return { success: true, plugin: entry };
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
this.app.delete<{ Params: { pluginId: string } }>('/api/plugins/external/:pluginId', async (request, reply) => {
|
||||
if (!this.containerManager) {
|
||||
return reply.status(503).send({ error: 'Container manager not initialized' });
|
||||
}
|
||||
try {
|
||||
await this.containerManager.removeExternalPlugin(request.params.pluginId);
|
||||
await this.getPluginManager?.()?.loadContainerPlugins();
|
||||
return { success: true };
|
||||
} catch (error: any) {
|
||||
return reply.status(400).send({ error: error.message });
|
||||
}
|
||||
});
|
||||
|
||||
// Get container plugin manifest
|
||||
this.app.get<{
|
||||
Params: { pluginId: string }
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
import { expect } from 'chai';
|
||||
import { UnifiedAPIServer } from './UnifiedAPIServer.js';
|
||||
import { UnifiedProcessingStateManager } from '../logic/UnifiedProcessingStateManager.js';
|
||||
import { performanceMetrics } from '../metrics/PerformanceMetrics.js';
|
||||
import type { IKVClient } from '../kv/IKVClient.js';
|
||||
import type { FastifyInstance } from 'fastify';
|
||||
|
||||
@@ -23,6 +24,25 @@ class MockKVClient implements IKVClient {
|
||||
private data: Map<string, Map<string, any>> = new Map();
|
||||
private rawStore: Map<string, string> = new Map();
|
||||
|
||||
// Stand-in for meta-core's GET /api/files/tuples. null = a meta-core that
|
||||
// predates the endpoint.
|
||||
fileTuples: Array<{ hashId: string; filePath: string; sizeByte: number; mtimeNano: number }> | null = null;
|
||||
fileTuplesDelayMs = 0;
|
||||
calls = { getFileTuples: 0, getAllHashIds: 0, getMetadataFlat: 0 };
|
||||
|
||||
async getFileTuples(opts: { summary?: boolean } = {}) {
|
||||
this.calls.getFileTuples++;
|
||||
if (this.fileTuplesDelayMs) {
|
||||
await new Promise(resolve => setTimeout(resolve, this.fileTuplesDelayMs));
|
||||
}
|
||||
if (!this.fileTuples) return null;
|
||||
const files = this.fileTuples;
|
||||
const totalSize = files.reduce((sum, f) => sum + f.sizeByte, 0);
|
||||
return opts.summary
|
||||
? { count: files.length, totalSize }
|
||||
: { count: files.length, totalSize, files };
|
||||
}
|
||||
|
||||
// Basic operations
|
||||
async set(key: string, value: any): Promise<void> {
|
||||
this.rawStore.set(key, JSON.stringify(value));
|
||||
@@ -99,6 +119,7 @@ class MockKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getMetadataFlat(hashId: string): Promise<any | null> {
|
||||
this.calls.getMetadataFlat++;
|
||||
const entry = this.data.get(hashId);
|
||||
if (!entry) return null;
|
||||
return Object.fromEntries(entry);
|
||||
@@ -125,6 +146,7 @@ class MockKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getAllHashIds(): Promise<string[]> {
|
||||
this.calls.getAllHashIds++;
|
||||
return Array.from(this.data.keys());
|
||||
}
|
||||
|
||||
@@ -736,3 +758,157 @@ describe('UnifiedAPIServer', function() {
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// File totals + recompute against meta-core's GET /api/files/tuples
|
||||
// =============================================================================
|
||||
//
|
||||
// Each test builds its own server: /api/stats caches for 30 s and recompute is
|
||||
// single-flight, so shared state between tests would hide both behaviours.
|
||||
|
||||
describe('UnifiedAPIServer file totals and recompute', function() {
|
||||
const tuple = (hashId: string, sizeByte: number) =>
|
||||
({ hashId, filePath: `/files/watch/${hashId}.mkv`, sizeByte, mtimeNano: 1 });
|
||||
|
||||
let app: FastifyInstance;
|
||||
let kv: MockKVClient;
|
||||
let queued: Array<{ pluginId: string; fileHash: string }>;
|
||||
|
||||
beforeEach(async function() {
|
||||
kv = new MockKVClient();
|
||||
queued = [];
|
||||
const pluginManager = { getPlugins: () => [{ id: 'ffmpeg', active: true }] };
|
||||
const taskScheduler = {
|
||||
createTasksForPluginOnFiles: (pluginId: string, files: Array<{ fileHash: string }>) =>
|
||||
files.map(f => ({ pluginId, fileHash: f.fileHash })),
|
||||
enqueueTasks: (tasks: Array<{ pluginId: string; fileHash: string }>) => { queued.push(...tasks); },
|
||||
};
|
||||
const server = new UnifiedAPIServer(
|
||||
{ port: 0, host: 'localhost', enableCors: false },
|
||||
new MockProcessingStateManager() as any,
|
||||
kv as any,
|
||||
4,
|
||||
16,
|
||||
() => ({ fast: { pending: 0, running: 0 }, background: { pending: 0, running: 0 } }),
|
||||
undefined,
|
||||
() => pluginManager as any,
|
||||
() => taskScheduler as any
|
||||
);
|
||||
app = server.getApp();
|
||||
await app.ready();
|
||||
});
|
||||
|
||||
afterEach(async function() {
|
||||
await app.close();
|
||||
});
|
||||
|
||||
it('GET /api/stats sums meta-core file tuples without walking hash ids', async function() {
|
||||
kv.fileTuples = [tuple('a', 100), tuple('b', 250)];
|
||||
|
||||
const response = await app.inject({ method: 'GET', url: '/api/stats' });
|
||||
|
||||
expect(response.statusCode).to.equal(200);
|
||||
expect(JSON.parse(response.payload)).to.deep.equal({ fileCount: 2, totalSize: 350, source: 'meta-core' });
|
||||
expect(kv.calls.getAllHashIds).to.equal(0);
|
||||
expect(kv.calls.getMetadataFlat).to.equal(0);
|
||||
});
|
||||
|
||||
it('GET /api/stats degrades to nulls on a meta-core without the endpoint', async function() {
|
||||
kv.fileTuples = null;
|
||||
|
||||
const response = await app.inject({ method: 'GET', url: '/api/stats' });
|
||||
|
||||
expect(response.statusCode).to.equal(200);
|
||||
expect(JSON.parse(response.payload)).to.deep.equal({ fileCount: null, totalSize: null, source: 'unavailable' });
|
||||
expect(kv.calls.getAllHashIds).to.equal(0);
|
||||
});
|
||||
|
||||
it('GET /api/stats shares one meta-core call between concurrent polls', async function() {
|
||||
kv.fileTuples = [tuple('a', 100)];
|
||||
kv.fileTuplesDelayMs = 50;
|
||||
|
||||
const responses = await Promise.all(
|
||||
Array.from({ length: 10 }, () => app.inject({ method: 'GET', url: '/api/stats' }))
|
||||
);
|
||||
|
||||
for (const response of responses) {
|
||||
expect(response.statusCode).to.equal(200);
|
||||
expect(JSON.parse(response.payload).totalSize).to.equal(100);
|
||||
}
|
||||
expect(kv.calls.getFileTuples).to.equal(1);
|
||||
});
|
||||
|
||||
it('POST recompute returns 501 on a meta-core without the endpoint', async function() {
|
||||
kv.fileTuples = null;
|
||||
|
||||
const response = await app.inject({ method: 'POST', url: '/api/plugins/ffmpeg/recompute' });
|
||||
|
||||
expect(response.statusCode).to.equal(501);
|
||||
expect(kv.calls.getAllHashIds).to.equal(0);
|
||||
expect(queued).to.have.length(0);
|
||||
});
|
||||
|
||||
it('POST recompute queues only file-backed records', async function() {
|
||||
kv.seedData('a', { filePath: '/files/watch/a.mkv', fileType: 'video' });
|
||||
kv.seedData('b', { filePath: '/files/watch/b.mkv', fileType: 'video' });
|
||||
kv.seedData('gateway-record', { title: 'Not a file' });
|
||||
kv.fileTuples = [tuple('a', 100), tuple('b', 250)];
|
||||
|
||||
const response = await app.inject({ method: 'POST', url: '/api/plugins/ffmpeg/recompute' });
|
||||
|
||||
expect(response.statusCode).to.equal(200);
|
||||
const body = JSON.parse(response.payload);
|
||||
expect(body.filesQueued).to.equal(2);
|
||||
expect(body.totalFiles).to.equal(2);
|
||||
expect(queued.map(q => q.fileHash).sort()).to.deep.equal(['a', 'b']);
|
||||
expect(kv.calls.getMetadataFlat).to.equal(2);
|
||||
expect(kv.calls.getAllHashIds).to.equal(0);
|
||||
});
|
||||
|
||||
it('POST recompute refuses a second run for the same plugin while one is queuing', async function() {
|
||||
kv.seedData('a', { filePath: '/files/watch/a.mkv' });
|
||||
kv.fileTuples = [tuple('a', 100)];
|
||||
kv.fileTuplesDelayMs = 50;
|
||||
|
||||
const [first, second] = await Promise.all([
|
||||
app.inject({ method: 'POST', url: '/api/plugins/ffmpeg/recompute' }),
|
||||
app.inject({ method: 'POST', url: '/api/plugins/ffmpeg/recompute' }),
|
||||
]);
|
||||
|
||||
expect([first.statusCode, second.statusCode].sort()).to.deep.equal([200, 409]);
|
||||
expect(queued).to.have.length(1);
|
||||
});
|
||||
});
|
||||
|
||||
// =============================================================================
|
||||
// File-level processing metric
|
||||
// =============================================================================
|
||||
|
||||
describe('UnifiedProcessingStateManager file metric', function() {
|
||||
// Files announced over SSE go straight to light processing; they are never
|
||||
// marked discovered, which is the path that used to leave "Processed" at 0.
|
||||
const runFile = (manager: UnifiedProcessingStateManager, filePath: string, error?: string) => {
|
||||
manager.startLightProcessing(filePath);
|
||||
manager.completeLightProcessing(filePath, 'bagacbabaeexamplehash');
|
||||
manager.startHashProcessing(filePath);
|
||||
manager.completeHashProcessing(filePath, 'bagacbabaeexamplehash', undefined, error);
|
||||
};
|
||||
|
||||
it('counts a file as processed when its hash phase completes cleanly', function() {
|
||||
const manager = new UnifiedProcessingStateManager();
|
||||
const before = performanceMetrics.getMetrics().totalFilesProcessed;
|
||||
|
||||
runFile(manager, '/files/watch/metric-ok.mkv');
|
||||
|
||||
expect(performanceMetrics.getMetrics().totalFilesProcessed).to.equal(before + 1);
|
||||
});
|
||||
|
||||
it('does not count a file whose hash phase failed', function() {
|
||||
const manager = new UnifiedProcessingStateManager();
|
||||
const before = performanceMetrics.getMetrics().totalFilesProcessed;
|
||||
|
||||
runFile(manager, '/files/watch/metric-failed.mkv', 'plugin failed');
|
||||
|
||||
expect(performanceMetrics.getMetrics().totalFilesProcessed).to.equal(before);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -69,6 +69,18 @@ interface EnvConfig {
|
||||
/** Concurrency for fast plugin queue. Default: 32 */
|
||||
FAST_QUEUE_CONCURRENCY: number;
|
||||
|
||||
/**
|
||||
* Maximum number of files concurrently in the light (fast-queue) phase.
|
||||
* Caps fan-out so the underlying file cache (WebDAV/nginx) only has to
|
||||
* hold N files at a time. Default: 16.
|
||||
*/
|
||||
FILE_LIGHT_SLOTS: number;
|
||||
|
||||
/**
|
||||
* Maximum number of files concurrently in the background phase. Default: 16.
|
||||
*/
|
||||
FILE_BG_SLOTS: number;
|
||||
|
||||
// ========================================================================
|
||||
// Container Plugins Configuration
|
||||
// All plugins run as Docker containers for isolation and flexibility
|
||||
@@ -77,6 +89,13 @@ interface EnvConfig {
|
||||
/** Path to container plugins config file (default: '/app/plugins.yml') */
|
||||
CONTAINER_PLUGINS_CONFIG: string;
|
||||
|
||||
/**
|
||||
* External plugins added by URL from the Plugins page's beacon Scan card
|
||||
* (default: '<CACHE_FOLDER_PATH>/state/external-plugins.json'). Kept apart
|
||||
* from plugins.yml, which the store app mounts read-only.
|
||||
*/
|
||||
EXTERNAL_PLUGINS_PATH: string;
|
||||
|
||||
/** Docker socket path (default: '/var/run/docker.sock') */
|
||||
DOCKER_SOCKET_PATH: string;
|
||||
|
||||
@@ -139,9 +158,13 @@ export const config: EnvConfig = {
|
||||
|
||||
// TaskScheduler Configuration
|
||||
FAST_QUEUE_CONCURRENCY: parseInt(process.env.FAST_QUEUE_CONCURRENCY || "32", 10),
|
||||
FILE_LIGHT_SLOTS: parseInt(process.env.FILE_LIGHT_SLOTS || "16", 10),
|
||||
FILE_BG_SLOTS: parseInt(process.env.FILE_BG_SLOTS || "16", 10),
|
||||
|
||||
// Container Plugins Configuration
|
||||
CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml',
|
||||
EXTERNAL_PLUGINS_PATH: process.env.EXTERNAL_PLUGINS_PATH
|
||||
|| `${process.env.CACHE_FOLDER_PATH || '/data/cache'}/state/external-plugins.json`,
|
||||
DOCKER_SOCKET_PATH: process.env.DOCKER_SOCKET_PATH || '/var/run/docker.sock',
|
||||
CONTAINER_CALLBACK_URL: process.env.CONTAINER_CALLBACK_URL || 'http://meta-sort:8180',
|
||||
CONTAINER_META_CORE_URL: process.env.CONTAINER_META_CORE_URL || 'http://meta-sort',
|
||||
|
||||
@@ -26,7 +26,8 @@ export function renamingRule(metamesh: Local, filepath?: string): string | null
|
||||
throw new Error(`No extension found for file: ${filepath}`);
|
||||
}
|
||||
|
||||
const title = metamesh?.titles?.eng || metamesh.originalTitle;
|
||||
// The display name — never the `titles/*` name set, which has no preferred member (METADATA_KEYS.md §3).
|
||||
const title = metamesh.title || metamesh.originalTitle;
|
||||
if (!title) {
|
||||
if (metamesh.fileType !== "torrent") {
|
||||
//don't warn for torrent files because they don't always have a title (eg representing a folder of files)
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm} from "@metazla/meta-hash";
|
||||
|
||||
// Target hashes for Piscina worker (full-file hash computation)
|
||||
// Note: midhash256 is NOT included here because it's computed in the main thread
|
||||
// during light processing, not by the Piscina worker
|
||||
export const targetHash = [
|
||||
CID_ALGORITHM_NAMES.sha1, // For torrent verification
|
||||
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
|
||||
CID_ALGORITHM_NAMES.md5, // For compatibility
|
||||
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.crc32, // Fast checksum
|
||||
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
|
||||
//
|
||||
// These are algorithm *selectors*, not metadata field names. The old
|
||||
// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
|
||||
// removed `cid_<algo>` record fields kept getting minted in the first place.
|
||||
// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
|
||||
// See METADATA_KEYS.md §2/§14.13.
|
||||
export const targetHash: CidAlgorithm[] = [
|
||||
'sha1', // For torrent verification
|
||||
'sha256', // For IPFS compatibility
|
||||
'md5', // For compatibility
|
||||
'sha3_256', // Additional cryptographic hash
|
||||
'sha3_384', // Additional cryptographic hash
|
||||
'crc32', // Fast checksum
|
||||
'btih_v2', // BitTorrent v2 info hash (BEP 52)
|
||||
]
|
||||
|
||||
// Target hashes for index-cid cache (includes all hashes we want to cache)
|
||||
// This includes midhash256 which is computed separately from the worker hashes
|
||||
export const targetHashForIndex = [
|
||||
export const targetHashForIndex: CidAlgorithm[] = [
|
||||
...targetHash,
|
||||
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
'midhash256', // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
|
||||
@@ -25,6 +25,9 @@ import type {
|
||||
PluginConfigureResponse,
|
||||
} from './types.js';
|
||||
import { config } from '../config/EnvConfig.js';
|
||||
import { existsSync } from 'fs';
|
||||
import { hostname } from 'os';
|
||||
import { ExternalPluginStore, type ExternalPluginEntry } from './ExternalPluginStore.js';
|
||||
|
||||
/**
|
||||
* Service info structure from meta-core service discovery
|
||||
@@ -96,6 +99,11 @@ export class ContainerManager extends EventEmitter {
|
||||
private stackName?: string;
|
||||
private callbackUrl: string;
|
||||
private metaCoreUrl: string;
|
||||
/** External plugins (added by URL); see ExternalPluginStore. */
|
||||
private externalStore: ExternalPluginStore;
|
||||
private externalEntries: ExternalPluginEntry[] = [];
|
||||
/** False when Docker is unreachable: only external plugins run then. */
|
||||
private dockerAvailable = false;
|
||||
|
||||
constructor(
|
||||
private configPath: string = config.CONTAINER_PLUGINS_CONFIG,
|
||||
@@ -116,6 +124,12 @@ export class ContainerManager extends EventEmitter {
|
||||
|
||||
// Configure stack name for Docker Desktop grouping
|
||||
this.stackName = config.PLUGIN_STACK_NAME;
|
||||
this.externalStore = new ExternalPluginStore(config.EXTERNAL_PLUGINS_PATH);
|
||||
}
|
||||
|
||||
/** Swap the external-plugin store (tests). */
|
||||
setExternalStore(store: ExternalPluginStore): void {
|
||||
this.externalStore = store;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -142,53 +156,18 @@ export class ContainerManager extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
// Fall back to service discovery for any missing URLs
|
||||
const serviceFile = join(config.META_CORE_PATH, 'services', 'meta-sort.json');
|
||||
|
||||
try {
|
||||
const content = await fs.readFile(serviceFile, 'utf-8');
|
||||
const serviceInfo: ServiceInfo = JSON.parse(content);
|
||||
|
||||
// Extract URLs from service info
|
||||
const api = serviceInfo.api;
|
||||
const endpoints = serviceInfo.endpoints || {};
|
||||
|
||||
// Use service discovery for callback URL if not set
|
||||
if (!this.callbackUrl) {
|
||||
if (endpoints.callback) {
|
||||
this.callbackUrl = endpoints.callback;
|
||||
console.log(`[ContainerManager] Discovered callback URL: ${this.callbackUrl}`);
|
||||
} else if (api) {
|
||||
// Fallback: construct from api base
|
||||
this.callbackUrl = `${api}/api/plugins/callback`;
|
||||
console.log(`[ContainerManager] Constructed callback URL: ${this.callbackUrl}`);
|
||||
}
|
||||
}
|
||||
|
||||
// Use service discovery for WebDAV URL if not set (fallback)
|
||||
if (!this.webdavUrl) {
|
||||
if (endpoints.webdav) {
|
||||
this.webdavUrl = endpoints.webdav;
|
||||
console.log(`[ContainerManager] Discovered WebDAV URL: ${this.webdavUrl}`);
|
||||
} else if (api) {
|
||||
// Fallback: construct from api base
|
||||
this.webdavUrl = `${api}/webdav`;
|
||||
console.log(`[ContainerManager] Constructed WebDAV URL: ${this.webdavUrl}`);
|
||||
}
|
||||
}
|
||||
|
||||
if (endpoints.health) {
|
||||
// Extract meta-core URL from health endpoint
|
||||
const healthUrl = new URL(endpoints.health);
|
||||
this.metaCoreUrl = `${healthUrl.protocol}//${healthUrl.host}`;
|
||||
console.log(`[ContainerManager] Discovered meta-core URL: ${this.metaCoreUrl}`);
|
||||
}
|
||||
|
||||
console.log('[ContainerManager] Service discovery successful');
|
||||
} catch (error) {
|
||||
console.warn(`[ContainerManager] Service discovery failed: ${error}`);
|
||||
console.log(`[ContainerManager] Using callback URL: ${this.callbackUrl}`);
|
||||
console.log(`[ContainerManager] Using WebDAV URL: ${this.webdavUrl || 'not set'}`);
|
||||
// Anything still missing is a configuration problem, not something to
|
||||
// guess at. The old fallback here read
|
||||
// `${META_CORE_PATH}/services/meta-sort.json` — a filename that stopped
|
||||
// existing when registration files became hostname-suffixed, using a
|
||||
// schema (`.api` / `.endpoints`) that no longer matched either. It had
|
||||
// been falling into its own catch block for a long time, and the
|
||||
// volume it read is gone since meta-discovery v1.
|
||||
if (!this.callbackUrl) {
|
||||
console.warn('[ContainerManager] No callback URL — set CONTAINER_CALLBACK_URL');
|
||||
}
|
||||
if (!this.webdavUrl) {
|
||||
console.warn('[ContainerManager] No WebDAV URL — meta-core did not supply one');
|
||||
}
|
||||
}
|
||||
|
||||
@@ -218,22 +197,41 @@ export class ContainerManager extends EventEmitter {
|
||||
console.warn(' Plugins will not have access to files.');
|
||||
}
|
||||
|
||||
// Initialize Docker client
|
||||
await this.dockerClient.initialize();
|
||||
// Docker is needed only for the plugins we spawn. Without it (no
|
||||
// socket, permission denied) external plugins still run.
|
||||
try {
|
||||
await this.dockerClient.initialize();
|
||||
await this.dockerClient.ensureNetwork(this.network);
|
||||
this.dockerAvailable = true;
|
||||
} catch (error) {
|
||||
console.warn('[ContainerManager] Docker unavailable — only external plugins will run:', error);
|
||||
}
|
||||
|
||||
// Ensure network exists
|
||||
await this.dockerClient.ensureNetwork(this.network);
|
||||
|
||||
// Load configuration
|
||||
this.pluginsConfig = await loadConfig(this.configPath);
|
||||
// Load configuration. A missing or broken plugins.yml means "no
|
||||
// spawned plugins", not "no plugin manager": external plugins can
|
||||
// still be added from the UI.
|
||||
if (existsSync(this.configPath)) {
|
||||
try {
|
||||
this.pluginsConfig = await loadConfig(this.configPath);
|
||||
} catch (error) {
|
||||
console.error(`[ContainerManager] Ignoring ${this.configPath}:`, error);
|
||||
}
|
||||
} else {
|
||||
console.log(`[ContainerManager] No ${this.configPath}; no spawned plugins`);
|
||||
}
|
||||
if (!this.pluginsConfig) {
|
||||
this.pluginsConfig = { version: '1.0', plugins: {} };
|
||||
}
|
||||
|
||||
// Get enabled plugins
|
||||
const enabledPlugins = getEnabledPlugins(this.pluginsConfig);
|
||||
const enabledPlugins = this.dockerAvailable ? getEnabledPlugins(this.pluginsConfig) : [];
|
||||
|
||||
console.log(`[ContainerManager] Found ${enabledPlugins.length} enabled plugins`);
|
||||
|
||||
// Clean up any stale containers from previous runs
|
||||
await this.cleanupStaleContainers();
|
||||
if (this.dockerAvailable) {
|
||||
// Clean up any stale containers from previous runs
|
||||
await this.cleanupStaleContainers();
|
||||
}
|
||||
|
||||
// Spawn containers for each enabled plugin
|
||||
for (const { id, config: pluginConfig } of enabledPlugins) {
|
||||
@@ -248,6 +246,21 @@ export class ContainerManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
// External plugins (added by URL). One unreachable at boot is
|
||||
// registered anyway and picked up by the health loop when it appears.
|
||||
this.externalEntries = await this.externalStore.load();
|
||||
for (const entry of this.externalEntries.filter((e) => e.enabled)) {
|
||||
if (this.instances.has(entry.pluginId)) {
|
||||
console.warn(
|
||||
`[ContainerManager] External plugin '${entry.pluginId}' (${entry.url}) skipped: ` +
|
||||
'a spawned plugin already has that id'
|
||||
);
|
||||
continue;
|
||||
}
|
||||
this.attachExternal(entry);
|
||||
}
|
||||
console.log(`[ContainerManager] ${this.externalEntries.length} external plugin(s) configured`);
|
||||
|
||||
this.initialized = true;
|
||||
this.emit('manager:initialized', { pluginCount: enabledPlugins.length });
|
||||
|
||||
@@ -450,6 +463,11 @@ export class ContainerManager extends EventEmitter {
|
||||
CALLBACK_URL: this.callbackUrl,
|
||||
WEBDAV_URL: this.webdavUrl || '',
|
||||
FILES_PATH: '/files', // Virtual path - files accessed via WebDAV
|
||||
// Beacon v2: advertise the URL we reach it on (so a Scan shows it
|
||||
// as configured), and only to this meta-sort (another one's
|
||||
// Scan shows it as bound elsewhere, never addable).
|
||||
BEACON_ADVERTISE_URL: `http://${containerName}:8080`,
|
||||
BEACON_BINDS: hostname(),
|
||||
},
|
||||
// Docker Desktop grouping
|
||||
stackName: this.stackName,
|
||||
@@ -462,6 +480,7 @@ export class ContainerManager extends EventEmitter {
|
||||
// Create instance record
|
||||
const instance: ContainerPluginInstance = {
|
||||
pluginId,
|
||||
kind: 'container',
|
||||
containerId,
|
||||
containerName,
|
||||
baseUrl: `http://${containerName}:8080`,
|
||||
@@ -480,6 +499,12 @@ export class ContainerManager extends EventEmitter {
|
||||
* Stop a container instance
|
||||
*/
|
||||
private async stopInstance(instance: ContainerPluginInstance): Promise<void> {
|
||||
if (instance.kind === 'external') {
|
||||
// Not ours to stop: just stop using it.
|
||||
instance.status = 'stopped';
|
||||
this.emit('plugin:stopped', { pluginId: instance.pluginId, instanceIndex: instance.instanceIndex });
|
||||
return;
|
||||
}
|
||||
console.log(`[ContainerManager] Stopping ${instance.containerName}...`);
|
||||
|
||||
try {
|
||||
@@ -525,6 +550,11 @@ export class ContainerManager extends EventEmitter {
|
||||
if (healthy) {
|
||||
instance.status = 'healthy';
|
||||
instance.lastHealthCheck = Date.now();
|
||||
// An external plugin unreachable at boot has no manifest
|
||||
// yet; the scheduler needs one.
|
||||
if (!instance.manifest) {
|
||||
instance.manifest = await this.fetchManifest(instance).catch(() => undefined);
|
||||
}
|
||||
|
||||
if (previousStatus !== 'healthy') {
|
||||
this.emit('plugin:healthy', { pluginId, instance });
|
||||
@@ -644,9 +674,12 @@ export class ContainerManager extends EventEmitter {
|
||||
throw new Error(`Failed to configure plugin: ${response.status}`);
|
||||
}
|
||||
|
||||
// Two contracts in the wild: `{ success: true }` and the TS plugins'
|
||||
// `{ status: 'ok' }`. Before both were accepted every configure of a
|
||||
// TS plugin "failed" (silently, behind a try/warn).
|
||||
const data = (await response.json()) as PluginConfigureResponse;
|
||||
if (!data.success) {
|
||||
throw new Error(`Plugin configuration failed: ${data.error}`);
|
||||
if (!data.success && data.status !== 'ok') {
|
||||
throw new Error(`Plugin configuration failed: ${data.error ?? JSON.stringify(data)}`);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -713,11 +746,14 @@ export class ContainerManager extends EventEmitter {
|
||||
for (const [pluginId, instances] of this.instances) {
|
||||
const pluginConfig = this.pluginsConfig?.plugins[pluginId];
|
||||
const healthyCount = instances.filter((i) => i.status === 'healthy').length;
|
||||
const external = instances[0]?.kind === 'external';
|
||||
|
||||
plugins.push({
|
||||
pluginId,
|
||||
enabled: pluginConfig?.enabled !== false,
|
||||
image: pluginConfig?.image || 'unknown',
|
||||
image: external ? instances[0].baseUrl : pluginConfig?.image || 'unknown',
|
||||
kind: external ? 'external' : 'container',
|
||||
url: instances[0]?.baseUrl,
|
||||
instances: instances.length,
|
||||
healthyInstances: healthyCount,
|
||||
manifest: instances[0]?.manifest,
|
||||
@@ -752,6 +788,16 @@ export class ContainerManager extends EventEmitter {
|
||||
const instances = this.instances.get(pluginId);
|
||||
const pluginConfig = this.pluginsConfig?.plugins[pluginId];
|
||||
|
||||
if (instances?.[0]?.kind === 'external') {
|
||||
// Nothing to respawn: re-probe it and refresh its manifest.
|
||||
for (const instance of instances) {
|
||||
const healthy = await this.checkHealth(instance);
|
||||
instance.status = healthy ? 'healthy' : 'unhealthy';
|
||||
if (healthy) instance.manifest = await this.fetchManifest(instance).catch(() => instance.manifest);
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!instances || !pluginConfig) {
|
||||
throw new Error(`Plugin '${pluginId}' not found`);
|
||||
}
|
||||
@@ -814,6 +860,93 @@ export class ContainerManager extends EventEmitter {
|
||||
}
|
||||
}
|
||||
|
||||
/** The external plugins, as persisted. */
|
||||
getExternalPlugins(): ExternalPluginEntry[] {
|
||||
return [...this.externalEntries];
|
||||
}
|
||||
|
||||
/**
|
||||
* Register an external plugin (one meta-sort did not spawn) by URL — what
|
||||
* the beacon Scan card's Add does. Its manifest id becomes its plugin id,
|
||||
* so it must be reachable now and its id must not clash with a loaded one.
|
||||
*/
|
||||
async addExternalPlugin(rawUrl: string, name?: string): Promise<ExternalPluginEntry> {
|
||||
const url = rawUrl.trim().replace(/\/+$/, '');
|
||||
if (!/^https?:\/\//.test(url)) {
|
||||
throw new Error('url must start with http:// or https://');
|
||||
}
|
||||
for (const list of this.instances.values()) {
|
||||
const holder = list.find((i) => i.baseUrl.replace(/\/+$/, '') === url);
|
||||
if (holder) throw new Error(`${url} is already registered as '${holder.pluginId}'`);
|
||||
}
|
||||
const probe: ContainerPluginInstance = {
|
||||
pluginId: '?', kind: 'external', containerId: '', containerName: url, baseUrl: url,
|
||||
instanceIndex: 0, status: 'starting', tasksProcessed: 0, tasksFailed: 0,
|
||||
};
|
||||
let manifest: ContainerPluginManifest;
|
||||
try {
|
||||
manifest = await this.fetchManifest(probe);
|
||||
} catch (error) {
|
||||
throw new Error(`could not read ${url}/manifest: ${error instanceof Error ? error.message : error}`);
|
||||
}
|
||||
const pluginId = manifest?.id;
|
||||
if (!pluginId || !/^[a-z][a-z0-9-]*$/.test(pluginId)) {
|
||||
throw new Error(`${url}/manifest has no usable plugin id`);
|
||||
}
|
||||
if (this.instances.has(pluginId)) {
|
||||
throw new Error(`a plugin with id '${pluginId}' is already loaded`);
|
||||
}
|
||||
const entry: ExternalPluginEntry = {
|
||||
pluginId, url, enabled: true, name: name?.trim() || undefined, addedAt: new Date().toISOString(),
|
||||
};
|
||||
this.externalEntries = [...this.externalEntries.filter((e) => e.pluginId !== pluginId), entry];
|
||||
await this.externalStore.save(this.externalEntries);
|
||||
this.attachExternal(entry, manifest);
|
||||
console.log(`[ContainerManager] External plugin '${pluginId}' added (${url})`);
|
||||
return entry;
|
||||
}
|
||||
|
||||
/** Forget an external plugin. Throws if `pluginId` is not one. */
|
||||
async removeExternalPlugin(pluginId: string): Promise<void> {
|
||||
const known = this.externalEntries.some((e) => e.pluginId === pluginId);
|
||||
const instances = this.instances.get(pluginId);
|
||||
if (!known && !(instances?.[0]?.kind === 'external')) {
|
||||
throw new Error(`'${pluginId}' is not an external plugin`);
|
||||
}
|
||||
if (instances?.[0]?.kind === 'external') {
|
||||
const interval = this.healthCheckIntervals.get(pluginId);
|
||||
if (interval) {
|
||||
clearInterval(interval);
|
||||
this.healthCheckIntervals.delete(pluginId);
|
||||
}
|
||||
for (const instance of instances) await this.stopInstance(instance);
|
||||
this.instances.delete(pluginId);
|
||||
this.roundRobinCounters.delete(pluginId);
|
||||
}
|
||||
this.externalEntries = this.externalEntries.filter((e) => e.pluginId !== pluginId);
|
||||
await this.externalStore.save(this.externalEntries);
|
||||
console.log(`[ContainerManager] External plugin '${pluginId}' removed`);
|
||||
}
|
||||
|
||||
/** One instance pointing at `entry.url`, kept alive by the health loop. */
|
||||
private attachExternal(entry: ExternalPluginEntry, manifest?: ContainerPluginManifest): void {
|
||||
const instance: ContainerPluginInstance = {
|
||||
pluginId: entry.pluginId,
|
||||
kind: 'external',
|
||||
containerId: '',
|
||||
// The scheduler reports completions by this name; the URL is unique.
|
||||
containerName: entry.url,
|
||||
baseUrl: entry.url,
|
||||
instanceIndex: 0,
|
||||
status: 'starting',
|
||||
manifest,
|
||||
tasksProcessed: 0,
|
||||
tasksFailed: 0,
|
||||
};
|
||||
this.instances.set(entry.pluginId, [instance]);
|
||||
this.startHealthCheck(entry.pluginId);
|
||||
}
|
||||
|
||||
/**
|
||||
* Add a new plugin dynamically
|
||||
*/
|
||||
@@ -873,6 +1006,11 @@ export class ContainerManager extends EventEmitter {
|
||||
console.log(`[ContainerManager] Removing plugin '${pluginId}'...`);
|
||||
|
||||
const instances = this.instances.get(pluginId);
|
||||
// An external plugin is forgotten (and un-persisted), never stopped.
|
||||
if (instances?.[0]?.kind === 'external' || this.externalEntries.some((e) => e.pluginId === pluginId)) {
|
||||
await this.removeExternalPlugin(pluginId);
|
||||
return;
|
||||
}
|
||||
|
||||
if (instances) {
|
||||
// Stop health check
|
||||
|
||||
@@ -21,6 +21,7 @@ import type { TaskQueueType } from '../plugin-engine/types.js';
|
||||
import type { IKVClient } from '../kv/IKVClient.js';
|
||||
import { type ExtendedContainerTask } from '../types/ExtendedInterfaces.js';
|
||||
import { config } from '../config/EnvConfig.js';
|
||||
import { FileSlotGate } from './FileSlotGate.js';
|
||||
|
||||
/**
|
||||
* Scheduler events
|
||||
@@ -55,6 +56,18 @@ export interface GateStatus {
|
||||
backgroundRunning: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-file holder snapshot exposed via getQueueStatus for the dashboard.
|
||||
* Lets operators see which 16 files are tying up cache slots and what plugin
|
||||
* each file is currently waiting on (the long-pole per file).
|
||||
*/
|
||||
export interface CacheSlotHolder {
|
||||
fileHash: string;
|
||||
filePath?: string;
|
||||
currentPlugin?: string;
|
||||
tasksRemaining: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Container Plugin Scheduler
|
||||
*/
|
||||
@@ -73,11 +86,30 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
private kvClient: IKVClient | null = null;
|
||||
private isAcceptingNewTasks: boolean = true; // Gate control for safe mount/unmount
|
||||
|
||||
// File-level cache slots — see FileSlotGate.ts. Caps the number of files
|
||||
// whose bytes are concurrently being touched in each phase, so the WebDAV
|
||||
// cache only has to hold N files at a time regardless of plugin fan-out.
|
||||
private lightFileGate: FileSlotGate;
|
||||
private bgFileGate: FileSlotGate;
|
||||
// Per-(file, phase) outstanding task counter — drives slot release.
|
||||
private lightTasksPerFile: Map<string, number> = new Map();
|
||||
private bgTasksPerFile: Map<string, number> = new Map();
|
||||
// Tasks staged while waiting for a slot. Once the gate grants the file a
|
||||
// slot we flush the buffered tasks into the underlying PQueue.
|
||||
private lightBufferedTasks: Map<string, ContainerTask[]> = new Map();
|
||||
private bgBufferedTasks: Map<string, ContainerTask[]> = new Map();
|
||||
// Currently-dispatched plugin per file in each phase — used by the
|
||||
// dashboard to show "this file is currently waiting on plugin X".
|
||||
private lightCurrentPlugin: Map<string, string> = new Map();
|
||||
private bgCurrentPlugin: Map<string, string> = new Map();
|
||||
|
||||
constructor(
|
||||
containerManager: ContainerManager,
|
||||
options?: {
|
||||
fastConcurrency?: number;
|
||||
backgroundConcurrency?: number;
|
||||
lightFileSlots?: number;
|
||||
bgFileSlots?: number;
|
||||
callbackUrl?: string;
|
||||
metaCoreUrl?: string;
|
||||
defaultTimeout?: number;
|
||||
@@ -98,6 +130,9 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
autoStart: false,
|
||||
});
|
||||
|
||||
this.lightFileGate = new FileSlotGate(options?.lightFileSlots ?? config.FILE_LIGHT_SLOTS);
|
||||
this.bgFileGate = new FileSlotGate(options?.bgFileSlots ?? config.FILE_BG_SLOTS);
|
||||
|
||||
// When fast queue becomes idle, start background queue
|
||||
this.fastQueue.on('idle', () => {
|
||||
this.onFastQueueIdle();
|
||||
@@ -246,6 +281,17 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
/**
|
||||
* Enqueue a task for execution
|
||||
* Returns false if the gate is closed and the task was rejected
|
||||
*
|
||||
* Tasks pass through the per-phase FileSlotGate before reaching PQueue:
|
||||
* - If the file already holds a slot in this phase, the task goes
|
||||
* straight to PQueue.
|
||||
* - Otherwise the task is buffered, and the file's first task triggers
|
||||
* a slot acquire. When the slot is granted, all buffered tasks for
|
||||
* that file are flushed into PQueue.
|
||||
*
|
||||
* This caps concurrent files per phase (and therefore concurrent file
|
||||
* fetches against CORN/SMB) at the gate's capacity, regardless of how
|
||||
* many plugin tasks each file fans out to.
|
||||
*/
|
||||
enqueueTask(task: ContainerTask): boolean {
|
||||
if (!this.isAcceptingNewTasks) {
|
||||
@@ -262,20 +308,90 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
return false;
|
||||
}
|
||||
|
||||
const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
|
||||
|
||||
// If adding to fast queue, pause background queue (fast takes priority)
|
||||
if (task.queue === 'fast' && !this.backgroundQueue.isPaused) {
|
||||
console.log('[ContainerScheduler] Fast task added, pausing background queue');
|
||||
this.backgroundQueue.pause();
|
||||
}
|
||||
|
||||
queue.add(async () => {
|
||||
await this.executeTask(task);
|
||||
const counterMap = task.queue === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const bufferMap = task.queue === 'fast' ? this.lightBufferedTasks : this.bgBufferedTasks;
|
||||
const gate = task.queue === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
|
||||
// Increment the per-file counter for this phase eagerly. Decrement
|
||||
// happens in the executeTask finally clause.
|
||||
counterMap.set(task.fileHash, (counterMap.get(task.fileHash) ?? 0) + 1);
|
||||
|
||||
if (gate.holds(task.fileHash)) {
|
||||
// File already has a slot — dispatch immediately.
|
||||
this.dispatchToPQueue(task);
|
||||
return true;
|
||||
}
|
||||
|
||||
// File doesn't hold a slot. Buffer the task. If this is the first
|
||||
// buffered task for the file, trigger gate.acquire and flush on grant.
|
||||
const existingBuffer = bufferMap.get(task.fileHash);
|
||||
if (existingBuffer) {
|
||||
existingBuffer.push(task);
|
||||
return true;
|
||||
}
|
||||
|
||||
bufferMap.set(task.fileHash, [task]);
|
||||
gate.acquire(task.fileHash).then(() => {
|
||||
const buffered = bufferMap.get(task.fileHash) ?? [];
|
||||
bufferMap.delete(task.fileHash);
|
||||
for (const t of buffered) {
|
||||
this.dispatchToPQueue(t);
|
||||
}
|
||||
// Edge case: if every buffered task got cancelled by clear() before
|
||||
// the slot was granted, release immediately so the slot doesn't
|
||||
// leak.
|
||||
if (buffered.length === 0 && (counterMap.get(task.fileHash) ?? 0) === 0) {
|
||||
gate.release(task.fileHash);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch a task into the underlying PQueue and release its file's gate
|
||||
* slot when the file's last task in that phase finishes.
|
||||
*/
|
||||
private dispatchToPQueue(task: ContainerTask): void {
|
||||
const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
|
||||
queue.add(async () => {
|
||||
try {
|
||||
await this.executeTask(task);
|
||||
} finally {
|
||||
this.onTaskFinished(task);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bookkeeping after a task leaves PQueue (success, failure, or thrown).
|
||||
* Decrements the per-(file, phase) task counter and releases the gate
|
||||
* slot when the counter hits zero.
|
||||
*/
|
||||
private onTaskFinished(task: ContainerTask): void {
|
||||
const counterMap = task.queue === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const currentMap = task.queue === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
const gate = task.queue === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
|
||||
const remaining = (counterMap.get(task.fileHash) ?? 1) - 1;
|
||||
if (remaining <= 0) {
|
||||
counterMap.delete(task.fileHash);
|
||||
currentMap.delete(task.fileHash);
|
||||
gate.release(task.fileHash);
|
||||
} else {
|
||||
counterMap.set(task.fileHash, remaining);
|
||||
// Clear current-plugin only if it points at this completed task.
|
||||
if (currentMap.get(task.fileHash) === task.pluginId) {
|
||||
currentMap.delete(task.fileHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when fast queue becomes idle - start background queue if it has work
|
||||
*/
|
||||
@@ -314,6 +430,11 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
task.dispatchedAt = Date.now();
|
||||
task.instanceName = instance.containerName;
|
||||
|
||||
// Record which plugin is currently running for this file (per phase)
|
||||
// so the dashboard can show the long-pole plugin per held cache slot.
|
||||
const currentMap = task.queue === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
currentMap.set(task.fileHash, task.pluginId);
|
||||
|
||||
this.emit('task:dispatched', { task });
|
||||
|
||||
try {
|
||||
@@ -581,6 +702,16 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
backgroundRunning: number; // unique files with background tasks running
|
||||
backgroundWaiting: number; // unique files with background tasks waiting
|
||||
};
|
||||
// Per-phase cache-slot view — bounded by FileSlotGate. This is the
|
||||
// metric operators actually care about: how many distinct files are
|
||||
// tying up the WebDAV cache right now, and which plugin is the
|
||||
// long-pole on each. `paused` reflects the underlying PQueue state so
|
||||
// the UI can distinguish "16 files reserved but bg PQueue paused"
|
||||
// from "16 files actively dispatching".
|
||||
cacheSlots: {
|
||||
fast: { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] };
|
||||
background: { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] };
|
||||
};
|
||||
} {
|
||||
// Count pending callbacks by queue type
|
||||
// These are tasks dispatched to containers awaiting completion - the real "running" count
|
||||
@@ -639,6 +770,56 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
backgroundRunning: backgroundRunningFiles.size,
|
||||
backgroundWaiting: backgroundWaitingFiles.size,
|
||||
},
|
||||
cacheSlots: {
|
||||
fast: this.snapshotCacheSlot('fast'),
|
||||
background: this.snapshotCacheSlot('background'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-phase cache-slot snapshot for the queue status payload.
|
||||
* `currentPlugin` is the plugin currently dispatched against the file
|
||||
* (the long-pole) — falls back to undefined if no task is mid-dispatch.
|
||||
* `tasksRemaining` is the count of light/bg tasks still outstanding for
|
||||
* the file in this phase, which is what holds the slot open.
|
||||
*/
|
||||
private snapshotCacheSlot(
|
||||
phase: TaskQueueType
|
||||
): { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] } {
|
||||
const gate = phase === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
const counterMap = phase === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const currentMap = phase === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
const queue = phase === 'fast' ? this.fastQueue : this.backgroundQueue;
|
||||
|
||||
const holders: CacheSlotHolder[] = gate.getHolders().map((fileHash) => {
|
||||
// Find any task for this file in this phase to recover filePath
|
||||
// (one filePath is enough for the UI; duplicates collapse).
|
||||
let filePath: string | undefined;
|
||||
const taskIds = this.fileTasks.get(fileHash);
|
||||
if (taskIds) {
|
||||
for (const tid of taskIds) {
|
||||
const t = this.pendingTasks.get(tid);
|
||||
if (t && t.queue === phase) {
|
||||
filePath = t.filePath;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
fileHash,
|
||||
filePath,
|
||||
currentPlugin: currentMap.get(fileHash),
|
||||
tasksRemaining: counterMap.get(fileHash) ?? 0,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
capacity: gate.capacity,
|
||||
inUse: gate.inUse(),
|
||||
waiting: gate.waiting(),
|
||||
paused: queue.isPaused,
|
||||
holders,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -751,6 +932,22 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
this.fileTasks.clear();
|
||||
this.fileCompletedPlugins.clear();
|
||||
this.dependencyWaiters.clear();
|
||||
|
||||
// Drop file-slot-gate state. Tasks that were buffered or holding
|
||||
// slots are no longer interesting once we've cleared the queues; a
|
||||
// fresh scheduler state implies fresh gate state.
|
||||
this.lightTasksPerFile.clear();
|
||||
this.bgTasksPerFile.clear();
|
||||
this.lightBufferedTasks.clear();
|
||||
this.bgBufferedTasks.clear();
|
||||
this.lightCurrentPlugin.clear();
|
||||
this.bgCurrentPlugin.clear();
|
||||
for (const fileHash of this.lightFileGate.getHolders()) {
|
||||
this.lightFileGate.release(fileHash);
|
||||
}
|
||||
for (const fileHash of this.bgFileGate.getHolders()) {
|
||||
this.bgFileGate.release(fileHash);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
/**
|
||||
* External plugins — enrichment plugins meta-sort did NOT spawn, reached by
|
||||
* URL (typically a MetaPlugin* store app found with the Plugins page's beacon
|
||||
* Scan card). Persisted as JSON next to the cache, because the store app
|
||||
* mounts plugins.yml read-only.
|
||||
*/
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
|
||||
export interface ExternalPluginEntry {
|
||||
/** The plugin's manifest id — its identity in the pipeline. */
|
||||
pluginId: string;
|
||||
/** Base URL of the plugin's HTTP contract (/manifest, /health, /process). */
|
||||
url: string;
|
||||
enabled: boolean;
|
||||
/** Name it was added under (the advertising instance), display only. */
|
||||
name?: string;
|
||||
addedAt?: string;
|
||||
}
|
||||
|
||||
export class ExternalPluginStore {
|
||||
constructor(private readonly path: string) {}
|
||||
|
||||
/** The persisted list; empty when the file is missing or unreadable. */
|
||||
async load(): Promise<ExternalPluginEntry[]> {
|
||||
let raw: string;
|
||||
try {
|
||||
raw = await fs.readFile(this.path, 'utf-8');
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
const list = Array.isArray(parsed?.plugins) ? parsed.plugins : [];
|
||||
return list.filter(
|
||||
(e: Partial<ExternalPluginEntry>) => typeof e?.pluginId === 'string' && typeof e?.url === 'string'
|
||||
).map((e: ExternalPluginEntry) => ({ ...e, enabled: e.enabled !== false }));
|
||||
} catch (error) {
|
||||
console.error(`[ExternalPluginStore] ${this.path} is not valid JSON; ignoring it:`, error);
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
/** Atomic write: a crash mid-save leaves the previous file intact. */
|
||||
async save(entries: ExternalPluginEntry[]): Promise<void> {
|
||||
await fs.mkdir(dirname(this.path), { recursive: true });
|
||||
const tmp = `${this.path}.tmp`;
|
||||
await fs.writeFile(tmp, JSON.stringify({ plugins: entries }, null, 2), 'utf-8');
|
||||
await fs.rename(tmp, this.path);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { createServer, type Server } from 'http';
|
||||
import { mkdtempSync, readFileSync } from 'fs';
|
||||
import { tmpdir } from 'os';
|
||||
import { join } from 'path';
|
||||
import { ContainerManager } from './ContainerManager.js';
|
||||
import { ExternalPluginStore } from './ExternalPluginStore.js';
|
||||
|
||||
/**
|
||||
* External plugins are the meta-sort side of the beacon Scan card's Add: a
|
||||
* plugin meta-sort did not spawn, reached by URL, identified by its manifest
|
||||
* id, persisted outside the read-only plugins.yml.
|
||||
*/
|
||||
describe('external plugins', () => {
|
||||
let server: Server;
|
||||
let url: string;
|
||||
|
||||
before(async () => {
|
||||
server = createServer((req, res) => {
|
||||
res.setHeader('Content-Type', 'application/json');
|
||||
if (req.url === '/manifest') {
|
||||
res.end(JSON.stringify({ id: 'language', name: 'Language', version: '1.0.2', defaultQueue: 'fast' }));
|
||||
} else if (req.url === '/health') {
|
||||
res.end(JSON.stringify({ status: 'healthy', ready: true, version: '1.0.2' }));
|
||||
} else {
|
||||
res.statusCode = 404;
|
||||
res.end('{}');
|
||||
}
|
||||
});
|
||||
await new Promise<void>((r) => server.listen(0, '127.0.0.1', () => r()));
|
||||
const addr = server.address();
|
||||
url = `http://127.0.0.1:${typeof addr === 'object' && addr ? addr.port : 0}`;
|
||||
});
|
||||
|
||||
after(() => server.close());
|
||||
|
||||
function manager(path: string): ContainerManager {
|
||||
const m = new ContainerManager('/nonexistent/plugins.yml', 'none', '/files', 'http://cb', 'http://core');
|
||||
m.setExternalStore(new ExternalPluginStore(path));
|
||||
return m;
|
||||
}
|
||||
|
||||
it('adds by URL under the manifest id, persists, and refuses duplicates', async () => {
|
||||
const path = join(mkdtempSync(join(tmpdir(), 'ext-')), 'state', 'external-plugins.json');
|
||||
const m = manager(path);
|
||||
const entry = await m.addExternalPlugin(`${url}/`, 'metapluginlanguage-plugin');
|
||||
assert.equal(entry.pluginId, 'language');
|
||||
assert.equal(entry.url, url, 'trailing slash normalised');
|
||||
|
||||
const saved = JSON.parse(readFileSync(path, 'utf-8'));
|
||||
assert.equal(saved.plugins[0].pluginId, 'language');
|
||||
|
||||
await assert.rejects(m.addExternalPlugin(url), /already registered/);
|
||||
const status = m.getStatus().plugins.find((p) => p.pluginId === 'language');
|
||||
assert.equal(status?.kind, 'external');
|
||||
assert.equal(status?.url, url);
|
||||
|
||||
await m.removeExternalPlugin('language');
|
||||
assert.equal(m.getStatus().plugins.length, 0);
|
||||
assert.deepEqual(JSON.parse(readFileSync(path, 'utf-8')).plugins, []);
|
||||
await assert.rejects(m.removeExternalPlugin('language'), /not an external plugin/);
|
||||
});
|
||||
|
||||
it('rejects a URL whose manifest cannot be read', async () => {
|
||||
const m = manager(join(mkdtempSync(join(tmpdir(), 'ext-')), 'x.json'));
|
||||
await assert.rejects(m.addExternalPlugin('http://127.0.0.1:9/'), /could not read/);
|
||||
await assert.rejects(m.addExternalPlugin('ftp://x'), /http/);
|
||||
});
|
||||
|
||||
it('a missing or corrupt store reads as empty', async () => {
|
||||
const dir = mkdtempSync(join(tmpdir(), 'ext-'));
|
||||
assert.deepEqual(await new ExternalPluginStore(join(dir, 'missing.json')).load(), []);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* FileSlotGate — bounds the number of files concurrently in a processing phase.
|
||||
*
|
||||
* Motivation: every plugin that runs against a file pulls bytes from CORN/SMB
|
||||
* via WebDAV. Without a per-file gate, the task queue happily fans 8 plugins
|
||||
* across hundreds of files at once, multiplying the WebDAV fetch cost by the
|
||||
* plugin count and overwhelming whatever local cache exists. The gate caps the
|
||||
* number of files that can be "in flight" in a given phase so the cache only
|
||||
* has to hold N files at a time.
|
||||
*
|
||||
* Semantics:
|
||||
* - Keyed by fileHash. A file holds at most one slot in a given phase.
|
||||
* - acquire(fileHash) resolves immediately if the file already holds a slot
|
||||
* or there is free capacity; otherwise it queues FIFO.
|
||||
* - release(fileHash) is idempotent for non-holders. When called on a real
|
||||
* holder it frees the slot and wakes the next waiter.
|
||||
*
|
||||
* The gate intentionally knows nothing about tasks or plugins — it is a thin
|
||||
* file-level semaphore. Counting tasks per (file, phase) and deciding when to
|
||||
* release is the caller's responsibility.
|
||||
*/
|
||||
export class FileSlotGate {
|
||||
private readonly holders: Set<string> = new Set();
|
||||
private readonly waiters: Array<{ fileHash: string; resolve: () => void }> = [];
|
||||
|
||||
constructor(public readonly capacity: number) {
|
||||
if (!Number.isFinite(capacity) || capacity < 1) {
|
||||
throw new Error(`FileSlotGate capacity must be a positive integer, got ${capacity}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire a slot for fileHash. If the file already holds a slot, resolves
|
||||
* immediately (re-entrant for the same file). Otherwise queues until a
|
||||
* slot is released.
|
||||
*/
|
||||
acquire(fileHash: string): Promise<void> {
|
||||
if (this.holders.has(fileHash)) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (this.holders.size < this.capacity) {
|
||||
this.holders.add(fileHash);
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.waiters.push({ fileHash, resolve });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the slot held by fileHash. No-op if the file doesn't hold one.
|
||||
* Wakes the next FIFO waiter, if any, and grants it a slot.
|
||||
*/
|
||||
release(fileHash: string): void {
|
||||
if (!this.holders.delete(fileHash)) {
|
||||
return;
|
||||
}
|
||||
const next = this.waiters.shift();
|
||||
if (!next) {
|
||||
return;
|
||||
}
|
||||
this.holders.add(next.fileHash);
|
||||
next.resolve();
|
||||
}
|
||||
|
||||
holds(fileHash: string): boolean {
|
||||
return this.holders.has(fileHash);
|
||||
}
|
||||
|
||||
inUse(): number {
|
||||
return this.holders.size;
|
||||
}
|
||||
|
||||
waiting(): number {
|
||||
return this.waiters.length;
|
||||
}
|
||||
|
||||
getHolders(): string[] {
|
||||
return Array.from(this.holders);
|
||||
}
|
||||
}
|
||||
@@ -164,7 +164,12 @@ export type ContainerInstanceStatus = 'starting' | 'healthy' | 'unhealthy' | 'st
|
||||
export interface ContainerPluginInstance {
|
||||
/** Plugin ID */
|
||||
pluginId: string;
|
||||
/** Docker container ID */
|
||||
/**
|
||||
* `container` = spawned by meta-sort from plugins.yml; `external` = added by
|
||||
* URL (beacon Scan card), not ours to start or stop. Absent = container.
|
||||
*/
|
||||
kind?: 'container' | 'external';
|
||||
/** Docker container ID ('' for an external plugin) */
|
||||
containerId: string;
|
||||
/** Container name (e.g., 'meta-plugin-ffmpeg-0') */
|
||||
containerName: string;
|
||||
@@ -211,7 +216,9 @@ export interface PluginConfigureRequest {
|
||||
* Configure response from POST /configure
|
||||
*/
|
||||
export interface PluginConfigureResponse {
|
||||
success: boolean;
|
||||
/** Older contract. The TS plugins answer `{ status: 'ok' }` instead. */
|
||||
success?: boolean;
|
||||
status?: string;
|
||||
error?: string;
|
||||
}
|
||||
|
||||
@@ -315,7 +322,11 @@ export interface ContainerManagerStatus {
|
||||
plugins: Array<{
|
||||
pluginId: string;
|
||||
enabled: boolean;
|
||||
/** Docker image, or the plugin URL for an external plugin. */
|
||||
image: string;
|
||||
kind: 'container' | 'external';
|
||||
/** Base URL of the first instance. */
|
||||
url?: string;
|
||||
instances: number;
|
||||
healthyInstances: number;
|
||||
manifest?: ContainerPluginManifest;
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { capMatches, classifyScan, suggestedName, type MeshNeighbor } from './meshdisco.js';
|
||||
|
||||
/** Mirrors meta-feeder-sdk `beacon::scan` tests: same rules, same JSON. */
|
||||
describe('beacon v2 scan classification', () => {
|
||||
const node = (instance: string, cap: string, http?: string, binds?: string): MeshNeighbor => ({
|
||||
name: 'plugin', instance, caps: [cap], addr: '10.0.0.1', lastSeen: 0,
|
||||
resources: [{ id: 'x', caps: [cap], ...(http ? { endpoints: { http } } : {}), ...(binds ? { binds } : {}) }],
|
||||
});
|
||||
|
||||
it('classifies configured, addable and bound-elsewhere', () => {
|
||||
const r = classifyScan('metamesh.enrich/*', 'metasort-app', [
|
||||
node('meta-plugin-language-0', 'metamesh.enrich/language', 'http://meta-plugin-language-0:8080/', 'metasort-app'),
|
||||
node('metapluginffmpeg-plugin', 'metamesh.enrich/ffmpeg', 'http://metapluginffmpeg-plugin:8080'),
|
||||
node('other-sort-plugin', 'metamesh.enrich/tmdb', 'http://x:8080', 'other-metasort'),
|
||||
node('no-endpoint', 'metamesh.enrich/tmdb'),
|
||||
node('a-service', 'metamesh.service/meta-core', 'http://core'),
|
||||
], [['language', 'http://meta-plugin-language-0:8080']]);
|
||||
assert.deepEqual(r.summary, { nodes: 5, capable: 3, configured: 1, boundElsewhere: 1, addable: 1 });
|
||||
assert.equal(r.candidates[0].instance, 'metapluginffmpeg-plugin');
|
||||
assert.equal(r.candidates[1].configuredAs, 'language');
|
||||
assert.equal(r.candidates[2].state, 'bound-elsewhere');
|
||||
});
|
||||
|
||||
it('caps and names', () => {
|
||||
assert.ok(capMatches('metamesh.enrich/*', 'metamesh.enrich/language@1'));
|
||||
assert.ok(!capMatches('metamesh.enrich/*', 'metamesh.enrichx/a'));
|
||||
assert.equal(suggestedName('MetaPlugin Language'), 'metaplugin-language');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,666 @@
|
||||
/**
|
||||
* Beacon v2 — local resource advertise / discover over UDP multicast
|
||||
* (239.255.99.1:9099).
|
||||
*
|
||||
* Every node advertises resources tagged with capability strings
|
||||
* (`metamesh.core`, `metamesh.service/meta-sort`, `metamesh.transport/nzb@1`)
|
||||
* and keeps a live view of everyone else's. This service uses it to locate
|
||||
* meta-core (the `metamesh.core` resource carries its /urls block) and to feed
|
||||
* the nav menu (`metamesh.service/*`), with no shared volume.
|
||||
*
|
||||
* The normative spec is docs/project-architecture/beacon-v2.md.
|
||||
*
|
||||
* ⚠ MIRRORED FILE. Byte-identical copies live in meta-sort-core, meta-fuse-core
|
||||
* and meta-dup-core. They cannot share a package: meta-fuse-core and
|
||||
* meta-dup-core do not depend on @metazla/meta-interface at all, and each
|
||||
* submodule is its own Docker build context, so a workspace dep resolves
|
||||
* outside the build. Edit one copy → edit all three, and
|
||||
* `scripts/check-mirrors.sh` will tell you if you forgot.
|
||||
*/
|
||||
|
||||
import dgram from 'dgram';
|
||||
import { hostname, networkInterfaces } from 'os';
|
||||
|
||||
export const PROTO = 'beacon';
|
||||
export const VERSION = 2;
|
||||
export const DEFAULT_GROUP = '239.255.99.1';
|
||||
export const DEFAULT_PORT = 9099;
|
||||
export const DEFAULT_INTERVAL_MS = 10_000;
|
||||
/** A node unheard for interval × this is dropped (evaluated at read time). */
|
||||
export const LIVENESS_FACTOR = 3;
|
||||
/** Senders stay under this so nothing fragments. */
|
||||
export const MAX_DATAGRAM = 1400;
|
||||
|
||||
export const TYPE_PROBE = 'probe';
|
||||
export const TYPE_ADVERTISE = 'advertise';
|
||||
export const TYPE_BYE = 'bye';
|
||||
|
||||
export const CAP_CORE = 'metamesh.core';
|
||||
export const CAP_SERVICE_PREFIX = 'metamesh.service/';
|
||||
export const CAP_ANY_SERVICE = 'metamesh.service/*';
|
||||
|
||||
/** Mirrors meta-core's GET /urls response minus redisUrl (retired). */
|
||||
export interface MeshUrls {
|
||||
hostname: string;
|
||||
baseUrl: string;
|
||||
apiUrl: string;
|
||||
webdavUrl: string;
|
||||
webdavUrlInternal: string;
|
||||
}
|
||||
|
||||
export interface BeaconNodeInfo {
|
||||
name: string;
|
||||
instance: string;
|
||||
version?: string;
|
||||
/** 'starting' | 'running'; absent reads as running. */
|
||||
status?: string;
|
||||
}
|
||||
|
||||
export interface BeaconResource {
|
||||
id: string;
|
||||
caps: string[];
|
||||
/** Well-known: http, ui, manifest, mcp. A '/…' value is relative to http. */
|
||||
endpoints?: Record<string, string>;
|
||||
rev?: string;
|
||||
/** The one consumer instance this resource belongs to. */
|
||||
binds?: string;
|
||||
data?: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export interface BeaconMessage {
|
||||
proto: string;
|
||||
v: number;
|
||||
type: string;
|
||||
node?: BeaconNodeInfo;
|
||||
resources?: BeaconResource[];
|
||||
from?: string;
|
||||
want?: string[];
|
||||
}
|
||||
|
||||
/** A node as served by /api/neighbors. */
|
||||
export interface MeshNeighbor extends BeaconNodeInfo {
|
||||
resources: BeaconResource[];
|
||||
/** Every resource's caps, flattened. */
|
||||
caps: string[];
|
||||
/** The metamesh.service/* resource's endpoints.ui (v1-compatible field). */
|
||||
baseUrl?: string;
|
||||
/** Source address taken from the packet, never from the payload. */
|
||||
addr: string;
|
||||
/** Unix seconds. */
|
||||
lastSeen: number;
|
||||
}
|
||||
|
||||
export interface MeshNodeConfig {
|
||||
name: string;
|
||||
instance?: string;
|
||||
version?: string;
|
||||
group?: string;
|
||||
port?: number;
|
||||
intervalMs?: number;
|
||||
enabled?: boolean;
|
||||
/** Rebuilt on every advertise so the payload never goes stale. */
|
||||
payload?: () => { status?: string; resources: BeaconResource[] };
|
||||
}
|
||||
|
||||
/** How a discovered resource relates to this consumer (Scan card). */
|
||||
export type ScanState = 'configured' | 'addable' | 'bound-elsewhere';
|
||||
|
||||
export interface ScanCandidate {
|
||||
instance: string;
|
||||
name: string;
|
||||
version?: string;
|
||||
resourceId: string;
|
||||
caps: string[];
|
||||
/** endpoints.http — what the consumer would register. */
|
||||
url: string;
|
||||
binds?: string;
|
||||
state: ScanState;
|
||||
/** The list entry that already holds this URL (state `configured`). */
|
||||
configuredAs?: string;
|
||||
/** A list-safe name to register it under. */
|
||||
suggestedName: string;
|
||||
}
|
||||
|
||||
/** Same JSON as meta-feeder-sdk `beacon::ScanReport`. */
|
||||
export interface ScanReport {
|
||||
cap: string;
|
||||
self: string;
|
||||
durationMs: number;
|
||||
summary: { nodes: number; capable: number; configured: number; boundElsewhere: number; addable: number };
|
||||
candidates: ScanCandidate[];
|
||||
}
|
||||
|
||||
/** Lower-case, `[a-z0-9._-]` only — accepted by every consumer's name rules. */
|
||||
export function suggestedName(instance: string): string {
|
||||
const s = instance.trim().toLowerCase().replace(/[^a-z0-9._-]/g, '-');
|
||||
return s || 'plugin';
|
||||
}
|
||||
|
||||
const normUrl = (u: string) => u.trim().replace(/\/+$/, '');
|
||||
|
||||
/**
|
||||
* Classify every resource matching `pattern` against the consumer's
|
||||
* `(name, url)` list. Mirrors meta-feeder-sdk `beacon::classify`.
|
||||
*/
|
||||
export function classifyScan(
|
||||
pattern: string,
|
||||
self: string,
|
||||
neighbors: MeshNeighbor[],
|
||||
configured: Array<[string, string]>,
|
||||
): ScanReport {
|
||||
const summary = { nodes: neighbors.length, capable: 0, configured: 0, boundElsewhere: 0, addable: 0 };
|
||||
const candidates: ScanCandidate[] = [];
|
||||
for (const n of neighbors) {
|
||||
for (const r of n.resources) {
|
||||
if (!resourceMatches(r, pattern)) continue;
|
||||
const http = resourceEndpoint(r, 'http');
|
||||
if (!http) continue; // nothing a consumer could register
|
||||
const url = normUrl(http);
|
||||
summary.capable++;
|
||||
const hit = configured.find(([, u]) => normUrl(u) === url);
|
||||
let state: ScanState;
|
||||
if (hit) { state = 'configured'; summary.configured++; }
|
||||
else if (r.binds && r.binds !== self) { state = 'bound-elsewhere'; summary.boundElsewhere++; }
|
||||
else { state = 'addable'; summary.addable++; }
|
||||
candidates.push({
|
||||
instance: n.instance,
|
||||
name: n.name,
|
||||
...(n.version ? { version: n.version } : {}),
|
||||
resourceId: r.id,
|
||||
caps: r.caps ?? [],
|
||||
url,
|
||||
...(r.binds ? { binds: r.binds } : {}),
|
||||
state,
|
||||
...(hit ? { configuredAs: hit[0] } : {}),
|
||||
suggestedName: suggestedName(n.instance),
|
||||
});
|
||||
}
|
||||
}
|
||||
const rank: Record<ScanState, number> = { addable: 0, configured: 1, 'bound-elsewhere': 2 };
|
||||
candidates.sort((a, b) => rank[a.state] - rank[b.state] || a.instance.localeCompare(b.instance));
|
||||
return { cap: pattern, self, durationMs: 0, summary, candidates };
|
||||
}
|
||||
|
||||
/**
|
||||
* Does capability `cap` satisfy `pattern`? `*` matches everything; an `@N` on
|
||||
* the pattern must equal the cap's contract (none ignores it); `x/*` matches
|
||||
* any variant of `x`; otherwise the bases must be equal.
|
||||
*/
|
||||
export function capMatches(pattern: string, cap: string): boolean {
|
||||
if (pattern === '*') return true;
|
||||
const split = (s: string): [string, string | null] => {
|
||||
const i = s.lastIndexOf('@');
|
||||
return i >= 0 ? [s.slice(0, i), s.slice(i + 1)] : [s, null];
|
||||
};
|
||||
const [pbase, pc] = split(pattern);
|
||||
const [cbase, cc] = split(cap);
|
||||
if (pc !== null && cc !== pc) return false;
|
||||
if (pbase.endsWith('/*')) {
|
||||
const prefix = pbase.slice(0, -2);
|
||||
if (!cbase.startsWith(prefix)) return false;
|
||||
const rest = cbase.slice(prefix.length);
|
||||
return rest.startsWith('/') && rest.length > 1;
|
||||
}
|
||||
return pbase === cbase;
|
||||
}
|
||||
|
||||
export function resourceMatches(r: BeaconResource, pattern: string): boolean {
|
||||
return (r.caps ?? []).some((c) => capMatches(pattern, c));
|
||||
}
|
||||
|
||||
/** The named endpoint as an absolute URL ('/…' joins onto endpoints.http). */
|
||||
export function resourceEndpoint(r: BeaconResource, name: string): string | undefined {
|
||||
const v = r.endpoints?.[name];
|
||||
if (!v) return undefined;
|
||||
if (!v.startsWith('/')) return v;
|
||||
const base = r.endpoints?.http;
|
||||
return base ? base.replace(/\/+$/, '') + v : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a datagram; null for anything that is not a well-formed beacon v2
|
||||
* message (beacon v1, meta-discovery v1, other versions, junk).
|
||||
*/
|
||||
export function parseMessage(buf: Buffer): BeaconMessage | null {
|
||||
let m: BeaconMessage;
|
||||
try {
|
||||
m = JSON.parse(buf.toString('utf8')) as BeaconMessage;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!m || m.proto !== PROTO || m.v !== VERSION) return null;
|
||||
if (m.type === TYPE_PROBE) return m;
|
||||
if (m.type === TYPE_ADVERTISE || m.type === TYPE_BYE) {
|
||||
if (!m.node?.name || !m.node?.instance) return null;
|
||||
return m;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** IPv4 addresses of every up, non-internal interface. */
|
||||
function localInterfaceAddresses(): string[] {
|
||||
const out: string[] = [];
|
||||
const nets = networkInterfaces();
|
||||
for (const name of Object.keys(nets)) {
|
||||
for (const ni of nets[name] ?? []) {
|
||||
// Node has flip-flopped between 'IPv4' and 4 for `family`.
|
||||
const isV4 = (ni as { family: string | number }).family === 'IPv4' ||
|
||||
(ni as { family: string | number }).family === 4;
|
||||
if (isV4 && !ni.internal && ni.address) out.push(ni.address);
|
||||
}
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/** First non-internal IPv4 address, or the hostname. Matches the Go helper. */
|
||||
export function localIPv4(): string {
|
||||
return localInterfaceAddresses()[0] ?? hostname();
|
||||
}
|
||||
|
||||
function envNum(key: string): number | undefined {
|
||||
const n = Number(process.env[key]);
|
||||
return Number.isFinite(n) && n > 0 ? n : undefined;
|
||||
}
|
||||
|
||||
interface Seen {
|
||||
node: BeaconNodeInfo;
|
||||
resources: BeaconResource[];
|
||||
addr: string;
|
||||
lastSeen: number; // ms
|
||||
}
|
||||
|
||||
function toNeighbor(s: Seen): MeshNeighbor {
|
||||
const caps: string[] = [];
|
||||
let baseUrl: string | undefined;
|
||||
for (const r of s.resources) {
|
||||
for (const c of r.caps ?? []) if (!caps.includes(c)) caps.push(c);
|
||||
if (!baseUrl && resourceMatches(r, CAP_ANY_SERVICE)) baseUrl = resourceEndpoint(r, 'ui');
|
||||
}
|
||||
return {
|
||||
...s.node,
|
||||
resources: s.resources,
|
||||
caps,
|
||||
...(baseUrl ? { baseUrl } : {}),
|
||||
addr: s.addr,
|
||||
lastSeen: s.lastSeen / 1000,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One beacon v2 participant: advertises its resources, answers probes, and
|
||||
* keeps a TTL map of every other node.
|
||||
*/
|
||||
export class MeshNode {
|
||||
private cfg: Required<Omit<MeshNodeConfig, 'payload'>> & Pick<MeshNodeConfig, 'payload'>;
|
||||
private socket: dgram.Socket | null = null;
|
||||
private timer: NodeJS.Timeout | null = null;
|
||||
private nodes = new Map<string, Seen>();
|
||||
private ifaceAddrs: string[] = [];
|
||||
private listeners: ((n: MeshNeighbor) => void)[] = [];
|
||||
private started = false;
|
||||
|
||||
constructor(config: MeshNodeConfig) {
|
||||
this.cfg = {
|
||||
name: config.name,
|
||||
instance: config.instance ?? hostname(),
|
||||
version: config.version ?? '',
|
||||
group: config.group ?? process.env.BEACON_GROUP ?? DEFAULT_GROUP,
|
||||
port: config.port ?? envNum('BEACON_PORT') ?? DEFAULT_PORT,
|
||||
intervalMs: config.intervalMs ?? envNum('BEACON_INTERVAL_MS') ?? DEFAULT_INTERVAL_MS,
|
||||
enabled: config.enabled ?? true,
|
||||
payload: config.payload,
|
||||
};
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.started || !this.cfg.enabled) {
|
||||
if (!this.cfg.enabled) console.log('[beacon] Disabled by configuration');
|
||||
return;
|
||||
}
|
||||
this.started = true;
|
||||
|
||||
await new Promise<void>((resolve) => {
|
||||
// reuseAddr sets SO_REUSEADDR (+ SO_REUSEPORT on Linux for udp4),
|
||||
// so several listeners can share 9099 on one host.
|
||||
const socket = dgram.createSocket({ type: 'udp4', reuseAddr: true });
|
||||
this.socket = socket;
|
||||
|
||||
socket.on('message', (buf, rinfo) => this.onMessage(buf, rinfo));
|
||||
socket.on('error', (err) => {
|
||||
// Never fatal: a host that blocks multicast must degrade, not
|
||||
// take the service down.
|
||||
console.error('[beacon] Socket error:', err.message);
|
||||
});
|
||||
|
||||
socket.bind(this.cfg.port, '0.0.0.0', () => {
|
||||
try {
|
||||
socket.setMulticastTTL(1); // link-local only
|
||||
} catch (e) {
|
||||
console.warn('[beacon] Could not set multicast TTL:', e);
|
||||
}
|
||||
|
||||
// Join on EVERY interface. A container on two docker networks
|
||||
// that joins only the default route silently sees one network.
|
||||
this.ifaceAddrs = localInterfaceAddresses();
|
||||
let joined = 0;
|
||||
for (const addr of this.ifaceAddrs) {
|
||||
try {
|
||||
socket.addMembership(this.cfg.group, addr);
|
||||
joined++;
|
||||
} catch (e) {
|
||||
console.warn(`[beacon] Join ${this.cfg.group} on ${addr} failed:`, e);
|
||||
}
|
||||
}
|
||||
console.log(
|
||||
`[beacon] Listening on ${this.cfg.group}:${this.cfg.port} as ` +
|
||||
`${this.cfg.name}/${this.cfg.instance} (joined ${joined} of ${this.ifaceAddrs.length} interfaces)`
|
||||
);
|
||||
resolve();
|
||||
});
|
||||
});
|
||||
|
||||
this.advertise();
|
||||
this.probe();
|
||||
this.timer = setInterval(() => this.advertise(), this.cfg.intervalMs);
|
||||
// Don't hold the event loop open on shutdown.
|
||||
this.timer.unref?.();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
if (!this.started) return;
|
||||
this.started = false;
|
||||
if (this.timer) { clearInterval(this.timer); this.timer = null; }
|
||||
// Bye, so neighbours drop us immediately rather than waiting out the
|
||||
// staleness window.
|
||||
this.multicast({ proto: PROTO, v: VERSION, type: TYPE_BYE, node: this.nodeInfo() });
|
||||
await new Promise<void>((resolve) => {
|
||||
if (!this.socket) return resolve();
|
||||
try { this.socket.close(() => resolve()); } catch { resolve(); }
|
||||
});
|
||||
this.socket = null;
|
||||
this.listeners = [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Multicast a probe. Every node owning a resource matching one of `want`
|
||||
* (every node, when empty) replies immediately.
|
||||
*/
|
||||
probe(want: string[] = []): void {
|
||||
const msg: BeaconMessage = { proto: PROTO, v: VERSION, type: TYPE_PROBE, from: this.cfg.instance };
|
||||
if (want.length) msg.want = want;
|
||||
this.multicast(msg);
|
||||
}
|
||||
|
||||
/** Every live node, sorted by (name, instance). */
|
||||
getNeighbors(): MeshNeighbor[] {
|
||||
const cutoff = Date.now() - this.cfg.intervalMs * LIVENESS_FACTOR;
|
||||
const out: MeshNeighbor[] = [];
|
||||
for (const [key, s] of this.nodes) {
|
||||
if (s.lastSeen < cutoff) { this.nodes.delete(key); continue; }
|
||||
out.push(toNeighbor(s));
|
||||
}
|
||||
return out.sort((a, b) => a.name.localeCompare(b.name) || a.instance.localeCompare(b.instance));
|
||||
}
|
||||
|
||||
/** One row per node name, most recently seen instance wins. */
|
||||
getNeighborsByName(): MeshNeighbor[] {
|
||||
const best = new Map<string, MeshNeighbor>();
|
||||
for (const nb of this.getNeighbors()) {
|
||||
const cur = best.get(nb.name);
|
||||
if (!cur || nb.lastSeen > cur.lastSeen) best.set(nb.name, nb);
|
||||
}
|
||||
return [...best.values()].sort((a, b) => a.name.localeCompare(b.name));
|
||||
}
|
||||
|
||||
/**
|
||||
* The Scan card: probe everyone, wait `waitMs` for the replies, then
|
||||
* classify what matches `pattern` against `configured` ([name, url]).
|
||||
*/
|
||||
async scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise<ScanReport> {
|
||||
const started = Date.now();
|
||||
this.probe();
|
||||
await new Promise((r) => setTimeout(r, waitMs));
|
||||
const report = classifyScan(pattern, this.cfg.instance, this.getNeighbors(), configured);
|
||||
report.durationMs = Date.now() - started;
|
||||
return report;
|
||||
}
|
||||
|
||||
/** This node as a row, so a UI can render itself without waiting. */
|
||||
self(): MeshNeighbor {
|
||||
const { status, resources } = this.payload();
|
||||
return toNeighbor({ node: this.nodeInfo(status), resources, addr: '', lastSeen: Date.now() });
|
||||
}
|
||||
|
||||
/** Fires for every advertise received (after the self-echo filter). */
|
||||
onAdvertise(cb: (n: MeshNeighbor) => void): void {
|
||||
this.listeners.push(cb);
|
||||
}
|
||||
|
||||
private payload(): { status?: string; resources: BeaconResource[] } {
|
||||
return this.cfg.payload ? this.cfg.payload() : { resources: [] };
|
||||
}
|
||||
|
||||
private nodeInfo(status?: string): BeaconNodeInfo {
|
||||
const n: BeaconNodeInfo = { name: this.cfg.name, instance: this.cfg.instance };
|
||||
if (this.cfg.version) n.version = this.cfg.version;
|
||||
if (status) n.status = status;
|
||||
return n;
|
||||
}
|
||||
|
||||
private advertiseMsg(): BeaconMessage {
|
||||
const { status, resources } = this.payload();
|
||||
return { proto: PROTO, v: VERSION, type: TYPE_ADVERTISE, node: this.nodeInfo(status), resources };
|
||||
}
|
||||
|
||||
private onMessage(buf: Buffer, rinfo: dgram.RemoteInfo): void {
|
||||
const msg = parseMessage(buf);
|
||||
if (!msg) return; // v1 or foreign traffic on the shared group
|
||||
|
||||
if (msg.type === TYPE_PROBE) {
|
||||
if (msg.from === this.cfg.instance) return;
|
||||
const ad = this.advertiseMsg();
|
||||
const want = msg.want ?? [];
|
||||
const wanted = want.length === 0 ||
|
||||
(ad.resources ?? []).some((r) => want.some((p) => resourceMatches(r, p)));
|
||||
if (wanted) this.replyTo(ad, rinfo);
|
||||
return;
|
||||
}
|
||||
|
||||
const node = msg.node!;
|
||||
if (node.instance === this.cfg.instance) return; // our own echo
|
||||
if (msg.type === TYPE_BYE) { this.nodes.delete(node.instance); return; }
|
||||
|
||||
const seen: Seen = { node, resources: msg.resources ?? [], addr: rinfo.address, lastSeen: Date.now() };
|
||||
this.nodes.set(node.instance, seen);
|
||||
const nb = toNeighbor(seen);
|
||||
for (const cb of this.listeners) {
|
||||
try { cb(nb); } catch (e) { console.error('[beacon] Listener threw:', e); }
|
||||
}
|
||||
}
|
||||
|
||||
private encode(msg: BeaconMessage): Buffer {
|
||||
const body = Buffer.from(JSON.stringify(msg));
|
||||
if (body.length > MAX_DATAGRAM) {
|
||||
console.warn(`[beacon] ${body.length}-byte datagram exceeds the fragmentation-safe ${MAX_DATAGRAM}`);
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
private advertise(): void {
|
||||
this.multicast(this.advertiseMsg());
|
||||
}
|
||||
|
||||
private replyTo(msg: BeaconMessage, rinfo: dgram.RemoteInfo): void {
|
||||
try {
|
||||
this.socket?.send(this.encode(msg), rinfo.port, rinfo.address);
|
||||
} catch (e) {
|
||||
console.warn('[beacon] Reply failed:', e);
|
||||
}
|
||||
}
|
||||
|
||||
/** Writes once per interface — the default route alone reaches one network. */
|
||||
private multicast(msg: BeaconMessage): void {
|
||||
const socket = this.socket;
|
||||
if (!socket) return;
|
||||
const body = this.encode(msg);
|
||||
for (const addr of this.ifaceAddrs) {
|
||||
try {
|
||||
socket.setMulticastInterface(addr);
|
||||
socket.send(body, this.cfg.port, this.cfg.group);
|
||||
} catch (e) {
|
||||
console.warn(`[beacon] Send on ${addr} failed:`, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export interface MetaCoreLocatorConfig {
|
||||
/** This service's name, for its own advertise. */
|
||||
serviceName: string;
|
||||
/** Browser-facing URL for the nav menu. */
|
||||
baseUrl?: string;
|
||||
version?: string;
|
||||
/**
|
||||
* Explicit meta-core API URL. When set this ALWAYS wins and the wire is
|
||||
* never consulted for core selection — see the spec's "the
|
||||
* metamesh.core pin". This is what stops a client box from latching onto a
|
||||
* gateway box's core when both are reachable on a shared network.
|
||||
*/
|
||||
metaCoreUrl?: string;
|
||||
group?: string;
|
||||
port?: number;
|
||||
intervalMs?: number;
|
||||
enabled?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Locates meta-core over beacon v2 (the first `metamesh.core` resource that
|
||||
* carries `data.urls`), and advertises this service as
|
||||
* `metamesh.service/<serviceName>`.
|
||||
*/
|
||||
export class MetaCoreLocator {
|
||||
private node: MeshNode;
|
||||
private cfg: MetaCoreLocatorConfig;
|
||||
private current: MeshUrls | null = null;
|
||||
private changeCallbacks: (() => void)[] = [];
|
||||
|
||||
constructor(config: MetaCoreLocatorConfig) {
|
||||
this.cfg = config;
|
||||
this.node = new MeshNode({
|
||||
name: config.serviceName,
|
||||
version: config.version,
|
||||
group: config.group,
|
||||
port: config.port,
|
||||
intervalMs: config.intervalMs,
|
||||
enabled: config.enabled,
|
||||
payload: () => ({
|
||||
resources: [{
|
||||
id: 'service',
|
||||
caps: [CAP_SERVICE_PREFIX + config.serviceName],
|
||||
endpoints: { ui: config.baseUrl ?? `http://${localIPv4()}` },
|
||||
}],
|
||||
}),
|
||||
});
|
||||
|
||||
this.node.onAdvertise((nb) => {
|
||||
if (this.cfg.metaCoreUrl) return; // pinned: the wire cannot move us
|
||||
const core = nb.resources.find((r) => resourceMatches(r, CAP_CORE));
|
||||
const urls = core?.data?.urls as MeshUrls | undefined;
|
||||
if (!urls?.apiUrl) return;
|
||||
|
||||
const prev = this.current;
|
||||
if (!prev) {
|
||||
this.current = urls;
|
||||
console.log(`[beacon] meta-core discovered at ${urls.apiUrl} (${nb.addr})`);
|
||||
this.notifyChange();
|
||||
return;
|
||||
}
|
||||
if (prev.apiUrl === urls.apiUrl) {
|
||||
this.current = urls; // refresh the rest of the fields
|
||||
return;
|
||||
}
|
||||
// A second, different core is advertising. Do not flap between
|
||||
// them — keep the first and say so loudly, because on a PCS box
|
||||
// this means a client container can see the gateway's core.
|
||||
console.warn(
|
||||
`[beacon] Ignoring a second meta-core at ${urls.apiUrl} (${nb.addr}); ` +
|
||||
`staying with ${prev.apiUrl}. Set META_CORE_URL to pin this explicitly.`
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
async start(): Promise<void> {
|
||||
if (this.cfg.metaCoreUrl) {
|
||||
console.log(`[beacon] meta-core pinned to ${this.cfg.metaCoreUrl}; discovery is advisory`);
|
||||
}
|
||||
await this.node.start();
|
||||
}
|
||||
|
||||
async stop(): Promise<void> {
|
||||
await this.node.stop();
|
||||
this.changeCallbacks = [];
|
||||
}
|
||||
|
||||
/** The pin if set, else whatever discovery elected. */
|
||||
getApiUrl(): string | null {
|
||||
if (this.cfg.metaCoreUrl) return this.cfg.metaCoreUrl;
|
||||
return this.current?.apiUrl ?? null;
|
||||
}
|
||||
|
||||
/** Full URLs block. null when pinned and /urls has not been fetched yet. */
|
||||
getUrls(): MeshUrls | null {
|
||||
return this.current;
|
||||
}
|
||||
|
||||
/**
|
||||
* Block until meta-core is located. Probes immediately and re-probes every
|
||||
* 500ms — the same loop shape as the old LeaderClient.waitForLeader, so the
|
||||
* 30s boot timeout semantics are preserved exactly.
|
||||
*/
|
||||
async waitForCore(timeoutMs = 30_000): Promise<string> {
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
let logged = false;
|
||||
while (Date.now() < deadline) {
|
||||
const url = this.getApiUrl();
|
||||
if (url) return url;
|
||||
if (!logged) { console.log('[beacon] Waiting for meta-core...'); logged = true; }
|
||||
this.node.probe([CAP_CORE]);
|
||||
await new Promise((r) => setTimeout(r, 500));
|
||||
}
|
||||
throw new Error(`[beacon] No meta-core found within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
/** Fires when the elected core's apiUrl changes. Replaces the fs.watch. */
|
||||
onChange(cb: () => void): void {
|
||||
this.changeCallbacks.push(cb);
|
||||
}
|
||||
|
||||
/**
|
||||
* Neighbours for /api/neighbors: one row per name (or every instance with
|
||||
* `all`), optionally only nodes with a resource matching `cap`.
|
||||
*/
|
||||
getNeighbors(opts: { all?: boolean; cap?: string } = {}): MeshNeighbor[] {
|
||||
const list = opts.all ? this.node.getNeighbors() : this.node.getNeighborsByName();
|
||||
const cap = opts.cap;
|
||||
return cap ? list.filter((n) => n.resources.some((r) => resourceMatches(r, cap))) : list;
|
||||
}
|
||||
|
||||
self(): MeshNeighbor {
|
||||
return this.node.self();
|
||||
}
|
||||
|
||||
/** The Scan card — see {@link MeshNode.scan}. */
|
||||
scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise<ScanReport> {
|
||||
return this.node.scan(pattern, configured, waitMs);
|
||||
}
|
||||
|
||||
probe(want: string[] = []): void {
|
||||
this.node.probe(want);
|
||||
}
|
||||
|
||||
private notifyChange(): void {
|
||||
for (const cb of this.changeCallbacks) {
|
||||
try { cb(); } catch (e) { console.error('[beacon] Change callback threw:', e); }
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,20 @@
|
||||
/**
|
||||
* FileEventConsumer
|
||||
*
|
||||
* Consumes file events from meta-core via Redis Stream.
|
||||
* Consumes file events from meta-core via SSE (Server-Sent Events).
|
||||
* When events are received, they are forwarded to the StreamingPipeline.
|
||||
*
|
||||
* This enables meta-sort to run without direct filesystem access - it receives
|
||||
* This enables meta-sort to run without direct filesystem access — it receives
|
||||
* file discovery events from meta-core and processes files via WebDAV.
|
||||
*
|
||||
* Previously consumed `file:events` via Redis Streams XREADGROUP. The
|
||||
* api-mediated-access migration (PR C) routes this through meta-core's HTTP
|
||||
* /api/events/files endpoint instead, so consumers no longer need direct
|
||||
* Redis access. Cursor durability is now the consumer's responsibility —
|
||||
* see SSEEventClient.
|
||||
*/
|
||||
|
||||
import type { RedisKVClient, StreamMessage } from '../kv/RedisClient.js';
|
||||
import { SSEEventClient, type SSEEvent } from './SSEEventClient.js';
|
||||
|
||||
// Pipeline interface (minimal, matching StreamingPipeline)
|
||||
interface Pipeline {
|
||||
@@ -18,13 +24,9 @@ interface Pipeline {
|
||||
reset(): Promise<void>;
|
||||
}
|
||||
|
||||
// Redis Stream configuration
|
||||
const EVENTS_STREAM = 'file:events';
|
||||
const CONSUMER_GROUP = 'meta-sort-processor';
|
||||
|
||||
export interface FileEventConsumerOptions {
|
||||
/** Redis KV client */
|
||||
kvClient: RedisKVClient;
|
||||
/** meta-core API base URL (e.g. http://metacore-app:9000). */
|
||||
metaCoreApiUrl: string;
|
||||
|
||||
/** Pipeline to receive events */
|
||||
pipeline: Pipeline;
|
||||
@@ -32,115 +34,104 @@ export interface FileEventConsumerOptions {
|
||||
/** Base path for files (e.g., /files). Paths from meta-core are relative and need this prefix. */
|
||||
filesPath?: string;
|
||||
|
||||
/** Block timeout in milliseconds for stream reads */
|
||||
blockMs?: number;
|
||||
/** Where to persist the SSE cursor. Defaults to /meta-core/cursors/meta-sort-files.cursor. */
|
||||
cursorPath?: string;
|
||||
}
|
||||
|
||||
export class FileEventConsumer {
|
||||
private kvClient: RedisKVClient;
|
||||
private pipeline: Pipeline;
|
||||
private filesPath: string;
|
||||
private blockMs: number;
|
||||
private sse: SSEEventClient;
|
||||
private isRunning = false;
|
||||
|
||||
constructor(options: FileEventConsumerOptions) {
|
||||
this.kvClient = options.kvClient;
|
||||
this.pipeline = options.pipeline;
|
||||
this.filesPath = options.filesPath ?? '/files';
|
||||
this.blockMs = options.blockMs ?? 5000;
|
||||
|
||||
const apiUrl = options.metaCoreApiUrl.replace(/\/+$/, '');
|
||||
const cursorPath = options.cursorPath ?? '/meta-core/cursors/meta-sort-files.cursor';
|
||||
|
||||
this.sse = new SSEEventClient({
|
||||
url: `${apiUrl}/api/events/files`,
|
||||
cursorPath,
|
||||
logTag: '[FileEventConsumer]',
|
||||
onEvent: (e) => this.handleSSEEvent(e),
|
||||
// gap on file:events means we missed retention. Easiest correct
|
||||
// response: ask meta-core for a watcher rescan so the StreamingPipeline
|
||||
// can rebuild from a known-good baseline.
|
||||
onGap: () => {
|
||||
console.warn('[FileEventConsumer] Stream cursor trimmed — pipeline will rebuild from next reset event');
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start consuming events from Redis Stream
|
||||
* Start consuming events from the meta-core SSE endpoint.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
console.log('[FileEventConsumer] Already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
console.log(`[FileEventConsumer] Starting stream consumer for ${EVENTS_STREAM}...`);
|
||||
|
||||
// Initialize consumer group
|
||||
await this.kvClient.initStreamConsumer(EVENTS_STREAM, CONSUMER_GROUP);
|
||||
|
||||
// Process any pending entries from crashed consumers
|
||||
await this.kvClient.processPendingEntries(
|
||||
EVENTS_STREAM,
|
||||
CONSUMER_GROUP,
|
||||
30000, // 30 second idle threshold
|
||||
async (message) => this.handleMessage(message)
|
||||
);
|
||||
|
||||
// Start consumer loop in background (don't await - runs indefinitely)
|
||||
this.kvClient.startStreamConsumer(
|
||||
EVENTS_STREAM,
|
||||
CONSUMER_GROUP,
|
||||
async (message) => this.handleMessage(message),
|
||||
this.blockMs
|
||||
).catch(error => {
|
||||
console.error('[FileEventConsumer] Stream consumer error:', error);
|
||||
console.log('[FileEventConsumer] Starting SSE stream consumer for /api/events/files...');
|
||||
// Fire-and-forget. SSEEventClient.start() returns only when stopped.
|
||||
this.sse.start().catch(error => {
|
||||
console.error('[FileEventConsumer] Stream consumer terminated:', error);
|
||||
});
|
||||
|
||||
console.log('[FileEventConsumer] Stream consumer started');
|
||||
console.log('[FileEventConsumer] SSE stream consumer started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop consuming events
|
||||
*/
|
||||
stop(): void {
|
||||
async stop(): Promise<void> {
|
||||
if (!this.isRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = false;
|
||||
this.kvClient.stopStreamConsumer();
|
||||
await this.sse.stop();
|
||||
console.log('[FileEventConsumer] Stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a stream message
|
||||
*/
|
||||
private async handleMessage(message: StreamMessage): Promise<void> {
|
||||
private async handleSSEEvent(e: SSEEvent): Promise<void> {
|
||||
// Convert relative path to absolute path
|
||||
const toAbsolutePath = (relativePath: string): string => {
|
||||
// If path is already absolute, return as-is
|
||||
if (relativePath.startsWith('/')) {
|
||||
return relativePath;
|
||||
}
|
||||
// Otherwise, prepend filesPath
|
||||
if (!relativePath) return relativePath;
|
||||
if (relativePath.startsWith('/')) return relativePath;
|
||||
return `${this.filesPath}/${relativePath}`;
|
||||
};
|
||||
|
||||
switch (message.type) {
|
||||
const path: string = e.data?.path ?? '';
|
||||
const midhash256: string | undefined = e.data?.midhash256 ?? undefined;
|
||||
const oldPath: string | undefined = e.data?.oldPath ?? undefined;
|
||||
const watcherId: string | undefined = e.data?.watcherId ?? undefined;
|
||||
|
||||
switch (e.event) {
|
||||
case 'reset':
|
||||
console.log(`[FileEventConsumer] Received reset event from watcher: ${message.watcherId || 'all'}`);
|
||||
console.log(`[FileEventConsumer] Received reset event from watcher: ${watcherId || 'all'}`);
|
||||
await this.pipeline.reset();
|
||||
break;
|
||||
|
||||
case 'add':
|
||||
this.pipeline.handleFileAdded(toAbsolutePath(message.path), message.midhash256);
|
||||
if (path) this.pipeline.handleFileAdded(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
case 'change':
|
||||
this.pipeline.handleFileChanged(toAbsolutePath(message.path), message.midhash256);
|
||||
if (path) this.pipeline.handleFileChanged(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
case 'delete':
|
||||
this.pipeline.handleFileDeleted(toAbsolutePath(message.path));
|
||||
if (path) this.pipeline.handleFileDeleted(toAbsolutePath(path));
|
||||
break;
|
||||
|
||||
case 'rename':
|
||||
// Handle rename as delete + add
|
||||
if (message.oldPath) {
|
||||
this.pipeline.handleFileDeleted(toAbsolutePath(message.oldPath));
|
||||
}
|
||||
this.pipeline.handleFileAdded(toAbsolutePath(message.path), message.midhash256);
|
||||
if (oldPath) this.pipeline.handleFileDeleted(toAbsolutePath(oldPath));
|
||||
if (path) this.pipeline.handleFileAdded(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
default:
|
||||
console.warn(`[FileEventConsumer] Unknown event type: ${message.type}`);
|
||||
console.warn(`[FileEventConsumer] Unknown event type: ${e.event}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,12 +140,12 @@ export class FileEventConsumer {
|
||||
* Create a FileEventConsumer with default configuration
|
||||
*/
|
||||
export function createFileEventConsumer(
|
||||
kvClient: RedisKVClient,
|
||||
metaCoreApiUrl: string,
|
||||
pipeline: Pipeline,
|
||||
options?: Partial<FileEventConsumerOptions>
|
||||
): FileEventConsumer {
|
||||
return new FileEventConsumer({
|
||||
kvClient,
|
||||
metaCoreApiUrl,
|
||||
pipeline,
|
||||
...options
|
||||
});
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* SSEEventClient — HTTP Server-Sent Events client for meta-core event streams.
|
||||
*
|
||||
* Replaces the Redis Streams XREADGROUP loop. Each consumer service holds
|
||||
* one long-lived HTTP connection to meta-core's /api/events/{files,meta}
|
||||
* endpoint and receives JSON events as the underlying Redis Stream is
|
||||
* appended to.
|
||||
*
|
||||
* Cursor semantics (see meta-core docs/api-mediated-access.md):
|
||||
* - The SSE wire id is the opaque Redis Stream entry ID (`<ms>-<seq>`).
|
||||
* We treat it as a black-box string.
|
||||
* - On reconnect we send Last-Event-ID so meta-core resumes from the
|
||||
* entry AFTER our cursor.
|
||||
* - We persist the cursor to disk so the loop survives a process
|
||||
* restart. The cursor is written every flushIntervalMs and on graceful
|
||||
* stop.
|
||||
* - On a `gap` event the server is telling us our cursor was trimmed
|
||||
* out of retention. Caller-supplied onGap handles it (typically a
|
||||
* re-bootstrap from a full HTTP fetch). If the caller doesn't handle
|
||||
* gaps, we just continue from the new cursor.
|
||||
*/
|
||||
import { promises as fs } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
|
||||
/** One SSE event parsed off the wire. */
|
||||
export interface SSEEvent {
|
||||
/** Opaque cursor (Redis Stream entry ID under the hood). */
|
||||
id: string;
|
||||
/** Event type — `add` / `change` / `delete` / `rename` / `reset` for
|
||||
* file:events; `set` / `del` / `expire` for meta:events; `gap` for
|
||||
* trim notifications; arbitrary otherwise. */
|
||||
event: string;
|
||||
/** Parsed JSON payload from the `data:` line. */
|
||||
data: any;
|
||||
}
|
||||
|
||||
export interface SSEClientOptions {
|
||||
/** Full URL to subscribe to (e.g. http://metacore-app:9000/api/events/files). */
|
||||
url: string;
|
||||
|
||||
/** Path where the cursor is persisted between runs.
|
||||
* Set to null to keep the cursor in-memory only (at-most-once on crash). */
|
||||
cursorPath: string | null;
|
||||
|
||||
/** Handler for every non-gap event. */
|
||||
onEvent: (e: SSEEvent) => Promise<void> | void;
|
||||
|
||||
/** Optional handler for the synthetic `gap` event. If omitted we just
|
||||
* log and continue from the resumeFrom cursor inside the payload. */
|
||||
onGap?: (payload: { requested: string; resumeFrom: string; reason: string }) => Promise<void> | void;
|
||||
|
||||
/** Logger prefix used for console.log lines. */
|
||||
logTag?: string;
|
||||
|
||||
/** Initial reconnect backoff in ms (doubles per failure, capped). Default 500. */
|
||||
reconnectBaseMs?: number;
|
||||
|
||||
/** Max reconnect backoff in ms. Default 30_000. */
|
||||
reconnectMaxMs?: number;
|
||||
|
||||
/** How often to flush the in-memory cursor to disk. Default 2000. */
|
||||
flushIntervalMs?: number;
|
||||
}
|
||||
|
||||
export class SSEEventClient {
|
||||
private readonly opts: Required<Omit<SSEClientOptions, 'onGap'>> & Pick<SSEClientOptions, 'onGap'>;
|
||||
private cursor: string = '';
|
||||
private cursorDirty = false;
|
||||
private abort: AbortController | null = null;
|
||||
private flushTimer: NodeJS.Timeout | null = null;
|
||||
private stopped = false;
|
||||
private backoff = 0;
|
||||
|
||||
constructor(options: SSEClientOptions) {
|
||||
this.opts = {
|
||||
url: options.url,
|
||||
cursorPath: options.cursorPath,
|
||||
onEvent: options.onEvent,
|
||||
onGap: options.onGap,
|
||||
logTag: options.logTag ?? '[SSE]',
|
||||
reconnectBaseMs: options.reconnectBaseMs ?? 500,
|
||||
reconnectMaxMs: options.reconnectMaxMs ?? 30_000,
|
||||
flushIntervalMs: options.flushIntervalMs ?? 2000,
|
||||
};
|
||||
this.backoff = this.opts.reconnectBaseMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the persisted cursor (if any) and begin streaming. The returned
|
||||
* promise resolves on stop()/permanent error; callers should not await
|
||||
* it directly — fire-and-forget and call stop() to shut down.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
await this.loadCursor();
|
||||
this.startFlushTimer();
|
||||
await this.runLoop();
|
||||
}
|
||||
|
||||
/** Graceful shutdown — closes the open connection and flushes the cursor. */
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true;
|
||||
if (this.abort) {
|
||||
this.abort.abort();
|
||||
this.abort = null;
|
||||
}
|
||||
if (this.flushTimer) {
|
||||
clearInterval(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
await this.flushCursor();
|
||||
}
|
||||
|
||||
/** Currently-known cursor — useful for tests. */
|
||||
getCursor(): string {
|
||||
return this.cursor;
|
||||
}
|
||||
|
||||
private async loadCursor(): Promise<void> {
|
||||
if (!this.opts.cursorPath) return;
|
||||
try {
|
||||
const data = await fs.readFile(this.opts.cursorPath, 'utf-8');
|
||||
const trimmed = data.trim();
|
||||
if (trimmed) {
|
||||
this.cursor = trimmed;
|
||||
console.log(`${this.opts.logTag} Resuming from cursor ${this.cursor}`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
console.warn(`${this.opts.logTag} Could not read cursor file ${this.opts.cursorPath}: ${err.message}`);
|
||||
}
|
||||
// ENOENT is expected first time; start at "$" (server-default).
|
||||
}
|
||||
}
|
||||
|
||||
private startFlushTimer(): void {
|
||||
this.flushTimer = setInterval(() => {
|
||||
if (this.cursorDirty) {
|
||||
this.flushCursor().catch(err =>
|
||||
console.warn(`${this.opts.logTag} Cursor flush failed: ${err.message}`)
|
||||
);
|
||||
}
|
||||
}, this.opts.flushIntervalMs);
|
||||
}
|
||||
|
||||
private async flushCursor(): Promise<void> {
|
||||
if (!this.opts.cursorPath || !this.cursorDirty) return;
|
||||
this.cursorDirty = false;
|
||||
try {
|
||||
await fs.mkdir(dirname(this.opts.cursorPath), { recursive: true });
|
||||
await fs.writeFile(this.opts.cursorPath, this.cursor, 'utf-8');
|
||||
} catch (err: any) {
|
||||
// Re-mark dirty so the next tick retries.
|
||||
this.cursorDirty = true;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(): Promise<void> {
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
await this.connect();
|
||||
// connect() only returns on graceful end-of-stream — reset
|
||||
// backoff so a long-running connection that finally
|
||||
// disconnects reconnects quickly.
|
||||
this.backoff = this.opts.reconnectBaseMs;
|
||||
} catch (err: any) {
|
||||
if (this.stopped) return;
|
||||
console.warn(`${this.opts.logTag} Stream error, reconnecting in ${this.backoff}ms: ${err?.message ?? err}`);
|
||||
await this.sleep(this.backoff);
|
||||
this.backoff = Math.min(this.backoff * 2, this.opts.reconnectMaxMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async connect(): Promise<void> {
|
||||
this.abort = new AbortController();
|
||||
const headers: Record<string, string> = {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
};
|
||||
if (this.cursor) {
|
||||
headers['Last-Event-ID'] = this.cursor;
|
||||
}
|
||||
|
||||
const response = await fetch(this.opts.url, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: this.abort.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
||||
}
|
||||
if (!response.body) {
|
||||
throw new Error('Response has no body');
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (!this.stopped) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
// Each SSE event ends with a blank line ("\n\n"). Split on
|
||||
// that boundary and process each chunk.
|
||||
let idx: number;
|
||||
while ((idx = buffer.indexOf('\n\n')) !== -1) {
|
||||
const rawChunk = buffer.slice(0, idx);
|
||||
buffer = buffer.slice(idx + 2);
|
||||
await this.handleChunk(rawChunk);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { reader.releaseLock(); } catch { /* nothing useful */ }
|
||||
}
|
||||
}
|
||||
|
||||
private async handleChunk(chunk: string): Promise<void> {
|
||||
let id = '';
|
||||
let event = '';
|
||||
const dataLines: string[] = [];
|
||||
for (const rawLine of chunk.split('\n')) {
|
||||
if (!rawLine || rawLine.startsWith(':')) {
|
||||
continue; // comment / heartbeat — ignore
|
||||
}
|
||||
const colon = rawLine.indexOf(':');
|
||||
if (colon === -1) continue;
|
||||
const field = rawLine.slice(0, colon);
|
||||
// SSE spec: ": " (single space after the colon) is stripped; if no space, keep as-is.
|
||||
let value = rawLine.slice(colon + 1);
|
||||
if (value.startsWith(' ')) value = value.slice(1);
|
||||
|
||||
switch (field) {
|
||||
case 'id':
|
||||
id = value;
|
||||
break;
|
||||
case 'event':
|
||||
event = value;
|
||||
break;
|
||||
case 'data':
|
||||
dataLines.push(value);
|
||||
break;
|
||||
// ignore other fields (retry, etc.)
|
||||
}
|
||||
}
|
||||
|
||||
if (dataLines.length === 0) {
|
||||
return; // bare heartbeat / no payload
|
||||
}
|
||||
|
||||
let data: any;
|
||||
try {
|
||||
data = JSON.parse(dataLines.join('\n'));
|
||||
} catch (err: any) {
|
||||
console.warn(`${this.opts.logTag} Skipping unparseable event ${id || '(no id)'}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const ev: SSEEvent = { id, event: event || 'message', data };
|
||||
|
||||
if (ev.event === 'gap') {
|
||||
if (this.opts.onGap) {
|
||||
await this.opts.onGap(ev.data);
|
||||
} else {
|
||||
console.warn(`${this.opts.logTag} Gap event (cursor ${ev.data?.requested} trimmed): resumed from ${ev.data?.resumeFrom}`);
|
||||
}
|
||||
if (typeof ev.data?.resumeFrom === 'string' && ev.data.resumeFrom) {
|
||||
this.cursor = ev.data.resumeFrom;
|
||||
this.cursorDirty = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.opts.onEvent(ev);
|
||||
} catch (err: any) {
|
||||
// Handler error doesn't tear down the connection — log and move on.
|
||||
console.error(`${this.opts.logTag} onEvent handler failed for ${ev.id}: ${err?.stack ?? err}`);
|
||||
}
|
||||
|
||||
if (ev.id) {
|
||||
this.cursor = ev.id;
|
||||
this.cursorDirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
@@ -1,7 +1,6 @@
|
||||
import {config} from "./config/EnvConfig.js";
|
||||
|
||||
import os from 'os';
|
||||
import {existsSync} from 'fs';
|
||||
import {WatchedFileProcessor} from "./logic/WatchedFileProcessor.js";
|
||||
import {UnifiedAPIServer} from "./api/UnifiedAPIServer.js";
|
||||
import {StreamingPipeline} from "./logic/pipeline/StreamingPipeline.js";
|
||||
@@ -104,7 +103,7 @@ process.on('SIGINT', async () => {
|
||||
// Stop File Event Consumer if running
|
||||
if (fileEventConsumer) {
|
||||
console.log('[Shutdown] Stopping File Event Consumer...');
|
||||
fileEventConsumer.stop();
|
||||
await fileEventConsumer.stop();
|
||||
}
|
||||
|
||||
// Stop Container Manager if running
|
||||
@@ -125,22 +124,18 @@ process.on('SIGINT', async () => {
|
||||
// Start initial discovery and processing (streaming pipeline)
|
||||
(async () => {
|
||||
try {
|
||||
// Validate META_CORE_PATH is configured (required for leader discovery)
|
||||
if (!config.META_CORE_PATH) {
|
||||
console.error('[Startup] ERROR: META_CORE_PATH is required. Set META_CORE_PATH environment variable.');
|
||||
console.error('[Startup] Example: META_CORE_PATH=/meta-core');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Initialize KV Manager (Redis via leader discovery)
|
||||
if (config.META_CORE_PATH) {
|
||||
console.log('[Startup] Initializing KV Manager (leader discovery)...');
|
||||
// meta-core is located over UDP (beacon v2) — no /meta-core
|
||||
// volume required. META_CORE_URL, when set, pins it and discovery
|
||||
// never overrides; see docs/project-architecture/service-discovery.md.
|
||||
{
|
||||
console.log('[Startup] Initializing KV Manager (meta-core discovery)...');
|
||||
kvManager = new KVManager({
|
||||
metaCorePath: config.META_CORE_PATH,
|
||||
filesPath: config.FILES_PATH,
|
||||
serviceName: config.SERVICE_NAME,
|
||||
apiPort: config.FUSE_API_PORT,
|
||||
baseUrl: config.BASE_URL,
|
||||
metaCoreUrl: config.META_CORE_URL,
|
||||
});
|
||||
|
||||
// Wait for KV to be ready
|
||||
@@ -172,8 +167,11 @@ process.on('SIGINT', async () => {
|
||||
const pluginManager = await initializePluginManager();
|
||||
console.log('[Startup] Plugin manager initialized');
|
||||
|
||||
// Initialize container plugins if config file exists
|
||||
if (existsSync(config.CONTAINER_PLUGINS_CONFIG)) {
|
||||
// Container plugins: the ones spawned from plugins.yml AND the external
|
||||
// ones added by URL (beacon Scan card). Always initialised — with no
|
||||
// plugins.yml and no Docker it still serves the external ones, and the
|
||||
// Plugins page can add the first one.
|
||||
{
|
||||
console.log('[Startup] Initializing container plugins...');
|
||||
try {
|
||||
// Get webdavUrlInternal from meta-core via LeaderClient (for container-to-container access)
|
||||
@@ -219,8 +217,6 @@ process.on('SIGINT', async () => {
|
||||
console.error('[Startup] Failed to initialize container plugins:', error);
|
||||
console.log('[Startup] Continuing without container plugins...');
|
||||
}
|
||||
} else {
|
||||
console.log(`[Startup] Container plugins config not found at ${config.CONTAINER_PLUGINS_CONFIG}, skipping`);
|
||||
}
|
||||
|
||||
// Initialize hash index cache (loads existing hashes from disk)
|
||||
@@ -256,21 +252,21 @@ process.on('SIGINT', async () => {
|
||||
console.log(`[Cache] midhash256 cache: ${hits} hits, ${misses} misses (${hitRate}% hit rate)`);
|
||||
}
|
||||
|
||||
// Connect to meta-core for file events via Redis Stream
|
||||
// KV client must be available
|
||||
if (!kvClient) {
|
||||
console.error('[Startup] ERROR: KV client not available. Cannot consume file events.');
|
||||
console.error('[Startup] Make sure meta-core is running and Redis is accessible.');
|
||||
// Connect to meta-core for file events via SSE
|
||||
const leaderInfoForSSE = kvManager?.getLeaderInfo();
|
||||
const metaCoreApiUrl = leaderInfoForSSE?.apiUrl;
|
||||
if (!metaCoreApiUrl) {
|
||||
console.error('[Startup] ERROR: meta-core apiUrl not available from leader info. Cannot consume file events.');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[Startup] Starting File Event Consumer (Redis Stream)...');
|
||||
console.log(`[Startup] Starting File Event Consumer (SSE → ${metaCoreApiUrl}/api/events/files)...`);
|
||||
fileEventConsumer = new FileEventConsumer({
|
||||
kvClient: kvClient as RedisKVClient,
|
||||
metaCoreApiUrl,
|
||||
pipeline: pipeline,
|
||||
filesPath: config.FILES_PATH,
|
||||
});
|
||||
await fileEventConsumer.start();
|
||||
console.log('[Startup] FileEventConsumer started, consuming from file:events stream');
|
||||
console.log('[Startup] FileEventConsumer started, consuming via SSE');
|
||||
|
||||
} catch (error) {
|
||||
console.error('[Startup] Error during startup:', error);
|
||||
|
||||
@@ -139,7 +139,7 @@ export interface KVClientConfig {
|
||||
}
|
||||
|
||||
/**
|
||||
* Leader lock file content format (kv-leader.info)
|
||||
* meta-core's identity and URL set (from its UDP announce, or /urls)
|
||||
* Written by meta-core leader, read by other services
|
||||
*/
|
||||
export interface LeaderLockInfo {
|
||||
|
||||
@@ -2,13 +2,12 @@
|
||||
* KV Manager - Unified manager for KV storage and service discovery
|
||||
*
|
||||
* This is the main entry point for the KV subsystem. It:
|
||||
* 1. Uses LeaderClient to read leader info from meta-core
|
||||
* 1. Uses LeaderClient to locate meta-core over UDP (beacon v2)
|
||||
* 2. Creates and manages the KV client (Redis)
|
||||
* 3. Handles service discovery registration
|
||||
* 4. Manages reconnection on leader changes
|
||||
*
|
||||
* Note: Leader election is now handled by meta-core (Go sidecar).
|
||||
* This service only reads the leader info and connects to Redis.
|
||||
* Note: there is no leader election. meta-core announces itself on the
|
||||
* discovery group; this service listens and connects to its HTTP API.
|
||||
*
|
||||
* Usage:
|
||||
* ```typescript
|
||||
@@ -28,7 +27,6 @@
|
||||
|
||||
import { hostname } from 'os';
|
||||
import { LeaderClient } from './LeaderClient.js';
|
||||
import { ServiceRegistration } from './ServiceRegistration.js';
|
||||
import { RedisKVClient } from './RedisClient.js';
|
||||
import type { IKVClient, LeaderLockInfo } from './IKVClient.js';
|
||||
import { configure as configureWebdav } from '../webdav/WebdavClient.js';
|
||||
@@ -59,7 +57,6 @@ interface KVManagerConfig {
|
||||
export class KVManager {
|
||||
private config: KVManagerConfig;
|
||||
private leaderClient: LeaderClient | null = null;
|
||||
private serviceRegistration: ServiceRegistration | null = null;
|
||||
private kvClient: IKVClient | null = null;
|
||||
private isStarted = false;
|
||||
private isShuttingDown = false;
|
||||
@@ -73,13 +70,17 @@ export class KVManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Redis client from URL
|
||||
* Create Redis client from URL. When apiUrl is supplied, the resulting
|
||||
* client routes writes through meta-core's HTTP API (PR B of the
|
||||
* api-mediated-access lockdown) while keeping reads + stream consumption
|
||||
* on the direct Redis connection.
|
||||
*/
|
||||
private async createRedisClient(redisUrl: string): Promise<RedisKVClient> {
|
||||
private async createRedisClient(redisUrl: string, apiUrl?: string): Promise<RedisKVClient> {
|
||||
const client = new RedisKVClient({
|
||||
url: redisUrl,
|
||||
timeout: 30000,
|
||||
prefix: '' // No prefix for shared access
|
||||
prefix: '', // No prefix for shared access
|
||||
metaCoreApiUrl: apiUrl,
|
||||
});
|
||||
|
||||
await client.connect();
|
||||
@@ -108,8 +109,14 @@ export class KVManager {
|
||||
|
||||
// Initialize leader client
|
||||
this.leaderClient = new LeaderClient({
|
||||
metaCorePath: this.config.metaCorePath,
|
||||
metaCoreUrl: this.config.metaCoreUrl
|
||||
metaCoreUrl: this.config.metaCoreUrl,
|
||||
serviceName: this.config.serviceName,
|
||||
// Browser-facing URL for the nav menu, in the order the spec requires:
|
||||
// PUBLIC_URL (reachable when there is no Caddy perimeter in front — a
|
||||
// debug-direct port) -> BASE_URL (the Caddy URL) -> container host.
|
||||
// Announcing a Caddy URL on a stack with no Caddy running is what puts
|
||||
// dead links in every neighbour's menu.
|
||||
baseUrl: process.env.PUBLIC_URL || this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`,
|
||||
});
|
||||
|
||||
// Watch for leader changes
|
||||
@@ -121,8 +128,16 @@ export class KVManager {
|
||||
// Wait for leader and connect
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||
if (leaderInfo.redisUrl) {
|
||||
// api-mediated-access PR D removes redisUrl from leader info.
|
||||
// Until that lands we still log it but services boot fine
|
||||
// when it's absent — all writes/reads route through HTTP.
|
||||
console.warn('[KVManager] Leader info still publishes redisUrl; direct Redis access is being deprecated, prefer meta-core HTTP');
|
||||
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
} else {
|
||||
console.log('[KVManager] redisUrl absent in leader info — running in HTTP-only mode');
|
||||
}
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl, leaderInfo.apiUrl);
|
||||
|
||||
// Configure WebDAV client with leader's WebDAV URL (prefer internal for container-to-container)
|
||||
const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl;
|
||||
@@ -137,15 +152,9 @@ export class KVManager {
|
||||
// Start watching for leader changes
|
||||
this.leaderClient.startWatching();
|
||||
|
||||
// Initialize and start service registration for heartbeat
|
||||
const apiUrl = this.config.baseUrl || `http://${hostname()}:${this.config.apiPort}`;
|
||||
this.serviceRegistration = new ServiceRegistration({
|
||||
metaCorePath: this.config.metaCorePath,
|
||||
serviceName: this.config.serviceName,
|
||||
version: '1.0.0',
|
||||
apiUrl
|
||||
});
|
||||
await this.serviceRegistration.start();
|
||||
// No separate service registration step any more: the LeaderClient's
|
||||
// discovery node announces this service on the same UDP group it
|
||||
// listens on, so being discoverable and discovering are one thing.
|
||||
|
||||
this.isStarted = true;
|
||||
this.notifyReady();
|
||||
@@ -169,7 +178,7 @@ export class KVManager {
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl, leaderInfo.apiUrl);
|
||||
|
||||
// Reconfigure WebDAV client with new leader's WebDAV URL (prefer internal for container-to-container)
|
||||
const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl;
|
||||
@@ -212,10 +221,6 @@ export class KVManager {
|
||||
}
|
||||
|
||||
// Service registration stop (if it was started)
|
||||
if (this.serviceRegistration) {
|
||||
await this.serviceRegistration.stop();
|
||||
this.serviceRegistration = null;
|
||||
}
|
||||
|
||||
this.isStarted = false;
|
||||
console.log('[KVManager] Stopped');
|
||||
@@ -312,9 +317,6 @@ export class KVManager {
|
||||
/**
|
||||
* Get service registration instance
|
||||
*/
|
||||
getServiceRegistration(): ServiceRegistration | null {
|
||||
return this.serviceRegistration;
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the KV store is healthy
|
||||
|
||||
@@ -1,26 +1,49 @@
|
||||
/**
|
||||
* LeaderClient - Client for reading leader info from meta-core
|
||||
* LeaderClient - locates meta-core and exposes its URLs.
|
||||
*
|
||||
* This replaces the old LeaderElection.ts by delegating leader election
|
||||
* to meta-core (Go sidecar) and just reading the results.
|
||||
* Since UDP discovery (meta-discovery v1, now beacon v2) this is a thin adapter
|
||||
* over MetaCoreLocator
|
||||
* (../discovery/meshdisco.js): meta-core is found by UDP announce instead of by
|
||||
* reading /meta-core/locks/kv-leader.info, which means this service no longer
|
||||
* needs the /meta-core volume mounted at all.
|
||||
*
|
||||
* Features:
|
||||
* - Reads kv-leader.info file for leader URLs
|
||||
* - Calls meta-core /urls API for current leader URLs
|
||||
* - Watches for leader changes via file system watcher
|
||||
* The public surface is deliberately unchanged so KVManager and its callers did
|
||||
* not have to move:
|
||||
* - getLeaderInfo() / getUrls() / getApiUrl() / getWebdavUrl*()
|
||||
* - waitForLeader(timeoutMs) — same 30s default, same "poll until it answers"
|
||||
* - startWatching() / onChange() — the fs.watch on the lock dir became "an
|
||||
* announce arrived carrying a different apiUrl", feeding the same callback.
|
||||
*
|
||||
* Precedence is the pin: when metaCoreUrl (META_CORE_URL) is set it always
|
||||
* wins and the wire is never consulted for core selection. See
|
||||
* docs/project-architecture/service-discovery.md.
|
||||
*
|
||||
* The name "leader" is legacy. meta-core's flock election is vestigial — the Go
|
||||
* binary only ever runs as leader — and redisUrl has been empty since the
|
||||
* api-mediated-access lockdown; everything routes over HTTP.
|
||||
*/
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import { watch, FSWatcher } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
import { MetaCoreLocator, type MeshNeighbor, type ScanReport } from '../discovery/meshdisco.js';
|
||||
import type { LeaderLockInfo } from './IKVClient.js';
|
||||
|
||||
export interface LeaderClientConfig {
|
||||
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
||||
metaCorePath: string;
|
||||
/**
|
||||
* @deprecated Unused since meta-discovery v1. Kept so existing callers
|
||||
* compile unchanged; nothing reads the volume any more.
|
||||
*/
|
||||
metaCorePath?: string;
|
||||
|
||||
/** meta-core API URL (e.g., http://meta-core:9000 or http://localhost:9000) */
|
||||
/** meta-core API URL. When set, discovery never overrides it. */
|
||||
metaCoreUrl?: string;
|
||||
|
||||
/** This service's name, used for its own announce. */
|
||||
serviceName?: string;
|
||||
|
||||
/** Browser-facing URL announced for the nav menu. */
|
||||
baseUrl?: string;
|
||||
|
||||
/** Service version, display only. */
|
||||
version?: string;
|
||||
}
|
||||
|
||||
export interface URLsResponse {
|
||||
@@ -35,59 +58,79 @@ export interface URLsResponse {
|
||||
|
||||
export class LeaderClient {
|
||||
private config: LeaderClientConfig;
|
||||
private infoFilePath: string;
|
||||
private locator: MetaCoreLocator;
|
||||
private leaderInfo: LeaderLockInfo | null = null;
|
||||
private watcher: FSWatcher | null = null;
|
||||
private onChangeCallbacks: (() => void)[] = [];
|
||||
private started = false;
|
||||
|
||||
// URL caching
|
||||
// URL caching — only used on the pinned path, where the URLs still come
|
||||
// from an HTTP GET /urls. A discovered core carries them in the announce.
|
||||
private cachedUrls: URLsResponse | null = null;
|
||||
private urlsCacheTime: number = 0;
|
||||
private readonly urlsCacheTTL: number = 5000; // 5 seconds
|
||||
private urlsCacheTime = 0;
|
||||
private readonly urlsCacheTTL = 5000;
|
||||
|
||||
constructor(config: LeaderClientConfig) {
|
||||
this.config = config;
|
||||
this.infoFilePath = `${config.metaCorePath}/locks/kv-leader.info`;
|
||||
this.locator = new MetaCoreLocator({
|
||||
serviceName: config.serviceName ?? 'meta-sort',
|
||||
baseUrl: config.baseUrl,
|
||||
version: config.version,
|
||||
metaCoreUrl: config.metaCoreUrl,
|
||||
enabled: process.env.ENABLE_UDP_DISCOVERY !== 'false' &&
|
||||
process.env.ENABLE_UDP_DISCOVERY !== '0',
|
||||
});
|
||||
this.locator.onChange(() => {
|
||||
this.cachedUrls = null;
|
||||
this.urlsCacheTime = 0;
|
||||
this.notifyChange();
|
||||
});
|
||||
}
|
||||
|
||||
/** Idempotent; safe to call from any entry point. */
|
||||
private async ensureStarted(): Promise<void> {
|
||||
if (this.started) return;
|
||||
this.started = true;
|
||||
await this.locator.start();
|
||||
}
|
||||
|
||||
/**
|
||||
* Read API URL from file (plain text format)
|
||||
* Guard against a rollback that reintroduces direct Redis exposure.
|
||||
* api-mediated-access PR D removed redisUrl; if it comes back, fail loudly
|
||||
* so it can't go unnoticed. ALLOW_LEGACY_REDIS_URL=1 downgrades to a warn.
|
||||
*/
|
||||
private async getApiUrlFromFile(): Promise<string | null> {
|
||||
try {
|
||||
const content = await fs.readFile(this.infoFilePath, 'utf-8');
|
||||
return content.trim() || null;
|
||||
} catch (error) {
|
||||
console.error('[LeaderClient] Failed to read API URL from file:', error);
|
||||
return null;
|
||||
private assertNoRedisUrl(redisUrl: string | undefined): void {
|
||||
if (!redisUrl) return;
|
||||
const msg =
|
||||
'[LeaderClient] meta-core still publishes redisUrl; direct Redis access ' +
|
||||
'was retired by the api-mediated-access lockdown. Verify meta-core version.';
|
||||
if (process.env.ALLOW_LEGACY_REDIS_URL === '1') {
|
||||
console.warn('WARNING: ' + msg);
|
||||
} else {
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fetch URLs from meta-core /urls API with caching
|
||||
*/
|
||||
/** HTTP GET {apiUrl}/urls — still needed when meta-core is pinned. */
|
||||
private async fetchUrls(apiUrl: string): Promise<URLsResponse | null> {
|
||||
// Check cache
|
||||
const now = Date.now();
|
||||
if (this.cachedUrls && (now - this.urlsCacheTime) < this.urlsCacheTTL) {
|
||||
if (this.cachedUrls && now - this.urlsCacheTime < this.urlsCacheTTL) {
|
||||
return this.cachedUrls;
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetch(`${apiUrl}/urls`, {
|
||||
method: 'GET',
|
||||
headers: { 'Accept': 'application/json' },
|
||||
signal: AbortSignal.timeout(5000)
|
||||
headers: { Accept: 'application/json' },
|
||||
signal: AbortSignal.timeout(5000),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.error('[LeaderClient] Failed to fetch URLs:', response.status, response.statusText);
|
||||
return null;
|
||||
}
|
||||
|
||||
this.cachedUrls = await response.json() as URLsResponse;
|
||||
const parsed = (await response.json()) as URLsResponse;
|
||||
this.assertNoRedisUrl(parsed.redisUrl);
|
||||
this.cachedUrls = parsed;
|
||||
this.urlsCacheTime = now;
|
||||
return this.cachedUrls;
|
||||
return parsed;
|
||||
} catch (error) {
|
||||
console.error('[LeaderClient] Error calling /urls API:', error);
|
||||
return null;
|
||||
@@ -95,23 +138,37 @@ export class LeaderClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Read leader info from file and /urls API
|
||||
* Resolve meta-core's URLs — from the announce when discovered (no HTTP
|
||||
* hop needed, the payload already carries them), or over HTTP when pinned.
|
||||
*/
|
||||
async getUrls(): Promise<URLsResponse | null> {
|
||||
await this.ensureStarted();
|
||||
|
||||
const wire = this.locator.getUrls();
|
||||
if (wire && !this.config.metaCoreUrl) {
|
||||
return {
|
||||
hostname: wire.hostname,
|
||||
baseUrl: wire.baseUrl,
|
||||
apiUrl: wire.apiUrl,
|
||||
redisUrl: '',
|
||||
webdavUrl: wire.webdavUrl,
|
||||
webdavUrlInternal: wire.webdavUrlInternal,
|
||||
isLeader: true,
|
||||
};
|
||||
}
|
||||
|
||||
const apiUrl = this.locator.getApiUrl();
|
||||
if (!apiUrl) {
|
||||
console.error('[LeaderClient] No meta-core API URL available');
|
||||
return null;
|
||||
}
|
||||
return this.fetchUrls(apiUrl);
|
||||
}
|
||||
|
||||
async getLeaderInfo(): Promise<LeaderLockInfo | null> {
|
||||
try {
|
||||
// Read API URL from file (plain text)
|
||||
const apiUrl = await this.getApiUrlFromFile();
|
||||
if (!apiUrl) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Fetch full info from /urls API
|
||||
const urls = await this.fetchUrls(apiUrl);
|
||||
if (!urls) {
|
||||
return null;
|
||||
}
|
||||
|
||||
// Convert URLsResponse to LeaderLockInfo
|
||||
const urls = await this.getUrls();
|
||||
if (!urls) return null;
|
||||
this.leaderInfo = {
|
||||
hostname: urls.hostname,
|
||||
baseUrl: urls.baseUrl,
|
||||
@@ -120,7 +177,7 @@ export class LeaderClient {
|
||||
webdavUrl: urls.webdavUrl,
|
||||
webdavUrlInternal: urls.webdavUrlInternal,
|
||||
timestamp: Date.now(),
|
||||
pid: 0 // Unknown for remote leader
|
||||
pid: 0,
|
||||
};
|
||||
return this.leaderInfo;
|
||||
} catch (error) {
|
||||
@@ -129,134 +186,67 @@ export class LeaderClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get Redis URL from leader info
|
||||
*/
|
||||
async getRedisUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.redisUrl ?? null;
|
||||
return (await this.getLeaderInfo())?.redisUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get WebDAV URL from leader info (external, via nginx/HTTPS)
|
||||
*/
|
||||
async getWebdavUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.webdavUrl ?? null;
|
||||
return (await this.getLeaderInfo())?.webdavUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get internal WebDAV URL from leader info (direct to port 9000)
|
||||
* Use this for container-to-container communication
|
||||
*/
|
||||
/** Internal WebDAV URL — use for container-to-container access. */
|
||||
async getWebdavUrlInternal(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.webdavUrlInternal ?? null;
|
||||
return (await this.getLeaderInfo())?.webdavUrlInternal ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get meta-core API URL from leader info
|
||||
*/
|
||||
async getApiUrl(): Promise<string | null> {
|
||||
const info = await this.getLeaderInfo();
|
||||
return info?.apiUrl ?? null;
|
||||
return (await this.getLeaderInfo())?.apiUrl ?? null;
|
||||
}
|
||||
|
||||
/**
|
||||
* Call meta-core /urls API to get current URLs
|
||||
* Useful for getting URLs when you don't want to read the file directly
|
||||
*/
|
||||
async getUrls(): Promise<URLsResponse | null> {
|
||||
// First try using configured metaCoreUrl
|
||||
let apiUrl = this.config.metaCoreUrl;
|
||||
|
||||
// Fall back to reading from file
|
||||
if (!apiUrl) {
|
||||
apiUrl = await this.getApiUrlFromFile();
|
||||
}
|
||||
|
||||
if (!apiUrl) {
|
||||
console.error('[LeaderClient] No meta-core API URL available');
|
||||
return null;
|
||||
}
|
||||
|
||||
return this.fetchUrls(apiUrl);
|
||||
}
|
||||
|
||||
/**
|
||||
* Wait for leader info to be available
|
||||
* Block until meta-core is reachable. Probes immediately then every 500ms,
|
||||
* matching the previous file-polling loop's timing exactly.
|
||||
*/
|
||||
async waitForLeader(timeoutMs: number = 30000): Promise<LeaderLockInfo> {
|
||||
const startTime = Date.now();
|
||||
const pollInterval = 500;
|
||||
await this.ensureStarted();
|
||||
const deadline = Date.now() + timeoutMs;
|
||||
|
||||
while (Date.now() - startTime < timeoutMs) {
|
||||
// Locate the core (pin returns instantly; discovery probes).
|
||||
await this.locator.waitForCore(timeoutMs);
|
||||
|
||||
// Then wait for a usable URLs payload — on the pinned path this is the
|
||||
// first successful GET /urls, so meta-core still has to be up.
|
||||
while (Date.now() < deadline) {
|
||||
const info = await this.getLeaderInfo();
|
||||
if (info) {
|
||||
console.log(`[LeaderClient] Leader found: ${info.hostname} at ${info.redisUrl}`);
|
||||
console.log(`[LeaderClient] meta-core found: ${info.hostname} at ${info.apiUrl}`);
|
||||
return info;
|
||||
}
|
||||
|
||||
console.log('[LeaderClient] Waiting for leader...');
|
||||
await new Promise(resolve => setTimeout(resolve, pollInterval));
|
||||
console.log('[LeaderClient] Waiting for meta-core...');
|
||||
await new Promise((resolve) => setTimeout(resolve, 500));
|
||||
}
|
||||
|
||||
throw new Error(`No leader found within ${timeoutMs}ms`);
|
||||
throw new Error(`No meta-core found within ${timeoutMs}ms`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Start watching for leader changes
|
||||
* Previously an fs.watch on the lock directory. Discovery is always
|
||||
* listening, so this only has to make sure the node is running.
|
||||
*/
|
||||
startWatching(): void {
|
||||
if (this.watcher) {
|
||||
return; // Already watching
|
||||
}
|
||||
|
||||
const lockDir = dirname(this.infoFilePath);
|
||||
|
||||
try {
|
||||
this.watcher = watch(lockDir, (eventType, filename) => {
|
||||
if (filename === 'kv-leader.info') {
|
||||
console.log('[LeaderClient] Leader info changed, invalidating cache...');
|
||||
// Invalidate cache to force fresh API call
|
||||
this.cachedUrls = null;
|
||||
this.urlsCacheTime = 0;
|
||||
|
||||
this.getLeaderInfo().then(() => {
|
||||
this.notifyChange();
|
||||
}).catch(console.error);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`[LeaderClient] Watching for leader changes in ${lockDir}`);
|
||||
} catch (error) {
|
||||
console.error('[LeaderClient] Failed to start watching:', error);
|
||||
}
|
||||
void this.ensureStarted();
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop watching for leader changes
|
||||
*/
|
||||
stopWatching(): void {
|
||||
if (this.watcher) {
|
||||
this.watcher.close();
|
||||
this.watcher = null;
|
||||
console.log('[LeaderClient] Stopped watching for leader changes');
|
||||
}
|
||||
// No-op: the locator keeps listening until close().
|
||||
}
|
||||
|
||||
/**
|
||||
* Register callback for leader changes
|
||||
*/
|
||||
onChange(callback: () => void): this {
|
||||
this.onChangeCallbacks.push(callback);
|
||||
return this;
|
||||
}
|
||||
|
||||
/**
|
||||
* Notify all change callbacks
|
||||
*/
|
||||
private notifyChange(): void {
|
||||
console.log('[LeaderClient] meta-core changed, invalidating cache...');
|
||||
for (const callback of this.onChangeCallbacks) {
|
||||
try {
|
||||
callback();
|
||||
@@ -266,18 +256,31 @@ export class LeaderClient {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Get cached leader info (without re-reading file)
|
||||
*/
|
||||
getCachedLeaderInfo(): LeaderLockInfo | null {
|
||||
return this.leaderInfo;
|
||||
}
|
||||
|
||||
/**
|
||||
* Clean up resources
|
||||
* Neighbours for /api/neighbors: one row per name (every instance with
|
||||
* `all`), optionally only nodes with a capability matching `cap`.
|
||||
*/
|
||||
getNeighbors(opts: { all?: boolean; cap?: string } = {}): MeshNeighbor[] {
|
||||
return this.locator.getNeighbors(opts);
|
||||
}
|
||||
|
||||
/** This service's own row, so the menu can show itself. */
|
||||
self(): MeshNeighbor {
|
||||
return this.locator.self();
|
||||
}
|
||||
|
||||
/** Beacon v2 Scan card: probe, wait, classify `pattern` against `configured`. */
|
||||
scan(pattern: string, configured: Array<[string, string]>, waitMs = 1500): Promise<ScanReport> {
|
||||
return this.locator.scan(pattern, configured, waitMs);
|
||||
}
|
||||
|
||||
close(): void {
|
||||
this.stopWatching();
|
||||
this.onChangeCallbacks = [];
|
||||
void this.locator.stop();
|
||||
this.started = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,262 @@
|
||||
/**
|
||||
* MetaCoreApiWriter — HTTP writer for metadata, replacing direct Redis writes.
|
||||
*
|
||||
* meta-core owns the Redis schema (UUID-rooted, reverse-indexed by CID). Any
|
||||
* service writing directly to Redis risks drifting the schema; the lockdown
|
||||
* plan (docs/api-mediated-access.md in meta-core) routes every write through
|
||||
* meta-core's HTTP /meta/{hash} family.
|
||||
*
|
||||
* This writer covers the three operations meta-sort actually performs in
|
||||
* production:
|
||||
* - setMetadataFlat → PATCH /meta/{hash} (merge semantics)
|
||||
* - setMetadataProperty → PUT /meta/{hash}/{key}
|
||||
* - deleteMetadataFlat → DELETE /meta/{hash}
|
||||
*
|
||||
* Reads and stream consumption stay on Redis in PR B; PR C migrates those.
|
||||
*/
|
||||
import {
|
||||
buildRecordFields,
|
||||
buildFilePrefix,
|
||||
} from './MetadataUtils.js';
|
||||
|
||||
/** One file-backed root as listed by meta-core's GET /api/files/tuples. */
|
||||
export interface FileTuple {
|
||||
hashId: string;
|
||||
filePath: string;
|
||||
sizeByte: number;
|
||||
mtimeNano: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/files/tuples body. `files` is present unless the summary form was
|
||||
* asked for.
|
||||
*/
|
||||
export interface FileTuplesResult {
|
||||
count: number;
|
||||
totalSize: number;
|
||||
files?: FileTuple[];
|
||||
}
|
||||
|
||||
export interface MetaCoreApiWriterConfig {
|
||||
/** Base URL for meta-core's HTTP API (e.g. http://metacore-app:9000) */
|
||||
apiUrl: string;
|
||||
|
||||
/** Optional per-request timeout in ms (default: 30000) */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export class MetaCoreApiWriter {
|
||||
private apiUrl: string;
|
||||
private timeoutMs: number;
|
||||
|
||||
constructor(config: MetaCoreApiWriterConfig) {
|
||||
// Trim trailing slash so we can join paths predictably.
|
||||
this.apiUrl = config.apiUrl.replace(/\/+$/, '');
|
||||
this.timeoutMs = config.timeoutMs ?? 30000;
|
||||
}
|
||||
|
||||
/**
|
||||
* PATCH /meta/{hash}.
|
||||
*
|
||||
* meta-core's PATCH calls MergeMetadataFlat — merge semantics, matches
|
||||
* what setMetadataFlat used to do via Redis pipeline of SETs.
|
||||
*/
|
||||
async setMetadataFlat(
|
||||
hashId: string,
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
// `buildRecordFields` is the single write-shape boundary, shared with the
|
||||
// direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
|
||||
// never a per-algorithm `cid_*` field — which meta-core would now reject
|
||||
// with a 400 anyway.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
|
||||
}
|
||||
|
||||
/**
|
||||
* PUT /meta/{hash}/{key}.
|
||||
*
|
||||
* meta-core accepts either JSON {"value":"..."} or plain text; we use
|
||||
* the JSON form to avoid ambiguity with values that happen to look like
|
||||
* JSON themselves.
|
||||
*/
|
||||
async setMetadataProperty(hashId: string, property: string, value: string): Promise<void> {
|
||||
await this.fetchJson(
|
||||
'PUT',
|
||||
`/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(property)}`,
|
||||
{ value }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /meta/{hash}.
|
||||
*
|
||||
* Removes every property and unregisters the hashId from file:__index__.
|
||||
*/
|
||||
async deleteMetadataFlat(hashId: string): Promise<number> {
|
||||
const body = await this.fetchJson(
|
||||
'DELETE',
|
||||
`/meta/${encodeURIComponent(hashId)}`,
|
||||
null
|
||||
);
|
||||
// meta-core's response is {success, hashId, deleted: N}.
|
||||
if (body && typeof body === 'object' && typeof (body as any).deleted === 'number') {
|
||||
return (body as any).deleted;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta/{hash}. Returns the flat metadata map (or null if absent).
|
||||
*/
|
||||
async getMetadataFlat(hashId: string): Promise<any | null> {
|
||||
const path = `/meta/${encodeURIComponent(hashId)}`;
|
||||
const body = await this.fetchJsonAllowing404('GET', path);
|
||||
if (body === null) return null;
|
||||
// meta-core wraps the flat map under `metadata`.
|
||||
const flat = (body as any)?.metadata;
|
||||
if (!flat || typeof flat !== 'object') return null;
|
||||
return flat;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta/{hash}/{propertyPath}. Returns the raw string value (or null
|
||||
* if absent). meta-core sends back text/plain.
|
||||
*/
|
||||
async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
|
||||
const path = `/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(propertyPath)}`;
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'text/plain' },
|
||||
});
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`[MetaCoreApiWriter] GET ${path} → ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
// Mirror the legacy RedisClient: try JSON-parse first, fall back to raw string.
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta. Returns the list of all hashIds known to meta-core.
|
||||
*/
|
||||
async getAllHashIds(): Promise<string[]> {
|
||||
const body = await this.fetchJson('GET', '/meta', null);
|
||||
const ids = (body as any)?.hashIds;
|
||||
return Array.isArray(ids) ? ids : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /api/files/tuples. meta-core's FILE-BACKED roots (records carrying
|
||||
* filePath, sizeByte and mtimeNano) with sizes, read server-side in batched
|
||||
* MGETs — the cheap alternative to walking getAllHashIds(), which on a
|
||||
* shared meta-core is every gateway record too. `summary` omits the rows.
|
||||
*
|
||||
* Returns null when meta-core predates the endpoint (404, or a body that is
|
||||
* not the expected JSON).
|
||||
*/
|
||||
async getFileTuples(opts: { summary?: boolean } = {}): Promise<FileTuplesResult | null> {
|
||||
const path = opts.summary ? '/api/files/tuples?summary=1' : '/api/files/tuples';
|
||||
const body = await this.fetchJsonAllowing404('GET', path);
|
||||
if (!body || typeof body !== 'object') return null;
|
||||
const b = body as { count?: unknown; totalSize?: unknown; files?: unknown };
|
||||
return {
|
||||
count: Number(b.count) || 0,
|
||||
totalSize: Number(b.totalSize) || 0,
|
||||
files: Array.isArray(b.files) ? (b.files as FileTuple[]) : undefined,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /health on meta-core. Used as a write-path health probe.
|
||||
*/
|
||||
async health(): Promise<boolean> {
|
||||
try {
|
||||
const response = await fetch(`${this.apiUrl}/health`, {
|
||||
method: 'GET',
|
||||
signal: AbortSignal.timeout(Math.min(this.timeoutMs, 5000)),
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchJsonAllowing404(method: string, path: string): Promise<unknown> {
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`[MetaCoreApiWriter] ${method} ${path} → ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
try { return JSON.parse(text); } catch { return text; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a slash-containing property path safely for the URL while still
|
||||
* letting meta-core's `{key:.*}` route match the multi-segment key.
|
||||
*/
|
||||
private encodePropertyPath(property: string): string {
|
||||
return property
|
||||
.split('/')
|
||||
.map(segment => encodeURIComponent(segment))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
private async fetchJson(
|
||||
method: string,
|
||||
path: string,
|
||||
body: unknown
|
||||
): Promise<unknown> {
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const init: RequestInit = {
|
||||
method,
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'application/json' },
|
||||
};
|
||||
if (body !== null && body !== undefined) {
|
||||
init.headers = { ...init.headers, 'Content-Type': 'application/json' };
|
||||
init.body = JSON.stringify(body);
|
||||
}
|
||||
|
||||
const response = await fetch(url, init);
|
||||
if (!response.ok) {
|
||||
// Surface meta-core's error envelope when it's structured.
|
||||
let detail = response.statusText;
|
||||
try {
|
||||
const errBody = await response.text();
|
||||
if (errBody) detail = `${detail}: ${errBody}`;
|
||||
} catch {
|
||||
/* ignore body-read errors; statusText still useful */
|
||||
}
|
||||
throw new Error(`[MetaCoreApiWriter] ${method} ${path} → ${response.status} ${detail}`);
|
||||
}
|
||||
|
||||
// 204 / empty body — return null so callers don't choke on JSON parse.
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { buildRecordFields } from './MetadataUtils.js';
|
||||
|
||||
/**
|
||||
* `buildRecordFields` is the single boundary where in-memory metadata becomes a
|
||||
* record. It is worth testing precisely because the bug it fixes had no coverage:
|
||||
* meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
|
||||
* fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
|
||||
* The other wrote a literal `cid_midhash256` field, which meta-core stores but
|
||||
* never reverse-indexes — so those records were unresolvable by their own CID,
|
||||
* silently. Both paths now go through this function.
|
||||
*/
|
||||
describe('buildRecordFields', () => {
|
||||
const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
|
||||
const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
|
||||
|
||||
it('turns cids[] into the bare-CID key-set, not array indices', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH, SHA256],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
assert.equal(flat[`cids/${MIDHASH}`], 'true');
|
||||
assert.equal(flat[`cids/${SHA256}`], 'true');
|
||||
|
||||
// The trap: the generic flattener would emit `cids/0`, `cids/1` for an
|
||||
// array. meta-core's reverse-index hook only matches `cids/<cid>`, so
|
||||
// those would be stored and never indexed.
|
||||
assert.equal(flat['cids/0'], undefined);
|
||||
assert.equal(flat['cids/1'], undefined);
|
||||
});
|
||||
|
||||
it('never emits a deprecated cid_* or midhash256 field', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
// meta-core returns 400 for any of these now — a record that carried one
|
||||
// was stored but unindexed.
|
||||
for (const field of Object.keys(flat)) {
|
||||
assert.ok(
|
||||
!field.startsWith('cid_'),
|
||||
`deprecated per-algorithm field reached the record: ${field}`
|
||||
);
|
||||
assert.notEqual(field, 'midhash256');
|
||||
assert.notEqual(field, 'canonical_cid');
|
||||
}
|
||||
});
|
||||
|
||||
it('does not store hashId as a property of its own record', () => {
|
||||
const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
|
||||
// hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
|
||||
assert.equal(flat['hashId'], undefined);
|
||||
});
|
||||
|
||||
it('still flattens ordinary nested metadata', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
|
||||
});
|
||||
|
||||
assert.equal(flat['title'], 'Inception');
|
||||
assert.equal(flat['fileinfo/duration'], '8878.5');
|
||||
assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
|
||||
});
|
||||
|
||||
it('honours excludeFields', () => {
|
||||
const flat = buildRecordFields(
|
||||
{hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
|
||||
['processingStatus']
|
||||
);
|
||||
assert.equal(flat['processingStatus'], undefined);
|
||||
assert.equal(flat['title'], 'x');
|
||||
});
|
||||
|
||||
it('tolerates a record with no cids at all', () => {
|
||||
const flat = buildRecordFields({title: 'x'});
|
||||
assert.equal(flat['title'], 'x');
|
||||
assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
|
||||
* // { key: "/file/abc123/video/codec", value: "h265" }
|
||||
* // ]
|
||||
*/
|
||||
/**
|
||||
* Fields that never go onto a record under their own name.
|
||||
*
|
||||
* `cids` is an **array of bare CIDs** and must become the key-set
|
||||
* `cids/<cid> = "true"`. If it fell through the generic flattener below it would
|
||||
* emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
|
||||
* reverse-index hook would not recognise, silently leaving the record
|
||||
* unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
|
||||
*
|
||||
* `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
|
||||
* storing it as a property inside itself is redundant.
|
||||
*/
|
||||
const STRUCTURAL_FIELDS = ['cids', 'hashId'];
|
||||
|
||||
/**
|
||||
* Turn an in-memory `FileMetadata` into the flat `field → value` map that a
|
||||
* record is made of.
|
||||
*
|
||||
* **This is the single write-shape boundary.** Both writers go through it — the
|
||||
* meta-core HTTP writer and the direct-Redis fallback — because they used not
|
||||
* to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
|
||||
* with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
|
||||
* meta-core never indexed. Records looked fine and could not be found by their
|
||||
* own midhash. One function, one shape, no second path to forget.
|
||||
*
|
||||
* Returns field names relative to the record root (no `/file/{hashId}` prefix).
|
||||
*/
|
||||
export function buildRecordFields(
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Record<string, string> {
|
||||
const prefix = 'r'; // throwaway root; stripped below
|
||||
const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
|
||||
if (field) {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
|
||||
// Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
|
||||
// The midhash is just the member whose multicodec is 0x1000 — there is no
|
||||
// `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
|
||||
const cids: unknown = metadata?.cids;
|
||||
if (Array.isArray(cids)) {
|
||||
for (const cid of cids) {
|
||||
if (typeof cid === 'string' && cid) {
|
||||
flat[`cids/${cid}`] = 'true';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return flat;
|
||||
}
|
||||
|
||||
export function flattenMetadata(
|
||||
metadata: any,
|
||||
prefix: string,
|
||||
|
||||
@@ -16,11 +16,12 @@ import * as IORedis from 'ioredis';
|
||||
import * as os from 'os';
|
||||
import type { IKVClient, KeyValuePair } from './IKVClient.js';
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
reconstructMetadata,
|
||||
buildFilePrefix,
|
||||
buildPropertyKey,
|
||||
} from './MetadataUtils.js';
|
||||
import { MetaCoreApiWriter, type FileTuplesResult } from './MetaCoreApiWriter.js';
|
||||
|
||||
// ESM/CJS interop for ioredis
|
||||
const Redis = (IORedis as any).default ?? IORedis;
|
||||
@@ -50,6 +51,18 @@ interface RedisClientConfig {
|
||||
url: string;
|
||||
timeout?: number;
|
||||
prefix?: string;
|
||||
|
||||
/**
|
||||
* meta-core HTTP API base URL. When set, write methods
|
||||
* (setMetadataFlat / setMetadataProperty / deleteMetadataFlat) route
|
||||
* through the /meta/{hash} HTTP family instead of writing to Redis
|
||||
* directly. Reads and stream consumption still use the Redis connection.
|
||||
*
|
||||
* Wiring this is the meta-sort half of the api-mediated-access lockdown
|
||||
* (see meta-core docs/api-mediated-access.md, PR B). Leaving it unset
|
||||
* preserves the legacy direct-write path for tests / one-off scripts.
|
||||
*/
|
||||
metaCoreApiUrl?: string;
|
||||
}
|
||||
|
||||
export class RedisKVClient implements IKVClient {
|
||||
@@ -58,6 +71,7 @@ export class RedisKVClient implements IKVClient {
|
||||
private timeout: number;
|
||||
private prefix: string;
|
||||
private isConnected: boolean = false;
|
||||
private writer: MetaCoreApiWriter | null = null;
|
||||
|
||||
// Stream consumer state
|
||||
private streamConsumerRunning = false;
|
||||
@@ -68,9 +82,47 @@ export class RedisKVClient implements IKVClient {
|
||||
this.timeout = config.timeout || 30000;
|
||||
this.prefix = config.prefix || '';
|
||||
|
||||
if (config.metaCoreApiUrl) {
|
||||
this.writer = new MetaCoreApiWriter({
|
||||
apiUrl: config.metaCoreApiUrl,
|
||||
timeoutMs: this.timeout,
|
||||
});
|
||||
console.log(`[Redis] Writes routed through meta-core API at ${config.metaCoreApiUrl}`);
|
||||
}
|
||||
|
||||
// Unique consumer name for this instance
|
||||
this.consumerName = `${os.hostname()}-${process.pid}`;
|
||||
|
||||
// HTTP-only mode: no Redis URL AND a writer is configured. Create a
|
||||
// disconnected stub Redis so accidental fallthrough into a Redis
|
||||
// method throws immediately instead of silently connecting to
|
||||
// localhost.
|
||||
if (!config.url && this.writer) {
|
||||
// ioredis requires a config object; lazyConnect prevents the
|
||||
// automatic connect. Any caller that tries to use the Redis
|
||||
// path (e.g. unmigrated stream code) will fail loudly rather
|
||||
// than silently misbehaving.
|
||||
this.redis = new Redis({
|
||||
host: '127.0.0.1',
|
||||
port: 0,
|
||||
lazyConnect: true,
|
||||
enableOfflineQueue: false,
|
||||
// Never reconnect. A stub that retries turns one stray call into
|
||||
// an endless "Unhandled error event: ECONNREFUSED" loop.
|
||||
retryStrategy: () => null,
|
||||
maxRetriesPerRequest: 0,
|
||||
});
|
||||
let stubWarned = false;
|
||||
this.redis.on('error', (err: Error) => {
|
||||
if (stubWarned) return;
|
||||
stubWarned = true;
|
||||
console.warn(`[Redis] HTTP-only stub was used and refused (${err.message}) — a caller still reaches for direct Redis`);
|
||||
});
|
||||
this.isConnected = false;
|
||||
console.log('[Redis] HTTP-only mode — no direct Redis connection');
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse Redis URL and create client
|
||||
this.redis = new Redis(config.url, {
|
||||
commandTimeout: this.timeout,
|
||||
@@ -102,10 +154,22 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to Redis (call before using other methods)
|
||||
* Connect to Redis (call before using other methods).
|
||||
*
|
||||
* In HTTP-only mode (no Redis URL + writer configured), connect is a
|
||||
* no-op — there's no underlying socket to open.
|
||||
*/
|
||||
async connect(): Promise<void> {
|
||||
if (this.isConnected) return;
|
||||
if (this.writer && !this.redis.options.host) {
|
||||
// HTTP-only mode stub — nothing to connect.
|
||||
return;
|
||||
}
|
||||
// Skip explicit connect when host is 127.0.0.1 with port 0 (HTTP-only stub).
|
||||
const opts = this.redis.options as { host?: string; port?: number };
|
||||
if (opts && opts.port === 0) {
|
||||
return;
|
||||
}
|
||||
await this.redis.connect();
|
||||
}
|
||||
|
||||
@@ -163,6 +227,11 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
if (this.writer) {
|
||||
// Probe meta-core /health rather than Redis PING; that's the
|
||||
// surface the application actually depends on after the lockdown.
|
||||
return this.writer.health();
|
||||
}
|
||||
try {
|
||||
const result = await this.redis.ping();
|
||||
return result === 'PONG';
|
||||
@@ -271,24 +340,30 @@ export class RedisKVClient implements IKVClient {
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
if (this.writer) {
|
||||
// PATCH /meta/{hash} — meta-core handles the flat-key fanout and
|
||||
// the file:__index__ membership atomically server-side.
|
||||
return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
|
||||
}
|
||||
|
||||
if (pairs.length === 0) return;
|
||||
// Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
|
||||
//
|
||||
// This path used to flatten the metadata itself, which meant it skipped
|
||||
// the `cid_* → cids/` transform that only the HTTP writer applied: it
|
||||
// wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
|
||||
// reverse-index hook only registers aliases for fields named `cids/<cid>`,
|
||||
// so those records were stored but never indexed — unresolvable by their
|
||||
// own midhash, with nothing logged. Both paths now build the record
|
||||
// through the same `buildRecordFields`, so the key-set cannot be skipped.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
// Use pipeline for efficient batch SET operations
|
||||
const pipeline = this.redis.pipeline();
|
||||
|
||||
for (const pair of pairs) {
|
||||
// Extract field name by removing the prefix
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
|
||||
if (field) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
const redisKey = this.buildKey(`file:${hashId}/${field}`);
|
||||
pipeline.set(redisKey, pair.value);
|
||||
}
|
||||
for (const [field, value] of Object.entries(flat)) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
|
||||
}
|
||||
|
||||
await pipeline.exec();
|
||||
|
||||
// Maintain index set for getAllHashIds
|
||||
@@ -296,6 +371,20 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getMetadataFlat(hashId: string): Promise<any | null> {
|
||||
if (this.writer) {
|
||||
// GET /meta/{hash} via HTTP — returns the flat map directly,
|
||||
// so we don't need a Redis SCAN/MGET round trip. Reconstruct
|
||||
// the nested structure consumers expect.
|
||||
const flat = await this.writer.getMetadataFlat(hashId);
|
||||
if (!flat) return null;
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs: KeyValuePair[] = Object.entries(flat).map(([field, value]) => ({
|
||||
key: `${prefix}/${field}`,
|
||||
value: value as string,
|
||||
}));
|
||||
return reconstructMetadata(pairs, prefix);
|
||||
}
|
||||
|
||||
const keyPrefix = this.buildKey(`file:${hashId}/`);
|
||||
|
||||
// Scan for all keys with this prefix
|
||||
@@ -329,6 +418,13 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
|
||||
if (this.writer) {
|
||||
// GET /meta/{hash}/{prop} via HTTP; meta-core returns text/plain
|
||||
// and the writer mirrors the JSON-or-raw fallback the Redis path
|
||||
// uses below.
|
||||
return this.writer.getMetadata(hashId, propertyPath);
|
||||
}
|
||||
|
||||
const key = this.buildKey(`file:${hashId}/${propertyPath}`);
|
||||
const value = await this.redis.get(key);
|
||||
|
||||
@@ -346,6 +442,10 @@ export class RedisKVClient implements IKVClient {
|
||||
* Writes to flat key: file:{hashId}/{property}
|
||||
*/
|
||||
async setMetadataProperty(hashId: string, property: string, value: string): Promise<void> {
|
||||
if (this.writer) {
|
||||
return this.writer.setMetadataProperty(hashId, property, value);
|
||||
}
|
||||
|
||||
const key = this.buildKey(`file:${hashId}/${property}`);
|
||||
await this.redis.set(key, value);
|
||||
|
||||
@@ -354,6 +454,10 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async deleteMetadataFlat(hashId: string): Promise<number> {
|
||||
if (this.writer) {
|
||||
return this.writer.deleteMetadataFlat(hashId);
|
||||
}
|
||||
|
||||
const keyPrefix = this.buildKey(`file:${hashId}/`);
|
||||
|
||||
// Scan and delete all keys with this prefix
|
||||
@@ -378,10 +482,22 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getAllHashIds(): Promise<string[]> {
|
||||
if (this.writer) {
|
||||
return this.writer.getAllHashIds();
|
||||
}
|
||||
const indexKey = this.buildKey('file:__index__');
|
||||
return await this.redis.smembers(indexKey);
|
||||
}
|
||||
|
||||
/**
|
||||
* File-backed roots from meta-core (GET /api/files/tuples). Only the HTTP
|
||||
* writer can serve this; direct-Redis mode returns null so callers take
|
||||
* their degraded path instead of sweeping every record.
|
||||
*/
|
||||
async getFileTuples(opts: { summary?: boolean } = {}): Promise<FileTuplesResult | null> {
|
||||
return this.writer ? this.writer.getFileTuples(opts) : null;
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Pub/Sub Operations
|
||||
// ========================================================================
|
||||
|
||||
@@ -1,199 +0,0 @@
|
||||
/**
|
||||
* Service Registration using shared filesystem
|
||||
*
|
||||
* Each service registers itself in /meta-core/services/{service-name}-{hostname}.json
|
||||
* with its base URL, status, and heartbeat timestamp.
|
||||
*
|
||||
* Service discovery is centralized in meta-core - this class only handles registration.
|
||||
* For service discovery, use the /api/services endpoint (proxied to meta-core via nginx).
|
||||
*
|
||||
* Each service must call start() to register itself and begin heartbeat updates.
|
||||
* meta-core runs a cleanup process that removes services with stale heartbeats.
|
||||
*/
|
||||
|
||||
import { promises as fs } from 'fs';
|
||||
import { hostname } from 'os';
|
||||
import type { ServiceInfo } from './IKVClient.js';
|
||||
import { isNodeError } from '../types/ExtendedInterfaces.js';
|
||||
|
||||
interface ServiceRegistrationConfig {
|
||||
/** Path to META_CORE_VOLUME (e.g., /meta-core) */
|
||||
metaCorePath: string;
|
||||
|
||||
/** Service name */
|
||||
serviceName: string;
|
||||
|
||||
/** Service version (optional, not used in simplified format) */
|
||||
version?: string;
|
||||
|
||||
/** Base URL for the service */
|
||||
apiUrl: string;
|
||||
|
||||
/** Heartbeat interval in ms (default: 30000) */
|
||||
heartbeatInterval?: number;
|
||||
|
||||
/** Stale threshold in ms (default: 60000) */
|
||||
staleThreshold?: number;
|
||||
}
|
||||
|
||||
export class ServiceRegistration {
|
||||
private config: ServiceRegistrationConfig;
|
||||
private servicesDir: string;
|
||||
private serviceFilePath: string;
|
||||
private heartbeatTimer: NodeJS.Timeout | null = null;
|
||||
private isShuttingDown = false;
|
||||
private currentHostname: string;
|
||||
|
||||
constructor(config: ServiceRegistrationConfig) {
|
||||
this.config = {
|
||||
heartbeatInterval: 30000,
|
||||
staleThreshold: 60000,
|
||||
...config
|
||||
};
|
||||
|
||||
this.currentHostname = hostname();
|
||||
this.servicesDir = `${this.config.metaCorePath}/services`;
|
||||
// Use hostname-based file naming
|
||||
this.serviceFilePath = `${this.servicesDir}/${this.config.serviceName}-${this.currentHostname}.json`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Ensure services directory exists
|
||||
*/
|
||||
private async ensureServicesDir(): Promise<void> {
|
||||
await fs.mkdir(this.servicesDir, { recursive: true });
|
||||
}
|
||||
|
||||
/**
|
||||
* Build service info for this service
|
||||
*/
|
||||
private buildServiceInfo(status: ServiceInfo['status']): ServiceInfo {
|
||||
return {
|
||||
name: this.config.serviceName,
|
||||
hostname: this.currentHostname,
|
||||
baseUrl: this.config.apiUrl,
|
||||
status,
|
||||
lastHeartbeat: new Date().toISOString()
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Register this service by writing service info to the shared filesystem.
|
||||
* meta-core's cleanup process monitors these files and removes stale entries.
|
||||
*/
|
||||
async register(): Promise<void> {
|
||||
await this.ensureServicesDir();
|
||||
|
||||
const info = this.buildServiceInfo('running');
|
||||
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
|
||||
|
||||
console.log(`[ServiceRegistration] Registered ${this.config.serviceName}-${this.currentHostname}`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Update service status
|
||||
*/
|
||||
async updateStatus(status: ServiceInfo['status']): Promise<void> {
|
||||
try {
|
||||
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
|
||||
const info = JSON.parse(content) as ServiceInfo;
|
||||
|
||||
info.status = status;
|
||||
info.lastHeartbeat = new Date().toISOString();
|
||||
|
||||
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
|
||||
} catch (error) {
|
||||
console.error('[ServiceRegistration] Failed to update status:', error);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Send heartbeat (update lastHeartbeat timestamp)
|
||||
*/
|
||||
async heartbeat(): Promise<void> {
|
||||
try {
|
||||
const content = await fs.readFile(this.serviceFilePath, 'utf-8');
|
||||
const info = JSON.parse(content) as ServiceInfo;
|
||||
|
||||
info.lastHeartbeat = new Date().toISOString();
|
||||
|
||||
await fs.writeFile(this.serviceFilePath, JSON.stringify(info, null, 2));
|
||||
} catch (error) {
|
||||
// If file was deleted, re-register
|
||||
if (isNodeError(error) && error.code === 'ENOENT') {
|
||||
await this.register();
|
||||
} else {
|
||||
console.error('[ServiceRegistration] Heartbeat failed:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Start heartbeat loop
|
||||
*/
|
||||
startHeartbeat(): void {
|
||||
this.heartbeatTimer = setInterval(async () => {
|
||||
if (this.isShuttingDown) return;
|
||||
await this.heartbeat();
|
||||
}, this.config.heartbeatInterval);
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop heartbeat loop
|
||||
*/
|
||||
stopHeartbeat(): void {
|
||||
if (this.heartbeatTimer) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Unregister this service (on shutdown)
|
||||
*/
|
||||
async unregister(): Promise<void> {
|
||||
this.isShuttingDown = true;
|
||||
this.stopHeartbeat();
|
||||
|
||||
try {
|
||||
// Remove the service file
|
||||
await fs.unlink(this.serviceFilePath);
|
||||
console.log(`[ServiceRegistration] Unregistered ${this.config.serviceName}-${this.currentHostname}`);
|
||||
} catch (error) {
|
||||
if (!isNodeError(error) || error.code !== 'ENOENT') {
|
||||
console.error('[ServiceRegistration] Failed to unregister:', error);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Lifecycle
|
||||
// ========================================================================
|
||||
|
||||
/**
|
||||
* Full startup sequence: register service and start heartbeat loop.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
await this.register();
|
||||
this.startHeartbeat();
|
||||
}
|
||||
|
||||
/**
|
||||
* Full shutdown sequence
|
||||
*/
|
||||
async stop(): Promise<void> {
|
||||
await this.unregister();
|
||||
}
|
||||
|
||||
// ========================================================================
|
||||
// Getters
|
||||
// ========================================================================
|
||||
|
||||
getServiceFilePath(): string {
|
||||
return this.serviceFilePath;
|
||||
}
|
||||
|
||||
getServicesDir(): string {
|
||||
return this.servicesDir;
|
||||
}
|
||||
}
|
||||
@@ -5,7 +5,8 @@
|
||||
* - IKVClient interface for KV storage abstraction
|
||||
* - RedisKVClient for Redis-based storage
|
||||
* - LeaderClient for reading leader info from meta-core
|
||||
* - ServiceRegistration for service registration (discovery is centralized in meta-core)
|
||||
* Services announce and discover over UDP (beacon v2); there is no
|
||||
* separate registration step, so no ServiceRegistration export.
|
||||
* - KVManager for unified management of all components
|
||||
* - MetadataUtils for flattening/reconstructing nested metadata
|
||||
*/
|
||||
@@ -13,6 +14,5 @@
|
||||
export * from './IKVClient.js';
|
||||
export * from './RedisClient.js';
|
||||
export * from './LeaderClient.js';
|
||||
export * from './ServiceRegistration.js';
|
||||
export * from './KVManager.js';
|
||||
export * from './MetadataUtils.js';
|
||||
|
||||
@@ -85,13 +85,10 @@ export class JellyfinFormatHandler implements IMetadataFormatHandler {
|
||||
if (metadata.originalTitle) jellyfinFields.originalTitle = metadata.originalTitle;
|
||||
if (metadata.movieYear) jellyfinFields.year = metadata.movieYear;
|
||||
|
||||
// Titles (first title or original title)
|
||||
if (metadata.titles) {
|
||||
const titleEntries = Object.entries(metadata.titles);
|
||||
if (titleEntries.length > 0) {
|
||||
jellyfinFields.title = titleEntries[0][1];
|
||||
}
|
||||
}
|
||||
// Title: the display name. `titles/*` is an unordered name set with no
|
||||
// preferred member, so it is never the NFO title (METADATA_KEYS.md §3).
|
||||
const title = metadata.title || metadata.originalTitle;
|
||||
if (title) jellyfinFields.title = title;
|
||||
|
||||
// IDs
|
||||
if (metadata.imdbid) jellyfinFields.imdbid = metadata.imdbid;
|
||||
|
||||
@@ -195,6 +195,16 @@ export class UnifiedProcessingStateManager {
|
||||
performanceMetrics.recordHashProcessing(hashProcessingTime);
|
||||
}
|
||||
|
||||
// File-level metric, one per file that finished cleanly. Nothing recorded
|
||||
// it before, so the monitor's "Processed" tile stayed at 0. Files announced
|
||||
// over SSE are never marked discovered, so fall back to the light phase.
|
||||
if (!error) {
|
||||
const fileTime = totalProcessingTime
|
||||
?? (hashState?.lightProcessingStartedAt !== undefined ? now - hashState.lightProcessingStartedAt : hashProcessingTime)
|
||||
?? 0;
|
||||
performanceMetrics.recordFileProcessing(filePath, fileTime);
|
||||
}
|
||||
|
||||
const doneState: UnifiedFileState = {
|
||||
filePath,
|
||||
state: 'done',
|
||||
|
||||
@@ -106,7 +106,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
// Map StreamingPipeline queues to the 3-queue architecture UI expects
|
||||
// PQueue: .pending = currently running, .size = waiting in queue
|
||||
// API expects: .pending = running, .size = waiting
|
||||
const result = {
|
||||
const result: any = {
|
||||
preProcessQueue: {
|
||||
// validation = pre-process (quick extension checks)
|
||||
size: queues.validation.size, // waiting in queue
|
||||
@@ -130,6 +130,15 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
// File-level counts (what users care about)
|
||||
files: queues.files
|
||||
};
|
||||
|
||||
// Layer in the per-file cache-slot view from the ContainerPluginScheduler
|
||||
// when available — this is the metric that reflects WebDAV/cache load
|
||||
// (number of distinct files concurrently being touched), not just task
|
||||
// throughput. Older deployments without container plugins simply omit it.
|
||||
const containerStatus = this.taskScheduler?.getContainerQueueStatus?.() ?? null;
|
||||
if (containerStatus?.cacheSlots) {
|
||||
result.cacheSlots = containerStatus.cacheSlots;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -273,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
);
|
||||
|
||||
let midHash256: string;
|
||||
if (indexLine?.cid_midhash256) {
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
if (indexLine?.midhash256) {
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
const computeStart = performance.now();
|
||||
@@ -288,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
return;
|
||||
}
|
||||
midHash256 = hash;
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
@@ -296,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
filePath,
|
||||
fileStats.size,
|
||||
fileStats.mtime.toISOString(),
|
||||
{ cid_midhash256: midHash256 }
|
||||
{ midhash256: midHash256 }
|
||||
);
|
||||
}
|
||||
|
||||
// STEP 2: Initialize metadata with midhash256
|
||||
// STEP 2: Initialize metadata with midhash256.
|
||||
//
|
||||
// The midhash plays two roles and now says so: it is the record
|
||||
// address (`hashId`, the `/file/{hashId}` key) *and* one member of
|
||||
// the record's bare-CID key-set (`cids`). It used to be a single
|
||||
// `cid_midhash256` property, which conflated the two and leaked a
|
||||
// deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: any = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
filePath,
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
@@ -432,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
const metadata = db.get(filePath);
|
||||
|
||||
// Collect hashId before deletion (for Stremio notification)
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
removedHashIds.push(metadata.cid_midhash256);
|
||||
if (metadata && metadata.hashId) {
|
||||
removedHashIds.push(metadata.hashId);
|
||||
}
|
||||
|
||||
db.delete(filePath);
|
||||
|
||||
@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
let midHash256: string | undefined;
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
|
||||
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -296,19 +302,16 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
metadata.processingStatus = 'complete';
|
||||
this.database.set(filePath, metadata);
|
||||
|
||||
// STEP 5: Update metadata in KV with full hashes
|
||||
// STEP 5: Update metadata in KV.
|
||||
// Full hashes (sha2-256, sha1, …) are computed by the full-hash
|
||||
// container plugin and written to meta-core directly as cids/<cid>
|
||||
// members — they are not held in this in-memory metadata. The
|
||||
// midhash256 is the primary id; on a (rare) midhash collision the
|
||||
// file simply shares the record address until the plugin's
|
||||
// distinct cids/* members disambiguate it.
|
||||
if (this.kvClient) {
|
||||
try {
|
||||
// Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
|
||||
let hashId: string;
|
||||
if (collisionDetected && metadata['cid_sha2-256']) {
|
||||
// Collision: Use full SHA-256 as primary ID
|
||||
hashId = metadata['cid_sha2-256'];
|
||||
console.log(` Using SHA-256 as primary ID for collision: ${hashId}`);
|
||||
} else {
|
||||
// Normal: Use midhash256 as primary ID
|
||||
hashId = midHash256;
|
||||
}
|
||||
const hashId = midHash256;
|
||||
|
||||
if (hashId) {
|
||||
// Prepare metadata with filePath added
|
||||
@@ -412,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Check cache or compute locally
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 locally (requires file access)
|
||||
@@ -426,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -527,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
throw new Error('Metadata not found - light processing may have failed');
|
||||
}
|
||||
|
||||
const midHash256 = metadata.cid_midhash256;
|
||||
const midHash256 = metadata.hashId;
|
||||
if (!midHash256) {
|
||||
throw new Error('midhash256 not found in metadata');
|
||||
}
|
||||
@@ -625,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Delete from KV if configured (using nested key architecture)
|
||||
if (this.kvClient && metadata) {
|
||||
try {
|
||||
const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
|
||||
const hashId = metadata.hashId;
|
||||
|
||||
// Delete all properties for this file using nested key architecture
|
||||
if (hashId) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
|
||||
import {HashIndexManager} from "@metazla/meta-hash";
|
||||
import {config} from "../../config/EnvConfig.js";
|
||||
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
|
||||
@@ -16,19 +16,34 @@ class SimpleFileToCid{
|
||||
if (!stats || !stats.exists) {
|
||||
return null;
|
||||
}
|
||||
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
return indexLine[CID_ALGORITHM_NAMES.sha256];
|
||||
}else {
|
||||
let hashs = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashs);
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
|
||||
return hashs[CID_ALGORITHM_NAMES.sha256];
|
||||
const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if (indexLine) {
|
||||
// The on-disk index keeps per-algorithm columns, keyed by bare
|
||||
// algorithm name (not the removed `cid_<algo>` record fields).
|
||||
return indexLine['sha256'] ?? null;
|
||||
}
|
||||
|
||||
// The computer returns bare CIDs; the algorithm is recovered from the
|
||||
// multicodec, never from a field name.
|
||||
const hashes: MultiHashData = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashes);
|
||||
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of targetHash) {
|
||||
const cid = pickCidByAlgorithm(hashes.cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
|
||||
return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
|
||||
@@ -166,14 +166,14 @@ export class StreamingPipeline {
|
||||
|
||||
// After light processing, dispatch to plugins and queue background tasks
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
if (metadata && metadata.hashId) {
|
||||
// Notify Stremio addon to add video
|
||||
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
|
||||
this.notifyStremioAdd(metadata.hashId).catch(err => {
|
||||
// Silent fail - Stremio addon may not be running
|
||||
});
|
||||
|
||||
// Dispatch tasks to container plugins (fire and forget)
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
|
||||
|
||||
// Stage 3: Queue for background processing (SHA-256 computation)
|
||||
this.backgroundQueue.add(() => this.processHashPhase(filePath))
|
||||
@@ -246,7 +246,7 @@ export class StreamingPipeline {
|
||||
// Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
|
||||
if (!this.config.containerPluginScheduler) {
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
const hashId = metadata?.cid_midhash256;
|
||||
const hashId = metadata?.hashId;
|
||||
let virtualPath: string | undefined;
|
||||
try {
|
||||
virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;
|
||||
|
||||
@@ -220,6 +220,17 @@ export class PluginManager extends EventEmitter {
|
||||
return;
|
||||
}
|
||||
|
||||
// Forget container-backed plugins that are gone (an external plugin
|
||||
// removed from the Plugins page); they have no local dir.
|
||||
const present = new Set(status.plugins.map((p) => p.pluginId));
|
||||
for (const [id, loaded] of this.loadedPlugins) {
|
||||
if (loaded.pluginDir === '' && !present.has(id)) {
|
||||
this.loadedPlugins.delete(id);
|
||||
this.manifests.delete(id);
|
||||
this.activePluginIds.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
// Load state for plugin active status
|
||||
const state = await this.loadState();
|
||||
|
||||
|
||||
@@ -584,6 +584,16 @@ export class TaskScheduler extends EventEmitter {
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass-through to ContainerPluginScheduler.getQueueStatus so callers like
|
||||
* WatchedFileProcessor can surface the file-cache-slot view (capacity,
|
||||
* holders, current plugin per file) without reaching past the scheduler
|
||||
* boundary themselves.
|
||||
*/
|
||||
getContainerQueueStatus(): ReturnType<ContainerPluginScheduler['getQueueStatus']> | null {
|
||||
return this.containerPluginScheduler?.getQueueStatus() ?? null;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Shutdown
|
||||
// =========================================================================
|
||||
|
||||
@@ -8,6 +8,7 @@
|
||||
|
||||
import type { HashMeta } from '@metazla/meta-interface';
|
||||
import type { IKVClient } from '../kv/IKVClient.js';
|
||||
import type { FileTuplesResult } from '../kv/MetaCoreApiWriter.js';
|
||||
import type { FileAnalyzerInterface } from '../logic/fileProcessor/FileAnalyzerInterface.js';
|
||||
import type { ContainerTask } from '../container-plugins/types.js';
|
||||
import type { UnifiedProcessingSnapshot } from '../logic/UnifiedProcessingStateManager.js';
|
||||
@@ -41,6 +42,21 @@ export function hasPublish(client: IKVClient): client is IKVClientWithPubSub {
|
||||
return typeof (client as IKVClientWithPubSub).publish === 'function';
|
||||
}
|
||||
|
||||
/**
|
||||
* IKVClient that can list meta-core's file-backed roots in one call
|
||||
* (GET /api/files/tuples). Resolves null when meta-core predates the endpoint.
|
||||
*/
|
||||
export interface IKVClientWithFileTuples extends IKVClient {
|
||||
getFileTuples(opts?: { summary?: boolean }): Promise<FileTuplesResult | null>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type guard to check if a KVClient can list file-backed roots
|
||||
*/
|
||||
export function hasFileTuples(client: IKVClient): client is IKVClientWithFileTuples {
|
||||
return typeof (client as IKVClientWithFileTuples).getFileTuples === 'function';
|
||||
}
|
||||
|
||||
/**
|
||||
* IKVClient with Redis Streams capabilities
|
||||
*
|
||||
@@ -102,13 +118,18 @@ export interface ExtendedHashMeta extends HashMeta {
|
||||
*/
|
||||
export interface IndexManager {
|
||||
/**
|
||||
* Get cached CID for a file based on path, size, and mtime
|
||||
* Get cached CID for a file based on path, size, and mtime.
|
||||
*
|
||||
* Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
|
||||
* by bare **algorithm** (`midhash256`, `sha256`), not by the removed
|
||||
* `cid_<algo>` record-field names — the local index and the record shape are
|
||||
* deliberately separate concerns now. See METADATA_KEYS.md §14.13.
|
||||
*/
|
||||
getCidForFile(
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string
|
||||
): { cid_midhash256?: string } | undefined;
|
||||
): { midhash256?: string } | undefined;
|
||||
|
||||
/**
|
||||
* Add a file CID to the cache
|
||||
@@ -117,7 +138,7 @@ export interface IndexManager {
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string,
|
||||
cid: { cid_midhash256: string }
|
||||
cid: { midhash256: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,4 +18,28 @@ export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
|
||||
export interface FileMetadata extends HashMeta {
|
||||
/** Current status of file processing */
|
||||
processingStatus?: ProcessingStatus;
|
||||
|
||||
/**
|
||||
* The record address: the file's midhash256 CID, computed in the light phase
|
||||
* and used as the `hashId` throughout meta-sort and as the key of the
|
||||
* `/file/{hashId}` record.
|
||||
*
|
||||
* This replaces the old `cid_midhash256` field. The rename is the point: that
|
||||
* name was a *metadata property* name, and it leaked — a `cid_*` field written
|
||||
* to a record is stored but never reverse-indexed by meta-core, leaving the
|
||||
* record unresolvable by its own CID. meta-core now rejects such fields with a
|
||||
* 400. See METADATA_KEYS.md §1/§14.13.
|
||||
*/
|
||||
hashId?: string;
|
||||
|
||||
/**
|
||||
* Every digest known for this file, as **bare CIDv1 strings** — the midhash
|
||||
* included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
|
||||
* shape meta-core indexes.
|
||||
*
|
||||
* A CIDv1 is self-describing, so a specific digest is recovered by decoding
|
||||
* the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
|
||||
* field name.
|
||||
*/
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
@@ -365,27 +365,115 @@ function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): b
|
||||
return false;
|
||||
}
|
||||
|
||||
// Human label per multihash code / CID codec. Source of truth for codes:
|
||||
// meta-hash MultiHashData + METADATA_KEYS.md §2.
|
||||
const CID_LABELS: Record<number, string> = {
|
||||
0x12: 'SHA-256',
|
||||
0x16: 'SHA3-256',
|
||||
0x15: 'SHA3-384',
|
||||
0x11: 'SHA-1',
|
||||
0xd5: 'MD5',
|
||||
0x132: 'CRC32',
|
||||
0x1000: 'MidHash256',
|
||||
0xb702: 'BT Pieces Root',
|
||||
0x70: 'IPFS (dag-pb)',
|
||||
0x10b7: 'BitTorrent v2',
|
||||
0x1001: 'BitTorrent v1 (file)',
|
||||
0x1002: 'BitTorrent v2 (file)',
|
||||
0x1005: 'Usenet release',
|
||||
0x1006: 'URL',
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract all hash values (cid_*) from metadata
|
||||
* Display order for a record's CIDs. This mirrors the canonical rank ladder
|
||||
* (`packages/meta-core/internal/cid/rank.go`, the two `cid_rank.rs`, and the
|
||||
* `/cid-rank-vectors.json` golden fixture) — same ordering, scaled down.
|
||||
*
|
||||
* It used to be its own invented ladder (5/4/3/2) with no dag-pb entry at all,
|
||||
* which meant an IPFS root fell through to 0 and sorted *below* MD5. Display-only,
|
||||
* so it never broke anything — but it was a seventh, silently divergent copy of a
|
||||
* rule that already has too many. Keep it in step: a code with no entry sorts last.
|
||||
*/
|
||||
const RANK_BY_LABEL: Record<string, number> = {
|
||||
'IPFS (dag-pb)': 6,
|
||||
'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5,
|
||||
'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4,
|
||||
MidHash256: 3,
|
||||
'Usenet release': 2, URL: 2,
|
||||
'SHA-1': 1, MD5: 1, CRC32: 1,
|
||||
};
|
||||
|
||||
const B32 = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
|
||||
function decodeBase32(s: string): Uint8Array | null {
|
||||
let bits = 0, value = 0;
|
||||
const out: number[] = [];
|
||||
for (const ch of s) {
|
||||
const idx = B32.indexOf(ch);
|
||||
if (idx < 0) return null;
|
||||
value = (value << 5) | idx;
|
||||
bits += 5;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out.push((value >> bits) & 0xff);
|
||||
}
|
||||
}
|
||||
return new Uint8Array(out);
|
||||
}
|
||||
|
||||
function readUvarint(buf: Uint8Array, pos: number): [number, number] {
|
||||
let result = 0, shift = 0, p = pos;
|
||||
while (p < buf.length) {
|
||||
const b = buf[p++];
|
||||
result |= (b & 0x7f) << shift;
|
||||
if ((b & 0x80) === 0) return [result >>> 0, p];
|
||||
shift += 7;
|
||||
}
|
||||
return [0, pos];
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a bare multibase-base32 CIDv1 into [codec, multihashCode], or null.
|
||||
* The algorithm is recovered from these — sibling CIDs are stored as a
|
||||
* key-set (cids/<cid>) with no per-algorithm field name.
|
||||
*/
|
||||
function decodeCid(cid: string): { codec: number; mh: number } | null {
|
||||
if (!cid.startsWith('b')) return null;
|
||||
const bytes = decodeBase32(cid.slice(1));
|
||||
if (!bytes || bytes.length < 3) return null;
|
||||
let v: number, pos = 0;
|
||||
[v, pos] = readUvarint(bytes, pos);
|
||||
if (v !== 1) return null;
|
||||
let codec: number;
|
||||
[codec, pos] = readUvarint(bytes, pos);
|
||||
let mh: number;
|
||||
[mh, pos] = readUvarint(bytes, pos);
|
||||
return { codec, mh };
|
||||
}
|
||||
|
||||
function cidLabel(cid: string): string {
|
||||
const d = decodeCid(cid);
|
||||
if (!d) return 'CID';
|
||||
// Per-file BitTorrent codecs are carried on the CID codec, not the mh code.
|
||||
return CID_LABELS[d.codec] ?? CID_LABELS[d.mh] ?? 'CID';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all sibling CIDs from the record's bare-CID key-set (cids/<cid>),
|
||||
* labelling each by its multicodec. Replaces the old per-algorithm cid_*
|
||||
* fields. See METADATA_KEYS.md §2/§14.13.
|
||||
*/
|
||||
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
|
||||
const hashOrder = [
|
||||
{ key: 'cid_sha2-256', label: 'SHA-256' },
|
||||
{ key: 'cid_sha3-256', label: 'SHA3-256' },
|
||||
{ key: 'cid_sha3-384', label: 'SHA3-384' },
|
||||
{ key: 'cid_sha1', label: 'SHA-1' },
|
||||
{ key: 'cid_md5', label: 'MD5' },
|
||||
{ key: 'cid_crc32', label: 'CRC32' },
|
||||
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
|
||||
];
|
||||
|
||||
return hashOrder
|
||||
.filter(({ key }) => (metadata as any)[key])
|
||||
.map(({ key, label }) => ({
|
||||
key,
|
||||
label,
|
||||
value: (metadata as any)[key],
|
||||
}));
|
||||
const out: { key: string; label: string; value: string }[] = [];
|
||||
for (const key of Object.keys(metadata as any)) {
|
||||
if (!key.startsWith('cids/')) continue;
|
||||
const cid = key.slice('cids/'.length);
|
||||
if (!cid) continue;
|
||||
out.push({ key, label: cidLabel(cid), value: cid });
|
||||
}
|
||||
// Most useful first (sha/ipfs > btih > midhash > weak digests), then by label.
|
||||
out.sort((a, b) => (RANK_BY_LABEL[b.label] ?? 0) - (RANK_BY_LABEL[a.label] ?? 0) || a.label.localeCompare(b.label));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -9,7 +9,7 @@ export interface FileMetadata {
|
||||
|
||||
// Filename parsing
|
||||
title?: string;
|
||||
titles?: string[];
|
||||
titles?: Record<string, Record<string, true | string>>; // key-set: lang3 → {name: "true"}
|
||||
originaltitle?: string;
|
||||
showtitle?: string;
|
||||
season?: number;
|
||||
@@ -41,13 +41,10 @@ export interface FileMetadata {
|
||||
collectionnumber?: number;
|
||||
criticrating?: number;
|
||||
|
||||
// Hashes
|
||||
cid_crc32?: string;
|
||||
cid_md5?: string;
|
||||
cid_sha1?: string;
|
||||
'cid_sha2-256'?: string;
|
||||
'cid_sha3-256'?: string;
|
||||
cid_sha3_384?: string;
|
||||
// Sibling CIDs are the bare-CID key-set `cids/<cid> = "true"` (dynamic
|
||||
// keys), not per-algorithm fields. Accessed via prefix-scan; the algorithm
|
||||
// is recovered from each CID's multicodec. See METADATA_KEYS.md §2/§14.13.
|
||||
[cidsKey: `cids/${string}`]: string | undefined;
|
||||
|
||||
// Video info
|
||||
video?: {
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
import { useEffect, useRef } from 'react';
|
||||
import '../meta-beacon-scan.js';
|
||||
|
||||
/**
|
||||
* The Plugins page's beacon v2 Scan card: enrichment plugins advertising on
|
||||
* the network (`metamesh.enrich/<id>`, typically MetaPlugin* store apps), with
|
||||
* an Add per new one. A thin wrapper around the framework-free
|
||||
* <meta-beacon-scan> element shared with meta-share and meta-gateway; the
|
||||
* --mm-scan-* tokens map this dashboard's palette onto it.
|
||||
*
|
||||
* Add registers the plugin as an EXTERNAL plugin (POST /api/plugins/external):
|
||||
* meta-sort calls it by URL and never starts or stops it.
|
||||
*/
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line @typescript-eslint/no-namespace
|
||||
namespace JSX {
|
||||
interface IntrinsicElements {
|
||||
'meta-beacon-scan': React.DetailedHTMLProps<React.HTMLAttributes<HTMLElement>, HTMLElement> & {
|
||||
'scan-endpoint'?: string;
|
||||
'add-endpoint'?: string;
|
||||
'add-extra'?: string;
|
||||
label?: string;
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const TOKENS = {
|
||||
display: 'block',
|
||||
'--mm-scan-fg': 'var(--text-primary)',
|
||||
'--mm-scan-bg': 'var(--bg-secondary)',
|
||||
'--mm-scan-border': 'var(--border-color)',
|
||||
'--mm-scan-accent': 'var(--accent-primary)',
|
||||
'--mm-scan-err': 'var(--accent-secondary)',
|
||||
} as React.CSSProperties;
|
||||
|
||||
function BeaconScanCard({ onAdded }: { onAdded: () => void }) {
|
||||
const ref = useRef<HTMLElement>(null);
|
||||
|
||||
useEffect(() => {
|
||||
const el = ref.current;
|
||||
if (!el) return;
|
||||
const handler = () => onAdded();
|
||||
el.addEventListener('beacon-added', handler);
|
||||
return () => el.removeEventListener('beacon-added', handler);
|
||||
}, [onAdded]);
|
||||
|
||||
return (
|
||||
<meta-beacon-scan
|
||||
ref={ref}
|
||||
style={TOKENS}
|
||||
scan-endpoint="/api/plugins/discover"
|
||||
add-endpoint="/api/plugins/external"
|
||||
label="enrichment plugins"
|
||||
/>
|
||||
);
|
||||
}
|
||||
|
||||
export default BeaconScanCard;
|
||||
@@ -1,165 +1,54 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useEffect } from 'react';
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
|
||||
// of defining the custom element.
|
||||
import '../meta-service-menu.js';
|
||||
|
||||
interface ServiceInfo {
|
||||
name: string;
|
||||
hostname: string;
|
||||
baseUrl: string;
|
||||
status: string;
|
||||
lastHeartbeat: string;
|
||||
}
|
||||
/**
|
||||
* ServiceNav — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
|
||||
* a framework-free custom element shared by every meta-* UI (see
|
||||
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
|
||||
* that existed in four near-identical copies and polled meta-core's
|
||||
* /api/services; the element polls this service's own /api/neighbors instead,
|
||||
* so the nav still renders when meta-core is down.
|
||||
*
|
||||
* The element lives in shadow DOM and cannot see this app's CSS variables, so
|
||||
* the four --mm-nav-* tokens below map our palette onto it.
|
||||
*/
|
||||
|
||||
interface ServicesResponse {
|
||||
services: ServiceInfo[];
|
||||
current: string;
|
||||
}
|
||||
|
||||
const serviceIcons: Record<string, string> = {
|
||||
'meta-sort': '📁',
|
||||
'meta-fuse': '🗂️',
|
||||
'meta-stremio': '🎬',
|
||||
'meta-core': '⚙️',
|
||||
'meta-dup': '🔍',
|
||||
'meta-orbit': '🌐',
|
||||
'default': '📦'
|
||||
};
|
||||
|
||||
function formatServiceName(name: string): string {
|
||||
return name.split('-').map(word =>
|
||||
word.charAt(0).toUpperCase() + word.slice(1)
|
||||
).join(' ');
|
||||
declare global {
|
||||
// eslint-disable-next-line @typescript-eslint/no-namespace
|
||||
namespace JSX {
|
||||
interface IntrinsicElements {
|
||||
'meta-service-menu': React.DetailedHTMLProps<
|
||||
React.HTMLAttributes<HTMLElement>,
|
||||
HTMLElement
|
||||
> & { current?: string; endpoint?: string; label?: string };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ServiceNav() {
|
||||
const [services, setServices] = useState<ServiceInfo[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const currentService = 'meta-sort';
|
||||
useEffect(() => {
|
||||
// Import above is for the side effect of defining the element; this
|
||||
// keeps bundlers from tree-shaking it away in production builds.
|
||||
}, []);
|
||||
|
||||
useEffect(() => {
|
||||
const fetchServices = async () => {
|
||||
try {
|
||||
const response = await fetch('/api/services');
|
||||
if (response.ok) {
|
||||
const data: ServicesResponse = await response.json();
|
||||
setServices(data.services || []);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch services:', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchServices();
|
||||
const interval = setInterval(fetchServices, 30000);
|
||||
return () => clearInterval(interval);
|
||||
}, []);
|
||||
|
||||
if (loading) return null;
|
||||
|
||||
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
if (sortedServices.length === 0) return null;
|
||||
|
||||
return (
|
||||
<nav className="services-nav">
|
||||
<span className="services-nav-label">Services:</span>
|
||||
<div className="services-nav-items">
|
||||
{sortedServices.map(service => {
|
||||
const icon = serviceIcons[service.name] || serviceIcons.default;
|
||||
const isActive = service.name === currentService;
|
||||
return (
|
||||
<a
|
||||
key={service.name}
|
||||
href={isActive ? '#' : service.baseUrl}
|
||||
className={`service-link${isActive ? ' active' : ''}`}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
if (!isActive) {
|
||||
window.location.href = service.baseUrl;
|
||||
return (
|
||||
<>
|
||||
<meta-service-menu current="meta-sort" />
|
||||
<style>{`
|
||||
meta-service-menu {
|
||||
--mm-nav-fg: var(--text-primary, #e0e0e0);
|
||||
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
|
||||
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
|
||||
--mm-nav-accent: var(--accent-primary, #4ecdc4);
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="service-icon">{icon}</span>
|
||||
<span>{formatServiceName(service.name)}</span>
|
||||
<span className="service-status"></span>
|
||||
</a>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<style>{`
|
||||
.services-nav {
|
||||
background: var(--bg-secondary);
|
||||
border-radius: 12px;
|
||||
padding: 0.5rem;
|
||||
margin-bottom: 1.5rem;
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
flex-wrap: wrap;
|
||||
border: 1px solid var(--border-color);
|
||||
}
|
||||
|
||||
.services-nav-label {
|
||||
color: var(--text-secondary);
|
||||
font-size: 0.85rem;
|
||||
padding: 0.5rem 0.75rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.services-nav-items {
|
||||
display: flex;
|
||||
gap: 0.5rem;
|
||||
flex-wrap: wrap;
|
||||
}
|
||||
|
||||
.service-link {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.4rem;
|
||||
padding: 0.5rem 1rem;
|
||||
border-radius: 8px;
|
||||
text-decoration: none;
|
||||
font-size: 0.9rem;
|
||||
font-weight: 500;
|
||||
transition: all 0.2s;
|
||||
color: var(--text-primary);
|
||||
background: var(--bg-tertiary);
|
||||
border: 1px solid var(--border-color);
|
||||
}
|
||||
|
||||
.service-link:hover {
|
||||
background: var(--border-color);
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
text-decoration: none;
|
||||
}
|
||||
|
||||
.service-link.active {
|
||||
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
}
|
||||
|
||||
.service-link .service-icon {
|
||||
font-size: 1.1rem;
|
||||
}
|
||||
|
||||
.service-link .service-status {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
border-radius: 50%;
|
||||
background: var(--accent-primary);
|
||||
margin-left: 0.25rem;
|
||||
}
|
||||
|
||||
@media (max-width: 600px) {
|
||||
.services-nav {
|
||||
flex-direction: column;
|
||||
align-items: flex-start;
|
||||
}
|
||||
}
|
||||
`}</style>
|
||||
</nav>
|
||||
);
|
||||
`}</style>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
export default ServiceNav;
|
||||
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* <meta-beacon-scan> — "what plugins are on the network?", as a card.
|
||||
*
|
||||
* The Scan button POSTs to the host's own discover endpoint, which probes the
|
||||
* beacon v2 group (239.255.99.1:9099), waits for the replies, and classifies
|
||||
* every resource matching the host's capability (transport plugins for
|
||||
* meta-share, feeders for meta-gateway, enrichment plugins for meta-sort):
|
||||
*
|
||||
* 10 nodes on the network · 3 transport plugins · 3 already configured
|
||||
*
|
||||
* Each `addable` row gets an Add button that POSTs {name, url} to the host's
|
||||
* existing add endpoint. Response shape: meta-feeder-sdk `beacon::ScanReport`
|
||||
* (docs/project-architecture/beacon-v2.md).
|
||||
*
|
||||
* ⚠ MIRRORED FILE. Byte-identical copies live in meta-share, meta-gateway and
|
||||
* meta-sort-ui; see `scripts/check-mirrors.sh`. Edit one → edit all.
|
||||
*
|
||||
* Shadow DOM for the same reason as <meta-service-menu>: the three hosts use
|
||||
* incompatible token vocabularies. Style it from the host with
|
||||
* --mm-scan-fg, --mm-scan-bg, --mm-scan-border, --mm-scan-accent,
|
||||
* --mm-scan-ok, --mm-scan-err
|
||||
*
|
||||
* Attributes:
|
||||
* scan-endpoint POST → ScanReport (required)
|
||||
* add-endpoint POST {name, url, ...add-extra} (required)
|
||||
* add-extra JSON object merged into the add body (optional)
|
||||
* label what the host calls a capable resource, plural
|
||||
* (default "plugins")
|
||||
*
|
||||
* Events (bubbling, composed):
|
||||
* beacon-added detail = { candidate, response }, after a successful Add
|
||||
*/
|
||||
|
||||
const STATE_TEXT = {
|
||||
addable: "new",
|
||||
configured: "configured",
|
||||
"bound-elsewhere": "bound elsewhere",
|
||||
};
|
||||
|
||||
const TEMPLATE = `
|
||||
<style>
|
||||
:host {
|
||||
--_fg: var(--mm-scan-fg, #e8eaed);
|
||||
--_bg: var(--mm-scan-bg, transparent);
|
||||
--_border: var(--mm-scan-border, rgba(255, 255, 255, 0.14));
|
||||
--_accent: var(--mm-scan-accent, #4ecdc4);
|
||||
--_ok: var(--mm-scan-ok, #3fb950);
|
||||
--_err: var(--mm-scan-err, #f85149);
|
||||
display: block;
|
||||
color: var(--_fg);
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
.card {
|
||||
background: var(--_bg);
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 10px;
|
||||
padding: 0.9rem 1rem;
|
||||
}
|
||||
.head { display: flex; align-items: center; gap: 0.75rem; flex-wrap: wrap; }
|
||||
.title { font-weight: 600; }
|
||||
.hint { opacity: 0.7; font-size: 0.85em; }
|
||||
.spacer { flex: 1; }
|
||||
button {
|
||||
font: inherit;
|
||||
font-size: 0.85em;
|
||||
cursor: pointer;
|
||||
color: var(--_fg);
|
||||
background: transparent;
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 6px;
|
||||
padding: 0.3rem 0.8rem;
|
||||
}
|
||||
button:hover:not(:disabled) { border-color: var(--_accent); color: var(--_accent); }
|
||||
button:disabled { opacity: 0.5; cursor: progress; }
|
||||
button.add { border-color: var(--_accent); color: var(--_accent); }
|
||||
.summary { margin-top: 0.6rem; font-size: 0.9em; }
|
||||
.summary b { color: var(--_accent); }
|
||||
.error { margin-top: 0.6rem; color: var(--_err); font-size: 0.85em; }
|
||||
ul { list-style: none; margin: 0.6rem 0 0; padding: 0; }
|
||||
li {
|
||||
display: grid;
|
||||
grid-template-columns: minmax(0, 1fr) auto;
|
||||
gap: 0.2rem 0.75rem;
|
||||
align-items: center;
|
||||
padding: 0.5rem 0;
|
||||
border-top: 1px solid var(--_border);
|
||||
}
|
||||
.who { display: flex; gap: 0.5rem; align-items: baseline; flex-wrap: wrap; min-width: 0; }
|
||||
.name { font-weight: 600; }
|
||||
.muted { opacity: 0.65; font-size: 0.85em; }
|
||||
code { font-size: 0.8em; opacity: 0.8; overflow-wrap: anywhere; }
|
||||
.chips { display: inline-flex; gap: 0.3rem; flex-wrap: wrap; }
|
||||
.chip {
|
||||
font-size: 0.72em;
|
||||
padding: 0 0.4rem;
|
||||
border-radius: 3px;
|
||||
border: 1px solid var(--_border);
|
||||
opacity: 0.85;
|
||||
}
|
||||
.state { font-size: 0.75em; padding: 0.05rem 0.45rem; border-radius: 999px; border: 1px solid currentColor; }
|
||||
.state.addable { color: var(--_accent); }
|
||||
.state.configured { color: var(--_ok); }
|
||||
.state.bound-elsewhere { opacity: 0.6; }
|
||||
.row-actions { display: flex; gap: 0.5rem; align-items: center; grid-row: span 2; }
|
||||
.row-err { color: var(--_err); font-size: 0.8em; }
|
||||
</style>
|
||||
<div class="card">
|
||||
<div class="head">
|
||||
<span class="title">Beacon discovery</span>
|
||||
<span class="hint">plugins advertising themselves on this network</span>
|
||||
<span class="spacer"></span>
|
||||
<button class="scan" type="button">Scan</button>
|
||||
</div>
|
||||
<div class="summary" hidden></div>
|
||||
<div class="error" hidden></div>
|
||||
<ul></ul>
|
||||
</div>
|
||||
`;
|
||||
|
||||
function h(tag, cls, text) {
|
||||
const n = document.createElement(tag);
|
||||
if (cls) n.className = cls;
|
||||
if (text != null) n.textContent = text;
|
||||
return n;
|
||||
}
|
||||
|
||||
async function readError(res) {
|
||||
const body = await res.text().catch(() => "");
|
||||
try {
|
||||
const j = JSON.parse(body);
|
||||
return j.error || j.message || body || `HTTP ${res.status}`;
|
||||
} catch {
|
||||
return body || `HTTP ${res.status}`;
|
||||
}
|
||||
}
|
||||
|
||||
class MetaBeaconScan extends HTMLElement {
|
||||
constructor() {
|
||||
super();
|
||||
this.attachShadow({ mode: "open" });
|
||||
this.shadowRoot.innerHTML = TEMPLATE;
|
||||
this._scanBtn = this.shadowRoot.querySelector("button.scan");
|
||||
this._summary = this.shadowRoot.querySelector(".summary");
|
||||
this._error = this.shadowRoot.querySelector(".error");
|
||||
this._list = this.shadowRoot.querySelector("ul");
|
||||
this._busy = false;
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
if (this._wired) return;
|
||||
this._wired = true;
|
||||
this._scanBtn.addEventListener("click", () => this.scan());
|
||||
}
|
||||
|
||||
get label() {
|
||||
return this.getAttribute("label") || "plugins";
|
||||
}
|
||||
|
||||
async scan() {
|
||||
const endpoint = this.getAttribute("scan-endpoint");
|
||||
if (!endpoint || this._busy) return;
|
||||
this._busy = true;
|
||||
this._scanBtn.disabled = true;
|
||||
this._scanBtn.textContent = "Scanning…";
|
||||
this._error.hidden = true;
|
||||
try {
|
||||
const res = await fetch(endpoint, { method: "POST", headers: { Accept: "application/json" } });
|
||||
if (!res.ok) throw new Error(await readError(res));
|
||||
this._render(await res.json());
|
||||
} catch (e) {
|
||||
// Keep the last result visible; just say what went wrong.
|
||||
this._error.textContent = `Scan failed: ${e.message}`;
|
||||
this._error.hidden = false;
|
||||
} finally {
|
||||
this._busy = false;
|
||||
this._scanBtn.disabled = false;
|
||||
this._scanBtn.textContent = "Scan again";
|
||||
}
|
||||
}
|
||||
|
||||
_render(report) {
|
||||
const s = report.summary || {};
|
||||
this._summary.replaceChildren();
|
||||
const part = (n, text) => {
|
||||
const span = h("span");
|
||||
span.append(h("b", null, String(n ?? 0)), ` ${text}`);
|
||||
return span;
|
||||
};
|
||||
const parts = [
|
||||
part(s.nodes, s.nodes === 1 ? "node on the network" : "nodes on the network"),
|
||||
part(s.capable, this.label),
|
||||
part(s.configured, "already configured"),
|
||||
part(s.addable, "new"),
|
||||
];
|
||||
if (s.boundElsewhere) parts.push(part(s.boundElsewhere, "bound to another consumer"));
|
||||
parts.forEach((p, i) => {
|
||||
if (i) this._summary.append(" · ");
|
||||
this._summary.append(p);
|
||||
});
|
||||
this._summary.hidden = false;
|
||||
|
||||
this._list.replaceChildren(...(report.candidates || []).map((c) => this._row(c)));
|
||||
}
|
||||
|
||||
_row(c) {
|
||||
const li = h("li");
|
||||
const who = h("div", "who");
|
||||
who.append(h("span", "name", c.instance));
|
||||
if (c.name && c.name !== c.instance) who.append(h("span", "muted", c.name));
|
||||
if (c.version) who.append(h("span", "muted", `v${c.version}`));
|
||||
const chips = h("span", "chips");
|
||||
for (const cap of c.caps || []) chips.append(h("span", "chip", cap));
|
||||
who.append(chips);
|
||||
|
||||
const actions = h("div", "row-actions");
|
||||
const state = h("span", `state ${c.state}`, STATE_TEXT[c.state] || c.state);
|
||||
if (c.state === "configured" && c.configuredAs) state.title = `listed as "${c.configuredAs}"`;
|
||||
if (c.state === "bound-elsewhere" && c.binds) state.title = `belongs to ${c.binds}`;
|
||||
actions.append(state);
|
||||
if (c.state === "addable") {
|
||||
const add = h("button", "add", "Add");
|
||||
add.type = "button";
|
||||
add.addEventListener("click", () => this._add(c, add, li));
|
||||
actions.append(add);
|
||||
}
|
||||
|
||||
const where = h("div");
|
||||
where.append(h("code", null, c.url));
|
||||
li.append(who, actions, where);
|
||||
return li;
|
||||
}
|
||||
|
||||
async _add(candidate, btn, li) {
|
||||
const endpoint = this.getAttribute("add-endpoint");
|
||||
if (!endpoint) return;
|
||||
let extra = {};
|
||||
try {
|
||||
extra = JSON.parse(this.getAttribute("add-extra") || "{}");
|
||||
} catch {
|
||||
/* a malformed attribute must not block the add */
|
||||
}
|
||||
btn.disabled = true;
|
||||
btn.textContent = "Adding…";
|
||||
li.querySelector(".row-err")?.remove();
|
||||
try {
|
||||
const res = await fetch(endpoint, {
|
||||
method: "POST",
|
||||
headers: { "Content-Type": "application/json", Accept: "application/json" },
|
||||
body: JSON.stringify({ ...extra, name: candidate.suggestedName, url: candidate.url }),
|
||||
});
|
||||
if (!res.ok) throw new Error(await readError(res));
|
||||
const text = await res.text();
|
||||
let response = null;
|
||||
try {
|
||||
response = text ? JSON.parse(text) : null;
|
||||
} catch {
|
||||
response = text;
|
||||
}
|
||||
this.dispatchEvent(
|
||||
new CustomEvent("beacon-added", { bubbles: true, composed: true, detail: { candidate, response } }),
|
||||
);
|
||||
btn.textContent = "Added";
|
||||
} catch (e) {
|
||||
btn.disabled = false;
|
||||
btn.textContent = "Add";
|
||||
li.append(h("div", "row-err", e.message));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get("meta-beacon-scan")) {
|
||||
customElements.define("meta-beacon-scan", MetaBeaconScan);
|
||||
}
|
||||
|
||||
export default MetaBeaconScan;
|
||||
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Replaces the eight divergent "services bar" implementations (four copies of
|
||||
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
|
||||
* ones) with a single framework-free custom element. It drops into the React
|
||||
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
|
||||
* as an ES module.
|
||||
*
|
||||
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
|
||||
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
|
||||
* make a fetch to a neighbour a dead end. Links are plain navigations.
|
||||
*
|
||||
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
|
||||
* `scripts/check-mirrors.sh`. Edit one → edit all.
|
||||
*
|
||||
* Shadow DOM is not optional here. The eight UIs use four incompatible token
|
||||
* vocabularies, and two of them define *different colours under the same
|
||||
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
|
||||
* #161b22 in meta-dup). meta-search and meta-share are also the only
|
||||
* light-mode-capable UIs, so a hardcoded dark palette would break them.
|
||||
* The element therefore styles itself from four tokens with dark defaults:
|
||||
*
|
||||
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
|
||||
*
|
||||
* Each host maps its own variables onto those, e.g.
|
||||
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
|
||||
*
|
||||
* Attributes:
|
||||
* current name of the host service, so it can be marked and not linked
|
||||
* endpoint override the default "/api/neighbors"
|
||||
* label button tooltip (default "Services")
|
||||
*
|
||||
* Property:
|
||||
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
|
||||
* translate internal hostnames into browser-reachable ones
|
||||
* (meta-gateway does this for its dev/prod split).
|
||||
*/
|
||||
|
||||
const ICONS = {
|
||||
"meta-core": "⚙️",
|
||||
"meta-sort": "📁",
|
||||
"meta-fuse": "🗂️",
|
||||
"meta-stremio": "🎬",
|
||||
"meta-dup": "🔍",
|
||||
"meta-search": "🔎",
|
||||
"meta-share": "🔗",
|
||||
"meta-gateway": "🌉",
|
||||
"meta-watch": "📺",
|
||||
"meta-listen": "🎧",
|
||||
"meta-read": "📖",
|
||||
};
|
||||
const DEFAULT_ICON = "📦";
|
||||
const POLL_MS = 30000;
|
||||
|
||||
function titleCase(name) {
|
||||
return String(name || "")
|
||||
.split("-")
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
const TEMPLATE = `
|
||||
<style>
|
||||
:host {
|
||||
--_fg: var(--mm-nav-fg, #e8eaed);
|
||||
--_bg: var(--mm-nav-bg, #1a1a2e);
|
||||
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
|
||||
--_accent: var(--mm-nav-accent, #4ecdc4);
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
button.trigger {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
width: 2.1rem;
|
||||
height: 2.1rem;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
color: var(--_fg);
|
||||
background: transparent;
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 8px;
|
||||
font: inherit;
|
||||
transition: border-color 0.15s, background 0.15s;
|
||||
}
|
||||
button.trigger:hover { border-color: var(--_accent); }
|
||||
button.trigger:focus-visible {
|
||||
outline: 2px solid var(--_accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
|
||||
.bars { display: block; width: 15px; }
|
||||
.bars span {
|
||||
display: block;
|
||||
height: 2px;
|
||||
margin: 3px 0;
|
||||
border-radius: 2px;
|
||||
background: currentColor;
|
||||
}
|
||||
.panel {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
right: 0;
|
||||
z-index: 1000;
|
||||
min-width: 13rem;
|
||||
max-height: 70vh;
|
||||
overflow-y: auto;
|
||||
padding: 0.3rem;
|
||||
background: var(--_bg);
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
.panel[hidden] { display: none; }
|
||||
a.item, .item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
padding: 0.45rem 0.6rem;
|
||||
border-radius: 6px;
|
||||
color: var(--_fg);
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
}
|
||||
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
|
||||
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
|
||||
.item.dead { opacity: 0.55; cursor: not-allowed; }
|
||||
.icon { font-size: 1rem; line-height: 1; }
|
||||
.dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
margin-left: auto;
|
||||
border-radius: 50%;
|
||||
background: var(--_accent);
|
||||
}
|
||||
.empty {
|
||||
padding: 0.6rem;
|
||||
opacity: 0.7;
|
||||
font-size: 0.85em;
|
||||
white-space: nowrap;
|
||||
}
|
||||
/* Respect a host that opts out of motion. */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
button.trigger { transition: none; }
|
||||
}
|
||||
</style>
|
||||
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
|
||||
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
|
||||
</button>
|
||||
<div class="panel" role="menu" hidden></div>
|
||||
`;
|
||||
|
||||
class MetaServiceMenu extends HTMLElement {
|
||||
constructor() {
|
||||
super();
|
||||
this.attachShadow({ mode: "open" });
|
||||
this.shadowRoot.innerHTML = TEMPLATE;
|
||||
this._btn = this.shadowRoot.querySelector("button.trigger");
|
||||
this._panel = this.shadowRoot.querySelector(".panel");
|
||||
this._neighbors = [];
|
||||
this._timer = null;
|
||||
this._open = false;
|
||||
|
||||
this._onDocClick = (e) => {
|
||||
if (!this._open) return;
|
||||
if (e.composedPath().includes(this)) return;
|
||||
this._setOpen(false);
|
||||
};
|
||||
this._onKey = (e) => {
|
||||
if (e.key === "Escape" && this._open) {
|
||||
this._setOpen(false);
|
||||
this._btn.focus();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
this._btn.title = this.getAttribute("label") || "Services";
|
||||
this._btn.setAttribute("aria-label", this._btn.title);
|
||||
this._btn.addEventListener("click", () => this._setOpen(!this._open));
|
||||
document.addEventListener("click", this._onDocClick);
|
||||
document.addEventListener("keydown", this._onKey);
|
||||
|
||||
this.refresh();
|
||||
this._timer = setInterval(() => this.refresh(), POLL_MS);
|
||||
}
|
||||
|
||||
disconnectedCallback() {
|
||||
if (this._timer) clearInterval(this._timer);
|
||||
this._timer = null;
|
||||
document.removeEventListener("click", this._onDocClick);
|
||||
document.removeEventListener("keydown", this._onKey);
|
||||
}
|
||||
|
||||
get endpoint() {
|
||||
return this.getAttribute("endpoint") || "/api/neighbors";
|
||||
}
|
||||
|
||||
get current() {
|
||||
return this.getAttribute("current") || "";
|
||||
}
|
||||
|
||||
async refresh() {
|
||||
try {
|
||||
const res = await fetch(this.endpoint, {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!res.ok) throw new Error(String(res.status));
|
||||
const data = await res.json();
|
||||
// `services` is the legacy alias some backends still emit.
|
||||
this._neighbors = data.neighbors || data.services || [];
|
||||
} catch {
|
||||
// Silent: the menu is navigation, not a health indicator. Keep whatever
|
||||
// we last had rather than blanking the list on one failed poll.
|
||||
}
|
||||
this._render();
|
||||
}
|
||||
|
||||
_setOpen(open) {
|
||||
this._open = open;
|
||||
this._panel.hidden = !open;
|
||||
this._btn.setAttribute("aria-expanded", String(open));
|
||||
if (open) this.refresh();
|
||||
}
|
||||
|
||||
_render() {
|
||||
const panel = this._panel;
|
||||
panel.textContent = "";
|
||||
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const n of this._neighbors) {
|
||||
const name = n.name || "";
|
||||
if (!name || seen.has(name)) continue;
|
||||
// Beacon v2 neighbours include plugins; the nav lists services only.
|
||||
// (A row without `caps` comes from a pre-v2 backend — keep it.)
|
||||
const isService = (c) => String(c).startsWith("metamesh.service/");
|
||||
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
|
||||
seen.add(name);
|
||||
rows.push(n);
|
||||
}
|
||||
// Always show the host itself, even before its own echo comes back.
|
||||
if (this.current && !seen.has(this.current)) {
|
||||
rows.push({ name: this.current, baseUrl: "" });
|
||||
}
|
||||
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
|
||||
|
||||
if (rows.length === 0) {
|
||||
const empty = document.createElement("div");
|
||||
empty.className = "empty";
|
||||
empty.textContent = "No services found";
|
||||
panel.appendChild(empty);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const n of rows) {
|
||||
const isActive = n.name === this.current;
|
||||
let url = n.baseUrl || "";
|
||||
if (typeof this.rewriteUrl === "function") {
|
||||
try {
|
||||
url = this.rewriteUrl(n.name, url, n) || "";
|
||||
} catch {
|
||||
/* a bad hook must not blank the menu */
|
||||
}
|
||||
}
|
||||
|
||||
// A neighbour with no reachable URL is shown but not linked — better
|
||||
// than a link that goes nowhere.
|
||||
const el = document.createElement(isActive || !url ? "span" : "a");
|
||||
el.className =
|
||||
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
|
||||
if (el.tagName === "A") {
|
||||
el.href = url;
|
||||
el.setAttribute("role", "menuitem");
|
||||
}
|
||||
if (!isActive && !url) el.title = "No reachable URL advertised";
|
||||
|
||||
const icon = document.createElement("span");
|
||||
icon.className = "icon";
|
||||
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = titleCase(n.name);
|
||||
|
||||
el.append(icon, label);
|
||||
// Beacon v2 omits `status` when running.
|
||||
if ((n.status || "running") === "running" || isActive) {
|
||||
const dot = document.createElement("span");
|
||||
dot.className = "dot";
|
||||
dot.title = n.status || "running";
|
||||
el.appendChild(dot);
|
||||
}
|
||||
panel.appendChild(el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get("meta-service-menu")) {
|
||||
customElements.define("meta-service-menu", MetaServiceMenu);
|
||||
}
|
||||
|
||||
export default MetaServiceMenu;
|
||||
@@ -1,11 +1,12 @@
|
||||
import { useState, useEffect, useCallback } from 'react';
|
||||
import { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import {
|
||||
ProcessingStatus,
|
||||
Metrics,
|
||||
RedisStats,
|
||||
QueueItem,
|
||||
HashTimingStats,
|
||||
FailedFile
|
||||
FailedFile,
|
||||
CacheSlotPhase
|
||||
} from '../types';
|
||||
import {
|
||||
formatMs,
|
||||
@@ -14,6 +15,43 @@ import {
|
||||
getFilename
|
||||
} from '../utils/format';
|
||||
|
||||
/**
|
||||
* Renders a single phase of the file-cache-slot view: capacity bar, count of
|
||||
* waiting files, and a list of currently-held files showing which plugin
|
||||
* each is currently waiting on. The plugin column is the long-pole — it tells
|
||||
* operators which plugin is the per-file bottleneck.
|
||||
*/
|
||||
function CacheSlotPanel({ label, phase }: { label: string; phase: CacheSlotPhase }) {
|
||||
const fillPct = Math.min(100, (phase.inUse / Math.max(1, phase.capacity)) * 100);
|
||||
return (
|
||||
<div style={{ marginTop: '0.75rem' }}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'baseline' }}>
|
||||
<strong>{label}</strong>
|
||||
<span style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
|
||||
{phase.inUse}/{phase.capacity} files
|
||||
{phase.waiting > 0 ? ` • ${phase.waiting} waiting for slot` : ''}
|
||||
</span>
|
||||
</div>
|
||||
<div style={{ height: '6px', background: '#0f3460', borderRadius: '3px', overflow: 'hidden', margin: '0.25rem 0 0.5rem' }}>
|
||||
<div style={{ width: `${fillPct}%`, height: '100%', background: '#4ade80', transition: 'width 0.5s linear' }} />
|
||||
</div>
|
||||
{phase.holders.length > 0 && (
|
||||
<div style={{ fontSize: '0.78rem', color: '#cbd5e1', maxHeight: '160px', overflow: 'auto' }}>
|
||||
{phase.holders.map((h) => (
|
||||
<div key={h.fileHash} style={{ display: 'flex', gap: '0.5rem', padding: '2px 0' }}>
|
||||
<span style={{ color: '#facc15', minWidth: '8rem' }}>{h.currentPlugin ?? '—'}</span>
|
||||
<span style={{ color: '#94a3b8', minWidth: '4rem' }}>×{h.tasksRemaining}</span>
|
||||
<span style={{ flex: 1, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }} title={h.filePath ?? h.fileHash}>
|
||||
{h.filePath ? getFilename(h.filePath) : h.fileHash}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Monitor() {
|
||||
const [status, setStatus] = useState<ProcessingStatus | null>(null);
|
||||
const [metrics, setMetrics] = useState<Metrics | null>(null);
|
||||
@@ -22,29 +60,35 @@ function Monitor() {
|
||||
const [failedFiles, setFailedFiles] = useState<FailedFile[]>([]);
|
||||
const [showPipelineHelp, setShowPipelineHelp] = useState(false);
|
||||
|
||||
const fetchData = useCallback(async () => {
|
||||
try {
|
||||
const [statusRes, metricsRes, queueRes, statsRes, failedRes] = await Promise.all([
|
||||
fetch('/api/processing/status'),
|
||||
fetch('/api/metrics'),
|
||||
fetch('/api/processing/queue'),
|
||||
fetch('/api/stats'),
|
||||
fetch('/api/processing/failed')
|
||||
]);
|
||||
// Each endpoint renders as soon as it answers, so one slow or failing call
|
||||
// cannot blank the page. A tick is skipped while the previous one is still
|
||||
// out, so a slow backend never collects a pile of overlapping polls.
|
||||
const pollInFlight = useRef(false);
|
||||
|
||||
if (statusRes.ok) setStatus(await statusRes.json());
|
||||
if (metricsRes.ok) setMetrics(await metricsRes.json());
|
||||
if (queueRes.ok) {
|
||||
const queueData = await queueRes.json();
|
||||
setQueue(queueData.items || []);
|
||||
const fetchData = useCallback(async () => {
|
||||
if (pollInFlight.current) return;
|
||||
pollInFlight.current = true;
|
||||
|
||||
const load = async <T,>(url: string, apply: (data: T) => void) => {
|
||||
const res = await fetch(url);
|
||||
if (res.ok) apply(await res.json());
|
||||
};
|
||||
|
||||
try {
|
||||
const results = await Promise.allSettled([
|
||||
load<ProcessingStatus>('/api/processing/status', setStatus),
|
||||
load<Metrics>('/api/metrics', setMetrics),
|
||||
load<{ items?: QueueItem[] }>('/api/processing/queue', (data) => setQueue(data.items || [])),
|
||||
load<RedisStats>('/api/stats', setRedisStats),
|
||||
load<{ failedFiles?: FailedFile[] }>('/api/processing/failed', (data) => setFailedFiles(data.failedFiles || [])),
|
||||
]);
|
||||
for (const result of results) {
|
||||
if (result.status === 'rejected') {
|
||||
console.error('Failed to fetch data:', result.reason);
|
||||
}
|
||||
}
|
||||
if (statsRes.ok) setRedisStats(await statsRes.json());
|
||||
if (failedRes.ok) {
|
||||
const failedData = await failedRes.json();
|
||||
setFailedFiles(failedData.failedFiles || []);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('Failed to fetch data:', err);
|
||||
} finally {
|
||||
pollInFlight.current = false;
|
||||
}
|
||||
}, []);
|
||||
|
||||
@@ -107,31 +151,50 @@ function Monitor() {
|
||||
// Get queue statistics for display
|
||||
// StreamingPipeline stages: validation → lightProcessing → hashProcessing
|
||||
// UI mapping: Validation → Fast Queue → Background Queue
|
||||
//
|
||||
// Fast/Background "Running/Max" represents file-cache slot occupancy
|
||||
// (cacheSlots.inUse / capacity), not raw task throughput. This matches the
|
||||
// operator mental model: "how many files are currently tying up the WebDAV
|
||||
// cache?" The task-callback count was misleading because dependency-wait
|
||||
// and per-plugin HTTP queueing made it under-report by ~5×.
|
||||
const getQueueStats = () => {
|
||||
const computed = status?.computed;
|
||||
const cacheSlots = status?.queueStatus?.cacheSlots;
|
||||
const fastQueueConcurrency = status?.fastQueueConcurrency || 16;
|
||||
const backgroundQueueConcurrency = status?.backgroundQueueConcurrency || 8;
|
||||
|
||||
return {
|
||||
// Stage 1: Validation (quick extension checks)
|
||||
// Concurrency: 2x fast queue workers
|
||||
// Stage 1: Validation (quick extension checks). Still task-level
|
||||
// — pre-process is a lightweight midhash256 scan, no cache impact.
|
||||
preProcess: {
|
||||
running: computed?.preProcessRunning || 0,
|
||||
pending: computed?.preProcessPending || 0,
|
||||
max: fastQueueConcurrency * 2,
|
||||
paused: computed?.preProcessPaused || false
|
||||
},
|
||||
// Stage 2: Fast Queue (midhash256 + FFmpeg + plugins)
|
||||
// Concurrency: fastQueueConcurrency workers
|
||||
fast: {
|
||||
// Stage 2: Fast Queue — files holding a light cache slot.
|
||||
// running = files in cache, pending = files waiting for a slot.
|
||||
// paused comes from the cacheSlots payload (true ContainerPluginScheduler
|
||||
// PQueue state) — the legacy `computed.fastQueuePaused` field is sourced
|
||||
// from a different StreamingPipeline queue and is unreliable here.
|
||||
fast: cacheSlots ? {
|
||||
running: cacheSlots.fast.inUse,
|
||||
pending: cacheSlots.fast.waiting,
|
||||
max: cacheSlots.fast.capacity,
|
||||
paused: cacheSlots.fast.paused
|
||||
} : {
|
||||
running: computed?.fastQueueRunning || 0,
|
||||
pending: computed?.fastQueuePending || 0,
|
||||
max: fastQueueConcurrency,
|
||||
paused: computed?.fastQueuePaused || false
|
||||
},
|
||||
// Stage 3: Background Queue (SHA-256 full file hash)
|
||||
// Concurrency: backgroundQueueConcurrency workers
|
||||
background: {
|
||||
// Stage 3: Background Queue — files holding a bg cache slot.
|
||||
background: cacheSlots ? {
|
||||
running: cacheSlots.background.inUse,
|
||||
pending: cacheSlots.background.waiting,
|
||||
max: cacheSlots.background.capacity,
|
||||
paused: cacheSlots.background.paused
|
||||
} : {
|
||||
running: computed?.backgroundQueueRunning || 0,
|
||||
pending: computed?.backgroundQueuePending || 0,
|
||||
max: backgroundQueueConcurrency,
|
||||
@@ -165,9 +228,9 @@ function Monitor() {
|
||||
{showPipelineHelp && (
|
||||
<div className="pipeline-help">
|
||||
<div className="help-item"><span className="help-dot discovered"></span><strong>Discovered:</strong> Files found on disk, waiting to be processed.</div>
|
||||
<div className="help-item"><span className="help-dot fast"></span><strong>Fast Queue:</strong> Files being processed for quick metadata extraction (hash, ffmpeg, etc.).</div>
|
||||
<div className="help-item"><span className="help-dot fast"></span><strong>Fast Queue:</strong> Files holding a light cache slot. The badge shows slots-in-use over the gate capacity (e.g. 16/16). Pending = files waiting for a slot to free.</div>
|
||||
<div className="help-item"><span className="help-dot awaiting-background"></span><strong>Awaiting BG:</strong> Files done with fast processing, queued for slower tasks.</div>
|
||||
<div className="help-item"><span className="help-dot background"></span><strong>Background:</strong> Files running slow tasks like full file hashing or TMDB lookups.</div>
|
||||
<div className="help-item"><span className="help-dot background"></span><strong>Background:</strong> Files holding a background cache slot (full-hash, TMDB, etc.). Same slot-occupancy semantics as Fast Queue.</div>
|
||||
<div className="help-item"><span className="help-dot done"></span><strong>Complete:</strong> Files fully processed and available in the virtual filesystem.</div>
|
||||
<div className="help-item"><span className="help-dot failed"></span><strong>Failed:</strong> Files that encountered errors during processing.</div>
|
||||
</div>
|
||||
@@ -301,6 +364,18 @@ function Monitor() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status?.queueStatus?.cacheSlots && (
|
||||
<div className="card cache-slot-section" style={{ marginTop: '1rem' }}>
|
||||
<h2>File cache slots</h2>
|
||||
<p style={{ color: '#888', fontSize: '0.85rem', marginTop: '-0.5rem' }}>
|
||||
Number of distinct files concurrently held by the WebDAV cache per phase. Caps how
|
||||
many file fetches CORN/SMB sees at once, regardless of plugin fan-out.
|
||||
</p>
|
||||
<CacheSlotPanel label="Light (fast queue)" phase={status.queueStatus.cacheSlots.fast} />
|
||||
<CacheSlotPanel label="Background" phase={status.queueStatus.cacheSlots.background} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid grid-2">
|
||||
{/* Metrics and Stats */}
|
||||
<div className="card">
|
||||
@@ -341,7 +416,7 @@ function Monitor() {
|
||||
<h3 className="section-title">File Coverage</h3>
|
||||
<div className="stats-grid">
|
||||
<div className="stat-box">
|
||||
<div className="stat-value">{status?.uniqueHashes ? formatNumber(status.uniqueHashes) : (redisStats ? formatNumber(redisStats.fileCount) : '-')}</div>
|
||||
<div className="stat-value">{status?.uniqueHashes ? formatNumber(status.uniqueHashes) : (redisStats?.fileCount != null ? formatNumber(redisStats.fileCount) : '-')}</div>
|
||||
<div className="stat-label">Unique Files</div>
|
||||
</div>
|
||||
<div className="stat-box">
|
||||
@@ -363,7 +438,7 @@ function Monitor() {
|
||||
<div className="card total-size-card">
|
||||
<h2>Total Size</h2>
|
||||
<div className="total-size-value">
|
||||
{redisStats ? formatBytes(redisStats.totalSize) : '-'}
|
||||
{redisStats?.totalSize != null ? formatBytes(redisStats.totalSize) : '-'}
|
||||
</div>
|
||||
<div className="total-size-label">Watch Folders</div>
|
||||
</div>
|
||||
|
||||
@@ -5,6 +5,7 @@ import {
|
||||
PluginTimingsResponse
|
||||
} from '../types';
|
||||
import { formatMs, formatNumber } from '../utils/format';
|
||||
import BeaconScanCard from '../components/BeaconScanCard';
|
||||
|
||||
interface AddPluginForm {
|
||||
pluginId: string;
|
||||
@@ -33,6 +34,8 @@ function Plugins() {
|
||||
cpus: '0.5',
|
||||
});
|
||||
const [restartAllStatus, setRestartAllStatus] = useState<string | null>(null);
|
||||
// Plugins meta-sort reaches by URL but did not spawn (added from the Scan card).
|
||||
const [externalUrls, setExternalUrls] = useState<Record<string, string>>({});
|
||||
|
||||
const fetchPlugins = useCallback(async () => {
|
||||
try {
|
||||
@@ -41,6 +44,14 @@ function Plugins() {
|
||||
const data = await res.json();
|
||||
setPluginsData(data);
|
||||
setError(null);
|
||||
fetch('/api/plugins/containers')
|
||||
.then((r) => (r.ok ? r.json() : null))
|
||||
.then((c: { plugins?: Array<{ id: string; kind?: string; url?: string }> } | null) => {
|
||||
const map: Record<string, string> = {};
|
||||
for (const p of c?.plugins ?? []) if (p.kind === 'external' && p.url) map[p.id] = p.url;
|
||||
setExternalUrls(map);
|
||||
})
|
||||
.catch(() => {});
|
||||
} else if (res.status === 503) {
|
||||
setError('Plugin manager not yet initialized. Process a file to initialize plugins.');
|
||||
}
|
||||
@@ -370,6 +381,9 @@ function Plugins() {
|
||||
<div className="plugin-info">
|
||||
<span className="plugin-name">{plugin.name}</span>
|
||||
<span className="plugin-version">v{plugin.version}</span>
|
||||
{externalUrls[plugin.id] && (
|
||||
<span className="plugin-external" title={`external: ${externalUrls[plugin.id]}`}>external</span>
|
||||
)}
|
||||
</div>
|
||||
<label className="toggle" onClick={(e) => e.stopPropagation()}>
|
||||
<input
|
||||
@@ -677,6 +691,17 @@ function Plugins() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Beacon v2: plugins advertising on the network, with Scan + Add */}
|
||||
<div className="card beacon-scan-card">
|
||||
<h2>Discover plugins</h2>
|
||||
<p className="card-description">
|
||||
Enrichment plugins installed as their own apps announce themselves on the network.
|
||||
Scan to list them; Add registers one as an external plugin (meta-sort calls it by URL
|
||||
and never starts or stops it). Remove it from the list like any other plugin.
|
||||
</p>
|
||||
<BeaconScanCard onAdded={fetchPlugins} />
|
||||
</div>
|
||||
|
||||
{/* Queue Assignment Section */}
|
||||
{pluginsData && pluginsData.activeCount > 0 && (
|
||||
<div className="card queue-assignment-card">
|
||||
@@ -920,6 +945,14 @@ function Plugins() {
|
||||
overflow-y: auto;
|
||||
}
|
||||
|
||||
.plugin-external {
|
||||
margin-left: 0.5rem;
|
||||
font-size: 0.7rem;
|
||||
padding: 0 0.4rem;
|
||||
border-radius: 3px;
|
||||
border: 1px dashed var(--accent-primary);
|
||||
color: var(--accent-primary);
|
||||
}
|
||||
.plugin-item {
|
||||
background: var(--bg-tertiary);
|
||||
border-radius: 8px;
|
||||
|
||||
@@ -75,6 +75,29 @@ export interface TaskSchedulerQueueStatus {
|
||||
preProcessQueue?: TaskQueueStatus;
|
||||
fastQueue?: TaskQueueStatus;
|
||||
backgroundQueue?: TaskQueueStatus;
|
||||
cacheSlots?: CacheSlotsStatus;
|
||||
}
|
||||
|
||||
export interface CacheSlotHolder {
|
||||
fileHash: string;
|
||||
filePath?: string;
|
||||
currentPlugin?: string;
|
||||
tasksRemaining: number;
|
||||
}
|
||||
|
||||
export interface CacheSlotPhase {
|
||||
capacity: number;
|
||||
inUse: number;
|
||||
waiting: number;
|
||||
/** True when the underlying PQueue is paused — bg phase is paused while
|
||||
* fast has work, so its slots can be "reserved" without active dispatch. */
|
||||
paused: boolean;
|
||||
holders: CacheSlotHolder[];
|
||||
}
|
||||
|
||||
export interface CacheSlotsStatus {
|
||||
fast: CacheSlotPhase;
|
||||
background: CacheSlotPhase;
|
||||
}
|
||||
|
||||
export interface ProcessingStatus {
|
||||
@@ -161,12 +184,12 @@ export interface Metrics {
|
||||
|
||||
// NOTE: DuplicateGroup and DuplicateData types have been moved to meta-dup service
|
||||
|
||||
/** GET /api/stats: totals over meta-core's file-backed roots. */
|
||||
export interface RedisStats {
|
||||
fileCount: number;
|
||||
keyCount: number;
|
||||
totalSize: number;
|
||||
memoryUsage?: string;
|
||||
memoryUsageBytes?: number;
|
||||
/** null when meta-core cannot supply file totals (source 'unavailable') */
|
||||
fileCount: number | null;
|
||||
totalSize: number | null;
|
||||
source?: 'meta-core' | 'unavailable';
|
||||
}
|
||||
|
||||
export interface QueueItem {
|
||||
|
||||
@@ -1,9 +1,23 @@
|
||||
/**
|
||||
* The `fileType` vocabulary — see METADATA_KEYS.md §1.
|
||||
*
|
||||
* ⚠ INVARIANT: a value belongs here only if an `(extension, mimeType)` lookup
|
||||
* can decide it on its own. Nothing that requires knowing what the content
|
||||
* *is* goes in this field — that is `contentKind`'s job — and nothing that
|
||||
* decides *which app gets the record* depends on it — that is `domain`'s job
|
||||
* (METADATA_KEYS.md §14.16).
|
||||
*
|
||||
* `card` is the one deliberate exception: it describes no bytes at all and is
|
||||
* therefore never produced by the tables here, only stamped by a card feeder.
|
||||
*/
|
||||
export type SimpleFileType =
|
||||
'audio'
|
||||
| 'video'
|
||||
| 'image'
|
||||
| 'document'
|
||||
| 'archive'
|
||||
| 'subtitle'
|
||||
| 'torrent'
|
||||
| 'card'
|
||||
| 'other'
|
||||
| 'undefined';
|
||||
| 'undefined';
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* `contentKind` → `domain` / `workForm` — the routing tables from
|
||||
* METADATA_KEYS.md §1.
|
||||
*
|
||||
* `domain` answers "which application is this record destined for?" and is the
|
||||
* key every client filters its catalogue on — **one value per app**, no
|
||||
* alternation. `workForm` answers the other question the old `film`/`tv` split
|
||||
* was secretly encoding: is this work complete in itself, or one instalment of
|
||||
* a run? It is cross-domain, so a comic issue is `serial` in `literature`
|
||||
* exactly as an episode is `serial` in `screen`.
|
||||
*
|
||||
* Both are **stored, not derived on read**: that is what lets the `contentKind`
|
||||
* vocabulary stay open, because a peer that has never heard of a kind still
|
||||
* routes the record correctly without having to interpret the token.
|
||||
*
|
||||
* ⚠ This binding is **permanent and one-way**. Add kinds freely; never re-map
|
||||
* an existing kind to a different domain. Because the value is stored, two
|
||||
* peers on different versions that disagree about where a kind belongs produce
|
||||
* a split-brain that redeploying cannot heal — the wrong domain is already
|
||||
* written into records across the mesh. If a kind has to move, mint a new kind.
|
||||
* The `film`+`tv` → `screen` merge (METADATA_KEYS.md §14.17) is the one
|
||||
* deliberate exception, taken once and paid for with a corpus sweep.
|
||||
*
|
||||
* ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain`. Edit both together.
|
||||
*/
|
||||
export type Domain =
|
||||
'screen'
|
||||
| 'music'
|
||||
| 'literature'
|
||||
| 'science';
|
||||
|
||||
/** Is the work complete in itself, or one instalment of a run? */
|
||||
export type WorkForm = 'standalone' | 'serial';
|
||||
|
||||
/**
|
||||
* The one-way binding. A kind that is absent here has no domain, which means
|
||||
* the record is **not independently routable** — correct for sidecars
|
||||
* (subtitles, artwork) and for un-anchored hits, a producer bug for anything
|
||||
* else.
|
||||
*
|
||||
* ⚠ `pack` is deliberately absent: a season pack and an album release are the
|
||||
* same structural thing, so the kind alone cannot tell `screen` from `music`.
|
||||
* The writer stamps `pack`'s domain from its own context — see
|
||||
* {@link domainForContentKind}'s contract.
|
||||
*
|
||||
* ⚠ `podcast` / `podcastEpisode` are absent too: the `spoken` domain was
|
||||
* retired unused, and the change that first *writes* those kinds picks their
|
||||
* domain.
|
||||
*/
|
||||
export const domainByContentKind: { [contentKind: string]: Domain } = {
|
||||
// screen — films and serials alike; the split lives in workFormByContentKind
|
||||
movie: 'screen',
|
||||
series: 'screen',
|
||||
episode: 'screen',
|
||||
// music
|
||||
track: 'music',
|
||||
album: 'music',
|
||||
artist: 'music',
|
||||
musicVideo: 'music',
|
||||
djMix: 'music',
|
||||
liveSet: 'music',
|
||||
// literature
|
||||
book: 'literature',
|
||||
// An audiobook's identity is the book — same author, same ISBN family,
|
||||
// same series as the ebook — so meta-read can group the two editions on
|
||||
// one page. The player does not decide the domain.
|
||||
audiobook: 'literature',
|
||||
comic: 'literature',
|
||||
manga: 'literature',
|
||||
magazine: 'literature',
|
||||
// science
|
||||
paper: 'science',
|
||||
};
|
||||
|
||||
/**
|
||||
* The `workForm` twin of {@link domainByContentKind}.
|
||||
*
|
||||
* ⚠ `pack` is absent for the same reason: a season pack is `serial`, an album
|
||||
* release is `standalone`, and only the writer knows which.
|
||||
*/
|
||||
export const workFormByContentKind: { [contentKind: string]: WorkForm } = {
|
||||
// screen
|
||||
movie: 'standalone',
|
||||
series: 'serial',
|
||||
episode: 'serial',
|
||||
// music — an album is a closed work, not an ongoing run, so every rung of
|
||||
// the music ladder is standalone.
|
||||
track: 'standalone',
|
||||
album: 'standalone',
|
||||
artist: 'standalone',
|
||||
musicVideo: 'standalone',
|
||||
djMix: 'standalone',
|
||||
liveSet: 'standalone',
|
||||
// literature — the split that proves workForm is not a screen-only axis.
|
||||
book: 'standalone',
|
||||
audiobook: 'standalone',
|
||||
comic: 'serial',
|
||||
manga: 'serial',
|
||||
magazine: 'serial',
|
||||
// science
|
||||
paper: 'standalone',
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the domain a `contentKind` belongs to.
|
||||
*
|
||||
* Returns `undefined` for `pack` (not derivable from the kind — the writer
|
||||
* supplies it), for sidecar/format kinds that route nowhere, and for any kind
|
||||
* this build has never heard of. A caller that already knows the domain from
|
||||
* its own context must prefer that over this table.
|
||||
*/
|
||||
export function domainForContentKind(contentKind: string | undefined | null): Domain | undefined {
|
||||
if (!contentKind) return undefined;
|
||||
return domainByContentKind[contentKind.trim()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the work form a `contentKind` sits on. Same contract as
|
||||
* {@link domainForContentKind}: `undefined` means "not derivable", never a
|
||||
* guess. Whoever writes `contentKind` writes both of these in the same breath.
|
||||
*/
|
||||
export function workFormForContentKind(contentKind: string | undefined | null): WorkForm | undefined {
|
||||
if (!contentKind) return undefined;
|
||||
return workFormByContentKind[contentKind.trim()];
|
||||
}
|
||||
@@ -1,5 +1,14 @@
|
||||
import {SimpleFileType} from "../SimpleFileType.js";
|
||||
|
||||
/**
|
||||
* Extension → `fileType`. Half of the canonical classification table
|
||||
* documented in METADATA_KEYS.md §1 (the other half is
|
||||
* {@link mimeTypeMappings}); this file is the source of truth and the doc
|
||||
* mirrors it.
|
||||
*
|
||||
* An extension absent from this map resolves to `other`, which makes the file
|
||||
* invisible to every typed query — so a missing entry is not a cosmetic gap.
|
||||
*/
|
||||
export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
// Audio
|
||||
'mp3': 'audio',
|
||||
@@ -7,7 +16,19 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'flac': 'audio',
|
||||
'aac': 'audio',
|
||||
'ogg': 'audio',
|
||||
'oga': 'audio',
|
||||
'opus': 'audio',
|
||||
'm4a': 'audio',
|
||||
// `m4b` is the audiobook container — without it `contentKind=audiobook`
|
||||
// had no extension that resolved to `audio` (METADATA_KEYS.md §14.15).
|
||||
'm4b': 'audio',
|
||||
'wma': 'audio',
|
||||
'aiff': 'audio',
|
||||
'aif': 'audio',
|
||||
'ape': 'audio',
|
||||
'wv': 'audio',
|
||||
'dsf': 'audio',
|
||||
'mka': 'audio',
|
||||
// Video
|
||||
'mp4': 'video',
|
||||
'mkv': 'video',
|
||||
@@ -27,6 +48,8 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'ts': 'video',
|
||||
'vob': 'video',
|
||||
'ogm': 'video',
|
||||
'ogv': 'video',
|
||||
'asf': 'video',
|
||||
'divx': 'video',
|
||||
'xvid': 'video',
|
||||
// Document
|
||||
@@ -37,31 +60,64 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'pptx': 'document',
|
||||
'xls': 'document',
|
||||
'xlsx': 'document',
|
||||
'odt': 'document',
|
||||
'ods': 'document',
|
||||
'odp': 'document',
|
||||
'txt': 'document',
|
||||
'md': 'document',
|
||||
'rtf': 'document',
|
||||
// Images (are document)
|
||||
'jpg': 'document',
|
||||
'jpeg': 'document',
|
||||
'png': 'document',
|
||||
'gif': 'document',
|
||||
'bmp': 'document',
|
||||
'tif': 'document',
|
||||
'tiff': 'document',
|
||||
'svg': 'document',
|
||||
'webp': 'document',
|
||||
// Document — literature formats. Their absence blocked meta-read outright
|
||||
// (METADATA_KEYS.md §14.15).
|
||||
'epub': 'document',
|
||||
'mobi': 'document',
|
||||
'azw': 'document',
|
||||
'azw3': 'document',
|
||||
'djvu': 'document',
|
||||
// Images
|
||||
// ⚠ These used to map to `document` ("Images (are document)"), which meant
|
||||
// the `image` bucket was unreachable from this table while Rust feeders
|
||||
// wrote it anyway. A consumer asking for documents must not receive
|
||||
// posters — see METADATA_KEYS.md §14.15.
|
||||
'jpg': 'image',
|
||||
'jpeg': 'image',
|
||||
'png': 'image',
|
||||
'gif': 'image',
|
||||
'bmp': 'image',
|
||||
'tif': 'image',
|
||||
'tiff': 'image',
|
||||
'svg': 'image',
|
||||
'webp': 'image',
|
||||
'avif': 'image',
|
||||
'heic': 'image',
|
||||
'heif': 'image',
|
||||
'ico': 'image',
|
||||
// Archive
|
||||
'zip': 'archive',
|
||||
'rar': 'archive',
|
||||
'7z': 'archive',
|
||||
'tar': 'archive',
|
||||
'tgz': 'archive',
|
||||
'gz': 'archive',
|
||||
'bz2': 'archive',
|
||||
'xz': 'archive',
|
||||
'zst': 'archive',
|
||||
// Archive — comic containers. A `.cbz` is literally a zip, so `archive` is
|
||||
// the honest byte-level answer; `contentKind=comic` + `domain=literature`
|
||||
// are what route it to meta-read (METADATA_KEYS.md §14.16).
|
||||
'cbz': 'archive',
|
||||
'cbr': 'archive',
|
||||
//subtitles
|
||||
'srt': 'subtitle',
|
||||
'sub': 'subtitle',
|
||||
'sbv': 'subtitle',
|
||||
'vtt': 'subtitle',
|
||||
// ⚠ ASS/SSA is what meta-watch renders through libass — the client's
|
||||
// primary subtitle format was the one this classifier did not know.
|
||||
'ass': 'subtitle',
|
||||
'ssa': 'subtitle',
|
||||
'ttml': 'subtitle',
|
||||
'dfxp': 'subtitle',
|
||||
'smi': 'subtitle',
|
||||
//torrent
|
||||
'torrent': 'torrent',
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,11 +1,32 @@
|
||||
import {SimpleFileType} from "../SimpleFileType.js";
|
||||
|
||||
/**
|
||||
* MIME type → `fileType`. The other half of the canonical classification
|
||||
* table (see {@link extensionMappings} and METADATA_KEYS.md §1).
|
||||
*
|
||||
* ⚠ Resolution is conservative on purpose: when a sniffed MIME maps to a
|
||||
* definite value that *disagrees* with the extension's value,
|
||||
* `FileTypeConfigurable.getFileType` returns `undefined` rather than guessing
|
||||
* — a wrong `fileType` routes a file to a handler that cannot open it. Adding
|
||||
* a mapping here therefore also creates a new way to disagree; only add ones
|
||||
* whose extension counterpart agrees.
|
||||
*/
|
||||
export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'text/plain': 'undefined',//text could be anything document or subtitle or other
|
||||
// Audio MIME types
|
||||
'audio/mpeg': 'audio',
|
||||
'audio/wav': 'audio',
|
||||
'audio/x-wav': 'audio',
|
||||
'audio/flac': 'audio',
|
||||
'audio/ogg': 'audio',
|
||||
'audio/opus': 'audio',
|
||||
'audio/aac': 'audio',
|
||||
// ftyp brands `M4A `/`M4B ` sniff as audio/mp4 while a real video mp4
|
||||
// sniffs as video/mp4, so this does not collide with the `mp4` extension.
|
||||
'audio/mp4': 'audio',
|
||||
'audio/x-ms-wma': 'audio',
|
||||
'audio/x-aiff': 'audio',
|
||||
'audio/x-ape': 'audio',
|
||||
// Video MIME types
|
||||
'video/mp4': 'video',
|
||||
'video/mpeg': 'video',
|
||||
@@ -13,8 +34,12 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'video/x-msvideo': 'video',
|
||||
'video/x-flv': 'video',
|
||||
'video/x-ms-wmv': 'video',
|
||||
'video/x-ms-asf': 'video',
|
||||
'video/x-matroska': 'video',
|
||||
'video/webm': 'video',
|
||||
'video/ogg': 'video',
|
||||
'video/3gpp': 'video',
|
||||
'video/mp2t': 'video',
|
||||
// Document MIME types
|
||||
'application/pdf': 'document',
|
||||
'application/msword': 'document',
|
||||
@@ -23,20 +48,47 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document',
|
||||
'application/vnd.ms-excel': 'document',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'document',
|
||||
'image/jpeg': 'document',
|
||||
'image/png': 'document',
|
||||
'image/gif': 'document',
|
||||
'application/vnd.oasis.opendocument.text': 'document',
|
||||
'application/vnd.oasis.opendocument.spreadsheet': 'document',
|
||||
'application/vnd.oasis.opendocument.presentation': 'document',
|
||||
'application/rtf': 'document',
|
||||
// Literature containers (METADATA_KEYS.md §14.15)
|
||||
'application/epub+zip': 'document',
|
||||
'application/x-mobipocket-ebook': 'document',
|
||||
'image/vnd.djvu': 'document',
|
||||
'text/markdown': 'document',
|
||||
// Image MIME types — no longer folded into `document` (§14.15)
|
||||
'image/jpeg': 'image',
|
||||
'image/png': 'image',
|
||||
'image/gif': 'image',
|
||||
'image/webp': 'image',
|
||||
'image/bmp': 'image',
|
||||
'image/tiff': 'image',
|
||||
'image/svg+xml': 'image',
|
||||
'image/avif': 'image',
|
||||
'image/heic': 'image',
|
||||
'image/x-icon': 'image',
|
||||
// Archive MIME types
|
||||
'application/zip': 'archive',
|
||||
'application/x-rar-compressed': 'archive',
|
||||
'application/vnd.rar': 'archive',
|
||||
'application/x-7z-compressed': 'archive',
|
||||
'application/gzip': 'archive',
|
||||
'application/x-bzip2': 'archive',
|
||||
'application/x-xz': 'archive',
|
||||
'application/zstd': 'archive',
|
||||
'application/x-tar': 'archive',
|
||||
// Comic containers sniff as their underlying zip/rar, which agrees with
|
||||
// the `cbz`/`cbr` extensions — both say `archive`.
|
||||
'application/vnd.comicbook+zip': 'archive',
|
||||
'application/vnd.comicbook-rar': 'archive',
|
||||
//torrent
|
||||
'application/x-bittorrent': 'torrent',
|
||||
//subtitles
|
||||
'text/vtt': 'subtitle',
|
||||
'text/srt': 'subtitle',
|
||||
'text/ssa': 'subtitle',
|
||||
'text/x-ssa': 'subtitle',
|
||||
'text/sub': 'subtitle',
|
||||
};
|
||||
'application/x-subrip': 'subtitle',
|
||||
};
|
||||
|
||||
@@ -13,6 +13,7 @@ export * from './tools/StringArrayOperation.js';
|
||||
export * from './tools/TwoStringsOperation.js';
|
||||
|
||||
export * from './config/AnimeKeywords.js';
|
||||
export * from './config/DomainMapping.js';
|
||||
export * from './config/ExtentionsMapping.js';
|
||||
export * from './config/KeywordsArray.js';
|
||||
export * from './config/MimeTypeMapping.js';
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
|
||||
|
||||
/**
|
||||
* Decoding a bare CIDv1 back into the algorithm that produced it.
|
||||
*
|
||||
* This is **the** TypeScript CID decoder for the platform. It exists because a
|
||||
* bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
|
||||
* "the sha2-256 of this file" has to ask the multicodec, not a string.
|
||||
*
|
||||
* Before this module, meta-dup carried a private hand-rolled base32+uvarint
|
||||
* decoder, the meta-sort editor carried a second one with a *different* rank
|
||||
* ladder, and meta-fuse had none at all (so it could only read the legacy
|
||||
* `cid_midhash256` field, and silently failed on any record that didn't have
|
||||
* one). One implementation, exported, replaces all three.
|
||||
*
|
||||
* ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
|
||||
* has two encoders that disagree about the codec: meta-hash writes
|
||||
* `codec == algorithm code`, while the fullhash plugin and the gateway wrap
|
||||
* every digest with the raw codec `0x55`. The multihash code is the algorithm in
|
||||
* both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
|
||||
*/
|
||||
|
||||
/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidMultihashCode(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).multihash.code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The content codec of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidCodec(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first member of `cids` whose multihash code is `mhCode`.
|
||||
*
|
||||
* Takes the raw code so callers can ask for digests meta-hash doesn't itself
|
||||
* compute (e.g. the gateway's per-file btih codes).
|
||||
*/
|
||||
export function pickCidByMulticodec(
|
||||
cids: readonly string[] | undefined,
|
||||
mhCode: number
|
||||
): string | undefined {
|
||||
return cids?.find(cid => cidMultihashCode(cid) === mhCode);
|
||||
}
|
||||
|
||||
/** The member of `cids` produced by `algo`, if present. */
|
||||
export function pickCidByAlgorithm(
|
||||
cids: readonly string[] | undefined,
|
||||
algo: CidAlgorithm
|
||||
): string | undefined {
|
||||
return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Has `algo` already been computed for this cid set?
|
||||
*
|
||||
* The "skip if already computed" check. It used to be an O(1) property lookup
|
||||
* (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
|
||||
* That is a handful of microseconds against a hash computation that streams the
|
||||
* whole file — the trade is not close.
|
||||
*/
|
||||
export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
|
||||
return pickCidByAlgorithm(cids, algo) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `cid` to `cids` unless an equal string is already there.
|
||||
*
|
||||
* Dedup is on the exact string. Two encoders can spell the same digest
|
||||
* differently (codec 0x55 vs codec == mh), and both spellings are legitimate
|
||||
* members — meta-core's reverse index maps each to the same record.
|
||||
*/
|
||||
export function addCid(cids: string[], cid: string): void {
|
||||
if (cid && !cids.includes(cid)) {
|
||||
cids.push(cid);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export const codeTable = {
|
||||
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
|
||||
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
|
||||
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
|
||||
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
|
||||
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
|
||||
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
|
||||
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
|
||||
[CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {create} from "multiformats/hashes/digest";
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {codeTable} from "@root/file-id/CodeTable";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param stream The Readable stream of the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @returns Array of bare CIDv1 strings, in the order of `algorithms`
|
||||
*/
|
||||
export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
export async function computeCIDs({stream, algorithms, createHasher}: {
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
algorithms: CID_ALGORITHM_NAMES[];
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
|
||||
algorithms: CidAlgorithm[];
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
|
||||
}): Promise<string[]> {
|
||||
const hashers = await hasherDefiner(algorithms,createHasher);
|
||||
const hashers = await hasherDefiner(algorithms, createHasher);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const {done, value} = await reader.read();
|
||||
@@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
return cidFinalize(hashers);
|
||||
}
|
||||
|
||||
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
|
||||
async function hasherDefiner(algorithms: CidAlgorithm[],
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
|
||||
hasher: SimpleHash,
|
||||
code: CID_ALGORITHM_CODES
|
||||
}[]> {
|
||||
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
|
||||
const hashers = algorithms
|
||||
.filter(algo => algo in CID_ALGORITHM_CODES)
|
||||
.map(async algo => ({
|
||||
hasher: await createHasher(algo),
|
||||
code: codeTable[algo]
|
||||
code: CID_ALGORITHM_CODES[algo]
|
||||
}));
|
||||
return await Promise.all(hashers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
|
||||
* plugin and the gateway instead use the raw codec (0x55) and carry the
|
||||
* algorithm only in the multihash. Both are valid CIDs for the same digest, and
|
||||
* both must decode to the same algorithm — which is why every consumer selects
|
||||
* on the **multihash code**, never the codec. See `CidDecode.ts`.
|
||||
*/
|
||||
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
|
||||
return await Promise.all(hashers.map(async ({code, hasher}) => {
|
||||
const hashBuffer = await hasher.digest();
|
||||
const digest = create(code, hashBuffer);
|
||||
return CID.createV1(code, digest).toString();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +1,32 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {createHash} from "crypto";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BtihV2Hasher} from "./BtihV2Hasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @param algo
|
||||
* Build the stream hasher for one algorithm.
|
||||
*
|
||||
* `midhash256` is absent on purpose: it is not a stream digest (it hashes the
|
||||
* file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
|
||||
*/
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return createHash("sha256");
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return createHash("sha1");
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return createHash("md5");
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return createHash("sha3-256");
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return createHash("sha3-384");
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
case CID_ALGORITHM_NAMES.btih_v2:
|
||||
case 'btih_v2':
|
||||
return new BtihV2Hasher();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BrowserHasher} from "@root/file-id/BrowserHasher";
|
||||
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
/**
|
||||
* Browser build. Note it supports fewer algorithms than the node one: there is
|
||||
* no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
|
||||
* either throws.
|
||||
*/
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
const hasher = (() => {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return new BrowserHasher('sha-256');
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return new BrowserHasher('sha-1');
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return new BrowserHasher('md5');
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return new BrowserHasher('sha3-256');
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return new BrowserHasher('sha3-384');
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
@@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHas
|
||||
}
|
||||
|
||||
return hasher;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export interface FileIDComputerInterface{
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
|
||||
}
|
||||
@@ -1,10 +1,16 @@
|
||||
import {Piscina} from "piscina";
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
|
||||
export class FileIDComputerWorker{
|
||||
private piscina: Piscina;
|
||||
|
||||
constructor(workerPath?:string) {
|
||||
// `WORKER_URL` is the escape hatch the tests have always set — it just
|
||||
// wasn't read. Honour it before falling back to path derivation, which
|
||||
// only works for a caller running from `dist/`.
|
||||
if (!workerPath && process.env.WORKER_URL) {
|
||||
workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
|
||||
}
|
||||
if(!workerPath) {
|
||||
// Construct the URL for the current module
|
||||
let distFolder = import.meta.dirname;
|
||||
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
|
||||
public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
|
||||
return this.piscina.run({filePath, algorithms});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {MultiHashData} from '../hash-compute/MultiHashData';
|
||||
import {pickCidByAlgorithm} from './CidDecode';
|
||||
import {CID} from 'multiformats/cid';
|
||||
|
||||
/**
|
||||
@@ -22,13 +23,13 @@ export class MagnetLinkGenerator {
|
||||
/**
|
||||
* Generate BitTorrent v2 magnet link from metadata
|
||||
*
|
||||
* @param metadata - File metadata containing cid_btih_v2
|
||||
* @param metadata - File metadata whose `cids` include a btih-v2 CID
|
||||
* @param fileName - Original filename for display name
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if btih_v2 not available
|
||||
*
|
||||
* @example
|
||||
* const metadata = {cid_btih_v2: 'bafyb...'};
|
||||
* const metadata = {cids: ['bafyb...']};
|
||||
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
|
||||
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
|
||||
*/
|
||||
@@ -37,7 +38,9 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
const cidString = metadata.cid_btih_v2;
|
||||
// The btih-v2 CID is recovered from the bare list by its multicodec
|
||||
// (0x10B7), not by a field name. See CidDecode.ts.
|
||||
const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
if (!cidString) {
|
||||
return null; // BitTorrent v2 hash not computed
|
||||
}
|
||||
@@ -178,6 +181,6 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
|
||||
return this.generate({cids: [cidString]}, fileName, fileSize);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
import fs from "fs";
|
||||
import {Readable} from 'stream';
|
||||
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
|
||||
@@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher";
|
||||
*/
|
||||
export default async function computeCIDs({filePath, algorithms}: {
|
||||
filePath: string;
|
||||
algorithms: CID_ALGORITHM_NAMES[]
|
||||
algorithms: CidAlgorithm[]
|
||||
}): Promise<string[]> {
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
return computeCIDsInternal({stream, algorithms, createHasher});
|
||||
|
||||
@@ -1,30 +1,29 @@
|
||||
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, MultiHashData} from "./MultiHashData";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class HashComputer{
|
||||
export class HashComputer {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[],
|
||||
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
|
||||
constructor(private targetHash: CidAlgorithm[],
|
||||
private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
|
||||
}
|
||||
|
||||
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
// Which digests are still missing? On a bare-CID list this costs a decode
|
||||
// per member rather than a property lookup — negligible against streaming
|
||||
// the whole file through a hasher.
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +1,28 @@
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {Readable} from "stream";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import fs from "fs";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
export class HashComputerFile implements ComputeInterface{
|
||||
export class HashComputerFile implements ComputeInterface {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
constructor(private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,59 @@
|
||||
import {HashComputerWorker} from "./HashComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {HashIndexManager} from "./HashIndexManager";
|
||||
import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {stat} from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* The translation point between the two shapes.
|
||||
*
|
||||
* `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
|
||||
* persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
|
||||
* **per-algorithm columns**, because its job is the O(1) "already computed?"
|
||||
* check. Neither shape leaks into the other; this class converts.
|
||||
*/
|
||||
export class HashComputerIndexCache implements ComputeInterface {
|
||||
private hashIndexManager: HashIndexManager;
|
||||
private hashComputer: HashComputerWorker;
|
||||
|
||||
constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash,workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
|
||||
constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
await this.hashIndexManager.init();
|
||||
let stats = await stat(filePath);
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
|
||||
const stats = await stat(filePath);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// Cache hit: lift the cached CIDs into the bare list.
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
|
||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!metadata[hash] && indexLine[hash]) {
|
||||
metadata[hash] = indexLine[hash];
|
||||
if (indexLine) {
|
||||
for (const algo of this.targetHash) {
|
||||
const cached = indexLine[algo];
|
||||
if (cached && !hasAlgorithm(cids, algo)) {
|
||||
addCid(cids, cached);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute whatever is still missing (a no-op on a full cache hit).
|
||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
||||
|
||||
// Write back: project the bare list onto the index's per-algorithm columns.
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of this.targetHash) {
|
||||
const cid = pickCidByAlgorithm(cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
}
|
||||
|
||||
async getHashIndexManager(): Promise<HashIndexManager> {
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
|
||||
export class HashComputerWorker implements ComputeInterface {
|
||||
private fileIDComputer: FileIDComputerWorker;
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
|
||||
constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
|
||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
||||
}
|
||||
|
||||
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
|
||||
import {existsAsync} from "../utils/ExistsAsync";
|
||||
import path from "path";
|
||||
import {clearInterval} from "node:timers";
|
||||
import {CID_ALGORITHM_NAMES} from "./MultiHashData";
|
||||
import {CidAlgorithm} from "./MultiHashData";
|
||||
import {stat} from "fs/promises";
|
||||
import {Stats} from "node:fs";
|
||||
|
||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
||||
/**
|
||||
* One row of the on-disk hash-index cache.
|
||||
*
|
||||
* This is the one place a **per-algorithm column** is still the right shape, and
|
||||
* it is not the deprecated one. The index answers "have I already computed sha256
|
||||
* for this file?" on every scan, which a column makes an O(1) lookup and a bare
|
||||
* CID list would make a decode-per-member. It is also strictly local: a wipeable
|
||||
* cache under `/data/cache/hash-index/` (see
|
||||
* `docs/project-architecture/cache-persistence.md`), never a record, never on the
|
||||
* wire.
|
||||
*
|
||||
* What changed is the *naming*: columns are keyed by bare algorithm
|
||||
* (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
|
||||
* so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
|
||||
* shape and the cache shape are now separate concerns that cannot leak into each
|
||||
* other — which is the whole point of the split (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
|
||||
path: string;
|
||||
size: string;
|
||||
mtime: string;
|
||||
@@ -21,15 +38,15 @@ export class HashIndexManager {
|
||||
private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
|
||||
private intervalId: any;
|
||||
private intervalTime: number = 30000;
|
||||
private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private indexOpsInProgress: boolean = false;
|
||||
private hasChanged: boolean = false;
|
||||
private initialLoad: Promise<void>;
|
||||
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
|
||||
private filePaths: { [key in CidAlgorithm]?: string } = {};
|
||||
|
||||
constructor(private indexFolderPath: string,
|
||||
private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
hasFileInCache(filePath: string,stats:Stats): boolean {
|
||||
@@ -81,7 +98,7 @@ export class HashIndexManager {
|
||||
}
|
||||
|
||||
// Function to check CSV headers
|
||||
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
|
||||
private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
|
||||
const records = parseSync(csvContent, {
|
||||
bom: true,
|
||||
columns: true,
|
||||
@@ -116,7 +133,7 @@ export class HashIndexManager {
|
||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
||||
}
|
||||
|
||||
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (await existsAsync(this.filePaths[hash])) {
|
||||
// check the file size and if it did not change, do not read the file
|
||||
const stats = await fs.stat(this.filePaths[hash]);
|
||||
@@ -147,7 +164,7 @@ export class HashIndexManager {
|
||||
return Array.from(this.cache.values());
|
||||
}
|
||||
|
||||
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (!(await existsAsync(this.filePaths[hash]))) {
|
||||
return [];
|
||||
}
|
||||
@@ -178,10 +195,8 @@ export class HashIndexManager {
|
||||
|
||||
public async saveCacheToFile(): Promise<void> {
|
||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`);
|
||||
return;
|
||||
}
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`);
|
||||
this.hasChanged = false;
|
||||
this.indexOpsInProgress = true;
|
||||
const start = performance.now();
|
||||
@@ -274,17 +289,15 @@ export class HashIndexManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
|
||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
||||
throw new Error('Invalid parameters');
|
||||
}
|
||||
// Check that at least one target hash is provided
|
||||
const providedHashes = this.targetHash.filter(hash => hashs[hash]);
|
||||
if (providedHashes.length === 0) {
|
||||
console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`);
|
||||
throw new Error('At least one target hash must be provided');
|
||||
}
|
||||
console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`);
|
||||
const size = fileSize + "";
|
||||
const baseName = path.basename(filePath);
|
||||
const cacheKey = `${baseName}-${size}-${mtime}`;
|
||||
|
||||
@@ -1,21 +1,26 @@
|
||||
export interface MultiHashData {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
|
||||
cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
|
||||
}
|
||||
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* according to
|
||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
||||
* Multicodec / multihash codes for every digest meta-hash can compute.
|
||||
*
|
||||
* Source of truth for these numbers. The same codes appear in the CID utility
|
||||
* RANK, which is a separate thing this file does not implement:
|
||||
* - `packages/meta-core/internal/cid/rank.go`
|
||||
* - the `cid_rank.rs` in each of meta-search and meta-share
|
||||
* - `/cid-rank-vectors.json` (the golden fixture pinning those three together)
|
||||
*
|
||||
* ⚠ This package used to vendor that fixture at `src/tests/cid-rank-vectors.json`
|
||||
* and no test ever loaded it, so every vector added cost a commit here for
|
||||
* nothing. Removed 2026-09-09. If a TypeScript rank implementation is ever
|
||||
* written, re-add the path to `scripts/check-cid-vectors.sh` *and* a test that
|
||||
* asserts it — a vendored fixture nobody reads is not a guard.
|
||||
*
|
||||
* ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
|
||||
* `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
|
||||
* the gateway wrap every digest with the **raw** codec `0x55` and put the
|
||||
* algorithm in the multihash code. The multihash code is the algorithm in both
|
||||
* conventions — so anything selecting a CID by algorithm must read the
|
||||
* **multihash code**, never the content codec. See `CidDecode.ts`.
|
||||
*
|
||||
* https://github.com/multiformats/multicodec/blob/master/table.csv
|
||||
*/
|
||||
export enum CID_ALGORITHM_CODES {
|
||||
crc32 = 0x0132,
|
||||
@@ -24,28 +29,55 @@ export enum CID_ALGORITHM_CODES {
|
||||
sha256 = 0x12,
|
||||
sha3_256 = 0x16,
|
||||
sha3_384 = 0x15,
|
||||
midhash256 = 0x1000, // Custom code for midhash256
|
||||
btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
|
||||
midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
|
||||
btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM_NAMES {
|
||||
crc32 = 'cid_crc32',
|
||||
md5 = 'cid_md5',
|
||||
sha1 = 'cid_sha1',
|
||||
sha256 = 'cid_sha2-256',
|
||||
sha3_256 = 'cid_sha3-256',
|
||||
sha3_384 = 'cid_sha3-384',
|
||||
midhash256 = 'cid_midhash256',
|
||||
btih_v2 = 'cid_btih_v2',
|
||||
/**
|
||||
* An algorithm *selector*: which digests to compute, and which column of the
|
||||
* local hash-index cache to read.
|
||||
*
|
||||
* This is deliberately **not** a metadata field name. The old
|
||||
* `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
|
||||
* duty — selector, record property name, and CSV column — and the middle duty
|
||||
* was the deprecated one: those `cid_<algo>` fields were stored on records but
|
||||
* never reverse-indexed by meta-core, leaving records unresolvable by their own
|
||||
* CID. meta-core now rejects them at the write boundary (400). See
|
||||
* METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* The triple duty also rotted on its own: the interface declared `cid_sha3_384`
|
||||
* (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
|
||||
* comment on it named the wrong multicodec. A self-describing CID cannot drift
|
||||
* from its own name that way.
|
||||
*/
|
||||
export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
|
||||
|
||||
/** Every algorithm meta-hash knows how to compute. */
|
||||
export const CID_ALGORITHMS: CidAlgorithm[] = [
|
||||
'crc32',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'sha3_256',
|
||||
'sha3_384',
|
||||
'midhash256',
|
||||
'btih_v2',
|
||||
];
|
||||
|
||||
/**
|
||||
* The output of a hash computation: **bare CIDv1 strings, nothing else**.
|
||||
*
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
|
||||
* per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
|
||||
* specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
|
||||
*
|
||||
* This is persisted onto a record as the bare-CID key-set
|
||||
* (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
|
||||
*/
|
||||
export interface MultiHashData {
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM {
|
||||
crc32 = 'crc32',
|
||||
md5 = 'md5',
|
||||
sha1 = 'sha1',
|
||||
sha256 = 'sha2-256',
|
||||
sha3_256 = 'sha3-256',
|
||||
sha3_384 = 'sha3-384',
|
||||
midhash256 = 'midhash256',
|
||||
btih_v2 = 'btih-v2',
|
||||
}
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker
|
||||
export * from './file-id/FastHash.js'; // Fast-hash computation
|
||||
export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
|
||||
export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
|
||||
export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
|
||||
|
||||
// The CID model: algorithm codes, the algorithm selector, and MultiHashData
|
||||
// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
|
||||
// key-set needs them to recover a specific digest — see CidDecode.
|
||||
export * from './hash-compute/MultiHashData.js';
|
||||
|
||||
//hash-compute
|
||||
export * from './hash-compute/HashComputerIndexCache.js';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher';
|
||||
import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib";
|
||||
import {HashComputerWorker, pickCidByAlgorithm} from "../lib";
|
||||
import {readFile} from 'fs/promises';
|
||||
import {createHash} from 'crypto';
|
||||
import * as process from "process";
|
||||
@@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should integrate with HashComputerWorker', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2,
|
||||
CID_ALGORITHM_NAMES.sha256
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
console.log('Computed metadata:', metadata);
|
||||
// The digests land in a bare CID list; each is recovered by its
|
||||
// multicodec, not by a field name.
|
||||
const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
expect(btih).toBeDefined();
|
||||
expect(typeof btih).toBe('string');
|
||||
expect(btih!.startsWith('b')).toBe(true);
|
||||
|
||||
// Verify btih_v2 CID was computed
|
||||
expect(metadata).toHaveProperty('cid_btih_v2');
|
||||
expect(typeof metadata.cid_btih_v2).toBe('string');
|
||||
expect(metadata.cid_btih_v2.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify CID format (should start with 'b' for base32 CIDv1)
|
||||
expect(metadata.cid_btih_v2.startsWith('b')).toBe(true);
|
||||
|
||||
// Should also have SHA-256 for comparison
|
||||
expect(metadata).toHaveProperty('cid_sha2-256');
|
||||
// Should also have SHA-256 for comparison.
|
||||
expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined();
|
||||
});
|
||||
|
||||
it('should compute piece length correctly', () => {
|
||||
@@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should produce valid CID format with codec 0x10B7', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
const cid = metadata.cid_btih_v2;
|
||||
console.log('BitTorrent v2 CID:', cid);
|
||||
const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
|
||||
// The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on
|
||||
// multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7"
|
||||
// assertion this test is named for.
|
||||
|
||||
// Verify CID format
|
||||
expect(cid).toBeDefined();
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
|
||||
import {
|
||||
CidAlgorithm,
|
||||
HashComputerIndexCache,
|
||||
HashComputerWorker,
|
||||
pickCidByAlgorithm,
|
||||
} from "../lib";
|
||||
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import * as process from "process";
|
||||
import { mkdir, rm, rmdir } from "fs/promises";
|
||||
import {mkdir, rm, stat} from "fs/promises";
|
||||
|
||||
process.env.WORKER_URL = "./dist/worker.js";
|
||||
|
||||
@@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js";
|
||||
* Remember to build the worker each time you change the worker code.
|
||||
*/
|
||||
|
||||
const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384'];
|
||||
|
||||
/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */
|
||||
const FIXTURE_BYTES = 490;
|
||||
|
||||
/**
|
||||
* The expected digests of `src/tests/test.txt`, as bare CIDv1 strings.
|
||||
*
|
||||
* Independently reproducible — these are not "whatever the code printed":
|
||||
*
|
||||
* $ sha256sum src/tests/test.txt
|
||||
* be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec
|
||||
* $ md5sum → c8971ace120d1393b658eabb45d08c88
|
||||
* $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7
|
||||
*
|
||||
* and each CID below wraps exactly that digest under its multicodec (meta-hash's
|
||||
* encoder sets `codec == multihash code`).
|
||||
*
|
||||
* ⚠ These values were **re-pinned**. The previous ones described a 499-byte
|
||||
* CRLF copy of the fixture, from before `.gitattributes` normalised it to LF —
|
||||
* so on any LF checkout the digests could never match and this suite was red.
|
||||
* The mismatch was invisible because the worker path was also broken, so the
|
||||
* assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the
|
||||
* fixture's bytes ever shift again, the test says *that*, instead of printing
|
||||
* six inscrutable CID diffs.
|
||||
*/
|
||||
const EXPECTED: Record<string, string> = {
|
||||
crc32: 'bagzafmqcat2hnot3',
|
||||
md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea',
|
||||
sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y',
|
||||
sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q',
|
||||
sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi',
|
||||
sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq',
|
||||
};
|
||||
|
||||
/**
|
||||
* Every expected digest is present in the bare list, and each is recoverable by
|
||||
* its algorithm **through its multicodec** — there is no field name to look up.
|
||||
* That is the whole point of the `cids: string[]` shape: a CIDv1 is
|
||||
* self-describing, so the record never carries a per-algorithm key that could
|
||||
* drift from the CID it names (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
function expectAllDigests(cids: string[] | undefined) {
|
||||
expect(cids).toBeDefined();
|
||||
expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort());
|
||||
|
||||
for (const algo of ALGOS) {
|
||||
expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('add', () => {
|
||||
it('meta data compute', async () => {
|
||||
let hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
it('the fixture has the bytes these CIDs were pinned against', async () => {
|
||||
const {size} = await stat('./src/tests/test.txt');
|
||||
expect(size).toBe(FIXTURE_BYTES);
|
||||
});
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp",{recursive:true});
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
|
||||
it('meta data compute', async () => {
|
||||
const hashComputer = new HashComputerWorker(ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp", {recursive: true, force: true});
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
await (await hashComputer.getHashIndexManager()).saveCacheToFile();
|
||||
});
|
||||
|
||||
it('cache', async () => {
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
|
||||
// The on-disk index is the one place per-algorithm columns survive — it
|
||||
// answers "already computed?" in O(1) and never leaves this machine. Note
|
||||
// the columns are bare algorithm names now (`sha256`), not the removed
|
||||
// record-field names (`cid_sha2-256`).
|
||||
const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
delete indexLine['mtime'];
|
||||
expect(indexLine).toEqual({
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
|
||||
"path": "test.txt",
|
||||
"size": "499",
|
||||
...EXPECTED,
|
||||
path: "test.txt",
|
||||
size: String(FIXTURE_BYTES),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('a second pass reuses the cache and recomputes nothing', async () => {
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
// Same digests, recovered from the index rather than by re-hashing.
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -22,13 +22,13 @@ describe('MetaData', () => {
|
||||
});
|
||||
|
||||
it('should set and get nested metadata using nested keys', () => {
|
||||
root.at('titles').at('eng').set("Star Wars");
|
||||
root.at('titles').at('jpn').set("Staru Warsu");
|
||||
expect(root.at('titles').at('eng').get()).to.equal("Star Wars");
|
||||
expect(root.at('titles').at('jpn').get()).to.equal("Staru Warsu");
|
||||
expect(root.at('titles').get()).to.deep.equal({eng: "Star Wars", jpn: "Staru Warsu"});
|
||||
root.at('titles').at('eng').at('Star Wars').set(true);
|
||||
root.at('titles').at('jpn').at('Staru Warsu').set(true);
|
||||
expect(root.at('titles').at('eng').at('Star Wars').get()).to.equal(true);
|
||||
expect(root.at('titles').at('jpn').at('Staru Warsu').get()).to.equal(true);
|
||||
expect(root.at('titles').get()).to.deep.equal({eng: {"Star Wars": true}, jpn: {"Staru Warsu": true}});
|
||||
expect(root.get()).to.deep.equal({
|
||||
titles: {eng: "Star Wars", jpn: "Staru Warsu"}
|
||||
titles: {eng: {"Star Wars": true}, jpn: {"Staru Warsu": true}}
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
@@ -4,13 +4,10 @@ export const FileMetaDataName = [
|
||||
"byteSize",
|
||||
]
|
||||
|
||||
// Sibling CIDs are stored as the bare-CID key-set `cids/<cid>` (see HashMeta),
|
||||
// not as per-algorithm cid_* fields. The single member prefix is `cids`.
|
||||
export const hashMetaDataName = [
|
||||
"cid_crc32",
|
||||
"cid_md5",
|
||||
"cid_sha1",
|
||||
"cid_sha2-256",
|
||||
"cid_sha3-256",
|
||||
"cid_sha3_384",
|
||||
"cids",
|
||||
];
|
||||
|
||||
export const AnimeMetaData = [
|
||||
|
||||
@@ -1,32 +1,23 @@
|
||||
/**
|
||||
* HashMeta - Content identification hashes
|
||||
* HashMeta - Content identification hashes.
|
||||
*
|
||||
* Primary Global ID: cid_midhash256 (SHA-256 hash of middle 1MB + file size)
|
||||
* - Format: CID v1 with custom multicodec 0x1000
|
||||
* - Example: bafkr4ih5kapbjzqvmj7jxr... (base32 CID encoding)
|
||||
* Sibling CIDs are NOT stored as per-algorithm fields anymore. Every digest
|
||||
* of the file (sha2-256, sha1, md5, crc32, sha3-*, midhash256, btih-v2, …)
|
||||
* lives in a single bare-CID key-set on the record:
|
||||
*
|
||||
* Legacy IDs: cid_sha2-256, cid_sha1 (computed optionally for compatibility)
|
||||
* cids/<cidV1> = "true"
|
||||
*
|
||||
* Migration Note: cid_midhash256 is the new primary identifier replacing full SHA-256.
|
||||
* Old hashes remain for backward compatibility and external system integration.
|
||||
* All CIDs now use proper CID v1 format (IPFS-compatible structure).
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is
|
||||
* no per-algorithm field name and no `<algo>:` prefix. The canonical /
|
||||
* network CID is derived by rank from this set on read, never stored. The
|
||||
* midhash stays the file's address (hashId). See METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* Because the members are dynamic keys (`cids/<cid>`), they can't be typed as
|
||||
* named fields here; consumers read them by prefix-scanning the flat record
|
||||
* and recover the algorithm from each CID's multicodec.
|
||||
*/
|
||||
export interface HashMeta {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (CID v1 format with custom multicodec)
|
||||
}
|
||||
|
||||
export const HashMetaFields = [
|
||||
"cid_crc32",
|
||||
"cid_md5",
|
||||
"cid_sha1",
|
||||
"cid_sha2-256",
|
||||
"cid_sha3-256",
|
||||
"cid_sha3_384",
|
||||
"cid_midhash256",
|
||||
];
|
||||
// Intentionally empty: sibling CIDs are the `cids/<cid>` key-set (above),
|
||||
// not typed fields. Kept as a named type so existing `extends HashMeta`
|
||||
// composition sites stay valid.
|
||||
}
|
||||
@@ -1,10 +1,25 @@
|
||||
import {TypedRecordSet} from "./basic/RecordSet.js";
|
||||
import {ISO639_1_CODE} from "./basic/LanguageData.js";
|
||||
import {LANG3_CODE} from "./basic/LanguageData.js";
|
||||
|
||||
/**
|
||||
* The language triad (METADATA_KEYS.md §9).
|
||||
*
|
||||
* `languages` is the **union** — every language the record offers by any route
|
||||
* — and the only one a query filters on. The two splits refine it and are
|
||||
* optional. Rule #7: a writer that adds a member to a split MUST add the same
|
||||
* member to the union in the same write; nothing reconciles them afterwards.
|
||||
*/
|
||||
export interface LanguagesMeta{
|
||||
languages?: TypedRecordSet<ISO639_1_CODE>;
|
||||
/** The union: audio ∪ subtitles ∪ bare language statements. Filterable. */
|
||||
languages?: TypedRecordSet<LANG3_CODE>;
|
||||
/** Spoken/audio-track languages only. Refinement, never filtered on. */
|
||||
audioLanguages?: TypedRecordSet<LANG3_CODE>;
|
||||
/** Subtitle languages only (§8). Refinement, never filtered on. */
|
||||
subtitleLanguages?: TypedRecordSet<LANG3_CODE>;
|
||||
}
|
||||
|
||||
export const LanguagesMetaFields = [
|
||||
"languages",
|
||||
"audioLanguages",
|
||||
"subtitleLanguages",
|
||||
];
|
||||
@@ -8,8 +8,9 @@ export interface VideoMeta {
|
||||
increment?: string;
|
||||
movieYear?: string;
|
||||
videoType?: VideoType;
|
||||
title?: string;//display title (localized); the one name to show
|
||||
originalTitle?: string;//title in original language
|
||||
titles?: LanguageData<string>;// <ISO 639-3 lang codes, title>
|
||||
titles?: LanguageData<Record<string, true>>;// key-set: <ISO 639-3 lang code, {name: true}> — every clean name incl. AKAs (METADATA_KEYS.md §3)
|
||||
}
|
||||
|
||||
export const VideoMetaFields = [
|
||||
|
||||
@@ -1,9 +1,21 @@
|
||||
/**
|
||||
* key is language ISO 639-3 code
|
||||
* https://en.wikipedia.org/wiki/List_of_ISO_639-3_codes
|
||||
* key is a `lang3` code: ISO 639-2/**B** — `eng`, `jpn`, `fre`/`ger`/`chi`,
|
||||
* NOT the 639-2/T (= 639-3) spellings `fra`/`deu`/`zho`.
|
||||
*
|
||||
* This is the vocabulary the stack compares against: `languages:` query filters
|
||||
* match it literally, and `meta_feeder_sdk::lang::normalize_lang_code` folds
|
||||
* incoming codes onto it. See METADATA_KEYS.md §"Value formats" and §14.3 —
|
||||
* the card tier (`titles/*`, `posters/*`) is still on /T and has not converged.
|
||||
* https://en.wikipedia.org/wiki/List_of_ISO_639-2_codes
|
||||
*/
|
||||
export type LanguageData<T> = Record<string, T>;
|
||||
export type ISO639_1_CODE = string;
|
||||
export type LANG3_CODE = string;
|
||||
|
||||
/**
|
||||
* @deprecated Misnamed: no writer has ever emitted 2-letter ISO 639-1 codes
|
||||
* here. Use {@link LANG3_CODE}. Kept as an alias so existing imports compile.
|
||||
*/
|
||||
export type ISO639_1_CODE = LANG3_CODE;
|
||||
|
||||
/**
|
||||
* jpn, jpr and jpa already exist. Creating jpl is for japan-latin. Used for japanese romaji NON ISO 639-3 standard.
|
||||
|
||||
@@ -1,2 +1,16 @@
|
||||
export type SimpleFileType = 'audio' | 'video' | 'document' | 'archive' | 'other' | 'undefined';
|
||||
|
||||
/**
|
||||
* The `fileType` vocabulary — METADATA_KEYS.md §1. Kept in lockstep with
|
||||
* `filename-tool`'s `SimpleFileType`, which is the classifier that produces
|
||||
* these values; this copy is the wire/type-level mirror.
|
||||
*/
|
||||
export type SimpleFileType =
|
||||
'audio'
|
||||
| 'video'
|
||||
| 'image'
|
||||
| 'document'
|
||||
| 'archive'
|
||||
| 'subtitle'
|
||||
| 'torrent'
|
||||
| 'card'
|
||||
| 'other'
|
||||
| 'undefined';
|
||||
|
||||
Reference in New Issue
Block a user