Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Worph
|
862f73d26d | ||
|
Worph
|
b855adb822 | ||
|
Worph
|
9bf2e623f9 | ||
|
Worph
|
7bdb54f9a7 | ||
|
Worph
|
e192632328 | ||
|
Worph
|
ba44daf7eb | ||
|
Worph
|
13fa36ffc7 | ||
|
Pierre
|
44b103c97e | ||
|
Pierre
|
31bb1245da | ||
|
Pierre
|
f77bb71b44 | ||
|
Pierre
|
b1d8a56c53 | ||
|
Pierre
|
f7315cbc3f | ||
|
Pierre
|
e3d26549d0 | ||
|
Pierre
|
58ced1e92a |
@@ -0,0 +1,6 @@
|
||||
# FIX CRLF always when developer has not set
|
||||
# Linux/Mac: git config --global core.autocrlf input
|
||||
# Windows: git config --global core.autocrlf true
|
||||
# Auto detect text files and perform LF normalization
|
||||
* text=auto
|
||||
* eol=lf
|
||||
@@ -1,386 +0,0 @@
|
||||
#!/bin/bash
|
||||
# =============================================================================
|
||||
# Mount Watcher Script
|
||||
#
|
||||
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
|
||||
# Runs as a supervisord program, polling every 5 seconds.
|
||||
#
|
||||
# Uses findmnt to check actual mount status (source of truth).
|
||||
# Writes errors to /meta-core/mounts/errors/{id}.error
|
||||
# =============================================================================
|
||||
|
||||
set -u
|
||||
|
||||
CONFIG_DIR="/meta-core/mounts"
|
||||
CONFIG_FILE="$CONFIG_DIR/mounts.json"
|
||||
ERROR_DIR="$CONFIG_DIR/errors"
|
||||
FILES_PATH="${FILES_PATH:-/files}"
|
||||
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
||||
|
||||
# ANSI colors
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
CYAN='\033[0;36m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
log() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] $1"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${GREEN}$1${NC}"
|
||||
}
|
||||
|
||||
log_warn() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${YELLOW}$1${NC}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "[$(date -Iseconds)] [${CYAN}mount-watcher${NC}] ${RED}$1${NC}"
|
||||
}
|
||||
|
||||
# Initialize directories
|
||||
init_dirs() {
|
||||
mkdir -p "$CONFIG_DIR" "$ERROR_DIR"
|
||||
|
||||
# Initialize empty config if not exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
echo '{"version":1,"mounts":[]}' > "$CONFIG_FILE"
|
||||
log "Initialized empty mounts config"
|
||||
fi
|
||||
}
|
||||
|
||||
# Check if path is mounted
|
||||
is_mounted() {
|
||||
local mount_path="$1"
|
||||
findmnt -n "$mount_path" > /dev/null 2>&1
|
||||
}
|
||||
|
||||
# Clear error file for mount
|
||||
clear_error() {
|
||||
local id="$1"
|
||||
rm -f "$ERROR_DIR/$id.error"
|
||||
}
|
||||
|
||||
# Write error to file
|
||||
write_error() {
|
||||
local id="$1"
|
||||
local error="$2"
|
||||
echo "$(date -Iseconds)" > "$ERROR_DIR/$id.error"
|
||||
echo "$error" >> "$ERROR_DIR/$id.error"
|
||||
}
|
||||
|
||||
# Mount NFS share (always read-only)
|
||||
do_mount_nfs() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
|
||||
|
||||
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
|
||||
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
|
||||
local rc=$?
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "NFS mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "NFS mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Mount SMB/CIFS share (always read-only)
|
||||
do_mount_smb() {
|
||||
local id="$1"
|
||||
local server="$2"
|
||||
local share="$3"
|
||||
local mount_path="$4"
|
||||
local username="${5:-}"
|
||||
local password_obscured="${6:-}"
|
||||
local domain="${7:-}"
|
||||
local options="${8:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Always mount read-only for safety
|
||||
local mount_opts="ro"
|
||||
|
||||
# If we have credentials, reveal password and build options
|
||||
if [ -n "$username" ]; then
|
||||
mount_opts="${mount_opts},username=${username}"
|
||||
|
||||
if [ -n "$password_obscured" ]; then
|
||||
# Reveal password using rclone (only exists in memory briefly)
|
||||
local password
|
||||
password=$(rclone reveal "$password_obscured" 2>/dev/null)
|
||||
if [ -n "$password" ]; then
|
||||
mount_opts="${mount_opts},password=${password}"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -n "$domain" ]; then
|
||||
mount_opts="${mount_opts},domain=${domain}"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Add any additional options
|
||||
if [ -n "$options" ]; then
|
||||
if [ -n "$mount_opts" ]; then
|
||||
mount_opts="${mount_opts},${options}"
|
||||
else
|
||||
mount_opts="$options"
|
||||
fi
|
||||
fi
|
||||
|
||||
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
|
||||
|
||||
# Execute mount directly (not via eval) to handle special chars in password
|
||||
local output
|
||||
local rc
|
||||
if [ -n "$mount_opts" ]; then
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
|
||||
rc=$?
|
||||
else
|
||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
|
||||
rc=$?
|
||||
fi
|
||||
|
||||
# Clear password from any variables (best effort)
|
||||
password=""
|
||||
mount_opts=""
|
||||
|
||||
if [ $rc -eq 0 ]; then
|
||||
clear_error "$id"
|
||||
log_success "SMB mount successful: $mount_path"
|
||||
else
|
||||
write_error "$id" "$output"
|
||||
log_error "SMB mount failed: $output"
|
||||
fi
|
||||
return $rc
|
||||
}
|
||||
|
||||
# Mount rclone remote using RC API (always read-only)
|
||||
do_mount_rclone() {
|
||||
local id="$1"
|
||||
local remote="$2"
|
||||
local remote_path="$3"
|
||||
local mount_path="$4"
|
||||
local options="${5:-}"
|
||||
|
||||
mkdir -p "$mount_path"
|
||||
|
||||
# Build the remote source string
|
||||
local fs="${remote}:${remote_path}"
|
||||
|
||||
# Use rclone RC API to mount (always read-only for safety)
|
||||
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
|
||||
|
||||
# Build JSON body for rclone mount with read-only option
|
||||
local json_body
|
||||
json_body=$(cat <<EOF
|
||||
{
|
||||
"fs": "${fs}",
|
||||
"mountPoint": "${mount_path}",
|
||||
"mountOpt": {
|
||||
"AllowOther": true,
|
||||
"ReadOnly": true
|
||||
},
|
||||
"vfsOpt": {
|
||||
"CacheMode": 2,
|
||||
"ReadOnly": true
|
||||
}
|
||||
}
|
||||
EOF
|
||||
)
|
||||
|
||||
output=$(curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-d "$json_body" \
|
||||
"http://127.0.0.1:5572/mount/mount" 2>&1)
|
||||
|
||||
# Wait a moment for mount to establish
|
||||
sleep 2
|
||||
|
||||
if is_mounted "$mount_path"; then
|
||||
clear_error "$id"
|
||||
log_success "rclone mount successful: $mount_path"
|
||||
return 0
|
||||
else
|
||||
local error_msg="rclone mount failed"
|
||||
# Try to extract error from response
|
||||
if echo "$output" | grep -q "error"; then
|
||||
error_msg="$output"
|
||||
fi
|
||||
write_error "$id" "$error_msg"
|
||||
log_error "rclone mount failed: $error_msg"
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Unmount a path
|
||||
do_unmount() {
|
||||
local id="$1"
|
||||
local mount_path="$2"
|
||||
local type="$3"
|
||||
|
||||
if ! is_mounted "$mount_path"; then
|
||||
log "Already unmounted: $mount_path"
|
||||
return 0
|
||||
fi
|
||||
|
||||
if [ "$type" = "rclone" ]; then
|
||||
# Use rclone RC API to unmount
|
||||
log "Unmounting rclone: $mount_path"
|
||||
curl -s -X POST \
|
||||
-H "Content-Type: application/json" \
|
||||
-u admin:admin \
|
||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
|
||||
else
|
||||
log "Unmounting: $mount_path"
|
||||
umount "$mount_path" 2>&1
|
||||
fi
|
||||
|
||||
# Wait a moment
|
||||
sleep 1
|
||||
|
||||
# Check if still mounted
|
||||
if is_mounted "$mount_path"; then
|
||||
# Force unmount with lazy option
|
||||
log_warn "Force unmounting (lazy): $mount_path"
|
||||
umount -l "$mount_path" 2>&1
|
||||
sleep 1
|
||||
fi
|
||||
|
||||
if is_mounted "$mount_path"; then
|
||||
log_error "Failed to unmount: $mount_path"
|
||||
return 1
|
||||
fi
|
||||
|
||||
log_success "Unmounted: $mount_path"
|
||||
return 0
|
||||
}
|
||||
|
||||
# Process all mounts from config
|
||||
process_mounts() {
|
||||
# Check if config file exists
|
||||
if [ ! -f "$CONFIG_FILE" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Parse JSON using python3
|
||||
local mounts
|
||||
mounts=$(python3 -c '
|
||||
import sys, json
|
||||
|
||||
try:
|
||||
with open("'"$CONFIG_FILE"'", "r") as f:
|
||||
data = json.load(f)
|
||||
except:
|
||||
sys.exit(0)
|
||||
|
||||
for m in data.get("mounts", []):
|
||||
# Output pipe-separated values
|
||||
fields = [
|
||||
m.get("id", ""),
|
||||
m.get("type", ""),
|
||||
str(m.get("enabled", True)),
|
||||
str(m.get("desiredMounted", False)),
|
||||
m.get("mountPath", ""),
|
||||
m.get("nfsServer", ""),
|
||||
m.get("nfsPath", ""),
|
||||
m.get("smbServer", ""),
|
||||
m.get("smbShare", ""),
|
||||
m.get("smbUsername", ""),
|
||||
m.get("smbPasswordObscured", ""),
|
||||
m.get("smbDomain", ""),
|
||||
m.get("rcloneRemote", ""),
|
||||
m.get("rclonePath", ""),
|
||||
m.get("options", "")
|
||||
]
|
||||
print("|".join(fields))
|
||||
' 2>/dev/null)
|
||||
|
||||
if [ -z "$mounts" ]; then
|
||||
return
|
||||
fi
|
||||
|
||||
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
|
||||
# Skip empty lines
|
||||
[ -z "$id" ] && continue
|
||||
|
||||
# Check actual mount status
|
||||
local is_mounted_now=false
|
||||
is_mounted "$mount_path" && is_mounted_now=true
|
||||
|
||||
# Handle disabled mounts - unmount if currently mounted
|
||||
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) disabled, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
fi
|
||||
continue
|
||||
fi
|
||||
|
||||
# Handle desired state
|
||||
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
||||
# Should be mounted
|
||||
if [ "$is_mounted_now" = false ]; then
|
||||
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
||||
case "$type" in
|
||||
nfs)
|
||||
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
|
||||
;;
|
||||
smb)
|
||||
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
|
||||
;;
|
||||
rclone)
|
||||
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
|
||||
;;
|
||||
*)
|
||||
log_error "Unknown mount type: $type"
|
||||
;;
|
||||
esac
|
||||
fi
|
||||
else
|
||||
# Should be unmounted
|
||||
if [ "$is_mounted_now" = true ]; then
|
||||
log "Mount $id ($mount_path) not desired, unmounting..."
|
||||
do_unmount "$id" "$mount_path" "$type"
|
||||
fi
|
||||
fi
|
||||
done <<< "$mounts"
|
||||
}
|
||||
|
||||
# Main loop
|
||||
main() {
|
||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
|
||||
|
||||
init_dirs
|
||||
|
||||
while true; do
|
||||
process_mounts
|
||||
sleep "$POLL_INTERVAL"
|
||||
done
|
||||
}
|
||||
|
||||
# Handle signals for graceful shutdown
|
||||
cleanup() {
|
||||
log "Shutting down mount watcher..."
|
||||
exit 0
|
||||
}
|
||||
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
main "$@"
|
||||
@@ -587,7 +587,7 @@ Deduplication logic remains identical. Fast-hash serves the same purpose as full
|
||||
|
||||
The Stremio addon retrieves files by metadata queries (title, season, episode), not by hash. The hash is used internally for metadata storage. Changing to fast-hash is transparent to the addon.
|
||||
|
||||
### Meta-Orbit (Distributed Synchronization)
|
||||
### meta-share (Distributed Synchronization)
|
||||
|
||||
**Impact:** Hash serves as content identifier for P2P synchronization.
|
||||
|
||||
@@ -716,7 +716,7 @@ MetaMesh implements a two-phase processing pipeline:
|
||||
**Phase 2: Hash Processing (Full Hashes - Background)**
|
||||
- Compute full-file hashes: SHA-256, SHA-1, MD5, CRC32 (slow, minutes)
|
||||
- Update etcd with additional hash metadata
|
||||
- Required for Meta-Orbit distributed synchronization
|
||||
- Required for meta-share distributed synchronization
|
||||
|
||||
### Why Both Are Needed
|
||||
|
||||
@@ -726,7 +726,7 @@ MetaMesh implements a two-phase processing pipeline:
|
||||
- **Fast deduplication**: Quickly identify potential duplicates
|
||||
|
||||
**Full Hash Purpose:**
|
||||
- **Meta-Orbit integration**: Distributed P2P synchronization requires cryptographically secure hashes
|
||||
- **meta-share integration**: Distributed P2P synchronization requires cryptographically secure hashes
|
||||
- **External system compatibility**: Torrent verification, IPFS CIDs
|
||||
- **Collision resolution**: If midhash256 collision detected, full SHA-256 provides definitive answer
|
||||
|
||||
@@ -744,7 +744,7 @@ Light Processing (32 workers, parallel)
|
||||
Hash Processing (8 workers, background)
|
||||
- Compute SHA-256, SHA-1, MD5, CRC32 (15+ minutes for 50GB)
|
||||
- Update etcd with full hashes
|
||||
- Enable Meta-Orbit synchronization ✓
|
||||
- Enable meta-share synchronization ✓
|
||||
```
|
||||
|
||||
### Collision Detection and Handling
|
||||
@@ -778,4 +778,4 @@ The trade-offs—weaker cryptographic guarantees, potential collisions for adver
|
||||
|
||||
This architecture aligns with Meta-Mesh's design philosophy: **optimize for the common case** (large media files, organic content, personal libraries) while maintaining **practicality and performance** over theoretical perfection.
|
||||
|
||||
Meta-Mesh leverages midhash256 for **immediate file availability** while computing full SHA-256 hashes **in the background** for Meta-Orbit distributed synchronization and external system compatibility. This dual-hash approach provides both speed and security.
|
||||
Meta-Mesh leverages midhash256 for **immediate file availability** while computing full SHA-256 hashes **in the background** for meta-share distributed synchronization and external system compatibility. This dual-hash approach provides both speed and security.
|
||||
|
||||
+27
-754
@@ -1,762 +1,35 @@
|
||||
# Meta-Sort Plugin System Architecture
|
||||
# Plugin System (Deprecated — See Container Plugins)
|
||||
|
||||
This document describes the plugin system architecture for meta-sort, enabling extensible metadata extraction through dynamically loaded plugins.
|
||||
> **This document is a historical pointer.** The in-process plugin system it
|
||||
> originally described — `plugins/<id>/{manifest.yml,index.ts}` directories
|
||||
> loaded via dynamic `import()` inside meta-sort-core — has been removed.
|
||||
> All plugins now run as Docker containers and communicate over HTTP.
|
||||
|
||||
## Overview
|
||||
For the current design, read:
|
||||
|
||||
The plugin system provides a modular architecture for metadata extraction. Plugins are self-contained units that:
|
||||
- **[containerized-plugin-architecture.md](./containerized-plugin-architecture.md)** — full container plugin design (HTTP contract, lifecycle, callbacks, KV access).
|
||||
- **[plugin-task-queue-architecture.md](./plugin-task-queue-architecture.md)** — how plugin work is dispatched across the fast / background queues.
|
||||
- Root **`CLAUDE.md`** (`Plugin Development` and `Runtime Plugin Configuration` sections) — operational quickstart.
|
||||
- Plugin sources at **`packages/plugins/metamesh-plugin-*`** (separate submodules) — `file-info`, `ffmpeg`, `filename-parser`, `jellyfin-nfo`, `tmdb`, `anime-detector`, `language`, `subtitle`, `subtitle-extractor`, `torrent`, `fullhash`.
|
||||
|
||||
- Are discovered and loaded from a plugins directory at startup
|
||||
- Can be activated/deactivated at runtime without restart
|
||||
- Process files sequentially in dependency order
|
||||
- Store metadata in a flat key-value format
|
||||
- Have access to their own persistent cache folder
|
||||
## What survived from the old design
|
||||
|
||||
## Directory Structure
|
||||
The `Plugin` / `PluginContext` / `PluginLifecycleContext` / `PluginKVStore`
|
||||
interfaces in `src/plugin-engine/types.ts` are still exported, but the
|
||||
container-only `PluginManager` (`src/plugin-engine/PluginManager.ts`) never
|
||||
constructs an in-process `Plugin` instance — manifests, lifecycle, and
|
||||
execution are owned by `ContainerManager` and `ContainerPluginScheduler`
|
||||
under `src/container-plugins/`. Treat the `Plugin`-interface types as
|
||||
vestigial and do not write new code against them.
|
||||
|
||||
```
|
||||
packages/meta-sort/packages/meta-sort-core/
|
||||
├── plugins/ # All plugins (shipped + custom)
|
||||
│ ├── file-info/
|
||||
│ │ ├── manifest.yml # Plugin metadata & config schema
|
||||
│ │ └── index.ts # Plugin implementation
|
||||
│ ├── ffmpeg/
|
||||
│ ├── filename-parser/
|
||||
│ ├── anime-detector/
|
||||
│ ├── jellyfin-nfo/
|
||||
│ ├── subtitle/
|
||||
│ ├── torrent/
|
||||
│ ├── language/
|
||||
│ ├── tmdb/
|
||||
│ └── debug/
|
||||
├── plugins.json # Runtime state (active/inactive + configs)
|
||||
└── src/
|
||||
└── plugin/
|
||||
├── types.ts # Type definitions
|
||||
├── PluginLoader.ts # Manifest parsing, dynamic import
|
||||
├── PluginContext.ts # Context factory
|
||||
├── PluginManager.ts # Lifecycle management
|
||||
└── PluginExecutor.ts # Dependency-ordered execution
|
||||
```
|
||||
## Why the change
|
||||
|
||||
Cache folders are stored separately under `CACHE_FOLDER_PATH` (default `/data/cache/plugins/`):
|
||||
| Limitation of in-process plugins | Container model |
|
||||
|----------------------------------|-----------------|
|
||||
| Heavy native deps (FFmpeg, Whisper, ML libs) inflated the meta-sort image | Each plugin ships its own image |
|
||||
| All plugins had to be TypeScript / Node | Any language with an HTTP server |
|
||||
| A crashing plugin took down meta-sort | Plugin crashes are isolated; meta-sort restarts the container |
|
||||
| Plugin updates required redeploying meta-sort | Plugins are versioned and pulled independently |
|
||||
|
||||
```
|
||||
/data/cache/plugins/
|
||||
├── ffmpeg/
|
||||
│ ├── {cid1}.json
|
||||
│ └── {cid2}.json
|
||||
├── tmdb/
|
||||
│ ├── {cid1}.json
|
||||
│ └── {cid2}.json
|
||||
└── ...
|
||||
```
|
||||
|
||||
## Plugin Manifest
|
||||
|
||||
Each plugin must have a `manifest.yml` file defining its metadata, dependencies, and configuration schema.
|
||||
|
||||
```yaml
|
||||
# manifest.yml
|
||||
id: tmdb # Unique identifier (matches folder name)
|
||||
name: TMDB Enricher # Human-readable name
|
||||
version: 1.0.0 # Semantic version
|
||||
description: Enriches metadata from The Movie Database API
|
||||
|
||||
# Plugins that must execute BEFORE this one
|
||||
dependencies:
|
||||
- filename-parser # Needs parsed title/year
|
||||
|
||||
# Configuration schema (rendered in dashboard)
|
||||
config:
|
||||
apiKey:
|
||||
type: string
|
||||
label: API Key
|
||||
required: true
|
||||
secret: true # Masked in UI
|
||||
language:
|
||||
type: string
|
||||
label: Language
|
||||
default: en-US
|
||||
includeAdult:
|
||||
type: boolean
|
||||
label: Include Adult Content
|
||||
default: false
|
||||
|
||||
# Metadata schema for editor UI (optional)
|
||||
# Describes keys this plugin writes
|
||||
schema:
|
||||
tmdb/id:
|
||||
label: TMDB ID
|
||||
type: string
|
||||
readonly: true
|
||||
tmdb/title:
|
||||
label: Title
|
||||
type: string
|
||||
tmdb/overview:
|
||||
label: Overview
|
||||
type: text
|
||||
tmdb/poster:
|
||||
label: Poster URL
|
||||
type: string
|
||||
format: url
|
||||
tmdb/genre/{n}:
|
||||
label: Genre
|
||||
type: string
|
||||
indexed: true # Indicates indexed keys (0, 1, 2...)
|
||||
```
|
||||
|
||||
### Manifest Fields
|
||||
|
||||
| Field | Required | Description |
|
||||
|-------|----------|-------------|
|
||||
| `id` | Yes | Unique plugin identifier, must match folder name |
|
||||
| `name` | Yes | Display name for dashboard |
|
||||
| `version` | Yes | Semantic version string |
|
||||
| `description` | No | Brief description |
|
||||
| `dependencies` | No | Array of plugin IDs that must run before this one |
|
||||
| `config` | No | Configuration schema for dashboard UI |
|
||||
| `schema` | No | Metadata keys written by this plugin (for editor) |
|
||||
|
||||
### Config Field Types
|
||||
|
||||
```yaml
|
||||
config:
|
||||
fieldName:
|
||||
type: string | number | boolean | select
|
||||
label: Display Label
|
||||
required: true | false # Default: false
|
||||
default: <value> # Default value
|
||||
secret: true | false # Mask in UI (default: false)
|
||||
options: # For type: select
|
||||
- value: opt1
|
||||
label: Option 1
|
||||
- value: opt2
|
||||
label: Option 2
|
||||
```
|
||||
|
||||
## Plugin Interface
|
||||
|
||||
Plugins export a default object implementing the `Plugin` interface.
|
||||
|
||||
```typescript
|
||||
// types.ts
|
||||
|
||||
export interface Plugin {
|
||||
/**
|
||||
* Called once when the plugin is loaded at startup.
|
||||
* Use for one-time initialization.
|
||||
*/
|
||||
onLoad?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is activated.
|
||||
* Triggered at startup (if previously active) or from dashboard.
|
||||
*/
|
||||
onActivate?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is deactivated from the dashboard.
|
||||
* Use for cleanup, releasing resources.
|
||||
*/
|
||||
onDeactivate?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Called when the plugin is unloaded at shutdown.
|
||||
*/
|
||||
onUnload?(ctx: PluginLifecycleContext): Promise<void>;
|
||||
|
||||
/**
|
||||
* Process a single file. Called for each file in the library.
|
||||
* Must be implemented by all plugins.
|
||||
*/
|
||||
process(ctx: PluginContext): Promise<void>;
|
||||
}
|
||||
|
||||
export interface PluginLifecycleContext {
|
||||
/** Plugin configuration from plugins.json */
|
||||
config: Record<string, unknown>;
|
||||
|
||||
/** Absolute path to plugin's cache directory */
|
||||
cacheDir: string;
|
||||
|
||||
/** Scoped logger */
|
||||
log: PluginLogger;
|
||||
}
|
||||
```
|
||||
|
||||
## Plugin Context
|
||||
|
||||
The `PluginContext` is injected into the `process()` function, providing access to file metadata and utilities.
|
||||
|
||||
```typescript
|
||||
export interface PluginContext {
|
||||
/**
|
||||
* Key-value storage for the current file's metadata.
|
||||
* Keys use path notation: "video/codec", "audio/0/language"
|
||||
*/
|
||||
kv: PluginKVStore;
|
||||
|
||||
/** Absolute path to the file being processed */
|
||||
filePath: string;
|
||||
|
||||
/** Plugin configuration values */
|
||||
config: Record<string, unknown>;
|
||||
|
||||
/** Absolute path to plugin's cache directory */
|
||||
cacheDir: string;
|
||||
|
||||
/** Cache utilities for persistent storage */
|
||||
cache: PluginCache;
|
||||
|
||||
/** Scoped logger (prefixed with plugin name) */
|
||||
log: PluginLogger;
|
||||
}
|
||||
```
|
||||
|
||||
### KV Store Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginKVStore {
|
||||
/**
|
||||
* Get a value by key.
|
||||
* @param key Path-style key, e.g., "video/codec" or "audio/0/language"
|
||||
*/
|
||||
get(key: string): string | undefined;
|
||||
|
||||
/**
|
||||
* Set a value. Only string values are supported.
|
||||
* @param key Path-style key
|
||||
* @param value String value
|
||||
*/
|
||||
set(key: string, value: string): void;
|
||||
|
||||
/**
|
||||
* Delete a key.
|
||||
*/
|
||||
delete(key: string): void;
|
||||
|
||||
/**
|
||||
* Get all keys, optionally filtered by prefix.
|
||||
* @param prefix Optional prefix filter, e.g., "audio/" returns ["audio/0/codec", "audio/0/language", ...]
|
||||
*/
|
||||
keys(prefix?: string): string[];
|
||||
|
||||
/**
|
||||
* Get all key-value pairs, optionally filtered by prefix.
|
||||
*/
|
||||
entries(prefix?: string): Array<[string, string]>;
|
||||
}
|
||||
```
|
||||
|
||||
### Cache Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginCache {
|
||||
/**
|
||||
* Get full path for a cache file.
|
||||
* @param filename Relative filename within cache directory
|
||||
*/
|
||||
getPath(filename: string): string;
|
||||
|
||||
/**
|
||||
* Read and parse a JSON cache file.
|
||||
* @returns Parsed data or null if not found
|
||||
*/
|
||||
readJson<T>(filename: string): Promise<T | null>;
|
||||
|
||||
/**
|
||||
* Write data as JSON to cache file.
|
||||
*/
|
||||
writeJson(filename: string, data: unknown): Promise<void>;
|
||||
|
||||
/**
|
||||
* Check if a cache file exists.
|
||||
*/
|
||||
exists(filename: string): Promise<boolean>;
|
||||
|
||||
/**
|
||||
* Delete a cache file.
|
||||
*/
|
||||
delete(filename: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Clear all files in the plugin's cache directory.
|
||||
*/
|
||||
clear(): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### Logger Interface
|
||||
|
||||
```typescript
|
||||
export interface PluginLogger {
|
||||
debug(message: string, ...args: unknown[]): void;
|
||||
info(message: string, ...args: unknown[]): void;
|
||||
warn(message: string, ...args: unknown[]): void;
|
||||
error(message: string, ...args: unknown[]): void;
|
||||
}
|
||||
```
|
||||
|
||||
## Plugin Implementation Example
|
||||
|
||||
```typescript
|
||||
// plugins/tmdb/index.ts
|
||||
import type { Plugin, PluginContext, PluginLifecycleContext } from '../../src/plugin/types';
|
||||
|
||||
interface TmdbConfig {
|
||||
apiKey: string;
|
||||
language: string;
|
||||
includeAdult: boolean;
|
||||
}
|
||||
|
||||
const plugin: Plugin = {
|
||||
async onLoad(ctx: PluginLifecycleContext) {
|
||||
ctx.log.info('TMDB plugin loaded');
|
||||
},
|
||||
|
||||
async onActivate(ctx: PluginLifecycleContext) {
|
||||
const config = ctx.config as TmdbConfig;
|
||||
if (!config.apiKey) {
|
||||
ctx.log.warn('No API key configured, plugin will skip processing');
|
||||
}
|
||||
},
|
||||
|
||||
async onDeactivate(ctx: PluginLifecycleContext) {
|
||||
ctx.log.info('TMDB plugin deactivated');
|
||||
},
|
||||
|
||||
async process(ctx: PluginContext) {
|
||||
const config = ctx.config as TmdbConfig;
|
||||
|
||||
// Skip if not configured
|
||||
if (!config.apiKey) return;
|
||||
|
||||
// Get data from previous plugins
|
||||
const title = ctx.kv.get('filename/title');
|
||||
const year = ctx.kv.get('filename/year');
|
||||
const videoType = ctx.kv.get('video/type');
|
||||
|
||||
if (!title) return;
|
||||
|
||||
// Check cache first (keyed by title+year)
|
||||
const cacheKey = `${sanitize(title)}-${year || 'unknown'}.json`;
|
||||
const cached = await ctx.cache.readJson<TmdbResult>(cacheKey);
|
||||
|
||||
if (cached) {
|
||||
applyResult(ctx.kv, cached);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fetch from TMDB API
|
||||
try {
|
||||
const result = await fetchTmdb(title, year, videoType, config);
|
||||
if (result) {
|
||||
await ctx.cache.writeJson(cacheKey, result);
|
||||
applyResult(ctx.kv, result);
|
||||
}
|
||||
} catch (error) {
|
||||
ctx.log.error('TMDB fetch failed', error);
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
function applyResult(kv: PluginContext['kv'], result: TmdbResult) {
|
||||
kv.set('tmdb/id', String(result.id));
|
||||
kv.set('tmdb/title', result.title);
|
||||
kv.set('tmdb/overview', result.overview);
|
||||
kv.set('tmdb/poster', result.poster_path);
|
||||
kv.set('tmdb/backdrop', result.backdrop_path);
|
||||
kv.set('tmdb/rating', String(result.vote_average));
|
||||
kv.set('tmdb/year', result.release_date?.slice(0, 4) || '');
|
||||
|
||||
// Genres as indexed keys
|
||||
result.genres?.forEach((genre, i) => {
|
||||
kv.set(`tmdb/genre/${i}`, genre.name);
|
||||
});
|
||||
}
|
||||
|
||||
export default plugin;
|
||||
```
|
||||
|
||||
## Metadata Key Format
|
||||
|
||||
All metadata is stored as flat key-value pairs with string values only. Keys use path notation with `/` as separator.
|
||||
|
||||
### Key Naming Convention
|
||||
|
||||
```
|
||||
{plugin-id}/{field} # Simple field
|
||||
{plugin-id}/{group}/{field} # Grouped field
|
||||
{plugin-id}/{group}/{index}/{field} # Indexed field (for lists)
|
||||
```
|
||||
|
||||
### Examples
|
||||
|
||||
```
|
||||
# File info plugin
|
||||
file/name = "Movie.2024.1080p.mkv"
|
||||
file/size = "4294967296"
|
||||
file/type = "video"
|
||||
file/mime = "video/x-matroska"
|
||||
|
||||
# FFmpeg plugin
|
||||
video/codec = "h264"
|
||||
video/width = "1920"
|
||||
video/height = "1080"
|
||||
video/duration = "7200.5"
|
||||
video/bitrate = "8000000"
|
||||
audio/0/codec = "aac"
|
||||
audio/0/language = "eng"
|
||||
audio/0/channels = "6"
|
||||
audio/1/codec = "dts"
|
||||
audio/1/language = "fra"
|
||||
audio/1/channels = "6"
|
||||
subtitle/0/language = "eng"
|
||||
subtitle/0/format = "subrip"
|
||||
subtitle/1/language = "fra"
|
||||
subtitle/1/format = "subrip"
|
||||
|
||||
# Filename parser plugin
|
||||
filename/title = "Movie Title"
|
||||
filename/year = "2024"
|
||||
filename/quality = "1080p"
|
||||
filename/source = "bluray"
|
||||
|
||||
# TMDB plugin
|
||||
tmdb/id = "12345"
|
||||
tmdb/title = "Movie Title"
|
||||
tmdb/overview = "A movie about..."
|
||||
tmdb/poster = "/path/to/poster.jpg"
|
||||
tmdb/genre/0 = "Action"
|
||||
tmdb/genre/1 = "Adventure"
|
||||
```
|
||||
|
||||
### Key Constraints
|
||||
|
||||
- Keys must be non-empty strings
|
||||
- Keys use `/` as path separator
|
||||
- Values must be strings (numbers converted via `String()`)
|
||||
- No nested objects or arrays (use indexed keys instead)
|
||||
- Keys are case-sensitive
|
||||
- Recommended: use lowercase with hyphens for multi-word keys
|
||||
|
||||
## Plugin Manager
|
||||
|
||||
The `PluginManager` handles plugin lifecycle and orchestration.
|
||||
|
||||
```typescript
|
||||
export class PluginManager extends EventEmitter {
|
||||
/**
|
||||
* Scan plugins directory and load manifests.
|
||||
* Called once at startup.
|
||||
*/
|
||||
async scanPlugins(): Promise<PluginInfo[]>;
|
||||
|
||||
/**
|
||||
* Load all plugins and activate those marked active in state.
|
||||
* Called once at startup after scanning.
|
||||
*/
|
||||
async loadAll(): Promise<void>;
|
||||
|
||||
/**
|
||||
* Activate a plugin (hot, no restart required).
|
||||
* Calls plugin.onActivate(), rebuilds execution order, saves state.
|
||||
* Emits 'plugin:activated' event.
|
||||
*/
|
||||
async activate(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Deactivate a plugin (hot, no restart required).
|
||||
* Calls plugin.onDeactivate(), rebuilds execution order, saves state.
|
||||
* Emits 'plugin:deactivated' event.
|
||||
*/
|
||||
async deactivate(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Update plugin configuration.
|
||||
* Saves to plugins.json.
|
||||
* Emits 'plugin:config-changed' event.
|
||||
*/
|
||||
async updateConfig(pluginId: string, config: Record<string, unknown>): Promise<void>;
|
||||
|
||||
/**
|
||||
* Clear a plugin's cache directory.
|
||||
*/
|
||||
async clearCache(pluginId: string): Promise<void>;
|
||||
|
||||
/**
|
||||
* Get all discovered plugins.
|
||||
*/
|
||||
getPlugins(): PluginInfo[];
|
||||
|
||||
/**
|
||||
* Get only active plugins.
|
||||
*/
|
||||
getActivePlugins(): PluginInfo[];
|
||||
|
||||
/**
|
||||
* Get execution order (dependency-sorted active plugin IDs).
|
||||
*/
|
||||
getExecutionOrder(): string[];
|
||||
|
||||
/**
|
||||
* Process a file through all active plugins.
|
||||
*/
|
||||
async processFile(filePath: string, kv: KVStore): Promise<ProcessingResult>;
|
||||
|
||||
/**
|
||||
* Shutdown: deactivate and unload all plugins.
|
||||
*/
|
||||
async shutdown(): Promise<void>;
|
||||
}
|
||||
```
|
||||
|
||||
### Events
|
||||
|
||||
| Event | Payload | Description |
|
||||
|-------|---------|-------------|
|
||||
| `plugin:activated` | `{ pluginId: string }` | Plugin was activated |
|
||||
| `plugin:deactivated` | `{ pluginId: string }` | Plugin was deactivated |
|
||||
| `plugin:config-changed` | `{ pluginId: string, config: object }` | Plugin config updated |
|
||||
| `plugin:error` | `{ pluginId: string, error: Error }` | Plugin error occurred |
|
||||
|
||||
### Plugin Info
|
||||
|
||||
```typescript
|
||||
export interface PluginInfo {
|
||||
id: string;
|
||||
name: string;
|
||||
version: string;
|
||||
description?: string;
|
||||
dependencies: string[];
|
||||
active: boolean;
|
||||
config: Record<string, unknown>;
|
||||
configSchema?: ConfigSchema;
|
||||
metadataSchema?: MetadataSchema;
|
||||
status: 'loaded' | 'error' | 'unloaded';
|
||||
error?: string;
|
||||
}
|
||||
```
|
||||
|
||||
## State Management
|
||||
|
||||
Plugin state is persisted in `plugins.json` located in the meta-sort-core package root.
|
||||
|
||||
```json
|
||||
{
|
||||
"plugins": {
|
||||
"file-info": {
|
||||
"active": true,
|
||||
"config": {}
|
||||
},
|
||||
"ffmpeg": {
|
||||
"active": true,
|
||||
"config": {}
|
||||
},
|
||||
"tmdb": {
|
||||
"active": true,
|
||||
"config": {
|
||||
"apiKey": "your-api-key",
|
||||
"language": "en-US",
|
||||
"includeAdult": false
|
||||
}
|
||||
},
|
||||
"custom-plugin": {
|
||||
"active": false,
|
||||
"config": {}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
### State Behavior
|
||||
|
||||
- **New plugins**: Discovered plugins not in state file are added with `active: false`
|
||||
- **Removed plugins**: Plugins in state but not on disk are kept (preserves config for reinstall)
|
||||
- **Config merge**: Default values from manifest are merged with saved config
|
||||
- **Persistence**: State is saved immediately on any change
|
||||
|
||||
## Execution Pipeline
|
||||
|
||||
### Startup Sequence
|
||||
|
||||
```
|
||||
1. PluginManager.scanPlugins()
|
||||
└── Scan plugins/ directory
|
||||
└── Parse each manifest.yml
|
||||
└── Validate manifest structure
|
||||
|
||||
2. PluginManager.loadAll()
|
||||
└── Load plugins.json state
|
||||
└── For each discovered plugin:
|
||||
├── Dynamic import(plugin/index.js)
|
||||
├── Validate Plugin interface
|
||||
├── Call plugin.onLoad()
|
||||
└── If active in state:
|
||||
└── Call plugin.onActivate()
|
||||
└── Resolve execution order (topological sort)
|
||||
```
|
||||
|
||||
### File Processing Sequence
|
||||
|
||||
```
|
||||
1. Create KVStore for file
|
||||
|
||||
2. For each active plugin (in dependency order):
|
||||
├── Create PluginContext
|
||||
├── Start timing
|
||||
├── try:
|
||||
│ └── await plugin.process(ctx)
|
||||
└── catch:
|
||||
└── Log error, continue to next plugin
|
||||
|
||||
3. Persist KV to Redis
|
||||
```
|
||||
|
||||
### Activation Sequence (Hot)
|
||||
|
||||
```
|
||||
1. PluginManager.activate(pluginId)
|
||||
├── Validate plugin exists and is loaded
|
||||
├── Check dependencies are active
|
||||
├── Call plugin.onActivate()
|
||||
├── Mark as active in memory
|
||||
├── Rebuild execution order
|
||||
├── Save plugins.json
|
||||
└── Emit 'plugin:activated' event
|
||||
|
||||
2. Caller receives event
|
||||
└── Trigger library rescan
|
||||
```
|
||||
|
||||
### Deactivation Sequence (Hot)
|
||||
|
||||
```
|
||||
1. PluginManager.deactivate(pluginId)
|
||||
├── Check no active plugins depend on this one
|
||||
├── Call plugin.onDeactivate()
|
||||
├── Mark as inactive in memory
|
||||
├── Rebuild execution order
|
||||
├── Save plugins.json
|
||||
└── Emit 'plugin:deactivated' event
|
||||
|
||||
2. Caller receives event
|
||||
└── Trigger library rescan
|
||||
|
||||
Note: Metadata written by the plugin remains in Redis.
|
||||
A separate "clean metadata" feature can remove orphaned keys.
|
||||
```
|
||||
|
||||
## Dependency Resolution
|
||||
|
||||
Plugins declare dependencies in their manifest. The system performs topological sort to determine execution order.
|
||||
|
||||
### Rules
|
||||
|
||||
1. A plugin cannot be activated if any of its dependencies are inactive
|
||||
2. A plugin cannot be deactivated if any active plugin depends on it
|
||||
3. Circular dependencies are detected and rejected at load time
|
||||
4. Missing dependencies are logged as warnings; plugin can still load but may fail
|
||||
|
||||
### Example
|
||||
|
||||
```yaml
|
||||
# filename-parser depends on file-info
|
||||
# tmdb depends on filename-parser
|
||||
# anime-detector depends on filename-parser
|
||||
|
||||
Resolved order: file-info → filename-parser → [tmdb, anime-detector]
|
||||
```
|
||||
|
||||
Plugins at the same dependency level may execute in any order (or parallel in future).
|
||||
|
||||
## REST API
|
||||
|
||||
The plugin system exposes REST endpoints for dashboard integration.
|
||||
|
||||
### Endpoints
|
||||
|
||||
```
|
||||
GET /api/plugins # List all plugins
|
||||
GET /api/plugins/:id # Get plugin details
|
||||
POST /api/plugins/:id/activate # Activate plugin
|
||||
POST /api/plugins/:id/deactivate # Deactivate plugin
|
||||
PUT /api/plugins/:id/config # Update plugin config
|
||||
DELETE /api/plugins/:id/cache # Clear plugin cache
|
||||
GET /api/plugins/:id/schema # Get metadata schema (for editor)
|
||||
```
|
||||
|
||||
### Response Format
|
||||
|
||||
```typescript
|
||||
// GET /api/plugins
|
||||
interface PluginsResponse {
|
||||
plugins: PluginInfo[];
|
||||
executionOrder: string[];
|
||||
}
|
||||
|
||||
// POST /api/plugins/:id/activate
|
||||
interface ActivateResponse {
|
||||
success: boolean;
|
||||
error?: string; # e.g., "Dependency 'x' is not active"
|
||||
}
|
||||
|
||||
// PUT /api/plugins/:id/config
|
||||
interface ConfigUpdateRequest {
|
||||
config: Record<string, unknown>;
|
||||
}
|
||||
```
|
||||
|
||||
## Error Handling
|
||||
|
||||
### Plugin Load Errors
|
||||
|
||||
- Invalid manifest: Plugin marked as `status: 'error'`, logged, skipped
|
||||
- Import failure: Plugin marked as `status: 'error'`, logged, skipped
|
||||
- Missing dependencies: Warning logged, plugin can load but activation may fail
|
||||
|
||||
### Plugin Execution Errors
|
||||
|
||||
- Errors in `process()` are caught, logged, and execution continues to next plugin
|
||||
- The file's metadata from previous plugins is preserved
|
||||
- Error count is tracked per plugin for monitoring
|
||||
|
||||
### Lifecycle Hook Errors
|
||||
|
||||
- Errors in `onActivate()` prevent activation, error returned to caller
|
||||
- Errors in `onDeactivate()` are logged but deactivation proceeds
|
||||
- Errors in `onLoad()`/`onUnload()` are logged, plugin marked as error state
|
||||
|
||||
## Future Considerations
|
||||
|
||||
### Metadata Cleanup
|
||||
|
||||
A future feature will allow cleaning metadata keys written by a specific plugin:
|
||||
|
||||
```typescript
|
||||
// Future API
|
||||
await pluginManager.cleanMetadata(pluginId);
|
||||
// Removes all keys matching "{pluginId}/*" from Redis
|
||||
```
|
||||
|
||||
### Parallel Execution
|
||||
|
||||
Plugins at the same dependency level could execute in parallel:
|
||||
|
||||
```typescript
|
||||
// Future: parallel execution of independent plugins
|
||||
const level1 = ['file-info'];
|
||||
const level2 = ['ffmpeg', 'filename-parser']; // parallel
|
||||
const level3 = ['tmdb', 'anime-detector']; // parallel
|
||||
```
|
||||
|
||||
### Plugin Hot Reload
|
||||
|
||||
Reloading a plugin's code without full restart (for development):
|
||||
|
||||
```typescript
|
||||
// Future API
|
||||
await pluginManager.reload(pluginId);
|
||||
// Re-imports plugin code, preserves state
|
||||
```
|
||||
See `containerized-plugin-architecture.md` for the migration rationale and
|
||||
the HTTP contract container plugins must implement.
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
+2
-1
@@ -12,5 +12,6 @@
|
||||
"test": "pnpm --filter @meta-sort/core test",
|
||||
"lint": "pnpm -r lint"
|
||||
},
|
||||
"license": "MIT"
|
||||
"license": "MIT",
|
||||
"packageManager": "[email protected]+sha512.10568bb4a6afb58c9eb3630da90cc9516417abebd3fabbe6739f0ae795728da1491e9db5a544c76ad8eb7570f5c4bb3d6c637b2cb41bfdcdb47fa823c8649319"
|
||||
}
|
||||
|
||||
@@ -35,7 +35,7 @@ lazyValue().then(value => console.log(value));
|
||||
### Listener Cleaner
|
||||
|
||||
```typescript
|
||||
import { ListenerCleaner } from "worph/async-utils";
|
||||
import { ListenerCleaner } from "@worph/async-utils";
|
||||
|
||||
const cleaner = new ListenerCleaner();
|
||||
cleaner.add(() => console.log("Cleanup action"));
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"spec": "dist/api/api.test.js",
|
||||
"spec": ["dist/api/api.test.js", "dist/kv/*.test.js"],
|
||||
"timeout": 30000,
|
||||
"exit": true
|
||||
}
|
||||
|
||||
@@ -69,6 +69,18 @@ interface EnvConfig {
|
||||
/** Concurrency for fast plugin queue. Default: 32 */
|
||||
FAST_QUEUE_CONCURRENCY: number;
|
||||
|
||||
/**
|
||||
* Maximum number of files concurrently in the light (fast-queue) phase.
|
||||
* Caps fan-out so the underlying file cache (WebDAV/nginx) only has to
|
||||
* hold N files at a time. Default: 16.
|
||||
*/
|
||||
FILE_LIGHT_SLOTS: number;
|
||||
|
||||
/**
|
||||
* Maximum number of files concurrently in the background phase. Default: 16.
|
||||
*/
|
||||
FILE_BG_SLOTS: number;
|
||||
|
||||
// ========================================================================
|
||||
// Container Plugins Configuration
|
||||
// All plugins run as Docker containers for isolation and flexibility
|
||||
@@ -139,6 +151,8 @@ export const config: EnvConfig = {
|
||||
|
||||
// TaskScheduler Configuration
|
||||
FAST_QUEUE_CONCURRENCY: parseInt(process.env.FAST_QUEUE_CONCURRENCY || "32", 10),
|
||||
FILE_LIGHT_SLOTS: parseInt(process.env.FILE_LIGHT_SLOTS || "16", 10),
|
||||
FILE_BG_SLOTS: parseInt(process.env.FILE_BG_SLOTS || "16", 10),
|
||||
|
||||
// Container Plugins Configuration
|
||||
CONTAINER_PLUGINS_CONFIG: process.env.CONTAINER_PLUGINS_CONFIG || '/app/plugins.yml',
|
||||
|
||||
@@ -1,21 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm} from "@metazla/meta-hash";
|
||||
|
||||
// Target hashes for Piscina worker (full-file hash computation)
|
||||
// Note: midhash256 is NOT included here because it's computed in the main thread
|
||||
// during light processing, not by the Piscina worker
|
||||
export const targetHash = [
|
||||
CID_ALGORITHM_NAMES.sha1, // For torrent verification
|
||||
CID_ALGORITHM_NAMES.sha256, // For IPFS compatibility
|
||||
CID_ALGORITHM_NAMES.md5, // For compatibility
|
||||
CID_ALGORITHM_NAMES.sha3_256, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.sha3_384, // Additional cryptographic hash
|
||||
CID_ALGORITHM_NAMES.crc32, // Fast checksum
|
||||
CID_ALGORITHM_NAMES.btih_v2 // BitTorrent v2 info hash (BEP 52)
|
||||
//
|
||||
// These are algorithm *selectors*, not metadata field names. The old
|
||||
// `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', …) served as both, which is how the
|
||||
// removed `cid_<algo>` record fields kept getting minted in the first place.
|
||||
// Digests are now persisted only as bare-CID key-set members (`cids/<cid>`).
|
||||
// See METADATA_KEYS.md §2/§14.13.
|
||||
export const targetHash: CidAlgorithm[] = [
|
||||
'sha1', // For torrent verification
|
||||
'sha256', // For IPFS compatibility
|
||||
'md5', // For compatibility
|
||||
'sha3_256', // Additional cryptographic hash
|
||||
'sha3_384', // Additional cryptographic hash
|
||||
'crc32', // Fast checksum
|
||||
'btih_v2', // BitTorrent v2 info hash (BEP 52)
|
||||
]
|
||||
|
||||
// Target hashes for index-cid cache (includes all hashes we want to cache)
|
||||
// This includes midhash256 which is computed separately from the worker hashes
|
||||
export const targetHashForIndex = [
|
||||
export const targetHashForIndex: CidAlgorithm[] = [
|
||||
...targetHash,
|
||||
CID_ALGORITHM_NAMES.midhash256 // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
'midhash256', // Fast hash for file identification (computed in light processing)
|
||||
]
|
||||
|
||||
@@ -21,6 +21,7 @@ import type { TaskQueueType } from '../plugin-engine/types.js';
|
||||
import type { IKVClient } from '../kv/IKVClient.js';
|
||||
import { type ExtendedContainerTask } from '../types/ExtendedInterfaces.js';
|
||||
import { config } from '../config/EnvConfig.js';
|
||||
import { FileSlotGate } from './FileSlotGate.js';
|
||||
|
||||
/**
|
||||
* Scheduler events
|
||||
@@ -55,6 +56,18 @@ export interface GateStatus {
|
||||
backgroundRunning: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Per-file holder snapshot exposed via getQueueStatus for the dashboard.
|
||||
* Lets operators see which 16 files are tying up cache slots and what plugin
|
||||
* each file is currently waiting on (the long-pole per file).
|
||||
*/
|
||||
export interface CacheSlotHolder {
|
||||
fileHash: string;
|
||||
filePath?: string;
|
||||
currentPlugin?: string;
|
||||
tasksRemaining: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Container Plugin Scheduler
|
||||
*/
|
||||
@@ -73,11 +86,30 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
private kvClient: IKVClient | null = null;
|
||||
private isAcceptingNewTasks: boolean = true; // Gate control for safe mount/unmount
|
||||
|
||||
// File-level cache slots — see FileSlotGate.ts. Caps the number of files
|
||||
// whose bytes are concurrently being touched in each phase, so the WebDAV
|
||||
// cache only has to hold N files at a time regardless of plugin fan-out.
|
||||
private lightFileGate: FileSlotGate;
|
||||
private bgFileGate: FileSlotGate;
|
||||
// Per-(file, phase) outstanding task counter — drives slot release.
|
||||
private lightTasksPerFile: Map<string, number> = new Map();
|
||||
private bgTasksPerFile: Map<string, number> = new Map();
|
||||
// Tasks staged while waiting for a slot. Once the gate grants the file a
|
||||
// slot we flush the buffered tasks into the underlying PQueue.
|
||||
private lightBufferedTasks: Map<string, ContainerTask[]> = new Map();
|
||||
private bgBufferedTasks: Map<string, ContainerTask[]> = new Map();
|
||||
// Currently-dispatched plugin per file in each phase — used by the
|
||||
// dashboard to show "this file is currently waiting on plugin X".
|
||||
private lightCurrentPlugin: Map<string, string> = new Map();
|
||||
private bgCurrentPlugin: Map<string, string> = new Map();
|
||||
|
||||
constructor(
|
||||
containerManager: ContainerManager,
|
||||
options?: {
|
||||
fastConcurrency?: number;
|
||||
backgroundConcurrency?: number;
|
||||
lightFileSlots?: number;
|
||||
bgFileSlots?: number;
|
||||
callbackUrl?: string;
|
||||
metaCoreUrl?: string;
|
||||
defaultTimeout?: number;
|
||||
@@ -98,6 +130,9 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
autoStart: false,
|
||||
});
|
||||
|
||||
this.lightFileGate = new FileSlotGate(options?.lightFileSlots ?? config.FILE_LIGHT_SLOTS);
|
||||
this.bgFileGate = new FileSlotGate(options?.bgFileSlots ?? config.FILE_BG_SLOTS);
|
||||
|
||||
// When fast queue becomes idle, start background queue
|
||||
this.fastQueue.on('idle', () => {
|
||||
this.onFastQueueIdle();
|
||||
@@ -246,6 +281,17 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
/**
|
||||
* Enqueue a task for execution
|
||||
* Returns false if the gate is closed and the task was rejected
|
||||
*
|
||||
* Tasks pass through the per-phase FileSlotGate before reaching PQueue:
|
||||
* - If the file already holds a slot in this phase, the task goes
|
||||
* straight to PQueue.
|
||||
* - Otherwise the task is buffered, and the file's first task triggers
|
||||
* a slot acquire. When the slot is granted, all buffered tasks for
|
||||
* that file are flushed into PQueue.
|
||||
*
|
||||
* This caps concurrent files per phase (and therefore concurrent file
|
||||
* fetches against CORN/SMB) at the gate's capacity, regardless of how
|
||||
* many plugin tasks each file fans out to.
|
||||
*/
|
||||
enqueueTask(task: ContainerTask): boolean {
|
||||
if (!this.isAcceptingNewTasks) {
|
||||
@@ -262,20 +308,90 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
return false;
|
||||
}
|
||||
|
||||
const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
|
||||
|
||||
// If adding to fast queue, pause background queue (fast takes priority)
|
||||
if (task.queue === 'fast' && !this.backgroundQueue.isPaused) {
|
||||
console.log('[ContainerScheduler] Fast task added, pausing background queue');
|
||||
this.backgroundQueue.pause();
|
||||
}
|
||||
|
||||
queue.add(async () => {
|
||||
await this.executeTask(task);
|
||||
const counterMap = task.queue === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const bufferMap = task.queue === 'fast' ? this.lightBufferedTasks : this.bgBufferedTasks;
|
||||
const gate = task.queue === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
|
||||
// Increment the per-file counter for this phase eagerly. Decrement
|
||||
// happens in the executeTask finally clause.
|
||||
counterMap.set(task.fileHash, (counterMap.get(task.fileHash) ?? 0) + 1);
|
||||
|
||||
if (gate.holds(task.fileHash)) {
|
||||
// File already has a slot — dispatch immediately.
|
||||
this.dispatchToPQueue(task);
|
||||
return true;
|
||||
}
|
||||
|
||||
// File doesn't hold a slot. Buffer the task. If this is the first
|
||||
// buffered task for the file, trigger gate.acquire and flush on grant.
|
||||
const existingBuffer = bufferMap.get(task.fileHash);
|
||||
if (existingBuffer) {
|
||||
existingBuffer.push(task);
|
||||
return true;
|
||||
}
|
||||
|
||||
bufferMap.set(task.fileHash, [task]);
|
||||
gate.acquire(task.fileHash).then(() => {
|
||||
const buffered = bufferMap.get(task.fileHash) ?? [];
|
||||
bufferMap.delete(task.fileHash);
|
||||
for (const t of buffered) {
|
||||
this.dispatchToPQueue(t);
|
||||
}
|
||||
// Edge case: if every buffered task got cancelled by clear() before
|
||||
// the slot was granted, release immediately so the slot doesn't
|
||||
// leak.
|
||||
if (buffered.length === 0 && (counterMap.get(task.fileHash) ?? 0) === 0) {
|
||||
gate.release(task.fileHash);
|
||||
}
|
||||
});
|
||||
return true;
|
||||
}
|
||||
|
||||
/**
|
||||
* Dispatch a task into the underlying PQueue and release its file's gate
|
||||
* slot when the file's last task in that phase finishes.
|
||||
*/
|
||||
private dispatchToPQueue(task: ContainerTask): void {
|
||||
const queue = task.queue === 'background' ? this.backgroundQueue : this.fastQueue;
|
||||
queue.add(async () => {
|
||||
try {
|
||||
await this.executeTask(task);
|
||||
} finally {
|
||||
this.onTaskFinished(task);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Bookkeeping after a task leaves PQueue (success, failure, or thrown).
|
||||
* Decrements the per-(file, phase) task counter and releases the gate
|
||||
* slot when the counter hits zero.
|
||||
*/
|
||||
private onTaskFinished(task: ContainerTask): void {
|
||||
const counterMap = task.queue === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const currentMap = task.queue === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
const gate = task.queue === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
|
||||
const remaining = (counterMap.get(task.fileHash) ?? 1) - 1;
|
||||
if (remaining <= 0) {
|
||||
counterMap.delete(task.fileHash);
|
||||
currentMap.delete(task.fileHash);
|
||||
gate.release(task.fileHash);
|
||||
} else {
|
||||
counterMap.set(task.fileHash, remaining);
|
||||
// Clear current-plugin only if it points at this completed task.
|
||||
if (currentMap.get(task.fileHash) === task.pluginId) {
|
||||
currentMap.delete(task.fileHash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Called when fast queue becomes idle - start background queue if it has work
|
||||
*/
|
||||
@@ -314,6 +430,11 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
task.dispatchedAt = Date.now();
|
||||
task.instanceName = instance.containerName;
|
||||
|
||||
// Record which plugin is currently running for this file (per phase)
|
||||
// so the dashboard can show the long-pole plugin per held cache slot.
|
||||
const currentMap = task.queue === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
currentMap.set(task.fileHash, task.pluginId);
|
||||
|
||||
this.emit('task:dispatched', { task });
|
||||
|
||||
try {
|
||||
@@ -581,6 +702,16 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
backgroundRunning: number; // unique files with background tasks running
|
||||
backgroundWaiting: number; // unique files with background tasks waiting
|
||||
};
|
||||
// Per-phase cache-slot view — bounded by FileSlotGate. This is the
|
||||
// metric operators actually care about: how many distinct files are
|
||||
// tying up the WebDAV cache right now, and which plugin is the
|
||||
// long-pole on each. `paused` reflects the underlying PQueue state so
|
||||
// the UI can distinguish "16 files reserved but bg PQueue paused"
|
||||
// from "16 files actively dispatching".
|
||||
cacheSlots: {
|
||||
fast: { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] };
|
||||
background: { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] };
|
||||
};
|
||||
} {
|
||||
// Count pending callbacks by queue type
|
||||
// These are tasks dispatched to containers awaiting completion - the real "running" count
|
||||
@@ -639,6 +770,56 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
backgroundRunning: backgroundRunningFiles.size,
|
||||
backgroundWaiting: backgroundWaitingFiles.size,
|
||||
},
|
||||
cacheSlots: {
|
||||
fast: this.snapshotCacheSlot('fast'),
|
||||
background: this.snapshotCacheSlot('background'),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the per-phase cache-slot snapshot for the queue status payload.
|
||||
* `currentPlugin` is the plugin currently dispatched against the file
|
||||
* (the long-pole) — falls back to undefined if no task is mid-dispatch.
|
||||
* `tasksRemaining` is the count of light/bg tasks still outstanding for
|
||||
* the file in this phase, which is what holds the slot open.
|
||||
*/
|
||||
private snapshotCacheSlot(
|
||||
phase: TaskQueueType
|
||||
): { capacity: number; inUse: number; waiting: number; paused: boolean; holders: CacheSlotHolder[] } {
|
||||
const gate = phase === 'fast' ? this.lightFileGate : this.bgFileGate;
|
||||
const counterMap = phase === 'fast' ? this.lightTasksPerFile : this.bgTasksPerFile;
|
||||
const currentMap = phase === 'fast' ? this.lightCurrentPlugin : this.bgCurrentPlugin;
|
||||
const queue = phase === 'fast' ? this.fastQueue : this.backgroundQueue;
|
||||
|
||||
const holders: CacheSlotHolder[] = gate.getHolders().map((fileHash) => {
|
||||
// Find any task for this file in this phase to recover filePath
|
||||
// (one filePath is enough for the UI; duplicates collapse).
|
||||
let filePath: string | undefined;
|
||||
const taskIds = this.fileTasks.get(fileHash);
|
||||
if (taskIds) {
|
||||
for (const tid of taskIds) {
|
||||
const t = this.pendingTasks.get(tid);
|
||||
if (t && t.queue === phase) {
|
||||
filePath = t.filePath;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
fileHash,
|
||||
filePath,
|
||||
currentPlugin: currentMap.get(fileHash),
|
||||
tasksRemaining: counterMap.get(fileHash) ?? 0,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
capacity: gate.capacity,
|
||||
inUse: gate.inUse(),
|
||||
waiting: gate.waiting(),
|
||||
paused: queue.isPaused,
|
||||
holders,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -751,6 +932,22 @@ export class ContainerPluginScheduler extends EventEmitter {
|
||||
this.fileTasks.clear();
|
||||
this.fileCompletedPlugins.clear();
|
||||
this.dependencyWaiters.clear();
|
||||
|
||||
// Drop file-slot-gate state. Tasks that were buffered or holding
|
||||
// slots are no longer interesting once we've cleared the queues; a
|
||||
// fresh scheduler state implies fresh gate state.
|
||||
this.lightTasksPerFile.clear();
|
||||
this.bgTasksPerFile.clear();
|
||||
this.lightBufferedTasks.clear();
|
||||
this.bgBufferedTasks.clear();
|
||||
this.lightCurrentPlugin.clear();
|
||||
this.bgCurrentPlugin.clear();
|
||||
for (const fileHash of this.lightFileGate.getHolders()) {
|
||||
this.lightFileGate.release(fileHash);
|
||||
}
|
||||
for (const fileHash of this.bgFileGate.getHolders()) {
|
||||
this.bgFileGate.release(fileHash);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -0,0 +1,81 @@
|
||||
/**
|
||||
* FileSlotGate — bounds the number of files concurrently in a processing phase.
|
||||
*
|
||||
* Motivation: every plugin that runs against a file pulls bytes from CORN/SMB
|
||||
* via WebDAV. Without a per-file gate, the task queue happily fans 8 plugins
|
||||
* across hundreds of files at once, multiplying the WebDAV fetch cost by the
|
||||
* plugin count and overwhelming whatever local cache exists. The gate caps the
|
||||
* number of files that can be "in flight" in a given phase so the cache only
|
||||
* has to hold N files at a time.
|
||||
*
|
||||
* Semantics:
|
||||
* - Keyed by fileHash. A file holds at most one slot in a given phase.
|
||||
* - acquire(fileHash) resolves immediately if the file already holds a slot
|
||||
* or there is free capacity; otherwise it queues FIFO.
|
||||
* - release(fileHash) is idempotent for non-holders. When called on a real
|
||||
* holder it frees the slot and wakes the next waiter.
|
||||
*
|
||||
* The gate intentionally knows nothing about tasks or plugins — it is a thin
|
||||
* file-level semaphore. Counting tasks per (file, phase) and deciding when to
|
||||
* release is the caller's responsibility.
|
||||
*/
|
||||
export class FileSlotGate {
|
||||
private readonly holders: Set<string> = new Set();
|
||||
private readonly waiters: Array<{ fileHash: string; resolve: () => void }> = [];
|
||||
|
||||
constructor(public readonly capacity: number) {
|
||||
if (!Number.isFinite(capacity) || capacity < 1) {
|
||||
throw new Error(`FileSlotGate capacity must be a positive integer, got ${capacity}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Acquire a slot for fileHash. If the file already holds a slot, resolves
|
||||
* immediately (re-entrant for the same file). Otherwise queues until a
|
||||
* slot is released.
|
||||
*/
|
||||
acquire(fileHash: string): Promise<void> {
|
||||
if (this.holders.has(fileHash)) {
|
||||
return Promise.resolve();
|
||||
}
|
||||
if (this.holders.size < this.capacity) {
|
||||
this.holders.add(fileHash);
|
||||
return Promise.resolve();
|
||||
}
|
||||
return new Promise<void>((resolve) => {
|
||||
this.waiters.push({ fileHash, resolve });
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Release the slot held by fileHash. No-op if the file doesn't hold one.
|
||||
* Wakes the next FIFO waiter, if any, and grants it a slot.
|
||||
*/
|
||||
release(fileHash: string): void {
|
||||
if (!this.holders.delete(fileHash)) {
|
||||
return;
|
||||
}
|
||||
const next = this.waiters.shift();
|
||||
if (!next) {
|
||||
return;
|
||||
}
|
||||
this.holders.add(next.fileHash);
|
||||
next.resolve();
|
||||
}
|
||||
|
||||
holds(fileHash: string): boolean {
|
||||
return this.holders.has(fileHash);
|
||||
}
|
||||
|
||||
inUse(): number {
|
||||
return this.holders.size;
|
||||
}
|
||||
|
||||
waiting(): number {
|
||||
return this.waiters.length;
|
||||
}
|
||||
|
||||
getHolders(): string[] {
|
||||
return Array.from(this.holders);
|
||||
}
|
||||
}
|
||||
@@ -1,14 +1,20 @@
|
||||
/**
|
||||
* FileEventConsumer
|
||||
*
|
||||
* Consumes file events from meta-core via Redis Stream.
|
||||
* Consumes file events from meta-core via SSE (Server-Sent Events).
|
||||
* When events are received, they are forwarded to the StreamingPipeline.
|
||||
*
|
||||
* This enables meta-sort to run without direct filesystem access - it receives
|
||||
* This enables meta-sort to run without direct filesystem access — it receives
|
||||
* file discovery events from meta-core and processes files via WebDAV.
|
||||
*
|
||||
* Previously consumed `file:events` via Redis Streams XREADGROUP. The
|
||||
* api-mediated-access migration (PR C) routes this through meta-core's HTTP
|
||||
* /api/events/files endpoint instead, so consumers no longer need direct
|
||||
* Redis access. Cursor durability is now the consumer's responsibility —
|
||||
* see SSEEventClient.
|
||||
*/
|
||||
|
||||
import type { RedisKVClient, StreamMessage } from '../kv/RedisClient.js';
|
||||
import { SSEEventClient, type SSEEvent } from './SSEEventClient.js';
|
||||
|
||||
// Pipeline interface (minimal, matching StreamingPipeline)
|
||||
interface Pipeline {
|
||||
@@ -18,13 +24,9 @@ interface Pipeline {
|
||||
reset(): Promise<void>;
|
||||
}
|
||||
|
||||
// Redis Stream configuration
|
||||
const EVENTS_STREAM = 'file:events';
|
||||
const CONSUMER_GROUP = 'meta-sort-processor';
|
||||
|
||||
export interface FileEventConsumerOptions {
|
||||
/** Redis KV client */
|
||||
kvClient: RedisKVClient;
|
||||
/** meta-core API base URL (e.g. http://metacore-app:9000). */
|
||||
metaCoreApiUrl: string;
|
||||
|
||||
/** Pipeline to receive events */
|
||||
pipeline: Pipeline;
|
||||
@@ -32,115 +34,104 @@ export interface FileEventConsumerOptions {
|
||||
/** Base path for files (e.g., /files). Paths from meta-core are relative and need this prefix. */
|
||||
filesPath?: string;
|
||||
|
||||
/** Block timeout in milliseconds for stream reads */
|
||||
blockMs?: number;
|
||||
/** Where to persist the SSE cursor. Defaults to /meta-core/cursors/meta-sort-files.cursor. */
|
||||
cursorPath?: string;
|
||||
}
|
||||
|
||||
export class FileEventConsumer {
|
||||
private kvClient: RedisKVClient;
|
||||
private pipeline: Pipeline;
|
||||
private filesPath: string;
|
||||
private blockMs: number;
|
||||
private sse: SSEEventClient;
|
||||
private isRunning = false;
|
||||
|
||||
constructor(options: FileEventConsumerOptions) {
|
||||
this.kvClient = options.kvClient;
|
||||
this.pipeline = options.pipeline;
|
||||
this.filesPath = options.filesPath ?? '/files';
|
||||
this.blockMs = options.blockMs ?? 5000;
|
||||
|
||||
const apiUrl = options.metaCoreApiUrl.replace(/\/+$/, '');
|
||||
const cursorPath = options.cursorPath ?? '/meta-core/cursors/meta-sort-files.cursor';
|
||||
|
||||
this.sse = new SSEEventClient({
|
||||
url: `${apiUrl}/api/events/files`,
|
||||
cursorPath,
|
||||
logTag: '[FileEventConsumer]',
|
||||
onEvent: (e) => this.handleSSEEvent(e),
|
||||
// gap on file:events means we missed retention. Easiest correct
|
||||
// response: ask meta-core for a watcher rescan so the StreamingPipeline
|
||||
// can rebuild from a known-good baseline.
|
||||
onGap: () => {
|
||||
console.warn('[FileEventConsumer] Stream cursor trimmed — pipeline will rebuild from next reset event');
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Start consuming events from Redis Stream
|
||||
* Start consuming events from the meta-core SSE endpoint.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
if (this.isRunning) {
|
||||
console.log('[FileEventConsumer] Already running');
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = true;
|
||||
console.log(`[FileEventConsumer] Starting stream consumer for ${EVENTS_STREAM}...`);
|
||||
|
||||
// Initialize consumer group
|
||||
await this.kvClient.initStreamConsumer(EVENTS_STREAM, CONSUMER_GROUP);
|
||||
|
||||
// Process any pending entries from crashed consumers
|
||||
await this.kvClient.processPendingEntries(
|
||||
EVENTS_STREAM,
|
||||
CONSUMER_GROUP,
|
||||
30000, // 30 second idle threshold
|
||||
async (message) => this.handleMessage(message)
|
||||
);
|
||||
|
||||
// Start consumer loop in background (don't await - runs indefinitely)
|
||||
this.kvClient.startStreamConsumer(
|
||||
EVENTS_STREAM,
|
||||
CONSUMER_GROUP,
|
||||
async (message) => this.handleMessage(message),
|
||||
this.blockMs
|
||||
).catch(error => {
|
||||
console.error('[FileEventConsumer] Stream consumer error:', error);
|
||||
console.log('[FileEventConsumer] Starting SSE stream consumer for /api/events/files...');
|
||||
// Fire-and-forget. SSEEventClient.start() returns only when stopped.
|
||||
this.sse.start().catch(error => {
|
||||
console.error('[FileEventConsumer] Stream consumer terminated:', error);
|
||||
});
|
||||
|
||||
console.log('[FileEventConsumer] Stream consumer started');
|
||||
console.log('[FileEventConsumer] SSE stream consumer started');
|
||||
}
|
||||
|
||||
/**
|
||||
* Stop consuming events
|
||||
*/
|
||||
stop(): void {
|
||||
async stop(): Promise<void> {
|
||||
if (!this.isRunning) {
|
||||
return;
|
||||
}
|
||||
|
||||
this.isRunning = false;
|
||||
this.kvClient.stopStreamConsumer();
|
||||
await this.sse.stop();
|
||||
console.log('[FileEventConsumer] Stopped');
|
||||
}
|
||||
|
||||
/**
|
||||
* Handle a stream message
|
||||
*/
|
||||
private async handleMessage(message: StreamMessage): Promise<void> {
|
||||
private async handleSSEEvent(e: SSEEvent): Promise<void> {
|
||||
// Convert relative path to absolute path
|
||||
const toAbsolutePath = (relativePath: string): string => {
|
||||
// If path is already absolute, return as-is
|
||||
if (relativePath.startsWith('/')) {
|
||||
return relativePath;
|
||||
}
|
||||
// Otherwise, prepend filesPath
|
||||
if (!relativePath) return relativePath;
|
||||
if (relativePath.startsWith('/')) return relativePath;
|
||||
return `${this.filesPath}/${relativePath}`;
|
||||
};
|
||||
|
||||
switch (message.type) {
|
||||
const path: string = e.data?.path ?? '';
|
||||
const midhash256: string | undefined = e.data?.midhash256 ?? undefined;
|
||||
const oldPath: string | undefined = e.data?.oldPath ?? undefined;
|
||||
const watcherId: string | undefined = e.data?.watcherId ?? undefined;
|
||||
|
||||
switch (e.event) {
|
||||
case 'reset':
|
||||
console.log(`[FileEventConsumer] Received reset event from watcher: ${message.watcherId || 'all'}`);
|
||||
console.log(`[FileEventConsumer] Received reset event from watcher: ${watcherId || 'all'}`);
|
||||
await this.pipeline.reset();
|
||||
break;
|
||||
|
||||
case 'add':
|
||||
this.pipeline.handleFileAdded(toAbsolutePath(message.path), message.midhash256);
|
||||
if (path) this.pipeline.handleFileAdded(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
case 'change':
|
||||
this.pipeline.handleFileChanged(toAbsolutePath(message.path), message.midhash256);
|
||||
if (path) this.pipeline.handleFileChanged(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
case 'delete':
|
||||
this.pipeline.handleFileDeleted(toAbsolutePath(message.path));
|
||||
if (path) this.pipeline.handleFileDeleted(toAbsolutePath(path));
|
||||
break;
|
||||
|
||||
case 'rename':
|
||||
// Handle rename as delete + add
|
||||
if (message.oldPath) {
|
||||
this.pipeline.handleFileDeleted(toAbsolutePath(message.oldPath));
|
||||
}
|
||||
this.pipeline.handleFileAdded(toAbsolutePath(message.path), message.midhash256);
|
||||
if (oldPath) this.pipeline.handleFileDeleted(toAbsolutePath(oldPath));
|
||||
if (path) this.pipeline.handleFileAdded(toAbsolutePath(path), midhash256);
|
||||
break;
|
||||
|
||||
default:
|
||||
console.warn(`[FileEventConsumer] Unknown event type: ${message.type}`);
|
||||
console.warn(`[FileEventConsumer] Unknown event type: ${e.event}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -149,12 +140,12 @@ export class FileEventConsumer {
|
||||
* Create a FileEventConsumer with default configuration
|
||||
*/
|
||||
export function createFileEventConsumer(
|
||||
kvClient: RedisKVClient,
|
||||
metaCoreApiUrl: string,
|
||||
pipeline: Pipeline,
|
||||
options?: Partial<FileEventConsumerOptions>
|
||||
): FileEventConsumer {
|
||||
return new FileEventConsumer({
|
||||
kvClient,
|
||||
metaCoreApiUrl,
|
||||
pipeline,
|
||||
...options
|
||||
});
|
||||
|
||||
@@ -0,0 +1,295 @@
|
||||
/**
|
||||
* SSEEventClient — HTTP Server-Sent Events client for meta-core event streams.
|
||||
*
|
||||
* Replaces the Redis Streams XREADGROUP loop. Each consumer service holds
|
||||
* one long-lived HTTP connection to meta-core's /api/events/{files,meta}
|
||||
* endpoint and receives JSON events as the underlying Redis Stream is
|
||||
* appended to.
|
||||
*
|
||||
* Cursor semantics (see meta-core docs/api-mediated-access.md):
|
||||
* - The SSE wire id is the opaque Redis Stream entry ID (`<ms>-<seq>`).
|
||||
* We treat it as a black-box string.
|
||||
* - On reconnect we send Last-Event-ID so meta-core resumes from the
|
||||
* entry AFTER our cursor.
|
||||
* - We persist the cursor to disk so the loop survives a process
|
||||
* restart. The cursor is written every flushIntervalMs and on graceful
|
||||
* stop.
|
||||
* - On a `gap` event the server is telling us our cursor was trimmed
|
||||
* out of retention. Caller-supplied onGap handles it (typically a
|
||||
* re-bootstrap from a full HTTP fetch). If the caller doesn't handle
|
||||
* gaps, we just continue from the new cursor.
|
||||
*/
|
||||
import { promises as fs } from 'fs';
|
||||
import { dirname } from 'path';
|
||||
|
||||
/** One SSE event parsed off the wire. */
|
||||
export interface SSEEvent {
|
||||
/** Opaque cursor (Redis Stream entry ID under the hood). */
|
||||
id: string;
|
||||
/** Event type — `add` / `change` / `delete` / `rename` / `reset` for
|
||||
* file:events; `set` / `del` / `expire` for meta:events; `gap` for
|
||||
* trim notifications; arbitrary otherwise. */
|
||||
event: string;
|
||||
/** Parsed JSON payload from the `data:` line. */
|
||||
data: any;
|
||||
}
|
||||
|
||||
export interface SSEClientOptions {
|
||||
/** Full URL to subscribe to (e.g. http://metacore-app:9000/api/events/files). */
|
||||
url: string;
|
||||
|
||||
/** Path where the cursor is persisted between runs.
|
||||
* Set to null to keep the cursor in-memory only (at-most-once on crash). */
|
||||
cursorPath: string | null;
|
||||
|
||||
/** Handler for every non-gap event. */
|
||||
onEvent: (e: SSEEvent) => Promise<void> | void;
|
||||
|
||||
/** Optional handler for the synthetic `gap` event. If omitted we just
|
||||
* log and continue from the resumeFrom cursor inside the payload. */
|
||||
onGap?: (payload: { requested: string; resumeFrom: string; reason: string }) => Promise<void> | void;
|
||||
|
||||
/** Logger prefix used for console.log lines. */
|
||||
logTag?: string;
|
||||
|
||||
/** Initial reconnect backoff in ms (doubles per failure, capped). Default 500. */
|
||||
reconnectBaseMs?: number;
|
||||
|
||||
/** Max reconnect backoff in ms. Default 30_000. */
|
||||
reconnectMaxMs?: number;
|
||||
|
||||
/** How often to flush the in-memory cursor to disk. Default 2000. */
|
||||
flushIntervalMs?: number;
|
||||
}
|
||||
|
||||
export class SSEEventClient {
|
||||
private readonly opts: Required<Omit<SSEClientOptions, 'onGap'>> & Pick<SSEClientOptions, 'onGap'>;
|
||||
private cursor: string = '';
|
||||
private cursorDirty = false;
|
||||
private abort: AbortController | null = null;
|
||||
private flushTimer: NodeJS.Timeout | null = null;
|
||||
private stopped = false;
|
||||
private backoff = 0;
|
||||
|
||||
constructor(options: SSEClientOptions) {
|
||||
this.opts = {
|
||||
url: options.url,
|
||||
cursorPath: options.cursorPath,
|
||||
onEvent: options.onEvent,
|
||||
onGap: options.onGap,
|
||||
logTag: options.logTag ?? '[SSE]',
|
||||
reconnectBaseMs: options.reconnectBaseMs ?? 500,
|
||||
reconnectMaxMs: options.reconnectMaxMs ?? 30_000,
|
||||
flushIntervalMs: options.flushIntervalMs ?? 2000,
|
||||
};
|
||||
this.backoff = this.opts.reconnectBaseMs;
|
||||
}
|
||||
|
||||
/**
|
||||
* Read the persisted cursor (if any) and begin streaming. The returned
|
||||
* promise resolves on stop()/permanent error; callers should not await
|
||||
* it directly — fire-and-forget and call stop() to shut down.
|
||||
*/
|
||||
async start(): Promise<void> {
|
||||
await this.loadCursor();
|
||||
this.startFlushTimer();
|
||||
await this.runLoop();
|
||||
}
|
||||
|
||||
/** Graceful shutdown — closes the open connection and flushes the cursor. */
|
||||
async stop(): Promise<void> {
|
||||
this.stopped = true;
|
||||
if (this.abort) {
|
||||
this.abort.abort();
|
||||
this.abort = null;
|
||||
}
|
||||
if (this.flushTimer) {
|
||||
clearInterval(this.flushTimer);
|
||||
this.flushTimer = null;
|
||||
}
|
||||
await this.flushCursor();
|
||||
}
|
||||
|
||||
/** Currently-known cursor — useful for tests. */
|
||||
getCursor(): string {
|
||||
return this.cursor;
|
||||
}
|
||||
|
||||
private async loadCursor(): Promise<void> {
|
||||
if (!this.opts.cursorPath) return;
|
||||
try {
|
||||
const data = await fs.readFile(this.opts.cursorPath, 'utf-8');
|
||||
const trimmed = data.trim();
|
||||
if (trimmed) {
|
||||
this.cursor = trimmed;
|
||||
console.log(`${this.opts.logTag} Resuming from cursor ${this.cursor}`);
|
||||
}
|
||||
} catch (err: any) {
|
||||
if (err.code !== 'ENOENT') {
|
||||
console.warn(`${this.opts.logTag} Could not read cursor file ${this.opts.cursorPath}: ${err.message}`);
|
||||
}
|
||||
// ENOENT is expected first time; start at "$" (server-default).
|
||||
}
|
||||
}
|
||||
|
||||
private startFlushTimer(): void {
|
||||
this.flushTimer = setInterval(() => {
|
||||
if (this.cursorDirty) {
|
||||
this.flushCursor().catch(err =>
|
||||
console.warn(`${this.opts.logTag} Cursor flush failed: ${err.message}`)
|
||||
);
|
||||
}
|
||||
}, this.opts.flushIntervalMs);
|
||||
}
|
||||
|
||||
private async flushCursor(): Promise<void> {
|
||||
if (!this.opts.cursorPath || !this.cursorDirty) return;
|
||||
this.cursorDirty = false;
|
||||
try {
|
||||
await fs.mkdir(dirname(this.opts.cursorPath), { recursive: true });
|
||||
await fs.writeFile(this.opts.cursorPath, this.cursor, 'utf-8');
|
||||
} catch (err: any) {
|
||||
// Re-mark dirty so the next tick retries.
|
||||
this.cursorDirty = true;
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
private async runLoop(): Promise<void> {
|
||||
while (!this.stopped) {
|
||||
try {
|
||||
await this.connect();
|
||||
// connect() only returns on graceful end-of-stream — reset
|
||||
// backoff so a long-running connection that finally
|
||||
// disconnects reconnects quickly.
|
||||
this.backoff = this.opts.reconnectBaseMs;
|
||||
} catch (err: any) {
|
||||
if (this.stopped) return;
|
||||
console.warn(`${this.opts.logTag} Stream error, reconnecting in ${this.backoff}ms: ${err?.message ?? err}`);
|
||||
await this.sleep(this.backoff);
|
||||
this.backoff = Math.min(this.backoff * 2, this.opts.reconnectMaxMs);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async connect(): Promise<void> {
|
||||
this.abort = new AbortController();
|
||||
const headers: Record<string, string> = {
|
||||
Accept: 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
};
|
||||
if (this.cursor) {
|
||||
headers['Last-Event-ID'] = this.cursor;
|
||||
}
|
||||
|
||||
const response = await fetch(this.opts.url, {
|
||||
method: 'GET',
|
||||
headers,
|
||||
signal: this.abort.signal,
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`HTTP ${response.status} ${response.statusText}`);
|
||||
}
|
||||
if (!response.body) {
|
||||
throw new Error('Response has no body');
|
||||
}
|
||||
|
||||
const reader = response.body.getReader();
|
||||
const decoder = new TextDecoder('utf-8');
|
||||
let buffer = '';
|
||||
|
||||
try {
|
||||
while (!this.stopped) {
|
||||
const { value, done } = await reader.read();
|
||||
if (done) {
|
||||
return;
|
||||
}
|
||||
buffer += decoder.decode(value, { stream: true });
|
||||
// Each SSE event ends with a blank line ("\n\n"). Split on
|
||||
// that boundary and process each chunk.
|
||||
let idx: number;
|
||||
while ((idx = buffer.indexOf('\n\n')) !== -1) {
|
||||
const rawChunk = buffer.slice(0, idx);
|
||||
buffer = buffer.slice(idx + 2);
|
||||
await this.handleChunk(rawChunk);
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
try { reader.releaseLock(); } catch { /* nothing useful */ }
|
||||
}
|
||||
}
|
||||
|
||||
private async handleChunk(chunk: string): Promise<void> {
|
||||
let id = '';
|
||||
let event = '';
|
||||
const dataLines: string[] = [];
|
||||
for (const rawLine of chunk.split('\n')) {
|
||||
if (!rawLine || rawLine.startsWith(':')) {
|
||||
continue; // comment / heartbeat — ignore
|
||||
}
|
||||
const colon = rawLine.indexOf(':');
|
||||
if (colon === -1) continue;
|
||||
const field = rawLine.slice(0, colon);
|
||||
// SSE spec: ": " (single space after the colon) is stripped; if no space, keep as-is.
|
||||
let value = rawLine.slice(colon + 1);
|
||||
if (value.startsWith(' ')) value = value.slice(1);
|
||||
|
||||
switch (field) {
|
||||
case 'id':
|
||||
id = value;
|
||||
break;
|
||||
case 'event':
|
||||
event = value;
|
||||
break;
|
||||
case 'data':
|
||||
dataLines.push(value);
|
||||
break;
|
||||
// ignore other fields (retry, etc.)
|
||||
}
|
||||
}
|
||||
|
||||
if (dataLines.length === 0) {
|
||||
return; // bare heartbeat / no payload
|
||||
}
|
||||
|
||||
let data: any;
|
||||
try {
|
||||
data = JSON.parse(dataLines.join('\n'));
|
||||
} catch (err: any) {
|
||||
console.warn(`${this.opts.logTag} Skipping unparseable event ${id || '(no id)'}: ${err.message}`);
|
||||
return;
|
||||
}
|
||||
|
||||
const ev: SSEEvent = { id, event: event || 'message', data };
|
||||
|
||||
if (ev.event === 'gap') {
|
||||
if (this.opts.onGap) {
|
||||
await this.opts.onGap(ev.data);
|
||||
} else {
|
||||
console.warn(`${this.opts.logTag} Gap event (cursor ${ev.data?.requested} trimmed): resumed from ${ev.data?.resumeFrom}`);
|
||||
}
|
||||
if (typeof ev.data?.resumeFrom === 'string' && ev.data.resumeFrom) {
|
||||
this.cursor = ev.data.resumeFrom;
|
||||
this.cursorDirty = true;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await this.opts.onEvent(ev);
|
||||
} catch (err: any) {
|
||||
// Handler error doesn't tear down the connection — log and move on.
|
||||
console.error(`${this.opts.logTag} onEvent handler failed for ${ev.id}: ${err?.stack ?? err}`);
|
||||
}
|
||||
|
||||
if (ev.id) {
|
||||
this.cursor = ev.id;
|
||||
this.cursorDirty = true;
|
||||
}
|
||||
}
|
||||
|
||||
private sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
}
|
||||
@@ -104,7 +104,7 @@ process.on('SIGINT', async () => {
|
||||
// Stop File Event Consumer if running
|
||||
if (fileEventConsumer) {
|
||||
console.log('[Shutdown] Stopping File Event Consumer...');
|
||||
fileEventConsumer.stop();
|
||||
await fileEventConsumer.stop();
|
||||
}
|
||||
|
||||
// Stop Container Manager if running
|
||||
@@ -256,21 +256,21 @@ process.on('SIGINT', async () => {
|
||||
console.log(`[Cache] midhash256 cache: ${hits} hits, ${misses} misses (${hitRate}% hit rate)`);
|
||||
}
|
||||
|
||||
// Connect to meta-core for file events via Redis Stream
|
||||
// KV client must be available
|
||||
if (!kvClient) {
|
||||
console.error('[Startup] ERROR: KV client not available. Cannot consume file events.');
|
||||
console.error('[Startup] Make sure meta-core is running and Redis is accessible.');
|
||||
// Connect to meta-core for file events via SSE
|
||||
const leaderInfoForSSE = kvManager?.getLeaderInfo();
|
||||
const metaCoreApiUrl = leaderInfoForSSE?.apiUrl;
|
||||
if (!metaCoreApiUrl) {
|
||||
console.error('[Startup] ERROR: meta-core apiUrl not available from leader info. Cannot consume file events.');
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('[Startup] Starting File Event Consumer (Redis Stream)...');
|
||||
console.log(`[Startup] Starting File Event Consumer (SSE → ${metaCoreApiUrl}/api/events/files)...`);
|
||||
fileEventConsumer = new FileEventConsumer({
|
||||
kvClient: kvClient as RedisKVClient,
|
||||
metaCoreApiUrl,
|
||||
pipeline: pipeline,
|
||||
filesPath: config.FILES_PATH,
|
||||
});
|
||||
await fileEventConsumer.start();
|
||||
console.log('[Startup] FileEventConsumer started, consuming from file:events stream');
|
||||
console.log('[Startup] FileEventConsumer started, consuming via SSE');
|
||||
|
||||
} catch (error) {
|
||||
console.error('[Startup] Error during startup:', error);
|
||||
|
||||
@@ -73,13 +73,17 @@ export class KVManager {
|
||||
}
|
||||
|
||||
/**
|
||||
* Create Redis client from URL
|
||||
* Create Redis client from URL. When apiUrl is supplied, the resulting
|
||||
* client routes writes through meta-core's HTTP API (PR B of the
|
||||
* api-mediated-access lockdown) while keeping reads + stream consumption
|
||||
* on the direct Redis connection.
|
||||
*/
|
||||
private async createRedisClient(redisUrl: string): Promise<RedisKVClient> {
|
||||
private async createRedisClient(redisUrl: string, apiUrl?: string): Promise<RedisKVClient> {
|
||||
const client = new RedisKVClient({
|
||||
url: redisUrl,
|
||||
timeout: 30000,
|
||||
prefix: '' // No prefix for shared access
|
||||
prefix: '', // No prefix for shared access
|
||||
metaCoreApiUrl: apiUrl,
|
||||
});
|
||||
|
||||
await client.connect();
|
||||
@@ -121,8 +125,16 @@ export class KVManager {
|
||||
// Wait for leader and connect
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||
if (leaderInfo.redisUrl) {
|
||||
// api-mediated-access PR D removes redisUrl from leader info.
|
||||
// Until that lands we still log it but services boot fine
|
||||
// when it's absent — all writes/reads route through HTTP.
|
||||
console.warn('[KVManager] Leader info still publishes redisUrl; direct Redis access is being deprecated, prefer meta-core HTTP');
|
||||
console.log(`[KVManager] Connecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
} else {
|
||||
console.log('[KVManager] redisUrl absent in leader info — running in HTTP-only mode');
|
||||
}
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl, leaderInfo.apiUrl);
|
||||
|
||||
// Configure WebDAV client with leader's WebDAV URL (prefer internal for container-to-container)
|
||||
const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl;
|
||||
@@ -169,7 +181,7 @@ export class KVManager {
|
||||
try {
|
||||
const leaderInfo = await this.leaderClient.waitForLeader(30000);
|
||||
console.log(`[KVManager] Reconnecting to Redis at ${leaderInfo.redisUrl}...`);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl);
|
||||
this.kvClient = await this.createRedisClient(leaderInfo.redisUrl, leaderInfo.apiUrl);
|
||||
|
||||
// Reconfigure WebDAV client with new leader's WebDAV URL (prefer internal for container-to-container)
|
||||
const webdavUrl = leaderInfo.webdavUrlInternal ?? leaderInfo.webdavUrl;
|
||||
|
||||
@@ -85,7 +85,24 @@ export class LeaderClient {
|
||||
return null;
|
||||
}
|
||||
|
||||
this.cachedUrls = await response.json() as URLsResponse;
|
||||
const parsed = await response.json() as URLsResponse;
|
||||
// api-mediated-access PR D: redisUrl is no longer published by
|
||||
// meta-core. If it shows up, an older build snuck back in —
|
||||
// fail loudly so the rollback can't go unnoticed. Set
|
||||
// ALLOW_LEGACY_REDIS_URL=1 to downgrade to a warning during a
|
||||
// deliberate temporary rollback.
|
||||
if (parsed.redisUrl) {
|
||||
const msg =
|
||||
'[LeaderClient] meta-core still publishes redisUrl; ' +
|
||||
'direct Redis access was retired by the api-mediated-access ' +
|
||||
'lockdown. Verify meta-core version.';
|
||||
if (process.env.ALLOW_LEGACY_REDIS_URL === '1') {
|
||||
console.warn('WARNING: ' + msg);
|
||||
} else {
|
||||
throw new Error(msg);
|
||||
}
|
||||
}
|
||||
this.cachedUrls = parsed;
|
||||
this.urlsCacheTime = now;
|
||||
return this.cachedUrls;
|
||||
} catch (error) {
|
||||
|
||||
@@ -0,0 +1,223 @@
|
||||
/**
|
||||
* MetaCoreApiWriter — HTTP writer for metadata, replacing direct Redis writes.
|
||||
*
|
||||
* meta-core owns the Redis schema (UUID-rooted, reverse-indexed by CID). Any
|
||||
* service writing directly to Redis risks drifting the schema; the lockdown
|
||||
* plan (docs/api-mediated-access.md in meta-core) routes every write through
|
||||
* meta-core's HTTP /meta/{hash} family.
|
||||
*
|
||||
* This writer covers the three operations meta-sort actually performs in
|
||||
* production:
|
||||
* - setMetadataFlat → PATCH /meta/{hash} (merge semantics)
|
||||
* - setMetadataProperty → PUT /meta/{hash}/{key}
|
||||
* - deleteMetadataFlat → DELETE /meta/{hash}
|
||||
*
|
||||
* Reads and stream consumption stay on Redis in PR B; PR C migrates those.
|
||||
*/
|
||||
import {
|
||||
buildRecordFields,
|
||||
buildFilePrefix,
|
||||
} from './MetadataUtils.js';
|
||||
|
||||
export interface MetaCoreApiWriterConfig {
|
||||
/** Base URL for meta-core's HTTP API (e.g. http://metacore-app:9000) */
|
||||
apiUrl: string;
|
||||
|
||||
/** Optional per-request timeout in ms (default: 30000) */
|
||||
timeoutMs?: number;
|
||||
}
|
||||
|
||||
export class MetaCoreApiWriter {
|
||||
private apiUrl: string;
|
||||
private timeoutMs: number;
|
||||
|
||||
constructor(config: MetaCoreApiWriterConfig) {
|
||||
// Trim trailing slash so we can join paths predictably.
|
||||
this.apiUrl = config.apiUrl.replace(/\/+$/, '');
|
||||
this.timeoutMs = config.timeoutMs ?? 30000;
|
||||
}
|
||||
|
||||
/**
|
||||
* PATCH /meta/{hash}.
|
||||
*
|
||||
* meta-core's PATCH calls MergeMetadataFlat — merge semantics, matches
|
||||
* what setMetadataFlat used to do via Redis pipeline of SETs.
|
||||
*/
|
||||
async setMetadataFlat(
|
||||
hashId: string,
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
// `buildRecordFields` is the single write-shape boundary, shared with the
|
||||
// direct-Redis path in RedisClient. It emits the `cids/<cid>` key-set and
|
||||
// never a per-algorithm `cid_*` field — which meta-core would now reject
|
||||
// with a 400 anyway.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
await this.fetchJson('PATCH', `/meta/${encodeURIComponent(hashId)}`, flat);
|
||||
}
|
||||
|
||||
/**
|
||||
* PUT /meta/{hash}/{key}.
|
||||
*
|
||||
* meta-core accepts either JSON {"value":"..."} or plain text; we use
|
||||
* the JSON form to avoid ambiguity with values that happen to look like
|
||||
* JSON themselves.
|
||||
*/
|
||||
async setMetadataProperty(hashId: string, property: string, value: string): Promise<void> {
|
||||
await this.fetchJson(
|
||||
'PUT',
|
||||
`/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(property)}`,
|
||||
{ value }
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* DELETE /meta/{hash}.
|
||||
*
|
||||
* Removes every property and unregisters the hashId from file:__index__.
|
||||
*/
|
||||
async deleteMetadataFlat(hashId: string): Promise<number> {
|
||||
const body = await this.fetchJson(
|
||||
'DELETE',
|
||||
`/meta/${encodeURIComponent(hashId)}`,
|
||||
null
|
||||
);
|
||||
// meta-core's response is {success, hashId, deleted: N}.
|
||||
if (body && typeof body === 'object' && typeof (body as any).deleted === 'number') {
|
||||
return (body as any).deleted;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta/{hash}. Returns the flat metadata map (or null if absent).
|
||||
*/
|
||||
async getMetadataFlat(hashId: string): Promise<any | null> {
|
||||
const path = `/meta/${encodeURIComponent(hashId)}`;
|
||||
const body = await this.fetchJsonAllowing404('GET', path);
|
||||
if (body === null) return null;
|
||||
// meta-core wraps the flat map under `metadata`.
|
||||
const flat = (body as any)?.metadata;
|
||||
if (!flat || typeof flat !== 'object') return null;
|
||||
return flat;
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta/{hash}/{propertyPath}. Returns the raw string value (or null
|
||||
* if absent). meta-core sends back text/plain.
|
||||
*/
|
||||
async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
|
||||
const path = `/meta/${encodeURIComponent(hashId)}/${this.encodePropertyPath(propertyPath)}`;
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const response = await fetch(url, {
|
||||
method: 'GET',
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'text/plain' },
|
||||
});
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`[MetaCoreApiWriter] GET ${path} → ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
// Mirror the legacy RedisClient: try JSON-parse first, fall back to raw string.
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /meta. Returns the list of all hashIds known to meta-core.
|
||||
*/
|
||||
async getAllHashIds(): Promise<string[]> {
|
||||
const body = await this.fetchJson('GET', '/meta', null);
|
||||
const ids = (body as any)?.hashIds;
|
||||
return Array.isArray(ids) ? ids : [];
|
||||
}
|
||||
|
||||
/**
|
||||
* GET /health on meta-core. Used as a write-path health probe.
|
||||
*/
|
||||
async health(): Promise<boolean> {
|
||||
try {
|
||||
const response = await fetch(`${this.apiUrl}/health`, {
|
||||
method: 'GET',
|
||||
signal: AbortSignal.timeout(Math.min(this.timeoutMs, 5000)),
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
return response.ok;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
private async fetchJsonAllowing404(method: string, path: string): Promise<unknown> {
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const response = await fetch(url, {
|
||||
method,
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'application/json' },
|
||||
});
|
||||
if (response.status === 404) return null;
|
||||
if (!response.ok) {
|
||||
throw new Error(`[MetaCoreApiWriter] ${method} ${path} → ${response.status} ${response.statusText}`);
|
||||
}
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
try { return JSON.parse(text); } catch { return text; }
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode a slash-containing property path safely for the URL while still
|
||||
* letting meta-core's `{key:.*}` route match the multi-segment key.
|
||||
*/
|
||||
private encodePropertyPath(property: string): string {
|
||||
return property
|
||||
.split('/')
|
||||
.map(segment => encodeURIComponent(segment))
|
||||
.join('/');
|
||||
}
|
||||
|
||||
private async fetchJson(
|
||||
method: string,
|
||||
path: string,
|
||||
body: unknown
|
||||
): Promise<unknown> {
|
||||
const url = `${this.apiUrl}${path}`;
|
||||
const init: RequestInit = {
|
||||
method,
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
headers: { Accept: 'application/json' },
|
||||
};
|
||||
if (body !== null && body !== undefined) {
|
||||
init.headers = { ...init.headers, 'Content-Type': 'application/json' };
|
||||
init.body = JSON.stringify(body);
|
||||
}
|
||||
|
||||
const response = await fetch(url, init);
|
||||
if (!response.ok) {
|
||||
// Surface meta-core's error envelope when it's structured.
|
||||
let detail = response.statusText;
|
||||
try {
|
||||
const errBody = await response.text();
|
||||
if (errBody) detail = `${detail}: ${errBody}`;
|
||||
} catch {
|
||||
/* ignore body-read errors; statusText still useful */
|
||||
}
|
||||
throw new Error(`[MetaCoreApiWriter] ${method} ${path} → ${response.status} ${detail}`);
|
||||
}
|
||||
|
||||
// 204 / empty body — return null so callers don't choke on JSON parse.
|
||||
const text = await response.text();
|
||||
if (!text) return null;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,86 @@
|
||||
import { strict as assert } from 'assert';
|
||||
import { buildRecordFields } from './MetadataUtils.js';
|
||||
|
||||
/**
|
||||
* `buildRecordFields` is the single boundary where in-memory metadata becomes a
|
||||
* record. It is worth testing precisely because the bug it fixes had no coverage:
|
||||
* meta-sort had *two* write paths (the meta-core HTTP writer and a direct-Redis
|
||||
* fallback), only one of which turned sibling CIDs into the `cids/<cid>` key-set.
|
||||
* The other wrote a literal `cid_midhash256` field, which meta-core stores but
|
||||
* never reverse-indexes — so those records were unresolvable by their own CID,
|
||||
* silently. Both paths now go through this function.
|
||||
*/
|
||||
describe('buildRecordFields', () => {
|
||||
const MIDHASH = 'bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4';
|
||||
const SHA256 = 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua';
|
||||
|
||||
it('turns cids[] into the bare-CID key-set, not array indices', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH, SHA256],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
assert.equal(flat[`cids/${MIDHASH}`], 'true');
|
||||
assert.equal(flat[`cids/${SHA256}`], 'true');
|
||||
|
||||
// The trap: the generic flattener would emit `cids/0`, `cids/1` for an
|
||||
// array. meta-core's reverse-index hook only matches `cids/<cid>`, so
|
||||
// those would be stored and never indexed.
|
||||
assert.equal(flat['cids/0'], undefined);
|
||||
assert.equal(flat['cids/1'], undefined);
|
||||
});
|
||||
|
||||
it('never emits a deprecated cid_* or midhash256 field', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
});
|
||||
|
||||
// meta-core returns 400 for any of these now — a record that carried one
|
||||
// was stored but unindexed.
|
||||
for (const field of Object.keys(flat)) {
|
||||
assert.ok(
|
||||
!field.startsWith('cid_'),
|
||||
`deprecated per-algorithm field reached the record: ${field}`
|
||||
);
|
||||
assert.notEqual(field, 'midhash256');
|
||||
assert.notEqual(field, 'canonical_cid');
|
||||
}
|
||||
});
|
||||
|
||||
it('does not store hashId as a property of its own record', () => {
|
||||
const flat = buildRecordFields({hashId: MIDHASH, cids: [MIDHASH], title: 'x'});
|
||||
// hashId IS the record key (`/file/{hashId}`); repeating it inside is noise.
|
||||
assert.equal(flat['hashId'], undefined);
|
||||
});
|
||||
|
||||
it('still flattens ordinary nested metadata', () => {
|
||||
const flat = buildRecordFields({
|
||||
hashId: MIDHASH,
|
||||
cids: [MIDHASH],
|
||||
title: 'Inception',
|
||||
fileinfo: {duration: 8878.5, streamdetails: {video: {codec: 'h264'}}},
|
||||
});
|
||||
|
||||
assert.equal(flat['title'], 'Inception');
|
||||
assert.equal(flat['fileinfo/duration'], '8878.5');
|
||||
assert.equal(flat['fileinfo/streamdetails/video/codec'], 'h264');
|
||||
});
|
||||
|
||||
it('honours excludeFields', () => {
|
||||
const flat = buildRecordFields(
|
||||
{hashId: MIDHASH, cids: [MIDHASH], title: 'x', processingStatus: 'processing'},
|
||||
['processingStatus']
|
||||
);
|
||||
assert.equal(flat['processingStatus'], undefined);
|
||||
assert.equal(flat['title'], 'x');
|
||||
});
|
||||
|
||||
it('tolerates a record with no cids at all', () => {
|
||||
const flat = buildRecordFields({title: 'x'});
|
||||
assert.equal(flat['title'], 'x');
|
||||
assert.ok(!Object.keys(flat).some(k => k.startsWith('cids/')));
|
||||
});
|
||||
});
|
||||
@@ -22,6 +22,63 @@ import type { KeyValuePair } from './IKVClient.js';
|
||||
* // { key: "/file/abc123/video/codec", value: "h265" }
|
||||
* // ]
|
||||
*/
|
||||
/**
|
||||
* Fields that never go onto a record under their own name.
|
||||
*
|
||||
* `cids` is an **array of bare CIDs** and must become the key-set
|
||||
* `cids/<cid> = "true"`. If it fell through the generic flattener below it would
|
||||
* emit `cids/0`, `cids/1`, … — array indices, not CIDs — which meta-core's
|
||||
* reverse-index hook would not recognise, silently leaving the record
|
||||
* unresolvable by any of its CIDs. `buildRecordFields` handles it explicitly.
|
||||
*
|
||||
* `hashId` is the record's own address (it *is* the `/file/{hashId}` key), so
|
||||
* storing it as a property inside itself is redundant.
|
||||
*/
|
||||
const STRUCTURAL_FIELDS = ['cids', 'hashId'];
|
||||
|
||||
/**
|
||||
* Turn an in-memory `FileMetadata` into the flat `field → value` map that a
|
||||
* record is made of.
|
||||
*
|
||||
* **This is the single write-shape boundary.** Both writers go through it — the
|
||||
* meta-core HTTP writer and the direct-Redis fallback — because they used not
|
||||
* to: only the HTTP path emitted the `cids/` key-set, so a meta-sort configured
|
||||
* with `redisUrl` and no `metaCoreApiUrl` wrote raw `cid_midhash256` fields that
|
||||
* meta-core never indexed. Records looked fine and could not be found by their
|
||||
* own midhash. One function, one shape, no second path to forget.
|
||||
*
|
||||
* Returns field names relative to the record root (no `/file/{hashId}` prefix).
|
||||
*/
|
||||
export function buildRecordFields(
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Record<string, string> {
|
||||
const prefix = 'r'; // throwaway root; stripped below
|
||||
const pairs = flattenMetadata(metadata, prefix, [...excludeFields, ...STRUCTURAL_FIELDS]);
|
||||
|
||||
const flat: Record<string, string> = {};
|
||||
for (const pair of pairs) {
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for the trailing '/'
|
||||
if (field) {
|
||||
flat[field] = pair.value;
|
||||
}
|
||||
}
|
||||
|
||||
// Sibling CIDs are a bare-CID key-set, never per-algorithm named fields.
|
||||
// The midhash is just the member whose multicodec is 0x1000 — there is no
|
||||
// `cid_midhash256`. See METADATA_KEYS.md §2/§14.13.
|
||||
const cids: unknown = metadata?.cids;
|
||||
if (Array.isArray(cids)) {
|
||||
for (const cid of cids) {
|
||||
if (typeof cid === 'string' && cid) {
|
||||
flat[`cids/${cid}`] = 'true';
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return flat;
|
||||
}
|
||||
|
||||
export function flattenMetadata(
|
||||
metadata: any,
|
||||
prefix: string,
|
||||
|
||||
@@ -16,11 +16,12 @@ import * as IORedis from 'ioredis';
|
||||
import * as os from 'os';
|
||||
import type { IKVClient, KeyValuePair } from './IKVClient.js';
|
||||
import {
|
||||
flattenMetadata,
|
||||
buildRecordFields,
|
||||
reconstructMetadata,
|
||||
buildFilePrefix,
|
||||
buildPropertyKey,
|
||||
} from './MetadataUtils.js';
|
||||
import { MetaCoreApiWriter } from './MetaCoreApiWriter.js';
|
||||
|
||||
// ESM/CJS interop for ioredis
|
||||
const Redis = (IORedis as any).default ?? IORedis;
|
||||
@@ -50,6 +51,18 @@ interface RedisClientConfig {
|
||||
url: string;
|
||||
timeout?: number;
|
||||
prefix?: string;
|
||||
|
||||
/**
|
||||
* meta-core HTTP API base URL. When set, write methods
|
||||
* (setMetadataFlat / setMetadataProperty / deleteMetadataFlat) route
|
||||
* through the /meta/{hash} HTTP family instead of writing to Redis
|
||||
* directly. Reads and stream consumption still use the Redis connection.
|
||||
*
|
||||
* Wiring this is the meta-sort half of the api-mediated-access lockdown
|
||||
* (see meta-core docs/api-mediated-access.md, PR B). Leaving it unset
|
||||
* preserves the legacy direct-write path for tests / one-off scripts.
|
||||
*/
|
||||
metaCoreApiUrl?: string;
|
||||
}
|
||||
|
||||
export class RedisKVClient implements IKVClient {
|
||||
@@ -58,6 +71,7 @@ export class RedisKVClient implements IKVClient {
|
||||
private timeout: number;
|
||||
private prefix: string;
|
||||
private isConnected: boolean = false;
|
||||
private writer: MetaCoreApiWriter | null = null;
|
||||
|
||||
// Stream consumer state
|
||||
private streamConsumerRunning = false;
|
||||
@@ -68,9 +82,32 @@ export class RedisKVClient implements IKVClient {
|
||||
this.timeout = config.timeout || 30000;
|
||||
this.prefix = config.prefix || '';
|
||||
|
||||
if (config.metaCoreApiUrl) {
|
||||
this.writer = new MetaCoreApiWriter({
|
||||
apiUrl: config.metaCoreApiUrl,
|
||||
timeoutMs: this.timeout,
|
||||
});
|
||||
console.log(`[Redis] Writes routed through meta-core API at ${config.metaCoreApiUrl}`);
|
||||
}
|
||||
|
||||
// Unique consumer name for this instance
|
||||
this.consumerName = `${os.hostname()}-${process.pid}`;
|
||||
|
||||
// HTTP-only mode: no Redis URL AND a writer is configured. Create a
|
||||
// disconnected stub Redis so accidental fallthrough into a Redis
|
||||
// method throws immediately instead of silently connecting to
|
||||
// localhost.
|
||||
if (!config.url && this.writer) {
|
||||
// ioredis requires a config object; lazyConnect prevents the
|
||||
// automatic connect. Any caller that tries to use the Redis
|
||||
// path (e.g. unmigrated stream code) will fail loudly rather
|
||||
// than silently misbehaving.
|
||||
this.redis = new Redis({ host: '127.0.0.1', port: 0, lazyConnect: true, enableOfflineQueue: false });
|
||||
this.isConnected = false;
|
||||
console.log('[Redis] HTTP-only mode — no direct Redis connection');
|
||||
return;
|
||||
}
|
||||
|
||||
// Parse Redis URL and create client
|
||||
this.redis = new Redis(config.url, {
|
||||
commandTimeout: this.timeout,
|
||||
@@ -102,10 +139,22 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
/**
|
||||
* Connect to Redis (call before using other methods)
|
||||
* Connect to Redis (call before using other methods).
|
||||
*
|
||||
* In HTTP-only mode (no Redis URL + writer configured), connect is a
|
||||
* no-op — there's no underlying socket to open.
|
||||
*/
|
||||
async connect(): Promise<void> {
|
||||
if (this.isConnected) return;
|
||||
if (this.writer && !this.redis.options.host) {
|
||||
// HTTP-only mode stub — nothing to connect.
|
||||
return;
|
||||
}
|
||||
// Skip explicit connect when host is 127.0.0.1 with port 0 (HTTP-only stub).
|
||||
const opts = this.redis.options as { host?: string; port?: number };
|
||||
if (opts && opts.port === 0) {
|
||||
return;
|
||||
}
|
||||
await this.redis.connect();
|
||||
}
|
||||
|
||||
@@ -163,6 +212,11 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
if (this.writer) {
|
||||
// Probe meta-core /health rather than Redis PING; that's the
|
||||
// surface the application actually depends on after the lockdown.
|
||||
return this.writer.health();
|
||||
}
|
||||
try {
|
||||
const result = await this.redis.ping();
|
||||
return result === 'PONG';
|
||||
@@ -271,24 +325,30 @@ export class RedisKVClient implements IKVClient {
|
||||
metadata: any,
|
||||
excludeFields: string[] = []
|
||||
): Promise<void> {
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs = flattenMetadata(metadata, prefix, excludeFields);
|
||||
if (this.writer) {
|
||||
// PATCH /meta/{hash} — meta-core handles the flat-key fanout and
|
||||
// the file:__index__ membership atomically server-side.
|
||||
return this.writer.setMetadataFlat(hashId, metadata, excludeFields);
|
||||
}
|
||||
|
||||
if (pairs.length === 0) return;
|
||||
// Direct-Redis fallback (no metaCoreApiUrl configured — see KVManager).
|
||||
//
|
||||
// This path used to flatten the metadata itself, which meant it skipped
|
||||
// the `cid_* → cids/` transform that only the HTTP writer applied: it
|
||||
// wrote a literal `file:<hash>/cid_midhash256` key. meta-core's
|
||||
// reverse-index hook only registers aliases for fields named `cids/<cid>`,
|
||||
// so those records were stored but never indexed — unresolvable by their
|
||||
// own midhash, with nothing logged. Both paths now build the record
|
||||
// through the same `buildRecordFields`, so the key-set cannot be skipped.
|
||||
const flat = buildRecordFields(metadata, excludeFields);
|
||||
if (Object.keys(flat).length === 0) return;
|
||||
|
||||
// Use pipeline for efficient batch SET operations
|
||||
const pipeline = this.redis.pipeline();
|
||||
|
||||
for (const pair of pairs) {
|
||||
// Extract field name by removing the prefix
|
||||
const field = pair.key.slice(prefix.length + 1); // +1 for trailing /
|
||||
if (field) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
const redisKey = this.buildKey(`file:${hashId}/${field}`);
|
||||
pipeline.set(redisKey, pair.value);
|
||||
}
|
||||
for (const [field, value] of Object.entries(flat)) {
|
||||
// Key format: file:{hashId}/{property}
|
||||
pipeline.set(this.buildKey(`file:${hashId}/${field}`), value);
|
||||
}
|
||||
|
||||
await pipeline.exec();
|
||||
|
||||
// Maintain index set for getAllHashIds
|
||||
@@ -296,6 +356,20 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getMetadataFlat(hashId: string): Promise<any | null> {
|
||||
if (this.writer) {
|
||||
// GET /meta/{hash} via HTTP — returns the flat map directly,
|
||||
// so we don't need a Redis SCAN/MGET round trip. Reconstruct
|
||||
// the nested structure consumers expect.
|
||||
const flat = await this.writer.getMetadataFlat(hashId);
|
||||
if (!flat) return null;
|
||||
const prefix = buildFilePrefix(hashId);
|
||||
const pairs: KeyValuePair[] = Object.entries(flat).map(([field, value]) => ({
|
||||
key: `${prefix}/${field}`,
|
||||
value: value as string,
|
||||
}));
|
||||
return reconstructMetadata(pairs, prefix);
|
||||
}
|
||||
|
||||
const keyPrefix = this.buildKey(`file:${hashId}/`);
|
||||
|
||||
// Scan for all keys with this prefix
|
||||
@@ -329,6 +403,13 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getMetadata(hashId: string, propertyPath: string): Promise<any | null> {
|
||||
if (this.writer) {
|
||||
// GET /meta/{hash}/{prop} via HTTP; meta-core returns text/plain
|
||||
// and the writer mirrors the JSON-or-raw fallback the Redis path
|
||||
// uses below.
|
||||
return this.writer.getMetadata(hashId, propertyPath);
|
||||
}
|
||||
|
||||
const key = this.buildKey(`file:${hashId}/${propertyPath}`);
|
||||
const value = await this.redis.get(key);
|
||||
|
||||
@@ -346,6 +427,10 @@ export class RedisKVClient implements IKVClient {
|
||||
* Writes to flat key: file:{hashId}/{property}
|
||||
*/
|
||||
async setMetadataProperty(hashId: string, property: string, value: string): Promise<void> {
|
||||
if (this.writer) {
|
||||
return this.writer.setMetadataProperty(hashId, property, value);
|
||||
}
|
||||
|
||||
const key = this.buildKey(`file:${hashId}/${property}`);
|
||||
await this.redis.set(key, value);
|
||||
|
||||
@@ -354,6 +439,10 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async deleteMetadataFlat(hashId: string): Promise<number> {
|
||||
if (this.writer) {
|
||||
return this.writer.deleteMetadataFlat(hashId);
|
||||
}
|
||||
|
||||
const keyPrefix = this.buildKey(`file:${hashId}/`);
|
||||
|
||||
// Scan and delete all keys with this prefix
|
||||
@@ -378,6 +467,9 @@ export class RedisKVClient implements IKVClient {
|
||||
}
|
||||
|
||||
async getAllHashIds(): Promise<string[]> {
|
||||
if (this.writer) {
|
||||
return this.writer.getAllHashIds();
|
||||
}
|
||||
const indexKey = this.buildKey('file:__index__');
|
||||
return await this.redis.smembers(indexKey);
|
||||
}
|
||||
|
||||
@@ -106,7 +106,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
// Map StreamingPipeline queues to the 3-queue architecture UI expects
|
||||
// PQueue: .pending = currently running, .size = waiting in queue
|
||||
// API expects: .pending = running, .size = waiting
|
||||
const result = {
|
||||
const result: any = {
|
||||
preProcessQueue: {
|
||||
// validation = pre-process (quick extension checks)
|
||||
size: queues.validation.size, // waiting in queue
|
||||
@@ -130,6 +130,15 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
// File-level counts (what users care about)
|
||||
files: queues.files
|
||||
};
|
||||
|
||||
// Layer in the per-file cache-slot view from the ContainerPluginScheduler
|
||||
// when available — this is the metric that reflects WebDAV/cache load
|
||||
// (number of distinct files concurrently being touched), not just task
|
||||
// throughput. Older deployments without container plugins simply omit it.
|
||||
const containerStatus = this.taskScheduler?.getContainerQueueStatus?.() ?? null;
|
||||
if (containerStatus?.cacheSlots) {
|
||||
result.cacheSlots = containerStatus.cacheSlots;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
@@ -273,8 +282,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
);
|
||||
|
||||
let midHash256: string;
|
||||
if (indexLine?.cid_midhash256) {
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
if (indexLine?.midhash256) {
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
const computeStart = performance.now();
|
||||
@@ -288,7 +297,7 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
return;
|
||||
}
|
||||
midHash256 = hash;
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordHashComputation('midhash256', Math.ceil(performance.now() - computeStart));
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
@@ -296,13 +305,20 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
filePath,
|
||||
fileStats.size,
|
||||
fileStats.mtime.toISOString(),
|
||||
{ cid_midhash256: midHash256 }
|
||||
{ midhash256: midHash256 }
|
||||
);
|
||||
}
|
||||
|
||||
// STEP 2: Initialize metadata with midhash256
|
||||
// STEP 2: Initialize metadata with midhash256.
|
||||
//
|
||||
// The midhash plays two roles and now says so: it is the record
|
||||
// address (`hashId`, the `/file/{hashId}` key) *and* one member of
|
||||
// the record's bare-CID key-set (`cids`). It used to be a single
|
||||
// `cid_midhash256` property, which conflated the two and leaked a
|
||||
// deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: any = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
filePath,
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
@@ -432,8 +448,8 @@ export class WatchedFileProcessor implements FileProcessorInterface {
|
||||
const metadata = db.get(filePath);
|
||||
|
||||
// Collect hashId before deletion (for Stremio notification)
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
removedHashIds.push(metadata.cid_midhash256);
|
||||
if (metadata && metadata.hashId) {
|
||||
removedHashIds.push(metadata.hashId);
|
||||
}
|
||||
|
||||
db.delete(filePath);
|
||||
|
||||
@@ -191,9 +191,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
let midHash256: string | undefined;
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 (< 1 second, but requires network I/O on network drives)
|
||||
@@ -203,16 +203,22 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -296,19 +302,16 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
metadata.processingStatus = 'complete';
|
||||
this.database.set(filePath, metadata);
|
||||
|
||||
// STEP 5: Update metadata in KV with full hashes
|
||||
// STEP 5: Update metadata in KV.
|
||||
// Full hashes (sha2-256, sha1, …) are computed by the full-hash
|
||||
// container plugin and written to meta-core directly as cids/<cid>
|
||||
// members — they are not held in this in-memory metadata. The
|
||||
// midhash256 is the primary id; on a (rare) midhash collision the
|
||||
// file simply shares the record address until the plugin's
|
||||
// distinct cids/* members disambiguate it.
|
||||
if (this.kvClient) {
|
||||
try {
|
||||
// Choose hash ID: Use SHA-256 if collision detected, otherwise midhash256
|
||||
let hashId: string;
|
||||
if (collisionDetected && metadata['cid_sha2-256']) {
|
||||
// Collision: Use full SHA-256 as primary ID
|
||||
hashId = metadata['cid_sha2-256'];
|
||||
console.log(` Using SHA-256 as primary ID for collision: ${hashId}`);
|
||||
} else {
|
||||
// Normal: Use midhash256 as primary ID
|
||||
hashId = midHash256;
|
||||
}
|
||||
const hashId = midHash256;
|
||||
|
||||
if (hashId) {
|
||||
// Prepare metadata with filePath added
|
||||
@@ -412,9 +415,9 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Check cache or compute locally
|
||||
const indexLine = this.indexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
|
||||
if (indexLine && indexLine.cid_midhash256) {
|
||||
if (indexLine && indexLine.midhash256) {
|
||||
// Cache hit! Use cached midhash256 (no network I/O needed)
|
||||
midHash256 = indexLine.cid_midhash256;
|
||||
midHash256 = indexLine.midhash256;
|
||||
performanceMetrics.recordCacheHit('midhash256');
|
||||
} else {
|
||||
// Cache miss - compute midhash256 locally (requires file access)
|
||||
@@ -426,17 +429,23 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
const computeTime = Math.ceil(performance.now() - computeStart);
|
||||
|
||||
// Record midhash256 computation time (excludes cache hits)
|
||||
performanceMetrics.recordHashComputation('cid_midhash256', computeTime);
|
||||
performanceMetrics.recordHashComputation('midhash256', computeTime);
|
||||
performanceMetrics.recordCacheMiss('midhash256');
|
||||
|
||||
// Store in cache for future lookups
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { cid_midhash256: midHash256 });
|
||||
this.indexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), { midhash256: midHash256 });
|
||||
}
|
||||
}
|
||||
|
||||
// Initialize metadata with midhash256
|
||||
// Initialize metadata with midhash256.
|
||||
//
|
||||
// Two roles, now named separately: the record address (`hashId`, the
|
||||
// `/file/{hashId}` key) and a member of the record's bare-CID key-set
|
||||
// (`cids`). The old single `cid_midhash256` property conflated them and
|
||||
// leaked a deprecated field name onto the record. See METADATA_KEYS.md §14.13.
|
||||
const metadata: FileMetadata = {
|
||||
cid_midhash256: midHash256,
|
||||
hashId: midHash256,
|
||||
cids: [midHash256],
|
||||
processingStatus: 'processing'
|
||||
};
|
||||
|
||||
@@ -527,7 +536,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
throw new Error('Metadata not found - light processing may have failed');
|
||||
}
|
||||
|
||||
const midHash256 = metadata.cid_midhash256;
|
||||
const midHash256 = metadata.hashId;
|
||||
if (!midHash256) {
|
||||
throw new Error('midhash256 not found in metadata');
|
||||
}
|
||||
@@ -625,7 +634,7 @@ export class FileProcessorPiscina implements FileAnalyzerInterface{
|
||||
// Delete from KV if configured (using nested key architecture)
|
||||
if (this.kvClient && metadata) {
|
||||
try {
|
||||
const hashId = metadata.cid_midhash256 || metadata['cid_sha2-256'] || metadata.cid_sha1 || metadata.cid_md5;
|
||||
const hashId = metadata.hashId;
|
||||
|
||||
// Delete all properties for this file using nested key architecture
|
||||
if (hashId) {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerFile} from "@metazla/meta-hash";
|
||||
import {CidAlgorithm, HashComputerFile, MultiHashData, pickCidByAlgorithm} from "@metazla/meta-hash";
|
||||
import {HashIndexManager} from "@metazla/meta-hash";
|
||||
import {config} from "../../config/EnvConfig.js";
|
||||
import {targetHash, targetHashForIndex} from "../../config/TargetHash.js";
|
||||
@@ -16,19 +16,34 @@ class SimpleFileToCid{
|
||||
if (!stats || !stats.exists) {
|
||||
return null;
|
||||
}
|
||||
let indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
return indexLine[CID_ALGORITHM_NAMES.sha256];
|
||||
}else {
|
||||
let hashs = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashs);
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), hashs);
|
||||
return hashs[CID_ALGORITHM_NAMES.sha256];
|
||||
const indexLine = globalHashIndex.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if (indexLine) {
|
||||
// The on-disk index keeps per-algorithm columns, keyed by bare
|
||||
// algorithm name (not the removed `cid_<algo>` record fields).
|
||||
return indexLine['sha256'] ?? null;
|
||||
}
|
||||
|
||||
// The computer returns bare CIDs; the algorithm is recovered from the
|
||||
// multicodec, never from a field name.
|
||||
const hashes: MultiHashData = {};
|
||||
await globalHashComputer.computeMissingHash(filePath, hashes);
|
||||
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of targetHash) {
|
||||
const cid = pickCidByAlgorithm(hashes.cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
globalHashIndex.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
|
||||
return pickCidByAlgorithm(hashes.cids, 'sha256') ?? null;
|
||||
} catch (e) {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
export const simpleFileToCid = new SimpleFileToCid();
|
||||
|
||||
@@ -166,14 +166,14 @@ export class StreamingPipeline {
|
||||
|
||||
// After light processing, dispatch to plugins and queue background tasks
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
if (metadata && metadata.cid_midhash256) {
|
||||
if (metadata && metadata.hashId) {
|
||||
// Notify Stremio addon to add video
|
||||
this.notifyStremioAdd(metadata.cid_midhash256).catch(err => {
|
||||
this.notifyStremioAdd(metadata.hashId).catch(err => {
|
||||
// Silent fail - Stremio addon may not be running
|
||||
});
|
||||
|
||||
// Dispatch tasks to container plugins (fire and forget)
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.cid_midhash256, metadata);
|
||||
this.dispatchContainerPluginTasks(filePath, metadata.hashId, metadata);
|
||||
|
||||
// Stage 3: Queue for background processing (SHA-256 computation)
|
||||
this.backgroundQueue.add(() => this.processHashPhase(filePath))
|
||||
@@ -246,7 +246,7 @@ export class StreamingPipeline {
|
||||
// Otherwise, wait for 'file:complete' event from ContainerPluginScheduler.
|
||||
if (!this.config.containerPluginScheduler) {
|
||||
const metadata = this.config.fileProcessor.getDatabase().get(filePath);
|
||||
const hashId = metadata?.cid_midhash256;
|
||||
const hashId = metadata?.hashId;
|
||||
let virtualPath: string | undefined;
|
||||
try {
|
||||
virtualPath = metadata ? this.config.metaDataToFolderStruct.renamingRule(metadata as any, filePath) : undefined;
|
||||
|
||||
@@ -584,6 +584,16 @@ export class TaskScheduler extends EventEmitter {
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Pass-through to ContainerPluginScheduler.getQueueStatus so callers like
|
||||
* WatchedFileProcessor can surface the file-cache-slot view (capacity,
|
||||
* holders, current plugin per file) without reaching past the scheduler
|
||||
* boundary themselves.
|
||||
*/
|
||||
getContainerQueueStatus(): ReturnType<ContainerPluginScheduler['getQueueStatus']> | null {
|
||||
return this.containerPluginScheduler?.getQueueStatus() ?? null;
|
||||
}
|
||||
|
||||
// =========================================================================
|
||||
// Shutdown
|
||||
// =========================================================================
|
||||
|
||||
@@ -102,13 +102,18 @@ export interface ExtendedHashMeta extends HashMeta {
|
||||
*/
|
||||
export interface IndexManager {
|
||||
/**
|
||||
* Get cached CID for a file based on path, size, and mtime
|
||||
* Get cached CID for a file based on path, size, and mtime.
|
||||
*
|
||||
* Structural mirror of meta-hash's `HashIndexManager`. Its columns are keyed
|
||||
* by bare **algorithm** (`midhash256`, `sha256`), not by the removed
|
||||
* `cid_<algo>` record-field names — the local index and the record shape are
|
||||
* deliberately separate concerns now. See METADATA_KEYS.md §14.13.
|
||||
*/
|
||||
getCidForFile(
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string
|
||||
): { cid_midhash256?: string } | undefined;
|
||||
): { midhash256?: string } | undefined;
|
||||
|
||||
/**
|
||||
* Add a file CID to the cache
|
||||
@@ -117,7 +122,7 @@ export interface IndexManager {
|
||||
path: string,
|
||||
size: number,
|
||||
mtime: string,
|
||||
cid: { cid_midhash256: string }
|
||||
cid: { midhash256: string }
|
||||
): void;
|
||||
}
|
||||
|
||||
|
||||
@@ -18,4 +18,28 @@ export type ProcessingStatus = 'pending' | 'processing' | 'complete' | 'error';
|
||||
export interface FileMetadata extends HashMeta {
|
||||
/** Current status of file processing */
|
||||
processingStatus?: ProcessingStatus;
|
||||
|
||||
/**
|
||||
* The record address: the file's midhash256 CID, computed in the light phase
|
||||
* and used as the `hashId` throughout meta-sort and as the key of the
|
||||
* `/file/{hashId}` record.
|
||||
*
|
||||
* This replaces the old `cid_midhash256` field. The rename is the point: that
|
||||
* name was a *metadata property* name, and it leaked — a `cid_*` field written
|
||||
* to a record is stored but never reverse-indexed by meta-core, leaving the
|
||||
* record unresolvable by its own CID. meta-core now rejects such fields with a
|
||||
* 400. See METADATA_KEYS.md §1/§14.13.
|
||||
*/
|
||||
hashId?: string;
|
||||
|
||||
/**
|
||||
* Every digest known for this file, as **bare CIDv1 strings** — the midhash
|
||||
* included. Persisted as the key-set `cids/<cid> = "true"`, which is the only
|
||||
* shape meta-core indexes.
|
||||
*
|
||||
* A CIDv1 is self-describing, so a specific digest is recovered by decoding
|
||||
* the multicodec (`pickCidByAlgorithm` from `@metazla/meta-hash`), never by a
|
||||
* field name.
|
||||
*/
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
@@ -365,27 +365,115 @@ function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): b
|
||||
return false;
|
||||
}
|
||||
|
||||
// Human label per multihash code / CID codec. Source of truth for codes:
|
||||
// meta-hash MultiHashData + METADATA_KEYS.md §2.
|
||||
const CID_LABELS: Record<number, string> = {
|
||||
0x12: 'SHA-256',
|
||||
0x16: 'SHA3-256',
|
||||
0x15: 'SHA3-384',
|
||||
0x11: 'SHA-1',
|
||||
0xd5: 'MD5',
|
||||
0x132: 'CRC32',
|
||||
0x1000: 'MidHash256',
|
||||
0xb702: 'BT Pieces Root',
|
||||
0x70: 'IPFS (dag-pb)',
|
||||
0x10b7: 'BitTorrent v2',
|
||||
0x1001: 'BitTorrent v1 (file)',
|
||||
0x1002: 'BitTorrent v2 (file)',
|
||||
0x1005: 'Usenet release',
|
||||
0x1006: 'URL',
|
||||
};
|
||||
|
||||
/**
|
||||
* Extract all hash values (cid_*) from metadata
|
||||
* Display order for a record's CIDs. This mirrors the canonical rank ladder
|
||||
* (`packages/meta-core/internal/cid/rank.go`, the two `cid_rank.rs`, and the
|
||||
* `/cid-rank-vectors.json` golden fixture) — same ordering, scaled down.
|
||||
*
|
||||
* It used to be its own invented ladder (5/4/3/2) with no dag-pb entry at all,
|
||||
* which meant an IPFS root fell through to 0 and sorted *below* MD5. Display-only,
|
||||
* so it never broke anything — but it was a seventh, silently divergent copy of a
|
||||
* rule that already has too many. Keep it in step: a code with no entry sorts last.
|
||||
*/
|
||||
const RANK_BY_LABEL: Record<string, number> = {
|
||||
'IPFS (dag-pb)': 6,
|
||||
'SHA-256': 5, 'SHA3-256': 5, 'SHA3-384': 5,
|
||||
'BitTorrent v2': 4, 'BitTorrent v1 (file)': 4, 'BitTorrent v2 (file)': 4,
|
||||
MidHash256: 3,
|
||||
'Usenet release': 2, URL: 2,
|
||||
'SHA-1': 1, MD5: 1, CRC32: 1,
|
||||
};
|
||||
|
||||
const B32 = 'abcdefghijklmnopqrstuvwxyz234567';
|
||||
|
||||
function decodeBase32(s: string): Uint8Array | null {
|
||||
let bits = 0, value = 0;
|
||||
const out: number[] = [];
|
||||
for (const ch of s) {
|
||||
const idx = B32.indexOf(ch);
|
||||
if (idx < 0) return null;
|
||||
value = (value << 5) | idx;
|
||||
bits += 5;
|
||||
if (bits >= 8) {
|
||||
bits -= 8;
|
||||
out.push((value >> bits) & 0xff);
|
||||
}
|
||||
}
|
||||
return new Uint8Array(out);
|
||||
}
|
||||
|
||||
function readUvarint(buf: Uint8Array, pos: number): [number, number] {
|
||||
let result = 0, shift = 0, p = pos;
|
||||
while (p < buf.length) {
|
||||
const b = buf[p++];
|
||||
result |= (b & 0x7f) << shift;
|
||||
if ((b & 0x80) === 0) return [result >>> 0, p];
|
||||
shift += 7;
|
||||
}
|
||||
return [0, pos];
|
||||
}
|
||||
|
||||
/**
|
||||
* Decode a bare multibase-base32 CIDv1 into [codec, multihashCode], or null.
|
||||
* The algorithm is recovered from these — sibling CIDs are stored as a
|
||||
* key-set (cids/<cid>) with no per-algorithm field name.
|
||||
*/
|
||||
function decodeCid(cid: string): { codec: number; mh: number } | null {
|
||||
if (!cid.startsWith('b')) return null;
|
||||
const bytes = decodeBase32(cid.slice(1));
|
||||
if (!bytes || bytes.length < 3) return null;
|
||||
let v: number, pos = 0;
|
||||
[v, pos] = readUvarint(bytes, pos);
|
||||
if (v !== 1) return null;
|
||||
let codec: number;
|
||||
[codec, pos] = readUvarint(bytes, pos);
|
||||
let mh: number;
|
||||
[mh, pos] = readUvarint(bytes, pos);
|
||||
return { codec, mh };
|
||||
}
|
||||
|
||||
function cidLabel(cid: string): string {
|
||||
const d = decodeCid(cid);
|
||||
if (!d) return 'CID';
|
||||
// Per-file BitTorrent codecs are carried on the CID codec, not the mh code.
|
||||
return CID_LABELS[d.codec] ?? CID_LABELS[d.mh] ?? 'CID';
|
||||
}
|
||||
|
||||
/**
|
||||
* Extract all sibling CIDs from the record's bare-CID key-set (cids/<cid>),
|
||||
* labelling each by its multicodec. Replaces the old per-algorithm cid_*
|
||||
* fields. See METADATA_KEYS.md §2/§14.13.
|
||||
*/
|
||||
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
|
||||
const hashOrder = [
|
||||
{ key: 'cid_sha2-256', label: 'SHA-256' },
|
||||
{ key: 'cid_sha3-256', label: 'SHA3-256' },
|
||||
{ key: 'cid_sha3-384', label: 'SHA3-384' },
|
||||
{ key: 'cid_sha1', label: 'SHA-1' },
|
||||
{ key: 'cid_md5', label: 'MD5' },
|
||||
{ key: 'cid_crc32', label: 'CRC32' },
|
||||
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
|
||||
];
|
||||
|
||||
return hashOrder
|
||||
.filter(({ key }) => (metadata as any)[key])
|
||||
.map(({ key, label }) => ({
|
||||
key,
|
||||
label,
|
||||
value: (metadata as any)[key],
|
||||
}));
|
||||
const out: { key: string; label: string; value: string }[] = [];
|
||||
for (const key of Object.keys(metadata as any)) {
|
||||
if (!key.startsWith('cids/')) continue;
|
||||
const cid = key.slice('cids/'.length);
|
||||
if (!cid) continue;
|
||||
out.push({ key, label: cidLabel(cid), value: cid });
|
||||
}
|
||||
// Most useful first (sha/ipfs > btih > midhash > weak digests), then by label.
|
||||
out.sort((a, b) => (RANK_BY_LABEL[b.label] ?? 0) - (RANK_BY_LABEL[a.label] ?? 0) || a.label.localeCompare(b.label));
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -41,13 +41,10 @@ export interface FileMetadata {
|
||||
collectionnumber?: number;
|
||||
criticrating?: number;
|
||||
|
||||
// Hashes
|
||||
cid_crc32?: string;
|
||||
cid_md5?: string;
|
||||
cid_sha1?: string;
|
||||
'cid_sha2-256'?: string;
|
||||
'cid_sha3-256'?: string;
|
||||
cid_sha3_384?: string;
|
||||
// Sibling CIDs are the bare-CID key-set `cids/<cid> = "true"` (dynamic
|
||||
// keys), not per-algorithm fields. Accessed via prefix-scan; the algorithm
|
||||
// is recovered from each CID's multicodec. See METADATA_KEYS.md §2/§14.13.
|
||||
[cidsKey: `cids/${string}`]: string | undefined;
|
||||
|
||||
// Video info
|
||||
video?: {
|
||||
|
||||
@@ -5,7 +5,8 @@ import {
|
||||
RedisStats,
|
||||
QueueItem,
|
||||
HashTimingStats,
|
||||
FailedFile
|
||||
FailedFile,
|
||||
CacheSlotPhase
|
||||
} from '../types';
|
||||
import {
|
||||
formatMs,
|
||||
@@ -14,6 +15,43 @@ import {
|
||||
getFilename
|
||||
} from '../utils/format';
|
||||
|
||||
/**
|
||||
* Renders a single phase of the file-cache-slot view: capacity bar, count of
|
||||
* waiting files, and a list of currently-held files showing which plugin
|
||||
* each is currently waiting on. The plugin column is the long-pole — it tells
|
||||
* operators which plugin is the per-file bottleneck.
|
||||
*/
|
||||
function CacheSlotPanel({ label, phase }: { label: string; phase: CacheSlotPhase }) {
|
||||
const fillPct = Math.min(100, (phase.inUse / Math.max(1, phase.capacity)) * 100);
|
||||
return (
|
||||
<div style={{ marginTop: '0.75rem' }}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'baseline' }}>
|
||||
<strong>{label}</strong>
|
||||
<span style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
|
||||
{phase.inUse}/{phase.capacity} files
|
||||
{phase.waiting > 0 ? ` • ${phase.waiting} waiting for slot` : ''}
|
||||
</span>
|
||||
</div>
|
||||
<div style={{ height: '6px', background: '#0f3460', borderRadius: '3px', overflow: 'hidden', margin: '0.25rem 0 0.5rem' }}>
|
||||
<div style={{ width: `${fillPct}%`, height: '100%', background: '#4ade80', transition: 'width 0.5s linear' }} />
|
||||
</div>
|
||||
{phase.holders.length > 0 && (
|
||||
<div style={{ fontSize: '0.78rem', color: '#cbd5e1', maxHeight: '160px', overflow: 'auto' }}>
|
||||
{phase.holders.map((h) => (
|
||||
<div key={h.fileHash} style={{ display: 'flex', gap: '0.5rem', padding: '2px 0' }}>
|
||||
<span style={{ color: '#facc15', minWidth: '8rem' }}>{h.currentPlugin ?? '—'}</span>
|
||||
<span style={{ color: '#94a3b8', minWidth: '4rem' }}>×{h.tasksRemaining}</span>
|
||||
<span style={{ flex: 1, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }} title={h.filePath ?? h.fileHash}>
|
||||
{h.filePath ? getFilename(h.filePath) : h.fileHash}
|
||||
</span>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function Monitor() {
|
||||
const [status, setStatus] = useState<ProcessingStatus | null>(null);
|
||||
const [metrics, setMetrics] = useState<Metrics | null>(null);
|
||||
@@ -107,31 +145,50 @@ function Monitor() {
|
||||
// Get queue statistics for display
|
||||
// StreamingPipeline stages: validation → lightProcessing → hashProcessing
|
||||
// UI mapping: Validation → Fast Queue → Background Queue
|
||||
//
|
||||
// Fast/Background "Running/Max" represents file-cache slot occupancy
|
||||
// (cacheSlots.inUse / capacity), not raw task throughput. This matches the
|
||||
// operator mental model: "how many files are currently tying up the WebDAV
|
||||
// cache?" The task-callback count was misleading because dependency-wait
|
||||
// and per-plugin HTTP queueing made it under-report by ~5×.
|
||||
const getQueueStats = () => {
|
||||
const computed = status?.computed;
|
||||
const cacheSlots = status?.queueStatus?.cacheSlots;
|
||||
const fastQueueConcurrency = status?.fastQueueConcurrency || 16;
|
||||
const backgroundQueueConcurrency = status?.backgroundQueueConcurrency || 8;
|
||||
|
||||
return {
|
||||
// Stage 1: Validation (quick extension checks)
|
||||
// Concurrency: 2x fast queue workers
|
||||
// Stage 1: Validation (quick extension checks). Still task-level
|
||||
// — pre-process is a lightweight midhash256 scan, no cache impact.
|
||||
preProcess: {
|
||||
running: computed?.preProcessRunning || 0,
|
||||
pending: computed?.preProcessPending || 0,
|
||||
max: fastQueueConcurrency * 2,
|
||||
paused: computed?.preProcessPaused || false
|
||||
},
|
||||
// Stage 2: Fast Queue (midhash256 + FFmpeg + plugins)
|
||||
// Concurrency: fastQueueConcurrency workers
|
||||
fast: {
|
||||
// Stage 2: Fast Queue — files holding a light cache slot.
|
||||
// running = files in cache, pending = files waiting for a slot.
|
||||
// paused comes from the cacheSlots payload (true ContainerPluginScheduler
|
||||
// PQueue state) — the legacy `computed.fastQueuePaused` field is sourced
|
||||
// from a different StreamingPipeline queue and is unreliable here.
|
||||
fast: cacheSlots ? {
|
||||
running: cacheSlots.fast.inUse,
|
||||
pending: cacheSlots.fast.waiting,
|
||||
max: cacheSlots.fast.capacity,
|
||||
paused: cacheSlots.fast.paused
|
||||
} : {
|
||||
running: computed?.fastQueueRunning || 0,
|
||||
pending: computed?.fastQueuePending || 0,
|
||||
max: fastQueueConcurrency,
|
||||
paused: computed?.fastQueuePaused || false
|
||||
},
|
||||
// Stage 3: Background Queue (SHA-256 full file hash)
|
||||
// Concurrency: backgroundQueueConcurrency workers
|
||||
background: {
|
||||
// Stage 3: Background Queue — files holding a bg cache slot.
|
||||
background: cacheSlots ? {
|
||||
running: cacheSlots.background.inUse,
|
||||
pending: cacheSlots.background.waiting,
|
||||
max: cacheSlots.background.capacity,
|
||||
paused: cacheSlots.background.paused
|
||||
} : {
|
||||
running: computed?.backgroundQueueRunning || 0,
|
||||
pending: computed?.backgroundQueuePending || 0,
|
||||
max: backgroundQueueConcurrency,
|
||||
@@ -165,9 +222,9 @@ function Monitor() {
|
||||
{showPipelineHelp && (
|
||||
<div className="pipeline-help">
|
||||
<div className="help-item"><span className="help-dot discovered"></span><strong>Discovered:</strong> Files found on disk, waiting to be processed.</div>
|
||||
<div className="help-item"><span className="help-dot fast"></span><strong>Fast Queue:</strong> Files being processed for quick metadata extraction (hash, ffmpeg, etc.).</div>
|
||||
<div className="help-item"><span className="help-dot fast"></span><strong>Fast Queue:</strong> Files holding a light cache slot. The badge shows slots-in-use over the gate capacity (e.g. 16/16). Pending = files waiting for a slot to free.</div>
|
||||
<div className="help-item"><span className="help-dot awaiting-background"></span><strong>Awaiting BG:</strong> Files done with fast processing, queued for slower tasks.</div>
|
||||
<div className="help-item"><span className="help-dot background"></span><strong>Background:</strong> Files running slow tasks like full file hashing or TMDB lookups.</div>
|
||||
<div className="help-item"><span className="help-dot background"></span><strong>Background:</strong> Files holding a background cache slot (full-hash, TMDB, etc.). Same slot-occupancy semantics as Fast Queue.</div>
|
||||
<div className="help-item"><span className="help-dot done"></span><strong>Complete:</strong> Files fully processed and available in the virtual filesystem.</div>
|
||||
<div className="help-item"><span className="help-dot failed"></span><strong>Failed:</strong> Files that encountered errors during processing.</div>
|
||||
</div>
|
||||
@@ -301,6 +358,18 @@ function Monitor() {
|
||||
</div>
|
||||
)}
|
||||
|
||||
{status?.queueStatus?.cacheSlots && (
|
||||
<div className="card cache-slot-section" style={{ marginTop: '1rem' }}>
|
||||
<h2>File cache slots</h2>
|
||||
<p style={{ color: '#888', fontSize: '0.85rem', marginTop: '-0.5rem' }}>
|
||||
Number of distinct files concurrently held by the WebDAV cache per phase. Caps how
|
||||
many file fetches CORN/SMB sees at once, regardless of plugin fan-out.
|
||||
</p>
|
||||
<CacheSlotPanel label="Light (fast queue)" phase={status.queueStatus.cacheSlots.fast} />
|
||||
<CacheSlotPanel label="Background" phase={status.queueStatus.cacheSlots.background} />
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="grid grid-2">
|
||||
{/* Metrics and Stats */}
|
||||
<div className="card">
|
||||
|
||||
@@ -75,6 +75,29 @@ export interface TaskSchedulerQueueStatus {
|
||||
preProcessQueue?: TaskQueueStatus;
|
||||
fastQueue?: TaskQueueStatus;
|
||||
backgroundQueue?: TaskQueueStatus;
|
||||
cacheSlots?: CacheSlotsStatus;
|
||||
}
|
||||
|
||||
export interface CacheSlotHolder {
|
||||
fileHash: string;
|
||||
filePath?: string;
|
||||
currentPlugin?: string;
|
||||
tasksRemaining: number;
|
||||
}
|
||||
|
||||
export interface CacheSlotPhase {
|
||||
capacity: number;
|
||||
inUse: number;
|
||||
waiting: number;
|
||||
/** True when the underlying PQueue is paused — bg phase is paused while
|
||||
* fast has work, so its slots can be "reserved" without active dispatch. */
|
||||
paused: boolean;
|
||||
holders: CacheSlotHolder[];
|
||||
}
|
||||
|
||||
export interface CacheSlotsStatus {
|
||||
fast: CacheSlotPhase;
|
||||
background: CacheSlotPhase;
|
||||
}
|
||||
|
||||
export interface ProcessingStatus {
|
||||
|
||||
@@ -1,9 +1,23 @@
|
||||
/**
|
||||
* The `fileType` vocabulary — see METADATA_KEYS.md §1.
|
||||
*
|
||||
* ⚠ INVARIANT: a value belongs here only if an `(extension, mimeType)` lookup
|
||||
* can decide it on its own. Nothing that requires knowing what the content
|
||||
* *is* goes in this field — that is `contentKind`'s job — and nothing that
|
||||
* decides *which app gets the record* depends on it — that is `domain`'s job
|
||||
* (METADATA_KEYS.md §14.16).
|
||||
*
|
||||
* `card` is the one deliberate exception: it describes no bytes at all and is
|
||||
* therefore never produced by the tables here, only stamped by a card feeder.
|
||||
*/
|
||||
export type SimpleFileType =
|
||||
'audio'
|
||||
| 'video'
|
||||
| 'image'
|
||||
| 'document'
|
||||
| 'archive'
|
||||
| 'subtitle'
|
||||
| 'torrent'
|
||||
| 'card'
|
||||
| 'other'
|
||||
| 'undefined';
|
||||
| 'undefined';
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
/**
|
||||
* `contentKind` → `domain` / `workForm` — the routing tables from
|
||||
* METADATA_KEYS.md §1.
|
||||
*
|
||||
* `domain` answers "which application is this record destined for?" and is the
|
||||
* key every client filters its catalogue on — **one value per app**, no
|
||||
* alternation. `workForm` answers the other question the old `film`/`tv` split
|
||||
* was secretly encoding: is this work complete in itself, or one instalment of
|
||||
* a run? It is cross-domain, so a comic issue is `serial` in `literature`
|
||||
* exactly as an episode is `serial` in `screen`.
|
||||
*
|
||||
* Both are **stored, not derived on read**: that is what lets the `contentKind`
|
||||
* vocabulary stay open, because a peer that has never heard of a kind still
|
||||
* routes the record correctly without having to interpret the token.
|
||||
*
|
||||
* ⚠ This binding is **permanent and one-way**. Add kinds freely; never re-map
|
||||
* an existing kind to a different domain. Because the value is stored, two
|
||||
* peers on different versions that disagree about where a kind belongs produce
|
||||
* a split-brain that redeploying cannot heal — the wrong domain is already
|
||||
* written into records across the mesh. If a kind has to move, mint a new kind.
|
||||
* The `film`+`tv` → `screen` merge (METADATA_KEYS.md §14.17) is the one
|
||||
* deliberate exception, taken once and paid for with a corpus sweep.
|
||||
*
|
||||
* ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain`. Edit both together.
|
||||
*/
|
||||
export type Domain =
|
||||
'screen'
|
||||
| 'music'
|
||||
| 'literature'
|
||||
| 'science';
|
||||
|
||||
/** Is the work complete in itself, or one instalment of a run? */
|
||||
export type WorkForm = 'standalone' | 'serial';
|
||||
|
||||
/**
|
||||
* The one-way binding. A kind that is absent here has no domain, which means
|
||||
* the record is **not independently routable** — correct for sidecars
|
||||
* (subtitles, artwork) and for un-anchored hits, a producer bug for anything
|
||||
* else.
|
||||
*
|
||||
* ⚠ `pack` is deliberately absent: a season pack and an album release are the
|
||||
* same structural thing, so the kind alone cannot tell `screen` from `music`.
|
||||
* The writer stamps `pack`'s domain from its own context — see
|
||||
* {@link domainForContentKind}'s contract.
|
||||
*
|
||||
* ⚠ `podcast` / `podcastEpisode` are absent too: the `spoken` domain was
|
||||
* retired unused, and the change that first *writes* those kinds picks their
|
||||
* domain.
|
||||
*/
|
||||
export const domainByContentKind: { [contentKind: string]: Domain } = {
|
||||
// screen — films and serials alike; the split lives in workFormByContentKind
|
||||
movie: 'screen',
|
||||
series: 'screen',
|
||||
episode: 'screen',
|
||||
// music
|
||||
track: 'music',
|
||||
album: 'music',
|
||||
artist: 'music',
|
||||
musicVideo: 'music',
|
||||
djMix: 'music',
|
||||
liveSet: 'music',
|
||||
// literature
|
||||
book: 'literature',
|
||||
// An audiobook's identity is the book — same author, same ISBN family,
|
||||
// same series as the ebook — so meta-read can group the two editions on
|
||||
// one page. The player does not decide the domain.
|
||||
audiobook: 'literature',
|
||||
comic: 'literature',
|
||||
manga: 'literature',
|
||||
magazine: 'literature',
|
||||
// science
|
||||
paper: 'science',
|
||||
};
|
||||
|
||||
/**
|
||||
* The `workForm` twin of {@link domainByContentKind}.
|
||||
*
|
||||
* ⚠ `pack` is absent for the same reason: a season pack is `serial`, an album
|
||||
* release is `standalone`, and only the writer knows which.
|
||||
*/
|
||||
export const workFormByContentKind: { [contentKind: string]: WorkForm } = {
|
||||
// screen
|
||||
movie: 'standalone',
|
||||
series: 'serial',
|
||||
episode: 'serial',
|
||||
// music — an album is a closed work, not an ongoing run, so every rung of
|
||||
// the music ladder is standalone.
|
||||
track: 'standalone',
|
||||
album: 'standalone',
|
||||
artist: 'standalone',
|
||||
musicVideo: 'standalone',
|
||||
djMix: 'standalone',
|
||||
liveSet: 'standalone',
|
||||
// literature — the split that proves workForm is not a screen-only axis.
|
||||
book: 'standalone',
|
||||
audiobook: 'standalone',
|
||||
comic: 'serial',
|
||||
manga: 'serial',
|
||||
magazine: 'serial',
|
||||
// science
|
||||
paper: 'standalone',
|
||||
};
|
||||
|
||||
/**
|
||||
* Resolve the domain a `contentKind` belongs to.
|
||||
*
|
||||
* Returns `undefined` for `pack` (not derivable from the kind — the writer
|
||||
* supplies it), for sidecar/format kinds that route nowhere, and for any kind
|
||||
* this build has never heard of. A caller that already knows the domain from
|
||||
* its own context must prefer that over this table.
|
||||
*/
|
||||
export function domainForContentKind(contentKind: string | undefined | null): Domain | undefined {
|
||||
if (!contentKind) return undefined;
|
||||
return domainByContentKind[contentKind.trim()];
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve the work form a `contentKind` sits on. Same contract as
|
||||
* {@link domainForContentKind}: `undefined` means "not derivable", never a
|
||||
* guess. Whoever writes `contentKind` writes both of these in the same breath.
|
||||
*/
|
||||
export function workFormForContentKind(contentKind: string | undefined | null): WorkForm | undefined {
|
||||
if (!contentKind) return undefined;
|
||||
return workFormByContentKind[contentKind.trim()];
|
||||
}
|
||||
@@ -1,5 +1,14 @@
|
||||
import {SimpleFileType} from "../SimpleFileType.js";
|
||||
|
||||
/**
|
||||
* Extension → `fileType`. Half of the canonical classification table
|
||||
* documented in METADATA_KEYS.md §1 (the other half is
|
||||
* {@link mimeTypeMappings}); this file is the source of truth and the doc
|
||||
* mirrors it.
|
||||
*
|
||||
* An extension absent from this map resolves to `other`, which makes the file
|
||||
* invisible to every typed query — so a missing entry is not a cosmetic gap.
|
||||
*/
|
||||
export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
// Audio
|
||||
'mp3': 'audio',
|
||||
@@ -7,7 +16,19 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'flac': 'audio',
|
||||
'aac': 'audio',
|
||||
'ogg': 'audio',
|
||||
'oga': 'audio',
|
||||
'opus': 'audio',
|
||||
'm4a': 'audio',
|
||||
// `m4b` is the audiobook container — without it `contentKind=audiobook`
|
||||
// had no extension that resolved to `audio` (METADATA_KEYS.md §14.15).
|
||||
'm4b': 'audio',
|
||||
'wma': 'audio',
|
||||
'aiff': 'audio',
|
||||
'aif': 'audio',
|
||||
'ape': 'audio',
|
||||
'wv': 'audio',
|
||||
'dsf': 'audio',
|
||||
'mka': 'audio',
|
||||
// Video
|
||||
'mp4': 'video',
|
||||
'mkv': 'video',
|
||||
@@ -27,6 +48,8 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'ts': 'video',
|
||||
'vob': 'video',
|
||||
'ogm': 'video',
|
||||
'ogv': 'video',
|
||||
'asf': 'video',
|
||||
'divx': 'video',
|
||||
'xvid': 'video',
|
||||
// Document
|
||||
@@ -37,31 +60,64 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
|
||||
'pptx': 'document',
|
||||
'xls': 'document',
|
||||
'xlsx': 'document',
|
||||
'odt': 'document',
|
||||
'ods': 'document',
|
||||
'odp': 'document',
|
||||
'txt': 'document',
|
||||
'md': 'document',
|
||||
'rtf': 'document',
|
||||
// Images (are document)
|
||||
'jpg': 'document',
|
||||
'jpeg': 'document',
|
||||
'png': 'document',
|
||||
'gif': 'document',
|
||||
'bmp': 'document',
|
||||
'tif': 'document',
|
||||
'tiff': 'document',
|
||||
'svg': 'document',
|
||||
'webp': 'document',
|
||||
// Document — literature formats. Their absence blocked meta-read outright
|
||||
// (METADATA_KEYS.md §14.15).
|
||||
'epub': 'document',
|
||||
'mobi': 'document',
|
||||
'azw': 'document',
|
||||
'azw3': 'document',
|
||||
'djvu': 'document',
|
||||
// Images
|
||||
// ⚠ These used to map to `document` ("Images (are document)"), which meant
|
||||
// the `image` bucket was unreachable from this table while Rust feeders
|
||||
// wrote it anyway. A consumer asking for documents must not receive
|
||||
// posters — see METADATA_KEYS.md §14.15.
|
||||
'jpg': 'image',
|
||||
'jpeg': 'image',
|
||||
'png': 'image',
|
||||
'gif': 'image',
|
||||
'bmp': 'image',
|
||||
'tif': 'image',
|
||||
'tiff': 'image',
|
||||
'svg': 'image',
|
||||
'webp': 'image',
|
||||
'avif': 'image',
|
||||
'heic': 'image',
|
||||
'heif': 'image',
|
||||
'ico': 'image',
|
||||
// Archive
|
||||
'zip': 'archive',
|
||||
'rar': 'archive',
|
||||
'7z': 'archive',
|
||||
'tar': 'archive',
|
||||
'tgz': 'archive',
|
||||
'gz': 'archive',
|
||||
'bz2': 'archive',
|
||||
'xz': 'archive',
|
||||
'zst': 'archive',
|
||||
// Archive — comic containers. A `.cbz` is literally a zip, so `archive` is
|
||||
// the honest byte-level answer; `contentKind=comic` + `domain=literature`
|
||||
// are what route it to meta-read (METADATA_KEYS.md §14.16).
|
||||
'cbz': 'archive',
|
||||
'cbr': 'archive',
|
||||
//subtitles
|
||||
'srt': 'subtitle',
|
||||
'sub': 'subtitle',
|
||||
'sbv': 'subtitle',
|
||||
'vtt': 'subtitle',
|
||||
// ⚠ ASS/SSA is what meta-watch renders through libass — the client's
|
||||
// primary subtitle format was the one this classifier did not know.
|
||||
'ass': 'subtitle',
|
||||
'ssa': 'subtitle',
|
||||
'ttml': 'subtitle',
|
||||
'dfxp': 'subtitle',
|
||||
'smi': 'subtitle',
|
||||
//torrent
|
||||
'torrent': 'torrent',
|
||||
};
|
||||
};
|
||||
|
||||
@@ -1,11 +1,32 @@
|
||||
import {SimpleFileType} from "../SimpleFileType.js";
|
||||
|
||||
/**
|
||||
* MIME type → `fileType`. The other half of the canonical classification
|
||||
* table (see {@link extensionMappings} and METADATA_KEYS.md §1).
|
||||
*
|
||||
* ⚠ Resolution is conservative on purpose: when a sniffed MIME maps to a
|
||||
* definite value that *disagrees* with the extension's value,
|
||||
* `FileTypeConfigurable.getFileType` returns `undefined` rather than guessing
|
||||
* — a wrong `fileType` routes a file to a handler that cannot open it. Adding
|
||||
* a mapping here therefore also creates a new way to disagree; only add ones
|
||||
* whose extension counterpart agrees.
|
||||
*/
|
||||
export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'text/plain': 'undefined',//text could be anything document or subtitle or other
|
||||
// Audio MIME types
|
||||
'audio/mpeg': 'audio',
|
||||
'audio/wav': 'audio',
|
||||
'audio/x-wav': 'audio',
|
||||
'audio/flac': 'audio',
|
||||
'audio/ogg': 'audio',
|
||||
'audio/opus': 'audio',
|
||||
'audio/aac': 'audio',
|
||||
// ftyp brands `M4A `/`M4B ` sniff as audio/mp4 while a real video mp4
|
||||
// sniffs as video/mp4, so this does not collide with the `mp4` extension.
|
||||
'audio/mp4': 'audio',
|
||||
'audio/x-ms-wma': 'audio',
|
||||
'audio/x-aiff': 'audio',
|
||||
'audio/x-ape': 'audio',
|
||||
// Video MIME types
|
||||
'video/mp4': 'video',
|
||||
'video/mpeg': 'video',
|
||||
@@ -13,8 +34,12 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'video/x-msvideo': 'video',
|
||||
'video/x-flv': 'video',
|
||||
'video/x-ms-wmv': 'video',
|
||||
'video/x-ms-asf': 'video',
|
||||
'video/x-matroska': 'video',
|
||||
'video/webm': 'video',
|
||||
'video/ogg': 'video',
|
||||
'video/3gpp': 'video',
|
||||
'video/mp2t': 'video',
|
||||
// Document MIME types
|
||||
'application/pdf': 'document',
|
||||
'application/msword': 'document',
|
||||
@@ -23,20 +48,47 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
|
||||
'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document',
|
||||
'application/vnd.ms-excel': 'document',
|
||||
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'document',
|
||||
'image/jpeg': 'document',
|
||||
'image/png': 'document',
|
||||
'image/gif': 'document',
|
||||
'application/vnd.oasis.opendocument.text': 'document',
|
||||
'application/vnd.oasis.opendocument.spreadsheet': 'document',
|
||||
'application/vnd.oasis.opendocument.presentation': 'document',
|
||||
'application/rtf': 'document',
|
||||
// Literature containers (METADATA_KEYS.md §14.15)
|
||||
'application/epub+zip': 'document',
|
||||
'application/x-mobipocket-ebook': 'document',
|
||||
'image/vnd.djvu': 'document',
|
||||
'text/markdown': 'document',
|
||||
// Image MIME types — no longer folded into `document` (§14.15)
|
||||
'image/jpeg': 'image',
|
||||
'image/png': 'image',
|
||||
'image/gif': 'image',
|
||||
'image/webp': 'image',
|
||||
'image/bmp': 'image',
|
||||
'image/tiff': 'image',
|
||||
'image/svg+xml': 'image',
|
||||
'image/avif': 'image',
|
||||
'image/heic': 'image',
|
||||
'image/x-icon': 'image',
|
||||
// Archive MIME types
|
||||
'application/zip': 'archive',
|
||||
'application/x-rar-compressed': 'archive',
|
||||
'application/vnd.rar': 'archive',
|
||||
'application/x-7z-compressed': 'archive',
|
||||
'application/gzip': 'archive',
|
||||
'application/x-bzip2': 'archive',
|
||||
'application/x-xz': 'archive',
|
||||
'application/zstd': 'archive',
|
||||
'application/x-tar': 'archive',
|
||||
// Comic containers sniff as their underlying zip/rar, which agrees with
|
||||
// the `cbz`/`cbr` extensions — both say `archive`.
|
||||
'application/vnd.comicbook+zip': 'archive',
|
||||
'application/vnd.comicbook-rar': 'archive',
|
||||
//torrent
|
||||
'application/x-bittorrent': 'torrent',
|
||||
//subtitles
|
||||
'text/vtt': 'subtitle',
|
||||
'text/srt': 'subtitle',
|
||||
'text/ssa': 'subtitle',
|
||||
'text/x-ssa': 'subtitle',
|
||||
'text/sub': 'subtitle',
|
||||
};
|
||||
'application/x-subrip': 'subtitle',
|
||||
};
|
||||
|
||||
@@ -13,6 +13,7 @@ export * from './tools/StringArrayOperation.js';
|
||||
export * from './tools/TwoStringsOperation.js';
|
||||
|
||||
export * from './config/AnimeKeywords.js';
|
||||
export * from './config/DomainMapping.js';
|
||||
export * from './config/ExtentionsMapping.js';
|
||||
export * from './config/KeywordsArray.js';
|
||||
export * from './config/MimeTypeMapping.js';
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from '../hash-compute/MultiHashData';
|
||||
|
||||
/**
|
||||
* Decoding a bare CIDv1 back into the algorithm that produced it.
|
||||
*
|
||||
* This is **the** TypeScript CID decoder for the platform. It exists because a
|
||||
* bare-CID key-set (`cids/<cid>`) carries no field names — a consumer that wants
|
||||
* "the sha2-256 of this file" has to ask the multicodec, not a string.
|
||||
*
|
||||
* Before this module, meta-dup carried a private hand-rolled base32+uvarint
|
||||
* decoder, the meta-sort editor carried a second one with a *different* rank
|
||||
* ladder, and meta-fuse had none at all (so it could only read the legacy
|
||||
* `cid_midhash256` field, and silently failed on any record that didn't have
|
||||
* one). One implementation, exported, replaces all three.
|
||||
*
|
||||
* ⚠ **Always match on the multihash code, never the content codec.** MetaMesh
|
||||
* has two encoders that disagree about the codec: meta-hash writes
|
||||
* `codec == algorithm code`, while the fullhash plugin and the gateway wrap
|
||||
* every digest with the raw codec `0x55`. The multihash code is the algorithm in
|
||||
* both. (`packages/meta-core/internal/cid/rank.go` documents the same trap.)
|
||||
*/
|
||||
|
||||
/** The multihash function code of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidMultihashCode(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).multihash.code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/** The content codec of a bare CID, or `undefined` if it won't parse. */
|
||||
export function cidCodec(cid: string): number | undefined {
|
||||
try {
|
||||
return CID.parse(cid).code;
|
||||
} catch {
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The first member of `cids` whose multihash code is `mhCode`.
|
||||
*
|
||||
* Takes the raw code so callers can ask for digests meta-hash doesn't itself
|
||||
* compute (e.g. the gateway's per-file btih codes).
|
||||
*/
|
||||
export function pickCidByMulticodec(
|
||||
cids: readonly string[] | undefined,
|
||||
mhCode: number
|
||||
): string | undefined {
|
||||
return cids?.find(cid => cidMultihashCode(cid) === mhCode);
|
||||
}
|
||||
|
||||
/** The member of `cids` produced by `algo`, if present. */
|
||||
export function pickCidByAlgorithm(
|
||||
cids: readonly string[] | undefined,
|
||||
algo: CidAlgorithm
|
||||
): string | undefined {
|
||||
return pickCidByMulticodec(cids, CID_ALGORITHM_CODES[algo]);
|
||||
}
|
||||
|
||||
/**
|
||||
* Has `algo` already been computed for this cid set?
|
||||
*
|
||||
* The "skip if already computed" check. It used to be an O(1) property lookup
|
||||
* (`metadata['cid_sha2-256']`); on a bare list it costs a decode per member.
|
||||
* That is a handful of microseconds against a hash computation that streams the
|
||||
* whole file — the trade is not close.
|
||||
*/
|
||||
export function hasAlgorithm(cids: readonly string[] | undefined, algo: CidAlgorithm): boolean {
|
||||
return pickCidByAlgorithm(cids, algo) !== undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Add `cid` to `cids` unless an equal string is already there.
|
||||
*
|
||||
* Dedup is on the exact string. Two encoders can spell the same digest
|
||||
* differently (codec 0x55 vs codec == mh), and both spellings are legitimate
|
||||
* members — meta-core's reverse index maps each to the same record.
|
||||
*/
|
||||
export function addCid(cids: string[], cid: string): void {
|
||||
if (cid && !cids.includes(cid)) {
|
||||
cids.push(cid);
|
||||
}
|
||||
}
|
||||
@@ -1,12 +0,0 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export const codeTable = {
|
||||
[CID_ALGORITHM_NAMES.crc32]: CID_ALGORITHM_CODES.crc32,
|
||||
[CID_ALGORITHM_NAMES.md5]: CID_ALGORITHM_CODES.md5,
|
||||
[CID_ALGORITHM_NAMES.sha1]: CID_ALGORITHM_CODES.sha1,
|
||||
[CID_ALGORITHM_NAMES.sha256]: CID_ALGORITHM_CODES.sha256,
|
||||
[CID_ALGORITHM_NAMES.sha3_256]: CID_ALGORITHM_CODES.sha3_256,
|
||||
[CID_ALGORITHM_NAMES.sha3_384]: CID_ALGORITHM_CODES.sha3_384,
|
||||
[CID_ALGORITHM_NAMES.midhash256]: CID_ALGORITHM_CODES.midhash256,
|
||||
[CID_ALGORITHM_NAMES.btih_v2]: CID_ALGORITHM_CODES.btih_v2
|
||||
}
|
||||
@@ -1,21 +1,20 @@
|
||||
import {CID_ALGORITHM_CODES, CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CID_ALGORITHM_CODES, CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {create} from "multiformats/hashes/digest";
|
||||
import {CID} from 'multiformats/cid';
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {codeTable} from "@root/file-id/CodeTable";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @param stream The Readable stream of the file
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1', etc.)
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @returns Array of bare CIDv1 strings, in the order of `algorithms`
|
||||
*/
|
||||
export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
export async function computeCIDs({stream, algorithms, createHasher}: {
|
||||
stream: ReadableStream<Uint8Array>;
|
||||
algorithms: CID_ALGORITHM_NAMES[];
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>
|
||||
algorithms: CidAlgorithm[];
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>
|
||||
}): Promise<string[]> {
|
||||
const hashers = await hasherDefiner(algorithms,createHasher);
|
||||
const hashers = await hasherDefiner(algorithms, createHasher);
|
||||
const reader = stream.getReader();
|
||||
while (true) {
|
||||
const {done, value} = await reader.read();
|
||||
@@ -28,23 +27,31 @@ export async function computeCIDs({stream, algorithms,createHasher}: {
|
||||
return cidFinalize(hashers);
|
||||
}
|
||||
|
||||
async function hasherDefiner(algorithms: CID_ALGORITHM_NAMES[],
|
||||
createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>): Promise<{
|
||||
async function hasherDefiner(algorithms: CidAlgorithm[],
|
||||
createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>): Promise<{
|
||||
hasher: SimpleHash,
|
||||
code: CID_ALGORITHM_CODES
|
||||
}[]> {
|
||||
const hashers = algorithms.filter(algo => Object.values(CID_ALGORITHM_NAMES).includes(algo))
|
||||
const hashers = algorithms
|
||||
.filter(algo => algo in CID_ALGORITHM_CODES)
|
||||
.map(async algo => ({
|
||||
hasher: await createHasher(algo),
|
||||
code: codeTable[algo]
|
||||
code: CID_ALGORITHM_CODES[algo]
|
||||
}));
|
||||
return await Promise.all(hashers);
|
||||
}
|
||||
|
||||
/**
|
||||
* Encoder convention: meta-hash sets `codec == multihash code`. The fullhash
|
||||
* plugin and the gateway instead use the raw codec (0x55) and carry the
|
||||
* algorithm only in the multihash. Both are valid CIDs for the same digest, and
|
||||
* both must decode to the same algorithm — which is why every consumer selects
|
||||
* on the **multihash code**, never the codec. See `CidDecode.ts`.
|
||||
*/
|
||||
async function cidFinalize(hashers: { hasher: SimpleHash, code: number }[]): Promise<string[]> {
|
||||
return await Promise.all(hashers.map(async ({code, hasher}) => {
|
||||
const hashBuffer = await hasher.digest();
|
||||
const digest = create(code, hashBuffer);
|
||||
return CID.createV1(code, digest).toString();
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,31 +1,32 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {createHash} from "crypto";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BtihV2Hasher} from "./BtihV2Hasher";
|
||||
|
||||
/**
|
||||
* Compute the CIDs of a file using specific algorithms
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
* @param algo
|
||||
* Build the stream hasher for one algorithm.
|
||||
*
|
||||
* `midhash256` is absent on purpose: it is not a stream digest (it hashes the
|
||||
* file size plus the middle 1 MB), so it is produced by `FastHash.ts` instead.
|
||||
*/
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return createHash("sha256");
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return createHash("sha1");
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return createHash("md5");
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return createHash("sha3-256");
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return createHash("sha3-384");
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
case CID_ALGORITHM_NAMES.btih_v2:
|
||||
case 'btih_v2':
|
||||
return new BtihV2Hasher();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,27 @@
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
import {SimpleHash} from "./SimpleHash";
|
||||
import {Crc32Hash} from "./Crc32Hash";
|
||||
import {BrowserHasher} from "@root/file-id/BrowserHasher";
|
||||
|
||||
export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHash> {
|
||||
/**
|
||||
* Browser build. Note it supports fewer algorithms than the node one: there is
|
||||
* no `btih_v2` (and no `midhash256`, which is never a stream hash) — asking for
|
||||
* either throws.
|
||||
*/
|
||||
export async function createHasher(algo: CidAlgorithm): Promise<SimpleHash> {
|
||||
const hasher = (() => {
|
||||
switch (algo) {
|
||||
case CID_ALGORITHM_NAMES.sha256:
|
||||
case 'sha256':
|
||||
return new BrowserHasher('sha-256');
|
||||
case CID_ALGORITHM_NAMES.sha1:
|
||||
case 'sha1':
|
||||
return new BrowserHasher('sha-1');
|
||||
case CID_ALGORITHM_NAMES.md5:
|
||||
case 'md5':
|
||||
return new BrowserHasher('md5');
|
||||
case CID_ALGORITHM_NAMES.sha3_256:
|
||||
case 'sha3_256':
|
||||
return new BrowserHasher('sha3-256');
|
||||
case CID_ALGORITHM_NAMES.sha3_384:
|
||||
case 'sha3_384':
|
||||
return new BrowserHasher('sha3-384');
|
||||
case CID_ALGORITHM_NAMES.crc32:
|
||||
case 'crc32':
|
||||
return new Crc32Hash();
|
||||
default:
|
||||
throw new Error(`Unsupported algorithm: ${algo}`);
|
||||
@@ -28,4 +33,4 @@ export async function createHasher(algo: CID_ALGORITHM_NAMES): Promise<SimpleHas
|
||||
}
|
||||
|
||||
return hasher;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
|
||||
export interface FileIDComputerInterface{
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]>;
|
||||
computeCIDs(filePath: ReadableStream<Uint8Array>, algorithms: CidAlgorithm[]): Promise<string[]>;
|
||||
}
|
||||
@@ -1,10 +1,16 @@
|
||||
import {Piscina} from "piscina";
|
||||
import {CID_ALGORITHM_NAMES} from "../hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "../hash-compute/MultiHashData";
|
||||
|
||||
export class FileIDComputerWorker{
|
||||
private piscina: Piscina;
|
||||
|
||||
constructor(workerPath?:string) {
|
||||
// `WORKER_URL` is the escape hatch the tests have always set — it just
|
||||
// wasn't read. Honour it before falling back to path derivation, which
|
||||
// only works for a caller running from `dist/`.
|
||||
if (!workerPath && process.env.WORKER_URL) {
|
||||
workerPath = new URL(process.env.WORKER_URL, `file://${process.cwd()}/`).href;
|
||||
}
|
||||
if(!workerPath) {
|
||||
// Construct the URL for the current module
|
||||
let distFolder = import.meta.dirname;
|
||||
@@ -25,7 +31,7 @@ export class FileIDComputerWorker{
|
||||
* @param algorithms Array of algorithms ('sha256', 'sha1')
|
||||
* @returns Array of CIDs (in the order of the algorithms)
|
||||
*/
|
||||
public async computeCIDs(filePath: string, algorithms: CID_ALGORITHM_NAMES[]): Promise<string[]> {
|
||||
public async computeCIDs(filePath: string, algorithms: CidAlgorithm[]): Promise<string[]> {
|
||||
return this.piscina.run({filePath, algorithms});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,5 @@
|
||||
import {MultiHashData} from '../hash-compute/MultiHashData';
|
||||
import {pickCidByAlgorithm} from './CidDecode';
|
||||
import {CID} from 'multiformats/cid';
|
||||
|
||||
/**
|
||||
@@ -22,13 +23,13 @@ export class MagnetLinkGenerator {
|
||||
/**
|
||||
* Generate BitTorrent v2 magnet link from metadata
|
||||
*
|
||||
* @param metadata - File metadata containing cid_btih_v2
|
||||
* @param metadata - File metadata whose `cids` include a btih-v2 CID
|
||||
* @param fileName - Original filename for display name
|
||||
* @param fileSize - File size in bytes
|
||||
* @returns Magnet link string, or null if btih_v2 not available
|
||||
*
|
||||
* @example
|
||||
* const metadata = {cid_btih_v2: 'bafyb...'};
|
||||
* const metadata = {cids: ['bafyb...']};
|
||||
* const magnet = MagnetLinkGenerator.generate(metadata, 'video.mp4', 1024000);
|
||||
* // Returns: magnet:?xt=urn:btmh:1220abc...&dn=video.mp4&xl=1024000
|
||||
*/
|
||||
@@ -37,7 +38,9 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
const cidString = metadata.cid_btih_v2;
|
||||
// The btih-v2 CID is recovered from the bare list by its multicodec
|
||||
// (0x10B7), not by a field name. See CidDecode.ts.
|
||||
const cidString = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
if (!cidString) {
|
||||
return null; // BitTorrent v2 hash not computed
|
||||
}
|
||||
@@ -178,6 +181,6 @@ export class MagnetLinkGenerator {
|
||||
fileName: string,
|
||||
fileSize: number
|
||||
): string | null {
|
||||
return this.generate({cid_btih_v2: cidString}, fileName, fileSize);
|
||||
return this.generate({cids: [cidString]}, fileName, fileSize);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import {CID_ALGORITHM_NAMES} from "@root/hash-compute/MultiHashData";
|
||||
import {CidAlgorithm} from "@root/hash-compute/MultiHashData";
|
||||
import fs from "fs";
|
||||
import {Readable} from 'stream';
|
||||
import {computeCIDs as computeCIDsInternal} from "./ComputeHash";
|
||||
@@ -12,7 +12,7 @@ import {createHasher} from "@root/file-id/CreateHasher";
|
||||
*/
|
||||
export default async function computeCIDs({filePath, algorithms}: {
|
||||
filePath: string;
|
||||
algorithms: CID_ALGORITHM_NAMES[]
|
||||
algorithms: CidAlgorithm[]
|
||||
}): Promise<string[]> {
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
return computeCIDsInternal({stream, algorithms, createHasher});
|
||||
|
||||
@@ -1,30 +1,29 @@
|
||||
import {CID_ALGORITHM_NAMES, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, MultiHashData} from "./MultiHashData";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {SimpleHash} from "@root/file-id/SimpleHash";
|
||||
|
||||
export class HashComputer{
|
||||
export class HashComputer {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[],
|
||||
private createHasher : (algo: CID_ALGORITHM_NAMES) => Promise<SimpleHash>) {
|
||||
constructor(private targetHash: CidAlgorithm[],
|
||||
private createHasher: (algo: CidAlgorithm) => Promise<SimpleHash>) {
|
||||
}
|
||||
|
||||
async computeMissingHash(stream: ReadableStream<Uint8Array>, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
// Which digests are still missing? On a bare-CID list this costs a decode
|
||||
// per member rather than a property lookup — negligible against streaming
|
||||
// the whole file through a hasher.
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher:this.createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher: this.createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,32 +1,28 @@
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {Readable} from "stream";
|
||||
|
||||
import {computeCIDs} from "@root/file-id/ComputeHash";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
import fs from "fs";
|
||||
import {createHasher} from "@root/file-id/CreateHasher";
|
||||
|
||||
export class HashComputerFile implements ComputeInterface{
|
||||
export class HashComputerFile implements ComputeInterface {
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
constructor(private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const stream = Readable.toWeb(fs.createReadStream(filePath)) as ReadableStream<Uint8Array>;
|
||||
const cids = await computeCIDs({stream, algorithms:neededHashes,createHasher});
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await computeCIDs({stream, algorithms: neededHashes, createHasher});
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,33 +1,59 @@
|
||||
import {HashComputerWorker} from "./HashComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {HashIndexManager} from "./HashIndexManager";
|
||||
import {addCid, hasAlgorithm, pickCidByAlgorithm} from "@root/file-id/CidDecode";
|
||||
import {stat} from "fs/promises";
|
||||
import path from "path";
|
||||
|
||||
/**
|
||||
* The translation point between the two shapes.
|
||||
*
|
||||
* `MultiHashData` carries **bare CIDs** (`cids: string[]`) — that is what gets
|
||||
* persisted onto a record as the `cids/<cid>` key-set. The on-disk index is
|
||||
* **per-algorithm columns**, because its job is the O(1) "already computed?"
|
||||
* check. Neither shape leaks into the other; this class converts.
|
||||
*/
|
||||
export class HashComputerIndexCache implements ComputeInterface {
|
||||
private hashIndexManager: HashIndexManager;
|
||||
private hashComputer: HashComputerWorker;
|
||||
|
||||
constructor(indexFolderPath: string,private targetHash: CID_ALGORITHM_NAMES[] = [CID_ALGORITHM_NAMES.sha1, CID_ALGORITHM_NAMES.sha256],workerPath?:string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash,workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath,targetHash);
|
||||
constructor(indexFolderPath: string, private targetHash: CidAlgorithm[] = ['sha1', 'sha256'], workerPath?: string) {
|
||||
this.hashComputer = new HashComputerWorker(targetHash, workerPath);
|
||||
this.hashIndexManager = new HashIndexManager(indexFolderPath, targetHash);
|
||||
}
|
||||
|
||||
public async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
await this.hashIndexManager.init();
|
||||
let stats = await stat(filePath);
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath),stats)) {
|
||||
const stats = await stat(filePath);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// Cache hit: lift the cached CIDs into the bare list.
|
||||
if (this.hashIndexManager.hasFileInCache(path.basename(filePath), stats)) {
|
||||
const indexLine = this.hashIndexManager.getCidForFile(filePath, stats.size, stats.mtime.toISOString());
|
||||
if(indexLine) {
|
||||
for (const hash of this.targetHash) {
|
||||
if (!metadata[hash] && indexLine[hash]) {
|
||||
metadata[hash] = indexLine[hash];
|
||||
if (indexLine) {
|
||||
for (const algo of this.targetHash) {
|
||||
const cached = indexLine[algo];
|
||||
if (cached && !hasAlgorithm(cids, algo)) {
|
||||
addCid(cids, cached);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Compute whatever is still missing (a no-op on a full cache hit).
|
||||
await this.hashComputer.computeMissingHash(filePath, metadata);
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), metadata);
|
||||
|
||||
// Write back: project the bare list onto the index's per-algorithm columns.
|
||||
const columns: Partial<Record<CidAlgorithm, string>> = {};
|
||||
for (const algo of this.targetHash) {
|
||||
const cid = pickCidByAlgorithm(cids, algo);
|
||||
if (cid) {
|
||||
columns[algo] = cid;
|
||||
}
|
||||
}
|
||||
if (Object.keys(columns).length > 0) {
|
||||
this.hashIndexManager.addFileCid(filePath, stats.size, stats.mtime.toISOString(), columns);
|
||||
}
|
||||
}
|
||||
|
||||
async getHashIndexManager(): Promise<HashIndexManager> {
|
||||
|
||||
@@ -1,29 +1,25 @@
|
||||
import {FileIDComputerWorker} from "../file-id/FileIDComputerWorker";
|
||||
import {CID_ALGORITHM_NAMES, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {CidAlgorithm, ComputeInterface, MultiHashData} from "./MultiHashData";
|
||||
import {addCid, hasAlgorithm} from "@root/file-id/CidDecode";
|
||||
|
||||
export class HashComputerWorker implements ComputeInterface {
|
||||
private fileIDComputer: FileIDComputerWorker;
|
||||
|
||||
constructor(private targetHash: CID_ALGORITHM_NAMES[], workerPath?: string) {
|
||||
constructor(private targetHash: CidAlgorithm[], workerPath?: string) {
|
||||
this.fileIDComputer = new FileIDComputerWorker(workerPath);
|
||||
}
|
||||
|
||||
async computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void> {
|
||||
// Dynamically determine which hashes are needed
|
||||
const neededHashes = this.targetHash.filter(hashName => !metadata[hashName]);
|
||||
const cids = (metadata.cids ??= []);
|
||||
|
||||
// If all hashes are already computed, skip the processing
|
||||
const neededHashes = this.targetHash.filter(algo => !hasAlgorithm(cids, algo));
|
||||
if (neededHashes.length === 0) {
|
||||
return;
|
||||
}
|
||||
|
||||
// Compute only the needed CIDs
|
||||
const cids = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
|
||||
// Map the computed CIDs back to their respective metadata properties
|
||||
for (const [index, cid] of cids.entries()) {
|
||||
const hashType = neededHashes[index];
|
||||
metadata[hashType] = cid;
|
||||
const computed = await this.fileIDComputer.computeCIDs(filePath, neededHashes);
|
||||
for (const cid of computed) {
|
||||
addCid(cids, cid);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,11 +5,28 @@ import {stringify} from 'csv-stringify/sync';
|
||||
import {existsAsync} from "../utils/ExistsAsync";
|
||||
import path from "path";
|
||||
import {clearInterval} from "node:timers";
|
||||
import {CID_ALGORITHM_NAMES} from "./MultiHashData";
|
||||
import {CidAlgorithm} from "./MultiHashData";
|
||||
import {stat} from "fs/promises";
|
||||
import {Stats} from "node:fs";
|
||||
|
||||
interface IndexLine extends Partial<Record<CID_ALGORITHM_NAMES, string>> {
|
||||
/**
|
||||
* One row of the on-disk hash-index cache.
|
||||
*
|
||||
* This is the one place a **per-algorithm column** is still the right shape, and
|
||||
* it is not the deprecated one. The index answers "have I already computed sha256
|
||||
* for this file?" on every scan, which a column makes an O(1) lookup and a bare
|
||||
* CID list would make a decode-per-member. It is also strictly local: a wipeable
|
||||
* cache under `/data/cache/hash-index/` (see
|
||||
* `docs/project-architecture/cache-persistence.md`), never a record, never on the
|
||||
* wire.
|
||||
*
|
||||
* What changed is the *naming*: columns are keyed by bare algorithm
|
||||
* (`sha256`, `midhash256`) instead of by the old `cid_<algo>` record-field names,
|
||||
* so the files are `index-sha256.csv`, not `index-cid_sha2-256.csv`. The record
|
||||
* shape and the cache shape are now separate concerns that cannot leak into each
|
||||
* other — which is the whole point of the split (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
interface IndexLine extends Partial<Record<CidAlgorithm, string>> {
|
||||
path: string;
|
||||
size: string;
|
||||
mtime: string;
|
||||
@@ -21,15 +38,15 @@ export class HashIndexManager {
|
||||
private cache: Map<string, IndexLine> = new Map<string, IndexLine>();//the key is the file `${file_name}-${filesize}-${mtime_ISOstr}`
|
||||
private intervalId: any;
|
||||
private intervalTime: number = 30000;
|
||||
private lastIndexFileSize: { [key in CID_ALGORITHM_NAMES]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CID_ALGORITHM_NAMES]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private lastIndexFileSize: { [key in CidAlgorithm]?: number } = {}; //size of the index file last time it was read
|
||||
private lastCacheFile: { [key in CidAlgorithm]?: IndexLine[] } = {}; //state of the file last time it was read
|
||||
private indexOpsInProgress: boolean = false;
|
||||
private hasChanged: boolean = false;
|
||||
private initialLoad: Promise<void>;
|
||||
private filePaths: { [key in CID_ALGORITHM_NAMES]?: string } = {};
|
||||
private filePaths: { [key in CidAlgorithm]?: string } = {};
|
||||
|
||||
constructor(private indexFolderPath: string,
|
||||
private targetHash: CID_ALGORITHM_NAMES[]) {
|
||||
private targetHash: CidAlgorithm[]) {
|
||||
}
|
||||
|
||||
hasFileInCache(filePath: string,stats:Stats): boolean {
|
||||
@@ -81,7 +98,7 @@ export class HashIndexManager {
|
||||
}
|
||||
|
||||
// Function to check CSV headers
|
||||
private checkCSVHeaders(csvContent: string, hash: CID_ALGORITHM_NAMES): boolean {
|
||||
private checkCSVHeaders(csvContent: string, hash: CidAlgorithm): boolean {
|
||||
const records = parseSync(csvContent, {
|
||||
bom: true,
|
||||
columns: true,
|
||||
@@ -116,7 +133,7 @@ export class HashIndexManager {
|
||||
this.intervalId = setInterval(() => this.saveCacheToFile(), time);
|
||||
}
|
||||
|
||||
public async loadIndex(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
public async loadIndex(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (await existsAsync(this.filePaths[hash])) {
|
||||
// check the file size and if it did not change, do not read the file
|
||||
const stats = await fs.stat(this.filePaths[hash]);
|
||||
@@ -147,7 +164,7 @@ export class HashIndexManager {
|
||||
return Array.from(this.cache.values());
|
||||
}
|
||||
|
||||
private async readCsv(hash: CID_ALGORITHM_NAMES): Promise<IndexLine[]> {
|
||||
private async readCsv(hash: CidAlgorithm): Promise<IndexLine[]> {
|
||||
if (!(await existsAsync(this.filePaths[hash]))) {
|
||||
return [];
|
||||
}
|
||||
@@ -178,10 +195,8 @@ export class HashIndexManager {
|
||||
|
||||
public async saveCacheToFile(): Promise<void> {
|
||||
if (this.indexOpsInProgress || !this.hasChanged) {
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Skipping (indexOpsInProgress=${this.indexOpsInProgress}, hasChanged=${this.hasChanged}, cacheSize=${this.cache.size})`);
|
||||
return;
|
||||
}
|
||||
console.log(`[HashIndexManager] saveCacheToFile: Starting save, cacheSize=${this.cache.size}`);
|
||||
this.hasChanged = false;
|
||||
this.indexOpsInProgress = true;
|
||||
const start = performance.now();
|
||||
@@ -274,17 +289,15 @@ export class HashIndexManager {
|
||||
return null;
|
||||
}
|
||||
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CID_ALGORITHM_NAMES, string>>): void {
|
||||
public addFileCid(filePath: string, fileSize: number, mtime: string, hashs: Partial<Record<CidAlgorithm, string>>): void {
|
||||
if (!filePath || !fileSize || !mtime || !hashs) {
|
||||
throw new Error('Invalid parameters');
|
||||
}
|
||||
// Check that at least one target hash is provided
|
||||
const providedHashes = this.targetHash.filter(hash => hashs[hash]);
|
||||
if (providedHashes.length === 0) {
|
||||
console.log(`[HashIndexManager] addFileCid: No target hashes provided for ${filePath}. targetHashes=${this.targetHash}, providedKeys=${Object.keys(hashs)}`);
|
||||
throw new Error('At least one target hash must be provided');
|
||||
}
|
||||
console.log(`[HashIndexManager] addFileCid: Adding ${filePath} with ${providedHashes.length} hashes: ${providedHashes.join(', ')}`);
|
||||
const size = fileSize + "";
|
||||
const baseName = path.basename(filePath);
|
||||
const cacheKey = `${baseName}-${size}-${mtime}`;
|
||||
|
||||
@@ -1,21 +1,26 @@
|
||||
export interface MultiHashData {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (SHA-256 of middle 1MB + file size) - CID v1 format
|
||||
cid_btih_v2?: string; // 0x10B7 BitTorrent v2 info hash (BEP 52) - CID v1 format
|
||||
}
|
||||
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
/**
|
||||
* according to
|
||||
* https://ipfs.io/ipfs/QmXec1jjwzxWJoNbxQF5KffL8q6hFXm9QwUGaa3wKGk6dT/#title=Multicodecs&src=https://raw.githubusercontent.com/multiformats/multicodec/master/table.csv
|
||||
* Multicodec / multihash codes for every digest meta-hash can compute.
|
||||
*
|
||||
* Source of truth for these numbers. The same codes appear in the CID utility
|
||||
* RANK, which is a separate thing this file does not implement:
|
||||
* - `packages/meta-core/internal/cid/rank.go`
|
||||
* - the `cid_rank.rs` in each of meta-search and meta-share
|
||||
* - `/cid-rank-vectors.json` (the golden fixture pinning those three together)
|
||||
*
|
||||
* ⚠ This package used to vendor that fixture at `src/tests/cid-rank-vectors.json`
|
||||
* and no test ever loaded it, so every vector added cost a commit here for
|
||||
* nothing. Removed 2026-09-09. If a TypeScript rank implementation is ever
|
||||
* written, re-add the path to `scripts/check-cid-vectors.sh` *and* a test that
|
||||
* asserts it — a vendored fixture nobody reads is not a guard.
|
||||
*
|
||||
* ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
|
||||
* `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
|
||||
* the gateway wrap every digest with the **raw** codec `0x55` and put the
|
||||
* algorithm in the multihash code. The multihash code is the algorithm in both
|
||||
* conventions — so anything selecting a CID by algorithm must read the
|
||||
* **multihash code**, never the content codec. See `CidDecode.ts`.
|
||||
*
|
||||
* https://github.com/multiformats/multicodec/blob/master/table.csv
|
||||
*/
|
||||
export enum CID_ALGORITHM_CODES {
|
||||
crc32 = 0x0132,
|
||||
@@ -24,28 +29,55 @@ export enum CID_ALGORITHM_CODES {
|
||||
sha256 = 0x12,
|
||||
sha3_256 = 0x16,
|
||||
sha3_384 = 0x15,
|
||||
midhash256 = 0x1000, // Custom code for midhash256
|
||||
btih_v2 = 0x10B7, // Custom code for BitTorrent v2 info hash
|
||||
midhash256 = 0x1000, // Custom — SHA-256(size ‖ middle 1MB)
|
||||
btih_v2 = 0x10B7, // Custom — BitTorrent v2 info hash (BEP 52)
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM_NAMES {
|
||||
crc32 = 'cid_crc32',
|
||||
md5 = 'cid_md5',
|
||||
sha1 = 'cid_sha1',
|
||||
sha256 = 'cid_sha2-256',
|
||||
sha3_256 = 'cid_sha3-256',
|
||||
sha3_384 = 'cid_sha3-384',
|
||||
midhash256 = 'cid_midhash256',
|
||||
btih_v2 = 'cid_btih_v2',
|
||||
/**
|
||||
* An algorithm *selector*: which digests to compute, and which column of the
|
||||
* local hash-index cache to read.
|
||||
*
|
||||
* This is deliberately **not** a metadata field name. The old
|
||||
* `CID_ALGORITHM_NAMES` enum ('cid_sha2-256', 'cid_midhash256', …) did triple
|
||||
* duty — selector, record property name, and CSV column — and the middle duty
|
||||
* was the deprecated one: those `cid_<algo>` fields were stored on records but
|
||||
* never reverse-indexed by meta-core, leaving records unresolvable by their own
|
||||
* CID. meta-core now rejects them at the write boundary (400). See
|
||||
* METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* The triple duty also rotted on its own: the interface declared `cid_sha3_384`
|
||||
* (underscore) while the enum emitted `'cid_sha3-384'` (hyphen), and the doc
|
||||
* comment on it named the wrong multicodec. A self-describing CID cannot drift
|
||||
* from its own name that way.
|
||||
*/
|
||||
export type CidAlgorithm = keyof typeof CID_ALGORITHM_CODES;
|
||||
|
||||
/** Every algorithm meta-hash knows how to compute. */
|
||||
export const CID_ALGORITHMS: CidAlgorithm[] = [
|
||||
'crc32',
|
||||
'md5',
|
||||
'sha1',
|
||||
'sha256',
|
||||
'sha3_256',
|
||||
'sha3_384',
|
||||
'midhash256',
|
||||
'btih_v2',
|
||||
];
|
||||
|
||||
/**
|
||||
* The output of a hash computation: **bare CIDv1 strings, nothing else**.
|
||||
*
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is no
|
||||
* per-algorithm field name to keep in sync, and no `<algo>:` prefix. To find a
|
||||
* specific digest, decode the members (`pickCidByAlgorithm` in `CidDecode.ts`).
|
||||
*
|
||||
* This is persisted onto a record as the bare-CID key-set
|
||||
* (`cids/<cid> = "true"`), which is the only shape meta-core indexes.
|
||||
*/
|
||||
export interface MultiHashData {
|
||||
cids?: string[];
|
||||
}
|
||||
|
||||
export enum CID_ALGORITHM {
|
||||
crc32 = 'crc32',
|
||||
md5 = 'md5',
|
||||
sha1 = 'sha1',
|
||||
sha256 = 'sha2-256',
|
||||
sha3_256 = 'sha3-256',
|
||||
sha3_384 = 'sha3-384',
|
||||
midhash256 = 'midhash256',
|
||||
btih_v2 = 'btih-v2',
|
||||
}
|
||||
export interface ComputeInterface {
|
||||
computeMissingHash(filePath: string, metadata: MultiHashData): Promise<void>;
|
||||
}
|
||||
|
||||
@@ -4,6 +4,12 @@ export * from './file-id/ShaComputeWorker.js';//worker
|
||||
export * from './file-id/FastHash.js'; // Fast-hash computation
|
||||
export * from './file-id/BtihV2Hasher.js'; // BitTorrent v2 info hash
|
||||
export * from './file-id/MagnetLinkGenerator.js'; // Magnet link generation
|
||||
export * from './file-id/CidDecode.js'; // Bare-CID → algorithm decoding (THE shared TS CID decoder)
|
||||
|
||||
// The CID model: algorithm codes, the algorithm selector, and MultiHashData
|
||||
// (bare `cids: string[]`). Exported because every consumer of a `cids/<cid>`
|
||||
// key-set needs them to recover a specific digest — see CidDecode.
|
||||
export * from './hash-compute/MultiHashData.js';
|
||||
|
||||
//hash-compute
|
||||
export * from './hash-compute/HashComputerIndexCache.js';
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import {BtihV2Hasher} from '../lib/file-id/BtihV2Hasher';
|
||||
import {CID_ALGORITHM_NAMES, HashComputerWorker} from "../lib";
|
||||
import {HashComputerWorker, pickCidByAlgorithm} from "../lib";
|
||||
import {readFile} from 'fs/promises';
|
||||
import {createHash} from 'crypto';
|
||||
import * as process from "process";
|
||||
@@ -157,26 +157,20 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should integrate with HashComputerWorker', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2,
|
||||
CID_ALGORITHM_NAMES.sha256
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2', 'sha256']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
console.log('Computed metadata:', metadata);
|
||||
// The digests land in a bare CID list; each is recovered by its
|
||||
// multicodec, not by a field name.
|
||||
const btih = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
expect(btih).toBeDefined();
|
||||
expect(typeof btih).toBe('string');
|
||||
expect(btih!.startsWith('b')).toBe(true);
|
||||
|
||||
// Verify btih_v2 CID was computed
|
||||
expect(metadata).toHaveProperty('cid_btih_v2');
|
||||
expect(typeof metadata.cid_btih_v2).toBe('string');
|
||||
expect(metadata.cid_btih_v2.length).toBeGreaterThan(0);
|
||||
|
||||
// Verify CID format (should start with 'b' for base32 CIDv1)
|
||||
expect(metadata.cid_btih_v2.startsWith('b')).toBe(true);
|
||||
|
||||
// Should also have SHA-256 for comparison
|
||||
expect(metadata).toHaveProperty('cid_sha2-256');
|
||||
// Should also have SHA-256 for comparison.
|
||||
expect(pickCidByAlgorithm(metadata.cids, 'sha256')).toBeDefined();
|
||||
});
|
||||
|
||||
it('should compute piece length correctly', () => {
|
||||
@@ -213,15 +207,16 @@ describe('BitTorrent v2 Info Hash (BEP 52)', () => {
|
||||
});
|
||||
|
||||
it('should produce valid CID format with codec 0x10B7', async () => {
|
||||
const hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.btih_v2
|
||||
]);
|
||||
const hashComputer = new HashComputerWorker(['btih_v2']);
|
||||
|
||||
const metadata: any = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
|
||||
const cid = metadata.cid_btih_v2;
|
||||
console.log('BitTorrent v2 CID:', cid);
|
||||
const cid = pickCidByAlgorithm(metadata.cids, 'btih_v2');
|
||||
|
||||
// The selector proves the codec: pickCidByAlgorithm('btih_v2') matches on
|
||||
// multihash 0x10B7, so a hit here IS the "valid CID with codec 0x10B7"
|
||||
// assertion this test is named for.
|
||||
|
||||
// Verify CID format
|
||||
expect(cid).toBeDefined();
|
||||
|
||||
@@ -1,8 +1,13 @@
|
||||
import {CID_ALGORITHM_NAMES, HashComputerIndexCache, HashComputerWorker} from "../lib";
|
||||
import {
|
||||
CidAlgorithm,
|
||||
HashComputerIndexCache,
|
||||
HashComputerWorker,
|
||||
pickCidByAlgorithm,
|
||||
} from "../lib";
|
||||
|
||||
import {describe, expect, it} from 'vitest';
|
||||
import * as process from "process";
|
||||
import { mkdir, rm, rmdir } from "fs/promises";
|
||||
import {mkdir, rm, stat} from "fs/promises";
|
||||
|
||||
process.env.WORKER_URL = "./dist/worker.js";
|
||||
|
||||
@@ -11,87 +16,102 @@ process.env.WORKER_URL = "./dist/worker.js";
|
||||
* Remember to build the worker each time you change the worker code.
|
||||
*/
|
||||
|
||||
const ALGOS: CidAlgorithm[] = ['sha1', 'sha256', 'md5', 'sha3_256', 'crc32', 'sha3_384'];
|
||||
|
||||
/** `src/tests/test.txt`, checked out with LF endings (`.gitattributes`: text=auto eol=lf). */
|
||||
const FIXTURE_BYTES = 490;
|
||||
|
||||
/**
|
||||
* The expected digests of `src/tests/test.txt`, as bare CIDv1 strings.
|
||||
*
|
||||
* Independently reproducible — these are not "whatever the code printed":
|
||||
*
|
||||
* $ sha256sum src/tests/test.txt
|
||||
* be25c28a7af0dfd58b3bd487858d0c40faf4c8fb3a721c9fe5d5db7fff3157ec
|
||||
* $ md5sum → c8971ace120d1393b658eabb45d08c88
|
||||
* $ sha1sum → fb6497eb882a7ce7773f4680fdd2b75cd73ebcf7
|
||||
*
|
||||
* and each CID below wraps exactly that digest under its multicodec (meta-hash's
|
||||
* encoder sets `codec == multihash code`).
|
||||
*
|
||||
* ⚠ These values were **re-pinned**. The previous ones described a 499-byte
|
||||
* CRLF copy of the fixture, from before `.gitattributes` normalised it to LF —
|
||||
* so on any LF checkout the digests could never match and this suite was red.
|
||||
* The mismatch was invisible because the worker path was also broken, so the
|
||||
* assertions never even ran. Hence the explicit `FIXTURE_BYTES` check: if the
|
||||
* fixture's bytes ever shift again, the test says *that*, instead of printing
|
||||
* six inscrutable CID diffs.
|
||||
*/
|
||||
const EXPECTED: Record<string, string> = {
|
||||
crc32: 'bagzafmqcat2hnot3',
|
||||
md5: 'bahkqdvibcdejogwocigrhe5wldvlwroqrsea',
|
||||
sha1: 'baeircfh3msl6xcbkpttxop2gqd65fn24247lz5y',
|
||||
sha256: 'baejbeif6exbiu6xq37kywo6uq6cy2dca7l2mr6z2oioj7zov3n776mkx5q',
|
||||
sha3_256: 'baelbmiemf56ripvp5zvx5rxljfr56fqfjpxv7xwyx3lphfyv6y5j2egdyi',
|
||||
sha3_384: 'baekrkmatuj3wvbkizsyfe6bb3kk7eoiglmsa6qhrr2jehdboxbauep2mppu56xaxxtdztagn7adtswhrw7wq',
|
||||
};
|
||||
|
||||
/**
|
||||
* Every expected digest is present in the bare list, and each is recoverable by
|
||||
* its algorithm **through its multicodec** — there is no field name to look up.
|
||||
* That is the whole point of the `cids: string[]` shape: a CIDv1 is
|
||||
* self-describing, so the record never carries a per-algorithm key that could
|
||||
* drift from the CID it names (METADATA_KEYS.md §14.13).
|
||||
*/
|
||||
function expectAllDigests(cids: string[] | undefined) {
|
||||
expect(cids).toBeDefined();
|
||||
expect([...cids!].sort()).toEqual(Object.values(EXPECTED).sort());
|
||||
|
||||
for (const algo of ALGOS) {
|
||||
expect(pickCidByAlgorithm(cids, algo)).toBe(EXPECTED[algo]);
|
||||
}
|
||||
}
|
||||
|
||||
describe('add', () => {
|
||||
it('meta data compute', async () => {
|
||||
let hashComputer = new HashComputerWorker([
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
it('the fixture has the bytes these CIDs were pinned against', async () => {
|
||||
const {size} = await stat('./src/tests/test.txt');
|
||||
expect(size).toBe(FIXTURE_BYTES);
|
||||
});
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp",{recursive:true});
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let metadata = {};
|
||||
|
||||
it('meta data compute', async () => {
|
||||
const hashComputer = new HashComputerWorker(ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
console.log(metadata);
|
||||
/**
|
||||
* test.txt CF20B62A (CRC32)
|
||||
* 26f1157f4349a6fb5d207ac5f5995140 *test.txt (MD5)
|
||||
* bb242b7b821074f4b9eb9a94d439ab668705aac4 ?SHA1*test.txt
|
||||
* 2aa7a80d2fd1eb96d31afc7cdda9211a97524202c63f04151d6163fba35061a0 ?SHA256*test.txt
|
||||
* 865a1ea4977b25e04b8ea84af93b21f1a031cb48359f3805089cdae1d89842b9a3faf87654989dcae418144a8bbb378c ?SHA3-384*test.txt
|
||||
*/
|
||||
expect(metadata).toEqual({
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga'
|
||||
});
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
|
||||
it('meta data compute with index', async () => {
|
||||
await rm("./tmp", {recursive: true, force: true});
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
await (await hashComputer.getHashIndexManager()).saveCacheToFile();
|
||||
});
|
||||
|
||||
it('cache', async () => {
|
||||
await mkdir("./tmp",{recursive:true});
|
||||
let hashComputer = new HashComputerIndexCache("./tmp/index.csv",[
|
||||
CID_ALGORITHM_NAMES.sha1,
|
||||
CID_ALGORITHM_NAMES.sha256,
|
||||
CID_ALGORITHM_NAMES.md5,
|
||||
CID_ALGORITHM_NAMES.sha3_256,
|
||||
CID_ALGORITHM_NAMES.crc32,
|
||||
CID_ALGORITHM_NAMES.sha3_384
|
||||
]);
|
||||
let indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
await mkdir("./tmp", {recursive: true});
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
|
||||
// The on-disk index is the one place per-algorithm columns survive — it
|
||||
// answers "already computed?" in O(1) and never leaves this machine. Note
|
||||
// the columns are bare algorithm names now (`sha256`), not the removed
|
||||
// record-field names (`cid_sha2-256`).
|
||||
const indexLine = await (await hashComputer.getHashIndexManager()).getCidForFileAsync('./src/tests/test.txt');
|
||||
delete indexLine['mtime'];
|
||||
expect(indexLine).toEqual({
|
||||
"cid_sha1": 'baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra',
|
||||
"cid_sha2-256": 'baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua',
|
||||
"cid_md5": 'bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa',
|
||||
"cid_sha3-256": 'baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq',
|
||||
"cid_crc32": "bagzafmqcathsbnrk",
|
||||
"cid_sha3-384": 'baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga',
|
||||
"path": "test.txt",
|
||||
"size": "499",
|
||||
...EXPECTED,
|
||||
path: "test.txt",
|
||||
size: String(FIXTURE_BYTES),
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
it('a second pass reuses the cache and recomputes nothing', async () => {
|
||||
const hashComputer = new HashComputerIndexCache("./tmp/index.csv", ALGOS);
|
||||
const metadata = {};
|
||||
await hashComputer.computeMissingHash('./src/tests/test.txt', metadata);
|
||||
// Same digests, recovered from the index rather than by re-hashing.
|
||||
expectAllDigests((metadata as { cids?: string[] }).cids);
|
||||
});
|
||||
});
|
||||
|
||||
@@ -4,13 +4,10 @@ export const FileMetaDataName = [
|
||||
"byteSize",
|
||||
]
|
||||
|
||||
// Sibling CIDs are stored as the bare-CID key-set `cids/<cid>` (see HashMeta),
|
||||
// not as per-algorithm cid_* fields. The single member prefix is `cids`.
|
||||
export const hashMetaDataName = [
|
||||
"cid_crc32",
|
||||
"cid_md5",
|
||||
"cid_sha1",
|
||||
"cid_sha2-256",
|
||||
"cid_sha3-256",
|
||||
"cid_sha3_384",
|
||||
"cids",
|
||||
];
|
||||
|
||||
export const AnimeMetaData = [
|
||||
|
||||
@@ -1,32 +1,23 @@
|
||||
/**
|
||||
* HashMeta - Content identification hashes
|
||||
* HashMeta - Content identification hashes.
|
||||
*
|
||||
* Primary Global ID: cid_midhash256 (SHA-256 hash of middle 1MB + file size)
|
||||
* - Format: CID v1 with custom multicodec 0x1000
|
||||
* - Example: bafkr4ih5kapbjzqvmj7jxr... (base32 CID encoding)
|
||||
* Sibling CIDs are NOT stored as per-algorithm fields anymore. Every digest
|
||||
* of the file (sha2-256, sha1, md5, crc32, sha3-*, midhash256, btih-v2, …)
|
||||
* lives in a single bare-CID key-set on the record:
|
||||
*
|
||||
* Legacy IDs: cid_sha2-256, cid_sha1 (computed optionally for compatibility)
|
||||
* cids/<cidV1> = "true"
|
||||
*
|
||||
* Migration Note: cid_midhash256 is the new primary identifier replacing full SHA-256.
|
||||
* Old hashes remain for backward compatibility and external system integration.
|
||||
* All CIDs now use proper CID v1 format (IPFS-compatible structure).
|
||||
* A CIDv1 is self-describing — its algorithm is the multicodec — so there is
|
||||
* no per-algorithm field name and no `<algo>:` prefix. The canonical /
|
||||
* network CID is derived by rank from this set on read, never stored. The
|
||||
* midhash stays the file's address (hashId). See METADATA_KEYS.md §2/§14.13.
|
||||
*
|
||||
* Because the members are dynamic keys (`cids/<cid>`), they can't be typed as
|
||||
* named fields here; consumers read them by prefix-scanning the flat record
|
||||
* and recover the algorithm from each CID's multicodec.
|
||||
*/
|
||||
export interface HashMeta {
|
||||
cid_crc32?: string; //0x0132 CRC32 (SFV)
|
||||
cid_md5?: string; // 0xd5 md5
|
||||
cid_sha1?: string; // 0x11 sha1
|
||||
"cid_sha2-256"?: string;//0x12 sha2-256
|
||||
"cid_sha3-256"?: string; // 0x16 sha3-256
|
||||
"cid_sha3_384"?: string; // 0x16 sha3-256
|
||||
cid_midhash256?: string; // 0x1000 midhash256 (CID v1 format with custom multicodec)
|
||||
}
|
||||
|
||||
export const HashMetaFields = [
|
||||
"cid_crc32",
|
||||
"cid_md5",
|
||||
"cid_sha1",
|
||||
"cid_sha2-256",
|
||||
"cid_sha3-256",
|
||||
"cid_sha3_384",
|
||||
"cid_midhash256",
|
||||
];
|
||||
// Intentionally empty: sibling CIDs are the `cids/<cid>` key-set (above),
|
||||
// not typed fields. Kept as a named type so existing `extends HashMeta`
|
||||
// composition sites stay valid.
|
||||
}
|
||||
@@ -1,2 +1,16 @@
|
||||
export type SimpleFileType = 'audio' | 'video' | 'document' | 'archive' | 'other' | 'undefined';
|
||||
|
||||
/**
|
||||
* The `fileType` vocabulary — METADATA_KEYS.md §1. Kept in lockstep with
|
||||
* `filename-tool`'s `SimpleFileType`, which is the classifier that produces
|
||||
* these values; this copy is the wire/type-level mirror.
|
||||
*/
|
||||
export type SimpleFileType =
|
||||
'audio'
|
||||
| 'video'
|
||||
| 'image'
|
||||
| 'document'
|
||||
| 'archive'
|
||||
| 'subtitle'
|
||||
| 'torrent'
|
||||
| 'card'
|
||||
| 'other'
|
||||
| 'undefined';
|
||||
|
||||
Reference in New Issue
Block a user