Compare commits

...
37 Commits
Author SHA1 Message Date
Worph ebc894fa42 deleting a cids/ member also drops its reverse alias
Build and Push Docker Image / build (push) Failing after 14s
2026-09-16 17:10:00 +00:00
Worph 515ec04369 bugfix and dashboard improvement
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 19:04:23 +02:00
Worph 0403a6f3bc replace file discovery by udp discovery
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 11:24:46 +02:00
Worph 355c666183 chnage service discovery and identity 2026-09-13 19:48:52 +02:00
Worph 0cccceccd9 align with metadata standard
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 15:57:30 +02:00
Worph 40b553016c cid vector
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 14:36:50 +02:00
Worph 4894810fd4 new file tuple API
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 12:01:33 +02:00
Worph ac21315255 improved cid resolution
Build and Push Docker Image / build (push) Failing after 13s
2026-09-12 21:37:38 +02:00
Worph ec1b227631 improved search 2026-09-12 16:37:51 +02:00
Worph 8f4e08b6f1 rework domain field
Build and Push Docker Image / build (push) Failing after 13s
2026-09-02 00:07:02 +02:00
Worph 0c4e3a2227 align to metadata keys doc
Build and Push Docker Image / build (push) Failing after 13s
2026-08-30 19:52:36 +02:00
Worph 1f41ea0166 bugfix scroll
Build and Push Docker Image / build (push) Failing after 14s
2026-08-25 18:09:44 +02:00
Worph 5efdd2c07d improved identity management 2026-08-25 17:48:23 +02:00
Worph 6ae87a2cdb audio support widening 2026-08-21 18:23:20 +02:00
Worph 604e718cdc add identity management
Build and Push Docker Image / build (push) Failing after 13s
2026-08-06 11:03:50 +02:00
Worph 0bbc656263 add identity page
Build and Push Docker Image / build (push) Failing after 13s
2026-08-05 23:37:12 +02:00
Worph a98734171c bugfix indexer field
Build and Push Docker Image / build (push) Failing after 13s
2026-08-03 19:44:22 +02:00
Worph a61057cd81 optimisation on assets retreive 2026-08-03 18:42:05 +02:00
Worph 55d08e77ed improvement on user management
Build and Push Docker Image / build (push) Failing after 14s
2026-08-02 21:35:39 +02:00
Worph 1f084b817c improve identity handling
Build and Push Docker Image / build (push) Failing after 14s
2026-08-01 23:31:49 +02:00
Worph 80050cd2a8 update vector with card locator
Build and Push Docker Image / build (push) Failing after 13s
2026-07-26 18:49:27 +02:00
Worph 75dda1e7d6 new NZB cid support
Build and Push Docker Image / build (push) Failing after 13s
2026-07-25 21:30:46 +02:00
Worph 76169920d6 improve meta core user management
Build and Push Docker Image / build (push) Failing after 13s
2026-07-18 17:30:17 +02:00
Worph df6a2d78f8 bug fix vector
Build and Push Docker Image / build (push) Failing after 14s
2026-07-14 16:02:26 +02:00
Worph 0ca03ac95a deprecate old cids fields 2026-07-13 22:02:29 +02:00
Worph 1798821c53 update metacore default 2026-07-12 14:47:21 +02:00
Worph d268bb83e6 search optimisation
Build and Push Docker Image / build (push) Failing after 13s
2026-07-10 22:56:59 +02:00
Worph 075d6f9bcb cleanup logs 2026-07-10 13:25:16 +02:00
Worph f1cd94b17e bugfix advertise url
Build and Push Docker Image / build (push) Failing after 13s
2026-06-14 18:47:28 +02:00
Pierre 52f5b5cedc multi identity support for metacore 2026-06-06 18:44:52 +02:00
Pierre f8a10091b8 remove in metadata cid_* to put in the cids/ set 2026-05-31 11:41:32 +02:00
Pierre 94ad0c240b bugfix and new internal API 2026-05-25 23:47:30 +02:00
Pierre 415595c1c6 new meta-core api, now redis is not exposed as an "API" 2026-05-24 17:43:17 +02:00
Pierre aa45f4f32e improve editor and add type detection 2026-05-14 23:31:07 +02:00
Pierre 3d65e7e951 doc improvement 2026-05-10 19:47:05 +02:00
Pierre bd833b7be4 improved kv metacore 2026-04-27 10:31:11 +02:00
Pierre 98da535335 improve editor for KV 2026-04-26 15:28:50 +02:00
159 changed files with 23565 additions and 7559 deletions
+6
View File
@@ -0,0 +1,6 @@
# FIX CRLF always when developer has not set
# Linux/Mac: git config --global core.autocrlf input
# Windows: git config --global core.autocrlf true
# Auto detect text files and perform LF normalization
* text=auto
* eol=lf
+2
View File
@@ -23,3 +23,5 @@ go.work
dashboard/node_modules/*
dist/*
dashboard/dist/*
editor/dist/
editor/node_modules/
+18 -9
View File
@@ -7,7 +7,7 @@
# - meta-core Go binary (HTTP API, Mount API, Watcher)
# - nginx (dashboard, WebDAV, API proxy)
# - Redis (managed by supervisord on leader)
# - rclone (remote mount daemon)
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
# - supervisord (process management - runs on leader only)
#
# Architecture:
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
LABEL org.opencontainers.image.licenses=MIT
# Install runtime dependencies
# Pinned rclone release. Alpine's package can lag the latest by several
# minor versions and we depend on VFS cache fixes shipped post-1.65.
ARG RCLONE_VERSION=1.73.5
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
# all remote mounts go through rclone now.
RUN apk add --no-cache \
# Core services
redis \
nginx \
nginx-mod-http-dav-ext \
supervisor \
rclone \
fuse3 \
# Mount utilities
nfs-utils \
cifs-utils \
# Tools
bash \
curl \
python3 \
ca-certificates \
apache2-utils \
# Process utilities
procps
# Install rclone from the official binary release (pinned).
RUN set -eux; \
apk add --no-cache --virtual .rclone-deps unzip; \
cd /tmp; \
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
unzip -q rclone.zip; \
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
apk del .rclone-deps; \
/usr/local/bin/rclone version | head -2
# Copy Go binary from builder
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
RUN chmod +x /usr/local/bin/meta-core
+14
View File
@@ -64,10 +64,24 @@ clean:
rm -f coverage.out coverage.html
# Build Docker image
#
# We tag THREE names:
# - meta-core:$(VERSION) — the canonical versioned tag
# - meta-core:latest — convenience for `docker run meta-core`
# - meta-core:local — the tag every sub-stack docker-compose.yml uses
# (meta-share federation, meta-gateway dev stacks).
# Without this, `make docker` produces an image
# the dev stacks can't see by name, and previously
# -tagged stale `:local` images linger, silently
# breaking downstream API fields (e.g. /urls
# missing webdavUrlInternal → meta-gateway's
# poster auto-store fails). See README.md →
# Quick start → Build for the audit recipe.
docker:
@echo "Building Docker image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) .
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local
# Run locally (for development)
run: build
+317 -427
View File
@@ -1,520 +1,410 @@
# meta-core
A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services.
Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
## Overview
meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul.
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
is not exposed outside the container — all metadata I/O is mediated by the
HTTP API.
### Key Capabilities
### Key capabilities
| Feature | Description |
|---------|-------------|
| **Leader Election** | flock-based leader election (Go binary only runs as leader) |
| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash |
| **HTTP API** | Language-agnostic REST interface for metadata and service discovery |
| **Service Discovery** | File-based registry with heartbeats and stale detection |
| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations |
| **File Watching** | Directory scanning, MidHash256 computation, event dispatch |
| **Mount Management** | SMB/rclone configuration, health monitoring |
| **WebDAV Server** | File access with nginx proxy_cache |
| **Event Publishing** | Redis keyspace notifications to meta:events stream |
| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, and file access by CID. |
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
| **Service discovery** | meta-discovery v1: UDP multicast announce/listen on `239.255.77.1:9399`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
### Design Characteristics
### Design characteristics
- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes)
- **Automatic failover** - Lock releases on process death, followers detect and re-elect
- **Thread-safe** - RWMutex protection on all shared state
- **Graceful shutdown** - Signal handling with reverse startup order
- **Static binary** - CGO disabled, single binary deployment
- **Single-leader** — flock on a shared volume; no external consensus.
- **HTTP-only externally** — Redis is bound to the container, never published.
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
on-disk Redis layout predates the current build (alpha clean-wipe policy;
see `docs/uuid-rooted-metadata.md`).
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
image with `redis-server`, `rclone`, nginx, and supervisord.
## Architecture
meta-core runs as a **standalone container** that other services connect to via HTTP API:
```
┌───────────────────────────────────────────────────────────────────────┐
│ meta-core container (standalone) │
│ │
│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │
│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │
│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │
│ │ │ │ │ │ │ │ /health, /services │ │
│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │
│ │
│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │
└──────────────────────────────────────────────┬────────────────────────┘
│
┌──────────────────────────────────────┴──────────────────────────┐
│ /meta-core (shared volume) │
│ ├── locks/kv-leader.info (leader URLs) │
│ ├── db/redis/ (Redis data) │
│ ├── services/ (hostname-based .json) │
│ └── mounts/ (mount configurations) │
└─────────────────────────────────────────────────────────────────┘
▲ ▲ ▲
┌────────────┘ │ └────────────┐
│ │ │
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
│ meta-sort │ │ meta-fuse │ │ meta-stremio │
│ (reads info, │ │ (reads info, │ │ (reads info, │
│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │
└───────────────┘ └───────────────┘ └───────────────┘
┌──────────────────────────────────────────────────────────────────────────┐
│ metacore-app (standalone container) │
│ │
│ supervisord │
│ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
│ │ writes kv-leader.info, exec supervisord) │
│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
│ ├─ nginx (TLS termination, proxy_cache for /webdav) │
│ ├─ rclone rcd (mount daemon, RC API) │
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
└──────────────────────────────────────────────────────────────────────────┘
│
┌──────────────────────┴──────────────────────┐
│ /meta-core (shared volume) │
│ locks/kv-leader.{lock,info} │
│ db/redis/ │
│ services/*.json │
│ mounts/{mounts.json, errors/} │
│ watchers.json │
│ cache/ (nginx proxy_cache) │
└─────────────────────────────────────────────┘
▲
┌──────────────────────────────┼────────────────────────────────┐
│ │ │
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
│ meta-sort │ │ meta-fuse │ │ meta-stremio│
│ │ │ │ │ │
│ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │
│ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│
└─────────────┘ └─────────────┘ └─────────────┘
```
## Quick Start
## Quick start
### Build
```bash
# Build binary
make build
# Build Docker image
make docker
# Run tests
make test
make build # build static binary to bin/meta-core
make docker # build container image as meta-core:1.0.0 + :latest + :local
make test # run Go unit tests
```
### Run
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
`:local` tag is the one every sub-stack docker-compose.yml references
(meta-share 10-peer federation, meta-gateway dev stacks); without it the
sub-stacks can't see a freshly-built image by name.
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
> image, but **already-running containers stay on the old image SHA** —
> `docker compose up -d` won't recreate them unless the compose file
> itself changed. After a meta-core source change, force-recreate the
> downstream containers so they pick up the new binary:
>
> ```bash
> make docker
> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
> up -d --force-recreate metagateway-generic-core
> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
> up -d --force-recreate metagateway-torznab-core
> docker compose -f packages/meta-share/dev/docker-compose.yml \
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
> ```
>
> The symptom of a missed recreate is missing API fields on `/urls`
> (e.g. `webdavUrlInternal` absent → meta-gateway's
> `store_poster_for_record` silently fails and torznab posters never
> land). Audit recipe:
>
> ```bash
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
> ```
### Dev container
In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`:
```bash
# With environment variables
META_CORE_PATH=/meta-core \
FILES_PATH=/files \
SERVICE_NAME=meta-sort \
META_CORE_HTTP_PORT=9000 \
./bin/meta-core
cd dev && ./scripts/start.sh
```
### Docker
```bash
docker build -t meta-core .
docker run -v meta-core:/meta-core -v files:/files meta-core
```
The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend
HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
## Configuration
| Environment Variable | Default | Description |
|---------------------|---------|-------------|
| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db |
| `FILES_PATH` | `/files` | Shared volume for media files |
| `SERVICE_NAME` | `meta-core` | Service identifier |
| `SERVICE_VERSION` | `1.0.0` | Service version |
| `API_PORT` | `8180` | Main service HTTP port |
| `BASE_URL` | - | Stable service URL |
| `REDIS_PORT` | `6379` | Redis port (leader only) |
| `META_CORE_HTTP_PORT` | `9000` | HTTP API port |
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval |
| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold |
| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning |
Environment variables consumed by `internal/config/config.go`:
## API Reference
| Variable | Default | Description |
|---|---|---|
| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
| `FILES_PATH` | `/files` | Files volume root. |
| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
| `REDIS_PORT` | `6379` | Local Redis port. |
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
### Health & Status
Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
retry interval (`ELECTION_RETRY_SECS`, default `5`).
## API reference
All endpoints below are served by `internal/api/server.go`. Examples assume
the in-container Go port `:9000`; from the host use
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
(debug-direct).
### Health, leader, URLs
```bash
# Health check
curl http://localhost:9000/health
# {"status":"ok","redis":true,"timestamp":"..."}
# Detailed status
curl http://localhost:9000/status
# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...}
# Current leader
curl http://localhost:9000/leader
# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...}
GET /health # storage + role
GET /status # version, uptime, etc.
GET /leader # current leader info (hostname, PID, timestamps)
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
GET /neighbors # neighbours heard over UDP (meta-discovery v1)
GET /api/neighbors # same, under /api
GET /api/stats # dashboard counters (see below)
```
### Metadata Operations
`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
are **gone** — the registration-and-heartbeat registry they served was replaced
by meta-discovery v1 (`/neighbors`). nginx answers an unknown non-`/api` path
with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
than a 404; that is what made the dashboard fail with
`SyntaxError: Unexpected token '<'` until 1.0.21.
#### `GET /api/stats`
Every headline counter in one request, split by cost:
```jsonc
{
"records": 88709, // SCARD file:__index__ — metadata roots, file or not
"redisKeys": 3115447, // DBSIZE — the flat per-field keys dominate
"redisMemory": "588.74M",
"identities": 3, // signing accounts on disk
"files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep
"udlUsers": 5, // accounts holding User Data Layer records
"sweptAt": 1789402401970,
"sweeping": false
}
```
The first four are O(1) commands and are live on every call. `files` and
`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
box), so they are served from a single-flight cache refreshed in the background
with a 60s TTL — the walk is **never** on the request path, because the
dashboard polls and ten open tabs must not cost ten Redis walks. Prefer this
over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
across records that share a file.
### Metadata — primary surface
These are the routes other services use. `{hash}` may be a UUID root or
any registered CID alias; CID aliases resolve to the underlying UUID
before reads/writes (`internal/storage/cid_resolution.go`).
```bash
# List all file hashes
curl http://localhost:9000/meta
# {"hashIds":["midhash256:abc123",...],"count":42}
# Get metadata for a file
curl http://localhost:9000/meta/{hash}
# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}}
# Update metadata
curl -X PUT http://localhost:9000/meta/{hash} \
-H "Content-Type: application/json" \
-d '{"title":"New Title"}'
# Delete metadata
curl -X DELETE http://localhost:9000/meta/{hash}
GET /meta # list of root IDs
GET /meta/{hash} # full document (flat keys → nested JSON)
PUT /meta/{hash} # replace document
PATCH /meta/{hash} # merge into document
DELETE /meta/{hash} # delete root + all CID aliases
GET /meta/{hash}/{key...} # single property (key may contain "/")
PUT /meta/{hash}/{key...} # set single property
DELETE /meta/{hash}/{key...} # delete single property
POST /meta/{hash}/_add/{key...} # add value to set-valued property
```
### Data Operations
CID-addressed access (public; auth-bypassed in the Caddy perimeter):
```bash
# Get file path
curl http://localhost:9000/data/{hash}/path
# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true}
# Check if file exists
curl -I http://localhost:9000/data/{hash}
# HTTP/1.1 200 OK (or 404)
GET /api/meta/{cid} # resolve CID → metadata document
GET /api/file/{cid}/info # CID resolution metadata
GET /file/{cid} # serve file bytes (range-aware)
POST /file/cid # compute CID for an uploaded file
HEAD /data/{hash} # existence + size
GET /data/{hash}/path # absolute path on /files
```
### File Operations (by CID)
(There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or
the `/file/{cid}` reverse-lookup path.)
### Editor / KV browser surface
```bash
# Serve file by CID (looks up poster/backdrop CIDs in metadata)
curl http://localhost:9000/file/{cid} --output poster.jpg
# Returns raw file bytes with appropriate Content-Type header
GET /api/metadata/hash-ids
GET /api/files/tuples
GET /api/metadata/list
POST /api/metadata/search
POST /api/metadata/batch
POST /api/metadata/clear
GET /api/metadata/{hashId}
PUT /api/metadata/{hashId}
DELETE /api/metadata/{hashId}
GET /api/metadata/{hashId}/property
PUT /api/metadata/{hashId}/property
# Example: fetch a poster image
curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4
# Content-Type: image/jpeg
# [binary image data]
GET /api/kv/info
GET /api/kv/keys
GET /api/kv/tree
GET /api/kv/search
GET /api/kv/find
GET /api/kv/value
PUT /api/kv/value
DELETE /api/kv/value
GET /api/kv/key/{key...}
GET /api/schema
POST /api/schema/rescan
GET /api/snapshot/export
POST /api/snapshot/import
POST /api/snapshot/wipe
```
The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval.
### SSE event streams
### Service Discovery
The HTTP-mirror of the Redis Streams. External services never read Redis
directly — they subscribe here with `Last-Event-ID` for resume. See
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
contract (one event per stream entry, opaque IDs, heartbeats, gap
signalling).
```bash
# List all services
curl http://localhost:9000/services
# {"services":[{"name":"meta-sort",...}],"count":3}
# Get specific service
curl http://localhost:9000/services/meta-sort
# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...}
GET /api/events/files # file:events stream (watcher events)
GET /api/events/meta # meta:events stream (metadata mutations)
```
### Mount Management
### Mounts
```bash
# List all mounts
curl http://localhost:9000/api/mounts/list
# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]}
# Add a new mount
curl -X POST http://localhost:9000/api/mounts/add \
-H "Content-Type: application/json" \
-d '{"type":"smb","path":"/files/corn/nas","config":{...}}'
# Remove a mount
curl -X DELETE http://localhost:9000/api/mounts/smb-nas
GET /api/mounts
POST /api/mounts
GET /api/mounts/{id}
PUT /api/mounts/{id} # also PATCH
DELETE /api/mounts/{id}
POST /api/mounts/{id}/mount
POST /api/mounts/{id}/unmount
POST /api/mounts/{id}/safe-unmount
POST /api/mounts/{id}/scan
GET /api/mounts/rclone/remotes
```
### File Watching
### Watchers
```bash
# Get scan status
curl http://localhost:9000/api/scan/status
# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847}
# Trigger rescan
curl -X POST http://localhost:9000/api/scan/trigger
# Stream file events (SSE)
curl http://localhost:9000/api/events/stream
# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."}
GET /api/watchers
POST /api/watchers
GET /api/watchers/{id}
PUT /api/watchers/{id}
DELETE /api/watchers/{id}
POST /api/watchers/{id}/scan
POST /api/watchers/{id}/reset
POST /api/watchers/scan-all
POST /api/watchers/reset-all
```
### KV Browser
The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
return a deprecation pointer at the `/api/watchers/*` routes
(`internal/watcher/handlers.go:32-33`).
### Admin / WebDAV
```bash
# Storage statistics
curl http://localhost:9000/api/kv/info
# {"totalKeys":50000,"memoryUsed":"256MB",...}
# List keys with cursor pagination
curl "http://localhost:9000/api/kv/keys?cursor=0&count=100"
# Get specific key value
curl http://localhost:9000/api/kv/key/file:bafkr.../title
POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
POST /api/admin/sweep-literature-echoes[?apply=true]
# strip query echoes off literature cards: anilistid on site cards,
# workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
# Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
GET/PUT/DELETE /webdav/... # mounted on /files
```
### Metadata Editor
No `/metrics` endpoint is exposed; observability is via container logs and
the `/health` / `/status` JSON.
```bash
# Get all hash IDs
curl http://localhost:9000/api/metadata/hash-ids
# {"hashIds":["bafkr...","bafks..."]}
## Leader gate
# Paginated metadata listing
curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50"
Leader election is a bash flock loop in `docker/leader-election.sh`, not a
Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
script writes the leader API URL to `kv-leader.info` and `exec`s
supervisord. When the process tree dies, the kernel releases the lock and
any standby instance can take over. There is no follower path inside the
Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
# Search metadata
curl -X POST http://localhost:9000/api/metadata/search \
-H "Content-Type: application/json" \
-d '{"query":"inception","fields":["title"]}'
# Batch update
curl -X POST http://localhost:9000/api/metadata/batch \
-H "Content-Type: application/json" \
-d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}'
# Clear all metadata
curl -X POST http://localhost:9000/api/metadata/clear
```
### WebDAV Access
```bash
# List directory (returns JSON)
curl http://localhost:9000/webdav/
# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}]
# Get file
curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv
# Upload file
curl -X PUT http://localhost:9000/webdav/plugin/output.txt \
-H "Content-Type: application/octet-stream" \
--data-binary @output.txt
# Create directory
curl -X MKCOL http://localhost:9000/webdav/newdir/
# Delete file/directory
curl -X DELETE http://localhost:9000/webdav/oldfile.txt
```
## Leader Election
meta-core uses POSIX file locking (flock) for distributed leader election:
1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock`
2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info`
3. Losers become **followers**, read leader info, connect to leader's Redis
4. Lock automatically releases when process dies (no stale locks)
### Election Flow
```
Instance 1 Instance 2 Instance 3
│ │ │
├── Try flock ──────────┼───────────────────────┤
│ │ │
[WIN] [LOSE] [LOSE]
LEADER FOLLOWER FOLLOWER
│ │ │
Start Redis Read leader.info Read leader.info
│ │ │
Write leader.info Connect to Redis Connect to Redis
│ │ │
[Health loop 5s] [Health loop 5s] [Health loop 5s]
- Update timestamp - Re-read leader - Re-read leader
- Check Redis alive - Detect changes - Detect changes
```
### Role Callbacks
The election system provides callbacks for role transitions:
```go
election.OnBecomeLeader(func(info LeaderLockInfo) {
// Start Redis, initialize resources
})
election.OnBecomeFollower(func(info LeaderLockInfo) {
// Connect to leader's Redis
})
election.OnLeaderLost(func() {
// Handle leader failure, attempt re-election
})
```
### Lock Info Format
The `kv-leader.info` file contains a plain text API URL:
`kv-leader.info` is a single plaintext line containing the API base URL:
```
http://10.0.1.50:9000
```
Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.).
Other services read this file, then call `GET /urls` for full discovery.
## Redis Management
The dev compose runs a single `metacore-app`; the flock primitive is
retained as a correctness anchor (one process owns Redis and the
writable state under `/meta-core`), not as live failover.
The leader is responsible for spawning and managing the Redis server.
## Metadata storage shape (one-line summary)
### Redis Configuration
Roots are opaque UUIDv7 strings; every property is a separate Redis
STRING at `file:<uuid>/<property>`; every known CID for a file is a
reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
The authoritative registry of field semantics and value formats lives in
the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
```bash
redis-server --port 6379 --bind 0.0.0.0 \
--dir /meta-core/db/redis \
--appendonly yes --appendfilename appendonly.aof \
--dbfilename dump.rdb --save 60 1 \
--loglevel warning
```
### Persistence Strategy
| Method | Configuration | Purpose |
|--------|--------------|---------|
| **AOF** | `appendonly yes` | Write-ahead log for durability |
| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed |
### Failure Handling
- **Redis crash**: Auto-detected via health check, automatically restarted
- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs
- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed
## Metadata Storage
meta-core uses a flat key-value schema in Redis for storing file metadata.
### Storage Format
```
Key: /file/{hashId}/{property}
Value: string
Examples:
/file/midhash256:abc123/title → "Movie Title"
/file/midhash256:abc123/year → "2024"
/file/midhash256:abc123/genres → "action,drama"
/file/midhash256:abc123/filePath → "movies/Movie.mkv"
```
### Storage Client Features
| Feature | Details |
|---------|---------|
| Connection Pool | 10 connections |
| Timeouts | 5s dial, 30s read/write |
| Batch Operations | Pipelined writes for performance |
| Key Scanning | SCAN-based iteration (non-blocking) |
| CID Lookup | Find file paths by poster/backdrop CID |
## Service Discovery
Services register by writing JSON to `/meta-core/services/{name}.json`.
### Registration Format
```json
{
"name": "meta-sort",
"version": "2.0.0",
"api": "http://10.0.1.50:8180",
"status": "running",
"pid": 12345,
"hostname": "meta-sort-dev",
"startedAt": "2024-01-01T00:00:00Z",
"lastHeartbeat": "2024-01-01T00:01:00Z",
"capabilities": ["meta-core"],
"endpoints": {
"health": "http://10.0.1.50:9000/health",
"meta": "http://10.0.1.50:9000/meta"
}
}
```
### Heartbeat & Stale Detection
| Parameter | Default | Purpose |
|-----------|---------|---------|
| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` |
| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this |
Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file.
## Development
```bash
# Format code
make fmt
# Run linter
make lint
# Run tests with coverage
make test-cover
# Clean build artifacts
make clean
```
## Internal Packages
## Internal packages
| Package | Purpose |
|---------|---------|
| `cmd/meta-core` | Main entry point, startup/shutdown orchestration |
| `internal/config` | Environment-based configuration with defaults |
| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) |
| `internal/storage` | Redis client wrapper with flat key-value operations |
| `internal/discovery` | Service registration, heartbeat loop, discovery |
| `internal/api` | HTTP server, router, and all endpoint handlers |
| `internal/events` | Redis keyspace notification publisher to meta:events |
| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch |
| `internal/watchers` | Polling-based watcher configuration management |
| `internal/mounts` | SMB/rclone mount configuration and health monitoring |
| `internal/webdav` | WebDAV protocol handler with caching integration |
| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
| `internal/config` | Env-driven configuration, path helpers. |
| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
| `internal/discovery` | Service registry + dead-service cleaner. |
| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
| `internal/snapshot` | Snapshot export / import / wipe. |
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. |
| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). |
### Startup Sequence
### Startup sequence
```
1. Load configuration from environment
2. Create storage client (Redis wrapper)
3. Initialize leader election
4. Register role transition callbacks
5. Start election (acquire lock or become follower)
6. Start service discovery (register + heartbeat loop)
7. Initialize dead service cleaner
8. Initialize file watcher (if enabled)
9. Initialize mount manager
10. Start HTTP API server (WebDAV caching handled by nginx)
11. Wait for SIGINT/SIGTERM
12. Shutdown in reverse order
```
## Integration
### As Sidecar in Docker
Add to existing service's supervisord config:
```ini
[program:meta-core]
command=/usr/local/bin/meta-core
priority=10
autostart=true
autorestart=true
```
### Client Usage (TypeScript)
```typescript
const response = await fetch('http://localhost:9000/meta/' + hashId);
const { metadata } = await response.json();
```
### Client Usage (Python)
```python
import requests
response = requests.get(f'http://localhost:9000/meta/{hash_id}')
metadata = response.json()['metadata']
2. Build LeaderInfoProvider (no election — the bash gate already won)
3. Connect to local Redis (retry up to 30× / 1s)
4. EnsureSchemaVersion — abort if the existing Redis layout is stale
5. Start service discovery + dead-service cleaner
6. Construct API server (initialises watcher, mounts, WebDAV)
7. Start API server; on first connect, start MetaPublisher + schema Indexer
and republish existing metadata onto meta:events
8. Wait for SIGINT/SIGTERM, shutdown in reverse order
```
## Dependencies
| Dependency | Version | Purpose |
|------------|---------|---------|
| `github.com/gorilla/mux` | v1.8.1 | HTTP router |
| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client |
| `golang.org/x/net` | v0.20.0 | WebDAV support |
| `github.com/google/uuid` | v1.6.0 | UUID generation |
| Go stdlib | - | context, encoding/json, net, os, sync, syscall |
| Dependency | Purpose |
|------------|---------|
| `github.com/gorilla/mux` | HTTP router |
| `github.com/redis/go-redis/v9` | Redis client |
| `github.com/google/uuid` | UUIDv7 minting |
| `golang.org/x/net` | WebDAV support |
| Go stdlib | context, net/http, os, sync, syscall |
### Runtime Requirements
- **Go 1.21+** for building
- **Redis** binary available in PATH (leader spawns it)
- **Shared filesystem** accessible by all instances (for flock)
Runtime requirements: Go 1.21+ at build time; the container ships Redis,
rclone, nginx, and supervisord alongside the binary.
## License
+64 -16
View File
@@ -10,8 +10,8 @@ import (
"github.com/metazla/meta-core/internal/api"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/storage"
)
@@ -51,20 +51,56 @@ func main() {
}
log.Println("[meta-core] Connected to Redis")
// Create and start service discovery
disc := discovery.NewService(cfg)
if err := disc.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
// the existing Redis store predates the UUID-rooted schema, so legacy
// data can't silently leak into the new code paths. See
// docs/uuid-rooted-metadata.md.
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
log.Fatalf("[meta-core] %v", err)
}
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
// Create and start dead service cleaner
cleaner := discovery.NewCleaner(cfg)
if err := cleaner.Start(); err != nil {
log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
// UDP service discovery (meta-discovery v1). Runs alongside the file
// registry above for now — nothing consumes the wire yet, so this cannot
// regress the existing path. The announce payload is rebuilt from
// leaderProvider on every tick, which is what keeps it and GET /urls from
// drifting apart.
var mesh *meshdisco.Node
if cfg.EnableUDPDiscovery {
hostname, _ := os.Hostname()
mesh = meshdisco.New(meshdisco.Config{
Name: cfg.ServiceName,
Instance: hostname,
Role: meshdisco.RoleCore,
Version: Version,
Group: cfg.DiscoveryGroup,
Port: cfg.DiscoveryPort,
Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
Payload: func() (string, string, *meshdisco.URLs) {
info := leaderProvider.LeaderInfo()
if info == nil {
return "", "starting", nil
}
return coreBaseURL(cfg), "running", &meshdisco.URLs{
Hostname: info.Hostname,
BaseUrl: info.BaseUrl,
ApiUrl: info.ApiUrl,
WebdavUrl: info.WebdavUrl,
WebdavUrlInternal: info.WebdavUrlInternal,
}
},
})
if err := mesh.Start(); err != nil {
log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
mesh = nil
}
} else {
log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
}
// Create and start API server
apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
apiServer := api.NewServer(cfg, leaderProvider, storageClient)
apiServer.SetMeshNode(mesh)
if err := apiServer.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start API server: %v", err)
}
@@ -81,12 +117,10 @@ func main() {
log.Printf("[meta-core] Error stopping API server: %v", err)
}
if err := cleaner.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service cleaner: %v", err)
}
if err := disc.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service discovery: %v", err)
if mesh != nil {
if err := mesh.Stop(); err != nil {
log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
}
}
if err := storageClient.Close(); err != nil {
@@ -103,3 +137,17 @@ func waitForShutdown() {
<-sigChan
}
// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
// the three-way fallback the file registry used (discovery/service.go):
// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
// the container IP.
func coreBaseURL(cfg *config.Config) string {
if cfg.PublicURL != "" {
return cfg.PublicURL
}
if cfg.BaseURL != "" {
return cfg.BaseURL
}
return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
}
+1 -1
View File
@@ -21,7 +21,7 @@
min-height: 100vh;
}
</style>
<script type="module" crossorigin src="/assets/index-DJlbGMsa.js"></script>
<script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
</head>
<body>
+10
View File
@@ -8,6 +8,7 @@
"name": "@meta-core/dashboard",
"version": "1.0.0",
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
@@ -744,6 +745,15 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@noble/secp256k1": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.1.0.tgz",
"integrity": "sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==",
"license": "MIT",
"funding": {
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/@remix-run/router": {
"version": "1.23.2",
"resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz",
+3 -2
View File
@@ -9,10 +9,11 @@
"preview": "vite preview"
},
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.20.0",
"iso-639-3": "^3.0.1"
"react-router-dom": "^6.20.0"
},
"devDependencies": {
"@types/react": "^18.2.38",
+8
View File
@@ -3,6 +3,7 @@ import Dashboard from './pages/Dashboard';
import Mounts from './pages/Mounts';
import Watchers from './pages/Watchers';
import Files from './pages/Files';
import Identity from './pages/Identity';
import ServiceNav from './components/ServiceNav';
const navStyle: React.CSSProperties = {
@@ -74,6 +75,12 @@ export default function App() {
>
Files
</NavLink>
<NavLink
to="/identity"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
>
Identity
</NavLink>
<NavLink
to="/editor"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
@@ -89,6 +96,7 @@ export default function App() {
<Route path="/mounts" element={<Mounts />} />
<Route path="/watchers" element={<Watchers />} />
<Route path="/files" element={<Files />} />
<Route path="/identity" element={<Identity />} />
<Route path="/editor" element={<EditorPage />} />
</Routes>
</div>
+44 -146
View File
@@ -1,156 +1,54 @@
import { useState, useEffect } from 'react';
import { useEffect } from 'react';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
// of defining the custom element.
import '../meta-service-menu.js';
interface ServiceInfo {
name: string;
hostname: string;
baseUrl: string;
status: string;
lastHeartbeat: string;
}
/**
* ServiceNav — the neighbouring meta-* services, as a burger menu.
*
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
* a framework-free custom element shared by every meta-* UI (see
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
* that existed in four near-identical copies and polled meta-core's
* /api/services; the element polls this service's own /api/neighbors instead,
* so the nav still renders when meta-core is down.
*
* The element lives in shadow DOM and cannot see this app's CSS variables, so
* the four --mm-nav-* tokens below map our palette onto it.
*/
interface ServicesResponse {
services: ServiceInfo[];
current: string;
}
const serviceIcons: Record<string, string> = {
'meta-core': '🗄️',
'meta-sort': '📁',
'meta-fuse': '🗂️',
'meta-stremio': '🎬',
'meta-orbit': '🌐',
'default': '📦'
};
function formatServiceName(name: string): string {
return name.split('-').map(word =>
word.charAt(0).toUpperCase() + word.slice(1)
).join(' ');
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace JSX {
interface IntrinsicElements {
'meta-service-menu': React.DetailedHTMLProps<
React.HTMLAttributes<HTMLElement>,
HTMLElement
> & { current?: string; endpoint?: string; label?: string };
}
}
}
function ServiceNav() {
const [services, setServices] = useState<ServiceInfo[]>([]);
const [loading, setLoading] = useState(true);
const currentService = 'meta-core';
useEffect(() => {
// Import above is for the side effect of defining the element; this
// keeps bundlers from tree-shaking it away in production builds.
}, []);
useEffect(() => {
const fetchServices = async () => {
try {
const response = await fetch('/services');
if (response.ok) {
const data: ServicesResponse = await response.json();
setServices(data.services || []);
}
} catch (error) {
console.error('Failed to fetch services:', error);
} finally {
setLoading(false);
}
};
fetchServices();
const interval = setInterval(fetchServices, 30000);
return () => clearInterval(interval);
}, []);
if (loading) return null;
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
if (sortedServices.length === 0) return null;
return (
<nav className="services-nav">
<span className="services-nav-label">Services:</span>
<div className="services-nav-items">
{sortedServices.map(service => {
const icon = serviceIcons[service.name] || serviceIcons.default;
const isActive = service.name === currentService;
return (
<a
key={service.name}
href={isActive ? '#' : service.baseUrl}
className={`service-link${isActive ? ' active' : ''}`}
onClick={(e) => {
e.preventDefault();
if (!isActive) {
window.location.href = service.baseUrl;
return (
<>
<meta-service-menu current="meta-core" />
<style>{`
meta-service-menu {
--mm-nav-fg: var(--text-primary, #e0e0e0);
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
--mm-nav-accent: var(--accent-primary, #4ecdc4);
}
}}
>
<span className="service-icon">{icon}</span>
<span>{formatServiceName(service.name)}</span>
<span className="service-status"></span>
</a>
);
})}
</div>
<style>{`
.services-nav {
display: flex;
align-items: center;
gap: 0.5rem;
}
.services-nav-label {
color: #888;
font-size: 0.8rem;
white-space: nowrap;
}
.services-nav-items {
display: flex;
gap: 0.35rem;
}
.service-link {
display: inline-flex;
align-items: center;
gap: 0.3rem;
padding: 0.35rem 0.6rem;
border-radius: 6px;
text-decoration: none;
font-size: 0.8rem;
font-weight: 500;
transition: all 0.2s;
color: #ccc;
background: #0f3460;
border: 1px solid #1a4a7a;
}
.service-link:hover {
background: #1a4a7a;
border-color: rgba(78, 205, 196, 0.5);
text-decoration: none;
color: #fff;
}
.service-link.active {
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
border-color: rgba(78, 205, 196, 0.5);
color: #fff;
}
.service-link .service-icon {
font-size: 0.9rem;
}
.service-link .service-status {
width: 5px;
height: 5px;
border-radius: 50%;
background: #4ecdc4;
}
@media (max-width: 800px) {
.services-nav-label {
display: none;
}
}
`}</style>
</nav>
);
`}</style>
</>
);
}
export default ServiceNav;
+28
View File
@@ -0,0 +1,28 @@
// JSON fetch that fails with a sentence instead of a parser error.
//
// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
// path it does not have a proxy rule for answers 200 text/html with the SPA
// shell. A caller that does `res.ok && res.json()` on such a path gets
// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
// no clue which request produced it — which is exactly how a route deleted
// from the Go router (`/services`, removed in favour of /api/neighbors) sat
// broken on the dashboard.
//
// So: check the content type, and name the URL in every error.
export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
const res = await fetch(url, init);
const ct = res.headers.get('content-type') || '';
if (!res.ok) {
throw new Error(`${url} → HTTP ${res.status}`);
}
if (!ct.includes('application/json')) {
// The SPA fallback is the overwhelmingly likely cause; say so.
throw new Error(
`${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
`missing and nginx served the dashboard shell instead)`
);
}
return (await res.json()) as T;
}
+127
View File
@@ -0,0 +1,127 @@
// Client-side proof of possession for account operations.
//
// meta-core stores every private key in plaintext, so it can sign as anybody —
// which means a signature it produced proves nothing about who asked. The only
// signature that carries authority is one made where the key actually is, and
// for a browser-driven admin console that is here. meta-core enforces the other
// half by refusing to sign anything in the challenge domain
// (identity.ChallengeDomain), so this path cannot be short-circuited.
//
// The secret key entered on this page is used for exactly one signature and is
// never sent anywhere: `/api/identity/challenge` receives only the uid, and the
// request that follows carries only the challenge and its signature.
//
// Same scheme, same curve and same encodings as meta-watch's sign-in
// (packages/meta-watch/ui/js/auth.js) — deliberately, so one implementation on
// the server verifies both.
import * as secp from '@noble/secp256k1';
export type IdentityAction = 'reveal' | 'delete';
function hexToBytes(hex: string): Uint8Array | null {
let h = hex.trim();
if (h.startsWith('0x') || h.startsWith('0X')) h = h.slice(2);
if (h.length !== 64 || /[^0-9a-fA-F]/.test(h)) return null;
const out = new Uint8Array(32);
for (let i = 0; i < 32; i++) out[i] = parseInt(h.slice(i * 2, i * 2 + 2), 16);
return out;
}
function bytesToBase64(bytes: Uint8Array): string {
let s = '';
for (const b of bytes) s += String.fromCharCode(b);
return btoa(s);
}
const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
/// base58btc, the multibase "z" alphabet. Spelled out rather than pulled in:
/// the alternative is a second dependency for thirty lines of arithmetic.
function base58Encode(bytes: Uint8Array): string {
const digits: number[] = [0];
for (const byte of bytes) {
let carry = byte;
for (let i = 0; i < digits.length; i++) {
carry += digits[i] << 8;
digits[i] = carry % 58;
carry = (carry / 58) | 0;
}
while (carry) {
digits.push(carry % 58);
carry = (carry / 58) | 0;
}
}
// Leading zero bytes encode as leading '1's rather than being dropped.
let out = '';
for (let i = 0; i < bytes.length && bytes[i] === 0; i++) out += B58[0];
for (let i = digits.length - 1; i >= 0; i--) out += B58[digits[i]];
return out;
}
/// The uid a secret key belongs to: "z" + base58btc(33-byte compressed pubkey).
/// Derived locally so a mistyped or wrong-account key is caught before any
/// request goes out. Mirrors `identityFromPrivKey` in meta-core's identity.go.
export function uidFromSecretKey(secretKeyHex: string): string | null {
const raw = hexToBytes(secretKeyHex);
if (!raw) return null;
try {
return 'z' + base58Encode(secp.getPublicKey(raw, true));
} catch {
return null; // not a valid scalar
}
}
/// Whether this page can sign at all.
///
/// noble's signAsync takes its sha256 from crypto.subtle, which browsers expose
/// only in a secure context. The dashboard behind Caddy is https and fine; the
/// debug-direct port (18083) is plain http, where every signature would fail
/// with a cryptic error unless we say so first.
export function canSign(): boolean {
return Boolean(globalThis.crypto?.subtle);
}
export const INSECURE_CONTEXT_MESSAGE =
'This page cannot sign over a plain-http connection — browsers only expose the ' +
'needed crypto on https (or localhost). Reach the dashboard through Caddy ' +
'(https://metacore-dev.localhost:8083) rather than the debug-direct port.';
/// Ask meta-core for a challenge, sign it locally, and hand back the proof.
///
/// Throws with a message fit to show the user. The caller passes the proof
/// straight into the reveal/delete request body.
export async function proveOwnership(
uid: string,
action: IdentityAction,
secretKeyHex: string
): Promise<{ challenge: string; signature: string }> {
if (!canSign()) throw new Error(INSECURE_CONTEXT_MESSAGE);
const raw = hexToBytes(secretKeyHex);
if (!raw) {
throw new Error('That is not a valid secret key: expected 64 hex characters.');
}
const derived = uidFromSecretKey(secretKeyHex);
if (!derived) throw new Error('That is not a valid secp256k1 secret key.');
if (derived !== uid) {
// Caught here rather than by the server, so the message can say which
// account the key actually is — the useful half of the answer.
throw new Error(`That key belongs to a different account (${derived}), not this one.`);
}
const cRes = await fetch('/api/identity/challenge', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ uid, action }),
});
const cBody = await cRes.json().catch(() => ({}));
if (!cRes.ok || !cBody.challenge) {
throw new Error(cBody.message ?? `Could not start verification (HTTP ${cRes.status}).`);
}
// signAsync prehashes with sha256 and returns a 64-byte compact (r‖s)
// signature — noble v3 dropped DER. meta-core's identity.Verify accepts both.
const sig = await secp.signAsync(new TextEncoder().encode(cBody.challenge), raw);
return { challenge: cBody.challenge, signature: bytesToBase64(sig) };
}
+303
View File
@@ -0,0 +1,303 @@
/**
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
*
* Replaces the eight divergent "services bar" implementations (four copies of
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
* ones) with a single framework-free custom element. It drops into the React
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
* as an ES module.
*
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
* make a fetch to a neighbour a dead end. Links are plain navigations.
*
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
* `scripts/check-mirrors.sh`. Edit one → edit all.
*
* Shadow DOM is not optional here. The eight UIs use four incompatible token
* vocabularies, and two of them define *different colours under the same
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
* #161b22 in meta-dup). meta-search and meta-share are also the only
* light-mode-capable UIs, so a hardcoded dark palette would break them.
* The element therefore styles itself from four tokens with dark defaults:
*
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
*
* Each host maps its own variables onto those, e.g.
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
*
* Attributes:
* current name of the host service, so it can be marked and not linked
* endpoint override the default "/api/neighbors"
* label button tooltip (default "Services")
*
* Property:
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
* translate internal hostnames into browser-reachable ones
* (meta-gateway does this for its dev/prod split).
*/
const ICONS = {
"meta-core": "⚙️",
"meta-sort": "📁",
"meta-fuse": "🗂️",
"meta-stremio": "🎬",
"meta-dup": "🔍",
"meta-search": "🔎",
"meta-share": "🔗",
"meta-gateway": "🌉",
"meta-watch": "📺",
"meta-listen": "🎧",
"meta-read": "📖",
};
const DEFAULT_ICON = "📦";
const POLL_MS = 30000;
function titleCase(name) {
return String(name || "")
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ");
}
const TEMPLATE = `
<style>
:host {
--_fg: var(--mm-nav-fg, #e8eaed);
--_bg: var(--mm-nav-bg, #1a1a2e);
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
--_accent: var(--mm-nav-accent, #4ecdc4);
position: relative;
display: inline-block;
font-family: inherit;
font-size: 14px;
line-height: 1.4;
}
button.trigger {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.4rem;
width: 2.1rem;
height: 2.1rem;
padding: 0;
cursor: pointer;
color: var(--_fg);
background: transparent;
border: 1px solid var(--_border);
border-radius: 8px;
font: inherit;
transition: border-color 0.15s, background 0.15s;
}
button.trigger:hover { border-color: var(--_accent); }
button.trigger:focus-visible {
outline: 2px solid var(--_accent);
outline-offset: 2px;
}
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
.bars { display: block; width: 15px; }
.bars span {
display: block;
height: 2px;
margin: 3px 0;
border-radius: 2px;
background: currentColor;
}
.panel {
position: absolute;
top: calc(100% + 6px);
right: 0;
z-index: 1000;
min-width: 13rem;
max-height: 70vh;
overflow-y: auto;
padding: 0.3rem;
background: var(--_bg);
border: 1px solid var(--_border);
border-radius: 10px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
}
.panel[hidden] { display: none; }
a.item, .item {
display: flex;
align-items: center;
gap: 0.55rem;
padding: 0.45rem 0.6rem;
border-radius: 6px;
color: var(--_fg);
text-decoration: none;
white-space: nowrap;
}
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
.item.dead { opacity: 0.55; cursor: not-allowed; }
.icon { font-size: 1rem; line-height: 1; }
.dot {
width: 6px;
height: 6px;
margin-left: auto;
border-radius: 50%;
background: var(--_accent);
}
.empty {
padding: 0.6rem;
opacity: 0.7;
font-size: 0.85em;
white-space: nowrap;
}
/* Respect a host that opts out of motion. */
@media (prefers-reduced-motion: reduce) {
button.trigger { transition: none; }
}
</style>
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
</button>
<div class="panel" role="menu" hidden></div>
`;
class MetaServiceMenu extends HTMLElement {
constructor() {
super();
this.attachShadow({ mode: "open" });
this.shadowRoot.innerHTML = TEMPLATE;
this._btn = this.shadowRoot.querySelector("button.trigger");
this._panel = this.shadowRoot.querySelector(".panel");
this._neighbors = [];
this._timer = null;
this._open = false;
this._onDocClick = (e) => {
if (!this._open) return;
if (e.composedPath().includes(this)) return;
this._setOpen(false);
};
this._onKey = (e) => {
if (e.key === "Escape" && this._open) {
this._setOpen(false);
this._btn.focus();
}
};
}
connectedCallback() {
this._btn.title = this.getAttribute("label") || "Services";
this._btn.setAttribute("aria-label", this._btn.title);
this._btn.addEventListener("click", () => this._setOpen(!this._open));
document.addEventListener("click", this._onDocClick);
document.addEventListener("keydown", this._onKey);
this.refresh();
this._timer = setInterval(() => this.refresh(), POLL_MS);
}
disconnectedCallback() {
if (this._timer) clearInterval(this._timer);
this._timer = null;
document.removeEventListener("click", this._onDocClick);
document.removeEventListener("keydown", this._onKey);
}
get endpoint() {
return this.getAttribute("endpoint") || "/api/neighbors";
}
get current() {
return this.getAttribute("current") || "";
}
async refresh() {
try {
const res = await fetch(this.endpoint, {
headers: { Accept: "application/json" },
});
if (!res.ok) throw new Error(String(res.status));
const data = await res.json();
// `services` is the legacy alias some backends still emit.
this._neighbors = data.neighbors || data.services || [];
} catch {
// Silent: the menu is navigation, not a health indicator. Keep whatever
// we last had rather than blanking the list on one failed poll.
}
this._render();
}
_setOpen(open) {
this._open = open;
this._panel.hidden = !open;
this._btn.setAttribute("aria-expanded", String(open));
if (open) this.refresh();
}
_render() {
const panel = this._panel;
panel.textContent = "";
const seen = new Set();
const rows = [];
for (const n of this._neighbors) {
const name = n.name || "";
if (!name || seen.has(name)) continue;
seen.add(name);
rows.push(n);
}
// Always show the host itself, even before its own echo comes back.
if (this.current && !seen.has(this.current)) {
rows.push({ name: this.current, baseUrl: "" });
}
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
if (rows.length === 0) {
const empty = document.createElement("div");
empty.className = "empty";
empty.textContent = "No services found";
panel.appendChild(empty);
return;
}
for (const n of rows) {
const isActive = n.name === this.current;
let url = n.baseUrl || "";
if (typeof this.rewriteUrl === "function") {
try {
url = this.rewriteUrl(n.name, url, n) || "";
} catch {
/* a bad hook must not blank the menu */
}
}
// A neighbour with no reachable URL is shown but not linked — better
// than a link that goes nowhere.
const el = document.createElement(isActive || !url ? "span" : "a");
el.className =
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
if (el.tagName === "A") {
el.href = url;
el.setAttribute("role", "menuitem");
}
if (!isActive && !url) el.title = "No reachable URL announced";
const icon = document.createElement("span");
icon.className = "icon";
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
const label = document.createElement("span");
label.textContent = titleCase(n.name);
el.append(icon, label);
if (n.status === "running" || isActive) {
const dot = document.createElement("span");
dot.className = "dot";
dot.title = n.status || "running";
el.appendChild(dot);
}
panel.appendChild(el);
}
}
}
if (!customElements.get("meta-service-menu")) {
customElements.define("meta-service-menu", MetaServiceMenu);
}
export default MetaServiceMenu;
+171 -71
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from 'react';
import { getJSON } from '../lib/api';
interface HealthStatus {
status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
timestamp: string;
}
interface FileStats {
count: number;
totalSize: number;
}
interface Stats {
records: number;
redisKeys: number;
redisMemory: string;
identities: number;
files: FileStats | null;
udlUsers: number | null;
sweptAt: number;
sweeping: boolean;
}
interface WatcherStatus {
id: string;
path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
count: number;
}
interface ServiceInfo {
name: string;
api: string;
capabilities: string[];
timestamp: number;
}
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
const gridStyle: React.CSSProperties = {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
// 200px, not 220: five tiles must fit one row inside the 1200px container
// (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
gap: '1rem',
marginBottom: '1rem',
};
const statStyle: React.CSSProperties = {
fontSize: '2rem',
fontWeight: 'bold',
color: '#4ade80',
lineHeight: 1.2,
};
const labelStyle: React.CSSProperties = {
marginBottom: '0.75rem',
color: '#888',
fontSize: '0.85rem',
textTransform: 'uppercase',
letterSpacing: '0.04em',
};
const noteStyle: React.CSSProperties = {
color: '#6b7a99',
fontSize: '0.8rem',
marginTop: '0.4rem',
};
const numberFmt = new Intl.NumberFormat();
function formatCount(n: number | null | undefined): string {
return n === null || n === undefined ? '—' : numberFmt.format(n);
}
// Binary units: this counts bytes on disk, and every other tool the operator
// compares against (du, docker, the mount page) reports GiB.
function formatBytes(bytes: number | null | undefined): string {
if (bytes === null || bytes === undefined) return '—';
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
let value = bytes;
let unit = 0;
while (value >= 1024 && unit < units.length - 1) {
value /= 1024;
unit++;
}
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
}
function Tile({
label,
value,
note,
pending,
}: {
label: string;
value: string;
note?: string;
pending?: boolean;
}) {
return (
<div style={cardStyle}>
<div style={labelStyle}>{label}</div>
<div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
{value}
</div>
{note && <div style={noteStyle}>{note}</div>}
</div>
);
}
export default function Dashboard() {
const [health, setHealth] = useState<HealthStatus | null>(null);
const [stats, setStats] = useState<Stats | null>(null);
const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
const [services, setServices] = useState<ServiceInfo[]>([]);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
let cancelled = false;
const fetchData = async () => {
try {
// Fetch health
const healthRes = await fetch('/health');
if (healthRes.ok) {
setHealth(await healthRes.json());
}
// Settled, not all: one dead endpoint must not blank every tile.
const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
getJSON<HealthStatus>('/health'),
getJSON<Stats>('/api/stats'),
getJSON<WatchersResponse>('/api/watchers'),
]);
if (cancelled) return;
// Fetch watchers status
const watchersRes = await fetch('/api/watchers');
if (watchersRes.ok) {
setWatchersData(await watchersRes.json());
}
if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
if (statsRes.status === 'fulfilled') setStats(statsRes.value);
if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
// Fetch services
const servicesRes = await fetch('/services');
if (servicesRes.ok) {
const data = await servicesRes.json();
setServices(data.services || []);
}
setError(null);
} catch (err) {
setError(String(err));
}
const failures = [healthRes, statsRes, watchersRes]
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
.map((r) => String(r.reason?.message ?? r.reason));
setError(failures.length ? failures.join(' · ') : null);
};
fetchData();
const interval = setInterval(fetchData, 5000);
return () => clearInterval(interval);
return () => {
cancelled = true;
clearInterval(interval);
};
}, []);
const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
}
};
// Aggregate watcher stats
const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
const lastScan =
watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
// Records we hold metadata for but have no local file: gateway/cid-rooted
// entries, plus anything whose file went away. The interesting number on a
// client box, where it should be most of them.
const fileless =
stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
const sweepPending = !!stats && !stats.files;
const sweptNote = (() => {
if (!stats) return undefined;
if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
if (stats.sweeping) return 'refreshing…';
if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
return undefined;
})();
return (
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
<h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
{error && (
<div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
Error: {error}
{error}
</div>
)}
<div style={gridStyle}>
{/* Health Card */}
<Tile
label="Files discovered"
value={formatCount(stats?.files?.count)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Library size"
value={formatBytes(stats?.files?.totalSize)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Metadata entries"
value={formatCount(stats?.records)}
note={
fileless === null
? 'records in the index'
: `${formatCount(fileless)} with no local file`
}
/>
<Tile
label="Redis keys"
value={formatCount(stats?.redisKeys)}
note={stats?.redisMemory ? `${stats.redisMemory} in use` : undefined}
/>
<Tile
label="Identities"
value={formatCount(stats?.identities)}
note={
stats?.udlUsers !== null && stats?.udlUsers !== undefined
? `${formatCount(stats.udlUsers)} with user data`
: 'signing accounts'
}
/>
</div>
<div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
<div style={labelStyle}>Service health</div>
{health ? (
<>
<div style={statStyle}>
{health.status === 'ok' ? 'Healthy' : health.status}
</div>
<p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
<p style={noteStyle}>
Redis: {health.redis ? 'Connected' : 'Disconnected'}
</p>
</>
) : (
<p>Loading...</p>
<p style={noteStyle}>Loading…</p>
)}
</div>
{/* Watchers Status Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
<div style={labelStyle}>File watchers</div>
{watchersData ? (
<>
<div style={statStyle}>{totalFiles}</div>
<p>Files discovered</p>
<p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
<p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
<div style={statStyle}>
{activeWatchers} / {watchersData.count}
</div>
<p style={noteStyle}>active watchers</p>
<p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
{lastScan > 0 && (
<p>Last scan: {new Date(lastScan).toLocaleString()}</p>
<p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
)}
<button
onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
</button>
</>
) : (
<p>Loading...</p>
)}
</div>
{/* Services Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
<div style={statStyle}>{services.length}</div>
<p>Services registered</p>
{services.length > 0 && (
<ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
{services.map((svc) => (
<li key={svc.name} style={{ marginBottom: '0.5rem' }}>
<strong>{svc.name}</strong>
<br />
<span style={{ color: '#888', fontSize: '0.9rem' }}>
{svc.capabilities?.join(', ') || 'No capabilities'}
</span>
</li>
))}
</ul>
<p style={noteStyle}>Loading…</p>
)}
</div>
</div>
+859
View File
@@ -0,0 +1,859 @@
import { useEffect, useRef, useState } from 'react';
import {
INSECURE_CONTEXT_MESSAGE,
canSign,
proveOwnership,
uidFromSecretKey,
type IdentityAction,
} from '../lib/sign';
// Admin console for meta-core's account keystore.
//
// meta-core stores one secp256k1 keypair per file under
// {META_CORE_PATH}/identity/accounts/<uid>.json and has NO notion of a current
// user — which account is "you" is decided by whichever client signs in
// (meta-watch). So this page lists accounts; it never speaks of an active one.
//
// The private key is returned by exactly two calls: POST /api/identity/generate
// (once, at creation) and POST /api/identity/reveal (on demand, for an account
// that already exists). Both land in the same exclusive SecretKeyPanel.
//
// Reveal and remove both require the account's own secret key, entered here and
// used to sign a one-shot challenge locally — the key itself is never sent. So
// this page can enumerate and create, but it holds no authority over an
// existing account that its operator does not already have the key for. There
// is deliberately no admin override: on this node the private key *is* the
// account.
interface Account {
uid: string;
curve: string;
createdAt: number;
}
// What one account holds in the User Data Layer. Absent from
// /api/udl/users/stats means zero, not unknown.
interface Stats {
records: number;
cids: number;
keys: number;
}
// A secret on screen. Carries no createdAt: /api/identity/reveal answers with
// {uid, curve, privateKeyHex} only.
interface SecretKeyView {
uid: string;
privateKeyHex: string;
kind: 'generated' | 'revealed';
}
// Which row has an open confirmation, and for what. One discriminated value
// rather than a pair of ids, so at most one confirm panel can ever be open.
interface Pending {
uid: string;
kind: 'remove' | 'reveal';
}
// Both confirmable operations map to a server-side action name, and the
// challenge is bound to it: a challenge minted to reveal a key cannot delete
// the account. Keeping the mapping in one place stops the two drifting.
const ACTION_FOR: Record<Pending['kind'], IdentityAction> = {
reveal: 'reveal',
remove: 'delete',
};
// meta-watch's sentinel cell for profile records: display name and avatar are
// about the *user*, not a content file, so they live at (uid, "self", ...).
// meta-core stores them opaquely and knows nothing of the convention.
const PROFILE_CID = 'self';
const PROFILE_NAME_KEY = 'profile:name';
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
padding: '1.5rem',
marginBottom: '1rem',
};
const buttonStyle: React.CSSProperties = {
padding: '0.5rem 1rem',
background: '#0f3460',
color: '#fff',
border: 'none',
borderRadius: '4px',
cursor: 'pointer',
marginRight: '0.5rem',
};
const dangerButtonStyle: React.CSSProperties = {
...buttonStyle,
background: '#7f1d1d',
};
const inputStyle: React.CSSProperties = {
width: '100%',
padding: '0.5rem',
marginBottom: '0.5rem',
background: '#1a1a2e',
border: '1px solid #0f3460',
borderRadius: '4px',
color: '#fff',
fontFamily: 'monospace',
};
const monoStyle: React.CSSProperties = {
fontFamily: 'monospace',
fontSize: '0.85rem',
wordBreak: 'break-all',
background: '#0a0a18',
border: '1px solid #0f3460',
padding: '0.75rem',
borderRadius: '4px',
};
const dangerPanelStyle: React.CSSProperties = {
background: '#1f0a0a',
border: '1px solid #7f1d1d',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginTop: '0.75rem',
};
const errorTextStyle: React.CSSProperties = {
color: '#f87171',
background: '#1f0a0a',
padding: '0.5rem 0.75rem',
borderRadius: '4px',
};
function formatCreatedAt(unixSec?: number): string {
if (!unixSec) return '';
try {
return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC');
} catch {
return String(unixSec);
}
}
// Read the JSON body of a non-2xx response if present, fall back to status text.
async function errorMessage(res: Response, fallback: string): Promise<string> {
try {
const j = await res.json();
if (j && (j.message || j.error)) return j.message ?? j.error;
} catch {
// not JSON
}
return `${fallback} (HTTP ${res.status})`;
}
// Copy text, falling back to selecting the element that displays it.
//
// The fallback is load-bearing, not politeness: the dashboard is served over
// plain HTTP, where navigator.clipboard is undefined — without it the copy
// buttons do nothing at all on a real deployment.
async function copyToClipboard(text: string, fallbackElementId: string): Promise<void> {
try {
await navigator.clipboard.writeText(text);
} catch {
const sel = window.getSelection();
const el = document.getElementById(fallbackElementId);
if (sel && el) {
const range = document.createRange();
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
}
}
}
async function fetchAccounts(): Promise<Account[]> {
const res = await fetch('/api/identity/accounts');
if (!res.ok) throw new Error(await errorMessage(res, 'Failed to load accounts'));
const data = (await res.json()) as { accounts?: Account[] };
// Backend order is CreatedAt ascending then uid (identity.List) — stable, so
// render it as given rather than re-sorting here.
return data.accounts ?? [];
}
// Resolve an account's display name from the User Data Layer, or null.
//
// Never throws and never reports an error upward: a name is decoration on top
// of the uid, and the account list must render identically whether Redis is up
// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
// — meta-core stores the plaintext beside the signed record and verifies
// nothing — so the uid stays on screen as the real identity, and the length cap
// bounds what a hostile writer can do to the layout.
async function fetchDisplayName(uid: string): Promise<string | null> {
const url =
`/api/udl/record?uid=${encodeURIComponent(uid)}` +
`&cid=${encodeURIComponent(PROFILE_CID)}` +
`&key=${encodeURIComponent(PROFILE_NAME_KEY)}`;
try {
const res = await fetch(url);
if (!res.ok) return null; // 503 when storage is down, 500, …
const data = (await res.json()) as { exists?: boolean; value?: unknown };
// Key off `value`, not `exists`: clearing a name writes a tombstone, which
// leaves the cell existing with no plaintext. Private-tier cells look the
// same, which is exactly right — there is nothing to show for either.
if (typeof data.value !== 'string') return null;
const name = data.value.trim();
return name ? name.slice(0, 64) : null;
} catch {
return null;
}
}
// How much each account holds. One call for the whole page, not one per row:
// meta-core answers it from a keyspace SCAN (there is no per-user cid index),
// so per-row calls would walk the keyspace once per account.
//
// Never throws, for the same reason fetchDisplayName does not: the counts are
// context on top of the uid, and the list must render identically when Redis is
// down — accounts are file-backed, their data is not.
//
// Returns null on failure rather than an empty map. The difference is
// load-bearing: an empty map renders as "no stored data" against every account,
// which inside a delete confirmation is a false statement about what is at
// stake. Unknown has to look like unknown.
async function fetchStats(): Promise<Record<string, Stats> | null> {
try {
const res = await fetch('/api/udl/users/stats');
if (!res.ok) return null;
const data = (await res.json()) as { users?: Record<string, Stats> };
return data.users ?? {};
} catch {
return null;
}
}
// The secret-key box shared by the reveal and remove confirmations.
//
// Separate from the surrounding panel so both operations ask the same way, and
// so the key lives in this component's state rather than the page's — it is
// discarded with the panel instead of lingering in a parent that outlives it.
function SecretKeyConfirm({
label,
actionLabel,
busy,
onConfirm,
onCancel,
}: {
label: string;
actionLabel: string;
busy: boolean;
onConfirm: (secretKeyHex: string) => void;
onCancel: () => void;
}) {
const [secret, setSecret] = useState('');
const signable = canSign();
// Derived locally, purely to tell the operator they have the wrong key before
// they hit the button. The server would refuse either way.
const derived = secret.trim() ? uidFromSecretKey(secret) : null;
const mismatch = Boolean(secret.trim() && derived && derived !== label);
return (
<form
onSubmit={(e) => {
e.preventDefault();
onConfirm(secret.trim());
}}
>
<p style={{ color: '#fecaca', margin: '0 0 0.5rem' }}>
Paste this account's secret key to prove it is yours. It is used to sign a one-time
challenge <em>in this browser</em> and is never sent to the server.
</p>
{!signable && <p style={errorTextStyle}>{INSECURE_CONTEXT_MESSAGE}</p>}
<input
style={inputStyle}
type="password"
autoComplete="off"
placeholder="64 hex characters"
value={secret}
onChange={(e) => setSecret(e.target.value)}
disabled={busy || !signable}
autoFocus
/>
{mismatch && (
<p style={{ ...errorTextStyle, marginTop: 0 }}>
That key belongs to a different account ({derived}).
</p>
)}
<button
type="submit"
style={dangerButtonStyle}
disabled={busy || !signable || !secret.trim() || mismatch}
>
{busy ? 'Verifying…' : actionLabel}
</button>
<button type="button" style={buttonStyle} onClick={onCancel} disabled={busy}>
Cancel
</button>
</form>
);
}
// The one place a private key is ever displayed. Rendered exclusively (the
// account list is hidden while it is open) so the "save it now" contract of a
// freshly generated key is not competing with anything, and so the hardcoded
// element id below stays unique.
function SecretKeyPanel({ data, onDismiss }: { data: SecretKeyView; onDismiss: () => void }) {
const [copied, setCopied] = useState(false);
const copy = async () => {
await copyToClipboard(data.privateKeyHex, 'mm-privkey-box');
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
<div
style={{
background: '#7f1d1d',
color: '#fee2e2',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginBottom: '1rem',
}}
>
{data.kind === 'generated' ? (
<>
<strong>This private key is shown ONCE.</strong> Save it now in a password manager.
You can ask for it again later with <em>Reveal private key</em>, but only for as long
as this account exists on this node — delete it and the key is gone for good.
</>
) : (
<>
<strong>This is the live private key for an existing account.</strong> Anyone holding
it can sign as this uid forever, on any machine. There is no revocation and no
rotation. Do not paste it into chat, tickets or screenshots.
</>
)}
</div>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
<div style={monoStyle}>{data.uid}</div>
<p style={{ color: '#9ca3af', margin: '1rem 0 0.5rem' }}>privateKeyHex (secret):</p>
<div id="mm-privkey-box" style={{ ...monoStyle, background: '#1f0a0a', borderColor: '#7f1d1d' }}>
{data.privateKeyHex}
</div>
<div style={{ marginTop: '1rem' }}>
<button style={buttonStyle} onClick={copy}>
{copied ? 'Copied' : 'Copy private key'}
</button>
<button style={dangerButtonStyle} onClick={onDismiss}>
{data.kind === 'generated' ? 'I have saved it' : 'Done — hide it'}
</button>
</div>
</div>
);
}
// Create / import. Always available, not just on an empty keystore: this is an
// admin console, so adding an account to a populated node has to be possible.
function NewAccountPanel({
onGenerated,
onImported,
}: {
onGenerated: (s: SecretKeyView) => void;
onImported: () => void;
}) {
const [busy, setBusy] = useState(false);
const [importing, setImporting] = useState(false);
const [importHex, setImportHex] = useState('');
const [error, setError] = useState<string | null>(null);
const [notice, setNotice] = useState<string | null>(null);
const generate = async () => {
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/generate', { method: 'POST' });
if (!res.ok) {
setError(await errorMessage(res, 'Generate failed'));
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
onGenerated({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'generated' });
} catch (e) {
setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
const importKey = async (e: React.FormEvent) => {
e.preventDefault();
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/import', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ privateKeyHex: importHex.trim() }),
});
if (!res.ok) {
setError(await errorMessage(res, 'Import failed'));
return;
}
// Import is idempotent — presenting the key IS the proof of ownership, so
// a key this node already holds is a success, not a conflict. `created`
// only decides the wording.
const data = (await res.json()) as { created?: boolean };
setNotice(
data.created
? 'Imported — a new account was added to this keystore.'
: 'That key was already in this keystore; nothing changed.'
);
setImportHex('');
setImporting(false);
onImported();
} catch (e) {
setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
return (
<div style={cardStyle}>
<h3 style={{ marginTop: 0 }}>Add an account</h3>
<p style={{ color: '#9ca3af', marginTop: 0 }}>
Generating mints a fresh secp256k1 keypair here. Importing adds a key created elsewhere —
it is also how an existing account is moved onto this node, so importing a key the
keystore already holds succeeds and changes nothing.
</p>
{error && <p style={errorTextStyle}>{error}</p>}
{notice && (
<p style={{ color: '#9ca3af', background: '#0a0a18', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
{notice}
</p>
)}
<div>
<button style={buttonStyle} disabled={busy} onClick={generate}>
{busy ? 'Working…' : 'Generate new keypair'}
</button>
<button
style={buttonStyle}
disabled={busy}
onClick={() => {
setImporting((x) => !x);
setError(null);
}}
>
{importing ? 'Cancel import' : 'Import existing private key'}
</button>
</div>
{importing && (
<form onSubmit={importKey} style={{ marginTop: '1rem' }}>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>
64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine.
</p>
<input
style={inputStyle}
placeholder="e.g. a3f1…"
value={importHex}
onChange={(e) => setImportHex(e.target.value)}
required
autoFocus
/>
<button type="submit" style={buttonStyle} disabled={busy || !importHex.trim()}>
{busy ? 'Importing…' : 'Import'}
</button>
</form>
)}
</div>
);
}
function AccountCard({
account,
name,
stats,
busy,
disabled,
pending,
error,
onAsk,
onCancel,
onReveal,
onRemove,
}: {
account: Account;
name?: string;
// undefined means "counted, and it holds nothing"; null means "could not be
// counted". The two must not read the same on a confirmation screen.
stats?: Stats | null;
busy: boolean;
disabled: boolean;
pending: Pending | null;
error: string | null;
onAsk: (p: Pending) => void;
onCancel: () => void;
onReveal: (uid: string, secretKeyHex: string) => void;
onRemove: (uid: string, secretKeyHex: string) => void;
}) {
const [copied, setCopied] = useState(false);
const uidBoxId = `mm-uid-${account.uid}`;
const label = name ?? account.uid;
const counted = stats !== null;
const records = stats?.records ?? 0;
const titles = stats?.cids ?? 0;
const copyUid = async () => {
await copyToClipboard(account.uid, uidBoxId);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={cardStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start', gap: '1rem' }}>
<div style={{ minWidth: 0, flex: 1 }}>
<h3 style={{ margin: '0 0 0.5rem', color: name ? '#fff' : '#888' }}>
{name ?? 'Unnamed account'}
</h3>
<div id={uidBoxId} style={monoStyle}>
{account.uid}
</div>
<p style={{ color: '#888', margin: '0.75rem 0 0', fontSize: '0.85rem' }}>
Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}
</p>
<p style={{ color: '#888', margin: '0.25rem 0 0', fontSize: '0.85rem' }}>
{/* An account that has never been used legitimately holds nothing, so
zero is a real answer — but only once the count actually ran. */}
{!counted
? 'Stored data: could not be counted (storage unavailable)'
: records === 0
? 'No stored data'
: `${records.toLocaleString()} stored ${records === 1 ? 'record' : 'records'}` +
` across ${titles.toLocaleString()} ${titles === 1 ? 'title' : 'titles'}`}
</p>
</div>
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem', flexShrink: 0 }}>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={copyUid} disabled={disabled}>
{copied ? 'Copied' : 'Copy uid'}
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'reveal' })}
disabled={disabled}
>
Reveal private key
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'remove' })}
disabled={disabled}
>
Remove
</button>
</div>
</div>
{error && <p style={{ ...errorTextStyle, marginBottom: 0 }}>{error}</p>}
{pending?.uid === account.uid && pending.kind === 'reveal' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Show the private key for <strong>{label}</strong>?
</p>
<p style={{ color: '#fecaca', marginTop: 0 }}>
The key <em>is</em> the account. Anyone who copies it can sign records as this uid on
any machine, forever — there is no way to revoke or rotate it afterwards. Because
proving ownership already requires the key, this only confirms that the copy this node
holds matches yours.
</p>
<SecretKeyConfirm
label={account.uid}
actionLabel="Show the private key"
busy={busy}
onConfirm={(secret) => onReveal(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
{pending?.uid === account.uid && pending.kind === 'remove' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Remove <strong>{label}</strong> from this node?
</p>
<ul style={{ color: '#fecaca', paddingLeft: '1.2rem', margin: '0 0 0.75rem' }}>
<li>
<strong>The key is gone for good.</strong> The uid <em>is</em> the public key, and
the private half exists only in this node's keystore. If nobody saved it, nothing —
not you, not meta-core — can ever sign as this uid again. Reveal and copy it first
if there is any doubt.
</li>
<li>
<strong>Nothing is retracted.</strong> Records already signed under this uid keep
verifying on every peer that replicated them. Deleting a key is not a recall.
</li>
<li>
<strong>Its data is destroyed, not orphaned.</strong>{' '}
{!counted
? 'This account\u2019s records could not be counted because storage is unavailable — ' +
'so meta-core will refuse this delete rather than remove the key and strand them.'
: records === 0
? 'This account holds no User Data Layer records, so there is nothing to purge.'
: `All ${records.toLocaleString()} User Data Layer ${
records === 1 ? 'record' : 'records'
} — likes, ratings, watch progress across ${titles.toLocaleString()} ${
titles === 1 ? 'title' : 'titles'
} — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
This cannot be undone and there is no backup.
</li>
<li>
<strong>This signs nobody out elsewhere.</strong> Any device that holds this key
still holds it, and can import it back here at any time.
</li>
</ul>
<SecretKeyConfirm
label={account.uid}
actionLabel="Remove this account and all its data"
busy={busy}
onConfirm={(secret) => onRemove(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
</div>
);
}
export default function Identity() {
const [accounts, setAccounts] = useState<Account[] | null>(null);
const [names, setNames] = useState<Record<string, string>>({});
const [stats, setStats] = useState<Record<string, Stats> | null>({});
const [notice, setNotice] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [fetchError, setFetchError] = useState<string | null>(null);
const [secret, setSecret] = useState<SecretKeyView | null>(null);
const [pending, setPending] = useState<Pending | null>(null);
const [busyUid, setBusyUid] = useState<string | null>(null);
const [rowError, setRowError] = useState<{ uid: string; message: string } | null>(null);
// Bumped per refresh so a slow response from a superseded load cannot
// overwrite a newer one. The page does not poll (unlike the rest of the
// dashboard) — a key list has no business changing under the cursor.
const reqGen = useRef(0);
const refresh = async () => {
const gen = ++reqGen.current;
setLoading(true);
setFetchError(null);
let list: Account[];
try {
list = await fetchAccounts();
} catch (e) {
if (gen !== reqGen.current) return;
setFetchError(e instanceof Error ? e.message : String(e));
setLoading(false);
return;
}
if (gen !== reqGen.current) return;
// Render the list before resolving names, so a down Redis delays nothing.
setAccounts(list);
setLoading(false);
// Names and counts both come after the list is on screen, and neither can
// fail the page: a down Redis costs decoration, not the account list.
const [settled, counts] = await Promise.all([
Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
fetchStats(),
]);
if (gen !== reqGen.current) return;
const next: Record<string, string> = {};
settled.forEach((r, i) => {
if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value;
});
// Replace rather than merge, so a removed account's name and counts go with
// it instead of lingering against a uid that no longer exists.
setNames(next);
setStats(counts);
};
useEffect(() => {
refresh();
}, []);
const reveal = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.reveal, secretKeyHex);
// POST with a body rather than ?uid=, so neither the target nor the
// signature reaches the proxy's access log. The handler accepts the uid
// either way; the proof must be in the body.
const res = await fetch('/api/identity/reveal', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Reveal failed') });
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
setPending(null);
setSecret({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'revealed' });
} catch (e) {
setRowError({ uid, message: `Reveal failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const remove = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
setNotice(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.remove, secretKeyHex);
// The uid is mandatory: the signature authorises one specific account, so
// the backend refuses an unqualified delete rather than guess a target.
const res = await fetch('/api/identity', {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Remove failed') });
return;
}
// Report what the purge actually removed. "Account removed" on its own
// would not distinguish a clean delete from one that stranded the data.
const data = (await res.json().catch(() => ({}))) as { purged?: Stats };
const purged = data.purged;
setNotice(
purged && purged.records > 0
? `Account removed. Purged ${purged.records.toLocaleString()} User Data Layer ` +
`${purged.records === 1 ? 'record' : 'records'} across ` +
`${purged.cids.toLocaleString()} ${purged.cids === 1 ? 'title' : 'titles'}.`
: 'Account removed. It held no User Data Layer records.'
);
setPending(null);
await refresh();
} catch (e) {
setRowError({ uid, message: `Remove failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const dismissSecret = () => {
setSecret(null);
refresh();
};
const ask = (p: Pending) => {
setRowError(null);
setNotice(null);
setPending(p);
};
const showList = !loading && !fetchError && !secret;
return (
// App.tsx's content area is `overflow: hidden`, so every page owns its own
// scrolling (Files.tsx does the same). Without this the account list is
// simply clipped at the viewport with no scrollbar — invisible on a node
// with two accounts, and a dead end on one with fifteen.
//
// The scroller is the outer element and the padding lives inside it, so the
// scrollbar sits at the edge of the content area rather than floating in
// 2rem of whitespace, and `margin: 0 auto` still centres the column.
<div style={{ height: '100%', overflowY: 'auto', minHeight: 0 }}>
<div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<h1 style={{ marginBottom: 0 }}>Identity</h1>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={refresh} disabled={loading}>
{loading ? 'Loading…' : 'Refresh'}
</button>
</div>
<p style={{ color: '#9ca3af' }}>
Every signing key in this node's keystore. meta-core has no "current user" — it stores
accounts, and which one is <em>you</em> is decided by whichever client signs in (e.g.
meta-watch). Keys live one JSON file per account under{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/&lt;uid&gt;.json</code>, and are used to sign
User Data Layer records (ratings, watch progress, "liked" flags).
</p>
<div style={{ ...dangerPanelStyle, marginTop: 0, marginBottom: '1rem' }}>
<p style={{ color: '#fecaca', margin: 0 }}>
<strong>The key is the only authority.</strong> Revealing or removing an account requires
that account's secret key, entered here and used to sign a one-time challenge in your
browser — there is no admin override, and holding this page is not one. Listing and
creating accounts are still open to anything that can reach this API, so the perimeter in
front of this port is what keeps the roster private.
</p>
<p style={{ color: '#fecaca', margin: '0.5rem 0 0' }}>
What this does <em>not</em> defend against: anyone with filesystem access to{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/</code> reads every private key directly.
The keys are stored in plaintext, matching the trust model of the rest of{' '}
<code>/meta-core</code>.
</p>
</div>
{notice && (
<div style={{ ...cardStyle, border: '1px solid #0f3460' }}>
<p style={{ color: '#9ca3af', margin: 0 }}>{notice}</p>
</div>
)}
{loading && <div style={cardStyle}>Loading…</div>}
{fetchError && (
<div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
<p style={{ color: '#f87171', margin: '0 0 1rem' }}>{fetchError}</p>
<button style={buttonStyle} onClick={refresh}>
Retry
</button>
</div>
)}
{!loading && !fetchError && secret && <SecretKeyPanel data={secret} onDismiss={dismissSecret} />}
{showList && (
<>
<NewAccountPanel onGenerated={(s) => setSecret(s)} onImported={refresh} />
<h2 style={{ fontSize: '1.1rem', color: '#9ca3af', margin: '1.5rem 0 1rem' }}>
Accounts ({accounts?.length ?? 0})
</h2>
{accounts && accounts.length === 0 && (
<div style={cardStyle}>
<p style={{ color: '#9ca3af', margin: 0 }}>
No accounts in this keystore. Accounts are normally created by a client when
someone signs up; use <em>Add an account</em> above if you are bootstrapping this
node.
</p>
</div>
)}
{accounts?.map((a) => (
<AccountCard
key={a.uid}
account={a}
name={names[a.uid]}
stats={stats === null ? null : stats[a.uid]}
busy={busyUid === a.uid}
disabled={busyUid !== null}
pending={pending}
error={rowError?.uid === a.uid ? rowError.message : null}
onAsk={ask}
onCancel={() => setPending(null)}
onReveal={reveal}
onRemove={remove}
/>
))}
</>
)}
</div>
</div>
);
}
+220 -42
View File
@@ -1,19 +1,47 @@
import { useEffect, useState } from 'react';
interface MountIOStats {
source: 'rclone';
bytesRead: number;
bytesWritten?: number;
readBps: number;
writeBps?: number;
readOps?: number;
writeOps?: number;
readIops?: number;
writeIops?: number;
transfersPerSec?: number;
lastSampleAt: number;
intervalMs?: number;
}
interface Mount {
id: string;
name: string;
type: 'nfs' | 'smb' | 'rclone';
type: 'smb' | 'rclone';
enabled: boolean;
desiredMounted: boolean;
mountPath: string;
mounted: boolean;
error?: string;
lastChecked: number;
// VFS cache configuration applied at mount time. Empty string means "use
// server default" (50G / 24h / 5m).
cacheMaxSize?: string;
cacheMaxAge?: string;
dirCacheTime?: string;
pollingEnabled?: boolean;
pollingIntervalMs?: number;
pollingActive?: boolean;
lastPolledScan?: number;
// Adaptive scan progress (set by the meta-core poller)
scanning?: boolean;
currentScanStartedAt?: number; // ms; only while scanning
lastScanDurationMs?: number;
nextScanAt?: number; // ms; planned start of next adaptive scan
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
// so the same numbers appear on every mount card.
ioStats?: MountIOStats;
}
interface RcloneRemote {
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
background: '#7f1d1d',
};
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
function formatBytes(n: number): string {
if (!isFinite(n) || n < 0) return '—';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let v = n;
while (v >= 1000 && i < units.length - 1) {
v /= 1000;
i++;
}
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
}
function formatRate(bps: number): string {
return `${formatBytes(bps)}/s`;
}
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
// is just noise.
function formatDuration(ms: number): string {
if (ms < 0 || !isFinite(ms)) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
const remS = s % 60;
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
const h = Math.floor(m / 60);
const remM = m % 60;
return remM ? `${h}h ${remM}m` : `${h}h`;
}
// IOStatsLine renders a single line of live throughput info beneath each
// mount card. Source is always rclone after the rclone-only consolidation,
// so we report file-level transfers/s rather than kernel IOPS.
function IOStatsLine({ stats }: { stats: MountIOStats }) {
const opsLabel = 'transfers/s';
const opsValue = stats.transfersPerSec;
const ageMs = Date.now() - stats.lastSampleAt;
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
return (
<p
style={{
color: stale ? '#6b7280' : '#9ca3af',
fontSize: '0.85rem',
margin: '0.25rem 0 0',
}}
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
>
↓ {formatRate(stats.readBps)}
{opsValue !== undefined && opsValue > 0 && (
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
)}
<> · {formatBytes(stats.bytesRead)} read total</>
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
<> · {formatBytes(stats.bytesWritten)} written</>
)}
</p>
);
}
interface ScanProgressProps {
mount: Mount;
now: number;
}
// Progress bar shows scan ETA when running, an idle row with a countdown to
// the next adaptive scan when waiting. We can't show true % complete (we
// don't know the file count up-front), so the running bar is filled by
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
// it pins at 95% to signal "taking longer than last time".
function ScanProgress({ mount, now }: ScanProgressProps) {
if (!mount.pollingEnabled) return null;
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
const lastDuration = mount.lastScanDurationMs ?? 0;
let fill = 0;
let label = '';
if (isScanning) {
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
if (lastDuration > 0) {
fill = Math.min(0.95, elapsed / lastDuration);
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
} else {
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
}
} else {
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
if (lastDuration > 0 && mount.lastPolledScan) {
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
} else if (mount.nextScanAt) {
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
} else {
label = 'Polling enabled — waiting for first scan';
}
}
return (
<div style={{ marginTop: '0.5rem' }}>
<div
style={{
height: '6px',
background: '#0f3460',
borderRadius: '3px',
overflow: 'hidden',
position: 'relative',
}}
>
<div
style={{
width: `${Math.round(fill * 100)}%`,
height: '100%',
background: isScanning ? '#4ade80' : '#374151',
transition: 'width 0.5s linear',
}}
/>
</div>
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
</div>
);
}
export default function Mounts() {
const [mounts, setMounts] = useState<Mount[]>([]);
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
@@ -50,19 +209,23 @@ export default function Mounts() {
const [showForm, setShowForm] = useState(false);
const [formData, setFormData] = useState({
name: '',
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
nfsServer: '',
nfsPath: '',
type: 'smb' as 'smb' | 'rclone',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
const [scanning, setScanning] = useState<string | null>(null);
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
// refetches (which run every 5s).
const [now, setNow] = useState<number>(() => Date.now());
const fetchMounts = async () => {
try {
@@ -94,7 +257,11 @@ export default function Mounts() {
fetchMounts();
fetchRemotes();
const interval = setInterval(fetchMounts, 5000);
return () => clearInterval(interval);
const tick = setInterval(() => setNow(Date.now()), 1000);
return () => {
clearInterval(interval);
clearInterval(tick);
};
}, []);
const handleMount = async (id: string) => {
@@ -143,10 +310,7 @@ export default function Mounts() {
type: formData.type,
};
if (formData.type === 'nfs') {
body.nfsServer = formData.nfsServer;
body.nfsPath = formData.nfsPath;
} else if (formData.type === 'smb') {
if (formData.type === 'smb') {
body.smbServer = formData.smbServer;
body.smbShare = formData.smbShare;
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
@@ -156,6 +320,12 @@ export default function Mounts() {
body.rclonePath = formData.rclonePath;
}
// Cache knobs — only forward non-empty values; the server applies defaults
// when fields are omitted.
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
// Add polling configuration
body.pollingEnabled = formData.pollingEnabled;
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
@@ -171,16 +341,17 @@ export default function Mounts() {
setShowForm(false);
setFormData({
name: '',
type: 'nfs',
nfsServer: '',
nfsPath: '',
type: 'smb',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
fetchMounts();
@@ -231,32 +402,15 @@ export default function Mounts() {
<select
style={inputStyle}
value={formData.type}
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
onChange={(e) => {
const type = e.target.value as 'smb' | 'rclone';
setFormData({ ...formData, type });
}}
>
<option value="nfs">NFS</option>
<option value="smb">SMB/CIFS</option>
<option value="rclone">rclone</option>
<option value="smb">SMB (via rclone)</option>
<option value="rclone">rclone (pre-defined remote)</option>
</select>
{formData.type === 'nfs' && (
<>
<input
style={inputStyle}
placeholder="NFS Server (e.g., 192.168.1.100)"
value={formData.nfsServer}
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
required
/>
<input
style={inputStyle}
placeholder="NFS Path (e.g., /export/media)"
value={formData.nfsPath}
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
required
/>
</>
)}
{formData.type === 'smb' && (
<>
<input
@@ -313,6 +467,30 @@ export default function Mounts() {
</>
)}
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
VFS cache (leave blank for defaults: 50G / 24h / 5m)
</p>
<input
style={inputStyle}
placeholder="Cache max size (e.g., 50G)"
value={formData.cacheMaxSize}
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Cache max age (e.g., 24h)"
value={formData.cacheMaxAge}
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Dir cache time (e.g., 5m)"
value={formData.dirCacheTime}
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
/>
</div>
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
<input
@@ -362,17 +540,17 @@ export default function Mounts() {
</span>
</p>
{mount.pollingEnabled && (
<p style={{ color: '#888', fontSize: '0.9rem' }}>
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
Polling:{' '}
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
{mount.pollingActive ? 'Active' : 'Inactive'}
</span>
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{mount.lastPolledScan && (
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
)}
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{' • Adaptive 10× idle'}
</p>
)}
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
<ScanProgress mount={mount} now={now} />
{mount.error && (
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
)}
-1
View File
@@ -10,7 +10,6 @@ set -e
# Create required directories
mkdir -p /meta-core/locks
mkdir -p /meta-core/db/redis
mkdir -p /meta-core/services
mkdir -p /meta-core/mounts/errors
mkdir -p /files
mkdir -p /var/log/supervisor
+7 -29
View File
@@ -2,9 +2,13 @@
# =============================================================================
# Leader Election Script for meta-core
#
# Uses flock for leader election. Only the leader runs the full application
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
# Followers remain dormant, periodically retrying to acquire the lock.
# The flock here is a MUTEX, not service discovery. It is the only thing
# stopping two containers that mount the same /meta-core volume from putting
# two Redis writers on one RDB/AOF. Sibling services no longer read anything
# from this directory — they locate meta-core over UDP (meta-discovery v1, see
# docs/project-architecture/service-discovery.md), so the kv-leader.info file
# this script used to publish is gone. Keep the lock; it costs nothing and the
# corruption it prevents is unrecoverable.
#
# Environment variables:
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +24,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
# Get the container's IP address
@@ -40,35 +43,13 @@ get_local_ip() {
echo "$ip"
}
# Write leader info file (plain text API URL)
write_leader_info() {
local ip=$(get_local_ip)
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
# Write atomically using temp file + rename
local temp_file="${INFO_FILE}.tmp"
echo "$api_url" > "$temp_file"
mv "$temp_file" "$INFO_FILE"
echo "[election] Wrote leader info: $api_url"
}
# Clean up leader info file on shutdown
cleanup() {
echo "[election] Cleaning up..."
rm -f "$INFO_FILE"
exit 0
}
# Set up signal handlers for graceful shutdown
trap cleanup SIGTERM SIGINT
# Ensure lock directory exists
mkdir -p "$(dirname "$LOCK_FILE")"
echo "[election] Starting leader election..."
echo "[election] Lock file: $LOCK_FILE"
echo "[election] Info file: $INFO_FILE"
# Open lock file for flock (file descriptor 200)
exec 200>"$LOCK_FILE"
@@ -79,9 +60,6 @@ while true; do
if flock -n 200; then
echo "[election] Acquired flock - transitioning to LEADER"
# Write leader info before starting services
write_leader_info
# exec supervisord - replaces this process
# When supervisord exits, the flock is released automatically
echo "[election] Starting supervisord..."
+138 -170
View File
@@ -2,11 +2,17 @@
# =============================================================================
# Mount Watcher Script
#
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
# Runs as a supervisord program, polling every 5 seconds.
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
# rclone RC daemon. After the rclone-only consolidation, this script no
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
# through rclone's mount/mount endpoint.
#
# Uses findmnt to check actual mount status (source of truth).
# Writes errors to /meta-core/mounts/errors/{id}.error
# Mount types:
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
# - "rclone" → references a pre-defined remote in rclone.conf
#
# All mounts are read-only by construction (no API surface to override).
# Errors surface in /meta-core/mounts/errors/{id}.error
# =============================================================================
set -u
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
FILES_PATH="${FILES_PATH:-/files}"
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
# Fall-back VFS cache settings — applied when a mount entry leaves the field
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
# packages/meta-core/internal/mounts/types.go.
DEFAULT_CACHE_MAX_SIZE="50G"
DEFAULT_CACHE_MAX_AGE="24h"
DEFAULT_DIR_CACHE_TIME="5m"
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
# ANSI colors
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -54,9 +70,7 @@ init_dirs() {
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
is_mounted() {
local mount_path="$1"
# Remove trailing slash for consistent matching
local clean_path="${mount_path%/}"
# Check if the path appears as a mount point in /proc/mounts (field 2)
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
}
@@ -74,145 +88,102 @@ write_error() {
echo "$error" >> "$ERROR_DIR/$id.error"
}
# Mount NFS share (always read-only)
do_mount_nfs() {
local id="$1"
local server="$2"
local path="$3"
local mount_path="$4"
local options="${5:-}"
# Build the rclone remote source string for a mount entry. Echoes the result.
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
# the optional remote path.
build_rclone_fs() {
local type="$1"
local smb_server="$2"
local smb_share="$3"
local smb_username="$4"
local smb_password_obscured="$5"
local smb_domain="$6"
local rclone_remote="$7"
local rclone_path="$8"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
local rc=$?
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "NFS mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "NFS mount failed: $output"
fi
return $rc
}
# Mount SMB/CIFS share (always read-only)
do_mount_smb() {
local id="$1"
local server="$2"
local share="$3"
local mount_path="$4"
local username="${5:-}"
local password_obscured="${6:-}"
local domain="${7:-}"
local options="${8:-}"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
# If we have credentials, reveal password and build options
if [ -n "$username" ]; then
mount_opts="${mount_opts},username=${username}"
if [ -n "$password_obscured" ]; then
# Reveal password using rclone (only exists in memory briefly)
local password
password=$(rclone reveal "$password_obscured" 2>/dev/null)
if [ -n "$password" ]; then
mount_opts="${mount_opts},password=${password}"
case "$type" in
smb)
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
# embed without quoting.
local opts="host=${smb_server}"
if [ -n "$smb_username" ]; then
opts="${opts},user=${smb_username}"
fi
fi
if [ -n "$domain" ]; then
mount_opts="${mount_opts},domain=${domain}"
fi
fi
# Add any additional options
if [ -n "$options" ]; then
if [ -n "$mount_opts" ]; then
mount_opts="${mount_opts},${options}"
else
mount_opts="$options"
fi
fi
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
# Execute mount directly (not via eval) to handle special chars in password
local output
local rc
if [ -n "$mount_opts" ]; then
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
rc=$?
else
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
rc=$?
fi
# Clear password from any variables (best effort)
password=""
mount_opts=""
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "SMB mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "SMB mount failed: $output"
fi
return $rc
if [ -n "$smb_password_obscured" ]; then
opts="${opts},pass=${smb_password_obscured}"
fi
if [ -n "$smb_domain" ]; then
opts="${opts},domain=${smb_domain}"
fi
echo ":smb,${opts}:${smb_share}"
;;
rclone)
echo "${rclone_remote}:${rclone_path}"
;;
*)
echo ""
;;
esac
}
# Mount rclone remote using RC API (always read-only)
# Mount via rclone RC API (always read-only).
do_mount_rclone() {
local id="$1"
local remote="$2"
local remote_path="$3"
local mount_path="$4"
local options="${5:-}"
local fs="$2"
local mount_path="$3"
local cache_max_size="$4"
local cache_max_age="$5"
local dir_cache_time="$6"
mkdir -p "$mount_path"
# Build the remote source string
local fs="${remote}:${remote_path}"
# Apply defaults for any blank cache fields.
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
# Use rclone RC API to mount (always read-only for safety)
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
# Build JSON body for rclone mount with read-only option
# Build JSON body via python3 — embedding the obscured password and
# display fs string into a hand-rolled heredoc is asking for an injection
# bug the next time we add a field with a quote in it.
local json_body
json_body=$(cat <<EOF
{
"fs": "${fs}",
"mountPoint": "${mount_path}",
json_body=$(python3 -c '
import json, sys
print(json.dumps({
"fs": sys.argv[1],
"mountPoint": sys.argv[2],
"mountOpt": {
"AllowOther": true,
"ReadOnly": true
"AllowOther": True,
"ReadOnly": True,
},
"vfsOpt": {
"CacheMode": 2,
"ReadOnly": true
}
}
EOF
)
# CacheMode 3 = "full" — cache both reads and writes on local disk.
# CacheMode 2 ("writes") would silently bypass the cache for reads,
# which is the opposite of what we want for a read-only mount.
"CacheMode": 3,
"CacheMaxSize": sys.argv[3],
"CacheMaxAge": sys.argv[4],
"DirCacheTime": sys.argv[5],
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
# throughput in isolation but, under the 4+ concurrent reads from
# the processing pipeline, multiplies into 16+ simultaneous SMB
# requests and tanks aggregate throughput. Left at default — rely on
# file-level concurrency from the pipeline for parallelism instead.
"ReadOnly": True,
},
}))
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
local output
output=$(curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-u "$RCLONE_RC_AUTH" \
-d "$json_body" \
"http://127.0.0.1:5572/mount/mount" 2>&1)
"${RCLONE_RC_URL}/mount/mount" 2>&1)
# Wait a moment for mount to establish
sleep 2
if is_mounted "$mount_path"; then
@@ -221,7 +192,6 @@ EOF
return 0
else
local error_msg="rclone mount failed"
# Try to extract error from response
if echo "$output" | grep -q "error"; then
error_msg="$output"
fi
@@ -231,36 +201,28 @@ EOF
fi
}
# Unmount a path
# Unmount a path. Always goes through the rclone RC API now — every mount we
# create lives behind rclone's FUSE driver.
do_unmount() {
local id="$1"
local mount_path="$2"
local type="$3"
if ! is_mounted "$mount_path"; then
log "Already unmounted: $mount_path"
return 0
fi
if [ "$type" = "rclone" ]; then
# Use rclone RC API to unmount
log "Unmounting rclone: $mount_path"
curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-d "{\"mountPoint\":\"${mount_path}\"}" \
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
else
log "Unmounting: $mount_path"
umount "$mount_path" 2>&1
fi
log "Unmounting: $mount_path"
curl -s -X POST \
-H "Content-Type: application/json" \
-u "$RCLONE_RC_AUTH" \
-d "{\"mountPoint\":\"${mount_path}\"}" \
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
# Wait a moment
sleep 1
# Check if still mounted
# Fallback to lazy umount in case the FUSE process is wedged.
if is_mounted "$mount_path"; then
# Force unmount with lazy option
log_warn "Force unmounting (lazy): $mount_path"
umount -l "$mount_path" 2>&1
sleep 1
@@ -277,12 +239,11 @@ do_unmount() {
# Process all mounts from config
process_mounts() {
# Check if config file exists
if [ ! -f "$CONFIG_FILE" ]; then
return
fi
# Parse JSON using python3
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
local mounts
mounts=$(python3 -c '
import sys, json
@@ -294,15 +255,12 @@ except:
sys.exit(0)
for m in data.get("mounts", []):
# Output pipe-separated values
fields = [
m.get("id", ""),
m.get("type", ""),
str(m.get("enabled", True)),
str(m.get("desiredMounted", False)),
m.get("mountPath", ""),
m.get("nfsServer", ""),
m.get("nfsPath", ""),
m.get("smbServer", ""),
m.get("smbShare", ""),
m.get("smbUsername", ""),
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
m.get("smbDomain", ""),
m.get("rcloneRemote", ""),
m.get("rclonePath", ""),
m.get("options", "")
m.get("cacheMaxSize", ""),
m.get("cacheMaxAge", ""),
m.get("dirCacheTime", ""),
]
print("|".join(fields))
' 2>/dev/null)
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
return
fi
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
while IFS='|' read -r id type enabled desired mount_path \
smb_server smb_share smb_username smb_password_obscured smb_domain \
rclone_remote rclone_path \
cache_max_size cache_max_age dir_cache_time; do
# Skip empty lines
[ -z "$id" ] && continue
# Check actual mount status
# Skip retired/legacy types — the Go-side migration disables NFS but a
# stale entry could still arrive here on a downgrade/upgrade race.
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
continue
fi
local is_mounted_now=false
is_mounted "$mount_path" && is_mounted_now=true
# Handle disabled mounts - unmount if currently mounted
# Disabled → unmount if currently up
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) disabled, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
continue
fi
# Handle desired state
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
# Should be mounted
if [ "$is_mounted_now" = false ]; then
local fs
fs=$(build_rclone_fs "$type" \
"$smb_server" "$smb_share" "$smb_username" \
"$smb_password_obscured" "$smb_domain" \
"$rclone_remote" "$rclone_path")
if [ -z "$fs" ]; then
log_error "Could not build rclone fs spec for mount $id (type=$type)"
write_error "$id" "Invalid mount config: missing required fields for type=$type"
continue
fi
log "Mount $id ($mount_path) desired but not mounted, mounting..."
case "$type" in
nfs)
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
;;
smb)
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
;;
rclone)
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
;;
*)
log_error "Unknown mount type: $type"
;;
esac
do_mount_rclone "$id" "$fs" "$mount_path" \
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
fi
else
# Should be unmounted
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) not desired, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
fi
done <<< "$mounts"
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
# Main loop
main() {
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
init_dirs
+19 -8
View File
@@ -70,6 +70,9 @@ http {
proxy_buffering off;
proxy_cache off;
proxy_read_timeout 86400s;
# Snapshot import accepts multi-MB ZIP uploads.
client_max_body_size 0;
}
# =========================================================================
@@ -120,14 +123,10 @@ http {
# =========================================================================
# Service discovery
# =========================================================================
location /services {
proxy_pass http://meta-core-api/services;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
location /services/ {
proxy_pass http://meta-core-api/services/;
# meta-discovery v1 neighbours (supersedes /services, which stays
# while consumers migrate).
location /neighbors {
proxy_pass http://meta-core-api/neighbors;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
@@ -192,6 +191,18 @@ http {
# WebDAV HTML browser (for iframe embedding in dashboard)
# Returns HTML directory listings instead of JSON
# =========================================================================
# At the browser root, strip the "../" parent-dir row that autoindex
# otherwise emits — clicking it inside the dashboard iframe would
# navigate to "/" (the dashboard itself), nesting the page inside itself.
location = /webdav-browser/ {
alias /files/;
autoindex on;
charset utf-8;
sub_filter '<a href="../">../</a>' '';
sub_filter_once on;
}
location /webdav-browser/ {
alias /files/;
autoindex on;
+2 -2
View File
@@ -71,7 +71,7 @@ startsecs=2
# rclone: Remote storage mount daemon (RC API on port 5572)
# =============================================================================
[program:rclone]
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
priority=30
autostart=true
autorestart=true
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
startsecs=3
# =============================================================================
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
# =============================================================================
[program:mount-watcher]
command=/app/scripts/mount-watcher.sh
+771
View File
@@ -0,0 +1,771 @@
# API-Mediated Access (Redis Lockdown)
## Status
**Landed** — all four PRs (A/B/C/D) deployed and verified end-to-end on the
dev stack. Redis is unreachable from outside `metacore-app`; every metadata
read, write, and event flows through meta-core's HTTP+SSE surface. Builds
on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
meta-core is the single writer to the metadata keyspace — an assumption
that's now actually enforced rather than aspirational.
See "Implementation outcome" at the bottom for the delta between this doc's
plan and what shipped (notably the write-path CID resolution and the
dual-root migration sweep, which weren't in the original plan but turned
out to be load-bearing).
## Motivation
meta-core was designed as the sidecar that mediates every metadata read and
write — a single point that owns the Redis schema, enforces invariants
(reverse index, `cids/` key-set integrity, schema-version sentinel),
and presents a stable HTTP contract. Other services were meant to talk to
meta-core, not to Redis.
In practice, meta-sort opens its own Redis connection and writes
`file:<hashId>/<property>` keys with raw pipeline SETs
(`packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts:269-296`).
It picked up this shortcut early, before meta-core had a complete write
API, and the shortcut stuck.
This bypass is a real problem in three ways:
1. **Schema invariants leak.** When meta-core landed the UUID-rooted schema
([uuid-rooted-metadata.md](uuid-rooted-metadata.md)), meta-sort kept
writing midhash-rooted entries. The result was *dual roots per file* —
one set of keys at the watcher's UUID, another at meta-sort's midhash.
`file:__index__` doubled in size. The bug was invisible to meta-sort
(its writes succeeded) but broke the reverse-index resolution
downstream services rely on.
2. **Schema evolution is impossible.** Changing the Redis layout requires
coordinating across every service that touches it. The UUID-root
migration would have been a single-PR change if writes already routed
through the API.
3. **Self-hosted distributed deployment widens the threat model.** Peers
are run by different operators. Plugin authors may ship their own
services. We can't expect every binary in the network to honour
undocumented conventions. A buggy or careless service that connects
to Redis and writes the wrong key shape can corrupt the schema for
everybody on that node.
The third point is the one that flipped the design from "soft lock by
convention" to "hard lock by network topology." We're not protecting
against malicious services — that's an out-of-scope threat. We're
protecting against *accidental* corruption from honest mistakes in code
we don't fully control.
## The architecture in one picture
```
meta-core (only Redis writer + reader)
┌──────────────────────────────────────┐
│ HTTP API surface: │
│ POST/PUT/PATCH /api/metadata/* │ ◄── writes
│ GET /api/meta/{cid} │ ◄── reads (by CID)
│ GET /api/metadata/* │ ◄── reads (by id)
│ GET /api/file/{cid} │ ◄── content
│ GET /api/events/files │ ◄── SSE stream
│ GET /api/events/meta │ ◄── SSE stream
│ │ │
│ ▼ │
│ Redis (file:*, cid:*, streams) │
│ ◄── container-local, no exposed │
│ port, no docker-network alias │
└──────────────────────────────────────┘
▲ ▲
│ HTTP │ HTTP SSE
│ │
┌───────────┴────┐ ┌──────┴─────────┐
│ meta-sort │ │ meta-fuse │
│ meta-stremio │ │ meta-share │
│ meta-dup │ │ (any future) │
└────────────────┘ └────────────────┘
```
**The single-writer rule.** meta-core has exclusive write authority over
the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
other service can write — not because of an honoured convention, but
because they can't reach Redis at all.
**Stream reads also mediated.** Events go out through SSE endpoints on
meta-core's HTTP surface. Consumers don't see the underlying Redis
Streams. This is the Kubernetes / Consul pattern (see "Industry context"
below). It costs an extra HTTP hop versus the current direct XREAD; the
benefit is that the storage layer is free to evolve (or be replaced)
without breaking every consumer.
**Redis is private.** No host port mapping, no docker-network alias, no
URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
## Eventing — HTTP Server-Sent Events
SSE is the chosen transport for cross-service event consumption. The
mapping from Redis Streams onto SSE is direct enough that no eventing
feature we currently use is lost.
### Feature mapping (what we keep, how we keep it)
| Redis Streams | SSE equivalent |
|---|---|
| Fan-out to N consumers | meta-core holds N SSE connections, broadcasts events to each |
| Replay from a cursor | `Last-Event-ID` header on reconnect — built into the SSE spec |
| Retention / MAXLEN | Underlying Redis Stream still does the trimming; SSE is a thin shim |
| Backpressure on slow consumers | meta-core's SSE handler blocks on the TCP write; events stay in Redis until trimmed |
| Multiple streams (file:events, meta:events) | Separate endpoints: `/api/events/files`, `/api/events/meta` |
| At-least-once via XACK | Implicit — client persists last-received `id`, sends it on reconnect (see "Cursor semantics and consumer-group migration" below) |
The one Streams feature we don't get is **consumer groups for
horizontally-scaled replicas of the same logical consumer.** We don't use
this today and don't plan to — meta-sort, meta-fuse, etc. each run one
instance. If horizontal scale ever happens, it'll be by sharding hash-IDs
across replicas, not by splitting an event stream. SSE stays adequate.
### Cursor semantics and consumer-group migration
Today three consumers use `XREADGROUP` against named consumer groups, not
bare `XREAD`:
- meta-sort: group `meta-sort-processor`
(`packages/meta-sort/.../events/FileEventConsumer.ts:23,66-74`)
- meta-dup: group on `file:events`
(`packages/meta-dup/.../MetaEventConsumer.ts:102-104`)
- meta-fuse: dual path — `XREADGROUP` on one consumer, plain `XREAD` on
another (`packages/meta-fuse/.../kv/RedisClient.ts:826,907`)
`XREADGROUP` keeps server-side state per group: a Pending Entries List
(PEL) and a `last-delivered-id` cursor. The load-bearing use today is
meta-sort's `processPendingEntries(idleMs=30000)`, which exists to claim
work from a crashed consumer's PEL on restart.
SSE has no equivalent server-side state. After the migration the contract
shifts to: **each consumer persists its own `Last-Event-ID` and resumes
from it on reconnect.** Implications:
- PEL-based crash recovery is gone. A crashed consumer that hadn't
persisted its cursor since entry X resumes from X (its last *durably
persisted* cursor), not from "whatever was in flight." Acceptable for
every current consumer — none run multiple instances per group — but
call out in the PR C migration of meta-sort that the recovery primitive
is changing.
- Each consumer must own its cursor persistence. Cheapest place is the
consumer's own state (a file, a Redis key written under its own
prefix, an in-memory variable if the consumer is OK with at-most-once
on its own crash).
- No server-side "ACK" call. The cursor *is* the ack — when the consumer
successfully writes Last-Event-ID = N to its own state, entries ≤ N
are considered handled. The SSE handler does nothing.
### Endpoint spec
```
GET /api/events/files
Request headers:
Last-Event-ID: <stream-entry-id> (optional; resume from this cursor)
Accept: text/event-stream
Response:
Content-Type: text/event-stream
Cache-Control: no-cache
Connection: keep-alive
X-Accel-Buffering: no (disable nginx response buffering)
```
Same shape for `/api/events/meta`.
### Event format
One SSE event per Redis Stream entry. Format:
```
id: 1747999999999-0
event: add
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1747999999999}
id: 1748000000123-0
event: change
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1748000000123}
```
- `id` — **opaque** cursor. Today it's the Redis Stream entry ID
(`<ms>-<seq>`); consumers MUST treat it as a black-box string they
echo back via `Last-Event-ID`. This contract lets meta-core swap the
backing store later without breaking every consumer's cursor handling.
- `event` — the `type` field from the underlying entry (`add` | `change`
| `delete` | `rename` | `reset` for `file:events`; `set` | `del` |
`expire` for `meta:events`).
- `data` — JSON-encoded rest of the entry. Keys match the existing stream
payload exactly so downstream consumers don't change their parser.
### Backing-store retention and lifecycle
**Contract: the SSE wire is a faithful mirror of the underlying Redis
Streams.** Consumers rely on `Last-Event-ID` resumption working across
*every* kind of meta-core hiccup, including a restart — that's the entire
point of having a cursor protocol. The backing store must therefore
preserve entries across meta-core process restart and bound its growth
explicitly, not implicitly.
Today's behaviour is wrong on both counts:
| Stream | Restart behaviour | Retention |
|---|---|---|
| `meta:events` | **Cleared on every MetaPublisher.Start** (`events/meta_publisher.go:50-58`) | No `MAXLEN` (unbounded) |
| `file:events` | Persists across restart | No `MAXLEN` (unbounded), cleared only on deliberate reset (`watcher/dispatcher.go:82-95`) |
The `meta:events` clear-on-restart is the load-bearing problem. With it
in place, every meta-core restart silently invalidates every consumer's
persisted cursor — the SSE `gap` event would fire on the first reconnect
after every restart, and the consumer would have to re-bootstrap. That
turns "restart meta-core to apply a config change" into "every downstream
service does a full re-sync," which is exactly the regression SSE is
supposed to prevent.
PR A must fix both streams:
1. **Remove the unconditional `Del(MetaEventsStream)` in
`MetaPublisher.Start`.** Entries persist across restart. The republish
path (`RepublishAllMetadata`) stays as a one-shot bootstrap for
missing entries, but it does not pre-truncate the stream.
2. **Add `MAXLEN ~ N` to every `XAdd` on both streams** so growth is
bounded. Approximate trim (`~`) is cheap. Default `N = 100_000` per
stream is fine for the foreseeable workload (a few KB per entry,
bounded total ≈ a few hundred MB) and easy to tune via config.
3. **Keep the deliberate `EmitReset → ClearStream` path on
`file:events`.** That clear is a feature: a reset is the consumer
contract for "throw out everything you knew, re-bootstrap." Consumers
detect it through the `event: reset` payload that's already emitted
as the first entry after the clear (`dispatcher.go:99-`). The SSE
`gap` event handles the race where a consumer's cursor was trimmed
*between* the clear and its reconnect.
After PR A, the cursor contract is **"durable across meta-core restart
and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
when they persist a Last-Event-ID.
### Heartbeats
Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
when no events are flowing. Keeps reverse proxies and load balancers
from killing the connection as idle. The client ignores comment lines
per the SSE spec.
### Reconnect semantics
Per the SSE spec the client (or `EventSource`) auto-reconnects after a
network error, sending `Last-Event-ID`. The server resumes from the next
entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
**Gap handling.** If the requested ID has been trimmed out of the Redis
Stream by retention, the server emits one synthetic event before resuming
from the oldest available entry:
```
event: gap
data: {"requested":"<id>","resumeFrom":"<oldest-id>","reason":"retention"}
```
Consumers that care about gap-free delivery (currently none — the
schema indexer tolerates gaps; ingest tasks rebuild from scratch) can
react. Consumers that don't care ignore it.
### Error handling
- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
- **Redis fails mid-stream**: close the SSE connection. Client reconnects
with `Last-Event-ID`; the cycle resumes when Redis recovers.
- **Client disconnects**: handler goroutine exits cleanly; no per-client
state on meta-core.
### Auth
`/api/events/*` bypasses the hash-lock perimeter, same pattern as
`/api/meta/{cid}` and `/api/file/{cid}`. Event streams are internal
infrastructure and need to be reachable by peer services without going
through Authelia.
**Bypass ≠ public.** "Hash-lock bypass" means the OIDC sidecar doesn't
gate the route; it does *not* automatically mean Caddy proxies it from
the public hostname. The two existing precedents differ on this:
`/api/file/{cid}` is publicly reachable (peer-to-peer content fetch is
its job), `/api/meta/{cid}` likewise. For the SSE endpoints, the choice
is deliberate:
- If kept inside-only (no Caddy upstream for `/api/events/files` and
`/api/events/meta`), only sibling services on the docker network can
subscribe. This matches the rest of the "Redis is private" model and
is the recommended default.
- If exposed publicly, any browser can open an `EventSource` against
the stream without authentication — i.e. anyone can watch every file
event in the library. Only do this if there's a concrete consumer
that needs it and the visibility is acceptable.
PR A's Caddy change (see PR A scope, item 4) makes this an explicit
choice rather than a default behaviour. The recommendation is
inside-only until a concrete external consumer appears.
### Implementation footprint
The SSE handler itself is ~80 lines of Go in meta-core: one handler per
stream, each running an `XREAD … BLOCK 30s` loop and writing
SSE-formatted lines to the response. Go's `net/http` natively supports
streaming responses with `http.Flusher.Flush()`. Standard SSE clients
exist for every language we use: Node `eventsource`, Rust
`eventsource-client` (or `reqwest` + `tokio_util::codec`), Python
`sseclient`, Go `r3labs/sse`.
PR A as a whole is larger than the handler — see the PR A scope below
for the full list (backing-store fixes, Caddy wiring, poll deprecation)
that has to land alongside the handler for the cursor contract to hold.
## Eventing — considered alternatives
Recorded for posterity. None of these are being implemented; if any are
ever revisited it should be a deliberate reopen, not a drift.
- **Long-polling.** Strictly worse than SSE — connection churn on every
event, latency floor at the poll interval, no built-in resumption.
Rejected.
- **Webhooks (meta-core POSTs to subscribers).** Symmetric coupling:
every consumer would need to run an HTTP server reachable from
meta-core. Out-of-order delivery, retry, dead-letter all need
reinventing. Right for cross-organization integrations, wrong for
in-cluster eventing. Rejected.
- **External broker (Kafka / NATS / RabbitMQ).** Industrial-grade
pub/sub. New dependency, new operational surface, overkill for the
scale (single-host dev, ≤20-peer POC). Revisit if MetaMesh ever needs
cross-WAN federation at high throughput. Rejected for now.
- **Two Redises (writes Redis private, events Redis exposed).** Strongest
separation. Doubles operational complexity for no benefit over the
SSE design — once events go through HTTP, the events-Redis isn't
exposed either. Rejected.
- **Redis ACLs (Option B from earlier brainstorm).** Allows direct Redis
access from clients while restricting commands by user. Adds
credential plumbing and ACL config to operate; lets a buggy service
still connect, just with limited verbs. The SSE design closes the
network entirely, which is strictly stronger. Rejected — but the
equivalent escape hatch (open Redis to other services again) stays
available if we ever need it.
## Industry context — where this lands
The pattern we're adopting is the **Kubernetes / etcd** pattern, and
that's the most informative comparison.
Kubernetes is structurally identical to MetaMesh:
- A single source-of-truth store (etcd) holds all cluster state.
- A single service (kube-apiserver) is the sole writer to etcd.
- Many polyglot clients (kubectl, controllers, schedulers, kubelets,
operators) read and react to changes.
- Clients subscribe to *watch streams* for events.
What Kubernetes does, point-for-point:
- **etcd is never reached by clients.** It runs on a separate port,
often a separate network. kube-apiserver is the only thing that holds
etcd credentials. The lock is by network topology, not authentication.
- **Watch is also mediated.** kube-apiserver exposes `/watch` as an HTTP
streaming endpoint (very similar to SSE in shape). Clients don't watch
etcd directly — even though etcd has its own native watch primitive.
- **Storage portability falls out for free.** etcd could be swapped for
another store without any client change. We don't plan to swap Redis,
but the optionality is real and free.
Other systems with the same shape:
- **HashiCorp Consul.** Clients talk to a local agent over HTTP. The
agent talks to the cluster. KV writes and watches both flow through
the agent. Direct raft-store access is impossible.
- **Apache ZooKeeper.** Clients use the ZooKeeper client library which
speaks the ZooKeeper wire protocol; the data store is private.
- **Strapi / Sanity / Contentful** (headless CMS as analog). All
consumers — frontends, backend services, build pipelines — go through
the CMS HTTP API. Direct Postgres or content-store access is impossible
by design.
Where MetaMesh sat *before*: closer to the bad-monolith pattern
(everyone connects directly to the shared Redis). Where it sits *after*
this design: aligned with the canonical microservices pattern, and
specifically with the Kubernetes flavour where streams are mediated too.
### Advantages we're picking up
- **Schema evolution.** One service can change its data layout without
coordinating releases across every other service. The UUID-root
migration is the cautionary tale of what happens without this.
- **Validation choke point.** Every write passes through one code path
that can enforce invariants (reverse index, `cids/` key-set, type
constraints, schema version).
- **Observability.** Audit logs, request metrics, slow-query detection
all live at one layer.
- **Threat-model fit.** Untrusted-service-in-the-network is now in
scope; network isolation makes the concern moot.
- **Storage portability.** Optionality only, but real.
### Drawbacks (acknowledged)
- **Latency.** One extra hop per write (TCP + HTTP parse + handler).
Local-host overhead is sub-millisecond; mitigations are batched writes
(PATCH the whole metadata blob per plugin completion, not per field)
and Keep-Alive / HTTP/2 connections.
- **SSE-handler plumbing.** ~80 lines per stream endpoint plus per-client
goroutines. Trivial at our scale, real at Kubernetes scale (which is
why they invested in chunked watch with bookmarks, resource versions,
etc.). We can defer the same optimisations forever.
- **API surface size.** Every operation needs a corresponding endpoint.
Most of it already exists; one new SSE endpoint per stream is the
only new surface area introduced here.
## Enforcement — what "lock" actually means
Three layers, in order of effectiveness:
1. **Network isolation (hard).** Redis is on a container-private port.
No host port mapping. No docker-network alias. `kv-leader.info` does
not publish a Redis URL. Any service that wants to talk to Redis
needs container privileges, and at that point the lock isn't the
concern — operating-system isolation is. This is the actual lock.
2. **Documentation (convention).** This file, plus a one-line note in
`CLAUDE.md` and in each consumer service's `CLAUDE.md`: "All metadata
I/O routes through meta-core's HTTP API. Redis is private to
metacore-app." Engineers reading the codebase find the rule on first
contact. This catches the case of a future meta-core PR that
accidentally exposes Redis again.
3. **Code review (convention).** PRs that re-expose Redis (a port
mapping, an alias, a credential leak via leader-info) get rejected
with a pointer to this doc.
No runtime canary or lint is proposed — the network isolation makes
violations physically impossible from outside, and inside meta-core the
small file count makes review sufficient. Layer 1 does the heavy
lifting.
## Migration plan
Four PRs, additive, ordered so the system stays working throughout.
Closing the network access (PR D) is the moment the lock becomes a
lock; everything before it is opt-in migration that consumers can adopt
at their own pace.
**All four PRs have landed.** What follows is the original plan as
written; see "Implementation outcome" at the bottom for the delta
between plan and ship.
### PR A — SSE endpoints in meta-core
Adds `GET /api/events/files` and `GET /api/events/meta`. Implements the
`Last-Event-ID` semantics, gap handling, heartbeats, and the per-stream
XREAD loop. Redis Streams stay reachable directly; consumers can
migrate one at a time.
Scope, in PR-order:
1. Backing-store fixes (see "Backing-store retention and lifecycle"
above) — remove the `meta:events` clear-on-restart, add `MAXLEN ~ N`
to every `XAdd`. These are prerequisites for the SSE cursor contract
to hold; without them PR A is not deliverable.
2. SSE handlers for the two streams.
3. Deprecate `GET /api/events/poll` (`watcher/handlers.go:29`). It
exists today as a long-poll variant over the same `file:events`
stream and is described in code as "always empty in practice"
(`watcher/handlers.go:55-58`). After SSE lands it's redundant — keep
it responding 200 for one release with a `Deprecation` header, then
remove. The `/api/scan/*` deprecation pattern in the same file is
the template.
4. Caddy perimeter wiring (see "Auth" subsection above) — explicit
allow-list for `/api/events/files` and `/api/events/meta` matching
whatever the perimeter policy decides. Either inside-only via the
docker network, or public via Caddy with deliberate authorisation.
Do not let the routes default-leak past the hash-lock.
Roughly 200 lines of Go + ~50 lines of test (covering reconnect from a
cursor, gap emission when the cursor is trimmed, heartbeat cadence,
durability across a meta-core restart).
### PR B — meta-sort writes via HTTP
Swap `RedisClient` for an `HttpKVClient` implementation of the existing
`IKVClient` interface in `packages/meta-sort-core`. All writes
(`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
calls against the **`/meta/{hash}` family** — the canonical write
surface for service-to-service traffic. Per-plugin write coalescing
uses `PATCH /meta/{hash}` (already calls `MergeMetadataFlat`
server-side with the partial-update semantics we need; see
`handlers.go:533-566`).
Reads from Redis stay direct in this PR (no consumer migration yet);
only writes move. Streams stay on direct XREAD.
**Pre-audit (already done):** meta-sort's production `kvClient.*`
surface is `setMetadataFlat` / `deleteMetadataFlat` / `getMetadataFlat`
/ `getMetadata` / `getAllHashIds` / `health` plus the `file:events`
stream consumer. No orphan keyspaces, no generic `kv.set(arbitrary_key)`
callers — the HTTP migration covers the entire write path with no
side bets.
### PR C — Stream consumers migrate to SSE
meta-sort's event consumer (`FileEventConsumer.ts`), meta-fuse's stream
reader, meta-stremio's event hook, meta-dup's monitor, meta-share's
ingest task — each swaps its `XREADGROUP` / `XREAD` loop for an SSE
client. Per-language standard libraries make this small (one HTTP
request, one event-handler closure).
In the same PR, each consumer's `LeaderClient` (TypeScript x2, Python,
plus any new Rust binding for meta-share if added) drops its
*requirement* on the `redisUrl` field — i.e. it stops opening a Redis
connection. It can still tolerate the field being present (PR D removes
it from the publisher side). This ordering matters: PR D's loud-failure
guard cannot ship until every consumer has been demonstrated to boot
without `redisUrl`.
The schema indexer is internal to meta-core and unaffected.
### PR D — Close Redis network access
PR D must land **after** every consumer in PR C has stopped requiring
`redisUrl` (verify by booting each service against a meta-core build
that already omits the field — staging environment or feature-flagged
release). Until that's true, this PR will deadlock the boot sequence.
Then, in this order:
1. Update consumer-side discovery (`LeaderClient` in each service) to
*log loudly* if it sees a `redisUrl` field — guards against
accidental rollback. (Loud log, not hard failure, while the field
is still being written — flip to hard failure once step 4 lands.)
2. Stop publishing `redisUrl` from `/urls` and from `kv-leader.info`.
3. Remove the `meta-core` docker-network alias on the metacore-app
service (other services no longer need DNS for the Redis port).
4. Remove `6380:6379` host port mapping in `dev/docker-compose.yml`.
5. Flip the LeaderClient guard from "warn" to "raise" — any future
consumer that tries to require the field fails loudly at boot.
After PR D, attempting to connect to Redis from anywhere except
metacore-app produces a connection refused. The lock is real.
### Out of scope for this work
- **Redis ACLs / proxy.** Network isolation makes these unnecessary.
Available as an escape hatch if the threat model ever changes.
- **Replacing Redis Streams under the hood.** SSE is the externalised
contract; the backing store is an implementation detail meta-core can
change later.
- **External brokers.** Out of scope absent a concrete driver
(cross-WAN federation at high throughput).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| Writes from non-meta-core services? | Forbidden by network topology | Redis unreachable; not a convention |
| Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
| Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
| Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
| Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
| Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
| **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
| **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
| **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
## Error envelope
Standard shape for all 4xx / 5xx responses from the metadata API:
```json
{
"error": "alias_collision",
"message": "CID midhash256:bafk… already resolves to UUID 01JKR…",
"retryable": false
}
```
Stable `error` slugs the client should switch on:
| Slug | HTTP status | Retryable | Meaning |
|---|---|---|---|
| `alias_collision` | 409 | no | A different UUID already owns the CID the caller is trying to register. Caller bug; escalate. |
| `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
| `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
| `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
| `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
**Implemented.** Every 4xx/5xx response across `internal/api`,
`internal/mounts`, `internal/watcher`, and `internal/watchers` now
emits the typed envelope. The slug-mapping helper is duplicated across
those packages (one ~15-line `slugForStatus` per package) to avoid an
import cycle on the api package; consolidating it into a shared
`internal/httperr` package is a worthwhile follow-up but not load-bearing.
## Remaining open questions
Mostly answered post-implementation:
- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
editor's React UI continues to call it; the surface itself was untouched
by this work. Worth a follow-up to consolidate the editor onto
`/meta/{hash}` and delete the parallel routes.
- **Initial bootstrap flood on `/api/events/files`.** Resolved in
practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
replay every entry on each restart and the StreamingStateBuilder
handles the resulting burst (~2.5k property events on a 50-file
library) without issue. meta-sort persists its cursor so only sees
the flood once per fresh deploy. meta-stremio filters to "interesting
fields" before triggering cache invalidation, so the burst is mostly
no-ops.
- **meta-dup SSE port.** Still pending — meta-dup is in no active
compose file. When revived, copy meta-fuse's pattern wholesale
(SSEEventClient + MetaCoreApiClient). Tracked separately.
## Non-goals
- **Authentication between internal services.** The hash-lock OIDC
perimeter handles external clients; internal service-to-service
trust is by network isolation (docker network for dev, equivalent
isolation in prod deployments). Adding service-to-service auth is a
separate concern.
- **Replacing the public HTTP API surface.** This proposal exercises
what already exists; the only new endpoints are the two SSE streams.
- **Introducing new transport, broker, proxy, or sidecar.** Everything
builds on what's deployed today.
- **Migrating meta-fuse / meta-stremio / meta-dup direct Redis reads
via XSCAN/GET** (separate from event-stream reads). ~~Was a non-goal~~
ended up being required for PR D's network closure to be safe.
meta-fuse and meta-stremio migrated; meta-dup deferred (not deployed).
See PR C / "Implementation outcome" for details.
## Implementation outcome
What actually shipped vs. what this doc planned.
### Delta against the original plan
- **Read-path migration for meta-fuse and meta-stremio is now in scope
and done.** The original "out of scope" carve-out was wrong: closing
the Redis network port doesn't just break stream consumers, it also
breaks every `kvClient.get` / `redis_storage.get_all_videos` call.
Both services grew their own `MetaCoreApiClient` (TS for meta-fuse,
Python for meta-stremio) and now route reads through `/meta/{hash}`
+ `/meta` + `/api/file/{cid}/info`.
- **meta-stremio's HTTP mode auto-recovers from transient meta-core
outages.** `is_connected` re-probes meta-core's `/health` on every
call (cheap; a stateless GET), so a metacore-app restart no longer
latches the addon into a permanent disconnected state. The
background reconnect thread is only needed for legacy Redis mode
where the socket itself is sticky.
- **Hard `redisUrl` guard is gated by `ALLOW_LEGACY_REDIS_URL=1`.** All
three consumer LeaderClients (TS x2 + Python) throw on seeing the
field by default. Setting the env var downgrades to a warn — escape
hatch for a deliberate temporary rollback to a meta-core that still
publishes the field.
- **Write-path CID resolution** turned out to be a hard prerequisite
for the lockdown to work in practice. Documented separately below.
- **PR D ordering matched the doc** (consumer migration first, then
publisher stops emitting, then network closure). The `meta-core`
docker-network alias and the `6380:6379` host port mapping are both
removed; sibling services reach the HTTP/SSE/WebDAV surface via the
canonical `metacore-app` hostname.
- **meta-dup is the one service that didn't migrate.** It's in no
active compose file. The TS port is straightforward when needed: copy
meta-fuse's pattern (SSEEventClient for streams, MetaCoreApiClient
for reads).
### Write-path CID resolution (`storage.ResolveRoot`)
Not in the original plan but load-bearing.
**The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed
every write through `/meta/{hash}`. But `{hash}` is the midhash256 CID
that meta-sort received via file:events, not the watcher's minted UUID.
With no resolution step in the handler, every write went to
`file:<midhash>/*` — a parallel root the watcher's UUID never saw. The
auto-alias hook on `cid_midhash256=<value>` then OVERWROTE the
watcher's `cid:midhash256:<value>` → UUID alias to point at the
midhash itself, leaving the UUID stranded with only the watcher's
sparse fields (filePath, sizeByte, mtimeNano, duplicates) and the
midhash root carrying the rich plugin output. This is the exact
"dual roots per file" failure the doc's Motivation section listed as
the prior bug — it just moved from "convention violation" to "API
contract violation."
**The fix.** Three layers in `internal/storage/cid_resolution.go`:
1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
read/write handler, look up `cid:<hash>` in the reverse index (bare
CID — no algorithm prefix). If found, use the UUID as the storage root.
Falls through to the bare hash for direct-UUID writes / legacy entries.
2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
register `cid:<v>` → `<v>`. The dual-root bug's chicken-and-
egg is: meta-sort writes first, registers self-pointing alias,
watcher then can't recover. Refusing the self-write keeps the
watcher's legitimate alias intact.
3. **`MigrateDualRoots` sweep** (`POST /api/admin/migrate-dual-roots`).
One-shot fixer for already-stranded entries: walks every UUID's
`midhash256` field to build the orphan index, merges each stranded
`file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
deletes the stranded root, and re-points the alias.
Verified on the dev stack: 48 stranded entries were reunited; the
editor now shows every file with its full plugin metadata under one
UUID; future writes stay unified.
### Typed error envelope
Done across all four handler packages (`api`, `mounts`, `watcher`,
`watchers`). Each has its own `writeError` returning
`{error: <slug>, message, retryable}`. The slug-mapping helper is
duplicated in each to avoid an import cycle on the `api` package.
### What's NOT done
- **meta-dup SSE + read migration.** Not deployed, no test harness.
When revived, copy meta-fuse's pattern.
- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate
editor-internal API surface is untouched; could be retired in a
follow-up by porting the editor UI to `/meta/{hash}`.
- **Consolidated `internal/httperr` package** for the typed envelope
helpers (currently duplicated 4× to avoid the import cycle). Code
hygiene; functionally a no-op.
### Verification done
- All four service dashboards load cleanly with no console errors.
- meta-sort SSE consumer + HTTP write path verified end-to-end by
delete + inject + reprocess of Sintel.
- meta-fuse SSE consumer rebuilds VFS to 48 files on each boot.
- meta-stremio catalog API now surfaces the full library (47 videos,
1 movie + 5 series + 46 episodes — was 0/0/0 before the Python
read-path migration because LeaderStorage broke when it tried Redis).
- `redis-cli` from any container other than `metacore-app` produces a
connection refused; `cid:midhash256:<H>` aliases all point to UUIDs
(no self-pointing); `file:__index__` has only UUID-style hashIds.
- ALLOW_LEGACY_REDIS_URL=1 escape hatch present in all three consumer
LeaderClients (TS x2 + Python).
### Known test-suite gaps (post-implementation bats run)
Running `docker exec meta-test-runner /app/test/test.sh` after the
lockdown shipped reveals a number of failing suites, almost all of
which are pre-existing test/environment drift unrelated to this work:
| Failure | Root cause | Relation to lockdown |
|---|---|---|
| meta-stremio (~35 tests), stremio-tmdb (6 tests) | Tests call `/manifest.json` directly; dev `.env` sets `STREMIO_HASH_API_SEED=dev-stremio-secret` which puts the addon behind a path-token prefix | Unrelated — pre-existing env mismatch |
| meta-fuse FUSE-mount (3 tests) | Tests use `docker exec meta-fuse` but the container is named `metafuse-app` (Yundera prod-parity rename) | Unrelated — predates this work |
| meta-fuse WebDAV PROPFIND (4 tests) | Tests use Caddy URL with WebDAV basic-auth credentials that no longer match the hash-lock perimeter | Unrelated — predates this work |
| meta-sort "API integration tests" (1 test) | Test runs `docker exec meta-sort pnpm test`; container is `metasort-app` | Unrelated — predates this work |
| meta-core "create and delete mount config" (1 test) | Test creates an NFS mount; production code dropped NFS support ("NFS mounts are no longer supported") | Unrelated — predates this work |
| service-discovery (9 tests) | Cascading from meta-stremio breakage (test expects unauth `/api/services` on stremio) | Unrelated to lockdown directly |
| meta-share / share-1c (multiple) | Tests assert `"phase":"1f"` in `/health`; meta-share has moved past that phase | Unrelated — meta-share evolution |
| flat-keys: "filePath field exists" (1 test) | Test SRANDMEMBERs file:__index__; if it lands on a UUID with no filePath (e.g. a manual test write) it fails | Was hit transiently after my interactive testing left stub entries; resolved by cleanup |
No failure traceable to a real regression from this work was found.
The dev-environment test harness needs an unrelated refresh to catch
up with the prod-parity container renames, the NFS removal, the
STREMIO_HASH_API_SEED default, and meta-share's current phase.
File diff suppressed because it is too large Load Diff
+234 -532
View File
@@ -1,595 +1,297 @@
# Redis Metadata Storage Architecture
# Redis Metadata Storage Structure
This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling.
How meta-core lays out file metadata in Redis. This document covers the
on-disk shape (UUID-rooted flat string keys, CID reverse index,
per-root sets) and the schema-version gate. It does **not** enumerate
field semantics — that's the job of the repo-root
[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key
registry for everyone who writes to or reads from metadata.
## Overview
For the design rationale (why UUIDs instead of content hashes, why a
reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
For the external-access contract (HTTP-only, SSE for events), see
[`api-mediated-access.md`](api-mediated-access.md).
MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables:
## Quick summary
- **Redis keyspace notifications** for field-level change detection
- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates)
- **meta:events stream** integration for real-time updates across services
- Individual field updates without affecting other properties
- Fast VFS lookups using content-based hash IDs
- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
not content hashes. Examples in this doc use the placeholder
`01JKR8XW5T4QMV7AHNJ2DEFGHK`.
- **Properties are flat Redis STRING keys**, one per leaf:
`file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
- **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
(a Redis STRING per known CID; no algorithm prefix — the multicodec
already carries it).
- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
`"true"` per known CID, midhash included), `file:<uuid>/duplicates`
(alternate paths with the same content), plus `file:__index__` (every UUID).
There is **no stored `canonical_cid`** — it is derived by rank on read.
- **A schema-version sentinel** (`meta-core:schema-version`) refuses to
boot against legacy data.
## Key Structure
Implementation lives in `internal/storage/client.go`,
`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
`internal/storage/schema_version.go`.
### Flat Key Storage
## Key structure
Each file property is stored as a separate Redis STRING key:
### Roots
```
Key Pattern: file:{hashId}/{property/path}
Key Type: STRING
Value: String value
file:<uuid>/<property/path> → Redis STRING (always)
```
**Example Redis commands:**
```bash
# Store file metadata (each property is a separate key)
SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..."
SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv"
SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5"
SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm"
UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
in a given window, which restored some of the debug-ability lost when
midhash256 stopped being the root.
# Retrieve single field
GET "file:bafkr4ih5kapbjzq.../fileinfo/duration"
# Retrieve all metadata for a file (scan by prefix)
SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000
```
### Why Flat Keys?
The flat key structure (vs. Redis Hashes) enables:
1. **Keyspace Notifications**: Redis publishes events when individual keys change
2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*`
3. **Selective Updates**: Update one field without touching others
4. **Stream Integration**: The `meta:events` stream captures field-level changes
### File Index
A Redis Set tracks all known file hash IDs:
Properties use `/` as the path separator. Nested objects in source code
are flattened to flat keys on write and reconstructed on read. Examples:
```
Key: file:__index__
Type: Set
Value: {hashId1, hashId2, hashId3, ...}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..."
```
This enables fast bulk queries like "get all files" without scanning keys.
### Reverse-index aliases (CIDs)
Every CID known for a file is registered as its own Redis STRING, keyed by
the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
function in the multicodec, so naming it again in the key would be
redundant (and would let the same digest be indexed under two spellings):
```
cid:<bareCid> → <uuid>
```
Examples:
```
cid:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:bagacbabaec7v3... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
registered CID, independent of which hash algorithm — there is no primary.
The auto-alias hook (`maybeAddAliasFromFieldLocked` in
`internal/storage/cid_resolution.go`, called from `SetProperty`,
`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
field whose name starts with `cids/`, so writers emit key-set members
without knowing the reverse index exists.
> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
> `midhash256` *named* field written to a root is stored verbatim and is
> **never indexed** — the record becomes unresolvable by that CID. Writers
> must collapse such fields to `cids/<cid>` members before the write. See
> `METADATA_KEYS.md` §14.13.
### Per-root sets
```
file:<uuid>/cids/<bareCid> → STRING "true" — one key per known CID (a key-set, not a Redis SET)
file:<uuid>/duplicates → Redis SET of alternate paths with same content
```
`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
which keeps it conflict-free to merge across peers (a SET would not be).
The record's own address (the midhash) is just the `0x1000` member — it has
no separate field.
There is **no stored `canonical_cid`**. The canonical CID is *derived on
read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the
key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns
the winner as the `canonical_cid` field of the JSON response only. Nothing
reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and
was removed with the flat `cid_*` keys.
### Global index
```
file:__index__ → Redis SET of all UUIDs
```
`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
every file without scanning the keyspace.
```bash
# Get all hash IDs
SMEMBERS "file:__index__"
# Count total files
SCARD "file:__index__"
# Check if a file exists
SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..."
SMEMBERS file:__index__ # all UUIDs
SCARD file:__index__ # total file count
SISMEMBER file:__index__ <uuid>
```
### Keyspace Notifications
Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration:
```bash
# Required Redis config (set by meta-core)
CONFIG SET notify-keyspace-events "K$"
# K = Keyspace events (published on __keyspace@<db>__:<key>)
# $ = String commands (SET, etc.)
```
Services can subscribe to keyspace events for real-time updates:
```bash
# Subscribe to all file key changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Subscribe to specific field pattern (e.g., TMDB updates)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### meta:events Stream
The `meta:events` stream provides reliable event delivery for file system events from meta-core:
### Schema-version sentinel
```
Stream Key: meta:events
Entry Fields:
- type: add | change | delete | rename | reset
- path: File path relative to /files
- midhash256: Content hash (for add/change events)
- oldPath: Previous path (for rename events)
- timestamp: Event timestamp
meta-core:schema-version → STRING (currently "2")
```
Consumer groups enable reliable processing with acknowledgment:
Set at first boot, checked on every subsequent boot
(`internal/storage/schema_version.go`). If the sentinel is missing but
`file:__index__` is non-empty, the gate refuses to boot (legacy data).
If the sentinel is present but doesn't match `SchemaVersion`, the gate
also refuses to boot. Alpha-stage policy: operators wipe and restart;
there is no automated migration.
```bash
# Create consumer group
XGROUP CREATE meta:events mygroup 0 MKSTREAM
## Why flat STRING keys?
# Read new events (blocking)
XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events >
The store does not use Redis Hashes. The reasons are:
# Acknowledge processed events
XACK meta:events mygroup <entry-id>
```
1. **Field-level keyspace notifications.** Redis publishes
`__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
you'd only get one notification on the hash itself. meta-core's
`MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
notifications and publishes structured events to the `meta:events`
Redis Stream.
2. **Selective updates without read-modify-write.** A plugin writing
`tmdb/poster` doesn't have to read the rest of the document.
3. **Compatibility with the CID alias hooks.** Field writes can trigger
alias-maintenance work in `SetProperty` without the plugin caring.
## Primary Hash ID
## Reads and writes
The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from:
The Go API in `internal/storage/client.go`:
1. SHA-256 hash of the middle 1MB of the file
2. File size in bytes
3. IPFS CID v1 encoding (multicodec 0x1000)
| Method | Behaviour |
|---|---|
| `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
| `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...`
## Reconstruction (flat → nested)
This provides:
- Content-based addressing (same file = same ID)
- Fast computation (only reads 1MB)
- IPFS compatibility for future distributed storage
### Collision Handling
If two different files produce the same midhash (rare), the system falls back to full SHA-256:
```typescript
// Normal case
hashId = midHash256; // "bafkr4ih5kapbjzq..."
// Collision detected (different filePath with same midhash)
hashId = metadata['cid_sha2-256']; // Full hash fallback
```
## Metadata Flattening
Nested objects are flattened into path-based keys for efficient Redis storage.
### Flattening Rules
```typescript
// Input (nested object)
{
title: "Inception",
video: {
codec: "h265",
width: "1920"
},
streams: [
{ language: "eng" },
{ language: "jpn" }
]
}
// Output (flattened for Redis)
title = "Inception"
video/codec = "h265"
video/width = "1920"
streams/0/language = "eng"
streams/1/language = "jpn"
```
**Key rules:**
- Nested objects use `/` as path separator
- Arrays use numeric indices (0, 1, 2...)
- All values stored as strings
- `null`/`undefined` stored as empty strings
- Numbers and booleans converted via `String()`
### Reconstruction
When reading, the flattened structure is reconstructed into nested objects:
```typescript
const metadata = await kvClient.getMetadataFlat(hashId);
// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... }
```
## Plugin Metadata Structure
Each plugin writes metadata using a prefixed key pattern:
When the HTTP API returns a document, it reconstructs the flattened
storage back into a nested JSON object:
```
{plugin-id}/{field} # Simple field
{plugin-id}/{group}/{field} # Grouped field
{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays)
flat nested
───────────────────────────────────── ───────────────────────────
title = "Inception" title: "Inception"
fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5",
fileinfo/streamdetails/ streamdetails: { video: { 0: {
video/0/codec = "h264" codec: "h264", ... } }, ... } }
streams/0/language = "eng" streams: [ { language: "eng" }, ... ]
```
### Plugin Contribution Flow
Rules:
- `/` is the path separator.
- Numeric path segments (`0`, `1`, ...) become array indices on read.
- All values are strings on the wire; type interpretation is the
caller's job (and is documented per-field in `METADATA_KEYS.md`).
- `null`/`undefined` round-trip as empty strings.
## Plugin contribution flow
Plugins don't talk to Redis. They write metadata via meta-core's HTTP
API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
writes under a prefix:
```
File Discovered
↓
[FileProcessorPiscina]
├─ Compute midhash256
├─ Store preliminary metadata
└─ Dispatch plugin tasks
↓
[Plugin Processing]
├─ file-info → fileType, mimeType, sizeByte
├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/...
├─ filename-parser → titles/eng, season, episode, videoType
├─ anime-detector → anime, titles/jpn, titles/jpl
├─ tmdb → tmdbid, plot/eng, genres
└─ full-hash → cid_sha2-256, cid_sha1
↓
[Plugin Callbacks]
└─ Store metadata via flat keys (SET per property)
{plugin-id}/<field> single value
{plugin-id}/<group>/<field> grouped
{plugin-id}/<group>/<n>/<field> indexed (array)
```
## Core Metadata Types
See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full,
authoritative list of fields each plugin writes (content identification,
file stats, video classification, anime detection, subtitle references,
Jellyfin metadata, FFmpeg stream details, etc.) and the value formats
(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc
deliberately does not duplicate that registry — it would drift.
### HashMeta - Content Identification
## Event streams
```
cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132)
cid_md5 = "bafkr..." # MD5 (multicodec 0xd5)
cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11)
cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12)
cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000)
```
Two Redis Streams are produced inside meta-core:
### FileStatMeta - File Properties
| Stream | Producer | Contents |
|---|---|---|
| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
```
fileType = "video" # video, audio, image, text
mimeType = "video/x-matroska" # MIME type
sizeByte = "4294967296" # File size in bytes
```
External consumers do **not** read these streams directly. They subscribe
via SSE:
### VideoMeta - Classification
- `GET /api/events/files` — mirror of `file:events`
- `GET /api/events/meta` — mirror of `meta:events`
```
videoType = "movie" # movie, episode, unknown
originalTitle = "Inception"
titles/eng = "Inception" # Localized titles
titles/jpn = "インセプション"
season = "1"
episode = "5"
movieYear = "2010"
```
The SSE wire contract uses `Last-Event-ID` for resume (the opaque
`<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
a client's cursor, the handler emits one `event: gap` payload before
resuming. See `internal/api/sse_events.go` and
[`api-mediated-access.md`](api-mediated-access.md) for the full
contract.
### AnimeMeta - Anime Detection
```
anime = "true" # "true" or "false"
titles/jpn = "進撃の巨人" # Japanese title
titles/jpl = "Shingeki no Kyojin" # Romaji
```
### SubtitleMeta - Subtitle References
```
subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID
subtitles/eng/1 = "bafks..."
subtitles/jpn/0 = "bafkt..."
```
### JellyfinMeta - Comprehensive Metadata
```
imdbid = "tt1375666"
tmdbid = "27205"
anidbid = "12345"
plot/eng = "A movie about..."
genre/0 = "Action"
genre/1 = "Sci-Fi"
rating = "8.8"
mpaa = "PG-13"
releasedate = "2010-07-16"
art/poster = "bafkr..." # Reference to poster image CID
art/fanart = "bafks..." # Reference to backdrop image CID
```
## FFmpeg Plugin - Special Case
The FFmpeg plugin extracts detailed stream information with a specific nested structure.
### Stream Details Structure
```
fileinfo/duration = "7200.5"
fileinfo/formatName = "matroska,webm"
fileinfo/streamdetails/video/0/codec = "h264"
fileinfo/streamdetails/video/0/width = "1920"
fileinfo/streamdetails/video/0/height = "1080"
fileinfo/streamdetails/video/0/aspect = "16:9"
fileinfo/streamdetails/video/0/bitrate = "8000000"
fileinfo/streamdetails/audio/0/codec = "aac"
fileinfo/streamdetails/audio/0/language = "eng"
fileinfo/streamdetails/audio/0/channels = "6"
fileinfo/streamdetails/audio/1/codec = "dts"
fileinfo/streamdetails/audio/1/language = "fra"
fileinfo/streamdetails/audio/1/channels = "6"
fileinfo/streamdetails/subtitle/0/language = "eng"
fileinfo/streamdetails/subtitle/0/format = "subrip"
fileinfo/streamdetails/subtitle/1/language = "jpn"
fileinfo/streamdetails/subtitle/1/format = "ass"
fileinfo/streamdetails/embeddedimage/0/type = "cover"
```
### Stream Type Keys
| Stream Type | Key Pattern | Description |
|------------|-------------|-------------|
| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) |
| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks |
| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles |
| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails |
### Common Stream Fields
**Video streams:**
- `codec` - h264, hevc, av1, vp9, etc.
- `width`, `height` - Resolution
- `aspect` - Aspect ratio
- `bitrate` - Video bitrate
- `profile` - Codec profile (main, high, etc.)
- `level` - Codec level
**Audio streams:**
- `codec` - aac, dts, ac3, flac, etc.
- `language` - ISO 639-2 code (eng, jpn, fra)
- `channels` - Channel count (2, 6, 8)
- `bitrate` - Audio bitrate
- `samplerate` - Sample rate
**Subtitle streams:**
- `language` - ISO 639-2 code
- `format` - subrip, ass, pgs, vobsub
## Real-Time Notifications
MetaMesh provides multiple mechanisms for real-time metadata change notifications:
### 1. meta:events Stream (Recommended)
The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups:
```bash
# File system events from meta-core watcher
XREAD STREAMS meta:events 0
# Event types: add, change, delete, rename, reset
```
Benefits:
- **Reliable delivery** via consumer groups and acknowledgment
- **Replay capability** for catching up after restarts
- **Ordered processing** with stream IDs
### 2. Keyspace Notifications
For field-level change detection, subscribe to Redis keyspace notifications:
```bash
# All file metadata changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Specific plugin updates (e.g., TMDB enrichment)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### Cache Invalidation
The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation:
```
┌─────────────────────────────────────────────────────────────────┐
│ Cache Invalidation Flow │
│ │
│ Redis Keyspace Cache │
│ ┌──────────────┐ Invalidator │
│ │ SET file:*/ │ ──────────► ┌──────────────┐ │
│ │ DEL file:*/ │ subscribe │ Listens to │ │
│ └──────────────┘ │ __keyspace@ │ │
│ │ 0__:file:* │ │
│ └──────┬───────┘ │
│ │ │
│ ▼ │
│ ┌──────────────┐ │
│ │ Remove from │ │
│ │ LRU cache │ │
│ └──────────────┘ │
└─────────────────────────────────────────────────────────────────┘
```
Flow:
1. Cache invalidator subscribes to `__keyspace@0__:file:*`
2. When metadata changes (SET/DEL), invalidator receives notification
3. Invalidator removes the corresponding file from the WebDAV cache
4. Next access fetches fresh copy from source filesystem
This ensures cached files stay synchronized with metadata changes.
### 3. Pub/Sub Channels (Legacy)
Batch notification channels are still supported:
```
meta-sort:file:batch # Batch file updates (every 5 seconds)
meta-sort:scan:reset # Full rescan notification
```
```json
{
"timestamp": 1704067200000,
"changes": [
{ "action": "add", "hashId": "bafkr4ih5..." },
{ "action": "update", "hashId": "bafkr4ih5..." },
{ "action": "remove", "hashId": "bafkr4ih5..." }
]
}
```
## Complete Example
Here's a complete metadata example for a movie file:
```bash
# Query all keys for a file
redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*'
# Example flat keys stored in Redis:
# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..."
# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv"
# file:bafkr4ih5kapbjzq.../fileType = "video"
# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5"
# file:bafkr4ih5kapbjzq.../titles/eng = "Inception"
# ...
# When reconstructed by the API (nested object representation)
{
# Content identification
"cid_midhash256": "bafkr4ih5kapbjzq...",
"cid_sha2-256": "bafks7ij2kbpck...",
"filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv",
# File properties
"fileType": "video",
"mimeType": "video/x-matroska",
"sizeByte": "4294967296",
# Classification
"videoType": "movie",
"originalTitle": "Inception",
"movieYear": "2010",
# Localized titles
"titles": {
"eng": "Inception",
"fra": "Inception",
"jpn": "インセプション"
},
# FFmpeg stream details
"fileinfo": {
"duration": "8878.5",
"formatName": "matroska,webm",
"streamdetails": {
"video": {
"0": {
"codec": "h264",
"width": "1920",
"height": "1080",
"aspect": "16:9",
"bitrate": "8000000"
}
},
"audio": {
"0": { "codec": "dts", "language": "eng", "channels": "6" },
"1": { "codec": "ac3", "language": "fra", "channels": "6" }
},
"subtitle": {
"0": { "language": "eng", "format": "subrip" },
"1": { "language": "fra", "format": "subrip" }
}
}
},
# External metadata (TMDB)
"tmdbid": "27205",
"imdbid": "tt1375666",
"plot": {
"eng": "A thief who steals corporate secrets through dream-sharing technology..."
},
"genre": ["Action", "Science Fiction", "Adventure"],
"rating": "8.4",
"releasedate": "2010-07-16",
# Media assets
"art": {
"poster": "bafkr4poster...",
"fanart": "bafkr4fanart..."
},
# Detection flags
"anime": "false"
}
```
## Key Implementation Files
| Component | File Path |
|-----------|-----------|
| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` |
| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` |
| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` |
| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` |
| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` |
| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` |
| meta-core Storage | `packages/meta-core/internal/storage/client.go` |
| meta-core Events | `packages/meta-core/internal/events/publisher.go` |
| meta-core Cache | `packages/meta-core/internal/cache/` |
Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
inside `MetaPublisher` to drain keyspace notifications, but this is not
the external surface — external services use SSE.
## Debugging
```bash
# List all file hash IDs
docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__'
# Roots
docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
docker exec metacore-app redis-cli SCARD 'file:__index__'
docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
# Count total files
docker exec meta-core-dev redis-cli SCARD 'file:__index__'
# Full document for one root (flat scan)
docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
# Get a sample hash ID
docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__'
# One property
docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
# List all keys for a specific file (flat key scan)
docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*'
# Confirm keys are STRING (not HASH)
docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
# string
# Get specific field (flat key)
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration'
# Reverse-index lookup (bare CID — no algorithm prefix)
docker exec metacore-app redis-cli GET 'cid:bafkreih...' # → uuid
# Get filePath for a file
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath'
# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
# Verify keys are STRING type (not HASH)
docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath'
# Should return: string
# canonical_cid is NOT stored. Derive it via the API instead:
curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
# Monitor metadata changes via stream
docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0
# Schema-version sentinel
docker exec metacore-app redis-cli GET 'meta-core:schema-version'
# Monitor keyspace notifications (requires notify-keyspace-events enabled)
docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*'
# Streams (server-side; external consumers should use SSE)
docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
# meta-core API debugging
# Cache status
curl http://localhost:9000/api/cache/status
# Clear cache
curl -X POST http://localhost:9000/api/cache/clear
# Watch events stream (SSE)
curl http://localhost:9000/api/events/stream
# KV browser - storage info
curl http://localhost:9000/api/kv/info
# Scan status
curl http://localhost:9000/api/scan/status
# HTTP equivalents — preferred
curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>' # CID → document
curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs
curl -k 'https://metacore-dev.localhost:8083/meta/<uuid>' # document by UUID
curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE
curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats
```
## meta:events Stream
## Key implementation files
The `meta:events` stream captures metadata change events for real-time notifications:
```bash
# View recent events
docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10
# Read events from a consumer group
docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' >
```
Stream events contain:
- `type`: Event type (add, change, delete, rename, reset)
- `path`: File path
- `midhash256`: Content hash (for add/change)
- `timestamp`: Event timestamp
| Component | Path |
|---|---|
| Redis client + flat-key ops | `internal/storage/client.go` |
| UUIDv7 minting | `internal/storage/uuid.go` |
| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
| Schema-version gate | `internal/storage/schema_version.go` |
| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
| SSE event mirror | `internal/api/sse_events.go` |
| Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
+362
View File
@@ -0,0 +1,362 @@
# UUID-Rooted Metadata with CID Aliasing
## Status
**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
> ⚠ **Two parts of this proposal were superseded before shipping.** This doc
> is the design record; it is *not* the schema reference. For what is actually
> stored, see `METADATA_KEYS.md` §2/§14.13 and
> [metadata-storage-structure.md](./metadata-storage-structure.md).
>
> 1. **`canonical_cid` is not a stored key.** The proposed reconciler
> (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't
> conflict-free to merge across peers. The canonical CID is **derived on
> read** by ranking the `cids` members (`internal/cid/rank.go` →
> `cid.Better`), and appears only in the `GET /api/meta/{cid}` response.
> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not
> `<algo>:<value>` tokens.** One flat STRING per member:
> `file:<uuid>/cids/<bareCid>` = `"true"`. The reverse index is likewise
> `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
> function, so the `<algo>:` prefix was redundant.
>
> Text below that references either shape describes the *proposal*, not the
> code. The rank ordering and everything else did ship as written.
## Motivation
Today, every metadata entry in Redis is keyed by a content hash, specifically
midhash256:
```
file:midhash256:abc.../filePath
file:midhash256:abc.../title
file:midhash256:abc.../fileinfo/duration
file:midhash256:abc.../tmdb/poster
...
```
Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields
*inside* that entry:
```
file:midhash256:abc.../cid_sha256 = sha256:def...
file:midhash256:abc.../cid_ipfs = bafy...
```
Two problems with this:
1. **midhash256 is privileged by accident.** It IS the key, while every other
CID is a secondary field. There's nothing intrinsically special about
midhash256 — it's just the first hash computed. Hash-algorithm changes, or
new digest types added by plugins, force rewrites of every root. The
"promotion" workarounds people reach for (re-keying entries when a stronger
hash arrives) solve the wrong problem; the privilege itself is the issue.
2. **CID lookup is O(n).** Resolving "given any CID, return that file's
metadata" currently scans every indexed entry (see
`storage/client.go::LookupPathByCID`, which acknowledges this in a
comment). Acceptable for admin tooling, unacceptable for a public API
surface — especially when peers on meta-share may query by any CID variant.
The product requirement is clear: **any CID a client knows about must resolve
to that file's metadata in O(1).** IPFS-compatible CIDs are the externally
meaningful identity for content retrieval; the internal storage layout should
not bias toward any one digest type.
## Goal
A single externally-meaningful contract:
```
GET /api/meta/<cid> → full metadata document for that file
```
Where `<cid>` may be any supported type (midhash256, sha256, IPFS, btih,
sha3, future additions). All CID types are treated symmetrically. No CID type
is "the key."
## Design
### Two-tier key structure
The root key becomes an opaque internal UUID; CIDs are aliases that point to
it.
```
file:<uuid>/<property...> ← canonical storage (UUIDv7, never disclosed externally)
cid:<cid> → <uuid> ← reverse index (one Redis key per known CID)
```
The metadata keyspace under `file:<uuid>/` remains open and flat, exactly as
today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
`file:<uuid>/fileinfo/duration`. Only the root identifier changes.
**Example layout:**
```
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = /watch/Inception.mkv
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size = 4500000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime = 1729872000000000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec… = "true" ← midhash (record address)
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd… = "true" ← sha2-256
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates = SET{/watch/dup1.mkv, /watch/dup2.mkv}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = bafy...
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng = ...
cid:bagacbabaec… → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:bafybeigd… → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
*(As shipped — the proposal originally wrote `cids` as a Redis SET of
`<algo>:<value>` tokens plus a `canonical_cid` scalar. See the Status note.)*
### UUID format: UUIDv7
UUIDv7 chosen over UUIDv4 or 64-bit random:
- **128 bits**: collision-free under distributed minting; no central allocator.
- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a
given window. Restores some of the debuggability lost by going opaque.
- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN
locality.
Encoded as a 26-character Crockford Base32 string (ULID-compatible) for
compactness and case-insensitive matching.
### New root fields owned by meta-core
The metadata schema is open — any writer can put any key under
`file:<uuid>/`. This proposal does not change that. It adds three keys whose
sole writer is meta-core itself:
| Key | Type (as shipped) | Purpose |
|-----|------|---------|
| `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
`subtitle/<lang>`, …) are unaffected.
~~Redis SET is a departure from the all-strings convention in the current
storage doc, but `file:__index__` is already a SET — so the precedent exists
for using sets where they're the right structure.~~ **Reversed for `cids`.**
The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
so that two peers merging the same record can never disagree about set
membership. `duplicates` remained a Redis SET (it is leader-local and never
merged across peers).
### Canonical CID selection
The canonical CID is a label, not a key — so nothing external breaks when it
changes. **As shipped it is not written at all**: it is computed on every read
from the `cids` members, which makes the "update" cost zero and removes the
reconciler entirely.
Selection rule, highest-wins (this part shipped as designed —
`internal/cid/rank.go`):
1. IPFS CID (any variant)
2. SHA-256 / SHA-3 family
3. BitTorrent infohash (v2 preferred over v1)
4. midhash256
Ranked by **utility for external retrieval**, not by cryptographic strength.
midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
externally meaningful for content addressing. btih ranks above midhash because
it enables swarm interop even if it's weaker as a digest.
~~Reconciler runs on the leader, recomputes `canonical_cid` whenever
`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no
reconciler. `GetMetadataDocument` folds the members through `cid.Better` on
each read.
## Internal operations
The meta-core storage interface adds five operations. None are exposed
externally — they're internal to meta-core. External callers see only the
CID-based API.
```go
// Mint creates a new root with no aliases. Called when the watcher
// computes a midhash for a previously-unseen file. Returns the new UUID.
Mint(filePath string, size int64, mtime int64) (uuid string, err error)
// AddAlias registers a CID as resolving to this UUID. Updates both the
// cid:<cid> reverse index and the file:<uuid>/cids/<cid> key-set member.
// (As shipped: no reconciliation step — canonical is derived on read.)
AddAlias(uuid string, cid string) error
// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
GetByCID(cid string) (uuid string, err error)
// MergeIfAliasExists is called when the watcher mints a fresh UUID for a
// file path but, on hashing, discovers the resulting CID already belongs
// to a different UUID. Merges the new (mostly-empty) UUID into the
// existing one, transferring filePath into the duplicates set, and
// returns the canonical UUID.
MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error)
// DeleteRoot removes the entire entry, including every cid:* alias and
// every file:<uuid>/* key. Used for file deletion events.
DeleteRoot(uuid string) error
```
### Lifecycle: file arrival
```
1. Watcher sees /watch/foo.mkv created.
2. Watcher computes midhash256 → "midhash256:abc...".
3. Watcher calls GetByCID("midhash256:abc...").
→ uuid_X (existing entry) or "" (new file).
4a. If existing (uuid_X):
- Add "/watch/foo.mkv" to file:<uuid_X>/duplicates.
- No new root created.
- meta-dup is notified via the existing keyspace event channel.
4b. If new:
- Mint("/watch/foo.mkv", size, mtime) → uuid_Y.
- AddAlias(uuid_Y, "midhash256:abc...").
- Plugins now process the file, referenced by midhash256:abc...
(UUID is never disclosed to them.)
5. Later, the full-hash plugin computes sha256.
- Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
- meta-core resolves midhash256:abc → uuid_Y via GetByCID.
- meta-core calls AddAlias(uuid_Y, "sha256:def...").
- Nothing else to do: the next read of this entry derives a canonical CID
of sha2-256, because it outranks the midhash.
```
### Lifecycle: file deletion
```
1. Watcher sees /watch/foo.mkv deleted.
2. Resolve which entry owns this path:
a. If /watch/foo.mkv is in some entry's duplicates set:
- Remove from set. Done.
b. If /watch/foo.mkv is the filePath of some entry:
- If duplicates is non-empty: promote one duplicate to filePath.
- If duplicates is empty: DeleteRoot(uuid).
- Iterates the cids set, deletes each cid:<cid> reverse-index key.
- Deletes every file:<uuid>/* key.
- Removes uuid from file:__index__.
```
## Public API
```
GET /api/meta/<cid>
→ 200 { "filePath": "...", "canonical_cid": "...",
"cids": [...], "title": "...", "tmdb": {...}, ... }
→ 404 if no such CID
GET /api/file/<cid> (already exists — implementation now O(1))
→ streams file content
```
The UUID is **not** included in the response. Clients address everything by
CID; meta-core's choice of internal identifier remains a private
implementation detail and may change.
Auth: `/api/meta/<cid>` follows the same pattern as `/api/file/<cid>` and
bypasses the auth perimeter. It's the metadata twin of the content endpoint,
and meta-share calls it from federated search responses.
## Cross-service impact
This is a meta-core schema change. Other services interact only through CIDs,
so the impact is contained but real.
- **meta-sort**: writers must resolve plugin callbacks to a UUID before
writing. The existing `LeaderClient` gains a `getByCID` cache.
- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the
ID format changes.
- **meta-stremio**: queries by CID; needs no logic change beyond its storage
client.
- **meta-dup**: gains a cleaner data source — duplicates are now explicit in
the `duplicates` set, not implicit in colliding writes.
- **meta-search / meta-share**: announce the *derived* canonical CID rather
than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata`
rather than reading a stored field. Federated search responses include all
CIDs from the `cids` key-set so any peer querying by any digest gets a hit.
(Written before the 2026-06 discovery/transport split; discovery is now
meta-search's job.)
- **Plugins**: no change. They speak CIDs only.
## Migration
> **Historical — this sweep was never run.** The alpha shipped behind a
> clean-wipe gate (Redis was flushed and the library re-ingested), so no
> in-place migration code exists. Kept for the record.
One-time sweep, idempotent, runs on first boot after upgrade:
```
For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …):
1. Mint a fresh uuid_Y.
2. AddAlias(uuid_Y, hashID). The existing primary becomes an alias.
3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value).
4. RENAME file:<hashID>/<property> → file:<uuid_Y>/<property> for every
property (Lua script, one entry at a time; non-atomic across entries,
but each entry stays internally consistent).
5. Reconcile canonical_cid.
6. Remove hashID from file:__index__, add uuid_Y.
```
Crash-recoverable: re-running the sweep finds entries with both old and new
roots and resumes by detecting which step completed (presence of
`file:<uuid>/cids` indicates step 3 finished; absence of any
`file:<hashID>/*` keys indicates step 4 finished).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. |
| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. |
| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/<cid>` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. |
| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. |
| Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
## Open questions
- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints
a UUID at discovery (before hashing), then `AddAlias` runs when the midhash
arrives. Requires an explicit "reconcile if alias already exists" step
because two paths with the same content would each get their own UUID until
hashing reveals the collision. Worth doing eventually; not required for
this iteration.
- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every*
member of `cids` on the wire (`Record.cids`) and addresses itself by the
single derived canonical (`Record.cid`), so a peer querying by any digest
gets a hit without the DHT footprint of announcing each one separately.
- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
content-streaming endpoint for external IPFS clients? Adjacent to this
design but not part of it.
- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`)
handle `file:<type>:<hash>/...` with a special split rule for the colon
inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies
— but any external consumer subscribed by hashID pattern needs updating.
Inventory those before flipping the migration.
## Non-goals
- **Not** a content-replication or IPFS-pinning system. Registering an IPFS
CID alias does not pin the file anywhere — it only states "if somebody
hashes this file with IPFS's chunker, this is the CID they'd get."
- **Not** a versioning scheme. Metadata is a snapshot of current state;
historical versions are out of scope.
- **Not** a strong-consistency system. Reverse-index writes happen under the
leader lock; reads from followers may briefly miss a freshly-added alias.
+57 -121
View File
@@ -1,140 +1,76 @@
# @metazla/meta-editor
# @meta-core/editor
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface.
React + Vite metadata browser/editor that ships inside the meta-core
container. Walks the UUID-rooted Redis keyspace, edits leaf values,
imports/exports snapshots, and consults the live schema indexer.
## Features
## What it talks to
- **Search**: Find files by title, filename, or hash ID
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more
- **Batch Operations**: Update multiple files simultaneously
- **Real-time**: Direct integration with etcd via REST API
- **User-friendly**: Clean, intuitive interface with form validation
The editor is a pure SPA — it has no server of its own. It calls the
meta-core HTTP API directly under three prefixes (see `src/api/`):
## Development
- `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`,
`value`, `key/{key...}`). This is the UUID-exposing surface — keys look
like `file:<uuid>/<property>` and `cid:<algo>:<value>`.
- `/api/schema` — live per-field schema (type hints, value breakdowns),
produced by meta-core's schema indexer that consumes `meta:events`.
- `/api/snapshot/*` — snapshot export / import / wipe.
### Prerequisites
In production the editor is served by meta-core's nginx (the Vite build
output is copied into the image at the same path) under
`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to
`/editor/` accordingly.
- Node.js 21.6.2+
- pnpm 10.19.0+
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
### Running Locally
```bash
# From the root directory
pnpm run start:editor
# Or from the package directory
cd packages/meta-editor
pnpm run dev
```
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
### Building
```bash
# From the root directory
pnpm run build:editor
# Or from the package directory
cd packages/meta-editor
pnpm run build
```
The built files will be in `dist/` directory.
## Docker Deployment
In the Docker development environment, the editor is automatically available at:
## Project layout
```
http://localhost/editor
```
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
## API Integration
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
- `POST /api/metadata/search` - Search for files
- `GET /api/metadata/:hashId` - Get file metadata
- `PUT /api/metadata/:hashId` - Update complete metadata
- `PUT /api/metadata/:hashId/property` - Update specific property
- `POST /api/metadata/batch` - Batch update multiple files
See `src/api/metadataApi.ts` for the API client implementation.
## Usage
### Searching for Files
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
2. Enter your search query
3. Click "Search"
4. Results will appear in the left sidebar
### Editing Metadata
1. Select a file from the search results
2. Modify any fields in the editor form
3. Click "Save Changes" to persist updates
4. Use "Reset" to discard changes
### Supported Metadata Fields
**Basic Information:**
- Title, Original Title, Show Title
- Video Type (Movie, TV, Anime, Other)
- Year, Season, Episode
**Jellyfin Metadata:**
- Plot, IMDb ID, TMDb ID, AniDB ID
- Director, Studio, Rating, Runtime
- Genres, Premiered Date, Release Date
**File Information:**
- Filename, File Size, MIME Type
- Hash values (SHA-256, MD5, etc.) - read-only
## Project Structure
```
packages/meta-editor/
├── src/
│ ├── api/ # API client for metadata operations
│ ├── components/ # React components
│ │ ├── SearchBar.tsx # Search interface
│ │ ├── FileList.tsx # Search results list
│ │ └── MetadataEditor.tsx # Main editing form
│ ├── types/ # TypeScript type definitions
│ ├── utils/ # Utility functions
│ ├── App.tsx # Main application component
│ ├── main.tsx # Entry point
│ └── index.css # Global styles
packages/meta-core/editor/
├── index.html
├── vite.config.ts
├── tsconfig.json
└── package.json
├── package.json @meta-core/editor
└── src/
├── main.tsx entry
├── api/
│ ├── kvApi.ts /api/kv client
│ ├── schemaApi.ts /api/schema client + cache
│ └── snapshotApi.ts /api/snapshot client
├── kv/ KV browser + leaf/branch editors
├── components/ shared widgets (language combobox, snapshot panel, ...)
└── index.css
```
## Technology Stack
## Local development
- **React 18** - UI framework
- **TypeScript 5.7** - Type safety
- **Vite 6** - Build tool and dev server
- **Native CSS** - Styling (no CSS framework)
Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`.
## Contributing
```bash
cd packages/meta-core/editor
pnpm install
pnpm dev # vite dev server on http://localhost:5173
```
When adding new features:
The Vite dev server proxies `/api/*` to `http://localhost:3000` (see
`vite.config.ts`). For local development against a running meta-core
container, change that target to wherever meta-core is reachable on your
host — e.g. `http://localhost:18083` for the dev compose's direct-backend
port.
1. Add type definitions to `src/types/index.ts`
2. Update API client in `src/api/metadataApi.ts`
3. Create/update components as needed
4. Update this README with new features
No root-level `start:editor` or `build:editor` scripts exist; run pnpm
commands from this directory.
## License
## Build
Part of the MetaMesh project.
```bash
pnpm build # tsc -b && vite build → dist/
```
The container build copies `dist/` into the nginx web root so the editor
is served at `/editor/`.
## Tech
- React 18, TypeScript 5
- Vite 5 (`base: '/editor/'`)
- Native CSS — no UI framework
- `iso-639-3` for language code rendering
+13 -2
View File
@@ -1,13 +1,14 @@
{
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"dependencies": {
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
@@ -1246,6 +1247,16 @@
"node": ">=6.9.0"
}
},
"node_modules/iso-639-3": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/iso-639-3/-/iso-639-3-3.0.1.tgz",
"integrity": "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-185
View File
@@ -1,185 +0,0 @@
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.app-header {
padding: 2rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.app-header h1 {
margin: 0 0 0.5rem 0;
color: #646cff;
}
.app-description {
margin: 0;
color: rgba(255, 255, 255, 0.6);
}
.app-main {
flex: 1;
padding: 2rem;
max-width: 1400px;
width: 100%;
margin: 0 auto;
}
.app-error {
padding: 1rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
border-radius: 8px;
color: #ff6b6b;
margin-bottom: 1rem;
}
.app-loading {
text-align: center;
padding: 3rem 2rem;
color: rgba(255, 255, 255, 0.7);
font-size: 1.1rem;
}
.app-welcome {
text-align: center;
padding: 4rem 2rem;
color: rgba(255, 255, 255, 0.6);
}
.app-welcome h2 {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.8);
}
.app-welcome-features {
margin-top: 1rem;
margin-bottom: 1.5rem;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
}
.app-kv-browser {
margin-top: 1.5rem;
text-align: left;
max-width: 900px;
margin-left: auto;
margin-right: auto;
}
.app-content {
display: grid;
grid-template-columns: 350px 1fr;
gap: 1.5rem;
margin-top: 1rem;
}
.app-sidebar {
min-height: 400px;
}
.app-editor {
min-height: 400px;
}
.app-no-selection {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
min-height: 400px;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
text-align: center;
padding: 2rem;
}
.app-footer {
padding: 1.5rem 2rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-size: 0.9rem;
}
.app-footer a {
color: #646cff;
text-decoration: none;
}
.app-footer a:hover {
text-decoration: underline;
}
@media (max-width: 1024px) {
.app-content {
grid-template-columns: 1fr;
}
.app-sidebar {
order: 2;
}
.app-editor {
order: 1;
}
}
@media (max-width: 768px) {
.app-header,
.app-main,
.app-footer {
padding: 1rem;
}
.app-content {
gap: 1rem;
}
}
@media (prefers-color-scheme: light) {
.app-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.app-description {
color: rgba(0, 0, 0, 0.6);
}
.app-error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.app-loading {
color: rgba(0, 0, 0, 0.7);
}
.app-welcome {
color: rgba(0, 0, 0, 0.6);
}
.app-welcome h2 {
color: rgba(0, 0, 0, 0.8);
}
.app-welcome-features {
color: rgba(0, 0, 0, 0.5);
}
.app-no-selection {
background-color: rgba(0, 0, 0, 0.02);
color: rgba(0, 0, 0, 0.5);
}
.app-footer {
border-top-color: rgba(0, 0, 0, 0.1);
color: rgba(0, 0, 0, 0.5);
}
}
-246
View File
@@ -1,246 +0,0 @@
import { useState, useEffect } from 'react';
import { SearchBar } from './components/SearchBar';
import { FileList } from './components/FileList';
import { MetadataEditor } from './components/MetadataEditor';
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
import { KVTreeView } from './components/KVTreeView';
import { MetadataAPI } from './api/metadataApi';
import { SearchResult } from './types';
import './App.css';
function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
const [bulkEditMode, setBulkEditMode] = useState(false);
const [isSearching, setIsSearching] = useState(false);
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
const [error, setError] = useState<string | null>(null);
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
const getHashFromUrl = (): string | null => {
const path = window.location.pathname;
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
if (fileMatch) {
return fileMatch[1];
}
// Fallback to query param for backwards compatibility
const params = new URLSearchParams(window.location.search);
return params.get('hash');
};
// Update URL when file is selected
const updateUrlForFile = (hashId: string | undefined) => {
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
if (hashId) {
const newPath = `${basePath}/file/${hashId}`;
window.history.pushState({ hashId }, '', newPath);
} else {
window.history.pushState({}, '', basePath || '/editor/');
}
};
// Check for hash parameter in URL on component mount
useEffect(() => {
const loadHashFromUrl = async () => {
const hashParam = getHashFromUrl();
if (hashParam) {
try {
setIsLoadingDirectHash(true);
setError(null);
// Try to get metadata for this hash
const metadata = await MetadataAPI.getMetadata(hashParam);
// Create a search result from the metadata
const searchResult: SearchResult = {
hashId: hashParam,
metadata: metadata,
score: 1.0,
};
// Set the search results and select this file
setSearchResults([searchResult]);
setSelectedHashId(hashParam);
} catch (err: any) {
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsLoadingDirectHash(false);
}
}
};
loadHashFromUrl();
// Handle browser back/forward
const handlePopState = (event: PopStateEvent) => {
const hashId = event.state?.hashId || getHashFromUrl();
if (hashId) {
setSelectedHashId(hashId);
} else {
setSelectedHashId(undefined);
}
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []); // Empty dependency array means this runs once on mount
const handleSearch = async (query: string) => {
try {
setIsSearching(true);
setError(null);
// Unified search - searches across all fields
const response = await MetadataAPI.search({ query });
setSearchResults(response.results);
// Auto-select first result if available
if (response.results.length > 0) {
setSelectedHashId(response.results[0].hashId);
} else {
setSelectedHashId(undefined);
}
} catch (err: any) {
setError(err.message);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsSearching(false);
}
};
const handleSelectFile = (hashId: string) => {
if (!bulkEditMode) {
setSelectedHashId(hashId);
setSelectedHashIds(new Set());
updateUrlForFile(hashId);
}
};
const handleToggleSelection = (hashId: string) => {
const newSelection = new Set(selectedHashIds);
if (newSelection.has(hashId)) {
newSelection.delete(hashId);
} else {
newSelection.add(hashId);
}
setSelectedHashIds(newSelection);
};
const handleSelectAll = () => {
const allHashIds = new Set(searchResults.map(r => r.hashId));
setSelectedHashIds(allHashIds);
};
const handleClearSelection = () => {
setSelectedHashIds(new Set());
setBulkEditMode(false);
};
const handleBulkEditMode = (enabled: boolean) => {
setBulkEditMode(enabled);
if (enabled) {
setSelectedHashId(undefined);
} else {
setSelectedHashIds(new Set());
}
};
const handleMetadataSaved = () => {
// Optionally refresh the search results to reflect changes
console.log('Metadata saved successfully');
};
const handleBulkMetadataSaved = () => {
console.log('Bulk metadata saved successfully');
handleClearSelection();
};
return (
<div className="app">
<header className="app-header">
<h1>MetaMesh Metadata Editor</h1>
<p className="app-description">
Search and edit metadata for your media files
</p>
</header>
<main className="app-main">
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
{error && <div className="app-error">{error}</div>}
{isLoadingDirectHash && (
<div className="app-loading">Loading file metadata...</div>
)}
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
<div className="app-welcome">
<h2>Welcome to MetaMesh Metadata Editor</h2>
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
<p className="app-welcome-features">
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
</p>
<div className="app-kv-browser">
<KVTreeView />
</div>
</div>
)}
{searchResults.length > 0 && (
<div className="app-content">
<div className="app-sidebar">
<FileList
results={searchResults}
selectedHashId={selectedHashId}
selectedHashIds={selectedHashIds}
bulkEditMode={bulkEditMode}
onSelectFile={handleSelectFile}
onToggleSelection={handleToggleSelection}
onSelectAll={handleSelectAll}
onClearSelection={handleClearSelection}
onBulkEditMode={handleBulkEditMode}
/>
</div>
<div className="app-editor">
{bulkEditMode && selectedHashIds.size > 0 ? (
<BulkMetadataEditor
hashIds={Array.from(selectedHashIds)}
onSaved={handleBulkMetadataSaved}
onCancel={handleClearSelection}
/>
) : selectedHashId ? (
<MetadataEditor
key={selectedHashId}
hashId={selectedHashId}
onSaved={handleMetadataSaved}
/>
) : (
<div className="app-no-selection">
{bulkEditMode
? 'Select files from the list to bulk edit metadata'
: 'Select a file from the list to edit its metadata'}
</div>
)}
</div>
</div>
)}
</main>
<footer className="app-footer">
<p>
MetaMesh Metadata Editor | Part of{' '}
<a href="/" target="_blank">
MetaMesh
</a>
</p>
</footer>
</div>
);
}
export default App;
+136
View File
@@ -0,0 +1,136 @@
// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of
// string-valued keys; this module wraps the S3-style listing, search, and
// per-key get/put/delete operations.
const API_BASE = '/api/kv';
export interface KVTreeResponse {
prefix: string;
delimiter: string;
branches: string[];
leaves: string[];
cursor: string;
hasMore: boolean;
}
export interface KVSearchResponse {
contains: string;
keys: string[];
truncated: boolean;
}
export interface KVFindMatch {
key: string;
value: string;
entryPath: string;
field: string;
}
export interface KVFindResponse {
contains: string;
fields: string[];
matches: KVFindMatch[];
truncated: boolean;
}
export interface KVValueResponse {
key: string;
type: string;
value: string;
exists: boolean;
}
export interface KVInfoResponse {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage: string;
}
async function jsonOrThrow<T>(res: Response): Promise<T> {
if (!res.ok) {
let msg = `HTTP ${res.status}`;
try {
const body = await res.json();
if (body?.message) msg = body.message;
else if (body?.error) msg = body.error;
} catch {}
throw new Error(msg);
}
return res.json();
}
export const KVAPI = {
async info(): Promise<KVInfoResponse> {
return jsonOrThrow(await fetch(`${API_BASE}/info`));
},
async tree(params: {
prefix?: string;
delimiter?: string;
cursor?: string;
max?: number;
}): Promise<KVTreeResponse> {
const q = new URLSearchParams();
if (params.prefix !== undefined) q.set('prefix', params.prefix);
if (params.delimiter !== undefined) q.set('delimiter', params.delimiter);
if (params.cursor) q.set('cursor', params.cursor);
if (params.max) q.set('max', String(params.max));
return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`));
},
// Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap
// is reached. Useful for branches with up to a few thousand children where
// pagination UX would just be a worse spinner.
async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
const seenLeaves = new Set<string>();
const seenBranches = new Set<string>();
let cursor = '';
while (true) {
const page = await this.tree({ prefix, delimiter, cursor, max: 1000 });
for (const k of page.leaves) seenLeaves.add(k);
for (const k of page.branches) seenBranches.add(k);
cursor = page.cursor;
if (!page.hasMore) break;
if (seenLeaves.size + seenBranches.size >= softCap) break;
}
return {
prefix,
delimiter,
branches: [...seenBranches].sort(),
leaves: [...seenLeaves].sort(),
cursor,
hasMore: cursor !== '0' && cursor !== '',
};
},
async search(contains: string, limit = 500): Promise<KVSearchResponse> {
const q = new URLSearchParams({ contains, limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`));
},
async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`));
},
async get(key: string): Promise<KVValueResponse> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
},
async put(key: string, value: string): Promise<KVValueResponse> {
const res = await fetch(`${API_BASE}/value`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ key, value }),
});
return jsonOrThrow(res);
},
async del(key: string): Promise<{ key: string; deleted: boolean }> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
},
};
-151
View File
@@ -1,151 +0,0 @@
import {
FileMetadata,
SearchResponse,
UpdateResult,
BatchUpdateResponse,
} from '../types';
const API_BASE = '/api/metadata';
export class MetadataAPI {
/**
* Search for files by query, hashId, or property
*/
static async search(params: {
query?: string;
hashId?: string;
property?: string;
propertyValue?: string;
limit?: number;
}): Promise<SearchResponse> {
const response = await fetch(`${API_BASE}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Search failed');
}
return response.json();
}
/**
* Get metadata for a specific file by hash ID
*/
static async getMetadata(hashId: string): Promise<FileMetadata> {
const response = await fetch(`${API_BASE}/${hashId}`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get metadata');
}
return response.json();
}
/**
* Update complete metadata for a file
*/
static async updateMetadata(
hashId: string,
metadata: FileMetadata
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(metadata),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update metadata');
}
return response.json();
}
/**
* Update a specific property for a file
*/
static async updateProperty(
hashId: string,
property: string,
value: string
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}/property`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ property, value }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update property');
}
return response.json();
}
/**
* Batch update multiple files
*/
static async batchUpdate(
updates: Array<{
hashId: string;
metadata?: FileMetadata;
properties?: Record<string, string>;
}>
): Promise<BatchUpdateResponse> {
const response = await fetch(`${API_BASE}/batch`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ updates }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Batch update failed');
}
return response.json();
}
/**
* Delete metadata for a file
*/
static async deleteMetadata(hashId: string): Promise<void> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'DELETE',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to delete metadata');
}
}
/**
* Get all hash IDs
*/
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
const response = await fetch(`${API_BASE}/hash-ids`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get hash IDs');
}
return response.json();
}
}
+74
View File
@@ -0,0 +1,74 @@
import { useEffect, useState } from 'react';
export type SchemaPrimitive =
| 'string'
| 'int'
| 'float'
| 'bool'
| 'json-object'
| 'json-array'
| 'undefined'
| 'mixed';
export type SchemaHint = 'cid' | 'timestamp';
export type SchemaKeyHint = 'language-code';
export interface SchemaFieldInfo {
type: SchemaPrimitive;
hint?: SchemaHint;
key_hint?: SchemaKeyHint;
breakdown: Record<string, number>;
}
export interface SchemaResponse {
fields: Record<string, SchemaFieldInfo>;
generated_at: string;
source: 'live' | 'rescan';
}
let cached: Promise<SchemaResponse> | null = null;
export async function fetchSchema(): Promise<SchemaResponse> {
if (!cached) {
cached = fetch('/api/schema').then((r) => {
if (!r.ok) {
cached = null;
throw new Error(`schema fetch failed: ${r.status}`);
}
return r.json();
});
}
return cached;
}
/**
* Returns the live schema entry for a field, or undefined if not yet observed.
* Loads /api/schema lazily on first mount and caches the result process-wide.
*/
export function useSchema(): {
schema: SchemaResponse | null;
getFieldHint: (key: string) => SchemaFieldInfo | undefined;
} {
const [schema, setSchema] = useState<SchemaResponse | null>(null);
useEffect(() => {
let alive = true;
fetchSchema()
.then((s) => {
if (alive) setSchema(s);
})
.catch(() => {
// Schema endpoint missing or errored — editor falls back to plugin schemas.
});
return () => {
alive = false;
};
}, []);
const getFieldHint = (key: string): SchemaFieldInfo | undefined => {
if (!schema) return undefined;
return schema.fields[key];
};
return { schema, getFieldHint };
}
+96
View File
@@ -0,0 +1,96 @@
const API_BASE = '/api/snapshot';
export type ImportMode = 'replace' | 'merge';
export type ImportConflict = 'mine' | 'source';
export interface FileImportResult {
hashId: string;
added: number;
updated: number;
unchanged: number;
kept: number;
deleted: number;
error?: string;
}
export interface PluginOutputResult {
written: number;
skipped: number;
}
export interface ImportResult {
mode: ImportMode;
conflict?: ImportConflict;
dryRun: boolean;
totalFiles: number;
filesOk: number;
filesFailed: number;
files?: FileImportResult[];
pluginOutput?: PluginOutputResult;
}
export interface WipeResult {
metadataDeleted: number;
}
export class SnapshotAPI {
/**
* Trigger a download of a metadata snapshot ZIP.
* Browser handles the download via a synthetic <a> click.
*/
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
const parts: string[] = [];
if (includes.pluginOutput) parts.push('plugin-output');
const qs = parts.length ? `?include=${parts.join(',')}` : '';
const a = document.createElement('a');
a.href = `${API_BASE}/export${qs}`;
a.rel = 'noopener';
document.body.appendChild(a);
a.click();
a.remove();
}
/**
* Upload a snapshot ZIP and apply it.
* Mode replace deletes and rewrites each CID's properties.
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
*/
static async runImport(
file: File,
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
): Promise<ImportResult> {
const params = new URLSearchParams();
params.set('mode', opts.mode);
if (opts.mode === 'merge') {
params.set('conflict', opts.conflict ?? 'source');
}
if (opts.dryRun) {
params.set('dry_run', 'true');
}
const fd = new FormData();
fd.append('snapshot', file);
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
method: 'POST',
body: fd,
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Import failed');
}
return response.json();
}
/**
* Wipe Redis metadata. Pass scope=metadata.
*/
static async wipe(scope: 'metadata'): Promise<WipeResult> {
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
method: 'POST',
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Wipe failed');
}
return response.json();
}
}
@@ -1,382 +0,0 @@
.bulk-metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.bulk-metadata-editor-header {
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-editor-header h2 {
margin: 0 0 0.5rem 0;
font-size: 1.5rem;
}
.bulk-info {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.6);
font-size: 0.95rem;
}
.bulk-metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.bulk-metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.bulk-metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.bulk-metadata-editor-form {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
.bulk-metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.bulk-metadata-section-header {
margin-bottom: 1.5rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.section-hint {
margin: 0.25rem 0 0 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.section-hint-bulk {
margin: 0.5rem 0 0 0;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
font-weight: 500;
}
/* Plugin-based bulk fields */
.bulk-fields-container {
display: flex;
flex-direction: column;
gap: 1rem;
}
.bulk-field-wrapper {
border: 1px solid rgba(255, 255, 255, 0.05);
border-radius: 4px;
padding: 0.75rem;
background-color: rgba(0, 0, 0, 0.1);
transition: all 0.2s ease;
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.bulk-field-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.field-key {
font-size: 0.85rem;
font-weight: 600;
color: rgba(255, 255, 255, 0.6);
text-transform: lowercase;
font-family: 'Courier New', monospace;
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.bulk-field-input {
/* Plugin component will render inside this */
}
/* Legacy bulk fields (for compatibility) */
.bulk-metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.bulk-metadata-field:last-child {
margin-bottom: 0;
}
.field-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.field-header label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.clear-field-btn {
width: 20px;
height: 20px;
padding: 0;
border: 1px solid rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
border-radius: 50%;
cursor: pointer;
font-size: 16px;
line-height: 1;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
.bulk-metadata-field input,
.bulk-metadata-field textarea,
.bulk-metadata-field select {
width: 100%;
}
.bulk-metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.bulk-metadata-field-row:last-child {
margin-bottom: 0;
}
/* Modified fields summary */
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-fields h3 {
color: #646cff;
}
.modified-fields-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.modified-field-item {
display: flex;
gap: 0.75rem;
align-items: center;
padding: 0.75rem;
background-color: rgba(100, 108, 255, 0.05);
border: 1px solid rgba(100, 108, 255, 0.2);
border-radius: 4px;
}
.field-name {
font-weight: 600;
color: #646cff;
min-width: 150px;
font-family: 'Courier New', monospace;
font-size: 0.9rem;
}
.field-value {
flex: 1;
word-break: break-word;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
}
.remove-field-btn {
padding: 0.25rem 0.75rem;
font-size: 0.85rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
white-space: nowrap;
}
.remove-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
@media (max-width: 768px) {
.bulk-metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.bulk-metadata-editor-actions {
width: 100%;
flex-direction: column;
}
.bulk-metadata-editor-actions button {
width: 100%;
}
.bulk-metadata-field-row {
grid-template-columns: 1fr;
}
.modified-field-item {
flex-direction: column;
align-items: flex-start;
}
.field-name {
min-width: auto;
}
.remove-field-btn {
align-self: flex-end;
}
}
@media (prefers-color-scheme: light) {
.bulk-metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.bulk-info {
color: rgba(0, 0, 0, 0.6);
}
.bulk-metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.section-hint {
color: rgba(0, 0, 0, 0.5);
}
.section-hint-bulk {
color: rgba(0, 0, 0, 0.4);
}
.bulk-field-wrapper {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.field-key {
color: rgba(0, 0, 0, 0.6);
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.field-header label {
color: rgba(0, 0, 0, 0.7);
}
.field-value {
color: rgba(0, 0, 0, 0.8);
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-field-item {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.2);
}
.clear-field-btn {
border-color: rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #cc0000;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
}
@@ -1,280 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './BulkMetadataEditor.css';
interface PluginsResponse {
plugins: PluginSchema[];
}
interface BulkMetadataEditorProps {
hashIds: string[];
onSaved?: () => void;
onCancel?: () => void;
}
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
// Load plugins and metadata on mount
useEffect(() => {
const loadData = async () => {
try {
setIsLoading(true);
setError(null);
// Fetch plugins
const pluginsResponse = await fetch('/api/plugins');
if (pluginsResponse.ok) {
const data: PluginsResponse = await pluginsResponse.json();
// Sort by priority
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
// Load metadata for all selected files
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
const metadata = await Promise.all(metadataPromises);
// Calculate common values
const commons: Record<string, any> = {};
// Get all editable fields from all plugins
const allFields: string[] = [];
plugins.forEach((plugin) => {
if (plugin.metadataSchema) {
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
if (!field.readonly) {
allFields.push(key);
}
});
}
});
allFields.forEach((field) => {
const values = metadata.map((m) => m[field as keyof FileMetadata]);
// Filter out undefined and null values
const definedValues = values.filter((v) => v !== undefined && v !== null);
if (definedValues.length === 0) {
return;
}
// Safely stringify values
const uniqueValues = Array.from(
new Set(
definedValues.map((v) => {
try {
return JSON.stringify(v);
} catch {
return String(v);
}
})
)
);
if (uniqueValues.length === 1) {
try {
commons[field] = JSON.parse(uniqueValues[0]);
} catch {
commons[field] = uniqueValues[0];
}
} else {
commons[field] = '*';
}
});
setCommonValues(commons);
} catch (err: any) {
setError(`Failed to load data: ${err.message}`);
} finally {
setIsLoading(false);
}
};
loadData();
}, [hashIds]);
const handleFieldChange = (field: string, value: any) => {
setEditedFields({
...editedFields,
[field]: value,
});
};
const handleClearField = (field: string) => {
const newFields = { ...editedFields };
delete newFields[field];
setEditedFields(newFields);
};
const handleSave = async () => {
if (Object.keys(editedFields).length === 0) {
setError('No fields have been edited');
return;
}
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
// Build batch update payload
const updates = hashIds.map((hashId) => ({
hashId,
metadata: editedFields,
}));
const result = await MetadataAPI.batchUpdate(updates);
if (result.errors > 0) {
setError(
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
);
console.error('Batch update errors:', result.results.filter((r) => r.error));
} else {
setSuccessMessage(`Successfully updated ${result.success} files!`);
}
if (onSaved) {
onSaved();
}
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const hasChanges = Object.keys(editedFields).length > 0;
// Filter to only editable plugins (those with non-readonly fields)
const editablePlugins = plugins.filter((plugin) => {
if (!plugin.metadataSchema) return false;
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
});
if (isLoading) {
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-loading">
Loading metadata for {hashIds.length} files...
</div>
</div>
);
}
// Build display metadata (edited values or common values)
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-header">
<h2>Bulk Edit Metadata</h2>
<p className="bulk-info">
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
be updated. Fields showing "*" have different values across files.
</p>
<div className="bulk-metadata-editor-actions">
{onCancel && (
<button onClick={onCancel} disabled={isSaving}>
Cancel
</button>
)}
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
</button>
</div>
</div>
{successMessage && (
<div className="bulk-metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
<div className="bulk-metadata-editor-form">
{editablePlugins.map((plugin) => {
if (!plugin.metadataSchema) return null;
// Get editable fields for this plugin
const editableFields = Object.entries(plugin.metadataSchema).filter(
([, field]) => !field.readonly
);
if (editableFields.length === 0) return null;
return (
<section
key={plugin.id}
className="bulk-metadata-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="bulk-metadata-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="section-hint">{plugin.description}</p>
)}
<p className="section-hint-bulk">
Common values shown. "*" indicates different values. Edit to update all files.
</p>
</div>
<div className="plugin-fields">
{editableFields.map(([key, field]) => (
<DynamicFieldRenderer
key={key}
fieldKey={key}
schema={field}
metadata={displayMetadata as FileMetadata}
onChange={handleFieldChange}
readonly={false}
/>
))}
</div>
</section>
);
})}
{/* Modified Fields Summary */}
{hasChanges && (
<section className="bulk-metadata-section modified-fields">
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
<div className="modified-fields-list">
{Object.entries(editedFields).map(([key, value]) => (
<div key={key} className="modified-field-item">
<span className="field-name">{key}:</span>
<span className="field-value">
{Array.isArray(value)
? value.join(', ')
: typeof value === 'object'
? JSON.stringify(value)
: String(value || '(empty)')}
</span>
<button
className="remove-field-btn"
onClick={() => handleClearField(key)}
title="Remove this field"
>
Remove
</button>
</div>
))}
</div>
</section>
)}
</div>
</div>
);
}
@@ -1,297 +0,0 @@
.dynamic-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.dynamic-field:last-child {
margin-bottom: 0;
}
.dynamic-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
width: 100%;
padding: 0.5rem 0.75rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-size: 0.9rem;
background: rgba(0, 0, 0, 0.3);
color: #fff;
transition: border-color 0.2s;
}
.dynamic-field input:focus,
.dynamic-field select:focus,
.dynamic-field textarea:focus {
outline: none;
border-color: #646cff;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: rgba(0, 0, 0, 0.2);
color: rgba(255, 255, 255, 0.5);
cursor: not-allowed;
}
.dynamic-field textarea {
resize: vertical;
min-height: 80px;
font-family: inherit;
}
.field-hint {
display: block;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
cursor: pointer;
font-weight: normal;
color: rgba(255, 255, 255, 0.7);
}
.checkbox-label input[type="checkbox"] {
width: 1.1rem;
height: 1.1rem;
cursor: pointer;
accent-color: #646cff;
}
.checkbox-label input:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.readonly-value {
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.7);
font-family: 'Courier New', monospace;
font-size: 0.9rem;
word-break: break-all;
}
.field-link {
color: #646cff;
text-decoration: none;
}
.field-link:hover {
text-decoration: underline;
}
/* CID field styling */
.cid-field .cid-value-container {
display: flex;
align-items: center;
gap: 0.5rem;
}
.cid-field .cid-value {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: #a0d8ef;
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cid-field .cid-value:hover {
background: rgba(0, 0, 0, 0.4);
border-color: #646cff;
}
.cid-field .cid-open-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 0.75rem;
background: #646cff;
color: #fff;
border-radius: 4px;
text-decoration: none;
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
transition: background 0.2s;
}
.cid-field .cid-open-button:hover {
background: #535bf2;
}
/* JSON field styling */
.json-field .json-viewer {
background: rgba(0, 0, 0, 0.4);
color: #d4d4d4;
padding: 1rem;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
overflow-x: auto;
max-height: 300px;
overflow-y: auto;
margin: 0;
white-space: pre-wrap;
word-break: break-all;
border: 1px solid rgba(255, 255, 255, 0.1);
}
/* Language string field styling */
.language-string-field .language-entries {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.language-string-field .language-entry {
display: flex;
align-items: center;
gap: 0.5rem;
}
.language-string-field .lang-input {
flex: 1;
min-width: 0;
}
.language-string-field .lang-code {
display: inline-block;
width: 50px;
padding: 0.4rem 0.5rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.8rem;
color: #a0d8ef;
text-align: center;
flex-shrink: 0;
}
.language-string-field .lang-text {
flex: 1;
padding: 0.4rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.language-string-field .lang-remove-btn {
width: 32px;
height: 32px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 100, 100, 0.2);
border: 1px solid rgba(255, 100, 100, 0.3);
border-radius: 4px;
color: #ff6b6b;
font-size: 1.2rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-remove-btn:hover {
background: rgba(255, 100, 100, 0.3);
border-color: rgba(255, 100, 100, 0.5);
}
.language-string-field .lang-add-btn {
align-self: flex-start;
padding: 0.4rem 0.75rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
color: #646cff;
font-size: 0.85rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-add-btn:hover {
background: rgba(100, 108, 255, 0.3);
border-color: rgba(100, 108, 255, 0.5);
}
/* Light theme */
@media (prefers-color-scheme: light) {
.dynamic-field label {
color: rgba(0, 0, 0, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
border-color: rgba(0, 0, 0, 0.1);
background: #fff;
color: #333;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: #f5f5f5;
color: rgba(0, 0, 0, 0.5);
}
.field-hint {
color: rgba(0, 0, 0, 0.5);
}
.checkbox-label {
color: rgba(0, 0, 0, 0.7);
}
.readonly-value {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
.json-field .json-viewer {
background: #1e1e1e;
border-color: rgba(0, 0, 0, 0.2);
}
.language-string-field .lang-code {
background: rgba(100, 108, 255, 0.1);
color: #4a50c8;
}
.language-string-field .lang-text {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
}
@@ -1,378 +0,0 @@
import { SchemaField, FileMetadata } from '../types';
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
import './DynamicFieldRenderer.css';
interface DynamicFieldRendererProps {
fieldKey: string;
schema: SchemaField;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
readonly?: boolean;
}
/**
* Renders a single metadata field based on its schema definition
*/
export function DynamicFieldRenderer({
fieldKey,
schema,
metadata,
onChange,
readonly = false,
}: DynamicFieldRendererProps) {
const isReadonly = readonly || schema.readonly;
const fieldType = schema.type || 'string';
// Get value from metadata using path-style key
const getValue = (): any => {
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = fieldKey.split('/');
let value: any = metadata;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
};
// Set value in metadata using path-style key
const setValue = (newValue: any) => {
onChange(fieldKey, newValue);
};
const value = getValue();
// Render link if configured
const renderValueWithLink = (displayValue: string) => {
if (schema.link && displayValue) {
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
return (
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
{displayValue}
</a>
);
}
return displayValue;
};
// Render based on field type
switch (fieldType) {
case 'boolean':
return (
<div className="dynamic-field">
<label className="checkbox-label">
<input
type="checkbox"
checked={Boolean(value)}
onChange={(e) => setValue(e.target.checked)}
disabled={isReadonly}
/>
<span>{schema.label}</span>
</label>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'number':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="number"
value={value ?? ''}
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
disabled={isReadonly}
min={schema.min}
max={schema.max}
placeholder={schema.placeholder}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'select':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<select
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
>
<option value="">Select...</option>
{schema.options?.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'text':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<textarea
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
placeholder={schema.placeholder}
rows={4}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'array':
// Convert RecordSet to comma-separated string for display
const arrayValue = (() => {
if (!value) return '';
if (typeof value === 'string') return value;
if (typeof value === 'object') {
return Object.values(value).join(', ');
}
return '';
})();
// Convert comma-separated string back to RecordSet
const handleArrayChange = (str: string) => {
const items = str.split(',').map(s => s.trim()).filter(Boolean);
if (items.length === 0) {
setValue(undefined);
return;
}
const recordSet: Record<string, string> = {};
items.forEach((item, idx) => {
recordSet[idx.toString()] = item;
});
setValue(recordSet);
};
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="text"
value={arrayValue}
onChange={(e) => handleArrayChange(e.target.value)}
disabled={isReadonly}
placeholder={schema.placeholder || 'Comma-separated values'}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'json':
// Aggregate matching keys based on pattern
const jsonData = (() => {
if (!schema.pattern) {
return value;
}
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
const result: Record<string, any> = {};
// Recursively find all matching keys
const findMatching = (obj: any, currentPath: string = '') => {
if (!obj || typeof obj !== 'object') return;
for (const [key, val] of Object.entries(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
// Store in result with relative path
const relativePath = newPath.substring(patternPrefix.length - 1);
if (relativePath) {
setNestedValue(result, relativePath, val);
}
}
if (typeof val === 'object' && val !== null) {
findMatching(val, newPath);
}
}
};
findMatching(metadata);
return Object.keys(result).length > 0 ? result : undefined;
})();
if (!jsonData) {
return null;
}
return (
<div className="dynamic-field json-field">
<label>{schema.label}</label>
<pre className="json-viewer">
{JSON.stringify(jsonData, null, 2)}
</pre>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'cid':
// CID field type - displays CID with "Open in Editor" button
if (!value) {
return null;
}
const cidValue = String(value);
const editorUrl = `/editor/file/${cidValue}`;
return (
<div className="dynamic-field cid-field">
<label>{schema.label}</label>
<div className="cid-value-container">
<code
className="cid-value"
onClick={() => navigator.clipboard.writeText(cidValue)}
title="Click to copy"
>
{cidValue}
</code>
<a
href={editorUrl}
className="cid-open-button"
title="Open in Editor"
target="_blank"
rel="noopener noreferrer"
>
↗ Open
</a>
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'languageString':
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
const langEntries: Array<[string, string]> = value && typeof value === 'object'
? Object.entries(value).map(([k, v]) => [k, String(v)])
: [];
const handleLangCodeChange = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const newValue = { ...value };
const text = newValue[oldLang];
delete newValue[oldLang];
newValue[newLang] = text;
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleLangTextChange = (lang: string, text: string) => {
const newValue = { ...value, [lang]: text };
setValue(newValue);
};
const handleRemoveLang = (lang: string) => {
const newValue = { ...value };
delete newValue[lang];
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleAddLang = () => {
// Find a language code not already used, default to 'eng'
const usedLangs = new Set(langEntries.map(([k]) => k));
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
setValue({ ...value, [defaultLang]: '' });
};
if (isReadonly && langEntries.length === 0) {
return null;
}
return (
<div className="dynamic-field language-string-field">
<label>{schema.label}</label>
<div className="language-entries">
{langEntries.map(([lang, text]) => (
<div key={lang} className="language-entry">
{isReadonly ? (
<>
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
<span className="lang-text">{text}</span>
</>
) : (
<>
<LanguageCombobox
value={lang}
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
/>
<input
type="text"
className="lang-input"
value={text}
onChange={(e) => handleLangTextChange(lang, e.target.value)}
placeholder={schema.placeholder || 'Text...'}
/>
<button
type="button"
className="lang-remove-btn"
onClick={() => handleRemoveLang(lang)}
title="Remove"
>
×
</button>
</>
)}
</div>
))}
{!isReadonly && (
<button
type="button"
className="lang-add-btn"
onClick={handleAddLang}
>
+ Add Language
</button>
)}
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'string':
default:
// Show nothing if no value and readonly
if (isReadonly && !value) {
return null;
}
return (
<div className="dynamic-field">
<label>{schema.label}</label>
{isReadonly ? (
<div className="readonly-value">
{renderValueWithLink(String(value ?? ''))}
</div>
) : (
<input
type="text"
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
placeholder={schema.placeholder}
/>
)}
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
}
}
// Helper to set nested value in object
function setNestedValue(obj: any, path: string, value: any) {
const parts = path.split('/').filter(Boolean);
let current = obj;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
}
current = current[parts[i]];
}
if (parts.length > 0) {
current[parts[parts.length - 1]] = value;
}
}
@@ -1,63 +0,0 @@
.plugin-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.plugin-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-section-header .plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.plugin-fields {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Responsive grid for fields */
@media (min-width: 768px) {
.plugin-fields {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1rem;
}
/* Make text and json fields span full width */
.plugin-fields .dynamic-field:has(textarea),
.plugin-fields .json-field {
grid-column: 1 / -1;
}
}
/* Light theme */
@media (prefers-color-scheme: light) {
.plugin-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-section-header .plugin-description {
color: rgba(0, 0, 0, 0.5);
}
}
@@ -1,126 +0,0 @@
import { PluginSchema, FileMetadata, SchemaField } from '../types';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './DynamicPluginSection.css';
interface DynamicPluginSectionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
}
/**
* Renders a plugin section with all its fields based on schema
*/
export function DynamicPluginSection({
plugin,
metadata,
onChange,
}: DynamicPluginSectionProps) {
const schema = plugin.metadataSchema;
if (!schema || Object.keys(schema).length === 0) {
return null;
}
// Check if plugin has any data in metadata
const hasData = checkPluginHasData(schema, metadata);
if (!hasData) {
return null;
}
// Sort fields: non-readonly first, then by key
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
// Non-readonly fields first
if (a.readonly !== b.readonly) {
return a.readonly ? 1 : -1;
}
// Then alphabetically
return keyA.localeCompare(keyB);
});
return (
<section
className="plugin-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="plugin-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="plugin-description">{plugin.description}</p>
)}
</div>
<div className="plugin-fields">
{sortedFields.map(([fieldKey, fieldSchema]) => (
<DynamicFieldRenderer
key={fieldKey}
fieldKey={fieldKey}
schema={fieldSchema}
metadata={metadata}
onChange={onChange}
/>
))}
</div>
</section>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function checkPluginHasData(
schema: Record<string, SchemaField>,
metadata: FileMetadata
): boolean {
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
/**
* Get nested value from object using path-style key
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
/**
* Check if object has any key starting with prefix
*/
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
-174
View File
@@ -1,174 +0,0 @@
.file-list {
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
overflow: hidden;
}
.file-list-header {
padding: 0.75rem 1rem;
background-color: rgba(255, 255, 255, 0.05);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
justify-content: space-between;
align-items: center;
}
.file-count {
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
}
.bulk-edit-toggle {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(100, 108, 255, 0.1);
border: 1px solid rgba(100, 108, 255, 0.3);
color: #646cff;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.bulk-edit-toggle:hover {
background-color: rgba(100, 108, 255, 0.2);
border-color: rgba(100, 108, 255, 0.5);
}
.bulk-edit-toggle.active {
background-color: rgba(100, 108, 255, 0.3);
border-color: #646cff;
font-weight: 500;
}
.file-list-bulk-actions {
padding: 0.75rem 1rem;
background-color: rgba(100, 108, 255, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
gap: 1rem;
align-items: center;
}
.select-all-btn {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(255, 255, 255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.2);
color: rgba(255, 255, 255, 0.9);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.select-all-btn:hover {
background-color: rgba(255, 255, 255, 0.2);
border-color: rgba(255, 255, 255, 0.3);
}
.selection-count {
font-size: 0.9em;
color: #646cff;
font-weight: 500;
}
.file-list-items {
max-height: 500px;
overflow-y: auto;
}
.file-list-item {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
cursor: pointer;
transition: background-color 0.2s;
display: flex;
gap: 0.75rem;
align-items: flex-start;
}
.file-list-item-checkbox {
padding-top: 0.2rem;
}
.file-list-item-checkbox input[type='checkbox'] {
width: 18px;
height: 18px;
cursor: pointer;
}
.file-list-item-content {
flex: 1;
}
.file-list-item:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.2);
border-left: 3px solid #646cff;
}
.file-list-item-title {
font-weight: 500;
margin-bottom: 0.5rem;
}
.file-list-item-details {
display: flex;
gap: 1rem;
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
margin-bottom: 0.25rem;
}
.file-list-item-hash {
font-size: 0.8em;
color: rgba(255, 255, 255, 0.4);
font-family: 'Courier New', monospace;
}
.file-list-empty {
padding: 3rem 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
@media (prefers-color-scheme: light) {
.file-list {
border-color: rgba(0, 0, 0, 0.1);
}
.file-list-header {
background-color: rgba(0, 0, 0, 0.02);
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.file-count {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.file-list-item:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.1);
}
.file-list-item-details {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item-hash {
color: rgba(0, 0, 0, 0.4);
}
.file-list-empty {
color: rgba(0, 0, 0, 0.5);
}
}
-123
View File
@@ -1,123 +0,0 @@
import { SearchResult } from '../types';
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
import './FileList.css';
interface FileListProps {
results: SearchResult[];
selectedHashId?: string;
selectedHashIds: Set<string>;
bulkEditMode: boolean;
onSelectFile: (hashId: string) => void;
onToggleSelection: (hashId: string) => void;
onSelectAll: () => void;
onClearSelection: () => void;
onBulkEditMode: (enabled: boolean) => void;
}
export function FileList({
results,
selectedHashId,
selectedHashIds,
bulkEditMode,
onSelectFile,
onToggleSelection,
onSelectAll,
onClearSelection,
onBulkEditMode,
}: FileListProps) {
if (results.length === 0) {
return (
<div className="file-list-empty">
<p>No files found. Try searching for a title, filename, or hash ID.</p>
</div>
);
}
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
// Sort results by file path
const sortedResults = [...results].sort((a, b) => {
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
return pathA.localeCompare(pathB);
});
return (
<div className="file-list">
<div className="file-list-header">
<span className="file-count">{results.length} files found</span>
<button
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
onClick={() => onBulkEditMode(!bulkEditMode)}
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
>
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
</button>
</div>
{bulkEditMode && (
<div className="file-list-bulk-actions">
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
{allSelected ? 'Deselect All' : 'Select All'}
</button>
{selectedHashIds.size > 0 && (
<span className="selection-count">
{selectedHashIds.size} selected
</span>
)}
</div>
)}
<div className="file-list-items">
{sortedResults.map((result) => {
const isSelected = bulkEditMode
? selectedHashIds.has(result.hashId)
: selectedHashId === result.hashId;
return (
<div
key={result.hashId}
className={`file-list-item ${isSelected ? 'selected' : ''}`}
onClick={() => {
if (bulkEditMode) {
onToggleSelection(result.hashId);
} else {
onSelectFile(result.hashId);
}
}}
>
{bulkEditMode && (
<div className="file-list-item-checkbox">
<input
type="checkbox"
checked={selectedHashIds.has(result.hashId)}
onChange={(e) => {
e.stopPropagation();
onToggleSelection(result.hashId);
}}
/>
</div>
)}
<div className="file-list-item-content">
<div className="file-list-item-title">
{getDisplayName(result.metadata)}
</div>
<div className="file-list-item-details">
<span className="file-list-item-type">
{result.metadata.videoType || 'Unknown'}
</span>
<span className="file-list-item-size">
{formatBytes(result.metadata.byteSize)}
</span>
</div>
<div className="file-list-item-hash">
Hash: {truncateHash(result.hashId)}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}
-247
View File
@@ -1,247 +0,0 @@
.kv-tree {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.1);
overflow: hidden;
}
.kv-tree-header {
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.kv-tree-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.8);
}
.kv-tree-info {
display: flex;
gap: 1rem;
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-refresh {
margin-left: auto;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
background-color: rgba(100, 108, 255, 0.2);
color: #646cff;
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.kv-tree-refresh:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.3);
}
.kv-tree-refresh:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-error {
padding: 0.5rem 1rem;
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
font-size: 0.8rem;
}
.kv-tree-content {
max-height: 400px;
overflow-y: auto;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 0.8rem;
}
.kv-tree-node {
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-node:last-child {
border-bottom: none;
}
.kv-tree-key {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-key:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-toggle {
width: 1rem;
color: rgba(255, 255, 255, 0.4);
font-size: 0.65rem;
}
.kv-tree-key-name {
flex: 1;
color: #88c0d0;
word-break: break-all;
}
.kv-tree-type {
padding: 0.1rem 0.4rem;
font-size: 0.65rem;
background-color: rgba(255, 255, 255, 0.1);
border-radius: 3px;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-value {
padding: 0.5rem 1rem 0.5rem 2.5rem;
background-color: rgba(0, 0, 0, 0.2);
border-top: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-fields {
width: 100%;
border-collapse: collapse;
}
.kv-tree-fields tr:hover {
background-color: rgba(255, 255, 255, 0.03);
}
.kv-field-name {
padding: 0.25rem 0.5rem 0.25rem 0;
color: #a3be8c;
white-space: nowrap;
vertical-align: top;
width: 1%;
}
.kv-field-value {
padding: 0.25rem 0;
color: rgba(255, 255, 255, 0.7);
word-break: break-all;
}
.kv-field-value pre {
margin: 0;
font-family: inherit;
white-space: pre-wrap;
font-size: 0.75rem;
}
.kv-tree-load-more {
display: block;
width: 100%;
padding: 0.75rem;
font-size: 0.8rem;
background-color: transparent;
color: #646cff;
border: none;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-empty {
padding: 2rem;
text-align: center;
color: rgba(255, 255, 255, 0.4);
}
/* Light mode */
@media (prefers-color-scheme: light) {
.kv-tree {
background-color: rgba(0, 0, 0, 0.02);
border-color: rgba(0, 0, 0, 0.1);
}
.kv-tree-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-header h3 {
color: rgba(0, 0, 0, 0.8);
}
.kv-tree-info {
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-error {
background-color: rgba(255, 0, 0, 0.08);
color: #cc0000;
}
.kv-tree-node {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-key:hover {
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-toggle {
color: rgba(0, 0, 0, 0.4);
}
.kv-tree-key-name {
color: #0077aa;
}
.kv-tree-type {
background-color: rgba(0, 0, 0, 0.08);
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-value {
background-color: rgba(0, 0, 0, 0.02);
border-top-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-fields tr:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.kv-field-name {
color: #228822;
}
.kv-field-value {
color: rgba(0, 0, 0, 0.7);
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-empty {
color: rgba(0, 0, 0, 0.4);
}
}
-216
View File
@@ -1,216 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import './KVTreeView.css';
interface KeyValue {
key: string;
type?: string;
value?: Record<string, any> | string;
expanded?: boolean;
loading?: boolean;
}
interface KVInfo {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage?: string;
}
const API_BASE = '/api/kv';
export function KVTreeView() {
const [keys, setKeys] = useState<KeyValue[]>([]);
const [cursor, setCursor] = useState<string>('0');
const [hasMore, setHasMore] = useState(true);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [info, setInfo] = useState<KVInfo | null>(null);
// Load KV info on mount
useEffect(() => {
fetch(`${API_BASE}/info`)
.then(res => res.json())
.then(data => setInfo(data))
.catch(err => console.error('Failed to load KV info:', err));
}, []);
// Load initial keys
const loadKeys = useCallback(async (reset = false) => {
if (loading) return;
const currentCursor = reset ? '0' : cursor;
if (!reset && currentCursor === '0' && keys.length > 0) return;
setLoading(true);
setError(null);
try {
const res = await fetch(`${API_BASE}/keys?cursor=${currentCursor}&count=50`);
if (!res.ok) throw new Error('Failed to load keys');
const data = await res.json();
const newKeys: KeyValue[] = data.keys.map((key: string) => ({
key,
expanded: false,
loading: false
}));
if (reset) {
setKeys(newKeys);
} else {
setKeys(prev => [...prev, ...newKeys]);
}
setCursor(data.cursor);
setHasMore(data.hasMore);
} catch (err: any) {
setError(err.message);
} finally {
setLoading(false);
}
}, [cursor, keys.length, loading]);
// Load on mount
useEffect(() => {
loadKeys(true);
}, []);
// Toggle key expansion and load value
const toggleKey = async (index: number) => {
const key = keys[index];
if (key.expanded) {
// Collapse
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: false } : k
));
return;
}
// Expand and load value if not already loaded
if (key.value === undefined) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: true } : k
));
try {
const res = await fetch(`${API_BASE}/key/${encodeURIComponent(key.key)}`);
if (!res.ok) throw new Error('Failed to load value');
const data = await res.json();
setKeys(prev => prev.map((k, i) =>
i === index ? {
...k,
type: data.type,
value: data.value,
expanded: true,
loading: false
} : k
));
} catch (err) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: false, value: 'Error loading value' } : k
));
}
} else {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: true } : k
));
}
};
const formatValue = (value: any): string => {
if (value === null) return 'null';
if (value === undefined) return 'undefined';
if (typeof value === 'object') {
return JSON.stringify(value, null, 2);
}
return String(value);
};
const formatSize = (bytes: number): string => {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
};
return (
<div className="kv-tree">
<div className="kv-tree-header">
<h3>KV Database Browser</h3>
{info && (
<div className="kv-tree-info">
<span>Keys: {info.keyCount}</span>
<span>Files: {formatSize(info.totalSize)}</span>
{info.memoryUsage && <span>Memory: {info.memoryUsage}</span>}
</div>
)}
<button
className="kv-tree-refresh"
onClick={() => loadKeys(true)}
disabled={loading}
>
{loading ? 'Loading...' : 'Refresh'}
</button>
</div>
{error && <div className="kv-tree-error">{error}</div>}
<div className="kv-tree-content">
{keys.map((item, index) => (
<div key={item.key} className="kv-tree-node">
<div
className={`kv-tree-key ${item.expanded ? 'expanded' : ''}`}
onClick={() => toggleKey(index)}
>
<span className="kv-tree-toggle">
{item.loading ? '...' : (item.expanded ? '▼' : '▶')}
</span>
<span className="kv-tree-key-name">{item.key}</span>
{item.type && <span className="kv-tree-type">{item.type}</span>}
</div>
{item.expanded && item.value && (
<div className="kv-tree-value">
{typeof item.value === 'object' ? (
<table className="kv-tree-fields">
<tbody>
{Object.entries(item.value).map(([field, val]) => (
<tr key={field}>
<td className="kv-field-name">{field}</td>
<td className="kv-field-value">
<pre>{formatValue(val)}</pre>
</td>
</tr>
))}
</tbody>
</table>
) : (
<pre>{formatValue(item.value)}</pre>
)}
</div>
)}
</div>
))}
{hasMore && (
<button
className="kv-tree-load-more"
onClick={() => loadKeys()}
disabled={loading}
>
{loading ? 'Loading...' : 'Load More'}
</button>
)}
{!loading && keys.length === 0 && (
<div className="kv-tree-empty">No keys found</div>
)}
</div>
</div>
);
}
@@ -8,15 +8,6 @@ for (const lang of iso6393) {
languagesByCode.set(lang.iso6393, lang);
}
// Get display name for a language code
export function getLanguageName(code: string): string {
const lang = languagesByCode.get(code);
if (lang) {
return lang.name;
}
return code.toUpperCase();
}
// Format for display: "English (eng)" or "eng" for unknown
export function formatLanguageOption(code: string): string {
const lang = languagesByCode.get(code);
-438
View File
@@ -1,438 +0,0 @@
.metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.metadata-editor-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-header h2 {
margin: 0;
font-size: 1.5rem;
}
.metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.metadata-editor-actions .download-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 1rem;
background-color: #28a745;
color: white;
border-radius: 4px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: background-color 0.2s;
}
.metadata-editor-actions .download-button:hover {
background-color: #218838;
}
.metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.metadata-editor-hashes {
margin-bottom: 1.5rem;
padding: 1rem;
background: rgba(0, 0, 0, 0.3);
border-radius: 8px;
border-left: 4px solid #888;
}
.hashes-header {
margin-bottom: 0.75rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hashes-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.7);
text-transform: uppercase;
letter-spacing: 0.05em;
}
.hashes-grid {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.hash-value {
display: flex;
flex-direction: column;
gap: 0.15rem;
}
.hash-value-primary {
margin-bottom: 0.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hash-value-primary .hash-label {
color: #4ecdc4;
}
.hash-value-primary .hash-code {
color: #4ecdc4;
font-size: 1rem;
}
.hash-label {
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.4);
text-transform: uppercase;
letter-spacing: 0.03em;
}
.hash-code {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.6);
word-break: break-all;
cursor: pointer;
transition: color 0.2s;
}
.hash-code:hover {
color: #fff;
}
/* Plugin Chips */
.metadata-editor-chips {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-bottom: 1.5rem;
}
.plugin-chip {
padding: 0.4rem 0.8rem;
border-radius: 16px;
font-size: 0.85rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
border: 2px solid var(--chip-color);
color: var(--chip-color);
background: transparent;
}
.plugin-chip:hover {
background-color: var(--chip-color);
color: white;
}
.plugin-chip.active {
color: white;
}
/* Plugin Accordion */
.plugin-accordion {
margin-bottom: 0.5rem;
border-radius: 8px;
overflow: hidden;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-accordion-header {
width: 100%;
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem 1.25rem;
border: none;
border-left: 4px solid #646cff;
background-color: rgba(255, 255, 255, 0.05);
color: inherit;
font-size: 1rem;
font-weight: 500;
cursor: pointer;
transition: background-color 0.2s ease;
}
.plugin-accordion-header:hover {
background-color: rgba(255, 255, 255, 0.1);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(255, 255, 255, 0.08);
}
.plugin-accordion-title {
color: inherit;
}
.plugin-accordion-icon {
font-size: 0.75rem;
opacity: 0.6;
transition: transform 0.2s ease;
}
.plugin-accordion-content {
padding: 1rem 1.25rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-form {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.metadata-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.metadata-field:last-child {
margin-bottom: 0;
}
.metadata-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.metadata-field input,
.metadata-field textarea,
.metadata-field select {
width: 100%;
}
.metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.metadata-field-row:last-child {
margin-bottom: 0;
}
.hash-field {
font-family: 'Courier New', monospace;
font-size: 0.9em;
}
.no-hash-info {
margin: 0;
padding: 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
padding: 3rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
.metadata-editor-error button {
margin-top: 1rem;
}
@media (max-width: 768px) {
.metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.metadata-editor-actions {
width: 100%;
}
.metadata-editor-actions button {
flex: 1;
}
.metadata-field-row {
grid-template-columns: 1fr;
}
}
@media (prefers-color-scheme: light) {
.metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-description {
color: rgba(0, 0, 0, 0.5);
}
.metadata-field label {
color: rgba(0, 0, 0, 0.7);
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
color: rgba(0, 0, 0, 0.5);
}
.no-hash-info {
color: rgba(0, 0, 0, 0.5);
}
.metadata-editor-hashes {
background: rgba(0, 0, 0, 0.05);
}
.hashes-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hashes-header h3 {
color: rgba(0, 0, 0, 0.7);
}
.hash-value-primary {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hash-value-primary .hash-label,
.hash-value-primary .hash-code {
color: #0a9e94;
}
.hash-label {
color: rgba(0, 0, 0, 0.4);
}
.hash-code {
color: rgba(0, 0, 0, 0.6);
}
.hash-code:hover {
color: #000;
}
/* Plugin Chips - Light theme */
.plugin-chip {
background: transparent;
}
.plugin-chip:hover,
.plugin-chip.active {
color: white;
}
/* Plugin Accordion - Light theme */
.plugin-accordion {
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.03);
}
.plugin-accordion-header:hover {
background-color: rgba(0, 0, 0, 0.06);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.05);
}
.plugin-accordion-content {
border-top-color: rgba(0, 0, 0, 0.1);
}
}
-392
View File
@@ -1,392 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema, PluginsResponse } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicPluginSection } from './DynamicPluginSection';
import './MetadataEditor.css';
interface PluginAccordionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (field: string, value: any) => void;
isExpanded: boolean;
onToggle: () => void;
}
function PluginAccordion({ plugin, metadata, onChange, isExpanded, onToggle }: PluginAccordionProps) {
return (
<div className={`plugin-accordion ${isExpanded ? 'expanded' : ''}`}>
<button
className="plugin-accordion-header"
onClick={onToggle}
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<span className="plugin-accordion-title">{plugin.name}</span>
<span className="plugin-accordion-icon">{isExpanded ? '▼' : '▶'}</span>
</button>
{isExpanded && (
<div className="plugin-accordion-content">
<DynamicPluginSection
plugin={plugin}
metadata={metadata}
onChange={onChange}
/>
</div>
)}
</div>
);
}
interface MetadataEditorProps {
hashId: string;
onSaved?: () => void;
}
export function MetadataEditor({ hashId, onSaved }: MetadataEditorProps) {
const [metadata, setMetadata] = useState<FileMetadata | null>(null);
const [editedMetadata, setEditedMetadata] = useState<FileMetadata | null>(null);
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
const [expandedPlugins, setExpandedPlugins] = useState<Set<string>>(new Set());
// Fetch plugins schema on mount
useEffect(() => {
fetchPlugins();
}, []);
// Fetch metadata when hashId changes
useEffect(() => {
loadMetadata();
}, [hashId]);
const fetchPlugins = async () => {
try {
const response = await fetch('/api/plugins');
if (response.ok) {
const data: PluginsResponse = await response.json();
// Sort by priority (lower = higher priority)
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
} catch (err) {
console.error('Failed to fetch plugin schemas:', err);
}
};
const loadMetadata = async () => {
try {
setIsLoading(true);
setError(null);
const data = await MetadataAPI.getMetadata(hashId);
setMetadata(data);
setEditedMetadata(data);
} catch (err: any) {
setError(err.message);
} finally {
setIsLoading(false);
}
};
const handleFieldChange = (field: string, value: any) => {
if (!editedMetadata) return;
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = field.split('/');
if (parts.length === 1) {
// Simple key
setEditedMetadata({
...editedMetadata,
[field]: value,
});
} else {
// Nested key - need to update nested structure
const newMetadata = { ...editedMetadata };
let current: any = newMetadata;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
} else {
current[parts[i]] = { ...current[parts[i]] };
}
current = current[parts[i]];
}
current[parts[parts.length - 1]] = value;
setEditedMetadata(newMetadata);
}
};
const handleSave = async () => {
if (!editedMetadata) return;
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
await MetadataAPI.updateMetadata(hashId, editedMetadata);
setMetadata(editedMetadata);
setSuccessMessage('Metadata saved successfully!');
if (onSaved) {
onSaved();
}
// Clear success message after 3 seconds
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleReset = () => {
setEditedMetadata(metadata);
setError(null);
setSuccessMessage(null);
};
const togglePlugin = (pluginId: string) => {
setExpandedPlugins(prev => {
const next = new Set(prev);
if (next.has(pluginId)) {
next.delete(pluginId);
} else {
next.add(pluginId);
}
return next;
});
};
const scrollToPlugin = (pluginId: string) => {
// Expand the plugin if not already expanded
if (!expandedPlugins.has(pluginId)) {
setExpandedPlugins(prev => new Set(prev).add(pluginId));
}
// Scroll to the plugin after a short delay to allow expansion
setTimeout(() => {
const element = document.getElementById(`plugin-${pluginId}`);
if (element) {
element.scrollIntoView({ behavior: 'smooth', block: 'start' });
}
}, 100);
};
if (isLoading) {
return <div className="metadata-editor-loading">Loading metadata...</div>;
}
if (error && !metadata) {
return (
<div className="metadata-editor-error">
<p>Error loading metadata: {error}</p>
<button onClick={loadMetadata}>Retry</button>
</div>
);
}
if (!metadata || !editedMetadata) {
return <div className="metadata-editor-empty">No metadata available</div>;
}
const hasChanges = JSON.stringify(metadata) !== JSON.stringify(editedMetadata);
// Filter plugins that have data
const pluginsWithData = plugins.filter(plugin => {
if (!plugin.metadataSchema) return false;
return hasPluginData(plugin, editedMetadata);
});
// Get file path for download
const filePath = editedMetadata?.filePath as string | undefined;
const downloadUrl = filePath ? `/api/file/download?path=${encodeURIComponent(filePath)}` : null;
return (
<div className="metadata-editor">
<div className="metadata-editor-header">
<h2>Edit Metadata</h2>
<div className="metadata-editor-actions">
{downloadUrl && (
<a
href={downloadUrl}
className="download-button"
title={`Download: ${filePath}`}
download
>
⬇ Download
</a>
)}
{hasChanges && (
<button onClick={handleReset} disabled={isSaving}>
Reset
</button>
)}
<button
onClick={handleSave}
disabled={isSaving || !hasChanges}
className="primary"
>
{isSaving ? 'Saving...' : 'Save Changes'}
</button>
</div>
</div>
{successMessage && (
<div className="metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="metadata-editor-message error">{error}</div>}
<div className="metadata-editor-hashes">
<div className="hashes-header">
<h3>Content Identifiers</h3>
</div>
<div className="hashes-grid">
<HashValue label="midhash256" value={hashId} primary />
{getHashValues(editedMetadata).map(({ key, label, value }) => (
<HashValue key={key} label={label} value={value} />
))}
</div>
</div>
{pluginsWithData.length > 0 && (
<div className="metadata-editor-chips">
{pluginsWithData.map((plugin) => (
<button
key={plugin.id}
className={`plugin-chip ${expandedPlugins.has(plugin.id) ? 'active' : ''}`}
style={{
'--chip-color': plugin.color || '#646cff',
backgroundColor: expandedPlugins.has(plugin.id) ? plugin.color || '#646cff' : 'transparent'
} as React.CSSProperties}
onClick={() => scrollToPlugin(plugin.id)}
>
{plugin.name}
</button>
))}
</div>
)}
<div className="metadata-editor-form">
{pluginsWithData.length === 0 ? (
<div className="metadata-editor-empty">
No metadata found for this file
</div>
) : (
pluginsWithData.map((plugin) => (
<div key={plugin.id} id={`plugin-${plugin.id}`}>
<PluginAccordion
plugin={plugin}
metadata={editedMetadata}
onChange={handleFieldChange}
isExpanded={expandedPlugins.has(plugin.id)}
onToggle={() => togglePlugin(plugin.id)}
/>
</div>
))
)}
</div>
</div>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function hasPluginData(plugin: PluginSchema, metadata: FileMetadata): boolean {
const schema = plugin.metadataSchema;
if (!schema) return false;
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
/**
* Extract all hash values (cid_*) from metadata
*/
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
const hashOrder = [
{ key: 'cid_sha2-256', label: 'SHA-256' },
{ key: 'cid_sha3-256', label: 'SHA3-256' },
{ key: 'cid_sha3-384', label: 'SHA3-384' },
{ key: 'cid_sha1', label: 'SHA-1' },
{ key: 'cid_md5', label: 'MD5' },
{ key: 'cid_crc32', label: 'CRC32' },
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
];
return hashOrder
.filter(({ key }) => (metadata as any)[key])
.map(({ key, label }) => ({
key,
label,
value: (metadata as any)[key],
}));
}
/**
* Display a single hash value with copy functionality
*/
function HashValue({ label, value, primary }: { label: string; value: string; primary?: boolean }) {
const handleCopy = () => {
navigator.clipboard.writeText(value);
};
return (
<div className={`hash-value ${primary ? 'hash-value-primary' : ''}`}>
<span className="hash-label">{label}</span>
<code className="hash-code" onClick={handleCopy} title="Click to copy">
{value}
</code>
</div>
);
}
-52
View File
@@ -1,52 +0,0 @@
.search-bar {
width: 100%;
padding: 1.5rem;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
margin-bottom: 1rem;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
}
.search-input {
flex: 1;
min-width: 200px;
}
.search-input.unified-search {
font-size: 1rem;
}
.search-bar button {
flex: 0 0 auto;
min-width: 100px;
}
.search-hint {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
@media (max-width: 768px) {
.search-controls {
flex-direction: column;
}
.search-input,
.search-bar button {
width: 100%;
}
}
@media (prefers-color-scheme: light) {
.search-hint {
color: rgba(0, 0, 0, 0.5);
}
}
-40
View File
@@ -1,40 +0,0 @@
import { useState } from 'react';
import './SearchBar.css';
interface SearchBarProps {
onSearch: (query: string) => void;
isLoading: boolean;
}
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
const [query, setQuery] = useState('');
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (query.trim()) {
onSearch(query.trim());
}
};
return (
<form className="search-bar" onSubmit={handleSubmit}>
<div className="search-controls">
<input
type="text"
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="Search by title, filename, filepath, or hash ID..."
className="search-input unified-search"
disabled={isLoading}
/>
<button type="submit" disabled={isLoading || !query.trim()}>
{isLoading ? 'Searching...' : 'Search'}
</button>
</div>
<p className="search-hint">
Search across all fields: title, filename, file path, and hash ID
</p>
</form>
);
}
+209
View File
@@ -0,0 +1,209 @@
.snapshot-panel {
max-width: 920px;
margin: 0 auto;
padding: 1.5rem;
}
.snapshot-panel h2 {
margin: 0 0 0.5rem;
}
.snapshot-intro {
color: var(--text-muted, #888);
margin-bottom: 1.5rem;
}
.snapshot-section {
border: 1px solid var(--border, #e0e0e0);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 1.25rem;
}
.snapshot-section.danger {
border-color: #d04a4a;
background: rgba(208, 74, 74, 0.04);
}
.snapshot-section h3 {
margin: 0 0 0.5rem;
}
.snapshot-row {
display: flex;
gap: 1rem;
align-items: flex-start;
margin: 0.75rem 0;
flex-wrap: wrap;
}
.snapshot-file {
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.snapshot-file span {
font-size: 0.85rem;
color: var(--text-muted, #888);
}
.snapshot-fieldset {
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.5rem 0.75rem;
min-width: 280px;
}
.snapshot-fieldset[disabled] {
opacity: 0.5;
}
.snapshot-fieldset legend {
font-weight: 600;
font-size: 0.85rem;
padding: 0 0.25rem;
}
.snapshot-fieldset label {
display: block;
margin: 0.25rem 0;
font-size: 0.9rem;
cursor: pointer;
}
.snapshot-fieldset input[type='radio'] {
margin-right: 0.5rem;
}
.snapshot-btn {
background: var(--bg-button, #f0f0f0);
border: 1px solid var(--border, #c0c0c0);
color: var(--text, #222);
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-btn:hover:not(:disabled) {
background: var(--bg-button-hover, #e0e0e0);
}
.snapshot-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.snapshot-btn.primary {
background: #2a6df4;
border-color: #2a6df4;
color: white;
}
.snapshot-btn.primary:hover:not(:disabled) {
background: #1a5de4;
}
.snapshot-btn.danger {
background: #d04a4a;
border-color: #d04a4a;
color: white;
}
.snapshot-btn.danger:hover:not(:disabled) {
background: #b83838;
}
.snapshot-error {
background: rgba(208, 74, 74, 0.1);
color: #b83838;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-info {
background: rgba(42, 109, 244, 0.08);
color: #1a5de4;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-result {
margin-top: 0.75rem;
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.75rem;
}
.snapshot-result-summary {
display: flex;
gap: 1rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
font-size: 0.9rem;
}
.snapshot-result-table {
width: 100%;
border-collapse: collapse;
font-size: 0.9rem;
}
.snapshot-result-table th,
.snapshot-result-table td {
border: 1px solid var(--border, #e0e0e0);
padding: 0.35rem 0.5rem;
text-align: right;
}
.snapshot-result-table th {
background: var(--bg-subtle, #f5f5f5);
font-weight: 600;
}
.snapshot-result-errors {
margin-top: 0.5rem;
font-size: 0.85rem;
}
.snapshot-result-errors summary {
cursor: pointer;
color: #b83838;
}
.snapshot-result-errors ul {
margin: 0.5rem 0 0;
padding-left: 1.25rem;
}
.snapshot-result-errors code {
font-size: 0.8rem;
word-break: break-all;
}
.snapshot-checkbox {
display: flex;
align-items: flex-start;
gap: 0.5rem;
margin: 0.5rem 0 0.75rem;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-checkbox input {
margin-top: 0.2rem;
}
.snapshot-hint {
color: var(--text-muted, #888);
font-size: 0.9rem;
}
.snapshot-result-aside {
margin-top: 0.5rem;
font-size: 0.85rem;
color: var(--text-muted, #555);
}
+271
View File
@@ -0,0 +1,271 @@
import { useState } from 'react';
import {
SnapshotAPI,
ImportMode,
ImportConflict,
ImportResult,
} from '../api/snapshotApi';
import './SnapshotPanel.css';
export function SnapshotPanel() {
const [exportPluginOutput, setExportPluginOutput] = useState(false);
const [importFile, setImportFile] = useState<File | null>(null);
const [importMode, setImportMode] = useState<ImportMode>('merge');
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
const [importing, setImporting] = useState(false);
const [importResult, setImportResult] = useState<ImportResult | null>(null);
const [importError, setImportError] = useState<string | null>(null);
const [wipeBusy, setWipeBusy] = useState(false);
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
const runImport = async (dryRun: boolean) => {
if (!importFile) {
setImportError('Pick a snapshot ZIP first.');
return;
}
setImporting(true);
setImportError(null);
setImportResult(null);
try {
const result = await SnapshotAPI.runImport(importFile, {
mode: importMode,
conflict: importMode === 'merge' ? importConflict : undefined,
dryRun,
});
setImportResult(result);
} catch (e: any) {
setImportError(e.message || String(e));
} finally {
setImporting(false);
}
};
const runWipe = async () => {
const ok = window.confirm(
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
'edit you made via this editor will be lost. Continue?'
);
if (!ok) return;
setWipeBusy(true);
setWipeMessage(null);
try {
const r = await SnapshotAPI.wipe('metadata');
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
} catch (e: any) {
setWipeMessage(`Failed: ${e.message || e}`);
} finally {
setWipeBusy(false);
}
};
return (
<div className="snapshot-panel">
<h2>Snapshot</h2>
<p className="snapshot-intro">
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
archives — one nested-JSON file per CID, plus a manifest.
</p>
<section className="snapshot-section">
<h3>Export</h3>
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
<label className="snapshot-checkbox">
<input
type="checkbox"
checked={exportPluginOutput}
onChange={(e) => setExportPluginOutput(e.target.checked)}
/>
<span>
Include plugin output
<span className="snapshot-hint">
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
</span>
</span>
</label>
<button
className="snapshot-btn primary"
onClick={() =>
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
}
>
Download snapshot.zip
</button>
</section>
<section className="snapshot-section">
<h3>Import</h3>
<div className="snapshot-row">
<label className="snapshot-file">
<span>Snapshot ZIP</span>
<input
type="file"
accept=".zip,application/zip"
onChange={(e) => {
setImportFile(e.target.files?.[0] ?? null);
setImportResult(null);
setImportError(null);
}}
/>
</label>
</div>
<div className="snapshot-row">
<fieldset className="snapshot-fieldset">
<legend>Mode</legend>
<label>
<input
type="radio"
name="mode"
value="merge"
checked={importMode === 'merge'}
onChange={() => setImportMode('merge')}
/>
Merge — keep existing fields not in the snapshot
</label>
<label>
<input
type="radio"
name="mode"
value="replace"
checked={importMode === 'replace'}
onChange={() => setImportMode('replace')}
/>
Replace — wipe each imported CID and rewrite from snapshot
</label>
</fieldset>
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
<legend>Conflict</legend>
<label>
<input
type="radio"
name="conflict"
value="source"
checked={importConflict === 'source'}
onChange={() => setImportConflict('source')}
/>
Source wins (overwrite my edits)
</label>
<label>
<input
type="radio"
name="conflict"
value="mine"
checked={importConflict === 'mine'}
onChange={() => setImportConflict('mine')}
/>
Mine wins (keep my edits, fill in only missing fields)
</label>
</fieldset>
</div>
<div className="snapshot-row">
<button
className="snapshot-btn"
disabled={!importFile || importing}
onClick={() => runImport(true)}
>
Dry run
</button>
<button
className="snapshot-btn primary"
disabled={!importFile || importing}
onClick={() => runImport(false)}
>
{importing ? 'Importing…' : 'Apply import'}
</button>
</div>
{importError && <div className="snapshot-error">{importError}</div>}
{importResult && <ImportResultView result={importResult} />}
</section>
<section className="snapshot-section danger">
<h3>Wipe metadata</h3>
<p>
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
Files on disk are untouched.
</p>
<button
className="snapshot-btn danger"
disabled={wipeBusy}
onClick={runWipe}
>
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
</button>
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
</section>
</div>
);
}
function ImportResultView({ result }: { result: ImportResult }) {
const totals = (result.files ?? []).reduce(
(acc, f) => {
acc.added += f.added;
acc.updated += f.updated;
acc.unchanged += f.unchanged;
acc.kept += f.kept;
acc.deleted += f.deleted;
return acc;
},
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
);
return (
<div className="snapshot-result">
<div className="snapshot-result-summary">
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
<span>
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
</span>
<span>
mode={result.mode}
{result.conflict ? `, conflict=${result.conflict}` : ''}
</span>
</div>
<table className="snapshot-result-table">
<thead>
<tr>
<th>added</th>
<th>updated</th>
<th>unchanged</th>
<th>kept (mine)</th>
<th>deleted (replace)</th>
</tr>
</thead>
<tbody>
<tr>
<td>{totals.added}</td>
<td>{totals.updated}</td>
<td>{totals.unchanged}</td>
<td>{totals.kept}</td>
<td>{totals.deleted}</td>
</tr>
</tbody>
</table>
{result.pluginOutput && (
<div className="snapshot-result-aside">
plugin-output: {result.pluginOutput.written} written
{result.pluginOutput.skipped > 0
? `, ${result.pluginOutput.skipped} skipped (existed)`
: ''}
</div>
)}
{result.files && result.files.some((f) => f.error) && (
<details className="snapshot-result-errors">
<summary>Per-file errors</summary>
<ul>
{result.files
.filter((f) => f.error)
.map((f) => (
<li key={f.hashId}>
<code>{f.hashId}</code>: {f.error}
</li>
))}
</ul>
</details>
)}
</div>
);
}
+233
View File
@@ -0,0 +1,233 @@
import { useEffect, useMemo, useState } from 'react';
import { KVAPI, KVFindMatch } from '../api/kvApi';
interface Props {
matches: KVFindMatch[];
truncated: boolean;
searching: boolean;
selectedKey: string | null;
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
// FilteredKVTree is the search-active counterpart to KVTree. Instead of
// replacing the tree with a flat results list, it shows only the file
// branches that contain matches, with the matched (field, value) pairs
// rendered inline as sub-lines under each branch.
//
// Branches remain expandable: clicking one fetches the full subtree via
// KVAPI.treeAll just like the main tree, so the user can drill into every
// key of a matching file without clearing the search first.
export function FilteredKVTree({
matches, truncated, searching,
selectedKey, expandedPaths,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: Props) {
// Group matches by their entryPath (file:<hash>/) so each entry is one row
// with potentially multiple hit previews.
const groups = useMemo(() => {
const m = new Map<string, KVFindMatch[]>();
for (const x of matches) {
const arr = m.get(x.entryPath);
if (arr) arr.push(x);
else m.set(x.entryPath, [x]);
}
return m;
}, [matches]);
return (
<div className="kv-filtered">
<div className="kv-search-header">
{searching
? 'searching…'
: `${matches.length} match${matches.length === 1 ? '' : 'es'} in ${groups.size} file${groups.size === 1 ? '' : 's'}${truncated ? ' (truncated)' : ''}`}
</div>
{[...groups.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([entryPath, hits]) => (
<FilteredEntry
key={entryPath}
entryPath={entryPath}
hits={hits}
selectedKey={selectedKey}
expanded={expandedPaths.has(entryPath)}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
))}
</div>
);
}
interface EntryProps {
entryPath: string;
hits: KVFindMatch[];
selectedKey: string | null;
expanded: boolean;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function FilteredEntry({
entryPath, hits, selectedKey, expanded,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: EntryProps) {
// We deliberately do NOT mark the matched-leaf rows as "selected" via the
// tree row's selected style — the inline hit lines are themselves the
// selected indicator (they're highlighted when their key is current).
const isBranchSelected = entryPath === selectedKey;
const HIT_PREVIEW = 4;
return (
<div className="kv-filtered-entry">
<div
className={`kv-tree-row kv-tree-branch${isBranchSelected ? ' selected' : ''}`}
style={{ paddingLeft: 4 }}
onClick={() => {
onToggleBranch(entryPath, !expanded);
onSelectBranch(entryPath);
}}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{entryPath}</span>
<span className="kv-filtered-count" title={`${hits.length} match${hits.length === 1 ? '' : 'es'}`}>
{hits.length}
</span>
</div>
<div className="kv-filtered-hits">
{hits.slice(0, HIT_PREVIEW).map((h) => (
<div
key={h.key}
className={`kv-filtered-hit${h.key === selectedKey ? ' selected' : ''}`}
onClick={(e) => { e.stopPropagation(); onSelectLeaf(h.key); }}
title={h.value}
>
<span className="kv-filtered-hit-field">{h.field}</span>
<span className="kv-filtered-hit-value">{h.value}</span>
</div>
))}
{hits.length > HIT_PREVIEW && (
<div className="kv-filtered-more">
+{hits.length - HIT_PREVIEW} more match{hits.length - HIT_PREVIEW === 1 ? '' : 'es'}
{!expanded && ' · expand to see all keys'}
</div>
)}
</div>
{expanded && (
<LazySubtree
path={entryPath}
depth={1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</div>
);
}
interface SubtreeProps {
path: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
interface SubtreeData {
loading: boolean;
branches: string[];
leaves: string[];
error?: string;
}
// LazySubtree fetches its children once on mount and renders them. Sub-
// branches are themselves LazyBranch components that fetch their own
// children on first expand — recursion all the way down without the global
// expandedPaths state of the main KVTree.
function LazySubtree({ path, depth, selectedKey, onSelectLeaf, onSelectBranch }: SubtreeProps) {
const [data, setData] = useState<SubtreeData>({ loading: true, branches: [], leaves: [] });
useEffect(() => {
let cancelled = false;
KVAPI.treeAll(path, '/', 5000)
.then((r) => { if (!cancelled) setData({ loading: false, branches: r.branches, leaves: r.leaves }); })
.catch((e) => { if (!cancelled) setData({ loading: false, branches: [], leaves: [], error: e.message }); });
return () => { cancelled = true; };
}, [path]);
if (data.loading) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (data.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>{data.error}</div>;
}
return (
<>
{data.branches.map((b) => (
<LazyBranch
key={b}
path={b}
parentPath={path}
depth={depth}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
))}
{data.leaves.map((l) => (
<div
key={l}
className={`kv-tree-row kv-tree-leaf${l === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(l)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{l.slice(path.length)}</span>
</div>
))}
</>
);
}
interface LazyBranchProps {
path: string;
parentPath: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
function LazyBranch({ path, parentPath, depth, selectedKey, onSelectLeaf, onSelectBranch }: LazyBranchProps) {
const [expanded, setExpanded] = useState(false);
const label = path.slice(parentPath.length);
return (
<>
<div
className={`kv-tree-row kv-tree-branch${path === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => { setExpanded(!expanded); onSelectBranch(path); }}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{label}</span>
</div>
{expanded && (
<LazySubtree
path={path}
depth={depth + 1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</>
);
}
+896
View File
@@ -0,0 +1,896 @@
/* KV Tree Editor — fills the iframe viewport with a 2-pane layout. */
.kv-app {
display: flex;
flex-direction: column;
height: 100vh;
width: 100%;
overflow: hidden;
background: #fafbfc;
color: #1f2330;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
font-size: 13px;
}
/* ── Header ───────────────────────────────────────────────────────────── */
.kv-app-header {
border-bottom: 1px solid #e2e6ee;
padding: 8px 12px;
background: #fff;
flex: 0 0 auto;
}
.kv-app-search {
display: flex;
align-items: center;
gap: 8px;
}
.kv-search-input {
flex: 1;
padding: 7px 11px;
border: 1px solid #d1d6e0;
border-radius: 6px;
font-size: 13px;
background: #fff;
outline: none;
}
.kv-search-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-app-stats {
font-size: 12px;
color: #8089a0;
white-space: nowrap;
}
.kv-app-breadcrumb {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 2px;
font-size: 12px;
}
.kv-crumb-label {
color: #8089a0;
margin-right: 4px;
}
.kv-crumb {
border: none;
background: transparent;
padding: 1px 4px;
border-radius: 3px;
cursor: pointer;
color: #4a5170;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-crumb:hover { background: #eef0fb; color: #2a2f48; }
/* ── Body ─────────────────────────────────────────────────────────────── */
.kv-app-body {
flex: 1 1 auto;
display: flex;
min-height: 0;
}
.kv-app-left {
flex: 0 0 38%;
min-width: 280px;
max-width: 600px;
border-right: 1px solid #e2e6ee;
background: #fff;
overflow: auto;
}
.kv-app-right {
flex: 1 1 auto;
overflow: auto;
padding: 16px 20px;
background: #fafbfc;
}
/* ── Tree ─────────────────────────────────────────────────────────────── */
.kv-tree {
padding: 4px 0;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-tree-list {
list-style: none;
margin: 0;
padding: 0;
}
.kv-tree-row {
display: flex;
align-items: center;
gap: 4px;
padding: 2px 4px;
cursor: pointer;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-row:hover { background: #f3f5fb; }
.kv-tree-toggle {
display: inline-block;
width: 12px;
color: #8089a0;
font-size: 10px;
}
.kv-tree-leaf-dot {
display: inline-block;
width: 12px;
color: #c5cad6;
}
.kv-tree-label {
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-branch .kv-tree-label {
color: #2a3050;
font-weight: 500;
}
.kv-tree-branch .kv-tree-filename {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85em;
color: #1e3a8a;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
}
.kv-tree-hash-pill {
margin-left: 0.5rem;
padding: 0.05rem 0.4rem;
background: rgba(100, 108, 255, 0.12);
color: #4a50c8;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.7em;
border-radius: 999px;
flex-shrink: 0;
}
.kv-tree-leaf .kv-tree-label {
color: #4a5170;
}
.kv-tree-row.selected {
background: #e7eafd;
color: #2532a3;
}
.kv-tree-row.selected:hover { background: #dde2fc; }
.kv-tree-loading,
.kv-tree-empty,
.kv-tree-error {
font-size: 12px;
color: #8089a0;
padding: 4px 8px;
font-style: italic;
}
.kv-tree-error { color: #c0392b; }
/* ── Right pane (leaf / branch) ───────────────────────────────────────── */
.kv-leaf, .kv-branch {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-leaf-key {
display: flex;
align-items: center;
gap: 6px;
background: #f0f2f8;
border: 1px solid #e2e6ee;
border-radius: 6px;
padding: 8px 10px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
}
.kv-leaf-key-text {
flex: 1;
word-break: break-all;
color: #2a3050;
}
.kv-leaf-meta {
display: flex;
align-items: center;
gap: 8px;
}
.kv-leaf-label {
font-size: 12px;
color: #6e7790;
font-weight: 500;
}
.kv-leaf-type {
border: 1px solid #d1d6e0;
border-radius: 5px;
padding: 4px 8px;
font-size: 12px;
background: #fff;
}
.kv-leaf-hint {
font-size: 12px;
color: #6e7790;
font-style: italic;
}
.kv-leaf-input {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
box-sizing: border-box;
}
.kv-leaf-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-textarea {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
resize: vertical;
box-sizing: border-box;
font-family: inherit;
}
.kv-leaf-textarea:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-mono {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-url {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-url-open {
font-size: 12px;
color: #6c7afa;
text-decoration: none;
white-space: nowrap;
}
.kv-leaf-actions {
display: flex;
gap: 8px;
}
/* ── CID leaf ─────────────────────────────────────────────────────────── */
.kv-leaf-cid {
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-leaf-cid-controls {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-cid-actions {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
}
.kv-leaf-cid-meta {
font-size: 12px;
color: #6b7280;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-cid-preview {
max-width: 100%;
max-height: 320px;
border-radius: 6px;
border: 1px solid #e2e6ee;
background: #f5f6fa;
object-fit: contain;
align-self: flex-start;
}
.kv-leaf-cid-preview-media {
width: 100%;
max-width: 560px;
}
/* ── File-detail view ─────────────────────────────────────────────────── */
.kv-file {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-file-header {
padding: 14px 16px;
background: #fff;
border: 1px solid #e2e6ee;
border-radius: 8px;
display: grid;
grid-template-columns: 1fr auto;
grid-template-areas:
"name actions"
"meta actions";
gap: 6px 16px;
align-items: center;
}
.kv-file-name {
grid-area: name;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-weight: 600;
font-size: 14px;
color: #1e3a8a;
word-break: break-all;
}
.kv-file-name-fallback {
color: #9ca3af;
font-style: italic;
}
.kv-file-meta {
grid-area: meta;
display: flex;
gap: 12px;
align-items: baseline;
font-size: 12px;
color: #6b7280;
min-width: 0;
}
.kv-file-hash {
background: rgba(100, 122, 250, 0.1);
color: #4a50c8;
padding: 1px 6px;
border-radius: 999px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
flex-shrink: 0;
}
.kv-file-path {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
}
.kv-file-actions {
grid-area: actions;
display: flex;
gap: 8px;
align-items: center;
}
.kv-file-rows {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row {
display: grid;
grid-template-columns: minmax(140px, 220px) 1fr auto;
gap: 12px;
padding: 8px 12px;
background: #fff;
border: 1px solid #e2e6ee;
border-left: 3px solid #d6deed;
border-radius: 6px;
align-items: start;
}
.kv-file-row.pending-delete {
border-left-color: #dc2626;
background: #fef2f2;
}
.kv-file-row.pending-delete .kv-file-row-label {
text-decoration: line-through;
color: #9ca3af;
}
.kv-file-row-deleted-note {
color: #b91c1c;
font-style: italic;
font-size: 12px;
padding: 6px 0;
}
.kv-file-row-delete {
width: 24px;
height: 24px;
align-self: center;
display: inline-flex;
align-items: center;
justify-content: center;
background: transparent;
border: 1px solid transparent;
border-radius: 4px;
color: #dc2626;
font-size: 16px;
line-height: 1;
cursor: pointer;
opacity: 0.4;
transition: opacity 0.15s, background 0.15s, border-color 0.15s;
}
.kv-file-row:hover .kv-file-row-delete,
.kv-file-row.pending-delete .kv-file-row-delete {
opacity: 1;
}
.kv-file-row-delete:hover {
background: rgba(220, 38, 38, 0.1);
border-color: rgba(220, 38, 38, 0.3);
}
.kv-file-add {
display: flex;
gap: 8px;
align-items: center;
padding: 10px 12px;
background: rgba(100, 122, 250, 0.05);
border: 1px dashed rgba(100, 122, 250, 0.3);
border-radius: 6px;
}
.kv-file-add-prefix {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
color: #6b7280;
white-space: nowrap;
flex-shrink: 0;
max-width: 220px;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-file-add > input {
flex: 1;
}
.kv-file-add > input:first-of-type {
flex: 0 0 200px;
}
.kv-file-row.dirty {
border-left-color: #fbbf24;
background: #fffbeb;
}
.kv-file-row.pulsed {
animation: kv-row-pulse 1.4s ease-out;
}
@keyframes kv-row-pulse {
0% { background: rgba(100, 122, 250, 0.35); border-left-color: #4a50c8; }
100% { background: transparent; }
}
.kv-file-row-label,
.kv-file-row-label-static {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
background: none;
border: none;
text-align: left;
cursor: pointer;
padding: 6px 0;
word-break: break-all;
}
.kv-file-row-label-static {
cursor: default;
}
.kv-file-row-label:hover {
color: #4a50c8;
text-decoration: underline;
}
.kv-file-row-value {
display: flex;
min-width: 0;
}
.kv-file-row-value > * {
flex: 1;
min-width: 0;
}
.kv-file-group {
display: flex;
flex-direction: column;
border: 1px solid #e2e6ee;
border-radius: 8px;
background: rgba(100, 122, 250, 0.03);
overflow: hidden;
}
.kv-file-group-header {
display: flex;
align-items: center;
gap: 8px;
padding: 6px 12px;
border: none;
background: rgba(100, 122, 250, 0.1);
cursor: pointer;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
}
.kv-file-group-header:hover {
background: rgba(100, 122, 250, 0.2);
}
.kv-file-group-chevron {
font-size: 10px;
color: #6b7280;
}
.kv-file-group-name {
font-weight: 600;
color: #1e3a8a;
}
.kv-file-group-count {
margin-left: auto;
padding: 1px 8px;
background: rgba(100, 122, 250, 0.2);
color: #4a50c8;
border-radius: 999px;
font-size: 11px;
}
.kv-file-group-body {
padding: 8px;
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row-lang .kv-file-lang-entries {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-lang-entry {
display: flex;
gap: 6px;
align-items: center;
}
.kv-file-lang-entry > .kv-leaf-input {
flex: 1;
}
.kv-file-lang-remove {
color: #dc2626;
}
.kv-file-lang-add {
align-self: flex-start;
background: rgba(100, 122, 250, 0.1);
}
/* ── DateTime leaf ────────────────────────────────────────────────────── */
.kv-leaf-datetime {
display: flex;
gap: 8px;
align-items: center;
}
.kv-leaf-datetime-shape {
font-size: 11px;
color: #6b7280;
background: rgba(100, 122, 250, 0.1);
padding: 2px 6px;
border-radius: 999px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
white-space: nowrap;
}
/* ── Branch ───────────────────────────────────────────────────────────── */
.kv-branch-section {
border: 1px solid #e2e6ee;
border-radius: 6px;
background: #fff;
padding: 12px 14px;
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-branch-section h4 {
margin: 0;
font-size: 13px;
color: #2a3050;
}
.kv-branch-row {
display: flex;
align-items: center;
gap: 6px;
}
.kv-branch-prefix {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
color: #6e7790;
background: #f0f2f8;
padding: 6px 8px;
border-radius: 4px;
white-space: nowrap;
max-width: 50%;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-branch-note {
margin: 0;
font-size: 12px;
color: #6e7790;
line-height: 1.4;
}
/* ── Buttons ──────────────────────────────────────────────────────────── */
.kv-btn {
border: 1px solid #d1d6e0;
background: #fff;
color: #2a3050;
padding: 6px 14px;
border-radius: 5px;
font-size: 12.5px;
cursor: pointer;
}
.kv-btn:hover:not(:disabled) { background: #f3f5fb; border-color: #b8c0d0; }
.kv-btn:disabled { opacity: 0.45; cursor: not-allowed; }
.kv-btn-primary {
background: #6c7afa;
border-color: #6c7afa;
color: #fff;
}
.kv-btn-primary:hover:not(:disabled) {
background: #5868e8;
border-color: #5868e8;
}
.kv-btn-danger {
border-color: #e0788a;
color: #c0392b;
}
.kv-btn-danger:hover:not(:disabled) {
background: #fdecef;
border-color: #c0392b;
}
.kv-btn-icon {
border: 1px solid transparent;
background: transparent;
color: #6e7790;
padding: 2px 6px;
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.kv-btn-icon:hover { background: #eef0fb; color: #2a3050; }
/* ── Pane status ──────────────────────────────────────────────────────── */
.kv-pane-status {
color: #6e7790;
font-size: 12.5px;
padding: 8px 0;
}
.kv-pane-empty {
text-align: center;
padding: 80px 20px;
font-size: 14px;
}
.kv-pane-hint {
margin-top: 8px;
font-size: 12px;
color: #8089a0;
}
/* ── Field chips (configure search-in fields) ─────────────────────────── */
.kv-chips {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 4px;
}
.kv-chips-label {
font-size: 12px;
color: #8089a0;
margin-right: 4px;
}
.kv-chip {
display: inline-flex;
align-items: center;
gap: 4px;
background: #eef0fb;
border: 1px solid #d5dafa;
color: #2a3050;
border-radius: 12px;
padding: 1px 4px 1px 8px;
font-size: 11.5px;
}
.kv-chip code {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11.5px;
}
.kv-chip-x {
border: none;
background: transparent;
color: #6e7790;
cursor: pointer;
padding: 0 4px;
font-size: 14px;
line-height: 1;
}
.kv-chip-x:hover { color: #c0392b; }
.kv-chip-input {
border: 1px dashed #c5cad6;
background: transparent;
border-radius: 12px;
padding: 2px 8px;
font-size: 11.5px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
outline: none;
min-width: 90px;
}
.kv-chip-input:focus { border-style: solid; border-color: #6c7afa; }
/* ── Search header (shared across filter mode) ────────────────────────── */
.kv-search-header {
padding: 6px 10px;
font-size: 11px;
color: #6e7790;
background: #f3f5fb;
border-bottom: 1px solid #e2e6ee;
position: sticky;
top: 0;
z-index: 1;
}
/* ── Filtered tree (search-active state) ──────────────────────────────── */
.kv-filtered {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-filtered-entry {
border-bottom: 1px solid #f0f2f8;
}
.kv-filtered-count {
margin-left: auto;
background: #eef0fb;
color: #4a5170;
padding: 0 6px;
border-radius: 9px;
font-size: 10.5px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-weight: 600;
}
.kv-filtered-hits {
padding: 2px 4px 4px 24px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 12px;
}
.kv-filtered-hit {
display: flex;
gap: 6px;
align-items: baseline;
padding: 2px 6px;
border-radius: 3px;
cursor: pointer;
line-height: 1.35;
}
.kv-filtered-hit:hover { background: #f3f5fb; }
.kv-filtered-hit.selected {
background: #e7eafd;
}
.kv-filtered-hit-field {
background: #eef0fb;
color: #4a5170;
padding: 0 5px;
border-radius: 3px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 10.5px;
flex: 0 0 auto;
}
.kv-filtered-hit-value {
color: #2a3050;
word-break: break-word;
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
}
.kv-filtered-more {
padding: 2px 6px 0;
font-size: 11px;
font-style: italic;
color: #8089a0;
}
/* ── Toasts ───────────────────────────────────────────────────────────── */
.kv-toast-stack {
position: fixed;
bottom: 16px;
right: 16px;
display: flex;
flex-direction: column-reverse;
gap: 6px;
z-index: 1000;
max-width: 480px;
}
.kv-toast {
background: #2a3050;
color: #fff;
border-radius: 6px;
padding: 9px 12px;
box-shadow: 0 4px 14px rgba(0,0,0,0.15);
font-size: 12.5px;
}
.kv-toast-success { background: #2f9e44; }
.kv-toast-error { background: #c0392b; }
.kv-toast-row {
display: flex;
align-items: flex-start;
gap: 8px;
justify-content: space-between;
}
.kv-toast-text {
flex: 1;
word-break: break-word;
}
.kv-toast-close {
border: none;
background: transparent;
color: rgba(255,255,255,0.8);
cursor: pointer;
font-size: 16px;
line-height: 1;
}
.kv-toast-close:hover { color: #fff; }
.kv-toast-details {
margin: 6px 0 0;
padding-left: 18px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11px;
color: rgba(255,255,255,0.85);
max-height: 160px;
overflow: auto;
}
.kv-app-gear {
margin-left: 0.25rem;
font-size: 1.05rem;
opacity: 0.6;
}
.kv-app-gear:hover {
opacity: 1;
}
.kv-modal-backdrop {
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.45);
z-index: 1000;
display: flex;
align-items: flex-start;
justify-content: center;
padding: 4vh 1rem;
overflow-y: auto;
}
.kv-modal {
position: relative;
background: white;
color: inherit;
border-radius: 8px;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
width: min(960px, 100%);
max-height: 92vh;
overflow-y: auto;
}
.kv-modal-close {
position: absolute;
top: 0.5rem;
right: 0.75rem;
background: transparent;
border: none;
font-size: 1.6rem;
line-height: 1;
cursor: pointer;
color: inherit;
opacity: 0.6;
z-index: 1;
}
.kv-modal-close:hover {
opacity: 1;
}
+313
View File
@@ -0,0 +1,313 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI, KVFindMatch, KVInfoResponse } from '../api/kvApi';
import { ancestorBranches, KVTree } from './KVTree';
import { FilteredKVTree } from './FilteredKVTree';
import { KVLeafEditor } from './KVLeafEditor';
import { KVBranchEditor } from './KVBranchEditor';
import { KVFileEditor } from './KVFileEditor';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
import { SearchFieldChips } from './SearchFieldChips';
import { ToastStack, makeToast } from './Toast';
import { ToastEntry } from './types';
import { SnapshotPanel } from '../components/SnapshotPanel';
import './KVApp.css';
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
const DEFAULT_SEARCH_FIELDS = ['originalTitle', 'filePath', 'fileName', 'plot/eng'];
function loadSearchFields(): string[] {
try {
const raw = localStorage.getItem(SEARCH_FIELDS_LS_KEY);
if (!raw) return DEFAULT_SEARCH_FIELDS;
const parsed = JSON.parse(raw);
if (Array.isArray(parsed) && parsed.every((x) => typeof x === 'string')) return parsed;
} catch {}
return DEFAULT_SEARCH_FIELDS;
}
// Top-level shell. Two panes inside the iframe:
// left — lazy tree of the keyspace
// right — leaf editor (when a leaf is selected) or branch editor (branch).
//
// A sticky header carries the search bar, the breadcrumb, and basic stats.
export default function KVApp() {
const [snapshotOpen, setSnapshotOpen] = useState(false);
// Selected node. branchPath is set when a branch is selected; selectedKey
// when a leaf is. Mutually exclusive.
const [selectedKey, setSelectedKey] = useState<string | null>(null);
const [branchPath, setBranchPath] = useState<string | null>(null);
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set(['']));
const [searchInput, setSearchInput] = useState('');
const [searchFields, setSearchFields] = useState<string[]>(loadSearchFields);
const [searchResults, setSearchResults] = useState<KVFindMatch[] | null>(null);
const [searchTruncated, setSearchTruncated] = useState(false);
const [searching, setSearching] = useState(false);
const [info, setInfo] = useState<KVInfoResponse | null>(null);
const [reloadToken, setReloadToken] = useState(0);
const [toasts, setToasts] = useState<ToastEntry[]>([]);
const searchTimerRef = useRef<number | null>(null);
// When the right pane is a file-detail view and the user clicks a leaf
// under the same file branch in the tree, we route the click as a
// scroll-anchor inside the file detail instead of a leaf-editor switch.
// The token bumps each click so identical anchors still trigger a scroll.
const [anchor, setAnchor] = useState<{ field: string; token: number } | null>(null);
const isFileBranch = (p: string | null) => !!p && /^file:[^/]+\/$/.test(p);
const handleLeafClick = (key: string) => {
if (isFileBranch(branchPath) && key.startsWith(branchPath!)) {
setAnchor({ field: key.slice(branchPath!.length), token: (anchor?.token ?? 0) + 1 });
return;
}
setSelectedKey(key);
setBranchPath(null);
setAnchor(null);
};
const handleBranchClick = (path: string) => {
setBranchPath(path);
setSelectedKey(null);
setAnchor(null);
};
const pushToast = (t: ToastEntry) => setToasts((cur) => [...cur, t]);
const dismissToast = (id: number) => setToasts((cur) => cur.filter((t) => t.id !== id));
useEffect(() => {
KVAPI.info().then(setInfo).catch(() => {});
}, [reloadToken]);
// Deep-link: /editor/file/<hashId> opens the tree pre-expanded at that hash.
useEffect(() => {
const m = window.location.pathname.match(/\/file\/([a-z0-9]+)$/i);
if (m) {
const hashId = m[1];
const branch = `file:${hashId}/`;
setExpandedPaths((cur) => new Set([...cur, '', 'file:', branch]));
setBranchPath(branch);
}
}, []);
// Persist field list across reloads.
useEffect(() => {
try { localStorage.setItem(SEARCH_FIELDS_LS_KEY, JSON.stringify(searchFields)); } catch {}
}, [searchFields]);
// Debounced live value-search across configured fields. Empty input clears.
useEffect(() => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
if (!searchInput.trim() || searchFields.length === 0) {
setSearchResults(null);
setSearchTruncated(false);
setSearching(false);
return;
}
setSearching(true);
searchTimerRef.current = window.setTimeout(async () => {
try {
const r = await KVAPI.find(searchInput.trim(), searchFields, 200);
setSearchResults(r.matches);
setSearchTruncated(r.truncated);
} catch (e: any) {
pushToast(makeToast('error', `Search failed: ${e.message}`));
} finally {
setSearching(false);
}
}, 250);
return () => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
};
}, [searchInput, searchFields]);
// Jump to a key from a search hit: select it, expand all ancestor branches.
const jumpToKey = (key: string) => {
const ancestors = ancestorBranches(key);
setExpandedPaths((cur) => new Set([...cur, '', ...ancestors]));
if (key.endsWith('/') || key.endsWith(':')) {
setBranchPath(key);
setSelectedKey(null);
} else {
setSelectedKey(key);
setBranchPath(null);
}
};
const breadcrumb = useMemo(() => {
const target = selectedKey ?? branchPath ?? '';
if (!target) return null;
const parts: { label: string; path: string }[] = [];
let i = 0;
while (i < target.length) {
let next = -1;
for (const d of [':', '/']) {
const idx = target.indexOf(d, i);
if (idx >= 0 && (next < 0 || idx < next)) next = idx;
}
if (next < 0) {
parts.push({ label: target.slice(i), path: target });
break;
}
parts.push({ label: target.slice(i, next + 1), path: target.slice(0, next + 1) });
i = next + 1;
}
return parts;
}, [selectedKey, branchPath]);
return (
<div className="kv-app">
<header className="kv-app-header">
<div className="kv-app-search">
<input
type="search"
className="kv-search-input"
placeholder="filter by value across configured fields…"
value={searchInput}
onChange={(e) => setSearchInput(e.target.value)}
autoFocus
/>
{searchInput && (
<button className="kv-btn-icon" onClick={() => setSearchInput('')} title="clear">×</button>
)}
<span className="kv-app-stats">
{info ? `${info.keyCount} files · ${info.memoryUsage}` : ''}
</span>
<button
className="kv-btn"
onClick={() => setReloadToken((n) => n + 1)}
title="refresh tree"
>Refresh</button>
<button
className="kv-btn-icon kv-app-gear"
onClick={() => setSnapshotOpen(true)}
title="Snapshot — export / import / wipe"
aria-label="Open snapshot settings"
>
⚙
</button>
</div>
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
{breadcrumb && breadcrumb.length > 0 && (
<div className="kv-app-breadcrumb">
<span className="kv-crumb-label">path:</span>
{breadcrumb.map((b, i) => (
<button
key={i}
className="kv-crumb"
onClick={() => jumpToKey(b.path)}
>{b.label}</button>
))}
</div>
)}
</header>
<div className="kv-app-body">
<aside className="kv-app-left">
{searchResults !== null ? (
<FilteredKVTree
matches={searchResults}
truncated={searchTruncated}
searching={searching}
selectedKey={selectedKey}
expandedPaths={expandedPaths}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
) : (
<KVTree
selectedKey={selectedKey}
expandedPaths={expandedPaths}
reloadToken={reloadToken}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
)}
</aside>
<main className="kv-app-right">
{selectedKey ? (
<KVLeafEditor
selectedKey={selectedKey}
reloadToken={reloadToken}
onSaved={() => {
pushToast(makeToast('success', `Saved ${selectedKey}`));
setReloadToken((n) => n + 1);
}}
onDeleted={() => {
pushToast(makeToast('success', `Deleted ${selectedKey}`));
setSelectedKey(null);
setReloadToken((n) => n + 1);
}}
onError={(msg) => pushToast(makeToast('error', msg))}
/>
) : branchPath && FILE_BRANCH_RE.test(branchPath) ? (
<KVFileEditor
key={branchPath}
branchPath={branchPath}
reloadToken={reloadToken}
scrollToField={anchor}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
onSelectLeaf={(k) => { setSelectedKey(k); setBranchPath(null); setAnchor(null); }}
/>
) : branchPath ? (
<KVBranchEditor
branchPath={branchPath}
reloadToken={reloadToken}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
/>
) : (
<div className="kv-pane-status kv-pane-empty">
Select a key from the tree.
<div className="kv-pane-hint">
Click a branch to add / rename / delete a subtree, or a leaf to edit its value.
</div>
</div>
)}
</main>
</div>
{snapshotOpen && (
<div
className="kv-modal-backdrop"
onClick={() => setSnapshotOpen(false)}
>
<div
className="kv-modal"
onClick={(e) => e.stopPropagation()}
role="dialog"
aria-modal="true"
>
<button
className="kv-modal-close"
onClick={() => setSnapshotOpen(false)}
aria-label="Close"
title="Close"
>×</button>
<SnapshotPanel />
</div>
</div>
)}
<ToastStack toasts={toasts} onDismiss={dismissToast} />
</div>
);
}
+194
View File
@@ -0,0 +1,194 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
interface Props {
// Branch path, ends with the delimiter (":" or "/").
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
}
// Branch operations are non-atomic — each per-key write/delete goes one round
// trip to Redis and may fail individually. Failures collect into a list and
// surface via the parent's toast stack. Per the design, we accept partial
// failure and surface what failed.
export function KVBranchEditor({ branchPath, reloadToken, onChanged }: Props) {
const [count, setCount] = useState<number | null>(null);
const [busy, setBusy] = useState(false);
const [newChild, setNewChild] = useState('');
const [newValue, setNewValue] = useState('');
const [renameTo, setRenameTo] = useState('');
// Use the search endpoint with the branch path as the substring to count
// descendants. Truncated when there are more than the limit; we just show
// an approximate count.
useEffect(() => {
let cancelled = false;
KVAPI.search(branchPath, 5000)
.then((r) => {
if (cancelled) return;
// Filter to true descendants (those starting with branchPath).
const descendants = r.keys.filter((k) => k.startsWith(branchPath));
setCount(descendants.length);
})
.catch(() => { if (!cancelled) setCount(null); });
return () => { cancelled = true; };
}, [branchPath, reloadToken]);
// Walk all descendants and run an op against each, collecting failures.
const forEachDescendant = async (op: (key: string) => Promise<void>): Promise<string[]> => {
const r = await KVAPI.search(branchPath, 5000);