Compare commits

...
43 Commits
Author SHA1 Message Date
Worph bb04a3f4dc release (first build on gitea-watch / IPCR)
Build and Push Docker Image / build (push) Successful in 1m0s
2026-10-13 11:02:23 +00:00
Worph 36eb3543b9 gitea workflow: push to IPCR's direct host (Cloudflare 100 MB cap) [skip ci] 2026-10-13 10:02:01 +00:00
Worph 155fd83392 gitea workflow: pull path via metadec.eth
Build and Push Docker Image / build (push) Successful in 56s
2026-10-12 18:30:11 +00:00
Worph cf63a8da68 gitea workflow: publish to IPCR (metadec.eth) instead of ghcr
Build and Push Docker Image / build (push) Successful in 55s
2026-10-12 18:26:08 +00:00
Worph 27ed45999b beacon v2 discovery: advertise metamesh.core + service resources on 239.255.99.1:9099, /api/neighbors ?cap= filter
Build and Push Docker Image / build (push) Failing after 14s
2026-10-04 11:51:03 +00:00
Worph 9254f043a4 README: sync with current code 2026-10-03 11:51:55 +00:00
Worph ebc894fa42 deleting a cids/ member also drops its reverse alias
Build and Push Docker Image / build (push) Failing after 14s
2026-09-16 17:10:00 +00:00
Worph 515ec04369 bugfix and dashboard improvement
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 19:04:23 +02:00
Worph 0403a6f3bc replace file discovery by udp discovery
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 11:24:46 +02:00
Worph 355c666183 chnage service discovery and identity 2026-09-13 19:48:52 +02:00
Worph 0cccceccd9 align with metadata standard
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 15:57:30 +02:00
Worph 40b553016c cid vector
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 14:36:50 +02:00
Worph 4894810fd4 new file tuple API
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 12:01:33 +02:00
Worph ac21315255 improved cid resolution
Build and Push Docker Image / build (push) Failing after 13s
2026-09-12 21:37:38 +02:00
Worph ec1b227631 improved search 2026-09-12 16:37:51 +02:00
Worph 8f4e08b6f1 rework domain field
Build and Push Docker Image / build (push) Failing after 13s
2026-09-02 00:07:02 +02:00
Worph 0c4e3a2227 align to metadata keys doc
Build and Push Docker Image / build (push) Failing after 13s
2026-08-30 19:52:36 +02:00
Worph 1f41ea0166 bugfix scroll
Build and Push Docker Image / build (push) Failing after 14s
2026-08-25 18:09:44 +02:00
Worph 5efdd2c07d improved identity management 2026-08-25 17:48:23 +02:00
Worph 6ae87a2cdb audio support widening 2026-08-21 18:23:20 +02:00
Worph 604e718cdc add identity management
Build and Push Docker Image / build (push) Failing after 13s
2026-08-06 11:03:50 +02:00
Worph 0bbc656263 add identity page
Build and Push Docker Image / build (push) Failing after 13s
2026-08-05 23:37:12 +02:00
Worph a98734171c bugfix indexer field
Build and Push Docker Image / build (push) Failing after 13s
2026-08-03 19:44:22 +02:00
Worph a61057cd81 optimisation on assets retreive 2026-08-03 18:42:05 +02:00
Worph 55d08e77ed improvement on user management
Build and Push Docker Image / build (push) Failing after 14s
2026-08-02 21:35:39 +02:00
Worph 1f084b817c improve identity handling
Build and Push Docker Image / build (push) Failing after 14s
2026-08-01 23:31:49 +02:00
Worph 80050cd2a8 update vector with card locator
Build and Push Docker Image / build (push) Failing after 13s
2026-07-26 18:49:27 +02:00
Worph 75dda1e7d6 new NZB cid support
Build and Push Docker Image / build (push) Failing after 13s
2026-07-25 21:30:46 +02:00
Worph 76169920d6 improve meta core user management
Build and Push Docker Image / build (push) Failing after 13s
2026-07-18 17:30:17 +02:00
Worph df6a2d78f8 bug fix vector
Build and Push Docker Image / build (push) Failing after 14s
2026-07-14 16:02:26 +02:00
Worph 0ca03ac95a deprecate old cids fields 2026-07-13 22:02:29 +02:00
Worph 1798821c53 update metacore default 2026-07-12 14:47:21 +02:00
Worph d268bb83e6 search optimisation
Build and Push Docker Image / build (push) Failing after 13s
2026-07-10 22:56:59 +02:00
Worph 075d6f9bcb cleanup logs 2026-07-10 13:25:16 +02:00
Worph f1cd94b17e bugfix advertise url
Build and Push Docker Image / build (push) Failing after 13s
2026-06-14 18:47:28 +02:00
Pierre 52f5b5cedc multi identity support for metacore 2026-06-06 18:44:52 +02:00
Pierre f8a10091b8 remove in metadata cid_* to put in the cids/ set 2026-05-31 11:41:32 +02:00
Pierre 94ad0c240b bugfix and new internal API 2026-05-25 23:47:30 +02:00
Pierre 415595c1c6 new meta-core api, now redis is not exposed as an "API" 2026-05-24 17:43:17 +02:00
Pierre aa45f4f32e improve editor and add type detection 2026-05-14 23:31:07 +02:00
Pierre 3d65e7e951 doc improvement 2026-05-10 19:47:05 +02:00
Pierre bd833b7be4 improved kv metacore 2026-04-27 10:31:11 +02:00
Pierre 98da535335 improve editor for KV 2026-04-26 15:28:50 +02:00
161 changed files with 24012 additions and 7588 deletions
+6
View File
@@ -0,0 +1,6 @@
# FIX CRLF always when developer has not set
# Linux/Mac: git config --global core.autocrlf input
# Windows: git config --global core.autocrlf true
# Auto detect text files and perform LF normalization
* text=auto
* eol=lf
+87
View File
@@ -0,0 +1,87 @@
# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
# folder exists, and GitHub never reads it.
#
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
name: Build and Push Docker Image
on:
push:
branches:
- main
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Tag for the image (optional, defaults to sha)'
required: false
type: string
env:
REGISTRY: ipcr-push-85-17-246-67.nip.io
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Docker CLI
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
- name: Trust the Mesh Router CA
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
env:
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
buildkitd-config-inline: |
[registry."${{ env.REGISTRY }}"]
ca = ["/tmp/ipcr-ca.pem"]
- name: Log in to IPCR
env:
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
run: |
mkdir -p ~/.docker
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
+2
View File
@@ -23,3 +23,5 @@ go.work
dashboard/node_modules/*
dist/*
dashboard/dist/*
editor/dist/
editor/node_modules/
+18 -9
View File
@@ -7,7 +7,7 @@
# - meta-core Go binary (HTTP API, Mount API, Watcher)
# - nginx (dashboard, WebDAV, API proxy)
# - Redis (managed by supervisord on leader)
# - rclone (remote mount daemon)
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
# - supervisord (process management - runs on leader only)
#
# Architecture:
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
LABEL org.opencontainers.image.licenses=MIT
# Install runtime dependencies
# Pinned rclone release. Alpine's package can lag the latest by several
# minor versions and we depend on VFS cache fixes shipped post-1.65.
ARG RCLONE_VERSION=1.73.5
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
# all remote mounts go through rclone now.
RUN apk add --no-cache \
# Core services
redis \
nginx \
nginx-mod-http-dav-ext \
supervisor \
rclone \
fuse3 \
# Mount utilities
nfs-utils \
cifs-utils \
# Tools
bash \
curl \
python3 \
ca-certificates \
apache2-utils \
# Process utilities
procps
# Install rclone from the official binary release (pinned).
RUN set -eux; \
apk add --no-cache --virtual .rclone-deps unzip; \
cd /tmp; \
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
unzip -q rclone.zip; \
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
apk del .rclone-deps; \
/usr/local/bin/rclone version | head -2
# Copy Go binary from builder
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
RUN chmod +x /usr/local/bin/meta-core
+14
View File
@@ -64,10 +64,24 @@ clean:
rm -f coverage.out coverage.html
# Build Docker image
#
# We tag THREE names:
# - meta-core:$(VERSION) — the canonical versioned tag
# - meta-core:latest — convenience for `docker run meta-core`
# - meta-core:local — the tag every sub-stack docker-compose.yml uses
# (meta-share federation, meta-gateway dev stacks).
# Without this, `make docker` produces an image
# the dev stacks can't see by name, and previously
# -tagged stale `:local` images linger, silently
# breaking downstream API fields (e.g. /urls
# missing webdavUrlInternal → meta-gateway's
# poster auto-store fails). See README.md →
# Quick start → Build for the audit recipe.
docker:
@echo "Building Docker image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) .
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local
# Run locally (for development)
run: build
+402 -456
View File
@@ -1,520 +1,466 @@
# meta-core
A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services.
Standalone Go service that owns Redis, the HTTP+SSE metadata surface, the
User Data Layer store and signing keystore, and a WebDAV file server for the
MetaMesh dev/prod stack.
## Overview
meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul.
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
its `metamesh.core` advertise carries the full `/urls` payload) and then call its
HTTP API. `META_CORE_URL` on a client pins it instead. Redis is not published
outside the container — all metadata I/O is mediated by the HTTP/SSE API
(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
### Key Capabilities
### Key capabilities
| Feature | Description |
|---------|-------------|
| **Leader Election** | flock-based leader election (Go binary only runs as leader) |
| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash |
| **HTTP API** | Language-agnostic REST interface for metadata and service discovery |
| **Service Discovery** | File-based registry with heartbeats and stale detection |
| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations |
| **File Watching** | Directory scanning, MidHash256 computation, event dispatch |
| **Mount Management** | SMB/rclone configuration, health monitoring |
| **WebDAV Server** | File access with nginx proxy_cache |
| **Event Publishing** | Redis keyspace notifications to meta:events stream |
| **Redis-volume mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two Redis writers. It is a mutex, not discovery — nothing reads the lock. |
| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
| **In-memory search index** | `POST /api/metadata/search` matches against an in-memory snapshot of every record, kept current by incremental refreshes (full reconcile every 30 min) — no Redis round-trip per record on the request path (`internal/api/search_index.go`). |
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
### Design Characteristics
### Design characteristics
- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes)
- **Automatic failover** - Lock releases on process death, followers detect and re-elect
- **Thread-safe** - RWMutex protection on all shared state
- **Graceful shutdown** - Signal handling with reverse startup order
- **Static binary** - CGO disabled, single binary deployment
- **Single writer** — flock on the `/meta-core` volume; no external consensus.
- **HTTP-only externally** — Redis is never published by the compose stacks;
debug with `docker exec metacore-app redis-cli …`.
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
on-disk Redis layout predates the current build (alpha clean-wipe policy;
see `docs/uuid-rooted-metadata.md`).
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
image with `redis-server`, `rclone`, nginx, and supervisord.
## Architecture
meta-core runs as a **standalone container** that other services connect to via HTTP API:
```
┌───────────────────────────────────────────────────────────────────────┐
│ meta-core container (standalone) │
│ │
│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │
│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │
│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │
│ │ │ │ │ │ │ │ /health, /services │ │
│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │
│ │
│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │
└──────────────────────────────────────────────┬────────────────────────┘
│
┌──────────────────────────────────────┴──────────────────────────┐
│ /meta-core (shared volume) │
│ ├── locks/kv-leader.info (leader URLs) │
│ ├── db/redis/ (Redis data) │
│ ├── services/ (hostname-based .json) │
│ └── mounts/ (mount configurations) │
└─────────────────────────────────────────────────────────────────┘
▲ ▲ ▲
┌────────────┘ │ └────────────┐
│ │ │
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
│ meta-sort │ │ meta-fuse │ │ meta-stremio │
│ (reads info, │ │ (reads info, │ │ (reads info, │
│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │
└───────────────┘ └───────────────┘ └───────────────┘
┌──────────────────────────────────────────────────────────────────────────┐
│ metacore-app (standalone container) │
│ │
│ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
│ then exec supervisord) │
│ supervisord │
│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
│ ├─ nginx :80 (dashboard, /editor, API proxy, │
│ │ proxy_cache for /webdav, rclone GUI /rclone) │
│ ├─ rclone rcd (mount daemon, RC API on :5572) │
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
│ ─► UDP advertise/listen 239.255.99.1:9099 │
└──────────────────────────────────────────────────────────────────────────┘
│
┌──────────────────────┴──────────────────────┐
│ /meta-core (meta-core's own volume) │
│ locks/kv-leader.lock │
│ db/redis/ │
│ identity/accounts/*.json │
│ mounts/{mounts.json, errors/} │
│ watchers.json │
│ cache/ (nginx proxy_cache) │
└─────────────────────────────────────────────┘
meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
│ 1. hear the metamesh.core advertise on UDP multicast (carries /urls)
└─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never Redis, never the volume
```
## Quick Start
## Quick start
### Build
```bash
# Build binary
make build
# Build Docker image
make docker
# Run tests
make test
make build # build static binary to bin/meta-core
make docker # build container image as meta-core:1.0.0 + :latest + :local
make test # run Go unit tests
```
### Run
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
`:local` tag is the one the sub-stack docker-compose files reference
(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
sub-stacks can't see a freshly-built image by name.
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
> image, but **already-running containers stay on the old image SHA** —
> `docker compose up -d` won't recreate them unless the compose file
> itself changed. After a meta-core source change, force-recreate the
> downstream containers so they pick up the new binary:
>
> ```bash
> make docker
> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
> up -d --force-recreate metagateway-hub-metacore
> docker compose -f packages/meta-share/dev/docker-compose.yml \
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
> docker compose -f packages/meta-read/dev/docker-compose.yml \
> up -d --force-recreate metaread-dev-core
> docker compose -f packages/meta-listen/dev/docker-compose.yml \
> up -d --force-recreate metalisten-dev-core
> ```
>
> The symptom of a missed recreate is missing API fields on `/urls`
> (e.g. `webdavUrlInternal` absent → meta-gateway's
> `store_poster_for_record` silently fails and torznab posters never
> land). Audit recipe:
>
> ```bash
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}' # repeat for :latest
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
> ```
### Dev container
In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`:
```bash
# With environment variables
META_CORE_PATH=/meta-core \
FILES_PATH=/files \
SERVICE_NAME=meta-sort \
META_CORE_HTTP_PORT=9000 \
./bin/meta-core
cd dev && ./scripts/start.sh
```
### Docker
```bash
docker build -t meta-core .
docker run -v meta-core:/meta-core -v files:/files meta-core
```
The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend
HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
## Configuration
| Environment Variable | Default | Description |
|---------------------|---------|-------------|
| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db |
| `FILES_PATH` | `/files` | Shared volume for media files |
| `SERVICE_NAME` | `meta-core` | Service identifier |
| `SERVICE_VERSION` | `1.0.0` | Service version |
| `API_PORT` | `8180` | Main service HTTP port |
| `BASE_URL` | - | Stable service URL |
| `REDIS_PORT` | `6379` | Redis port (leader only) |
| `META_CORE_HTTP_PORT` | `9000` | HTTP API port |
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval |
| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold |
| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning |
Environment variables (from `internal/config/config.go` unless noted):
## API Reference
| Variable | Default | Description |
|---|---|---|
| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, Redis data, identity, mounts, watchers, cache). No other service mounts it. |
| `FILES_PATH` | `/files` | Files volume root. |
| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
| `REDIS_PORT` | `6379` | Local Redis port. |
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
| `META_CORE_INDEX_EXCLUDE_PREFIXES` | `categories/newznab/` | Comma-separated field prefixes left out of the in-memory search index; set empty to index everything (`internal/storage/client.go`). |
| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
### Health & Status
`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
them (leftovers of the file-based registry); setting them has no effect. There
is no `META_CORE_REDIS_PORT` / `META_CORE_SERVICE_NAME` /
`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
configs created via `/api/watchers`.
## API reference
All endpoints below are registered in `internal/api/server.go` (mounts,
watcher and watchers routes in their own packages' `handlers.go`). Examples assume
the in-container Go port `:9000`; from the host use
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
(debug-direct).
### Health, leader, URLs
```bash
# Health check
curl http://localhost:9000/health
# {"status":"ok","redis":true,"timestamp":"..."}
# Detailed status
curl http://localhost:9000/status
# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...}
# Current leader
curl http://localhost:9000/leader
# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...}
GET /health # storage + role
GET /status # version, uptime, etc.
GET /leader # current leader info (hostname, PID, timestamps)
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
GET /neighbors # neighbours heard over UDP (beacon v2)
GET /api/neighbors # same, under /api
GET /api/stats # dashboard counters (see below)
```
### Metadata Operations
`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
are **gone** — the registration-and-heartbeat registry they served was replaced
by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
than a 404; that is what made the dashboard fail with
`SyntaxError: Unexpected token '<'` until 1.0.21.
```bash
# List all file hashes
curl http://localhost:9000/meta
# {"hashIds":["midhash256:abc123",...],"count":42}
#### `GET /api/stats`
# Get metadata for a file
curl http://localhost:9000/meta/{hash}
# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}}
Every headline counter in one request, split by cost:
# Update metadata
curl -X PUT http://localhost:9000/meta/{hash} \
-H "Content-Type: application/json" \
-d '{"title":"New Title"}'
# Delete metadata
curl -X DELETE http://localhost:9000/meta/{hash}
```
### Data Operations
```bash
# Get file path
curl http://localhost:9000/data/{hash}/path
# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true}
# Check if file exists
curl -I http://localhost:9000/data/{hash}
# HTTP/1.1 200 OK (or 404)
```
### File Operations (by CID)
```bash
# Serve file by CID (looks up poster/backdrop CIDs in metadata)
curl http://localhost:9000/file/{cid} --output poster.jpg
# Returns raw file bytes with appropriate Content-Type header
# Example: fetch a poster image
curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4
# Content-Type: image/jpeg
# [binary image data]
```
The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval.
### Service Discovery
```bash
# List all services
curl http://localhost:9000/services
# {"services":[{"name":"meta-sort",...}],"count":3}
# Get specific service
curl http://localhost:9000/services/meta-sort
# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...}
```
### Mount Management
```bash
# List all mounts
curl http://localhost:9000/api/mounts/list
# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]}
# Add a new mount
curl -X POST http://localhost:9000/api/mounts/add \
-H "Content-Type: application/json" \
-d '{"type":"smb","path":"/files/corn/nas","config":{...}}'
# Remove a mount
curl -X DELETE http://localhost:9000/api/mounts/smb-nas
```
### File Watching
```bash
# Get scan status
curl http://localhost:9000/api/scan/status
# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847}
# Trigger rescan
curl -X POST http://localhost:9000/api/scan/trigger
# Stream file events (SSE)
curl http://localhost:9000/api/events/stream
# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."}
```
### KV Browser
```bash
# Storage statistics
curl http://localhost:9000/api/kv/info
# {"totalKeys":50000,"memoryUsed":"256MB",...}
# List keys with cursor pagination
curl "http://localhost:9000/api/kv/keys?cursor=0&count=100"
# Get specific key value
curl http://localhost:9000/api/kv/key/file:bafkr.../title
```
### Metadata Editor
```bash
# Get all hash IDs
curl http://localhost:9000/api/metadata/hash-ids
# {"hashIds":["bafkr...","bafks..."]}
# Paginated metadata listing
curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50"
# Search metadata
curl -X POST http://localhost:9000/api/metadata/search \
-H "Content-Type: application/json" \
-d '{"query":"inception","fields":["title"]}'
# Batch update
curl -X POST http://localhost:9000/api/metadata/batch \
-H "Content-Type: application/json" \
-d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}'
# Clear all metadata
curl -X POST http://localhost:9000/api/metadata/clear
```
### WebDAV Access
```bash
# List directory (returns JSON)
curl http://localhost:9000/webdav/
# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}]
# Get file
curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv
# Upload file
curl -X PUT http://localhost:9000/webdav/plugin/output.txt \
-H "Content-Type: application/octet-stream" \
--data-binary @output.txt
# Create directory
curl -X MKCOL http://localhost:9000/webdav/newdir/
# Delete file/directory
curl -X DELETE http://localhost:9000/webdav/oldfile.txt
```
## Leader Election
meta-core uses POSIX file locking (flock) for distributed leader election:
1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock`
2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info`
3. Losers become **followers**, read leader info, connect to leader's Redis
4. Lock automatically releases when process dies (no stale locks)
### Election Flow
```
Instance 1 Instance 2 Instance 3
│ │ │
├── Try flock ──────────┼───────────────────────┤
│ │ │
[WIN] [LOSE] [LOSE]
LEADER FOLLOWER FOLLOWER
│ │ │
Start Redis Read leader.info Read leader.info
│ │ │
Write leader.info Connect to Redis Connect to Redis
│ │ │
[Health loop 5s] [Health loop 5s] [Health loop 5s]
- Update timestamp - Re-read leader - Re-read leader
- Check Redis alive - Detect changes - Detect changes
```
### Role Callbacks
The election system provides callbacks for role transitions:
```go
election.OnBecomeLeader(func(info LeaderLockInfo) {
// Start Redis, initialize resources
})
election.OnBecomeFollower(func(info LeaderLockInfo) {
// Connect to leader's Redis
})
election.OnLeaderLost(func() {
// Handle leader failure, attempt re-election
})
```
### Lock Info Format
The `kv-leader.info` file contains a plain text API URL:
```
http://10.0.1.50:9000
```
Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.).
## Redis Management
The leader is responsible for spawning and managing the Redis server.
### Redis Configuration
```bash
redis-server --port 6379 --bind 0.0.0.0 \
--dir /meta-core/db/redis \
--appendonly yes --appendfilename appendonly.aof \
--dbfilename dump.rdb --save 60 1 \
--loglevel warning
```
### Persistence Strategy
| Method | Configuration | Purpose |
|--------|--------------|---------|
| **AOF** | `appendonly yes` | Write-ahead log for durability |
| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed |
### Failure Handling
- **Redis crash**: Auto-detected via health check, automatically restarted
- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs
- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed
## Metadata Storage
meta-core uses a flat key-value schema in Redis for storing file metadata.
### Storage Format
```
Key: /file/{hashId}/{property}
Value: string
Examples:
/file/midhash256:abc123/title → "Movie Title"
/file/midhash256:abc123/year → "2024"
/file/midhash256:abc123/genres → "action,drama"
/file/midhash256:abc123/filePath → "movies/Movie.mkv"
```
### Storage Client Features
| Feature | Details |
|---------|---------|
| Connection Pool | 10 connections |
| Timeouts | 5s dial, 30s read/write |
| Batch Operations | Pipelined writes for performance |
| Key Scanning | SCAN-based iteration (non-blocking) |
| CID Lookup | Find file paths by poster/backdrop CID |
## Service Discovery
Services register by writing JSON to `/meta-core/services/{name}.json`.
### Registration Format
```json
```jsonc
{
"name": "meta-sort",
"version": "2.0.0",
"api": "http://10.0.1.50:8180",
"status": "running",
"pid": 12345,
"hostname": "meta-sort-dev",
"startedAt": "2024-01-01T00:00:00Z",
"lastHeartbeat": "2024-01-01T00:01:00Z",
"capabilities": ["meta-core"],
"endpoints": {
"health": "http://10.0.1.50:9000/health",
"meta": "http://10.0.1.50:9000/meta"
}
"records": 88709, // SCARD file:__index__ — metadata roots, file or not
"redisKeys": 3115447, // DBSIZE — the flat per-field keys dominate
"redisMemory": "588.74M",
"identities": 3, // signing accounts on disk
"files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep
"udlUsers": 5, // accounts holding User Data Layer records
"sweptAt": 1789402401970,
"sweeping": false
}
```
### Heartbeat & Stale Detection
The first four are O(1) commands and are live on every call. `files` and
`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
box), so they are served from a single-flight cache refreshed in the background
with a 60s TTL — the walk is **never** on the request path, because the
dashboard polls and ten open tabs must not cost ten Redis walks. Prefer this
over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
across records that share a file.
| Parameter | Default | Purpose |
|-----------|---------|---------|
| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` |
| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this |
### Metadata — primary surface
Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file.
## Development
These are the routes other services use. `{hash}` may be a UUID root or
any registered CID alias; CID aliases resolve to the underlying UUID
before reads/writes (`internal/storage/cid_resolution.go`).
```bash
# Format code
make fmt
# Run linter
make lint
# Run tests with coverage
make test-cover
# Clean build artifacts
make clean
GET /meta # list of root IDs
GET /meta/{hash} # full document (flat keys → nested JSON)
PUT /meta/{hash} # replace document
PATCH /meta/{hash} # merge into document
DELETE /meta/{hash} # delete root + all CID aliases
GET /meta/{hash}/{key...} # single property (key may contain "/")
PUT /meta/{hash}/{key...} # set single property
DELETE /meta/{hash}/{key...} # delete single property
POST /meta/{hash}/_add/{key...} # add value to set-valued property
```
## Internal Packages
CID-addressed access (public; auth-bypassed in the Caddy perimeter):
```bash
GET /api/meta/{cid} # resolve CID → metadata document
GET /api/file/{cid}/info # CID resolution metadata
GET /file/{cid} # serve file bytes (range-aware)
POST /file/cid # compute CID for an uploaded file
POST /api/files/register # mint + alias one file under /files/plugin by midhash
# (the watcher's path for one file; works with the watcher OFF)
HEAD /data/{hash} # existence + size
GET /data/{hash}/path # absolute path on /files
```
(There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or
the `/file/{cid}` reverse-lookup path.)
### Editor / KV browser surface
```bash
GET /api/metadata/hash-ids
GET /api/files/tuples
GET /api/metadata/list
POST /api/metadata/search
POST /api/metadata/batch
POST /api/metadata/clear
GET /api/metadata/{hashId}
PUT /api/metadata/{hashId}
DELETE /api/metadata/{hashId}
GET /api/metadata/{hashId}/property
PUT /api/metadata/{hashId}/property
GET /api/kv/info
GET /api/kv/keys
GET /api/kv/tree
GET /api/kv/search
GET /api/kv/find
GET /api/kv/value
PUT /api/kv/value
DELETE /api/kv/value
GET /api/kv/key/{key...}
GET /api/schema
POST /api/schema/rescan
GET /api/snapshot/export
POST /api/snapshot/import
POST /api/snapshot/wipe
```
### SSE event streams
The HTTP-mirror of the Redis Streams. External services never read Redis
directly — they subscribe here with `Last-Event-ID` for resume. See
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
contract (one event per stream entry, opaque IDs, heartbeats, gap
signalling).
```bash
GET /api/events/files # file:events stream (watcher events)
GET /api/events/meta # meta:events stream (metadata mutations)
GET /api/events/poll # DEPRECATED long-poll over file:events — use SSE
```
### Identity (signing keystore)
Reached through meta-search's identity proxy (and directly by meta-gateway,
meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
```bash
GET /api/identity # status (never the private key)
GET /api/identity/accounts
POST /api/identity/generate # the only call that returns a new private key
POST /api/identity/import
POST /api/identity/challenge # one-shot challenge for proof-of-possession
POST /api/identity/reveal # needs proof-of-possession
DELETE /api/identity # needs proof-of-possession; reports purged UDL counts
POST /api/identity/sign
POST /api/identity/sign-batch
GET /api/identity/aead-key
```
### User Data Layer
Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
cell in a batch). Redis model in `internal/storage/udl.go`.
```bash
GET /api/udl/record?uid&cid&key
PUT /api/udl/record
PUT /api/udl/records # batch
GET /api/udl/user/{uid}/key/{key}
GET /api/udl/user/{uid}/cid/{cid}
GET /api/udl/user/{uid}/all
GET /api/udl/user/{uid}/count
GET /api/udl/cid/{cid}/users
GET /api/udl/cid/{cid}/aggregate # meta-watch Aggregate wire shape
GET /api/udl/users/stats
```
### Mounts
```bash
GET /api/mounts
POST /api/mounts
GET /api/mounts/{id}
PUT /api/mounts/{id} # also PATCH
DELETE /api/mounts/{id}
POST /api/mounts/{id}/mount
POST /api/mounts/{id}/unmount
POST /api/mounts/{id}/safe-unmount
POST /api/mounts/{id}/scan
GET /api/mounts/rclone/remotes
```
### Watchers
```bash
GET /api/watchers
POST /api/watchers
GET /api/watchers/{id}
PUT /api/watchers/{id}
DELETE /api/watchers/{id}
POST /api/watchers/{id}/scan
POST /api/watchers/{id}/reset
POST /api/watchers/scan-all
POST /api/watchers/reset-all
```
The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
return a deprecation pointer at the `/api/watchers/*` routes
(`internal/watcher/handlers.go:32-33`).
### Admin / WebDAV
```bash
POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
POST /api/admin/migrate-domain-screen # domain film|tv → screen, workForm backfill (idempotent)
POST /api/admin/migrate-user-cids # backfill udl:idx:user:<uid>:cids; until run, pre-index
# profiles export as EMPTY. Returns {fixed}
POST /api/admin/sweep-literature-echoes[?apply=true]
# strip query echoes off literature cards: anilistid on site cards,
# workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
# Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
GET/PUT/DELETE /webdav/... # mounted on /files
```
No `/metrics` endpoint is exposed; observability is via container logs and
the `/health` / `/status` JSON.
## Redis-volume mutex
`docker/leader-election.sh` is a bash flock loop, not a Go-level state
machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
script `exec`s supervisord; a second container mounting the same volume
retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
dies, the kernel releases the lock. There is no follower path inside the Go
binary.
The lock is a **mutex, not service discovery** — it only stops two Redis
writers landing on one RDB/AOF. Nothing reads it, and the `kv-leader.info`
file it used to publish is gone: siblings locate meta-core over UDP
(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
`internal/leader` keeps its name for history, but only builds the `/urls`
payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
and the UDP announce share.
## Metadata storage shape (one-line summary)
Roots are opaque UUIDv7 strings; every property is a separate Redis
STRING at `file:<uuid>/<property>`; every known CID is a key-set member
`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
(the canonical CID is derived by rank on read, never stored). Full details in
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
The authoritative registry of field semantics and value formats lives in
the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
## Internal packages
| Package | Purpose |
|---------|---------|
| `cmd/meta-core` | Main entry point, startup/shutdown orchestration |
| `internal/config` | Environment-based configuration with defaults |
| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) |
| `internal/storage` | Redis client wrapper with flat key-value operations |
| `internal/discovery` | Service registration, heartbeat loop, discovery |
| `internal/api` | HTTP server, router, and all endpoint handlers |
| `internal/events` | Redis keyspace notification publisher to meta:events |
| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch |
| `internal/watchers` | Polling-based watcher configuration management |
| `internal/mounts` | SMB/rclone mount configuration and health monitoring |
| `internal/webdav` | WebDAV protocol handler with caching integration |
| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + UDP discovery + service wiring. |
| `internal/config` | Env-driven configuration, path helpers. |
| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, field indexes + search-index bulk reads, UDL store (`udl.go`, `udl_admin.go`), stats, one-shot migrations/sweeps. |
| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
| `internal/api` | HTTP server + all handlers + SSE event endpoints + in-memory search index + `/api/stats`. |
| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
| `internal/snapshot` | Snapshot export / import / wipe. |
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. |
| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). |
### Startup Sequence
### Startup sequence
```
1. Load configuration from environment
2. Create storage client (Redis wrapper)
3. Initialize leader election
4. Register role transition callbacks
5. Start election (acquire lock or become follower)
6. Start service discovery (register + heartbeat loop)
7. Initialize dead service cleaner
8. Initialize file watcher (if enabled)
9. Initialize mount manager
10. Start HTTP API server (WebDAV caching handled by nginx)
11. Wait for SIGINT/SIGTERM
12. Shutdown in reverse order
```
## Integration
### As Sidecar in Docker
Add to existing service's supervisord config:
```ini
[program:meta-core]
command=/usr/local/bin/meta-core
priority=10
autostart=true
autorestart=true
```
### Client Usage (TypeScript)
```typescript
const response = await fetch('http://localhost:9000/meta/' + hashId);
const { metadata } = await response.json();
```
### Client Usage (Python)
```python
import requests
response = requests.get(f'http://localhost:9000/meta/{hash_id}')
metadata = response.json()['metadata']
2. Build LeaderInfoProvider (no election — the bash gate already won)
3. Connect to local Redis (retry up to 30× / 1s)
4. EnsureSchemaVersion — abort if the existing Redis layout is stale
5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
6. Construct API server (identity keystore migration, watcher, watchers,
mounts, WebDAV)
7. Start API server + pollers + search-index refresh loop; start
MetaPublisher + schema Indexer, then warm field indexes and republish
existing metadata onto meta:events in the background
8. Wait for SIGINT/SIGTERM, shutdown in reverse order
```
## Dependencies
| Dependency | Version | Purpose |
|------------|---------|---------|
| `github.com/gorilla/mux` | v1.8.1 | HTTP router |
| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client |
| `golang.org/x/net` | v0.20.0 | WebDAV support |
| `github.com/google/uuid` | v1.6.0 | UUID generation |
| Go stdlib | - | context, encoding/json, net, os, sync, syscall |
| Dependency | Purpose |
|------------|---------|
| `github.com/gorilla/mux` | HTTP router |
| `github.com/redis/go-redis/v9` | Redis client |
| `github.com/google/uuid` | UUIDv7 minting |
| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
| `golang.org/x/net` | WebDAV support |
| `github.com/alicebob/miniredis/v2` | In-memory Redis for tests |
### Runtime Requirements
- **Go 1.21+** for building
- **Redis** binary available in PATH (leader spawns it)
- **Shared filesystem** accessible by all instances (for flock)
Runtime requirements: Go 1.21+ at build time; the container ships Redis,
rclone, nginx, and supervisord alongside the binary, plus the React dashboard
and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
## License
+75 -16
View File
@@ -1,6 +1,7 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
@@ -10,8 +11,8 @@ import (
"github.com/metazla/meta-core/internal/api"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/storage"
)
@@ -51,20 +52,66 @@ func main() {
}
log.Println("[meta-core] Connected to Redis")
// Create and start service discovery
disc := discovery.NewService(cfg)
if err := disc.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
// the existing Redis store predates the UUID-rooted schema, so legacy
// data can't silently leak into the new code paths. See
// docs/uuid-rooted-metadata.md.
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
log.Fatalf("[meta-core] %v", err)
}
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
// Create and start dead service cleaner
cleaner := discovery.NewCleaner(cfg)
if err := cleaner.Start(); err != nil {
log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
// as metamesh.core — carrying the /urls block services locate us by — plus
// its own metamesh.service/<name> nav entry. The payload is rebuilt from
// leaderProvider on every tick, which keeps it and GET /urls from drifting.
var mesh *meshdisco.Node
if cfg.EnableUDPDiscovery {
hostname, _ := os.Hostname()
mesh = meshdisco.New(meshdisco.Config{
Name: cfg.ServiceName,
Instance: hostname,
Version: Version,
Group: cfg.DiscoveryGroup,
Port: cfg.DiscoveryPort,
Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
Payload: func() (string, []meshdisco.Resource) {
service := meshdisco.Resource{
ID: "service",
Caps: []string{meshdisco.CapServicePrefix + cfg.ServiceName},
Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
}
info := leaderProvider.LeaderInfo()
if info == nil {
return "starting", []meshdisco.Resource{service}
}
data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
Hostname: info.Hostname,
BaseUrl: info.BaseUrl,
ApiUrl: info.ApiUrl,
WebdavUrl: info.WebdavUrl,
WebdavUrlInternal: info.WebdavUrlInternal,
}})
core := meshdisco.Resource{
ID: "core",
Caps: []string{meshdisco.CapCore},
Endpoints: map[string]string{"http": info.ApiUrl},
Data: data,
}
return "running", []meshdisco.Resource{core, service}
},
})
if err := mesh.Start(); err != nil {
log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
mesh = nil
}
} else {
log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
}
// Create and start API server
apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
apiServer := api.NewServer(cfg, leaderProvider, storageClient)
apiServer.SetMeshNode(mesh)
if err := apiServer.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start API server: %v", err)
}
@@ -81,12 +128,10 @@ func main() {
log.Printf("[meta-core] Error stopping API server: %v", err)
}
if err := cleaner.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service cleaner: %v", err)
}
if err := disc.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service discovery: %v", err)
if mesh != nil {
if err := mesh.Stop(); err != nil {
log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
}
}
if err := storageClient.Close(); err != nil {
@@ -103,3 +148,17 @@ func waitForShutdown() {
<-sigChan
}
// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
// the three-way fallback the file registry used (discovery/service.go):
// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
// the container IP.
func coreBaseURL(cfg *config.Config) string {
if cfg.PublicURL != "" {
return cfg.PublicURL
}
if cfg.BaseURL != "" {
return cfg.BaseURL
}
return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
}
+1 -1
View File
@@ -21,7 +21,7 @@
min-height: 100vh;
}
</style>
<script type="module" crossorigin src="/assets/index-DJlbGMsa.js"></script>
<script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
</head>
<body>
+10
View File
@@ -8,6 +8,7 @@
"name": "@meta-core/dashboard",
"version": "1.0.0",
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
@@ -744,6 +745,15 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@noble/secp256k1": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.1.0.tgz",
"integrity": "sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==",
"license": "MIT",
"funding": {
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/@remix-run/router": {
"version": "1.23.2",
"resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz",
+3 -2
View File
@@ -9,10 +9,11 @@
"preview": "vite preview"
},
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.20.0",
"iso-639-3": "^3.0.1"
"react-router-dom": "^6.20.0"
},
"devDependencies": {
"@types/react": "^18.2.38",
+8
View File
@@ -3,6 +3,7 @@ import Dashboard from './pages/Dashboard';
import Mounts from './pages/Mounts';
import Watchers from './pages/Watchers';
import Files from './pages/Files';
import Identity from './pages/Identity';
import ServiceNav from './components/ServiceNav';
const navStyle: React.CSSProperties = {
@@ -74,6 +75,12 @@ export default function App() {
>
Files
</NavLink>
<NavLink
to="/identity"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
>
Identity
</NavLink>
<NavLink
to="/editor"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
@@ -89,6 +96,7 @@ export default function App() {
<Route path="/mounts" element={<Mounts />} />
<Route path="/watchers" element={<Watchers />} />
<Route path="/files" element={<Files />} />
<Route path="/identity" element={<Identity />} />
<Route path="/editor" element={<EditorPage />} />
</Routes>
</div>
+44 -146
View File
@@ -1,156 +1,54 @@
import { useState, useEffect } from 'react';
import { useEffect } from 'react';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
// of defining the custom element.
import '../meta-service-menu.js';
interface ServiceInfo {
name: string;
hostname: string;
baseUrl: string;
status: string;
lastHeartbeat: string;
}
/**
* ServiceNav — the neighbouring meta-* services, as a burger menu.
*
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
* a framework-free custom element shared by every meta-* UI (see
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
* that existed in four near-identical copies and polled meta-core's
* /api/services; the element polls this service's own /api/neighbors instead,
* so the nav still renders when meta-core is down.
*
* The element lives in shadow DOM and cannot see this app's CSS variables, so
* the four --mm-nav-* tokens below map our palette onto it.
*/
interface ServicesResponse {
services: ServiceInfo[];
current: string;
}
const serviceIcons: Record<string, string> = {
'meta-core': '🗄️',
'meta-sort': '📁',
'meta-fuse': '🗂️',
'meta-stremio': '🎬',
'meta-orbit': '🌐',
'default': '📦'
};
function formatServiceName(name: string): string {
return name.split('-').map(word =>
word.charAt(0).toUpperCase() + word.slice(1)
).join(' ');
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace JSX {
interface IntrinsicElements {
'meta-service-menu': React.DetailedHTMLProps<
React.HTMLAttributes<HTMLElement>,
HTMLElement
> & { current?: string; endpoint?: string; label?: string };
}
}
}
function ServiceNav() {
const [services, setServices] = useState<ServiceInfo[]>([]);
const [loading, setLoading] = useState(true);
const currentService = 'meta-core';
useEffect(() => {
// Import above is for the side effect of defining the element; this
// keeps bundlers from tree-shaking it away in production builds.
}, []);
useEffect(() => {
const fetchServices = async () => {
try {
const response = await fetch('/services');
if (response.ok) {
const data: ServicesResponse = await response.json();
setServices(data.services || []);
}
} catch (error) {
console.error('Failed to fetch services:', error);
} finally {
setLoading(false);
}
};
fetchServices();
const interval = setInterval(fetchServices, 30000);
return () => clearInterval(interval);
}, []);
if (loading) return null;
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
if (sortedServices.length === 0) return null;
return (
<nav className="services-nav">
<span className="services-nav-label">Services:</span>
<div className="services-nav-items">
{sortedServices.map(service => {
const icon = serviceIcons[service.name] || serviceIcons.default;
const isActive = service.name === currentService;
return (
<a
key={service.name}
href={isActive ? '#' : service.baseUrl}
className={`service-link${isActive ? ' active' : ''}`}
onClick={(e) => {
e.preventDefault();
if (!isActive) {
window.location.href = service.baseUrl;
return (
<>
<meta-service-menu current="meta-core" />
<style>{`
meta-service-menu {
--mm-nav-fg: var(--text-primary, #e0e0e0);
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
--mm-nav-accent: var(--accent-primary, #4ecdc4);
}
}}
>
<span className="service-icon">{icon}</span>
<span>{formatServiceName(service.name)}</span>
<span className="service-status"></span>
</a>
);
})}
</div>
<style>{`
.services-nav {
display: flex;
align-items: center;
gap: 0.5rem;
}
.services-nav-label {
color: #888;
font-size: 0.8rem;
white-space: nowrap;
}
.services-nav-items {
display: flex;
gap: 0.35rem;
}
.service-link {
display: inline-flex;
align-items: center;
gap: 0.3rem;
padding: 0.35rem 0.6rem;
border-radius: 6px;
text-decoration: none;
font-size: 0.8rem;
font-weight: 500;
transition: all 0.2s;
color: #ccc;
background: #0f3460;
border: 1px solid #1a4a7a;
}
.service-link:hover {
background: #1a4a7a;
border-color: rgba(78, 205, 196, 0.5);
text-decoration: none;
color: #fff;
}
.service-link.active {
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
border-color: rgba(78, 205, 196, 0.5);
color: #fff;
}
.service-link .service-icon {
font-size: 0.9rem;
}
.service-link .service-status {
width: 5px;
height: 5px;
border-radius: 50%;
background: #4ecdc4;
}
@media (max-width: 800px) {
.services-nav-label {
display: none;
}
}
`}</style>
</nav>
);
`}</style>
</>
);
}
export default ServiceNav;
+28
View File
@@ -0,0 +1,28 @@
// JSON fetch that fails with a sentence instead of a parser error.
//
// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
// path it does not have a proxy rule for answers 200 text/html with the SPA
// shell. A caller that does `res.ok && res.json()` on such a path gets
// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
// no clue which request produced it — which is exactly how a route deleted
// from the Go router (`/services`, removed in favour of /api/neighbors) sat
// broken on the dashboard.
//
// So: check the content type, and name the URL in every error.
export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
const res = await fetch(url, init);
const ct = res.headers.get('content-type') || '';
if (!res.ok) {
throw new Error(`${url} → HTTP ${res.status}`);
}
if (!ct.includes('application/json')) {
// The SPA fallback is the overwhelmingly likely cause; say so.
throw new Error(
`${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
`missing and nginx served the dashboard shell instead)`
);
}
return (await res.json()) as T;
}
+127
View File
@@ -0,0 +1,127 @@
// Client-side proof of possession for account operations.
//
// meta-core stores every private key in plaintext, so it can sign as anybody —
// which means a signature it produced proves nothing about who asked. The only
// signature that carries authority is one made where the key actually is, and
// for a browser-driven admin console that is here. meta-core enforces the other
// half by refusing to sign anything in the challenge domain
// (identity.ChallengeDomain), so this path cannot be short-circuited.
//
// The secret key entered on this page is used for exactly one signature and is
// never sent anywhere: `/api/identity/challenge` receives only the uid, and the
// request that follows carries only the challenge and its signature.
//
// Same scheme, same curve and same encodings as meta-watch's sign-in
// (packages/meta-watch/ui/js/auth.js) — deliberately, so one implementation on
// the server verifies both.
import * as secp from '@noble/secp256k1';
export type IdentityAction = 'reveal' | 'delete';
function hexToBytes(hex: string): Uint8Array | null {
let h = hex.trim();
if (h.startsWith('0x') || h.startsWith('0X')) h = h.slice(2);
if (h.length !== 64 || /[^0-9a-fA-F]/.test(h)) return null;
const out = new Uint8Array(32);
for (let i = 0; i < 32; i++) out[i] = parseInt(h.slice(i * 2, i * 2 + 2), 16);
return out;
}
function bytesToBase64(bytes: Uint8Array): string {
let s = '';
for (const b of bytes) s += String.fromCharCode(b);
return btoa(s);
}
const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
/// base58btc, the multibase "z" alphabet. Spelled out rather than pulled in:
/// the alternative is a second dependency for thirty lines of arithmetic.
function base58Encode(bytes: Uint8Array): string {
const digits: number[] = [0];
for (const byte of bytes) {
let carry = byte;
for (let i = 0; i < digits.length; i++) {
carry += digits[i] << 8;
digits[i] = carry % 58;
carry = (carry / 58) | 0;
}
while (carry) {
digits.push(carry % 58);
carry = (carry / 58) | 0;
}
}
// Leading zero bytes encode as leading '1's rather than being dropped.
let out = '';
for (let i = 0; i < bytes.length && bytes[i] === 0; i++) out += B58[0];
for (let i = digits.length - 1; i >= 0; i--) out += B58[digits[i]];
return out;
}
/// The uid a secret key belongs to: "z" + base58btc(33-byte compressed pubkey).
/// Derived locally so a mistyped or wrong-account key is caught before any
/// request goes out. Mirrors `identityFromPrivKey` in meta-core's identity.go.
export function uidFromSecretKey(secretKeyHex: string): string | null {
const raw = hexToBytes(secretKeyHex);
if (!raw) return null;
try {
return 'z' + base58Encode(secp.getPublicKey(raw, true));
} catch {
return null; // not a valid scalar
}
}
/// Whether this page can sign at all.
///
/// noble's signAsync takes its sha256 from crypto.subtle, which browsers expose
/// only in a secure context. The dashboard behind Caddy is https and fine; the
/// debug-direct port (18083) is plain http, where every signature would fail
/// with a cryptic error unless we say so first.
export function canSign(): boolean {
return Boolean(globalThis.crypto?.subtle);
}
export const INSECURE_CONTEXT_MESSAGE =
'This page cannot sign over a plain-http connection — browsers only expose the ' +
'needed crypto on https (or localhost). Reach the dashboard through Caddy ' +
'(https://metacore-dev.localhost:8083) rather than the debug-direct port.';
/// Ask meta-core for a challenge, sign it locally, and hand back the proof.
///
/// Throws with a message fit to show the user. The caller passes the proof
/// straight into the reveal/delete request body.
export async function proveOwnership(
uid: string,
action: IdentityAction,
secretKeyHex: string
): Promise<{ challenge: string; signature: string }> {
if (!canSign()) throw new Error(INSECURE_CONTEXT_MESSAGE);
const raw = hexToBytes(secretKeyHex);
if (!raw) {
throw new Error('That is not a valid secret key: expected 64 hex characters.');
}
const derived = uidFromSecretKey(secretKeyHex);
if (!derived) throw new Error('That is not a valid secp256k1 secret key.');
if (derived !== uid) {
// Caught here rather than by the server, so the message can say which
// account the key actually is — the useful half of the answer.
throw new Error(`That key belongs to a different account (${derived}), not this one.`);
}
const cRes = await fetch('/api/identity/challenge', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ uid, action }),
});
const cBody = await cRes.json().catch(() => ({}));
if (!cRes.ok || !cBody.challenge) {
throw new Error(cBody.message ?? `Could not start verification (HTTP ${cRes.status}).`);
}
// signAsync prehashes with sha256 and returns a 64-byte compact (r‖s)
// signature — noble v3 dropped DER. meta-core's identity.Verify accepts both.
const sig = await secp.signAsync(new TextEncoder().encode(cBody.challenge), raw);
return { challenge: cBody.challenge, signature: bytesToBase64(sig) };
}
+308
View File
@@ -0,0 +1,308 @@
/**
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
*
* Replaces the eight divergent "services bar" implementations (four copies of
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
* ones) with a single framework-free custom element. It drops into the React
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
* as an ES module.
*
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
* make a fetch to a neighbour a dead end. Links are plain navigations.
*
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
* `scripts/check-mirrors.sh`. Edit one → edit all.
*
* Shadow DOM is not optional here. The eight UIs use four incompatible token
* vocabularies, and two of them define *different colours under the same
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
* #161b22 in meta-dup). meta-search and meta-share are also the only
* light-mode-capable UIs, so a hardcoded dark palette would break them.
* The element therefore styles itself from four tokens with dark defaults:
*
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
*
* Each host maps its own variables onto those, e.g.
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
*
* Attributes:
* current name of the host service, so it can be marked and not linked
* endpoint override the default "/api/neighbors"
* label button tooltip (default "Services")
*
* Property:
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
* translate internal hostnames into browser-reachable ones
* (meta-gateway does this for its dev/prod split).
*/
const ICONS = {
"meta-core": "⚙️",
"meta-sort": "📁",
"meta-fuse": "🗂️",
"meta-stremio": "🎬",
"meta-dup": "🔍",
"meta-search": "🔎",
"meta-share": "🔗",
"meta-gateway": "🌉",
"meta-watch": "📺",
"meta-listen": "🎧",
"meta-read": "📖",
};
const DEFAULT_ICON = "📦";
const POLL_MS = 30000;
function titleCase(name) {
return String(name || "")
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ");
}
const TEMPLATE = `
<style>
:host {
--_fg: var(--mm-nav-fg, #e8eaed);
--_bg: var(--mm-nav-bg, #1a1a2e);
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
--_accent: var(--mm-nav-accent, #4ecdc4);
position: relative;
display: inline-block;
font-family: inherit;
font-size: 14px;
line-height: 1.4;
}
button.trigger {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.4rem;
width: 2.1rem;
height: 2.1rem;
padding: 0;
cursor: pointer;
color: var(--_fg);
background: transparent;
border: 1px solid var(--_border);
border-radius: 8px;
font: inherit;
transition: border-color 0.15s, background 0.15s;
}
button.trigger:hover { border-color: var(--_accent); }
button.trigger:focus-visible {
outline: 2px solid var(--_accent);
outline-offset: 2px;
}
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
.bars { display: block; width: 15px; }
.bars span {
display: block;
height: 2px;
margin: 3px 0;
border-radius: 2px;
background: currentColor;
}
.panel {
position: absolute;
top: calc(100% + 6px);
right: 0;
z-index: 1000;
min-width: 13rem;
max-height: 70vh;
overflow-y: auto;
padding: 0.3rem;
background: var(--_bg);
border: 1px solid var(--_border);
border-radius: 10px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
}
.panel[hidden] { display: none; }
a.item, .item {
display: flex;
align-items: center;
gap: 0.55rem;
padding: 0.45rem 0.6rem;
border-radius: 6px;
color: var(--_fg);
text-decoration: none;
white-space: nowrap;
}
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
.item.dead { opacity: 0.55; cursor: not-allowed; }
.icon { font-size: 1rem; line-height: 1; }
.dot {
width: 6px;
height: 6px;
margin-left: auto;
border-radius: 50%;
background: var(--_accent);
}
.empty {
padding: 0.6rem;
opacity: 0.7;
font-size: 0.85em;
white-space: nowrap;
}
/* Respect a host that opts out of motion. */
@media (prefers-reduced-motion: reduce) {
button.trigger { transition: none; }
}
</style>
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
</button>
<div class="panel" role="menu" hidden></div>
`;
class MetaServiceMenu extends HTMLElement {
constructor() {
super();
this.attachShadow({ mode: "open" });
this.shadowRoot.innerHTML = TEMPLATE;
this._btn = this.shadowRoot.querySelector("button.trigger");
this._panel = this.shadowRoot.querySelector(".panel");
this._neighbors = [];
this._timer = null;
this._open = false;
this._onDocClick = (e) => {
if (!this._open) return;
if (e.composedPath().includes(this)) return;
this._setOpen(false);
};
this._onKey = (e) => {
if (e.key === "Escape" && this._open) {
this._setOpen(false);
this._btn.focus();
}
};
}
connectedCallback() {
this._btn.title = this.getAttribute("label") || "Services";
this._btn.setAttribute("aria-label", this._btn.title);
this._btn.addEventListener("click", () => this._setOpen(!this._open));
document.addEventListener("click", this._onDocClick);
document.addEventListener("keydown", this._onKey);
this.refresh();
this._timer = setInterval(() => this.refresh(), POLL_MS);
}
disconnectedCallback() {
if (this._timer) clearInterval(this._timer);
this._timer = null;
document.removeEventListener("click", this._onDocClick);
document.removeEventListener("keydown", this._onKey);
}
get endpoint() {
return this.getAttribute("endpoint") || "/api/neighbors";
}
get current() {
return this.getAttribute("current") || "";
}
async refresh() {
try {
const res = await fetch(this.endpoint, {
headers: { Accept: "application/json" },
});
if (!res.ok) throw new Error(String(res.status));
const data = await res.json();
// `services` is the legacy alias some backends still emit.
this._neighbors = data.neighbors || data.services || [];
} catch {
// Silent: the menu is navigation, not a health indicator. Keep whatever
// we last had rather than blanking the list on one failed poll.
}
this._render();
}
_setOpen(open) {
this._open = open;
this._panel.hidden = !open;
this._btn.setAttribute("aria-expanded", String(open));
if (open) this.refresh();
}
_render() {
const panel = this._panel;
panel.textContent = "";
const seen = new Set();
const rows = [];
for (const n of this._neighbors) {
const name = n.name || "";
if (!name || seen.has(name)) continue;
// Beacon v2 neighbours include plugins; the nav lists services only.
// (A row without `caps` comes from a pre-v2 backend — keep it.)
const isService = (c) => String(c).startsWith("metamesh.service/");
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
seen.add(name);
rows.push(n);
}
// Always show the host itself, even before its own echo comes back.
if (this.current && !seen.has(this.current)) {
rows.push({ name: this.current, baseUrl: "" });
}
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
if (rows.length === 0) {
const empty = document.createElement("div");
empty.className = "empty";
empty.textContent = "No services found";
panel.appendChild(empty);
return;
}
for (const n of rows) {
const isActive = n.name === this.current;
let url = n.baseUrl || "";
if (typeof this.rewriteUrl === "function") {
try {
url = this.rewriteUrl(n.name, url, n) || "";
} catch {
/* a bad hook must not blank the menu */
}
}
// A neighbour with no reachable URL is shown but not linked — better
// than a link that goes nowhere.
const el = document.createElement(isActive || !url ? "span" : "a");
el.className =
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
if (el.tagName === "A") {
el.href = url;
el.setAttribute("role", "menuitem");
}
if (!isActive && !url) el.title = "No reachable URL advertised";
const icon = document.createElement("span");
icon.className = "icon";
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
const label = document.createElement("span");
label.textContent = titleCase(n.name);
el.append(icon, label);
// Beacon v2 omits `status` when running.
if ((n.status || "running") === "running" || isActive) {
const dot = document.createElement("span");
dot.className = "dot";
dot.title = n.status || "running";
el.appendChild(dot);
}
panel.appendChild(el);
}
}
}
if (!customElements.get("meta-service-menu")) {
customElements.define("meta-service-menu", MetaServiceMenu);
}
export default MetaServiceMenu;
+171 -71
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from 'react';
import { getJSON } from '../lib/api';
interface HealthStatus {
status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
timestamp: string;
}
interface FileStats {
count: number;
totalSize: number;
}
interface Stats {
records: number;
redisKeys: number;
redisMemory: string;
identities: number;
files: FileStats | null;
udlUsers: number | null;
sweptAt: number;
sweeping: boolean;
}
interface WatcherStatus {
id: string;
path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
count: number;
}
interface ServiceInfo {
name: string;
api: string;
capabilities: string[];
timestamp: number;
}
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
const gridStyle: React.CSSProperties = {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
// 200px, not 220: five tiles must fit one row inside the 1200px container
// (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
gap: '1rem',
marginBottom: '1rem',
};
const statStyle: React.CSSProperties = {
fontSize: '2rem',
fontWeight: 'bold',
color: '#4ade80',
lineHeight: 1.2,
};
const labelStyle: React.CSSProperties = {
marginBottom: '0.75rem',
color: '#888',
fontSize: '0.85rem',
textTransform: 'uppercase',
letterSpacing: '0.04em',
};
const noteStyle: React.CSSProperties = {
color: '#6b7a99',
fontSize: '0.8rem',
marginTop: '0.4rem',
};
const numberFmt = new Intl.NumberFormat();
function formatCount(n: number | null | undefined): string {
return n === null || n === undefined ? '—' : numberFmt.format(n);
}
// Binary units: this counts bytes on disk, and every other tool the operator
// compares against (du, docker, the mount page) reports GiB.
function formatBytes(bytes: number | null | undefined): string {
if (bytes === null || bytes === undefined) return '—';
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
let value = bytes;
let unit = 0;
while (value >= 1024 && unit < units.length - 1) {
value /= 1024;
unit++;
}
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
}
function Tile({
label,
value,
note,
pending,
}: {
label: string;
value: string;
note?: string;
pending?: boolean;
}) {
return (
<div style={cardStyle}>
<div style={labelStyle}>{label}</div>
<div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
{value}
</div>
{note && <div style={noteStyle}>{note}</div>}
</div>
);
}
export default function Dashboard() {
const [health, setHealth] = useState<HealthStatus | null>(null);
const [stats, setStats] = useState<Stats | null>(null);
const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
const [services, setServices] = useState<ServiceInfo[]>([]);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
let cancelled = false;
const fetchData = async () => {
try {
// Fetch health
const healthRes = await fetch('/health');
if (healthRes.ok) {
setHealth(await healthRes.json());
}
// Settled, not all: one dead endpoint must not blank every tile.
const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
getJSON<HealthStatus>('/health'),
getJSON<Stats>('/api/stats'),
getJSON<WatchersResponse>('/api/watchers'),
]);
if (cancelled) return;
// Fetch watchers status
const watchersRes = await fetch('/api/watchers');
if (watchersRes.ok) {
setWatchersData(await watchersRes.json());
}
if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
if (statsRes.status === 'fulfilled') setStats(statsRes.value);
if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
// Fetch services
const servicesRes = await fetch('/services');
if (servicesRes.ok) {
const data = await servicesRes.json();
setServices(data.services || []);
}
setError(null);
} catch (err) {
setError(String(err));
}
const failures = [healthRes, statsRes, watchersRes]
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
.map((r) => String(r.reason?.message ?? r.reason));
setError(failures.length ? failures.join(' · ') : null);
};
fetchData();
const interval = setInterval(fetchData, 5000);
return () => clearInterval(interval);
return () => {
cancelled = true;
clearInterval(interval);
};
}, []);
const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
}
};
// Aggregate watcher stats
const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
const lastScan =
watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
// Records we hold metadata for but have no local file: gateway/cid-rooted
// entries, plus anything whose file went away. The interesting number on a
// client box, where it should be most of them.
const fileless =
stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
const sweepPending = !!stats && !stats.files;
const sweptNote = (() => {
if (!stats) return undefined;
if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
if (stats.sweeping) return 'refreshing…';
if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
return undefined;
})();
return (
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
<h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
{error && (
<div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
Error: {error}
{error}
</div>
)}
<div style={gridStyle}>
{/* Health Card */}
<Tile
label="Files discovered"
value={formatCount(stats?.files?.count)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Library size"
value={formatBytes(stats?.files?.totalSize)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Metadata entries"
value={formatCount(stats?.records)}
note={
fileless === null
? 'records in the index'
: `${formatCount(fileless)} with no local file`
}
/>
<Tile
label="Redis keys"
value={formatCount(stats?.redisKeys)}
note={stats?.redisMemory ? `${stats.redisMemory} in use` : undefined}
/>
<Tile
label="Identities"
value={formatCount(stats?.identities)}
note={
stats?.udlUsers !== null && stats?.udlUsers !== undefined
? `${formatCount(stats.udlUsers)} with user data`
: 'signing accounts'
}
/>
</div>
<div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
<div style={labelStyle}>Service health</div>
{health ? (
<>
<div style={statStyle}>
{health.status === 'ok' ? 'Healthy' : health.status}
</div>
<p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
<p style={noteStyle}>
Redis: {health.redis ? 'Connected' : 'Disconnected'}
</p>
</>
) : (
<p>Loading...</p>
<p style={noteStyle}>Loading…</p>
)}
</div>
{/* Watchers Status Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
<div style={labelStyle}>File watchers</div>
{watchersData ? (
<>
<div style={statStyle}>{totalFiles}</div>
<p>Files discovered</p>
<p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
<p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
<div style={statStyle}>
{activeWatchers} / {watchersData.count}
</div>
<p style={noteStyle}>active watchers</p>
<p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
{lastScan > 0 && (
<p>Last scan: {new Date(lastScan).toLocaleString()}</p>
<p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
)}
<button
onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
</button>
</>
) : (
<p>Loading...</p>
)}
</div>
{/* Services Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
<div style={statStyle}>{services.length}</div>
<p>Services registered</p>
{services.length > 0 && (
<ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
{services.map((svc) => (
<li key={svc.name} style={{ marginBottom: '0.5rem' }}>
<strong>{svc.name}</strong>
<br />
<span style={{ color: '#888', fontSize: '0.9rem' }}>
{svc.capabilities?.join(', ') || 'No capabilities'}
</span>
</li>
))}
</ul>
<p style={noteStyle}>Loading…</p>
)}
</div>
</div>
+859
View File
@@ -0,0 +1,859 @@
import { useEffect, useRef, useState } from 'react';
import {
INSECURE_CONTEXT_MESSAGE,
canSign,
proveOwnership,
uidFromSecretKey,
type IdentityAction,
} from '../lib/sign';
// Admin console for meta-core's account keystore.
//
// meta-core stores one secp256k1 keypair per file under
// {META_CORE_PATH}/identity/accounts/<uid>.json and has NO notion of a current
// user — which account is "you" is decided by whichever client signs in
// (meta-watch). So this page lists accounts; it never speaks of an active one.
//
// The private key is returned by exactly two calls: POST /api/identity/generate
// (once, at creation) and POST /api/identity/reveal (on demand, for an account
// that already exists). Both land in the same exclusive SecretKeyPanel.
//
// Reveal and remove both require the account's own secret key, entered here and
// used to sign a one-shot challenge locally — the key itself is never sent. So
// this page can enumerate and create, but it holds no authority over an
// existing account that its operator does not already have the key for. There
// is deliberately no admin override: on this node the private key *is* the
// account.
interface Account {
uid: string;
curve: string;
createdAt: number;
}
// What one account holds in the User Data Layer. Absent from
// /api/udl/users/stats means zero, not unknown.
interface Stats {
records: number;
cids: number;
keys: number;
}
// A secret on screen. Carries no createdAt: /api/identity/reveal answers with
// {uid, curve, privateKeyHex} only.
interface SecretKeyView {
uid: string;
privateKeyHex: string;
kind: 'generated' | 'revealed';
}
// Which row has an open confirmation, and for what. One discriminated value
// rather than a pair of ids, so at most one confirm panel can ever be open.
interface Pending {
uid: string;
kind: 'remove' | 'reveal';
}
// Both confirmable operations map to a server-side action name, and the
// challenge is bound to it: a challenge minted to reveal a key cannot delete
// the account. Keeping the mapping in one place stops the two drifting.
const ACTION_FOR: Record<Pending['kind'], IdentityAction> = {
reveal: 'reveal',
remove: 'delete',
};
// meta-watch's sentinel cell for profile records: display name and avatar are
// about the *user*, not a content file, so they live at (uid, "self", ...).
// meta-core stores them opaquely and knows nothing of the convention.
const PROFILE_CID = 'self';
const PROFILE_NAME_KEY = 'profile:name';
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
padding: '1.5rem',
marginBottom: '1rem',
};
const buttonStyle: React.CSSProperties = {
padding: '0.5rem 1rem',
background: '#0f3460',
color: '#fff',
border: 'none',
borderRadius: '4px',
cursor: 'pointer',
marginRight: '0.5rem',
};
const dangerButtonStyle: React.CSSProperties = {
...buttonStyle,
background: '#7f1d1d',
};
const inputStyle: React.CSSProperties = {
width: '100%',
padding: '0.5rem',
marginBottom: '0.5rem',
background: '#1a1a2e',
border: '1px solid #0f3460',
borderRadius: '4px',
color: '#fff',
fontFamily: 'monospace',
};
const monoStyle: React.CSSProperties = {
fontFamily: 'monospace',
fontSize: '0.85rem',
wordBreak: 'break-all',
background: '#0a0a18',
border: '1px solid #0f3460',
padding: '0.75rem',
borderRadius: '4px',
};
const dangerPanelStyle: React.CSSProperties = {
background: '#1f0a0a',
border: '1px solid #7f1d1d',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginTop: '0.75rem',
};
const errorTextStyle: React.CSSProperties = {
color: '#f87171',
background: '#1f0a0a',
padding: '0.5rem 0.75rem',
borderRadius: '4px',
};
function formatCreatedAt(unixSec?: number): string {
if (!unixSec) return '';
try {
return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC');
} catch {
return String(unixSec);
}
}
// Read the JSON body of a non-2xx response if present, fall back to status text.
async function errorMessage(res: Response, fallback: string): Promise<string> {
try {
const j = await res.json();
if (j && (j.message || j.error)) return j.message ?? j.error;
} catch {
// not JSON
}
return `${fallback} (HTTP ${res.status})`;
}
// Copy text, falling back to selecting the element that displays it.
//
// The fallback is load-bearing, not politeness: the dashboard is served over
// plain HTTP, where navigator.clipboard is undefined — without it the copy
// buttons do nothing at all on a real deployment.
async function copyToClipboard(text: string, fallbackElementId: string): Promise<void> {
try {
await navigator.clipboard.writeText(text);
} catch {
const sel = window.getSelection();
const el = document.getElementById(fallbackElementId);
if (sel && el) {
const range = document.createRange();
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
}
}
}
async function fetchAccounts(): Promise<Account[]> {
const res = await fetch('/api/identity/accounts');
if (!res.ok) throw new Error(await errorMessage(res, 'Failed to load accounts'));
const data = (await res.json()) as { accounts?: Account[] };
// Backend order is CreatedAt ascending then uid (identity.List) — stable, so
// render it as given rather than re-sorting here.
return data.accounts ?? [];
}
// Resolve an account's display name from the User Data Layer, or null.
//
// Never throws and never reports an error upward: a name is decoration on top
// of the uid, and the account list must render identically whether Redis is up
// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
// — meta-core stores the plaintext beside the signed record and verifies
// nothing — so the uid stays on screen as the real identity, and the length cap
// bounds what a hostile writer can do to the layout.
async function fetchDisplayName(uid: string): Promise<string | null> {
const url =
`/api/udl/record?uid=${encodeURIComponent(uid)}` +
`&cid=${encodeURIComponent(PROFILE_CID)}` +
`&key=${encodeURIComponent(PROFILE_NAME_KEY)}`;
try {
const res = await fetch(url);
if (!res.ok) return null; // 503 when storage is down, 500, …
const data = (await res.json()) as { exists?: boolean; value?: unknown };
// Key off `value`, not `exists`: clearing a name writes a tombstone, which
// leaves the cell existing with no plaintext. Private-tier cells look the
// same, which is exactly right — there is nothing to show for either.
if (typeof data.value !== 'string') return null;
const name = data.value.trim();
return name ? name.slice(0, 64) : null;
} catch {
return null;
}
}
// How much each account holds. One call for the whole page, not one per row:
// meta-core answers it from a keyspace SCAN (there is no per-user cid index),
// so per-row calls would walk the keyspace once per account.
//
// Never throws, for the same reason fetchDisplayName does not: the counts are
// context on top of the uid, and the list must render identically when Redis is
// down — accounts are file-backed, their data is not.
//
// Returns null on failure rather than an empty map. The difference is
// load-bearing: an empty map renders as "no stored data" against every account,
// which inside a delete confirmation is a false statement about what is at
// stake. Unknown has to look like unknown.
async function fetchStats(): Promise<Record<string, Stats> | null> {
try {
const res = await fetch('/api/udl/users/stats');
if (!res.ok) return null;
const data = (await res.json()) as { users?: Record<string, Stats> };
return data.users ?? {};
} catch {
return null;
}
}
// The secret-key box shared by the reveal and remove confirmations.
//
// Separate from the surrounding panel so both operations ask the same way, and
// so the key lives in this component's state rather than the page's — it is
// discarded with the panel instead of lingering in a parent that outlives it.
function SecretKeyConfirm({
label,
actionLabel,
busy,
onConfirm,
onCancel,
}: {
label: string;
actionLabel: string;
busy: boolean;
onConfirm: (secretKeyHex: string) => void;
onCancel: () => void;
}) {
const [secret, setSecret] = useState('');
const signable = canSign();
// Derived locally, purely to tell the operator they have the wrong key before
// they hit the button. The server would refuse either way.
const derived = secret.trim() ? uidFromSecretKey(secret) : null;
const mismatch = Boolean(secret.trim() && derived && derived !== label);
return (
<form
onSubmit={(e) => {
e.preventDefault();
onConfirm(secret.trim());
}}
>
<p style={{ color: '#fecaca', margin: '0 0 0.5rem' }}>
Paste this account's secret key to prove it is yours. It is used to sign a one-time
challenge <em>in this browser</em> and is never sent to the server.
</p>
{!signable && <p style={errorTextStyle}>{INSECURE_CONTEXT_MESSAGE}</p>}
<input
style={inputStyle}
type="password"
autoComplete="off"
placeholder="64 hex characters"
value={secret}
onChange={(e) => setSecret(e.target.value)}
disabled={busy || !signable}
autoFocus
/>
{mismatch && (
<p style={{ ...errorTextStyle, marginTop: 0 }}>
That key belongs to a different account ({derived}).
</p>
)}
<button
type="submit"
style={dangerButtonStyle}
disabled={busy || !signable || !secret.trim() || mismatch}
>
{busy ? 'Verifying…' : actionLabel}
</button>
<button type="button" style={buttonStyle} onClick={onCancel} disabled={busy}>
Cancel
</button>
</form>
);
}
// The one place a private key is ever displayed. Rendered exclusively (the
// account list is hidden while it is open) so the "save it now" contract of a
// freshly generated key is not competing with anything, and so the hardcoded
// element id below stays unique.
function SecretKeyPanel({ data, onDismiss }: { data: SecretKeyView; onDismiss: () => void }) {
const [copied, setCopied] = useState(false);
const copy = async () => {
await copyToClipboard(data.privateKeyHex, 'mm-privkey-box');
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
<div
style={{
background: '#7f1d1d',
color: '#fee2e2',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginBottom: '1rem',
}}
>
{data.kind === 'generated' ? (
<>
<strong>This private key is shown ONCE.</strong> Save it now in a password manager.
You can ask for it again later with <em>Reveal private key</em>, but only for as long
as this account exists on this node — delete it and the key is gone for good.
</>
) : (
<>
<strong>This is the live private key for an existing account.</strong> Anyone holding
it can sign as this uid forever, on any machine. There is no revocation and no
rotation. Do not paste it into chat, tickets or screenshots.
</>
)}
</div>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
<div style={monoStyle}>{data.uid}</div>
<p style={{ color: '#9ca3af', margin: '1rem 0 0.5rem' }}>privateKeyHex (secret):</p>
<div id="mm-privkey-box" style={{ ...monoStyle, background: '#1f0a0a', borderColor: '#7f1d1d' }}>
{data.privateKeyHex}
</div>
<div style={{ marginTop: '1rem' }}>
<button style={buttonStyle} onClick={copy}>
{copied ? 'Copied' : 'Copy private key'}
</button>
<button style={dangerButtonStyle} onClick={onDismiss}>
{data.kind === 'generated' ? 'I have saved it' : 'Done — hide it'}
</button>
</div>
</div>
);
}
// Create / import. Always available, not just on an empty keystore: this is an
// admin console, so adding an account to a populated node has to be possible.
function NewAccountPanel({
onGenerated,
onImported,
}: {
onGenerated: (s: SecretKeyView) => void;
onImported: () => void;
}) {
const [busy, setBusy] = useState(false);
const [importing, setImporting] = useState(false);
const [importHex, setImportHex] = useState('');
const [error, setError] = useState<string | null>(null);
const [notice, setNotice] = useState<string | null>(null);
const generate = async () => {
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/generate', { method: 'POST' });
if (!res.ok) {
setError(await errorMessage(res, 'Generate failed'));
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
onGenerated({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'generated' });
} catch (e) {
setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
const importKey = async (e: React.FormEvent) => {
e.preventDefault();
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/import', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ privateKeyHex: importHex.trim() }),
});
if (!res.ok) {
setError(await errorMessage(res, 'Import failed'));
return;
}
// Import is idempotent — presenting the key IS the proof of ownership, so
// a key this node already holds is a success, not a conflict. `created`
// only decides the wording.
const data = (await res.json()) as { created?: boolean };
setNotice(
data.created
? 'Imported — a new account was added to this keystore.'
: 'That key was already in this keystore; nothing changed.'
);
setImportHex('');
setImporting(false);
onImported();
} catch (e) {
setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
return (
<div style={cardStyle}>
<h3 style={{ marginTop: 0 }}>Add an account</h3>
<p style={{ color: '#9ca3af', marginTop: 0 }}>
Generating mints a fresh secp256k1 keypair here. Importing adds a key created elsewhere —
it is also how an existing account is moved onto this node, so importing a key the
keystore already holds succeeds and changes nothing.
</p>
{error && <p style={errorTextStyle}>{error}</p>}
{notice && (
<p style={{ color: '#9ca3af', background: '#0a0a18', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
{notice}
</p>
)}
<div>
<button style={buttonStyle} disabled={busy} onClick={generate}>
{busy ? 'Working…' : 'Generate new keypair'}
</button>
<button
style={buttonStyle}
disabled={busy}
onClick={() => {
setImporting((x) => !x);
setError(null);
}}
>
{importing ? 'Cancel import' : 'Import existing private key'}
</button>
</div>
{importing && (
<form onSubmit={importKey} style={{ marginTop: '1rem' }}>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>
64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine.
</p>
<input
style={inputStyle}
placeholder="e.g. a3f1…"
value={importHex}
onChange={(e) => setImportHex(e.target.value)}
required
autoFocus
/>
<button type="submit" style={buttonStyle} disabled={busy || !importHex.trim()}>
{busy ? 'Importing…' : 'Import'}
</button>
</form>
)}
</div>
);
}
function AccountCard({
account,
name,
stats,
busy,
disabled,
pending,
error,
onAsk,
onCancel,
onReveal,
onRemove,
}: {
account: Account;
name?: string;
// undefined means "counted, and it holds nothing"; null means "could not be
// counted". The two must not read the same on a confirmation screen.
stats?: Stats | null;
busy: boolean;
disabled: boolean;
pending: Pending | null;
error: string | null;
onAsk: (p: Pending) => void;
onCancel: () => void;
onReveal: (uid: string, secretKeyHex: string) => void;
onRemove: (uid: string, secretKeyHex: string) => void;
}) {
const [copied, setCopied] = useState(false);
const uidBoxId = `mm-uid-${account.uid}`;
const label = name ?? account.uid;
const counted = stats !== null;
const records = stats?.records ?? 0;
const titles = stats?.cids ?? 0;
const copyUid = async () => {
await copyToClipboard(account.uid, uidBoxId);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={cardStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start', gap: '1rem' }}>
<div style={{ minWidth: 0, flex: 1 }}>
<h3 style={{ margin: '0 0 0.5rem', color: name ? '#fff' : '#888' }}>
{name ?? 'Unnamed account'}
</h3>
<div id={uidBoxId} style={monoStyle}>
{account.uid}
</div>
<p style={{ color: '#888', margin: '0.75rem 0 0', fontSize: '0.85rem' }}>
Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}
</p>
<p style={{ color: '#888', margin: '0.25rem 0 0', fontSize: '0.85rem' }}>
{/* An account that has never been used legitimately holds nothing, so
zero is a real answer — but only once the count actually ran. */}
{!counted
? 'Stored data: could not be counted (storage unavailable)'
: records === 0
? 'No stored data'
: `${records.toLocaleString()} stored ${records === 1 ? 'record' : 'records'}` +
` across ${titles.toLocaleString()} ${titles === 1 ? 'title' : 'titles'}`}
</p>
</div>
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem', flexShrink: 0 }}>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={copyUid} disabled={disabled}>
{copied ? 'Copied' : 'Copy uid'}
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'reveal' })}
disabled={disabled}
>
Reveal private key
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'remove' })}
disabled={disabled}
>
Remove
</button>
</div>
</div>
{error && <p style={{ ...errorTextStyle, marginBottom: 0 }}>{error}</p>}
{pending?.uid === account.uid && pending.kind === 'reveal' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Show the private key for <strong>{label}</strong>?
</p>
<p style={{ color: '#fecaca', marginTop: 0 }}>
The key <em>is</em> the account. Anyone who copies it can sign records as this uid on
any machine, forever — there is no way to revoke or rotate it afterwards. Because
proving ownership already requires the key, this only confirms that the copy this node
holds matches yours.
</p>
<SecretKeyConfirm
label={account.uid}
actionLabel="Show the private key"
busy={busy}
onConfirm={(secret) => onReveal(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
{pending?.uid === account.uid && pending.kind === 'remove' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Remove <strong>{label}</strong> from this node?
</p>
<ul style={{ color: '#fecaca', paddingLeft: '1.2rem', margin: '0 0 0.75rem' }}>
<li>
<strong>The key is gone for good.</strong> The uid <em>is</em> the public key, and
the private half exists only in this node's keystore. If nobody saved it, nothing —
not you, not meta-core — can ever sign as this uid again. Reveal and copy it first
if there is any doubt.
</li>
<li>
<strong>Nothing is retracted.</strong> Records already signed under this uid keep
verifying on every peer that replicated them. Deleting a key is not a recall.
</li>
<li>
<strong>Its data is destroyed, not orphaned.</strong>{' '}
{!counted
? 'This account\u2019s records could not be counted because storage is unavailable — ' +
'so meta-core will refuse this delete rather than remove the key and strand them.'
: records === 0
? 'This account holds no User Data Layer records, so there is nothing to purge.'
: `All ${records.toLocaleString()} User Data Layer ${
records === 1 ? 'record' : 'records'
} — likes, ratings, watch progress across ${titles.toLocaleString()} ${
titles === 1 ? 'title' : 'titles'
} — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
This cannot be undone and there is no backup.
</li>
<li>
<strong>This signs nobody out elsewhere.</strong> Any device that holds this key
still holds it, and can import it back here at any time.
</li>
</ul>
<SecretKeyConfirm
label={account.uid}
actionLabel="Remove this account and all its data"
busy={busy}
onConfirm={(secret) => onRemove(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
</div>
);
}
export default function Identity() {
const [accounts, setAccounts] = useState<Account[] | null>(null);
const [names, setNames] = useState<Record<string, string>>({});
const [stats, setStats] = useState<Record<string, Stats> | null>({});
const [notice, setNotice] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [fetchError, setFetchError] = useState<string | null>(null);
const [secret, setSecret] = useState<SecretKeyView | null>(null);
const [pending, setPending] = useState<Pending | null>(null);
const [busyUid, setBusyUid] = useState<string | null>(null);
const [rowError, setRowError] = useState<{ uid: string; message: string } | null>(null);
// Bumped per refresh so a slow response from a superseded load cannot
// overwrite a newer one. The page does not poll (unlike the rest of the
// dashboard) — a key list has no business changing under the cursor.
const reqGen = useRef(0);
const refresh = async () => {
const gen = ++reqGen.current;
setLoading(true);
setFetchError(null);
let list: Account[];
try {
list = await fetchAccounts();
} catch (e) {
if (gen !== reqGen.current) return;
setFetchError(e instanceof Error ? e.message : String(e));
setLoading(false);
return;
}
if (gen !== reqGen.current) return;
// Render the list before resolving names, so a down Redis delays nothing.
setAccounts(list);
setLoading(false);
// Names and counts both come after the list is on screen, and neither can
// fail the page: a down Redis costs decoration, not the account list.
const [settled, counts] = await Promise.all([
Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
fetchStats(),
]);
if (gen !== reqGen.current) return;
const next: Record<string, string> = {};
settled.forEach((r, i) => {
if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value;
});
// Replace rather than merge, so a removed account's name and counts go with
// it instead of lingering against a uid that no longer exists.
setNames(next);
setStats(counts);
};
useEffect(() => {
refresh();
}, []);
const reveal = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.reveal, secretKeyHex);
// POST with a body rather than ?uid=, so neither the target nor the
// signature reaches the proxy's access log. The handler accepts the uid
// either way; the proof must be in the body.
const res = await fetch('/api/identity/reveal', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Reveal failed') });
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
setPending(null);
setSecret({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'revealed' });
} catch (e) {
setRowError({ uid, message: `Reveal failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const remove = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
setNotice(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.remove, secretKeyHex);
// The uid is mandatory: the signature authorises one specific account, so
// the backend refuses an unqualified delete rather than guess a target.
const res = await fetch('/api/identity', {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Remove failed') });
return;
}
// Report what the purge actually removed. "Account removed" on its own
// would not distinguish a clean delete from one that stranded the data.
const data = (await res.json().catch(() => ({}))) as { purged?: Stats };
const purged = data.purged;
setNotice(
purged && purged.records > 0
? `Account removed. Purged ${purged.records.toLocaleString()} User Data Layer ` +
`${purged.records === 1 ? 'record' : 'records'} across ` +
`${purged.cids.toLocaleString()} ${purged.cids === 1 ? 'title' : 'titles'}.`
: 'Account removed. It held no User Data Layer records.'
);
setPending(null);
await refresh();
} catch (e) {
setRowError({ uid, message: `Remove failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const dismissSecret = () => {
setSecret(null);
refresh();
};
const ask = (p: Pending) => {
setRowError(null);
setNotice(null);
setPending(p);
};
const showList = !loading && !fetchError && !secret;
return (
// App.tsx's content area is `overflow: hidden`, so every page owns its own
// scrolling (Files.tsx does the same). Without this the account list is
// simply clipped at the viewport with no scrollbar — invisible on a node
// with two accounts, and a dead end on one with fifteen.
//
// The scroller is the outer element and the padding lives inside it, so the
// scrollbar sits at the edge of the content area rather than floating in
// 2rem of whitespace, and `margin: 0 auto` still centres the column.
<div style={{ height: '100%', overflowY: 'auto', minHeight: 0 }}>
<div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<h1 style={{ marginBottom: 0 }}>Identity</h1>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={refresh} disabled={loading}>
{loading ? 'Loading…' : 'Refresh'}
</button>
</div>
<p style={{ color: '#9ca3af' }}>
Every signing key in this node's keystore. meta-core has no "current user" — it stores
accounts, and which one is <em>you</em> is decided by whichever client signs in (e.g.
meta-watch). Keys live one JSON file per account under{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/&lt;uid&gt;.json</code>, and are used to sign
User Data Layer records (ratings, watch progress, "liked" flags).
</p>
<div style={{ ...dangerPanelStyle, marginTop: 0, marginBottom: '1rem' }}>
<p style={{ color: '#fecaca', margin: 0 }}>
<strong>The key is the only authority.</strong> Revealing or removing an account requires
that account's secret key, entered here and used to sign a one-time challenge in your
browser — there is no admin override, and holding this page is not one. Listing and
creating accounts are still open to anything that can reach this API, so the perimeter in
front of this port is what keeps the roster private.
</p>
<p style={{ color: '#fecaca', margin: '0.5rem 0 0' }}>
What this does <em>not</em> defend against: anyone with filesystem access to{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/</code> reads every private key directly.
The keys are stored in plaintext, matching the trust model of the rest of{' '}
<code>/meta-core</code>.
</p>
</div>
{notice && (
<div style={{ ...cardStyle, border: '1px solid #0f3460' }}>
<p style={{ color: '#9ca3af', margin: 0 }}>{notice}</p>
</div>
)}
{loading && <div style={cardStyle}>Loading…</div>}
{fetchError && (
<div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
<p style={{ color: '#f87171', margin: '0 0 1rem' }}>{fetchError}</p>
<button style={buttonStyle} onClick={refresh}>
Retry
</button>
</div>
)}
{!loading && !fetchError && secret && <SecretKeyPanel data={secret} onDismiss={dismissSecret} />}
{showList && (
<>
<NewAccountPanel onGenerated={(s) => setSecret(s)} onImported={refresh} />
<h2 style={{ fontSize: '1.1rem', color: '#9ca3af', margin: '1.5rem 0 1rem' }}>
Accounts ({accounts?.length ?? 0})
</h2>
{accounts && accounts.length === 0 && (
<div style={cardStyle}>
<p style={{ color: '#9ca3af', margin: 0 }}>
No accounts in this keystore. Accounts are normally created by a client when
someone signs up; use <em>Add an account</em> above if you are bootstrapping this
node.
</p>
</div>
)}
{accounts?.map((a) => (
<AccountCard
key={a.uid}
account={a}
name={names[a.uid]}
stats={stats === null ? null : stats[a.uid]}
busy={busyUid === a.uid}
disabled={busyUid !== null}
pending={pending}
error={rowError?.uid === a.uid ? rowError.message : null}
onAsk={ask}
onCancel={() => setPending(null)}
onReveal={reveal}
onRemove={remove}
/>
))}
</>
)}
</div>
</div>
);
}
+220 -42
View File
@@ -1,19 +1,47 @@
import { useEffect, useState } from 'react';
interface MountIOStats {
source: 'rclone';
bytesRead: number;
bytesWritten?: number;
readBps: number;
writeBps?: number;
readOps?: number;
writeOps?: number;
readIops?: number;
writeIops?: number;
transfersPerSec?: number;
lastSampleAt: number;
intervalMs?: number;
}
interface Mount {
id: string;
name: string;
type: 'nfs' | 'smb' | 'rclone';
type: 'smb' | 'rclone';
enabled: boolean;
desiredMounted: boolean;
mountPath: string;
mounted: boolean;
error?: string;
lastChecked: number;
// VFS cache configuration applied at mount time. Empty string means "use
// server default" (50G / 24h / 5m).
cacheMaxSize?: string;
cacheMaxAge?: string;
dirCacheTime?: string;
pollingEnabled?: boolean;
pollingIntervalMs?: number;
pollingActive?: boolean;
lastPolledScan?: number;
// Adaptive scan progress (set by the meta-core poller)
scanning?: boolean;
currentScanStartedAt?: number; // ms; only while scanning
lastScanDurationMs?: number;
nextScanAt?: number; // ms; planned start of next adaptive scan
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
// so the same numbers appear on every mount card.
ioStats?: MountIOStats;
}
interface RcloneRemote {
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
background: '#7f1d1d',
};
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
function formatBytes(n: number): string {
if (!isFinite(n) || n < 0) return '—';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let v = n;
while (v >= 1000 && i < units.length - 1) {
v /= 1000;
i++;
}
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
}
function formatRate(bps: number): string {
return `${formatBytes(bps)}/s`;
}
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
// is just noise.
function formatDuration(ms: number): string {
if (ms < 0 || !isFinite(ms)) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
const remS = s % 60;
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
const h = Math.floor(m / 60);
const remM = m % 60;
return remM ? `${h}h ${remM}m` : `${h}h`;
}
// IOStatsLine renders a single line of live throughput info beneath each
// mount card. Source is always rclone after the rclone-only consolidation,
// so we report file-level transfers/s rather than kernel IOPS.
function IOStatsLine({ stats }: { stats: MountIOStats }) {
const opsLabel = 'transfers/s';
const opsValue = stats.transfersPerSec;
const ageMs = Date.now() - stats.lastSampleAt;
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
return (
<p
style={{
color: stale ? '#6b7280' : '#9ca3af',
fontSize: '0.85rem',
margin: '0.25rem 0 0',
}}
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
>
↓ {formatRate(stats.readBps)}
{opsValue !== undefined && opsValue > 0 && (
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
)}
<> · {formatBytes(stats.bytesRead)} read total</>
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
<> · {formatBytes(stats.bytesWritten)} written</>
)}
</p>
);
}
interface ScanProgressProps {
mount: Mount;
now: number;
}
// Progress bar shows scan ETA when running, an idle row with a countdown to
// the next adaptive scan when waiting. We can't show true % complete (we
// don't know the file count up-front), so the running bar is filled by
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
// it pins at 95% to signal "taking longer than last time".
function ScanProgress({ mount, now }: ScanProgressProps) {
if (!mount.pollingEnabled) return null;
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
const lastDuration = mount.lastScanDurationMs ?? 0;
let fill = 0;
let label = '';
if (isScanning) {
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
if (lastDuration > 0) {
fill = Math.min(0.95, elapsed / lastDuration);
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
} else {
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
}
} else {
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
if (lastDuration > 0 && mount.lastPolledScan) {
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
} else if (mount.nextScanAt) {
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
} else {
label = 'Polling enabled — waiting for first scan';
}
}
return (
<div style={{ marginTop: '0.5rem' }}>
<div
style={{
height: '6px',
background: '#0f3460',
borderRadius: '3px',
overflow: 'hidden',
position: 'relative',
}}
>
<div
style={{
width: `${Math.round(fill * 100)}%`,
height: '100%',
background: isScanning ? '#4ade80' : '#374151',
transition: 'width 0.5s linear',
}}
/>
</div>
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
</div>
);
}
export default function Mounts() {
const [mounts, setMounts] = useState<Mount[]>([]);
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
@@ -50,19 +209,23 @@ export default function Mounts() {
const [showForm, setShowForm] = useState(false);
const [formData, setFormData] = useState({
name: '',
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
nfsServer: '',
nfsPath: '',
type: 'smb' as 'smb' | 'rclone',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
const [scanning, setScanning] = useState<string | null>(null);
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
// refetches (which run every 5s).
const [now, setNow] = useState<number>(() => Date.now());
const fetchMounts = async () => {
try {
@@ -94,7 +257,11 @@ export default function Mounts() {
fetchMounts();
fetchRemotes();
const interval = setInterval(fetchMounts, 5000);
return () => clearInterval(interval);
const tick = setInterval(() => setNow(Date.now()), 1000);
return () => {
clearInterval(interval);
clearInterval(tick);
};
}, []);
const handleMount = async (id: string) => {
@@ -143,10 +310,7 @@ export default function Mounts() {
type: formData.type,
};
if (formData.type === 'nfs') {
body.nfsServer = formData.nfsServer;
body.nfsPath = formData.nfsPath;
} else if (formData.type === 'smb') {
if (formData.type === 'smb') {
body.smbServer = formData.smbServer;
body.smbShare = formData.smbShare;
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
@@ -156,6 +320,12 @@ export default function Mounts() {
body.rclonePath = formData.rclonePath;
}
// Cache knobs — only forward non-empty values; the server applies defaults
// when fields are omitted.
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
// Add polling configuration
body.pollingEnabled = formData.pollingEnabled;
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
@@ -171,16 +341,17 @@ export default function Mounts() {
setShowForm(false);
setFormData({
name: '',
type: 'nfs',
nfsServer: '',
nfsPath: '',
type: 'smb',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
fetchMounts();
@@ -231,32 +402,15 @@ export default function Mounts() {
<select
style={inputStyle}
value={formData.type}
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
onChange={(e) => {
const type = e.target.value as 'smb' | 'rclone';
setFormData({ ...formData, type });
}}
>
<option value="nfs">NFS</option>
<option value="smb">SMB/CIFS</option>
<option value="rclone">rclone</option>
<option value="smb">SMB (via rclone)</option>
<option value="rclone">rclone (pre-defined remote)</option>
</select>
{formData.type === 'nfs' && (
<>
<input
style={inputStyle}
placeholder="NFS Server (e.g., 192.168.1.100)"
value={formData.nfsServer}
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
required
/>
<input
style={inputStyle}
placeholder="NFS Path (e.g., /export/media)"
value={formData.nfsPath}
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
required
/>
</>
)}
{formData.type === 'smb' && (
<>
<input
@@ -313,6 +467,30 @@ export default function Mounts() {
</>
)}
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
VFS cache (leave blank for defaults: 50G / 24h / 5m)
</p>
<input
style={inputStyle}
placeholder="Cache max size (e.g., 50G)"
value={formData.cacheMaxSize}
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Cache max age (e.g., 24h)"
value={formData.cacheMaxAge}
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Dir cache time (e.g., 5m)"
value={formData.dirCacheTime}
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
/>
</div>
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
<input
@@ -362,17 +540,17 @@ export default function Mounts() {
</span>
</p>
{mount.pollingEnabled && (
<p style={{ color: '#888', fontSize: '0.9rem' }}>
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
Polling:{' '}
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
{mount.pollingActive ? 'Active' : 'Inactive'}
</span>
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{mount.lastPolledScan && (
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
)}
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{' • Adaptive 10× idle'}
</p>
)}
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
<ScanProgress mount={mount} now={now} />
{mount.error && (
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
)}
-1
View File
@@ -10,7 +10,6 @@ set -e
# Create required directories
mkdir -p /meta-core/locks
mkdir -p /meta-core/db/redis
mkdir -p /meta-core/services
mkdir -p /meta-core/mounts/errors
mkdir -p /files
mkdir -p /var/log/supervisor
+7 -29
View File
@@ -2,9 +2,13 @@
# =============================================================================
# Leader Election Script for meta-core
#
# Uses flock for leader election. Only the leader runs the full application
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
# Followers remain dormant, periodically retrying to acquire the lock.
# The flock here is a MUTEX, not service discovery. It is the only thing
# stopping two containers that mount the same /meta-core volume from putting
# two Redis writers on one RDB/AOF. Sibling services no longer read anything
# from this directory — they locate meta-core over UDP (meta-discovery v1, see
# docs/project-architecture/service-discovery.md), so the kv-leader.info file
# this script used to publish is gone. Keep the lock; it costs nothing and the
# corruption it prevents is unrecoverable.
#
# Environment variables:
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +24,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
# Get the container's IP address
@@ -40,35 +43,13 @@ get_local_ip() {
echo "$ip"
}
# Write leader info file (plain text API URL)
write_leader_info() {
local ip=$(get_local_ip)
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
# Write atomically using temp file + rename
local temp_file="${INFO_FILE}.tmp"
echo "$api_url" > "$temp_file"
mv "$temp_file" "$INFO_FILE"
echo "[election] Wrote leader info: $api_url"
}
# Clean up leader info file on shutdown
cleanup() {
echo "[election] Cleaning up..."
rm -f "$INFO_FILE"
exit 0
}
# Set up signal handlers for graceful shutdown
trap cleanup SIGTERM SIGINT
# Ensure lock directory exists
mkdir -p "$(dirname "$LOCK_FILE")"
echo "[election] Starting leader election..."
echo "[election] Lock file: $LOCK_FILE"
echo "[election] Info file: $INFO_FILE"
# Open lock file for flock (file descriptor 200)
exec 200>"$LOCK_FILE"
@@ -79,9 +60,6 @@ while true; do
if flock -n 200; then
echo "[election] Acquired flock - transitioning to LEADER"
# Write leader info before starting services
write_leader_info
# exec supervisord - replaces this process
# When supervisord exits, the flock is released automatically
echo "[election] Starting supervisord..."
+138 -170
View File
@@ -2,11 +2,17 @@
# =============================================================================
# Mount Watcher Script
#
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
# Runs as a supervisord program, polling every 5 seconds.
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
# rclone RC daemon. After the rclone-only consolidation, this script no
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
# through rclone's mount/mount endpoint.
#
# Uses findmnt to check actual mount status (source of truth).
# Writes errors to /meta-core/mounts/errors/{id}.error
# Mount types:
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
# - "rclone" → references a pre-defined remote in rclone.conf
#
# All mounts are read-only by construction (no API surface to override).
# Errors surface in /meta-core/mounts/errors/{id}.error
# =============================================================================
set -u
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
FILES_PATH="${FILES_PATH:-/files}"
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
# Fall-back VFS cache settings — applied when a mount entry leaves the field
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
# packages/meta-core/internal/mounts/types.go.
DEFAULT_CACHE_MAX_SIZE="50G"
DEFAULT_CACHE_MAX_AGE="24h"
DEFAULT_DIR_CACHE_TIME="5m"
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
# ANSI colors
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -54,9 +70,7 @@ init_dirs() {
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
is_mounted() {
local mount_path="$1"
# Remove trailing slash for consistent matching
local clean_path="${mount_path%/}"
# Check if the path appears as a mount point in /proc/mounts (field 2)
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
}
@@ -74,145 +88,102 @@ write_error() {
echo "$error" >> "$ERROR_DIR/$id.error"
}
# Mount NFS share (always read-only)
do_mount_nfs() {
local id="$1"
local server="$2"
local path="$3"
local mount_path="$4"
local options="${5:-}"
# Build the rclone remote source string for a mount entry. Echoes the result.
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
# the optional remote path.
build_rclone_fs() {
local type="$1"
local smb_server="$2"
local smb_share="$3"
local smb_username="$4"
local smb_password_obscured="$5"
local smb_domain="$6"
local rclone_remote="$7"
local rclone_path="$8"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
local rc=$?
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "NFS mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "NFS mount failed: $output"
fi
return $rc
}
# Mount SMB/CIFS share (always read-only)
do_mount_smb() {
local id="$1"
local server="$2"
local share="$3"
local mount_path="$4"
local username="${5:-}"
local password_obscured="${6:-}"
local domain="${7:-}"
local options="${8:-}"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
# If we have credentials, reveal password and build options
if [ -n "$username" ]; then
mount_opts="${mount_opts},username=${username}"
if [ -n "$password_obscured" ]; then
# Reveal password using rclone (only exists in memory briefly)
local password
password=$(rclone reveal "$password_obscured" 2>/dev/null)
if [ -n "$password" ]; then
mount_opts="${mount_opts},password=${password}"
case "$type" in
smb)
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
# embed without quoting.
local opts="host=${smb_server}"
if [ -n "$smb_username" ]; then
opts="${opts},user=${smb_username}"
fi
fi
if [ -n "$domain" ]; then
mount_opts="${mount_opts},domain=${domain}"
fi
fi
# Add any additional options
if [ -n "$options" ]; then
if [ -n "$mount_opts" ]; then
mount_opts="${mount_opts},${options}"
else
mount_opts="$options"
fi
fi
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
# Execute mount directly (not via eval) to handle special chars in password
local output
local rc
if [ -n "$mount_opts" ]; then
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
rc=$?
else
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
rc=$?
fi
# Clear password from any variables (best effort)
password=""
mount_opts=""
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "SMB mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "SMB mount failed: $output"
fi
return $rc
if [ -n "$smb_password_obscured" ]; then
opts="${opts},pass=${smb_password_obscured}"
fi
if [ -n "$smb_domain" ]; then
opts="${opts},domain=${smb_domain}"
fi
echo ":smb,${opts}:${smb_share}"
;;
rclone)
echo "${rclone_remote}:${rclone_path}"
;;
*)
echo ""
;;
esac
}
# Mount rclone remote using RC API (always read-only)
# Mount via rclone RC API (always read-only).
do_mount_rclone() {
local id="$1"
local remote="$2"
local remote_path="$3"
local mount_path="$4"
local options="${5:-}"
local fs="$2"
local mount_path="$3"
local cache_max_size="$4"
local cache_max_age="$5"
local dir_cache_time="$6"
mkdir -p "$mount_path"
# Build the remote source string
local fs="${remote}:${remote_path}"
# Apply defaults for any blank cache fields.
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
# Use rclone RC API to mount (always read-only for safety)
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
# Build JSON body for rclone mount with read-only option
# Build JSON body via python3 — embedding the obscured password and
# display fs string into a hand-rolled heredoc is asking for an injection
# bug the next time we add a field with a quote in it.
local json_body
json_body=$(cat <<EOF
{
"fs": "${fs}",
"mountPoint": "${mount_path}",
json_body=$(python3 -c '
import json, sys
print(json.dumps({
"fs": sys.argv[1],
"mountPoint": sys.argv[2],
"mountOpt": {
"AllowOther": true,
"ReadOnly": true
"AllowOther": True,
"ReadOnly": True,
},
"vfsOpt": {
"CacheMode": 2,
"ReadOnly": true
}
}
EOF
)
# CacheMode 3 = "full" — cache both reads and writes on local disk.
# CacheMode 2 ("writes") would silently bypass the cache for reads,
# which is the opposite of what we want for a read-only mount.
"CacheMode": 3,
"CacheMaxSize": sys.argv[3],
"CacheMaxAge": sys.argv[4],
"DirCacheTime": sys.argv[5],
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
# throughput in isolation but, under the 4+ concurrent reads from
# the processing pipeline, multiplies into 16+ simultaneous SMB
# requests and tanks aggregate throughput. Left at default — rely on
# file-level concurrency from the pipeline for parallelism instead.
"ReadOnly": True,
},
}))
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
local output
output=$(curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-u "$RCLONE_RC_AUTH" \
-d "$json_body" \
"http://127.0.0.1:5572/mount/mount" 2>&1)
"${RCLONE_RC_URL}/mount/mount" 2>&1)
# Wait a moment for mount to establish
sleep 2
if is_mounted "$mount_path"; then
@@ -221,7 +192,6 @@ EOF
return 0
else
local error_msg="rclone mount failed"
# Try to extract error from response
if echo "$output" | grep -q "error"; then
error_msg="$output"
fi
@@ -231,36 +201,28 @@ EOF
fi
}
# Unmount a path
# Unmount a path. Always goes through the rclone RC API now — every mount we
# create lives behind rclone's FUSE driver.
do_unmount() {
local id="$1"
local mount_path="$2"
local type="$3"
if ! is_mounted "$mount_path"; then
log "Already unmounted: $mount_path"
return 0
fi
if [ "$type" = "rclone" ]; then
# Use rclone RC API to unmount
log "Unmounting rclone: $mount_path"
curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-d "{\"mountPoint\":\"${mount_path}\"}" \
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
else
log "Unmounting: $mount_path"
umount "$mount_path" 2>&1
fi
log "Unmounting: $mount_path"
curl -s -X POST \
-H "Content-Type: application/json" \
-u "$RCLONE_RC_AUTH" \
-d "{\"mountPoint\":\"${mount_path}\"}" \
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
# Wait a moment
sleep 1
# Check if still mounted
# Fallback to lazy umount in case the FUSE process is wedged.
if is_mounted "$mount_path"; then
# Force unmount with lazy option
log_warn "Force unmounting (lazy): $mount_path"
umount -l "$mount_path" 2>&1
sleep 1
@@ -277,12 +239,11 @@ do_unmount() {
# Process all mounts from config
process_mounts() {
# Check if config file exists
if [ ! -f "$CONFIG_FILE" ]; then
return
fi
# Parse JSON using python3
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
local mounts
mounts=$(python3 -c '
import sys, json
@@ -294,15 +255,12 @@ except:
sys.exit(0)
for m in data.get("mounts", []):
# Output pipe-separated values
fields = [
m.get("id", ""),
m.get("type", ""),
str(m.get("enabled", True)),
str(m.get("desiredMounted", False)),
m.get("mountPath", ""),
m.get("nfsServer", ""),
m.get("nfsPath", ""),
m.get("smbServer", ""),
m.get("smbShare", ""),
m.get("smbUsername", ""),
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
m.get("smbDomain", ""),
m.get("rcloneRemote", ""),
m.get("rclonePath", ""),
m.get("options", "")
m.get("cacheMaxSize", ""),
m.get("cacheMaxAge", ""),
m.get("dirCacheTime", ""),
]
print("|".join(fields))
' 2>/dev/null)
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
return
fi
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
while IFS='|' read -r id type enabled desired mount_path \
smb_server smb_share smb_username smb_password_obscured smb_domain \
rclone_remote rclone_path \
cache_max_size cache_max_age dir_cache_time; do
# Skip empty lines
[ -z "$id" ] && continue
# Check actual mount status
# Skip retired/legacy types — the Go-side migration disables NFS but a
# stale entry could still arrive here on a downgrade/upgrade race.
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
continue
fi
local is_mounted_now=false
is_mounted "$mount_path" && is_mounted_now=true
# Handle disabled mounts - unmount if currently mounted
# Disabled → unmount if currently up
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) disabled, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
continue
fi
# Handle desired state
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
# Should be mounted
if [ "$is_mounted_now" = false ]; then
local fs
fs=$(build_rclone_fs "$type" \
"$smb_server" "$smb_share" "$smb_username" \
"$smb_password_obscured" "$smb_domain" \
"$rclone_remote" "$rclone_path")
if [ -z "$fs" ]; then
log_error "Could not build rclone fs spec for mount $id (type=$type)"
write_error "$id" "Invalid mount config: missing required fields for type=$type"
continue
fi
log "Mount $id ($mount_path) desired but not mounted, mounting..."
case "$type" in
nfs)
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
;;
smb)
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
;;
rclone)
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
;;
*)
log_error "Unknown mount type: $type"
;;
esac
do_mount_rclone "$id" "$fs" "$mount_path" \
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
fi
else
# Should be unmounted
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) not desired, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
fi
done <<< "$mounts"
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
# Main loop
main() {
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
init_dirs
+19 -8
View File
@@ -70,6 +70,9 @@ http {
proxy_buffering off;
proxy_cache off;
proxy_read_timeout 86400s;
# Snapshot import accepts multi-MB ZIP uploads.
client_max_body_size 0;
}
# =========================================================================
@@ -120,14 +123,10 @@ http {
# =========================================================================
# Service discovery
# =========================================================================
location /services {
proxy_pass http://meta-core-api/services;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
location /services/ {
proxy_pass http://meta-core-api/services/;
# meta-discovery v1 neighbours (supersedes /services, which stays
# while consumers migrate).
location /neighbors {
proxy_pass http://meta-core-api/neighbors;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
@@ -192,6 +191,18 @@ http {
# WebDAV HTML browser (for iframe embedding in dashboard)
# Returns HTML directory listings instead of JSON
# =========================================================================
# At the browser root, strip the "../" parent-dir row that autoindex
# otherwise emits — clicking it inside the dashboard iframe would
# navigate to "/" (the dashboard itself), nesting the page inside itself.
location = /webdav-browser/ {
alias /files/;
autoindex on;
charset utf-8;
sub_filter '<a href="../">../</a>' '';
sub_filter_once on;
}
location /webdav-browser/ {
alias /files/;
autoindex on;
+2 -2
View File
@@ -71,7 +71,7 @@ startsecs=2
# rclone: Remote storage mount daemon (RC API on port 5572)
# =============================================================================
[program:rclone]
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
priority=30
autostart=true
autorestart=true
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
startsecs=3
# =============================================================================
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
# =============================================================================
[program:mount-watcher]
command=/app/scripts/mount-watcher.sh
+771
View File
@@ -0,0 +1,771 @@
# API-Mediated Access (Redis Lockdown)
## Status
**Landed** — all four PRs (A/B/C/D) deployed and verified end-to-end on the
dev stack. Redis is unreachable from outside `metacore-app`; every metadata
read, write, and event flows through meta-core's HTTP+SSE surface. Builds
on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
meta-core is the single writer to the metadata keyspace — an assumption
that's now actually enforced rather than aspirational.
See "Implementation outcome" at the bottom for the delta between this doc's
plan and what shipped (notably the write-path CID resolution and the
dual-root migration sweep, which weren't in the original plan but turned
out to be load-bearing).
## Motivation
meta-core was designed as the sidecar that mediates every metadata read and
write — a single point that owns the Redis schema, enforces invariants
(reverse index, `cids/` key-set integrity, schema-version sentinel),
and presents a stable HTTP contract. Other services were meant to talk to
meta-core, not to Redis.
In practice, meta-sort opens its own Redis connection and writes
`file:<hashId>/<property>` keys with raw pipeline SETs
(`packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts:269-296`).
It picked up this shortcut early, before meta-core had a complete write
API, and the shortcut stuck.
This bypass is a real problem in three ways:
1. **Schema invariants leak.** When meta-core landed the UUID-rooted schema
([uuid-rooted-metadata.md](uuid-rooted-metadata.md)), meta-sort kept
writing midhash-rooted entries. The result was *dual roots per file* —
one set of keys at the watcher's UUID, another at meta-sort's midhash.
`file:__index__` doubled in size. The bug was invisible to meta-sort
(its writes succeeded) but broke the reverse-index resolution
downstream services rely on.
2. **Schema evolution is impossible.** Changing the Redis layout requires
coordinating across every service that touches it. The UUID-root
migration would have been a single-PR change if writes already routed
through the API.
3. **Self-hosted distributed deployment widens the threat model.** Peers
are run by different operators. Plugin authors may ship their own
services. We can't expect every binary in the network to honour
undocumented conventions. A buggy or careless service that connects
to Redis and writes the wrong key shape can corrupt the schema for
everybody on that node.
The third point is the one that flipped the design from "soft lock by
convention" to "hard lock by network topology." We're not protecting
against malicious services — that's an out-of-scope threat. We're
protecting against *accidental* corruption from honest mistakes in code
we don't fully control.
## The architecture in one picture
```
meta-core (only Redis writer + reader)
┌──────────────────────────────────────┐
│ HTTP API surface: │
│ POST/PUT/PATCH /api/metadata/* │ ◄── writes
│ GET /api/meta/{cid} │ ◄── reads (by CID)
│ GET /api/metadata/* │ ◄── reads (by id)
│ GET /api/file/{cid} │ ◄── content
│ GET /api/events/files │ ◄── SSE stream
│ GET /api/events/meta │ ◄── SSE stream
│ │ │
│ ▼ │
│ Redis (file:*, cid:*, streams) │
│ ◄── container-local, no exposed │
│ port, no docker-network alias │
└──────────────────────────────────────┘
▲ ▲
│ HTTP │ HTTP SSE
│ │
┌───────────┴────┐ ┌──────┴─────────┐
│ meta-sort │ │ meta-fuse │
│ meta-stremio │ │ meta-share │
│ meta-dup │ │ (any future) │
└────────────────┘ └────────────────┘
```
**The single-writer rule.** meta-core has exclusive write authority over
the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
other service can write — not because of an honoured convention, but
because they can't reach Redis at all.
**Stream reads also mediated.** Events go out through SSE endpoints on
meta-core's HTTP surface. Consumers don't see the underlying Redis
Streams. This is the Kubernetes / Consul pattern (see "Industry context"
below). It costs an extra HTTP hop versus the current direct XREAD; the
benefit is that the storage layer is free to evolve (or be replaced)
without breaking every consumer.
**Redis is private.** No host port mapping, no docker-network alias, no
URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
## Eventing — HTTP Server-Sent Events
SSE is the chosen transport for cross-service event consumption. The
mapping from Redis Streams onto SSE is direct enough that no eventing
feature we currently use is lost.
### Feature mapping (what we keep, how we keep it)
| Redis Streams | SSE equivalent |
|---|---|
| Fan-out to N consumers | meta-core holds N SSE connections, broadcasts events to each |
| Replay from a cursor | `Last-Event-ID` header on reconnect — built into the SSE spec |
| Retention / MAXLEN | Underlying Redis Stream still does the trimming; SSE is a thin shim |
| Backpressure on slow consumers | meta-core's SSE handler blocks on the TCP write; events stay in Redis until trimmed |
| Multiple streams (file:events, meta:events) | Separate endpoints: `/api/events/files`, `/api/events/meta` |
| At-least-once via XACK | Implicit — client persists last-received `id`, sends it on reconnect (see "Cursor semantics and consumer-group migration" below) |
The one Streams feature we don't get is **consumer groups for
horizontally-scaled replicas of the same logical consumer.** We don't use
this today and don't plan to — meta-sort, meta-fuse, etc. each run one
instance. If horizontal scale ever happens, it'll be by sharding hash-IDs
across replicas, not by splitting an event stream. SSE stays adequate.
### Cursor semantics and consumer-group migration
Today three consumers use `XREADGROUP` against named consumer groups, not
bare `XREAD`:
- meta-sort: group `meta-sort-processor`
(`packages/meta-sort/.../events/FileEventConsumer.ts:23,66-74`)
- meta-dup: group on `file:events`
(`packages/meta-dup/.../MetaEventConsumer.ts:102-104`)
- meta-fuse: dual path — `XREADGROUP` on one consumer, plain `XREAD` on
another (`packages/meta-fuse/.../kv/RedisClient.ts:826,907`)
`XREADGROUP` keeps server-side state per group: a Pending Entries List
(PEL) and a `last-delivered-id` cursor. The load-bearing use today is
meta-sort's `processPendingEntries(idleMs=30000)`, which exists to claim
work from a crashed consumer's PEL on restart.
SSE has no equivalent server-side state. After the migration the contract
shifts to: **each consumer persists its own `Last-Event-ID` and resumes
from it on reconnect.** Implications:
- PEL-based crash recovery is gone. A crashed consumer that hadn't
persisted its cursor since entry X resumes from X (its last *durably
persisted* cursor), not from "whatever was in flight." Acceptable for
every current consumer — none run multiple instances per group — but
call out in the PR C migration of meta-sort that the recovery primitive
is changing.
- Each consumer must own its cursor persistence. Cheapest place is the
consumer's own state (a file, a Redis key written under its own
prefix, an in-memory variable if the consumer is OK with at-most-once
on its own crash).
- No server-side "ACK" call. The cursor *is* the ack — when the consumer
successfully writes Last-Event-ID = N to its own state, entries ≤ N
are considered handled. The SSE handler does nothing.
### Endpoint spec
```
GET /api/events/files
Request headers:
Last-Event-ID: <stream-entry-id> (optional; resume from this cursor)
Accept: text/event-stream
Response:
Content-Type: text/event-stream
Cache-Control: no-cache
Connection: keep-alive
X-Accel-Buffering: no (disable nginx response buffering)
```
Same shape for `/api/events/meta`.
### Event format
One SSE event per Redis Stream entry. Format:
```
id: 1747999999999-0
event: add
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1747999999999}
id: 1748000000123-0
event: change
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1748000000123}
```
- `id` — **opaque** cursor. Today it's the Redis Stream entry ID
(`<ms>-<seq>`); consumers MUST treat it as a black-box string they
echo back via `Last-Event-ID`. This contract lets meta-core swap the
backing store later without breaking every consumer's cursor handling.
- `event` — the `type` field from the underlying entry (`add` | `change`
| `delete` | `rename` | `reset` for `file:events`; `set` | `del` |
`expire` for `meta:events`).
- `data` — JSON-encoded rest of the entry. Keys match the existing stream
payload exactly so downstream consumers don't change their parser.
### Backing-store retention and lifecycle
**Contract: the SSE wire is a faithful mirror of the underlying Redis
Streams.** Consumers rely on `Last-Event-ID` resumption working across
*every* kind of meta-core hiccup, including a restart — that's the entire
point of having a cursor protocol. The backing store must therefore
preserve entries across meta-core process restart and bound its growth
explicitly, not implicitly.
Today's behaviour is wrong on both counts:
| Stream | Restart behaviour | Retention |
|---|---|---|
| `meta:events` | **Cleared on every MetaPublisher.Start** (`events/meta_publisher.go:50-58`) | No `MAXLEN` (unbounded) |
| `file:events` | Persists across restart | No `MAXLEN` (unbounded), cleared only on deliberate reset (`watcher/dispatcher.go:82-95`) |
The `meta:events` clear-on-restart is the load-bearing problem. With it
in place, every meta-core restart silently invalidates every consumer's
persisted cursor — the SSE `gap` event would fire on the first reconnect
after every restart, and the consumer would have to re-bootstrap. That
turns "restart meta-core to apply a config change" into "every downstream
service does a full re-sync," which is exactly the regression SSE is
supposed to prevent.
PR A must fix both streams:
1. **Remove the unconditional `Del(MetaEventsStream)` in
`MetaPublisher.Start`.** Entries persist across restart. The republish
path (`RepublishAllMetadata`) stays as a one-shot bootstrap for
missing entries, but it does not pre-truncate the stream.
2. **Add `MAXLEN ~ N` to every `XAdd` on both streams** so growth is
bounded. Approximate trim (`~`) is cheap. Default `N = 100_000` per
stream is fine for the foreseeable workload (a few KB per entry,
bounded total ≈ a few hundred MB) and easy to tune via config.
3. **Keep the deliberate `EmitReset → ClearStream` path on
`file:events`.** That clear is a feature: a reset is the consumer
contract for "throw out everything you knew, re-bootstrap." Consumers
detect it through the `event: reset` payload that's already emitted
as the first entry after the clear (`dispatcher.go:99-`). The SSE
`gap` event handles the race where a consumer's cursor was trimmed
*between* the clear and its reconnect.
After PR A, the cursor contract is **"durable across meta-core restart
and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
when they persist a Last-Event-ID.
### Heartbeats
Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
when no events are flowing. Keeps reverse proxies and load balancers
from killing the connection as idle. The client ignores comment lines
per the SSE spec.
### Reconnect semantics
Per the SSE spec the client (or `EventSource`) auto-reconnects after a
network error, sending `Last-Event-ID`. The server resumes from the next
entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
**Gap handling.** If the requested ID has been trimmed out of the Redis
Stream by retention, the server emits one synthetic event before resuming
from the oldest available entry:
```
event: gap
data: {"requested":"<id>","resumeFrom":"<oldest-id>","reason":"retention"}
```
Consumers that care about gap-free delivery (currently none — the
schema indexer tolerates gaps; ingest tasks rebuild from scratch) can
react. Consumers that don't care ignore it.
### Error handling
- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
- **Redis fails mid-stream**: close the SSE connection. Client reconnects
with `Last-Event-ID`; the cycle resumes when Redis recovers.
- **Client disconnects**: handler goroutine exits cleanly; no per-client
state on meta-core.
### Auth
`/api/events/*` bypasses the hash-lock perimeter, same pattern as
`/api/meta/{cid}` and `/api/file/{cid}`. Event streams are internal
infrastructure and need to be reachable by peer services without going
through Authelia.
**Bypass ≠ public.** "Hash-lock bypass" means the OIDC sidecar doesn't
gate the route; it does *not* automatically mean Caddy proxies it from
the public hostname. The two existing precedents differ on this:
`/api/file/{cid}` is publicly reachable (peer-to-peer content fetch is
its job), `/api/meta/{cid}` likewise. For the SSE endpoints, the choice
is deliberate:
- If kept inside-only (no Caddy upstream for `/api/events/files` and
`/api/events/meta`), only sibling services on the docker network can
subscribe. This matches the rest of the "Redis is private" model and
is the recommended default.
- If exposed publicly, any browser can open an `EventSource` against
the stream without authentication — i.e. anyone can watch every file
event in the library. Only do this if there's a concrete consumer
that needs it and the visibility is acceptable.
PR A's Caddy change (see PR A scope, item 4) makes this an explicit
choice rather than a default behaviour. The recommendation is
inside-only until a concrete external consumer appears.
### Implementation footprint
The SSE handler itself is ~80 lines of Go in meta-core: one handler per
stream, each running an `XREAD … BLOCK 30s` loop and writing
SSE-formatted lines to the response. Go's `net/http` natively supports
streaming responses with `http.Flusher.Flush()`. Standard SSE clients
exist for every language we use: Node `eventsource`, Rust
`eventsource-client` (or `reqwest` + `tokio_util::codec`), Python
`sseclient`, Go `r3labs/sse`.
PR A as a whole is larger than the handler — see the PR A scope below
for the full list (backing-store fixes, Caddy wiring, poll deprecation)
that has to land alongside the handler for the cursor contract to hold.
## Eventing — considered alternatives
Recorded for posterity. None of these are being implemented; if any are
ever revisited it should be a deliberate reopen, not a drift.
- **Long-polling.** Strictly worse than SSE — connection churn on every
event, latency floor at the poll interval, no built-in resumption.
Rejected.
- **Webhooks (meta-core POSTs to subscribers).** Symmetric coupling:
every consumer would need to run an HTTP server reachable from
meta-core. Out-of-order delivery, retry, dead-letter all need
reinventing. Right for cross-organization integrations, wrong for
in-cluster eventing. Rejected.
- **External broker (Kafka / NATS / RabbitMQ).** Industrial-grade
pub/sub. New dependency, new operational surface, overkill for the
scale (single-host dev, ≤20-peer POC). Revisit if MetaMesh ever needs
cross-WAN federation at high throughput. Rejected for now.
- **Two Redises (writes Redis private, events Redis exposed).** Strongest
separation. Doubles operational complexity for no benefit over the
SSE design — once events go through HTTP, the events-Redis isn't
exposed either. Rejected.
- **Redis ACLs (Option B from earlier brainstorm).** Allows direct Redis
access from clients while restricting commands by user. Adds
credential plumbing and ACL config to operate; lets a buggy service
still connect, just with limited verbs. The SSE design closes the
network entirely, which is strictly stronger. Rejected — but the
equivalent escape hatch (open Redis to other services again) stays
available if we ever need it.
## Industry context — where this lands
The pattern we're adopting is the **Kubernetes / etcd** pattern, and
that's the most informative comparison.
Kubernetes is structurally identical to MetaMesh:
- A single source-of-truth store (etcd) holds all cluster state.
- A single service (kube-apiserver) is the sole writer to etcd.
- Many polyglot clients (kubectl, controllers, schedulers, kubelets,
operators) read and react to changes.
- Clients subscribe to *watch streams* for events.
What Kubernetes does, point-for-point:
- **etcd is never reached by clients.** It runs on a separate port,
often a separate network. kube-apiserver is the only thing that holds
etcd credentials. The lock is by network topology, not authentication.
- **Watch is also mediated.** kube-apiserver exposes `/watch` as an HTTP
streaming endpoint (very similar to SSE in shape). Clients don't watch
etcd directly — even though etcd has its own native watch primitive.
- **Storage portability falls out for free.** etcd could be swapped for
another store without any client change. We don't plan to swap Redis,
but the optionality is real and free.
Other systems with the same shape:
- **HashiCorp Consul.** Clients talk to a local agent over HTTP. The
agent talks to the cluster. KV writes and watches both flow through
the agent. Direct raft-store access is impossible.
- **Apache ZooKeeper.** Clients use the ZooKeeper client library which
speaks the ZooKeeper wire protocol; the data store is private.
- **Strapi / Sanity / Contentful** (headless CMS as analog). All
consumers — frontends, backend services, build pipelines — go through
the CMS HTTP API. Direct Postgres or content-store access is impossible
by design.
Where MetaMesh sat *before*: closer to the bad-monolith pattern
(everyone connects directly to the shared Redis). Where it sits *after*
this design: aligned with the canonical microservices pattern, and
specifically with the Kubernetes flavour where streams are mediated too.
### Advantages we're picking up
- **Schema evolution.** One service can change its data layout without
coordinating releases across every other service. The UUID-root
migration is the cautionary tale of what happens without this.
- **Validation choke point.** Every write passes through one code path
that can enforce invariants (reverse index, `cids/` key-set, type
constraints, schema version).
- **Observability.** Audit logs, request metrics, slow-query detection
all live at one layer.
- **Threat-model fit.** Untrusted-service-in-the-network is now in
scope; network isolation makes the concern moot.
- **Storage portability.** Optionality only, but real.
### Drawbacks (acknowledged)
- **Latency.** One extra hop per write (TCP + HTTP parse + handler).
Local-host overhead is sub-millisecond; mitigations are batched writes
(PATCH the whole metadata blob per plugin completion, not per field)
and Keep-Alive / HTTP/2 connections.
- **SSE-handler plumbing.** ~80 lines per stream endpoint plus per-client
goroutines. Trivial at our scale, real at Kubernetes scale (which is
why they invested in chunked watch with bookmarks, resource versions,
etc.). We can defer the same optimisations forever.
- **API surface size.** Every operation needs a corresponding endpoint.
Most of it already exists; one new SSE endpoint per stream is the
only new surface area introduced here.
## Enforcement — what "lock" actually means
Three layers, in order of effectiveness:
1. **Network isolation (hard).** Redis is on a container-private port.
No host port mapping. No docker-network alias. `kv-leader.info` does
not publish a Redis URL. Any service that wants to talk to Redis
needs container privileges, and at that point the lock isn't the
concern — operating-system isolation is. This is the actual lock.
2. **Documentation (convention).** This file, plus a one-line note in
`CLAUDE.md` and in each consumer service's `CLAUDE.md`: "All metadata
I/O routes through meta-core's HTTP API. Redis is private to
metacore-app." Engineers reading the codebase find the rule on first
contact. This catches the case of a future meta-core PR that
accidentally exposes Redis again.
3. **Code review (convention).** PRs that re-expose Redis (a port
mapping, an alias, a credential leak via leader-info) get rejected
with a pointer to this doc.
No runtime canary or lint is proposed — the network isolation makes
violations physically impossible from outside, and inside meta-core the
small file count makes review sufficient. Layer 1 does the heavy
lifting.
## Migration plan
Four PRs, additive, ordered so the system stays working throughout.
Closing the network access (PR D) is the moment the lock becomes a
lock; everything before it is opt-in migration that consumers can adopt
at their own pace.
**All four PRs have landed.** What follows is the original plan as
written; see "Implementation outcome" at the bottom for the delta
between plan and ship.
### PR A — SSE endpoints in meta-core
Adds `GET /api/events/files` and `GET /api/events/meta`. Implements the
`Last-Event-ID` semantics, gap handling, heartbeats, and the per-stream
XREAD loop. Redis Streams stay reachable directly; consumers can
migrate one at a time.
Scope, in PR-order:
1. Backing-store fixes (see "Backing-store retention and lifecycle"
above) — remove the `meta:events` clear-on-restart, add `MAXLEN ~ N`
to every `XAdd`. These are prerequisites for the SSE cursor contract
to hold; without them PR A is not deliverable.
2. SSE handlers for the two streams.
3. Deprecate `GET /api/events/poll` (`watcher/handlers.go:29`). It
exists today as a long-poll variant over the same `file:events`
stream and is described in code as "always empty in practice"
(`watcher/handlers.go:55-58`). After SSE lands it's redundant — keep
it responding 200 for one release with a `Deprecation` header, then
remove. The `/api/scan/*` deprecation pattern in the same file is
the template.
4. Caddy perimeter wiring (see "Auth" subsection above) — explicit
allow-list for `/api/events/files` and `/api/events/meta` matching
whatever the perimeter policy decides. Either inside-only via the
docker network, or public via Caddy with deliberate authorisation.
Do not let the routes default-leak past the hash-lock.
Roughly 200 lines of Go + ~50 lines of test (covering reconnect from a
cursor, gap emission when the cursor is trimmed, heartbeat cadence,
durability across a meta-core restart).
### PR B — meta-sort writes via HTTP
Swap `RedisClient` for an `HttpKVClient` implementation of the existing
`IKVClient` interface in `packages/meta-sort-core`. All writes
(`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
calls against the **`/meta/{hash}` family** — the canonical write
surface for service-to-service traffic. Per-plugin write coalescing
uses `PATCH /meta/{hash}` (already calls `MergeMetadataFlat`
server-side with the partial-update semantics we need; see
`handlers.go:533-566`).
Reads from Redis stay direct in this PR (no consumer migration yet);
only writes move. Streams stay on direct XREAD.
**Pre-audit (already done):** meta-sort's production `kvClient.*`
surface is `setMetadataFlat` / `deleteMetadataFlat` / `getMetadataFlat`
/ `getMetadata` / `getAllHashIds` / `health` plus the `file:events`
stream consumer. No orphan keyspaces, no generic `kv.set(arbitrary_key)`
callers — the HTTP migration covers the entire write path with no
side bets.
### PR C — Stream consumers migrate to SSE
meta-sort's event consumer (`FileEventConsumer.ts`), meta-fuse's stream
reader, meta-stremio's event hook, meta-dup's monitor, meta-share's
ingest task — each swaps its `XREADGROUP` / `XREAD` loop for an SSE
client. Per-language standard libraries make this small (one HTTP
request, one event-handler closure).
In the same PR, each consumer's `LeaderClient` (TypeScript x2, Python,
plus any new Rust binding for meta-share if added) drops its
*requirement* on the `redisUrl` field — i.e. it stops opening a Redis
connection. It can still tolerate the field being present (PR D removes
it from the publisher side). This ordering matters: PR D's loud-failure
guard cannot ship until every consumer has been demonstrated to boot
without `redisUrl`.
The schema indexer is internal to meta-core and unaffected.
### PR D — Close Redis network access
PR D must land **after** every consumer in PR C has stopped requiring
`redisUrl` (verify by booting each service against a meta-core build
that already omits the field — staging environment or feature-flagged
release). Until that's true, this PR will deadlock the boot sequence.
Then, in this order:
1. Update consumer-side discovery (`LeaderClient` in each service) to
*log loudly* if it sees a `redisUrl` field — guards against
accidental rollback. (Loud log, not hard failure, while the field
is still being written — flip to hard failure once step 4 lands.)
2. Stop publishing `redisUrl` from `/urls` and from `kv-leader.info`.
3. Remove the `meta-core` docker-network alias on the metacore-app
service (other services no longer need DNS for the Redis port).
4. Remove `6380:6379` host port mapping in `dev/docker-compose.yml`.
5. Flip the LeaderClient guard from "warn" to "raise" — any future
consumer that tries to require the field fails loudly at boot.
After PR D, attempting to connect to Redis from anywhere except
metacore-app produces a connection refused. The lock is real.
### Out of scope for this work
- **Redis ACLs / proxy.** Network isolation makes these unnecessary.
Available as an escape hatch if the threat model ever changes.
- **Replacing Redis Streams under the hood.** SSE is the externalised
contract; the backing store is an implementation detail meta-core can
change later.
- **External brokers.** Out of scope absent a concrete driver
(cross-WAN federation at high throughput).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| Writes from non-meta-core services? | Forbidden by network topology | Redis unreachable; not a convention |
| Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
| Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
| Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
| Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
| Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
| **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
| **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
| **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
## Error envelope
Standard shape for all 4xx / 5xx responses from the metadata API:
```json
{
"error": "alias_collision",
"message": "CID midhash256:bafk… already resolves to UUID 01JKR…",
"retryable": false
}
```
Stable `error` slugs the client should switch on:
| Slug | HTTP status | Retryable | Meaning |
|---|---|---|---|
| `alias_collision` | 409 | no | A different UUID already owns the CID the caller is trying to register. Caller bug; escalate. |
| `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
| `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
| `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
| `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
**Implemented.** Every 4xx/5xx response across `internal/api`,
`internal/mounts`, `internal/watcher`, and `internal/watchers` now
emits the typed envelope. The slug-mapping helper is duplicated across
those packages (one ~15-line `slugForStatus` per package) to avoid an
import cycle on the api package; consolidating it into a shared
`internal/httperr` package is a worthwhile follow-up but not load-bearing.
## Remaining open questions
Mostly answered post-implementation:
- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
editor's React UI continues to call it; the surface itself was untouched
by this work. Worth a follow-up to consolidate the editor onto
`/meta/{hash}` and delete the parallel routes.
- **Initial bootstrap flood on `/api/events/files`.** Resolved in
practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
replay every entry on each restart and the StreamingStateBuilder
handles the resulting burst (~2.5k property events on a 50-file
library) without issue. meta-sort persists its cursor so only sees
the flood once per fresh deploy. meta-stremio filters to "interesting
fields" before triggering cache invalidation, so the burst is mostly
no-ops.
- **meta-dup SSE port.** Still pending — meta-dup is in no active
compose file. When revived, copy meta-fuse's pattern wholesale
(SSEEventClient + MetaCoreApiClient). Tracked separately.
## Non-goals
- **Authentication between internal services.** The hash-lock OIDC
perimeter handles external clients; internal service-to-service
trust is by network isolation (docker network for dev, equivalent
isolation in prod deployments). Adding service-to-service auth is a
separate concern.
- **Replacing the public HTTP API surface.** This proposal exercises
what already exists; the only new endpoints are the two SSE streams.
- **Introducing new transport, broker, proxy, or sidecar.** Everything
builds on what's deployed today.
- **Migrating meta-fuse / meta-stremio / meta-dup direct Redis reads
via XSCAN/GET** (separate from event-stream reads). ~~Was a non-goal~~
ended up being required for PR D's network closure to be safe.
meta-fuse and meta-stremio migrated; meta-dup deferred (not deployed).
See PR C / "Implementation outcome" for details.
## Implementation outcome
What actually shipped vs. what this doc planned.
### Delta against the original plan
- **Read-path migration for meta-fuse and meta-stremio is now in scope
and done.** The original "out of scope" carve-out was wrong: closing
the Redis network port doesn't just break stream consumers, it also
breaks every `kvClient.get` / `redis_storage.get_all_videos` call.
Both services grew their own `MetaCoreApiClient` (TS for meta-fuse,
Python for meta-stremio) and now route reads through `/meta/{hash}`
+ `/meta` + `/api/file/{cid}/info`.
- **meta-stremio's HTTP mode auto-recovers from transient meta-core
outages.** `is_connected` re-probes meta-core's `/health` on every
call (cheap; a stateless GET), so a metacore-app restart no longer
latches the addon into a permanent disconnected state. The
background reconnect thread is only needed for legacy Redis mode
where the socket itself is sticky.
- **Hard `redisUrl` guard is gated by `ALLOW_LEGACY_REDIS_URL=1`.** All
three consumer LeaderClients (TS x2 + Python) throw on seeing the
field by default. Setting the env var downgrades to a warn — escape
hatch for a deliberate temporary rollback to a meta-core that still
publishes the field.
- **Write-path CID resolution** turned out to be a hard prerequisite
for the lockdown to work in practice. Documented separately below.
- **PR D ordering matched the doc** (consumer migration first, then
publisher stops emitting, then network closure). The `meta-core`
docker-network alias and the `6380:6379` host port mapping are both
removed; sibling services reach the HTTP/SSE/WebDAV surface via the
canonical `metacore-app` hostname.
- **meta-dup is the one service that didn't migrate.** It's in no
active compose file. The TS port is straightforward when needed: copy
meta-fuse's pattern (SSEEventClient for streams, MetaCoreApiClient
for reads).
### Write-path CID resolution (`storage.ResolveRoot`)
Not in the original plan but load-bearing.
**The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed
every write through `/meta/{hash}`. But `{hash}` is the midhash256 CID
that meta-sort received via file:events, not the watcher's minted UUID.
With no resolution step in the handler, every write went to
`file:<midhash>/*` — a parallel root the watcher's UUID never saw. The
auto-alias hook on `cid_midhash256=<value>` then OVERWROTE the
watcher's `cid:midhash256:<value>` → UUID alias to point at the
midhash itself, leaving the UUID stranded with only the watcher's
sparse fields (filePath, sizeByte, mtimeNano, duplicates) and the
midhash root carrying the rich plugin output. This is the exact
"dual roots per file" failure the doc's Motivation section listed as
the prior bug — it just moved from "convention violation" to "API
contract violation."
**The fix.** Three layers in `internal/storage/cid_resolution.go`:
1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
read/write handler, look up `cid:<hash>` in the reverse index (bare
CID — no algorithm prefix). If found, use the UUID as the storage root.
Falls through to the bare hash for direct-UUID writes / legacy entries.
2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
register `cid:<v>` → `<v>`. The dual-root bug's chicken-and-
egg is: meta-sort writes first, registers self-pointing alias,
watcher then can't recover. Refusing the self-write keeps the
watcher's legitimate alias intact.
3. **`MigrateDualRoots` sweep** (`POST /api/admin/migrate-dual-roots`).
One-shot fixer for already-stranded entries: walks every UUID's
`midhash256` field to build the orphan index, merges each stranded
`file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
deletes the stranded root, and re-points the alias.
Verified on the dev stack: 48 stranded entries were reunited; the
editor now shows every file with its full plugin metadata under one
UUID; future writes stay unified.
### Typed error envelope
Done across all four handler packages (`api`, `mounts`, `watcher`,
`watchers`). Each has its own `writeError` returning
`{error: <slug>, message, retryable}`. The slug-mapping helper is
duplicated in each to avoid an import cycle on the `api` package.
### What's NOT done
- **meta-dup SSE + read migration.** Not deployed, no test harness.
When revived, copy meta-fuse's pattern.
- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate
editor-internal API surface is untouched; could be retired in a
follow-up by porting the editor UI to `/meta/{hash}`.
- **Consolidated `internal/httperr` package** for the typed envelope
helpers (currently duplicated 4× to avoid the import cycle). Code
hygiene; functionally a no-op.
### Verification done
- All four service dashboards load cleanly with no console errors.
- meta-sort SSE consumer + HTTP write path verified end-to-end by
delete + inject + reprocess of Sintel.
- meta-fuse SSE consumer rebuilds VFS to 48 files on each boot.
- meta-stremio catalog API now surfaces the full library (47 videos,
1 movie + 5 series + 46 episodes — was 0/0/0 before the Python
read-path migration because LeaderStorage broke when it tried Redis).
- `redis-cli` from any container other than `metacore-app` produces a
connection refused; `cid:midhash256:<H>` aliases all point to UUIDs
(no self-pointing); `file:__index__` has only UUID-style hashIds.
- ALLOW_LEGACY_REDIS_URL=1 escape hatch present in all three consumer
LeaderClients (TS x2 + Python).
### Known test-suite gaps (post-implementation bats run)
Running `docker exec meta-test-runner /app/test/test.sh` after the
lockdown shipped reveals a number of failing suites, almost all of
which are pre-existing test/environment drift unrelated to this work:
| Failure | Root cause | Relation to lockdown |
|---|---|---|
| meta-stremio (~35 tests), stremio-tmdb (6 tests) | Tests call `/manifest.json` directly; dev `.env` sets `STREMIO_HASH_API_SEED=dev-stremio-secret` which puts the addon behind a path-token prefix | Unrelated — pre-existing env mismatch |
| meta-fuse FUSE-mount (3 tests) | Tests use `docker exec meta-fuse` but the container is named `metafuse-app` (Yundera prod-parity rename) | Unrelated — predates this work |
| meta-fuse WebDAV PROPFIND (4 tests) | Tests use Caddy URL with WebDAV basic-auth credentials that no longer match the hash-lock perimeter | Unrelated — predates this work |
| meta-sort "API integration tests" (1 test) | Test runs `docker exec meta-sort pnpm test`; container is `metasort-app` | Unrelated — predates this work |
| meta-core "create and delete mount config" (1 test) | Test creates an NFS mount; production code dropped NFS support ("NFS mounts are no longer supported") | Unrelated — predates this work |
| service-discovery (9 tests) | Cascading from meta-stremio breakage (test expects unauth `/api/services` on stremio) | Unrelated to lockdown directly |
| meta-share / share-1c (multiple) | Tests assert `"phase":"1f"` in `/health`; meta-share has moved past that phase | Unrelated — meta-share evolution |
| flat-keys: "filePath field exists" (1 test) | Test SRANDMEMBERs file:__index__; if it lands on a UUID with no filePath (e.g. a manual test write) it fails | Was hit transiently after my interactive testing left stub entries; resolved by cleanup |
No failure traceable to a real regression from this work was found.
The dev-environment test harness needs an unrelated refresh to catch
up with the prod-parity container renames, the NFS removal, the
STREMIO_HASH_API_SEED default, and meta-share's current phase.
File diff suppressed because it is too large Load Diff
+234 -532
View File
@@ -1,595 +1,297 @@
# Redis Metadata Storage Architecture
# Redis Metadata Storage Structure
This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling.
How meta-core lays out file metadata in Redis. This document covers the
on-disk shape (UUID-rooted flat string keys, CID reverse index,
per-root sets) and the schema-version gate. It does **not** enumerate
field semantics — that's the job of the repo-root
[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key
registry for everyone who writes to or reads from metadata.
## Overview
For the design rationale (why UUIDs instead of content hashes, why a
reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
For the external-access contract (HTTP-only, SSE for events), see
[`api-mediated-access.md`](api-mediated-access.md).
MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables:
## Quick summary
- **Redis keyspace notifications** for field-level change detection
- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates)
- **meta:events stream** integration for real-time updates across services
- Individual field updates without affecting other properties
- Fast VFS lookups using content-based hash IDs
- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
not content hashes. Examples in this doc use the placeholder
`01JKR8XW5T4QMV7AHNJ2DEFGHK`.
- **Properties are flat Redis STRING keys**, one per leaf:
`file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
- **CIDs are reverse-index aliases**: `cid:<bareCid>` → `<uuid>`
(a Redis STRING per known CID; no algorithm prefix — the multicodec
already carries it).
- **Per-root sets**: `file:<uuid>/cids/<bareCid>` (a *key-set*: one STRING
`"true"` per known CID, midhash included), `file:<uuid>/duplicates`
(alternate paths with the same content), plus `file:__index__` (every UUID).
There is **no stored `canonical_cid`** — it is derived by rank on read.
- **A schema-version sentinel** (`meta-core:schema-version`) refuses to
boot against legacy data.
## Key Structure
Implementation lives in `internal/storage/client.go`,
`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
`internal/storage/schema_version.go`.
### Flat Key Storage
## Key structure
Each file property is stored as a separate Redis STRING key:
### Roots
```
Key Pattern: file:{hashId}/{property/path}
Key Type: STRING
Value: String value
file:<uuid>/<property/path> → Redis STRING (always)
```
**Example Redis commands:**
```bash
# Store file metadata (each property is a separate key)
SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..."
SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv"
SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5"
SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm"
UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
in a given window, which restored some of the debug-ability lost when
midhash256 stopped being the root.
# Retrieve single field
GET "file:bafkr4ih5kapbjzq.../fileinfo/duration"
# Retrieve all metadata for a file (scan by prefix)
SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000
```
### Why Flat Keys?
The flat key structure (vs. Redis Hashes) enables:
1. **Keyspace Notifications**: Redis publishes events when individual keys change
2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*`
3. **Selective Updates**: Update one field without touching others
4. **Stream Integration**: The `meta:events` stream captures field-level changes
### File Index
A Redis Set tracks all known file hash IDs:
Properties use `/` as the path separator. Nested objects in source code
are flattened to flat keys on write and reconstructed on read. Examples:
```
Key: file:__index__
Type: Set
Value: {hashId1, hashId2, hashId3, ...}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..."
```
This enables fast bulk queries like "get all files" without scanning keys.
### Reverse-index aliases (CIDs)
Every CID known for a file is registered as its own Redis STRING, keyed by
the **bare CID** — no algorithm prefix. A CIDv1 already encodes its hash
function in the multicodec, so naming it again in the key would be
redundant (and would let the same digest be indexed under two spellings):
```
cid:<bareCid> → <uuid>
```
Examples:
```
cid:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:bagacbabaec7v3... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
registered CID, independent of which hash algorithm — there is no primary.
The auto-alias hook (`maybeAddAliasFromFieldLocked` in
`internal/storage/cid_resolution.go`, called from `SetProperty`,
`SetMetadataFlat` and `MergeMetadataFlat`) registers the alias for any
field whose name starts with `cids/`, so writers emit key-set members
without knowing the reverse index exists.
> ⚠ The hook matches **only** the `cids/` prefix. A legacy `cid_*` or
> `midhash256` *named* field written to a root is stored verbatim and is
> **never indexed** — the record becomes unresolvable by that CID. Writers
> must collapse such fields to `cids/<cid>` members before the write. See
> `METADATA_KEYS.md` §14.13.
### Per-root sets
```
file:<uuid>/cids/<bareCid> → STRING "true" — one key per known CID (a key-set, not a Redis SET)
file:<uuid>/duplicates → Redis SET of alternate paths with same content
```
`cids` is a **key-set**: one flat `file:<uuid>/cids/<cid>` STRING per member,
which keeps it conflict-free to merge across peers (a SET would not be).
The record's own address (the midhash) is just the `0x1000` member — it has
no separate field.
There is **no stored `canonical_cid`**. The canonical CID is *derived on
read*: `GetMetadataDocument` (`internal/storage/cid_resolution.go`) ranks the
key-set's members through `internal/cid/rank.go` (`cid.Better`) and returns
the winner as the `canonical_cid` field of the JSON response only. Nothing
reconciles or persists it — an earlier `reconcileCanonicalCIDLocked` did, and
was removed with the flat `cid_*` keys.
### Global index
```
file:__index__ → Redis SET of all UUIDs
```
`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
every file without scanning the keyspace.
```bash
# Get all hash IDs
SMEMBERS "file:__index__"
# Count total files
SCARD "file:__index__"
# Check if a file exists
SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..."
SMEMBERS file:__index__ # all UUIDs
SCARD file:__index__ # total file count
SISMEMBER file:__index__ <uuid>
```
### Keyspace Notifications
Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration:
```bash
# Required Redis config (set by meta-core)
CONFIG SET notify-keyspace-events "K$"
# K = Keyspace events (published on __keyspace@<db>__:<key>)
# $ = String commands (SET, etc.)
```
Services can subscribe to keyspace events for real-time updates:
```bash
# Subscribe to all file key changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Subscribe to specific field pattern (e.g., TMDB updates)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### meta:events Stream
The `meta:events` stream provides reliable event delivery for file system events from meta-core:
### Schema-version sentinel
```
Stream Key: meta:events
Entry Fields:
- type: add | change | delete | rename | reset
- path: File path relative to /files
- midhash256: Content hash (for add/change events)
- oldPath: Previous path (for rename events)
- timestamp: Event timestamp
meta-core:schema-version → STRING (currently "2")
```
Consumer groups enable reliable processing with acknowledgment:
Set at first boot, checked on every subsequent boot
(`internal/storage/schema_version.go`). If the sentinel is missing but
`file:__index__` is non-empty, the gate refuses to boot (legacy data).
If the sentinel is present but doesn't match `SchemaVersion`, the gate
also refuses to boot. Alpha-stage policy: operators wipe and restart;
there is no automated migration.
```bash
# Create consumer group
XGROUP CREATE meta:events mygroup 0 MKSTREAM
## Why flat STRING keys?
# Read new events (blocking)
XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events >
The store does not use Redis Hashes. The reasons are:
# Acknowledge processed events
XACK meta:events mygroup <entry-id>
```
1. **Field-level keyspace notifications.** Redis publishes
`__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
you'd only get one notification on the hash itself. meta-core's
`MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
notifications and publishes structured events to the `meta:events`
Redis Stream.
2. **Selective updates without read-modify-write.** A plugin writing
`tmdb/poster` doesn't have to read the rest of the document.
3. **Compatibility with the CID alias hooks.** Field writes can trigger
alias-maintenance work in `SetProperty` without the plugin caring.
## Primary Hash ID
## Reads and writes
The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from:
The Go API in `internal/storage/client.go`:
1. SHA-256 hash of the middle 1MB of the file
2. File size in bytes
3. IPFS CID v1 encoding (multicodec 0x1000)
| Method | Behaviour |
|---|---|
| `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register an alias for every `cids/<cid>` member. |
| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
| `AddAlias(uuid, "<bareCid>")` | Set `cid:<bareCid>` → `<uuid>` and the key-set member `file:<uuid>/cids/<bareCid>` = `"true"`. Nothing to reconcile — canonical is derived on read. |
| `ResolveRoot("<bareCid>")` | `GET cid:<bareCid>` → uuid (returns the input unchanged if it is already a UUID root). |
| `GetMetadataDocument(uuid)` | Flat fields + `cids` hoisted to an array + `canonical_cid` **derived** by `cid.Better` rank + `duplicates`. Backs `GET /api/meta/{cid}`. |
**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...`
## Reconstruction (flat → nested)
This provides:
- Content-based addressing (same file = same ID)
- Fast computation (only reads 1MB)
- IPFS compatibility for future distributed storage
### Collision Handling
If two different files produce the same midhash (rare), the system falls back to full SHA-256:
```typescript
// Normal case
hashId = midHash256; // "bafkr4ih5kapbjzq..."
// Collision detected (different filePath with same midhash)
hashId = metadata['cid_sha2-256']; // Full hash fallback
```
## Metadata Flattening
Nested objects are flattened into path-based keys for efficient Redis storage.
### Flattening Rules
```typescript
// Input (nested object)
{
title: "Inception",
video: {
codec: "h265",
width: "1920"
},
streams: [
{ language: "eng" },
{ language: "jpn" }
]
}
// Output (flattened for Redis)
title = "Inception"
video/codec = "h265"
video/width = "1920"
streams/0/language = "eng"
streams/1/language = "jpn"
```
**Key rules:**
- Nested objects use `/` as path separator
- Arrays use numeric indices (0, 1, 2...)
- All values stored as strings
- `null`/`undefined` stored as empty strings
- Numbers and booleans converted via `String()`
### Reconstruction
When reading, the flattened structure is reconstructed into nested objects:
```typescript
const metadata = await kvClient.getMetadataFlat(hashId);
// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... }
```
## Plugin Metadata Structure
Each plugin writes metadata using a prefixed key pattern:
When the HTTP API returns a document, it reconstructs the flattened
storage back into a nested JSON object:
```
{plugin-id}/{field} # Simple field
{plugin-id}/{group}/{field} # Grouped field
{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays)
flat nested
───────────────────────────────────── ───────────────────────────
title = "Inception" title: "Inception"
fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5",
fileinfo/streamdetails/ streamdetails: { video: { 0: {
video/0/codec = "h264" codec: "h264", ... } }, ... } }
streams/0/language = "eng" streams: [ { language: "eng" }, ... ]
```
### Plugin Contribution Flow
Rules:
- `/` is the path separator.
- Numeric path segments (`0`, `1`, ...) become array indices on read.
- All values are strings on the wire; type interpretation is the
caller's job (and is documented per-field in `METADATA_KEYS.md`).
- `null`/`undefined` round-trip as empty strings.
## Plugin contribution flow
Plugins don't talk to Redis. They write metadata via meta-core's HTTP
API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
writes under a prefix:
```
File Discovered
↓
[FileProcessorPiscina]
├─ Compute midhash256
├─ Store preliminary metadata
└─ Dispatch plugin tasks
↓
[Plugin Processing]
├─ file-info → fileType, mimeType, sizeByte
├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/...
├─ filename-parser → titles/eng, season, episode, videoType
├─ anime-detector → anime, titles/jpn, titles/jpl
├─ tmdb → tmdbid, plot/eng, genres
└─ full-hash → cid_sha2-256, cid_sha1
↓
[Plugin Callbacks]
└─ Store metadata via flat keys (SET per property)
{plugin-id}/<field> single value
{plugin-id}/<group>/<field> grouped
{plugin-id}/<group>/<n>/<field> indexed (array)
```
## Core Metadata Types
See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full,
authoritative list of fields each plugin writes (content identification,
file stats, video classification, anime detection, subtitle references,
Jellyfin metadata, FFmpeg stream details, etc.) and the value formats
(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc
deliberately does not duplicate that registry — it would drift.
### HashMeta - Content Identification
## Event streams
```
cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132)
cid_md5 = "bafkr..." # MD5 (multicodec 0xd5)
cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11)
cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12)
cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000)
```
Two Redis Streams are produced inside meta-core:
### FileStatMeta - File Properties
| Stream | Producer | Contents |
|---|---|---|
| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
```
fileType = "video" # video, audio, image, text
mimeType = "video/x-matroska" # MIME type
sizeByte = "4294967296" # File size in bytes
```
External consumers do **not** read these streams directly. They subscribe
via SSE:
### VideoMeta - Classification
- `GET /api/events/files` — mirror of `file:events`
- `GET /api/events/meta` — mirror of `meta:events`
```
videoType = "movie" # movie, episode, unknown
originalTitle = "Inception"
titles/eng = "Inception" # Localized titles
titles/jpn = "インセプション"
season = "1"
episode = "5"
movieYear = "2010"
```
The SSE wire contract uses `Last-Event-ID` for resume (the opaque
`<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
a client's cursor, the handler emits one `event: gap` payload before
resuming. See `internal/api/sse_events.go` and
[`api-mediated-access.md`](api-mediated-access.md) for the full
contract.
### AnimeMeta - Anime Detection
```
anime = "true" # "true" or "false"
titles/jpn = "進撃の巨人" # Japanese title
titles/jpl = "Shingeki no Kyojin" # Romaji
```
### SubtitleMeta - Subtitle References
```
subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID
subtitles/eng/1 = "bafks..."
subtitles/jpn/0 = "bafkt..."
```
### JellyfinMeta - Comprehensive Metadata
```
imdbid = "tt1375666"
tmdbid = "27205"
anidbid = "12345"
plot/eng = "A movie about..."
genre/0 = "Action"
genre/1 = "Sci-Fi"
rating = "8.8"
mpaa = "PG-13"
releasedate = "2010-07-16"
art/poster = "bafkr..." # Reference to poster image CID
art/fanart = "bafks..." # Reference to backdrop image CID
```
## FFmpeg Plugin - Special Case
The FFmpeg plugin extracts detailed stream information with a specific nested structure.
### Stream Details Structure
```
fileinfo/duration = "7200.5"
fileinfo/formatName = "matroska,webm"
fileinfo/streamdetails/video/0/codec = "h264"
fileinfo/streamdetails/video/0/width = "1920"
fileinfo/streamdetails/video/0/height = "1080"
fileinfo/streamdetails/video/0/aspect = "16:9"
fileinfo/streamdetails/video/0/bitrate = "8000000"
fileinfo/streamdetails/audio/0/codec = "aac"
fileinfo/streamdetails/audio/0/language = "eng"
fileinfo/streamdetails/audio/0/channels = "6"
fileinfo/streamdetails/audio/1/codec = "dts"
fileinfo/streamdetails/audio/1/language = "fra"
fileinfo/streamdetails/audio/1/channels = "6"
fileinfo/streamdetails/subtitle/0/language = "eng"
fileinfo/streamdetails/subtitle/0/format = "subrip"
fileinfo/streamdetails/subtitle/1/language = "jpn"
fileinfo/streamdetails/subtitle/1/format = "ass"
fileinfo/streamdetails/embeddedimage/0/type = "cover"
```
### Stream Type Keys
| Stream Type | Key Pattern | Description |
|------------|-------------|-------------|
| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) |
| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks |
| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles |
| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails |
### Common Stream Fields
**Video streams:**
- `codec` - h264, hevc, av1, vp9, etc.
- `width`, `height` - Resolution
- `aspect` - Aspect ratio
- `bitrate` - Video bitrate
- `profile` - Codec profile (main, high, etc.)
- `level` - Codec level
**Audio streams:**
- `codec` - aac, dts, ac3, flac, etc.
- `language` - ISO 639-2 code (eng, jpn, fra)
- `channels` - Channel count (2, 6, 8)
- `bitrate` - Audio bitrate
- `samplerate` - Sample rate
**Subtitle streams:**
- `language` - ISO 639-2 code
- `format` - subrip, ass, pgs, vobsub
## Real-Time Notifications
MetaMesh provides multiple mechanisms for real-time metadata change notifications:
### 1. meta:events Stream (Recommended)
The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups:
```bash
# File system events from meta-core watcher
XREAD STREAMS meta:events 0
# Event types: add, change, delete, rename, reset
```
Benefits:
- **Reliable delivery** via consumer groups and acknowledgment
- **Replay capability** for catching up after restarts
- **Ordered processing** with stream IDs
### 2. Keyspace Notifications
For field-level change detection, subscribe to Redis keyspace notifications:
```bash
# All file metadata changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Specific plugin updates (e.g., TMDB enrichment)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### Cache Invalidation
The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation:
```
┌─────────────────────────────────────────────────────────────────┐
│ Cache Invalidation Flow │
│ │
│ Redis Keyspace Cache │
│ ┌──────────────┐ Invalidator │
│ │ SET file:*/ │ ──────────► ┌──────────────┐ │
│ │ DEL file:*/ │ subscribe │ Listens to │ │
│ └──────────────┘ │ __keyspace@ │ │
│ │ 0__:file:* │ │
│ └──────┬───────┘ │
│ │ │
│ ▼ │
│ ┌──────────────┐ │
│ │ Remove from │ │
│ │ LRU cache │ │
│ └──────────────┘ │
└─────────────────────────────────────────────────────────────────┘
```
Flow:
1. Cache invalidator subscribes to `__keyspace@0__:file:*`
2. When metadata changes (SET/DEL), invalidator receives notification
3. Invalidator removes the corresponding file from the WebDAV cache
4. Next access fetches fresh copy from source filesystem
This ensures cached files stay synchronized with metadata changes.
### 3. Pub/Sub Channels (Legacy)
Batch notification channels are still supported:
```
meta-sort:file:batch # Batch file updates (every 5 seconds)
meta-sort:scan:reset # Full rescan notification
```
```json
{
"timestamp": 1704067200000,
"changes": [
{ "action": "add", "hashId": "bafkr4ih5..." },
{ "action": "update", "hashId": "bafkr4ih5..." },
{ "action": "remove", "hashId": "bafkr4ih5..." }
]
}
```
## Complete Example
Here's a complete metadata example for a movie file:
```bash
# Query all keys for a file
redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*'
# Example flat keys stored in Redis:
# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..."
# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv"
# file:bafkr4ih5kapbjzq.../fileType = "video"
# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5"
# file:bafkr4ih5kapbjzq.../titles/eng = "Inception"
# ...
# When reconstructed by the API (nested object representation)
{
# Content identification
"cid_midhash256": "bafkr4ih5kapbjzq...",
"cid_sha2-256": "bafks7ij2kbpck...",
"filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv",
# File properties
"fileType": "video",
"mimeType": "video/x-matroska",
"sizeByte": "4294967296",
# Classification
"videoType": "movie",
"originalTitle": "Inception",
"movieYear": "2010",
# Localized titles
"titles": {
"eng": "Inception",
"fra": "Inception",
"jpn": "インセプション"
},
# FFmpeg stream details
"fileinfo": {
"duration": "8878.5",
"formatName": "matroska,webm",
"streamdetails": {
"video": {
"0": {
"codec": "h264",
"width": "1920",
"height": "1080",
"aspect": "16:9",
"bitrate": "8000000"
}
},
"audio": {
"0": { "codec": "dts", "language": "eng", "channels": "6" },
"1": { "codec": "ac3", "language": "fra", "channels": "6" }
},
"subtitle": {
"0": { "language": "eng", "format": "subrip" },
"1": { "language": "fra", "format": "subrip" }
}
}
},
# External metadata (TMDB)
"tmdbid": "27205",
"imdbid": "tt1375666",
"plot": {
"eng": "A thief who steals corporate secrets through dream-sharing technology..."
},
"genre": ["Action", "Science Fiction", "Adventure"],
"rating": "8.4",
"releasedate": "2010-07-16",
# Media assets
"art": {
"poster": "bafkr4poster...",
"fanart": "bafkr4fanart..."
},
# Detection flags
"anime": "false"
}
```
## Key Implementation Files
| Component | File Path |
|-----------|-----------|
| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` |
| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` |
| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` |
| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` |
| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` |
| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` |
| meta-core Storage | `packages/meta-core/internal/storage/client.go` |
| meta-core Events | `packages/meta-core/internal/events/publisher.go` |
| meta-core Cache | `packages/meta-core/internal/cache/` |
Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
inside `MetaPublisher` to drain keyspace notifications, but this is not
the external surface — external services use SSE.
## Debugging
```bash
# List all file hash IDs
docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__'
# Roots
docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
docker exec metacore-app redis-cli SCARD 'file:__index__'
docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
# Count total files
docker exec meta-core-dev redis-cli SCARD 'file:__index__'
# Full document for one root (flat scan)
docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
# Get a sample hash ID
docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__'
# One property
docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
# List all keys for a specific file (flat key scan)
docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*'
# Confirm keys are STRING (not HASH)
docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
# string
# Get specific field (flat key)
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration'
# Reverse-index lookup (bare CID — no algorithm prefix)
docker exec metacore-app redis-cli GET 'cid:bafkreih...' # → uuid
# Get filePath for a file
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath'
# The cids key-set (one STRING per member, NOT a Redis SET — SMEMBERS won't work)
docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/*'
# Verify keys are STRING type (not HASH)
docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath'
# Should return: string
# canonical_cid is NOT stored. Derive it via the API instead:
curl -k https://metacore-dev.localhost:8083/api/meta/<cid> | jq '.canonical_cid, .cids'
# Monitor metadata changes via stream
docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0
# Schema-version sentinel
docker exec metacore-app redis-cli GET 'meta-core:schema-version'
# Monitor keyspace notifications (requires notify-keyspace-events enabled)
docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*'
# Streams (server-side; external consumers should use SSE)
docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
# meta-core API debugging
# Cache status
curl http://localhost:9000/api/cache/status
# Clear cache
curl -X POST http://localhost:9000/api/cache/clear
# Watch events stream (SSE)
curl http://localhost:9000/api/events/stream
# KV browser - storage info
curl http://localhost:9000/api/kv/info
# Scan status
curl http://localhost:9000/api/scan/status
# HTTP equivalents — preferred
curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>' # CID → document
curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs
curl -k 'https://metacore-dev.localhost:8083/meta/<uuid>' # document by UUID
curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE
curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats
```
## meta:events Stream
## Key implementation files
The `meta:events` stream captures metadata change events for real-time notifications:
```bash
# View recent events
docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10
# Read events from a consumer group
docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' >
```
Stream events contain:
- `type`: Event type (add, change, delete, rename, reset)
- `path`: File path
- `midhash256`: Content hash (for add/change)
- `timestamp`: Event timestamp
| Component | Path |
|---|---|
| Redis client + flat-key ops | `internal/storage/client.go` |
| UUIDv7 minting | `internal/storage/uuid.go` |
| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
| Schema-version gate | `internal/storage/schema_version.go` |
| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
| SSE event mirror | `internal/api/sse_events.go` |
| Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
+362
View File
@@ -0,0 +1,362 @@
# UUID-Rooted Metadata with CID Aliasing
## Status
**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
> ⚠ **Two parts of this proposal were superseded before shipping.** This doc
> is the design record; it is *not* the schema reference. For what is actually
> stored, see `METADATA_KEYS.md` §2/§14.13 and
> [metadata-storage-structure.md](./metadata-storage-structure.md).
>
> 1. **`canonical_cid` is not a stored key.** The proposed reconciler
> (`reconcileCanonicalCIDLocked`) was never kept — a stored scalar isn't
> conflict-free to merge across peers. The canonical CID is **derived on
> read** by ranking the `cids` members (`internal/cid/rank.go` →
> `cid.Better`), and appears only in the `GET /api/meta/{cid}` response.
> 2. **`cids` is a key-set, not a Redis SET, and holds bare CIDs, not
> `<algo>:<value>` tokens.** One flat STRING per member:
> `file:<uuid>/cids/<bareCid>` = `"true"`. The reverse index is likewise
> `cid:<bareCid>` → uuid. A CIDv1's multicodec already names its hash
> function, so the `<algo>:` prefix was redundant.
>
> Text below that references either shape describes the *proposal*, not the
> code. The rank ordering and everything else did ship as written.
## Motivation
Today, every metadata entry in Redis is keyed by a content hash, specifically
midhash256:
```
file:midhash256:abc.../filePath
file:midhash256:abc.../title
file:midhash256:abc.../fileinfo/duration
file:midhash256:abc.../tmdb/poster
...
```
Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields
*inside* that entry:
```
file:midhash256:abc.../cid_sha256 = sha256:def...
file:midhash256:abc.../cid_ipfs = bafy...
```
Two problems with this:
1. **midhash256 is privileged by accident.** It IS the key, while every other
CID is a secondary field. There's nothing intrinsically special about
midhash256 — it's just the first hash computed. Hash-algorithm changes, or
new digest types added by plugins, force rewrites of every root. The
"promotion" workarounds people reach for (re-keying entries when a stronger
hash arrives) solve the wrong problem; the privilege itself is the issue.
2. **CID lookup is O(n).** Resolving "given any CID, return that file's
metadata" currently scans every indexed entry (see
`storage/client.go::LookupPathByCID`, which acknowledges this in a
comment). Acceptable for admin tooling, unacceptable for a public API
surface — especially when peers on meta-share may query by any CID variant.
The product requirement is clear: **any CID a client knows about must resolve
to that file's metadata in O(1).** IPFS-compatible CIDs are the externally
meaningful identity for content retrieval; the internal storage layout should
not bias toward any one digest type.
## Goal
A single externally-meaningful contract:
```
GET /api/meta/<cid> → full metadata document for that file
```
Where `<cid>` may be any supported type (midhash256, sha256, IPFS, btih,
sha3, future additions). All CID types are treated symmetrically. No CID type
is "the key."
## Design
### Two-tier key structure
The root key becomes an opaque internal UUID; CIDs are aliases that point to
it.
```
file:<uuid>/<property...> ← canonical storage (UUIDv7, never disclosed externally)
cid:<cid> → <uuid> ← reverse index (one Redis key per known CID)
```
The metadata keyspace under `file:<uuid>/` remains open and flat, exactly as
today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
`file:<uuid>/fileinfo/duration`. Only the root identifier changes.
**Example layout:**
```
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = /watch/Inception.mkv
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size = 4500000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime = 1729872000000000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bagacbabaec… = "true" ← midhash (record address)
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids/bafybeigd… = "true" ← sha2-256
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates = SET{/watch/dup1.mkv, /watch/dup2.mkv}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = bafy...
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng = ...
cid:bagacbabaec… → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:bafybeigd… → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
*(As shipped — the proposal originally wrote `cids` as a Redis SET of
`<algo>:<value>` tokens plus a `canonical_cid` scalar. See the Status note.)*
### UUID format: UUIDv7
UUIDv7 chosen over UUIDv4 or 64-bit random:
- **128 bits**: collision-free under distributed minting; no central allocator.
- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a
given window. Restores some of the debuggability lost by going opaque.
- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN
locality.
Encoded as a 26-character Crockford Base32 string (ULID-compatible) for
compactness and case-insensitive matching.
### New root fields owned by meta-core
The metadata schema is open — any writer can put any key under
`file:<uuid>/`. This proposal does not change that. It adds three keys whose
sole writer is meta-core itself:
| Key | Type (as shipped) | Purpose |
|-----|------|---------|
| `cids/<bareCid>` | STRING `"true"` — a *key-set*, one key per member | All CIDs known for this file, midhash included. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. **Proposed as a Redis SET of `<algo>:<value>` tokens; shipped as a key-set of bare CIDs** — a key-set merges conflict-free across peers and the multicodec already names the algorithm. |
| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
| ~~`canonical_cid`~~ | *(not stored)* | **Dropped.** Was to hold the externally-presented CID. Now derived on read by ranking the `cids` members; surfaced only on `GET /api/meta/{cid}`. |
Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
`subtitle/<lang>`, …) are unaffected.
~~Redis SET is a departure from the all-strings convention in the current
storage doc, but `file:__index__` is already a SET — so the precedent exists
for using sets where they're the right structure.~~ **Reversed for `cids`.**
The all-strings convention won: `cids` ships as flat `cids/<cid>` STRING keys
so that two peers merging the same record can never disagree about set
membership. `duplicates` remained a Redis SET (it is leader-local and never
merged across peers).
### Canonical CID selection
The canonical CID is a label, not a key — so nothing external breaks when it
changes. **As shipped it is not written at all**: it is computed on every read
from the `cids` members, which makes the "update" cost zero and removes the
reconciler entirely.
Selection rule, highest-wins (this part shipped as designed —
`internal/cid/rank.go`):
1. IPFS CID (any variant)
2. SHA-256 / SHA-3 family
3. BitTorrent infohash (v2 preferred over v1)
4. midhash256
Ranked by **utility for external retrieval**, not by cryptographic strength.
midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
externally meaningful for content addressing. btih ranks above midhash because
it enables swarm interop even if it's weaker as a digest.
~~Reconciler runs on the leader, recomputes `canonical_cid` whenever
`AddAlias` registers a new CID for an entry.~~ **Not shipped** — there is no
reconciler. `GetMetadataDocument` folds the members through `cid.Better` on
each read.
## Internal operations
The meta-core storage interface adds five operations. None are exposed
externally — they're internal to meta-core. External callers see only the
CID-based API.
```go
// Mint creates a new root with no aliases. Called when the watcher
// computes a midhash for a previously-unseen file. Returns the new UUID.
Mint(filePath string, size int64, mtime int64) (uuid string, err error)
// AddAlias registers a CID as resolving to this UUID. Updates both the
// cid:<cid> reverse index and the file:<uuid>/cids/<cid> key-set member.
// (As shipped: no reconciliation step — canonical is derived on read.)
AddAlias(uuid string, cid string) error
// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
GetByCID(cid string) (uuid string, err error)
// MergeIfAliasExists is called when the watcher mints a fresh UUID for a
// file path but, on hashing, discovers the resulting CID already belongs
// to a different UUID. Merges the new (mostly-empty) UUID into the
// existing one, transferring filePath into the duplicates set, and
// returns the canonical UUID.
MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error)
// DeleteRoot removes the entire entry, including every cid:* alias and
// every file:<uuid>/* key. Used for file deletion events.
DeleteRoot(uuid string) error
```
### Lifecycle: file arrival
```
1. Watcher sees /watch/foo.mkv created.
2. Watcher computes midhash256 → "midhash256:abc...".
3. Watcher calls GetByCID("midhash256:abc...").
→ uuid_X (existing entry) or "" (new file).
4a. If existing (uuid_X):
- Add "/watch/foo.mkv" to file:<uuid_X>/duplicates.
- No new root created.
- meta-dup is notified via the existing keyspace event channel.
4b. If new:
- Mint("/watch/foo.mkv", size, mtime) → uuid_Y.
- AddAlias(uuid_Y, "midhash256:abc...").
- Plugins now process the file, referenced by midhash256:abc...
(UUID is never disclosed to them.)
5. Later, the full-hash plugin computes sha256.
- Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
- meta-core resolves midhash256:abc → uuid_Y via GetByCID.
- meta-core calls AddAlias(uuid_Y, "sha256:def...").
- Nothing else to do: the next read of this entry derives a canonical CID
of sha2-256, because it outranks the midhash.
```
### Lifecycle: file deletion
```
1. Watcher sees /watch/foo.mkv deleted.
2. Resolve which entry owns this path:
a. If /watch/foo.mkv is in some entry's duplicates set:
- Remove from set. Done.
b. If /watch/foo.mkv is the filePath of some entry:
- If duplicates is non-empty: promote one duplicate to filePath.
- If duplicates is empty: DeleteRoot(uuid).
- Iterates the cids set, deletes each cid:<cid> reverse-index key.
- Deletes every file:<uuid>/* key.
- Removes uuid from file:__index__.
```
## Public API
```
GET /api/meta/<cid>
→ 200 { "filePath": "...", "canonical_cid": "...",
"cids": [...], "title": "...", "tmdb": {...}, ... }
→ 404 if no such CID
GET /api/file/<cid> (already exists — implementation now O(1))
→ streams file content
```
The UUID is **not** included in the response. Clients address everything by
CID; meta-core's choice of internal identifier remains a private
implementation detail and may change.
Auth: `/api/meta/<cid>` follows the same pattern as `/api/file/<cid>` and
bypasses the auth perimeter. It's the metadata twin of the content endpoint,
and meta-share calls it from federated search responses.
## Cross-service impact
This is a meta-core schema change. Other services interact only through CIDs,
so the impact is contained but real.
- **meta-sort**: writers must resolve plugin callbacks to a UUID before
writing. The existing `LeaderClient` gains a `getByCID` cache.
- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the
ID format changes.
- **meta-stremio**: queries by CID; needs no logic change beyond its storage
client.
- **meta-dup**: gains a cleaner data source — duplicates are now explicit in
the `duplicates` set, not implicit in colliding writes.
- **meta-search / meta-share**: announce the *derived* canonical CID rather
than the midhash — each mirrors the Go rank in `ingest::canonical_cid_from_metadata`
rather than reading a stored field. Federated search responses include all
CIDs from the `cids` key-set so any peer querying by any digest gets a hit.
(Written before the 2026-06 discovery/transport split; discovery is now
meta-search's job.)
- **Plugins**: no change. They speak CIDs only.
## Migration
> **Historical — this sweep was never run.** The alpha shipped behind a
> clean-wipe gate (Redis was flushed and the library re-ingested), so no
> in-place migration code exists. Kept for the record.
One-time sweep, idempotent, runs on first boot after upgrade:
```
For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …):
1. Mint a fresh uuid_Y.
2. AddAlias(uuid_Y, hashID). The existing primary becomes an alias.
3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value).
4. RENAME file:<hashID>/<property> → file:<uuid_Y>/<property> for every
property (Lua script, one entry at a time; non-atomic across entries,
but each entry stays internally consistent).
5. Reconcile canonical_cid.
6. Remove hashID from file:__index__, add uuid_Y.
```
Crash-recoverable: re-running the sweep finds entries with both old and new
roots and resumes by detecting which step completed (presence of
`file:<uuid>/cids` indicates step 3 finished; absence of any
`file:<hashID>/*` keys indicates step 4 finished).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. |
| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. |
| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
| ~~`cids` and `duplicates` field type~~ | ~~Redis SET~~ → **`cids` is a key-set of bare CIDs; `duplicates` stayed a SET** | Reversed before shipping. Flat `cids/<cid>` STRING keys merge conflict-free across peers; a SET does not. See §14.13 of `METADATA_KEYS.md`. |
| ~~Stored `canonical_cid`~~ | **Dropped — derived on read** | A stored scalar is not conflict-free to merge. `GetMetadataDocument` ranks the `cids` members per request. |
| Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
## Open questions
- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints
a UUID at discovery (before hashing), then `AddAlias` runs when the midhash
arrives. Requires an explicit "reconcile if alias already exists" step
because two paths with the same content would each get their own UUID until
hashing reveals the collision. Worth doing eventually; not required for
this iteration.
- ~~**meta-share announce strategy.**~~ **Settled.** The record carries *every*
member of `cids` on the wire (`Record.cids`) and addresses itself by the
single derived canonical (`Record.cid`), so a peer querying by any digest
gets a hit without the DHT footprint of announcing each one separately.
- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
content-streaming endpoint for external IPFS clients? Adjacent to this
design but not part of it.
- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`)
handle `file:<type>:<hash>/...` with a special split rule for the colon
inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies
— but any external consumer subscribed by hashID pattern needs updating.
Inventory those before flipping the migration.
## Non-goals
- **Not** a content-replication or IPFS-pinning system. Registering an IPFS
CID alias does not pin the file anywhere — it only states "if somebody
hashes this file with IPFS's chunker, this is the CID they'd get."
- **Not** a versioning scheme. Metadata is a snapshot of current state;
historical versions are out of scope.
- **Not** a strong-consistency system. Reverse-index writes happen under the
leader lock; reads from followers may briefly miss a freshly-added alias.
+57 -121
View File
@@ -1,140 +1,76 @@
# @metazla/meta-editor
# @meta-core/editor
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface.
React + Vite metadata browser/editor that ships inside the meta-core
container. Walks the UUID-rooted Redis keyspace, edits leaf values,
imports/exports snapshots, and consults the live schema indexer.
## Features
## What it talks to
- **Search**: Find files by title, filename, or hash ID
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more
- **Batch Operations**: Update multiple files simultaneously
- **Real-time**: Direct integration with etcd via REST API
- **User-friendly**: Clean, intuitive interface with form validation
The editor is a pure SPA — it has no server of its own. It calls the
meta-core HTTP API directly under three prefixes (see `src/api/`):
## Development
- `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`,
`value`, `key/{key...}`). This is the UUID-exposing surface — keys look
like `file:<uuid>/<property>` and `cid:<algo>:<value>`.
- `/api/schema` — live per-field schema (type hints, value breakdowns),
produced by meta-core's schema indexer that consumes `meta:events`.
- `/api/snapshot/*` — snapshot export / import / wipe.
### Prerequisites
In production the editor is served by meta-core's nginx (the Vite build
output is copied into the image at the same path) under
`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to
`/editor/` accordingly.
- Node.js 21.6.2+
- pnpm 10.19.0+
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
### Running Locally
```bash
# From the root directory
pnpm run start:editor
# Or from the package directory
cd packages/meta-editor
pnpm run dev
```
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
### Building
```bash
# From the root directory
pnpm run build:editor
# Or from the package directory
cd packages/meta-editor
pnpm run build
```
The built files will be in `dist/` directory.
## Docker Deployment
In the Docker development environment, the editor is automatically available at:
## Project layout
```
http://localhost/editor
```
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
## API Integration
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
- `POST /api/metadata/search` - Search for files
- `GET /api/metadata/:hashId` - Get file metadata
- `PUT /api/metadata/:hashId` - Update complete metadata
- `PUT /api/metadata/:hashId/property` - Update specific property
- `POST /api/metadata/batch` - Batch update multiple files
See `src/api/metadataApi.ts` for the API client implementation.
## Usage
### Searching for Files
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
2. Enter your search query
3. Click "Search"
4. Results will appear in the left sidebar
### Editing Metadata
1. Select a file from the search results
2. Modify any fields in the editor form
3. Click "Save Changes" to persist updates
4. Use "Reset" to discard changes
### Supported Metadata Fields
**Basic Information:**
- Title, Original Title, Show Title
- Video Type (Movie, TV, Anime, Other)
- Year, Season, Episode
**Jellyfin Metadata:**
- Plot, IMDb ID, TMDb ID, AniDB ID
- Director, Studio, Rating, Runtime
- Genres, Premiered Date, Release Date
**File Information:**
- Filename, File Size, MIME Type
- Hash values (SHA-256, MD5, etc.) - read-only
## Project Structure
```
packages/meta-editor/
├── src/
│ ├── api/ # API client for metadata operations
│ ├── components/ # React components
│ │ ├── SearchBar.tsx # Search interface
│ │ ├── FileList.tsx # Search results list
│ │ └── MetadataEditor.tsx # Main editing form
│ ├── types/ # TypeScript type definitions
│ ├── utils/ # Utility functions
│ ├── App.tsx # Main application component
│ ├── main.tsx # Entry point
│ └── index.css # Global styles
packages/meta-core/editor/
├── index.html
├── vite.config.ts
├── tsconfig.json
└── package.json
├── package.json @meta-core/editor
└── src/
├── main.tsx entry
├── api/
│ ├── kvApi.ts /api/kv client
│ ├── schemaApi.ts /api/schema client + cache
│ └── snapshotApi.ts /api/snapshot client
├── kv/ KV browser + leaf/branch editors
├── components/ shared widgets (language combobox, snapshot panel, ...)
└── index.css
```
## Technology Stack
## Local development
- **React 18** - UI framework
- **TypeScript 5.7** - Type safety
- **Vite 6** - Build tool and dev server
- **Native CSS** - Styling (no CSS framework)
Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`.
## Contributing
```bash
cd packages/meta-core/editor
pnpm install
pnpm dev # vite dev server on http://localhost:5173
```
When adding new features:
The Vite dev server proxies `/api/*` to `http://localhost:3000` (see
`vite.config.ts`). For local development against a running meta-core
container, change that target to wherever meta-core is reachable on your
host — e.g. `http://localhost:18083` for the dev compose's direct-backend
port.
1. Add type definitions to `src/types/index.ts`
2. Update API client in `src/api/metadataApi.ts`
3. Create/update components as needed
4. Update this README with new features
No root-level `start:editor` or `build:editor` scripts exist; run pnpm
commands from this directory.
## License
## Build
Part of the MetaMesh project.
```bash
pnpm build # tsc -b && vite build → dist/
```
The container build copies `dist/` into the nginx web root so the editor
is served at `/editor/`.
## Tech
- React 18, TypeScript 5
- Vite 5 (`base: '/editor/'`)
- Native CSS — no UI framework
- `iso-639-3` for language code rendering
+13 -2
View File
@@ -1,13 +1,14 @@
{
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"dependencies": {
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
@@ -1246,6 +1247,16 @@
"node": ">=6.9.0"
}
},
"node_modules/iso-639-3": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/iso-639-3/-/iso-639-3-3.0.1.tgz",
"integrity": "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-185
View File
@@ -1,185 +0,0 @@
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.app-header {
padding: 2rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.app-header h1 {
margin: 0 0 0.5rem 0;
color: #646cff;
}
.app-description {
margin: 0;
color: rgba(255, 255, 255, 0.6);
}
.app-main {
flex: 1;
padding: 2rem;
max-width: 1400px;
width: 100%;
margin: 0 auto;
}
.app-error {
padding: 1rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
border-radius: 8px;
color: #ff6b6b;
margin-bottom: 1rem;
}
.app-loading {
text-align: center;
padding: 3rem 2rem;
color: rgba(255, 255, 255, 0.7);
font-size: 1.1rem;
}
.app-welcome {
text-align: center;
padding: 4rem 2rem;
color: rgba(255, 255, 255, 0.6);
}
.app-welcome h2 {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.8);
}
.app-welcome-features {
margin-top: 1rem;
margin-bottom: 1.5rem;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
}
.app-kv-browser {
margin-top: 1.5rem;
text-align: left;
max-width: 900px;
margin-left: auto;
margin-right: auto;
}
.app-content {
display: grid;
grid-template-columns: 350px 1fr;
gap: 1.5rem;
margin-top: 1rem;
}
.app-sidebar {
min-height: 400px;
}
.app-editor {
min-height: 400px;
}
.app-no-selection {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
min-height: 400px;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
text-align: center;
padding: 2rem;
}
.app-footer {
padding: 1.5rem 2rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-size: 0.9rem;
}
.app-footer a {
color: #646cff;
text-decoration: none;
}
.app-footer a:hover {
text-decoration: underline;
}
@media (max-width: 1024px) {
.app-content {
grid-template-columns: 1fr;
}
.app-sidebar {
order: 2;
}
.app-editor {
order: 1;
}
}
@media (max-width: 768px) {
.app-header,
.app-main,
.app-footer {
padding: 1rem;
}
.app-content {
gap: 1rem;
}
}
@media (prefers-color-scheme: light) {
.app-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.app-description {
color: rgba(0, 0, 0, 0.6);
}
.app-error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.app-loading {
color: rgba(0, 0, 0, 0.7);
}
.app-welcome {
color: rgba(0, 0, 0, 0.6);
}
.app-welcome h2 {
color: rgba(0, 0, 0, 0.8);
}
.app-welcome-features {
color: rgba(0, 0, 0, 0.5);
}
.app-no-selection {
background-color: rgba(0, 0, 0, 0.02);
color: rgba(0, 0, 0, 0.5);
}
.app-footer {
border-top-color: rgba(0, 0, 0, 0.1);
color: rgba(0, 0, 0, 0.5);
}
}
-246
View File
@@ -1,246 +0,0 @@
import { useState, useEffect } from 'react';
import { SearchBar } from './components/SearchBar';
import { FileList } from './components/FileList';
import { MetadataEditor } from './components/MetadataEditor';
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
import { KVTreeView } from './components/KVTreeView';
import { MetadataAPI } from './api/metadataApi';
import { SearchResult } from './types';
import './App.css';
function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
const [bulkEditMode, setBulkEditMode] = useState(false);
const [isSearching, setIsSearching] = useState(false);
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
const [error, setError] = useState<string | null>(null);
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
const getHashFromUrl = (): string | null => {
const path = window.location.pathname;
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
if (fileMatch) {
return fileMatch[1];
}
// Fallback to query param for backwards compatibility
const params = new URLSearchParams(window.location.search);
return params.get('hash');
};
// Update URL when file is selected
const updateUrlForFile = (hashId: string | undefined) => {
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
if (hashId) {
const newPath = `${basePath}/file/${hashId}`;
window.history.pushState({ hashId }, '', newPath);
} else {
window.history.pushState({}, '', basePath || '/editor/');
}
};
// Check for hash parameter in URL on component mount
useEffect(() => {
const loadHashFromUrl = async () => {
const hashParam = getHashFromUrl();
if (hashParam) {
try {
setIsLoadingDirectHash(true);
setError(null);
// Try to get metadata for this hash
const metadata = await MetadataAPI.getMetadata(hashParam);
// Create a search result from the metadata
const searchResult: SearchResult = {
hashId: hashParam,
metadata: metadata,
score: 1.0,
};
// Set the search results and select this file
setSearchResults([searchResult]);
setSelectedHashId(hashParam);
} catch (err: any) {
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsLoadingDirectHash(false);
}
}
};
loadHashFromUrl();
// Handle browser back/forward
const handlePopState = (event: PopStateEvent) => {
const hashId = event.state?.hashId || getHashFromUrl();
if (hashId) {
setSelectedHashId(hashId);
} else {
setSelectedHashId(undefined);
}
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []); // Empty dependency array means this runs once on mount
const handleSearch = async (query: string) => {
try {
setIsSearching(true);
setError(null);
// Unified search - searches across all fields
const response = await MetadataAPI.search({ query });
setSearchResults(response.results);
// Auto-select first result if available
if (response.results.length > 0) {
setSelectedHashId(response.results[0].hashId);
} else {
setSelectedHashId(undefined);
}
} catch (err: any) {
setError(err.message);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsSearching(false);
}
};
const handleSelectFile = (hashId: string) => {
if (!bulkEditMode) {
setSelectedHashId(hashId);
setSelectedHashIds(new Set());
updateUrlForFile(hashId);
}
};
const handleToggleSelection = (hashId: string) => {
const newSelection = new Set(selectedHashIds);
if (newSelection.has(hashId)) {
newSelection.delete(hashId);
} else {
newSelection.add(hashId);
}
setSelectedHashIds(newSelection);
};
const handleSelectAll = () => {
const allHashIds = new Set(searchResults.map(r => r.hashId));
setSelectedHashIds(allHashIds);
};
const handleClearSelection = () => {
setSelectedHashIds(new Set());
setBulkEditMode(false);
};
const handleBulkEditMode = (enabled: boolean) => {
setBulkEditMode(enabled);
if (enabled) {
setSelectedHashId(undefined);
} else {
setSelectedHashIds(new Set());
}
};
const handleMetadataSaved = () => {
// Optionally refresh the search results to reflect changes
console.log('Metadata saved successfully');
};
const handleBulkMetadataSaved = () => {
console.log('Bulk metadata saved successfully');
handleClearSelection();
};
return (
<div className="app">
<header className="app-header">
<h1>MetaMesh Metadata Editor</h1>
<p className="app-description">
Search and edit metadata for your media files
</p>
</header>
<main className="app-main">
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
{error && <div className="app-error">{error}</div>}
{isLoadingDirectHash && (
<div className="app-loading">Loading file metadata...</div>
)}
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
<div className="app-welcome">
<h2>Welcome to MetaMesh Metadata Editor</h2>
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
<p className="app-welcome-features">
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
</p>
<div className="app-kv-browser">
<KVTreeView />
</div>
</div>
)}
{searchResults.length > 0 && (
<div className="app-content">
<div className="app-sidebar">
<FileList
results={searchResults}
selectedHashId={selectedHashId}
selectedHashIds={selectedHashIds}
bulkEditMode={bulkEditMode}
onSelectFile={handleSelectFile}
onToggleSelection={handleToggleSelection}
onSelectAll={handleSelectAll}
onClearSelection={handleClearSelection}
onBulkEditMode={handleBulkEditMode}
/>
</div>
<div className="app-editor">
{bulkEditMode && selectedHashIds.size > 0 ? (
<BulkMetadataEditor
hashIds={Array.from(selectedHashIds)}
onSaved={handleBulkMetadataSaved}
onCancel={handleClearSelection}
/>
) : selectedHashId ? (
<MetadataEditor
key={selectedHashId}
hashId={selectedHashId}
onSaved={handleMetadataSaved}
/>
) : (
<div className="app-no-selection">
{bulkEditMode
? 'Select files from the list to bulk edit metadata'
: 'Select a file from the list to edit its metadata'}
</div>
)}
</div>
</div>
)}
</main>
<footer className="app-footer">
<p>
MetaMesh Metadata Editor | Part of{' '}
<a href="/" target="_blank">
MetaMesh
</a>
</p>
</footer>
</div>
);
}
export default App;
+136
View File
@@ -0,0 +1,136 @@
// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of
// string-valued keys; this module wraps the S3-style listing, search, and
// per-key get/put/delete operations.
const API_BASE = '/api/kv';
export interface KVTreeResponse {
prefix: string;
delimiter: string;
branches: string[];
leaves: string[];
cursor: string;
hasMore: boolean;
}
export interface KVSearchResponse {
contains: string;
keys: string[];
truncated: boolean;
}
export interface KVFindMatch {
key: string;
value: string;
entryPath: string;
field: string;
}
export interface KVFindResponse {
contains: string;
fields: string[];
matches: KVFindMatch[];
truncated: boolean;
}
export interface KVValueResponse {
key: string;
type: string;
value: string;
exists: boolean;
}
export interface KVInfoResponse {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage: string;
}
async function jsonOrThrow<T>(res: Response): Promise<T> {
if (!res.ok) {
let msg = `HTTP ${res.status}`;
try {
const body = await res.json();
if (body?.message) msg = body.message;
else if (body?.error) msg = body.error;
} catch {}
throw new Error(msg);
}
return res.json();
}
export const KVAPI = {
async info(): Promise<KVInfoResponse> {
return jsonOrThrow(await fetch(`${API_BASE}/info`));
},
async tree(params: {
prefix?: string;
delimiter?: string;
cursor?: string;
max?: number;
}): Promise<KVTreeResponse> {
const q = new URLSearchParams();
if (params.prefix !== undefined) q.set('prefix', params.prefix);
if (params.delimiter !== undefined) q.set('delimiter', params.delimiter);
if (params.cursor) q.set('cursor', params.cursor);
if (params.max) q.set('max', String(params.max));
return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`));
},
// Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap
// is reached. Useful for branches with up to a few thousand children where
// pagination UX would just be a worse spinner.
async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
const seenLeaves = new Set<string>();
const seenBranches = new Set<string>();
let cursor = '';
while (true) {
const page = await this.tree({ prefix, delimiter, cursor, max: 1000 });
for (const k of page.leaves) seenLeaves.add(k);
for (const k of page.branches) seenBranches.add(k);
cursor = page.cursor;
if (!page.hasMore) break;
if (seenLeaves.size + seenBranches.size >= softCap) break;
}
return {
prefix,
delimiter,
branches: [...seenBranches].sort(),
leaves: [...seenLeaves].sort(),
cursor,
hasMore: cursor !== '0' && cursor !== '',
};
},
async search(contains: string, limit = 500): Promise<KVSearchResponse> {
const q = new URLSearchParams({ contains, limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`));
},
async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`));
},
async get(key: string): Promise<KVValueResponse> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
},
async put(key: string, value: string): Promise<KVValueResponse> {
const res = await fetch(`${API_BASE}/value`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ key, value }),
});
return jsonOrThrow(res);
},
async del(key: string): Promise<{ key: string; deleted: boolean }> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
},
};
-151
View File
@@ -1,151 +0,0 @@
import {
FileMetadata,
SearchResponse,
UpdateResult,
BatchUpdateResponse,
} from '../types';
const API_BASE = '/api/metadata';
export class MetadataAPI {
/**
* Search for files by query, hashId, or property
*/
static async search(params: {
query?: string;
hashId?: string;
property?: string;
propertyValue?: string;
limit?: number;
}): Promise<SearchResponse> {
const response = await fetch(`${API_BASE}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Search failed');
}
return response.json();
}
/**
* Get metadata for a specific file by hash ID
*/
static async getMetadata(hashId: string): Promise<FileMetadata> {
const response = await fetch(`${API_BASE}/${hashId}`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get metadata');
}
return response.json();
}
/**
* Update complete metadata for a file
*/
static async updateMetadata(
hashId: string,
metadata: FileMetadata
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(metadata),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update metadata');
}
return response.json();
}
/**
* Update a specific property for a file
*/
static async updateProperty(
hashId: string,
property: string,
value: string
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}/property`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ property, value }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update property');
}
return response.json();
}
/**
* Batch update multiple files
*/
static async batchUpdate(
updates: Array<{
hashId: string;
metadata?: FileMetadata;
properties?: Record<string, string>;
}>
): Promise<BatchUpdateResponse> {
const response = await fetch(`${API_BASE}/batch`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ updates }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Batch update failed');
}
return response.json();
}
/**
* Delete metadata for a file
*/
static async deleteMetadata(hashId: string): Promise<void> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'DELETE',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to delete metadata');
}
}
/**
* Get all hash IDs
*/
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
const response = await fetch(`${API_BASE}/hash-ids`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get hash IDs');
}
return response.json();
}
}
+74
View File
@@ -0,0 +1,74 @@
import { useEffect, useState } from 'react';
export type SchemaPrimitive =
| 'string'
| 'int'
| 'float'
| 'bool'
| 'json-object'
| 'json-array'
| 'undefined'
| 'mixed';
export type SchemaHint = 'cid' | 'timestamp';
export type SchemaKeyHint = 'language-code';
export interface SchemaFieldInfo {
type: SchemaPrimitive;
hint?: SchemaHint;
key_hint?: SchemaKeyHint;
breakdown: Record<string, number>;
}
export interface SchemaResponse {
fields: Record<string, SchemaFieldInfo>;
generated_at: string;
source: 'live' | 'rescan';
}
let cached: Promise<SchemaResponse> | null = null;
export async function fetchSchema(): Promise<SchemaResponse> {
if (!cached) {
cached = fetch('/api/schema').then((r) => {
if (!r.ok) {
cached = null;
throw new Error(`schema fetch failed: ${r.status}`);
}
return r.json();
});
}
return cached;
}
/**
* Returns the live schema entry for a field, or undefined if not yet observed.
* Loads /api/schema lazily on first mount and caches the result process-wide.
*/
export function useSchema(): {
schema: SchemaResponse | null;
getFieldHint: (key: string) => SchemaFieldInfo | undefined;
} {
const [schema, setSchema] = useState<SchemaResponse | null>(null);
useEffect(() => {
let alive = true;
fetchSchema()
.then((s) => {
if (alive) setSchema(s);
})
.catch(() => {
// Schema endpoint missing or errored — editor falls back to plugin schemas.
});
return () => {
alive = false;
};
}, []);
const getFieldHint = (key: string): SchemaFieldInfo | undefined => {
if (!schema) return undefined;
return schema.fields[key];
};
return { schema, getFieldHint };
}
+96
View File
@@ -0,0 +1,96 @@
const API_BASE = '/api/snapshot';
export type ImportMode = 'replace' | 'merge';
export type ImportConflict = 'mine' | 'source';
export interface FileImportResult {
hashId: string;
added: number;
updated: number;
unchanged: number;
kept: number;
deleted: number;
error?: string;
}
export interface PluginOutputResult {
written: number;
skipped: number;
}
export interface ImportResult {
mode: ImportMode;
conflict?: ImportConflict;
dryRun: boolean;
totalFiles: number;
filesOk: number;
filesFailed: number;
files?: FileImportResult[];
pluginOutput?: PluginOutputResult;
}
export interface WipeResult {
metadataDeleted: number;
}
export class SnapshotAPI {
/**
* Trigger a download of a metadata snapshot ZIP.
* Browser handles the download via a synthetic <a> click.
*/
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
const parts: string[] = [];
if (includes.pluginOutput) parts.push('plugin-output');
const qs = parts.length ? `?include=${parts.join(',')}` : '';
const a = document.createElement('a');
a.href = `${API_BASE}/export${qs}`;
a.rel = 'noopener';
document.body.appendChild(a);
a.click();
a.remove();
}
/**
* Upload a snapshot ZIP and apply it.
* Mode replace deletes and rewrites each CID's properties.
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
*/
static async runImport(
file: File,
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
): Promise<ImportResult> {
const params = new URLSearchParams();
params.set('mode', opts.mode);
if (opts.mode === 'merge') {
params.set('conflict', opts.conflict ?? 'source');
}
if (opts.dryRun) {
params.set('dry_run', 'true');
}
const fd = new FormData();
fd.append('snapshot', file);
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
method: 'POST',
body: fd,
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Import failed');
}
return response.json();
}
/**
* Wipe Redis metadata. Pass scope=metadata.
*/
static async wipe(scope: 'metadata'): Promise<WipeResult> {
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
method: 'POST',
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Wipe failed');
}
return response.json();
}
}
@@ -1,382 +0,0 @@
.bulk-metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.bulk-metadata-editor-header {
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-editor-header h2 {
margin: 0 0 0.5rem 0;
font-size: 1.5rem;
}
.bulk-info {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.6);
font-size: 0.95rem;
}
.bulk-metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.bulk-metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.bulk-metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.bulk-metadata-editor-form {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
.bulk-metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.bulk-metadata-section-header {
margin-bottom: 1.5rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.section-hint {
margin: 0.25rem 0 0 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.section-hint-bulk {
margin: 0.5rem 0 0 0;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
font-weight: 500;
}
/* Plugin-based bulk fields */
.bulk-fields-container {
display: flex;
flex-direction: column;
gap: 1rem;
}
.bulk-field-wrapper {
border: 1px solid rgba(255, 255, 255, 0.05);
border-radius: 4px;
padding: 0.75rem;
background-color: rgba(0, 0, 0, 0.1);
transition: all 0.2s ease;
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.bulk-field-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.field-key {
font-size: 0.85rem;
font-weight: 600;
color: rgba(255, 255, 255, 0.6);
text-transform: lowercase;
font-family: 'Courier New', monospace;
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.bulk-field-input {
/* Plugin component will render inside this */
}
/* Legacy bulk fields (for compatibility) */
.bulk-metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.bulk-metadata-field:last-child {
margin-bottom: 0;
}
.field-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.field-header label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.clear-field-btn {
width: 20px;
height: 20px;
padding: 0;
border: 1px solid rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
border-radius: 50%;
cursor: pointer;
font-size: 16px;
line-height: 1;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
.bulk-metadata-field input,
.bulk-metadata-field textarea,
.bulk-metadata-field select {
width: 100%;
}
.bulk-metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.bulk-metadata-field-row:last-child {
margin-bottom: 0;
}
/* Modified fields summary */
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-fields h3 {
color: #646cff;
}
.modified-fields-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.modified-field-item {
display: flex;
gap: 0.75rem;
align-items: center;
padding: 0.75rem;
background-color: rgba(100, 108, 255, 0.05);
border: 1px solid rgba(100, 108, 255, 0.2);
border-radius: 4px;
}
.field-name {
font-weight: 600;
color: #646cff;
min-width: 150px;
font-family: 'Courier New', monospace;
font-size: 0.9rem;
}
.field-value {
flex: 1;
word-break: break-word;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
}
.remove-field-btn {
padding: 0.25rem 0.75rem;
font-size: 0.85rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
white-space: nowrap;
}
.remove-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
@media (max-width: 768px) {
.bulk-metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.bulk-metadata-editor-actions {
width: 100%;
flex-direction: column;
}
.bulk-metadata-editor-actions button {
width: 100%;
}
.bulk-metadata-field-row {
grid-template-columns: 1fr;
}
.modified-field-item {
flex-direction: column;
align-items: flex-start;
}
.field-name {
min-width: auto;
}
.remove-field-btn {
align-self: flex-end;
}
}
@media (prefers-color-scheme: light) {
.bulk-metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.bulk-info {
color: rgba(0, 0, 0, 0.6);
}
.bulk-metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.section-hint {
color: rgba(0, 0, 0, 0.5);
}
.section-hint-bulk {
color: rgba(0, 0, 0, 0.4);
}
.bulk-field-wrapper {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.field-key {
color: rgba(0, 0, 0, 0.6);
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.field-header label {
color: rgba(0, 0, 0, 0.7);
}
.field-value {
color: rgba(0, 0, 0, 0.8);
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-field-item {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.2);
}
.clear-field-btn {
border-color: rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #cc0000;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
}
@@ -1,280 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './BulkMetadataEditor.css';
interface PluginsResponse {
plugins: PluginSchema[];
}
interface BulkMetadataEditorProps {
hashIds: string[];
onSaved?: () => void;
onCancel?: () => void;
}
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
// Load plugins and metadata on mount
useEffect(() => {
const loadData = async () => {
try {
setIsLoading(true);
setError(null);
// Fetch plugins
const pluginsResponse = await fetch('/api/plugins');
if (pluginsResponse.ok) {
const data: PluginsResponse = await pluginsResponse.json();
// Sort by priority
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
// Load metadata for all selected files
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
const metadata = await Promise.all(metadataPromises);
// Calculate common values
const commons: Record<string, any> = {};
// Get all editable fields from all plugins
const allFields: string[] = [];
plugins.forEach((plugin) => {
if (plugin.metadataSchema) {
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
if (!field.readonly) {
allFields.push(key);
}
});
}
});
allFields.forEach((field) => {
const values = metadata.map((m) => m[field as keyof FileMetadata]);
// Filter out undefined and null values
const definedValues = values.filter((v) => v !== undefined && v !== null);
if (definedValues.length === 0) {
return;
}
// Safely stringify values
const uniqueValues = Array.from(
new Set(
definedValues.map((v) => {
try {
return JSON.stringify(v);
} catch {
return String(v);
}
})
)
);
if (uniqueValues.length === 1) {
try {
commons[field] = JSON.parse(uniqueValues[0]);
} catch {
commons[field] = uniqueValues[0];
}
} else {
commons[field] = '*';
}
});
setCommonValues(commons);
} catch (err: any) {
setError(`Failed to load data: ${err.message}`);
} finally {
setIsLoading(false);
}
};
loadData();
}, [hashIds]);
const handleFieldChange = (field: string, value: any) => {
setEditedFields({
...editedFields,
[field]: value,
});
};
const handleClearField = (field: string) => {
const newFields = { ...editedFields };
delete newFields[field];
setEditedFields(newFields);
};
const handleSave = async () => {
if (Object.keys(editedFields).length === 0) {
setError('No fields have been edited');
return;
}
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
// Build batch update payload
const updates = hashIds.map((hashId) => ({
hashId,
metadata: editedFields,
}));
const result = await MetadataAPI.batchUpdate(updates);
if (result.errors > 0) {
setError(
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
);
console.error('Batch update errors:', result.results.filter((r) => r.error));
} else {
setSuccessMessage(`Successfully updated ${result.success} files!`);
}
if (onSaved) {
onSaved();
}
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const hasChanges = Object.keys(editedFields).length > 0;
// Filter to only editable plugins (those with non-readonly fields)
const editablePlugins = plugins.filter((plugin) => {
if (!plugin.metadataSchema) return false;
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
});
if (isLoading) {
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-loading">
Loading metadata for {hashIds.length} files...
</div>
</div>
);
}
// Build display metadata (edited values or common values)
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-header">
<h2>Bulk Edit Metadata</h2>
<p className="bulk-info">
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
be updated. Fields showing "*" have different values across files.
</p>
<div className="bulk-metadata-editor-actions">
{onCancel && (
<button onClick={onCancel} disabled={isSaving}>
Cancel
</button>
)}
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
</button>
</div>
</div>
{successMessage && (
<div className="bulk-metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
<div className="bulk-metadata-editor-form">
{editablePlugins.map((plugin) => {
if (!plugin.metadataSchema) return null;
// Get editable fields for this plugin
const editableFields = Object.entries(plugin.metadataSchema).filter(
([, field]) => !field.readonly
);
if (editableFields.length === 0) return null;
return (
<section
key={plugin.id}
className="bulk-metadata-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="bulk-metadata-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="section-hint">{plugin.description}</p>
)}
<p className="section-hint-bulk">
Common values shown. "*" indicates different values. Edit to update all files.
</p>
</div>
<div className="plugin-fields">
{editableFields.map(([key, field]) => (
<DynamicFieldRenderer
key={key}
fieldKey={key}
schema={field}
metadata={displayMetadata as FileMetadata}
onChange={handleFieldChange}
readonly={false}
/>
))}
</div>
</section>
);
})}
{/* Modified Fields Summary */}
{hasChanges && (
<section className="bulk-metadata-section modified-fields">
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
<div className="modified-fields-list">
{Object.entries(editedFields).map(([key, value]) => (
<div key={key} className="modified-field-item">
<span className="field-name">{key}:</span>
<span className="field-value">
{Array.isArray(value)
? value.join(', ')
: typeof value === 'object'
? JSON.stringify(value)
: String(value || '(empty)')}
</span>
<button
className="remove-field-btn"
onClick={() => handleClearField(key)}
title="Remove this field"
>
Remove
</button>
</div>
))}
</div>
</section>
)}
</div>
</div>
);
}
@@ -1,297 +0,0 @@
.dynamic-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.dynamic-field:last-child {
margin-bottom: 0;
}
.dynamic-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
width: 100%;
padding: 0.5rem 0.75rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-size: 0.9rem;
background: rgba(0, 0, 0, 0.3);
color: #fff;
transition: border-color 0.2s;
}
.dynamic-field input:focus,
.dynamic-field select:focus,
.dynamic-field textarea:focus {
outline: none;
border-color: #646cff;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: rgba(0, 0, 0, 0.2);
color: rgba(255, 255, 255, 0.5);
cursor: not-allowed;
}
.dynamic-field textarea {
resize: vertical;
min-height: 80px;
font-family: inherit;
}
.field-hint {
display: block;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
cursor: pointer;
font-weight: normal;
color: rgba(255, 255, 255, 0.7);
}
.checkbox-label input[type="checkbox"] {
width: 1.1rem;
height: 1.1rem;
cursor: pointer;
accent-color: #646cff;
}
.checkbox-label input:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.readonly-value {
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.7);
font-family: 'Courier New', monospace;
font-size: 0.9rem;
word-break: break-all;
}
.field-link {
color: #646cff;
text-decoration: none;
}
.field-link:hover {
text-decoration: underline;
}
/* CID field styling */
.cid-field .cid-value-container {
display: flex;
align-items: center;
gap: 0.5rem;
}
.cid-field .cid-value {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: #a0d8ef;
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cid-field .cid-value:hover {
background: rgba(0, 0, 0, 0.4);
border-color: #646cff;
}
.cid-field .cid-open-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 0.75rem;
background: #646cff;
color: #fff;
border-radius: 4px;
text-decoration: none;
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
transition: background 0.2s;
}
.cid-field .cid-open-button:hover {
background: #535bf2;
}
/* JSON field styling */
.json-field .json-viewer {
background: rgba(0, 0, 0, 0.4);
color: #d4d4d4;
padding: 1rem;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
overflow-x: auto;
max-height: 300px;
overflow-y: auto;
margin: 0;
white-space: pre-wrap;
word-break: break-all;
border: 1px solid rgba(255, 255, 255, 0.1);
}
/* Language string field styling */
.language-string-field .language-entries {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.language-string-field .language-entry {
display: flex;
align-items: center;
gap: 0.5rem;
}
.language-string-field .lang-input {
flex: 1;
min-width: 0;
}
.language-string-field .lang-code {
display: inline-block;
width: 50px;
padding: 0.4rem 0.5rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.8rem;
color: #a0d8ef;
text-align: center;
flex-shrink: 0;
}
.language-string-field .lang-text {
flex: 1;
padding: 0.4rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.language-string-field .lang-remove-btn {
width: 32px;
height: 32px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 100, 100, 0.2);
border: 1px solid rgba(255, 100, 100, 0.3);
border-radius: 4px;
color: #ff6b6b;
font-size: 1.2rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-remove-btn:hover {
background: rgba(255, 100, 100, 0.3);
border-color: rgba(255, 100, 100, 0.5);
}
.language-string-field .lang-add-btn {
align-self: flex-start;
padding: 0.4rem 0.75rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
color: #646cff;
font-size: 0.85rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-add-btn:hover {
background: rgba(100, 108, 255, 0.3);
border-color: rgba(100, 108, 255, 0.5);
}
/* Light theme */
@media (prefers-color-scheme: light) {
.dynamic-field label {
color: rgba(0, 0, 0, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
border-color: rgba(0, 0, 0, 0.1);
background: #fff;
color: #333;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: #f5f5f5;
color: rgba(0, 0, 0, 0.5);
}
.field-hint {
color: rgba(0, 0, 0, 0.5);
}
.checkbox-label {
color: rgba(0, 0, 0, 0.7);
}
.readonly-value {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
.json-field .json-viewer {
background: #1e1e1e;
border-color: rgba(0, 0, 0, 0.2);
}
.language-string-field .lang-code {
background: rgba(100, 108, 255, 0.1);
color: #4a50c8;
}
.language-string-field .lang-text {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
}
@@ -1,378 +0,0 @@
import { SchemaField, FileMetadata } from '../types';
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
import './DynamicFieldRenderer.css';
interface DynamicFieldRendererProps {
fieldKey: string;
schema: SchemaField;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
readonly?: boolean;
}
/**
* Renders a single metadata field based on its schema definition
*/
export function DynamicFieldRenderer({
fieldKey,
schema,
metadata,
onChange,
readonly = false,
}: DynamicFieldRendererProps) {
const isReadonly = readonly || schema.readonly;
const fieldType = schema.type || 'string';
// Get value from metadata using path-style key
const getValue = (): any => {
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = fieldKey.split('/');
let value: any = metadata;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
};
// Set value in metadata using path-style key
const setValue = (newValue: any) => {
onChange(fieldKey, newValue);
};
const value = getValue();
// Render link if configured
const renderValueWithLink = (displayValue: string) => {
if (schema.link && displayValue) {
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
return (
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
{displayValue}
</a>
);
}
return displayValue;
};
// Render based on field type
switch (fieldType) {
case 'boolean':
return (
<div className="dynamic-field">
<label className="checkbox-label">
<input
type="checkbox"
checked={Boolean(value)}
onChange={(e) => setValue(e.target.checked)}
disabled={isReadonly}
/>
<span>{schema.label}</span>
</label>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'number':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="number"
value={value ?? ''}
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
disabled={isReadonly}
min={schema.min}
max={schema.max}
placeholder={schema.placeholder}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'select':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<select
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
>
<option value="">Select...</option>
{schema.options?.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'text':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<textarea
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
placeholder={schema.placeholder}
rows={4}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'array':
// Convert RecordSet to comma-separated string for display
const arrayValue = (() => {
if (!value) return '';
if (typeof value === 'string') return value;
if (typeof value === 'object') {
return Object.values(value).join(', ');
}
return '';
})();
// Convert comma-separated string back to RecordSet
const handleArrayChange = (str: string) => {
const items = str.split(',').map(s => s.trim()).filter(Boolean);
if (items.length === 0) {
setValue(undefined);
return;
}
const recordSet: Record<string, string> = {};
items.forEach((item, idx) => {
recordSet[idx.toString()] = item;
});
setValue(recordSet);
};
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="text"
value={arrayValue}
onChange={(e) => handleArrayChange(e.target.value)}
disabled={isReadonly}
placeholder={schema.placeholder || 'Comma-separated values'}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'json':
// Aggregate matching keys based on pattern
const jsonData = (() => {
if (!schema.pattern) {
return value;
}
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
const result: Record<string, any> = {};
// Recursively find all matching keys
const findMatching = (obj: any, currentPath: string = '') => {
if (!obj || typeof obj !== 'object') return;
for (const [key, val] of Object.entries(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
// Store in result with relative path
const relativePath = newPath.substring(patternPrefix.length - 1);
if (relativePath) {
setNestedValue(result, relativePath, val);
}
}
if (typeof val === 'object' && val !== null) {
findMatching(val, newPath);
}
}
};
findMatching(metadata);
return Object.keys(result).length > 0 ? result : undefined;
})();
if (!jsonData) {
return null;
}
return (
<div className="dynamic-field json-field">
<label>{schema.label}</label>
<pre className="json-viewer">
{JSON.stringify(jsonData, null, 2)}
</pre>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'cid':
// CID field type - displays CID with "Open in Editor" button
if (!value) {
return null;
}
const cidValue = String(value);
const editorUrl = `/editor/file/${cidValue}`;
return (
<div className="dynamic-field cid-field">
<label>{schema.label}</label>
<div className="cid-value-container">
<code
className="cid-value"
onClick={() => navigator.clipboard.writeText(cidValue)}
title="Click to copy"
>
{cidValue}
</code>
<a
href={editorUrl}
className="cid-open-button"
title="Open in Editor"
target="_blank"
rel="noopener noreferrer"
>
↗ Open
</a>
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'languageString':
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
const langEntries: Array<[string, string]> = value && typeof value === 'object'
? Object.entries(value).map(([k, v]) => [k, String(v)])
: [];
const handleLangCodeChange = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const newValue = { ...value };
const text = newValue[oldLang];
delete newValue[oldLang];
newValue[newLang] = text;
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleLangTextChange = (lang: string, text: string) => {
const newValue = { ...value, [lang]: text };
setValue(newValue);
};
const handleRemoveLang = (lang: string) => {
const newValue = { ...value };
delete newValue[lang];
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleAddLang = () => {
// Find a language code not already used, default to 'eng'
const usedLangs = new Set(langEntries.map(([k]) => k));
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
setValue({ ...value, [defaultLang]: '' });
};
if (isReadonly && langEntries.length === 0) {
return null;
}
return (
<div className="dynamic-field language-string-field">
<label>{schema.label}</label>
<div className="language-entries">
{langEntries.map(([lang, text]) => (
<div key={lang} className="language-entry">
{isReadonly ? (
<>
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
<span className="lang-text">{text}</span>
</>
) : (
<>
<LanguageCombobox
value={lang}
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
/>
<input
type="text"
className="lang-input"
value={text}
onChange={(e) => handleLangTextChange(lang, e.target.value)}
placeholder={schema.placeholder || 'Text...'}
/>
<button
type="button"
className="lang-remove-btn"
onClick={() => handleRemoveLang(lang)}
title="Remove"
>
×
</button>
</>
)}
</div>
))}
{!isReadonly && (
<button
type="button"
className="lang-add-btn"
onClick={handleAddLang}
>
+ Add Language
</button>
)}
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'string':
default:
// Show nothing if no value and readonly
if (isReadonly && !value) {
return null;
}
return (
<div className="dynamic-field">
<label>{schema.label}</label>
{isReadonly ? (
<div className="readonly-value">
{renderValueWithLink(String(value ?? ''))}
</div>
) : (
<input
type="text"
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
placeholder={schema.placeholder}
/>
)}
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
}
}
// Helper to set nested value in object
function setNestedValue(obj: any, path: string, value: any) {
const parts = path.split('/').filter(Boolean);
let current = obj;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
}
current = current[parts[i]];
}
if (parts.length > 0) {
current[parts[parts.length - 1]] = value;
}
}
@@ -1,63 +0,0 @@
.plugin-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.plugin-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-section-header .plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.plugin-fields {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Responsive grid for fields */
@media (min-width: 768px) {
.plugin-fields {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1rem;
}
/* Make text and json fields span full width */
.plugin-fields .dynamic-field:has(textarea),
.plugin-fields .json-field {
grid-column: 1 / -1;
}
}
/* Light theme */
@media (prefers-color-scheme: light) {
.plugin-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-section-header .plugin-description {
color: rgba(0, 0, 0, 0.5);
}
}
@@ -1,126 +0,0 @@
import { PluginSchema, FileMetadata, SchemaField } from '../types';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './DynamicPluginSection.css';
interface DynamicPluginSectionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
}
/**
* Renders a plugin section with all its fields based on schema
*/
export function DynamicPluginSection({
plugin,
metadata,
onChange,
}: DynamicPluginSectionProps) {
const schema = plugin.metadataSchema;
if (!schema || Object.keys(schema).length === 0) {
return null;
}
// Check if plugin has any data in metadata
const hasData = checkPluginHasData(schema, metadata);
if (!hasData) {
return null;
}
// Sort fields: non-readonly first, then by key
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
// Non-readonly fields first
if (a.readonly !== b.readonly) {
return a.readonly ? 1 : -1;
}
// Then alphabetically
return keyA.localeCompare(keyB);
});
return (
<section
className="plugin-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="plugin-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="plugin-description">{plugin.description}</p>
)}
</div>
<div className="plugin-fields">
{sortedFields.map(([fieldKey, fieldSchema]) => (
<DynamicFieldRenderer
key={fieldKey}
fieldKey={fieldKey}
schema={fieldSchema}
metadata={metadata}
onChange={onChange}
/>
))}
</div>
</section>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function checkPluginHasData(
schema: Record<string, SchemaField>,
metadata: FileMetadata
): boolean {
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
/**
* Get nested value from object using path-style key
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
/**
* Check if object has any key starting with prefix
*/
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
-174
View File
@@ -1,174 +0,0 @@
.file-list {
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
overflow: hidden;
}
.file-list-header {
padding: 0.75rem 1rem;
background-color: rgba(255, 255, 255, 0.05);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
justify-content: space-between;
align-items: center;
}
.file-count {
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
}
.bulk-edit-toggle {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(100, 108, 255, 0.1);
border: 1px solid rgba(100, 108, 255, 0.3);
color: #646cff;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.bulk-edit-toggle:hover {
background-color: rgba(100, 108, 255, 0.2);
border-color: rgba(100, 108, 255, 0.5);
}
.bulk-edit-toggle.active {
background-color: rgba(100, 108, 255, 0.3);
border-color: #646cff;
font-weight: 500;
}
.file-list-bulk-actions {
padding: 0.75rem 1rem;
background-color: rgba(100, 108, 255, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
gap: 1rem;
align-items: center;
}
.select-all-btn {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(255, 255, 255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.2);
color: rgba(255, 255, 255, 0.9);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.select-all-btn:hover {
background-color: rgba(255, 255, 255, 0.2);
border-color: rgba(255, 255, 255, 0.3);
}
.selection-count {
font-size: 0.9em;
color: #646cff;
font-weight: 500;
}
.file-list-items {
max-height: 500px;
overflow-y: auto;
}
.file-list-item {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
cursor: pointer;
transition: background-color 0.2s;
display: flex;
gap: 0.75rem;
align-items: flex-start;
}
.file-list-item-checkbox {
padding-top: 0.2rem;
}
.file-list-item-checkbox input[type='checkbox'] {
width: 18px;
height: 18px;
cursor: pointer;
}
.file-list-item-content {
flex: 1;
}
.file-list-item:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.2);
border-left: 3px solid #646cff;
}
.file-list-item-title {
font-weight: 500;
margin-bottom: 0.5rem;
}
.file-list-item-details {
display: flex;
gap: 1rem;
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
margin-bottom: 0.25rem;
}
.file-list-item-hash {
font-size: 0.8em;
color: rgba(255, 255, 255, 0.4);
font-family: 'Courier New', monospace;
}
.file-list-empty {
padding: 3rem 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
@media (prefers-color-scheme: light) {
.file-list {
border-color: rgba(0, 0, 0, 0.1);
}
.file-list-header {
background-color: rgba(0, 0, 0, 0.02);
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.file-count {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.file-list-item:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.1);
}
.file-list-item-details {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item-hash {
color: rgba(0, 0, 0, 0.4);
}
.file-list-empty {
color: rgba(0, 0, 0, 0.5);
}
}
-123
View File
@@ -1,123 +0,0 @@
import { SearchResult } from '../types';
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
import './FileList.css';
interface FileListProps {
results: SearchResult[];
selectedHashId?: string;
selectedHashIds: Set<string>;
bulkEditMode: boolean;
onSelectFile: (hashId: string) => void;
onToggleSelection: (hashId: string) => void;
onSelectAll: () => void;
onClearSelection: () => void;
onBulkEditMode: (enabled: boolean) => void;
}
export function FileList({
results,
selectedHashId,
selectedHashIds,
bulkEditMode,
onSelectFile,
onToggleSelection,
onSelectAll,
onClearSelection,
onBulkEditMode,
}: FileListProps) {
if (results.length === 0) {
return (
<div className="file-list-empty">
<p>No files found. Try searching for a title, filename, or hash ID.</p>
</div>
);
}
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
// Sort results by file path
const sortedResults = [...results].sort((a, b) => {
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
return pathA.localeCompare(pathB);
});
return (
<div className="file-list">
<div className="file-list-header">
<span className="file-count">{results.length} files found</span>
<button
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
onClick={() => onBulkEditMode(!bulkEditMode)}
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
>
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
</button>
</div>
{bulkEditMode && (
<div className="file-list-bulk-actions">
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
{allSelected ? 'Deselect All' : 'Select All'}
</button>
{selectedHashIds.size > 0 && (
<span className="selection-count">
{selectedHashIds.size} selected
</span>
)}
</div>
)}
<div className="file-list-items">
{sortedResults.map((result) => {
const isSelected = bulkEditMode
? selectedHashIds.has(result.hashId)
: selectedHashId === result.hashId;
return (
<div
key={result.hashId}
className={`file-list-item ${isSelected ? 'selected' : ''}`}
onClick={() => {
if (bulkEditMode) {
onToggleSelection(result.hashId);
} else {
onSelectFile(result.hashId);
}
}}
>
{bulkEditMode && (
<div className="file-list-item-checkbox">
<input
type="checkbox"
checked={selectedHashIds.has(result.hashId)}
onChange={(e) => {
e.stopPropagation();
onToggleSelection(result.hashId);
}}
/>
</div>
)}
<div className="file-list-item-content">
<div className="file-list-item-title">
{getDisplayName(result.metadata)}
</div>
<div className="file-list-item-details">
<span className="file-list-item-type">
{result.metadata.videoType || 'Unknown'}
</span>
<span className="file-list-item-size">
{formatBytes(result.metadata.byteSize)}
</span>
</div>
<div className="file-list-item-hash">
Hash: {truncateHash(result.hashId)}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}
-247
View File
@@ -1,247 +0,0 @@
.kv-tree {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.1);
overflow: hidden;
}
.kv-tree-header {
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.kv-tree-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.8);
}
.kv-tree-info {
display: flex;
gap: 1rem;
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-refresh {
margin-left: auto;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
background-color: rgba(100, 108, 255, 0.2);
color: #646cff;
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.kv-tree-refresh:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.3);
}
.kv-tree-refresh:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-error {
padding: 0.5rem 1rem;
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
font-size: 0.8rem;
}
.kv-tree-content {
max-height: 400px;
overflow-y: auto;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 0.8rem;
}
.kv-tree-node {
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-node:last-child {
border-bottom: none;
}
.kv-tree-key {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-key:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-toggle {
width: 1rem;
color: rgba(255, 255, 255, 0.4);
font-size: 0.65rem;
}
.kv-tree-key-name {
flex: 1;
color: #88c0d0;
word-break: break-all;
}
.kv-tree-type {
padding: 0.1rem 0.4rem;
font-size: 0.65rem;
background-color: rgba(255, 255, 255, 0.1);
border-radius: 3px;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-value {
padding: 0.5rem 1rem 0.5rem 2.5rem;
background-color: rgba(0, 0, 0, 0.2);
border-top: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-fields {
width: 100%;
border-collapse: collapse;
}
.kv-tree-fields tr:hover {
background-color: rgba(255, 255, 255, 0.03);
}
.kv-field-name {
padding: 0.25rem 0.5rem 0.25rem 0;
color: #a3be8c;
white-space: nowrap;
vertical-align: top;
width: 1%;
}
.kv-field-value {
padding: 0.25rem 0;
color: rgba(255, 255, 255, 0.7);
word-break: break-all;
}
.kv-field-value pre {
margin: 0;
font-family: inherit;
white-space: pre-wrap;
font-size: 0.75rem;
}
.kv-tree-load-more {
display: block;
width: 100%;
padding: 0.75rem;
font-size: 0.8rem;
background-color: transparent;
color: #646cff;
border: none;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-empty {
padding: 2rem;
text-align: center;
color: rgba(255, 255, 255, 0.4);
}
/* Light mode */
@media (prefers-color-scheme: light) {
.kv-tree {
background-color: rgba(0, 0, 0, 0.02);
border-color: rgba(0, 0, 0, 0.1);
}
.kv-tree-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-header h3 {
color: rgba(0, 0, 0, 0.8);
}
.kv-tree-info {
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-error {
background-color: rgba(255, 0, 0, 0.08);
color: #cc0000;
}
.kv-tree-node {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-key:hover {
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-toggle {
color: rgba(0, 0, 0, 0.4);
}
.kv-tree-key-name {
color: #0077aa;
}
.kv-tree-type {
background-color: rgba(0, 0, 0, 0.08);
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-value {
background-color: rgba(0, 0, 0, 0.02);
border-top-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-fields tr:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.kv-field-name {
color: #228822;
}
.kv-field-value {
color: rgba(0, 0, 0, 0.7);
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-empty {
color: rgba(0, 0, 0, 0.4);
}
}
-216
View File
@@ -1,216 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import './KVTreeView.css';
interface KeyValue {
key: string;
type?: string;
value?: Record<string, any> | string;
expanded?: boolean;
loading?: boolean;
}
interface KVInfo {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage?: string;
}
const API_BASE = '/api/kv';
export function KVTreeView() {
const [keys, setKeys] = useState<KeyValue[]>([]);
const [cursor, setCursor] = useState<string>('0');
const [hasMore, setHasMore] = useState(true);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [info, setInfo] = useState<KVInfo | null>(null);
// Load KV info on mount
useEffect(() => {
fetch(`${API_BASE}/info`)
.then(res => res.json())
.then(data => setInfo(data))
.catch(err => console.error('Failed to load KV info:', err));
}, []);
// Load initial keys
const loadKeys = useCallback(async (reset = false) => {
if (loading) return;
const currentCursor = reset ? '0' : cursor;
if (!reset && currentCursor === '0' && keys.length > 0) return;
setLoading(true);
setError(null);
try {
const res = await fetch(`${API_BASE}/keys?cursor=${currentCursor}&count=50`);
if (!res.ok) throw new Error('Failed to load keys');
const data = await res.json();
const newKeys: KeyValue[] = data.keys.map((key: string) => ({
key,
expanded: false,
loading: false
}));
if (reset) {
setKeys(newKeys);
} else {
setKeys(prev => [...prev, ...newKeys]);
}
setCursor(data.cursor);
setHasMore(data.hasMore);
} catch (err: any) {
setError(err.message);
} finally {
setLoading(false);
}
}, [cursor, keys.length, loading]);
// Load on mount
useEffect(() => {
loadKeys(true);
}, []);
// Toggle key expansion and load value
const toggleKey = async (index: number) => {
const key = keys[index];
if (key.expanded) {
// Collapse
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: false } : k
));
return;
}
// Expand and load value if not already loaded
if (key.value === undefined) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: true } : k
));
try {
const res = await fetch(`${API_BASE}/key/${encodeURIComponent(key.key)}`);
if (!res.ok) throw new Error('Failed to load value');
const data = await res.json();
setKeys(prev => prev.map((k, i) =>
i === index ? {
...k,
type: data.type,
value: data.value,
expanded: true,
loading: false
} : k
));
} catch (err) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: false, value: 'Error loading value' } : k
));
}
} else {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: true } : k
));
}
};
const formatValue = (value: any): string => {
if (value === null) return 'null';
if (value === undefined) return 'undefined';
if (typeof value === 'object') {
return JSON.stringify(value, null, 2);
}
return String(value);
};
const formatSize = (bytes: number): string => {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
};
return (
<div className="kv-tree">
<div className="kv-tree-header">
<h3>KV Database Browser</h3>
{info && (
<div className="kv-tree-info">
<span>Keys: {info.keyCount}</span>
<span>Files: {formatSize(info.totalSize)}</span>
{info.memoryUsage && <span>Memory: {info.memoryUsage}</span>}
</div>
)}
<button
className="kv-tree-refresh"
onClick={() => loadKeys(true)}
disabled={loading}
>
{loading ? 'Loading...' : 'Refresh'}
</button>
</div>
{error && <div className="kv-tree-error">{error}</div>}
<div className="kv-tree-content">
{keys.map((item, index) => (
<div key={item.key} className="kv-tree-node">
<div
className={`kv-tree-key ${item.expanded ? 'expanded' : ''}`}
onClick={() => toggleKey(index)}
>
<span className="kv-tree-toggle">
{item.loading ? '...' : (item.expanded ? '▼' : '▶')}
</span>
<span className="kv-tree-key-name">{item.key}</span>
{item.type && <span className="kv-tree-type">{item.type}</span>}
</div>
{item.expanded && item.value && (
<div className="kv-tree-value">
{typeof item.value === 'object' ? (
<table className="kv-tree-fields">
<tbody>
{Object.entries(item.value).map(([field, val]) => (
<tr key={field}>
<td className="kv-field-name">{field}</td>
<td className="kv-field-value">
<pre>{formatValue(val)}</pre>
</td>
</tr>
))}
</tbody>
</table>
) : (
<pre>{formatValue(item.value)}</pre>
)}
</div>
)}
</div>
))}
{hasMore && (
<button
className="kv-tree-load-more"
onClick={() => loadKeys()}
disabled={loading}
>
{loading ? 'Loading...' : 'Load More'}
</button>
)}
{!loading && keys.length === 0 && (
<div className="kv-tree-empty">No keys found</div>
)}
</div>
</div>
);
}
@@ -8,15 +8,6 @@ for (const lang of iso6393) {
languagesByCode.set(lang.iso6393, lang);
}
// Get display name for a language code
export function getLanguageName(code: string): string {
const lang = languagesByCode.get(code);
if (lang) {
return lang.name;
}
return code.toUpperCase();
}
// Format for display: "English (eng)" or "eng" for unknown
export function formatLanguageOption(code: string): string {
const lang = languagesByCode.get(code);
-438
View File
@@ -1,438 +0,0 @@
.metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.metadata-editor-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-header h2 {
margin: 0;
font-size: 1.5rem;
}
.metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.metadata-editor-actions .download-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 1rem;
background-color: #28a745;
color: white;
border-radius: 4px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: background-color 0.2s;
}
.metadata-editor-actions .download-button:hover {
background-color: #218838;
}
.metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.metadata-editor-hashes {
margin-bottom: 1.5rem;
padding: 1rem;
background: rgba(0, 0, 0, 0.3);
border-radius: 8px;
border-left: 4px solid #888;
}
.hashes-header {
margin-bottom: 0.75rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hashes-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.7);
text-transform: uppercase;
letter-spacing: 0.05em;
}
.hashes-grid {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.hash-value {
display: flex;
flex-direction: column;
gap: 0.15rem;
}
.hash-value-primary {
margin-bottom: 0.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hash-value-primary .hash-label {
color: #4ecdc4;
}
.hash-value-primary .hash-code {
color: #4ecdc4;
font-size: 1rem;
}
.hash-label {
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.4);
text-transform: uppercase;
letter-spacing: 0.03em;
}
.hash-code {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.6);
word-break: break-all;
cursor: pointer;
transition: color 0.2s;
}
.hash-code:hover {
color: #fff;
}
/* Plugin Chips */
.metadata-editor-chips {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-bottom: 1.5rem;
}
.plugin-chip {
padding: 0.4rem 0.8rem;
border-radius: 16px;
font-size: 0.85rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
border: 2px solid var(--chip-color);
color: var(--chip-color);
background: transparent;
}
.plugin-chip:hover {
background-color: var(--chip-color);
color: white;
}
.plugin-chip.active {
color: white;
}
/* Plugin Accordion */
.plugin-accordion {
margin-bottom: 0.5rem;
border-radius: 8px;
overflow: hidden;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-accordion-header {
width: 100%;
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem 1.25rem;
border: none;
border-left: 4px solid #646cff;
background-color: rgba(255, 255, 255, 0.05);
color: inherit;
font-size: 1rem;
font-weight: 500;
cursor: pointer;
transition: background-color 0.2s ease;
}
.plugin-accordion-header:hover {
background-color: rgba(255, 255, 255, 0.1);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(255, 255, 255, 0.08);
}
.plugin-accordion-title {
color: inherit;
}
.plugin-accordion-icon {
font-size: 0.75rem;
opacity: 0.6;
transition: transform 0.2s ease;
}
.plugin-accordion-content {
padding: 1rem 1.25rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-form {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.metadata-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.metadata-field:last-child {
margin-bottom: 0;
}
.metadata-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.metadata-field input,
.metadata-field textarea,
.metadata-field select {
width: 100%;
}
.metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.metadata-field-row:last-child {
margin-bottom: 0;
}
.hash-field {
font-family: 'Courier New', monospace;
font-size: 0.9em;
}
.no-hash-info {
margin: 0;
padding: 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
padding: 3rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
.metadata-editor-error button {
margin-top: 1rem;
}
@media (max-width: 768px) {
.metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.metadata-editor-actions {
width: 100%;
}
.metadata-editor-actions button {
flex: 1;
}
.metadata-field-row {
grid-template-columns: 1fr;
}
}
@media (prefers-color-scheme: light) {
.metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-description {
color: rgba(0, 0, 0, 0.5);
}
.metadata-field label {
color: rgba(0, 0, 0, 0.7);
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
color: rgba(0, 0, 0, 0.5);
}
.no-hash-info {
color: rgba(0, 0, 0, 0.5);
}
.metadata-editor-hashes {
background: rgba(0, 0, 0, 0.05);
}
.hashes-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hashes-header h3 {
color: rgba(0, 0, 0, 0.7);
}
.hash-value-primary {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hash-value-primary .hash-label,
.hash-value-primary .hash-code {
color: #0a9e94;
}
.hash-label {
color: rgba(0, 0, 0, 0.4);
}
.hash-code {
color: rgba(0, 0, 0, 0.6);
}
.hash-code:hover {
color: #000;
}
/* Plugin Chips - Light theme */
.plugin-chip {
background: transparent;
}
.plugin-chip:hover,
.plugin-chip.active {
color: white;
}
/* Plugin Accordion - Light theme */
.plugin-accordion {
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.03);
}
.plugin-accordion-header:hover {
background-color: rgba(0, 0, 0, 0.06);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.05);
}
.plugin-accordion-content {
border-top-color: rgba(0, 0, 0, 0.1);
}
}
-392
View File
@@ -1,392 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema, PluginsResponse } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicPluginSection } from './DynamicPluginSection';
import './MetadataEditor.css';
interface PluginAccordionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (field: string, value: any) => void;
isExpanded: boolean;
onToggle: () => void;
}
function PluginAccordion({ plugin, metadata, onChange, isExpanded, onToggle }: PluginAccordionProps) {
return (
<div className={`plugin-accordion ${isExpanded ? 'expanded' : ''}`}>
<button
className="plugin-accordion-header"
onClick={onToggle}
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<span className="plugin-accordion-title">{plugin.name}</span>
<span className="plugin-accordion-icon">{isExpanded ? '▼' : '▶'}</span>
</button>
{isExpanded && (
<div className="plugin-accordion-content">
<DynamicPluginSection
plugin={plugin}
metadata={metadata}
onChange={onChange}
/>
</div>
)}
</div>
);
}
interface MetadataEditorProps {
hashId: string;
onSaved?: () => void;
}
export function MetadataEditor({ hashId, onSaved }: MetadataEditorProps) {
const [metadata, setMetadata] = useState<FileMetadata | null>(null);
const [editedMetadata, setEditedMetadata] = useState<FileMetadata | null>(null);
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
const [expandedPlugins, setExpandedPlugins] = useState<Set<string>>(new Set());
// Fetch plugins schema on mount
useEffect(() => {
fetchPlugins();
}, []);
// Fetch metadata when hashId changes
useEffect(() => {
loadMetadata();
}, [hashId]);
const fetchPlugins = async () => {
try {
const response = await fetch('/api/plugins');
if (response.ok) {
const data: PluginsResponse = await response.json();
// Sort by priority (lower = higher priority)
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
} catch (err) {
console.error('Failed to fetch plugin schemas:', err);
}
};
const loadMetadata = async () => {
try {
setIsLoading(true);
setError(null);
const data = await MetadataAPI.getMetadata(hashId);
setMetadata(data);
setEditedMetadata(data);
} catch (err: any) {
setError(err.message);
} finally {
setIsLoading(false);
}
};
const handleFieldChange = (field: string, value: any) => {
if (!editedMetadata) return;
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = field.split('/');
if (parts.length === 1) {
// Simple key
setEditedMetadata({
...editedMetadata,
[field]: value,
});
} else {
// Nested key - need to update nested structure
const newMetadata = { ...editedMetadata };
let current: any = newMetadata;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
} else {
current[parts[i]] = { ...current[parts[i]] };
}
current = current[parts[i]];
}
current[parts[parts.length - 1]] = value;
setEditedMetadata(newMetadata);
}
};
const handleSave = async () => {
if (!editedMetadata) return;
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
await MetadataAPI.updateMetadata(hashId, editedMetadata);
setMetadata(editedMetadata);
setSuccessMessage('Metadata saved successfully!');
if (onSaved) {
onSaved();
}
// Clear success message after 3 seconds
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleReset = () => {
setEditedMetadata(metadata);
setError(null);
setSuccessMessage(null);
};
const togglePlugin = (pluginId: string) => {
setExpandedPlugins(prev => {
const next = new Set(prev);
if (next.has(pluginId)) {
next.delete(pluginId);
} else {
next.add(pluginId);
}
return next;
});
};
const scrollToPlugin = (pluginId: string) => {
// Expand the plugin if not already expanded
if (!expandedPlugins.has(pluginId)) {
setExpandedPlugins(prev => new Set(prev).add(pluginId));
}
// Scroll to the plugin after a short delay to allow expansion
setTimeout(() => {
const element = document.getElementById(`plugin-${pluginId}`);
if (element) {
element.scrollIntoView({ behavior: 'smooth', block: 'start' });
}
}, 100);
};
if (isLoading) {
return <div className="metadata-editor-loading">Loading metadata...</div>;
}
if (error && !metadata) {
return (
<div className="metadata-editor-error">
<p>Error loading metadata: {error}</p>
<button onClick={loadMetadata}>Retry</button>
</div>
);
}
if (!metadata || !editedMetadata) {
return <div className="metadata-editor-empty">No metadata available</div>;
}
const hasChanges = JSON.stringify(metadata) !== JSON.stringify(editedMetadata);
// Filter plugins that have data
const pluginsWithData = plugins.filter(plugin => {
if (!plugin.metadataSchema) return false;
return hasPluginData(plugin, editedMetadata);
});
// Get file path for download
const filePath = editedMetadata?.filePath as string | undefined;
const downloadUrl = filePath ? `/api/file/download?path=${encodeURIComponent(filePath)}` : null;
return (
<div className="metadata-editor">
<div className="metadata-editor-header">
<h2>Edit Metadata</h2>
<div className="metadata-editor-actions">
{downloadUrl && (
<a
href={downloadUrl}
className="download-button"
title={`Download: ${filePath}`}
download
>
⬇ Download
</a>
)}
{hasChanges && (
<button onClick={handleReset} disabled={isSaving}>
Reset
</button>
)}
<button
onClick={handleSave}
disabled={isSaving || !hasChanges}
className="primary"
>
{isSaving ? 'Saving...' : 'Save Changes'}
</button>
</div>
</div>
{successMessage && (
<div className="metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="metadata-editor-message error">{error}</div>}
<div className="metadata-editor-hashes">
<div className="hashes-header">
<h3>Content Identifiers</h3>
</div>
<div className="hashes-grid">
<HashValue label="midhash256" value={hashId} primary />
{getHashValues(editedMetadata).map(({ key, label, value }) => (
<HashValue key={key} label={label} value={value} />
))}
</div>
</div>
{pluginsWithData.length > 0 && (
<div className="metadata-editor-chips">
{pluginsWithData.map((plugin) => (
<button
key={plugin.id}
className={`plugin-chip ${expandedPlugins.has(plugin.id) ? 'active' : ''}`}
style={{
'--chip-color': plugin.color || '#646cff',
backgroundColor: expandedPlugins.has(plugin.id) ? plugin.color || '#646cff' : 'transparent'
} as React.CSSProperties}
onClick={() => scrollToPlugin(plugin.id)}
>
{plugin.name}
</button>
))}
</div>
)}
<div className="metadata-editor-form">
{pluginsWithData.length === 0 ? (
<div className="metadata-editor-empty">
No metadata found for this file
</div>
) : (
pluginsWithData.map((plugin) => (
<div key={plugin.id} id={`plugin-${plugin.id}`}>
<PluginAccordion
plugin={plugin}
metadata={editedMetadata}
onChange={handleFieldChange}
isExpanded={expandedPlugins.has(plugin.id)}
onToggle={() => togglePlugin(plugin.id)}
/>
</div>
))
)}
</div>
</div>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function hasPluginData(plugin: PluginSchema, metadata: FileMetadata): boolean {
const schema = plugin.metadataSchema;
if (!schema) return false;
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
/**
* Extract all hash values (cid_*) from metadata
*/
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
const hashOrder = [
{ key: 'cid_sha2-256', label: 'SHA-256' },
{ key: 'cid_sha3-256', label: 'SHA3-256' },
{ key: 'cid_sha3-384', label: 'SHA3-384' },
{ key: 'cid_sha1', label: 'SHA-1' },
{ key: 'cid_md5', label: 'MD5' },
{ key: 'cid_crc32', label: 'CRC32' },
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
];
return hashOrder
.filter(({ key }) => (metadata as any)[key])
.map(({ key, label }) => ({
key,
label,
value: (metadata as any)[key],
}));
}
/**
* Display a single hash value with copy functionality
*/
function HashValue({ label, value, primary }: { label: string; value: string; primary?: boolean }) {
const handleCopy = () => {
navigator.clipboard.writeText(value);
};
return (
<div className={`hash-value ${primary ? 'hash-value-primary' : ''}`}>
<span className="hash-label">{label}</span>
<code className="hash-code" onClick={handleCopy} title="Click to copy">
{value}
</code>
</div>
);
}
-52
View File
@@ -1,52 +0,0 @@
.search-bar {
width: 100%;
padding: 1.5rem;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
margin-bottom: 1rem;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
}
.search-input {
flex: 1;
min-width: 200px;
}
.search-input.unified-search {
font-size: 1rem;
}
.search-bar button {
flex: 0 0 auto;
min-width: 100px;
}
.search-hint {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
@media (max-width: 768px) {
.search-controls {
flex-direction: column;
}
.search-input,
.search-bar button {
width: 100%;
}
}
@media (prefers-color-scheme: light) {
.search-hint {
color: rgba(0, 0, 0, 0.5);
}
}
-40
View File
@@ -1,40 +0,0 @@
import { useState } from 'react';
import './SearchBar.css';
interface SearchBarProps {
onSearch: (query: string) => void;
isLoading: boolean;
}
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
const [query, setQuery] = useState('');
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (query.trim()) {
onSearch(query.trim());
}
};
return (
<form className="search-bar" onSubmit={handleSubmit}>
<div className="search-controls">
<input
type="text"
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="Search by title, filename, filepath, or hash ID..."
className="search-input unified-search"
disabled={isLoading}
/>
<button type="submit" disabled={isLoading || !query.trim()}>
{isLoading ? 'Searching...' : 'Search'}
</button>
</div>
<p className="search-hint">
Search across all fields: title, filename, file path, and hash ID
</p>
</form>
);
}
+209
View File
@@ -0,0 +1,209 @@
.snapshot-panel {
max-width: 920px;
margin: 0 auto;
padding: 1.5rem;
}
.snapshot-panel h2 {
margin: 0 0 0.5rem;
}
.snapshot-intro {
color: var(--text-muted, #888);
margin-bottom: 1.5rem;
}
.snapshot-section {
border: 1px solid var(--border, #e0e0e0);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 1.25rem;
}
.snapshot-section.danger {
border-color: #d04a4a;
background: rgba(208, 74, 74, 0.04);
}
.snapshot-section h3 {
margin: 0 0 0.5rem;
}
.snapshot-row {
display: flex;
gap: 1rem;
align-items: flex-start;
margin: 0.75rem 0;
flex-wrap: wrap;
}
.snapshot-file {
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.snapshot-file span {
font-size: 0.85rem;
color: var(--text-muted, #888);
}
.snapshot-fieldset {
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.5rem 0.75rem;
min-width: 280px;
}
.snapshot-fieldset[disabled] {
opacity: 0.5;
}
.snapshot-fieldset legend {
font-weight: 600;
font-size: 0.85rem;
padding: 0 0.25rem;
}
.snapshot-fieldset label {
display: block;
margin: 0.25rem 0;
font-size: 0.9rem;
cursor: pointer;
}
.snapshot-fieldset input[type='radio'] {
margin-right: 0.5rem;
}
.snapshot-btn {
background: var(--bg-button, #f0f0f0);
border: 1px solid var(--border, #c0c0c0);
color: var(--text, #222);
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-btn:hover:not(:disabled) {
background: var(--bg-button-hover, #e0e0e0);
}
.snapshot-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.snapshot-btn.primary {
background: #2a6df4;
border-color: #2a6df4;
color: white;
}
.snapshot-btn.primary:hover:not(:disabled) {
background: #1a5de4;
}
.snapshot-btn.danger {
background: #d04a4a;
border-color: #d04a4a;
color: white;
}
.snapshot-btn.danger:hover:not(:disabled) {
background: #b83838;
}
.snapshot-error {
background: rgba(208, 74, 74, 0.1);
color: #b83838;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-info {
background: rgba(42, 109, 244, 0.08);
color: #1a5de4;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-result {
margin-top: 0.75rem;
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.75rem;
}
.snapshot-result-summary {
display: flex;
gap: 1rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
font-size: 0.9rem;
}
.snapshot-result-table {
width: 100%;
border-collapse: collapse;
font-size: 0.9rem;
}
.snapshot-result-table th,
.snapshot-result-table td {
border: 1px solid var(--border, #e0e0e0);
padding: 0.35rem 0.5rem;
text-align: right;
}
.snapshot-result-table th {
background: var(--bg-subtle, #f5f5f5);
font-weight: 600;
}
.snapshot-result-errors {
margin-top: 0.5rem;
font-size: 0.85rem;
}
.snapshot-result-errors summary {
cursor: pointer;
color: #b83838;
}
.snapshot-result-errors ul {
margin: 0.5rem 0 0;
padding-left: 1.25rem;
}
.snapshot-result-errors code {
font-size: 0.8rem;
word-break: break-all;
}
.snapshot-checkbox {
display: flex;
align-items: flex-start;
gap: 0.5rem;
margin: 0.5rem 0 0.75rem;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-checkbox input {
margin-top: 0.2rem;
}
.snapshot-hint {
color: var(--text-muted, #888);
font-size: 0.9rem;
}
.snapshot-result-aside {
margin-top: 0.5rem;
font-size: 0.85rem;
color: var(--text-muted, #555);
}
+271
View File
@@ -0,0 +1,271 @@
import { useState } from 'react';
import {
SnapshotAPI,
ImportMode,
ImportConflict,
ImportResult,
} from '../api/snapshotApi';
import './SnapshotPanel.css';
export function SnapshotPanel() {
const [exportPluginOutput, setExportPluginOutput] = useState(false);
const [importFile, setImportFile] = useState<File | null>(null);
const [importMode, setImportMode] = useState<ImportMode>('merge');
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
const [importing, setImporting] = useState(false);
const [importResult, setImportResult] = useState<ImportResult | null>(null);
const [importError, setImportError] = useState<string | null>(null);
const [wipeBusy, setWipeBusy] = useState(false);
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
const runImport = async (dryRun: boolean) => {
if (!importFile) {
setImportError('Pick a snapshot ZIP first.');
return;
}
setImporting(true);
setImportError(null);
setImportResult(null);
try {
const result = await SnapshotAPI.runImport(importFile, {
mode: importMode,
conflict: importMode === 'merge' ? importConflict : undefined,
dryRun,
});
setImportResult(result);
} catch (e: any) {
setImportError(e.message || String(e));
} finally {
setImporting(false);
}
};
const runWipe = async () => {
const ok = window.confirm(
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
'edit you made via this editor will be lost. Continue?'
);
if (!ok) return;
setWipeBusy(true);
setWipeMessage(null);
try {
const r = await SnapshotAPI.wipe('metadata');
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
} catch (e: any) {
setWipeMessage(`Failed: ${e.message || e}`);
} finally {
setWipeBusy(false);
}
};
return (
<div className="snapshot-panel">
<h2>Snapshot</h2>
<p className="snapshot-intro">
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
archives — one nested-JSON file per CID, plus a manifest.
</p>
<section className="snapshot-section">
<h3>Export</h3>
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
<label className="snapshot-checkbox">
<input
type="checkbox"
checked={exportPluginOutput}
onChange={(e) => setExportPluginOutput(e.target.checked)}
/>
<span>
Include plugin output
<span className="snapshot-hint">
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
</span>
</span>
</label>
<button
className="snapshot-btn primary"
onClick={() =>
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
}
>
Download snapshot.zip
</button>
</section>
<section className="snapshot-section">
<h3>Import</h3>
<div className="snapshot-row">
<label className="snapshot-file">
<span>Snapshot ZIP</span>
<input
type="file"
accept=".zip,application/zip"
onChange={(e) => {
setImportFile(e.target.files?.[0] ?? null);
setImportResult(null);
setImportError(null);
}}
/>
</label>
</div>
<div className="snapshot-row">
<fieldset className="snapshot-fieldset">
<legend>Mode</legend>
<label>
<input
type="radio"
name="mode"
value="merge"
checked={importMode === 'merge'}
onChange={() => setImportMode('merge')}
/>
Merge — keep existing fields not in the snapshot
</label>
<label>
<input
type="radio"
name="mode"
value="replace"
checked={importMode === 'replace'}
onChange={() => setImportMode('replace')}
/>
Replace — wipe each imported CID and rewrite from snapshot
</label>
</fieldset>
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
<legend>Conflict</legend>
<label>
<input
type="radio"
name="conflict"
value="source"
checked={importConflict === 'source'}
onChange={() => setImportConflict('source')}
/>
Source wins (overwrite my edits)
</label>
<label>
<input
type="radio"
name="conflict"
value="mine"
checked={importConflict === 'mine'}
onChange={() => setImportConflict('mine')}
/>
Mine wins (keep my edits, fill in only missing fields)
</label>
</fieldset>
</div>
<div className="snapshot-row">
<button
className="snapshot-btn"
disabled={!importFile || importing}
onClick={() => runImport(true)}
>
Dry run
</button>
<button
className="snapshot-btn primary"
disabled={!importFile || importing}
onClick={() => runImport(false)}
>
{importing ? 'Importing…' : 'Apply import'}
</button>
</div>
{importError && <div className="snapshot-error">{importError}</div>}
{importResult && <ImportResultView result={importResult} />}
</section>
<section className="snapshot-section danger">
<h3>Wipe metadata</h3>
<p>
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
Files on disk are untouched.
</p>
<button
className="snapshot-btn danger"
disabled={wipeBusy}
onClick={runWipe}
>
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
</button>
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
</section>
</div>
);
}
function ImportResultView({ result }: { result: ImportResult }) {
const totals = (result.files ?? []).reduce(
(acc, f) => {
acc.added += f.added;
acc.updated += f.updated;
acc.unchanged += f.unchanged;
acc.kept += f.kept;
acc.deleted += f.deleted;
return acc;
},
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
);
return (
<div className="snapshot-result">
<div className="snapshot-result-summary">
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
<span>
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
</span>
<span>
mode={result.mode}
{result.conflict ? `, conflict=${result.conflict}` : ''}
</span>
</div>
<table className="snapshot-result-table">
<thead>
<tr>
<th>added</th>
<th>updated</th>
<th>unchanged</th>
<th>kept (mine)</th>
<th>deleted (replace)</th>
</tr>
</thead>
<tbody>
<tr>
<td>{totals.added}</td>
<td>{totals.updated}</td>
<td>{totals.unchanged}</td>
<td>{totals.kept}</td>
<td>{totals.deleted}</td>
</tr>
</tbody>
</table>
{result.pluginOutput && (
<div className="snapshot-result-aside">
plugin-output: {result.pluginOutput.written} written
{result.pluginOutput.skipped > 0
? `, ${result.pluginOutput.skipped} skipped (existed)`
: ''}
</div>
)}
{result.files && result.files.some((f) => f.error) && (
<details className="snapshot-result-errors">
<summary>Per-file errors</summary>
<ul>
{result.files
.filter((f) => f.error)
.map((f) => (
<li key={f.hashId}>
<code>{f.hashId}</code>: {f.error}
</li>
))}
</ul>
</details>
)}
</div>
);
}
+233
View File
@@ -0,0 +1,233 @@
import { useEffect, useMemo, useState } from 'react';
import { KVAPI, KVFindMatch } from '../api/kvApi';
interface Props {
matches: KVFindMatch[];
truncated: boolean;
searching: boolean;
selectedKey: string | null;
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
// FilteredKVTree is the search-active counterpart to KVTree. Instead of
// replacing the tree with a flat results list, it shows only the file
// branches that contain matches, with the matched (field, value) pairs
// rendered inline as sub-lines under each branch.
//
// Branches remain expandable: clicking one fetches the full subtree via
// KVAPI.treeAll just like the main tree, so the user can drill into every
// key of a matching file without clearing the search first.
export function FilteredKVTree({
matches, truncated, searching,
selectedKey, expandedPaths,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: Props) {
// Group matches by their entryPath (file:<hash>/) so each entry is one row
// with potentially multiple hit previews.
const groups = useMemo(() => {
const m = new Map<string, KVFindMatch[]>();
for (const x of matches) {
const arr = m.get(x.entryPath);
if (arr) arr.push(x);
else m.set(x.entryPath, [x]);
}
return m;
}, [matches]);
return (
<div className="kv-filtered">
<div className="kv-search-header">
{searching
? 'searching…'
: `${matches.length} match${matches.length === 1 ? '' : 'es'} in ${groups.size} file${groups.size === 1 ? '' : 's'}${truncated ? ' (truncated)' : ''}`}
</div>
{[...groups.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([entryPath, hits]) => (
<FilteredEntry
key={entryPath}
entryPath={entryPath}
hits={hits}
selectedKey={selectedKey}
expanded={expandedPaths.has(entryPath)}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
))}
</div>
);
}
interface EntryProps {
entryPath: string;
hits: KVFindMatch[];
selectedKey: string | null;
expanded: boolean;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function FilteredEntry({
entryPath, hits, selectedKey, expanded,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: EntryProps) {
// We deliberately do NOT mark the matched-leaf rows as "selected" via the
// tree row's selected style — the inline hit lines are themselves the
// selected indicator (they're highlighted when their key is current).
const isBranchSelected = entryPath === selectedKey;
const HIT_PREVIEW = 4;
return (
<div className="kv-filtered-entry">
<div
className={`kv-tree-row kv-tree-branch${isBranchSelected ? ' selected' : ''}`}
style={{ paddingLeft: 4 }}
onClick={() => {
onToggleBranch(entryPath, !expanded);
onSelectBranch(entryPath);
}}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{entryPath}</span>
<span className="kv-filtered-count" title={`${hits.length} match${hits.length === 1 ? '' : 'es'}`}>
{hits.length}
</span>
</div>
<div className="kv-filtered-hits">
{hits.slice(0, HIT_PREVIEW).map((h) => (
<div
key={h.key}
className={`kv-filtered-hit${h.key === selectedKey ? ' selected' : ''}`}
onClick={(e) => { e.stopPropagation(); onSelectLeaf(h.key); }}
title={h.value}
>
<span className="kv-filtered-hit-field">{h.field}</span>
<span className="kv-filtered-hit-value">{h.value}</span>
</div>
))}
{hits.length > HIT_PREVIEW && (
<div className="kv-filtered-more">
+{hits.length - HIT_PREVIEW} more match{hits.length - HIT_PREVIEW === 1 ? '' : 'es'}
{!expanded && ' · expand to see all keys'}
</div>
)}
</div>
{expanded && (
<LazySubtree
path={entryPath}
depth={1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</div>
);
}
interface SubtreeProps {
path: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
interface SubtreeData {
loading: boolean;
branches: string[];
leaves: string[];
error?: string;
}
// LazySubtree fetches its children once on mount and renders them. Sub-
// branches are themselves LazyBranch components that fetch their own
// children on first expand — recursion all the way down without the global
// expandedPaths state of the main KVTree.
function LazySubtree({ path, depth, selectedKey, onSelectLeaf, onSelectBranch }: SubtreeProps) {
const [data, setData] = useState<SubtreeData>({ loading: true, branches: [], leaves: [] });
useEffect(() => {
let cancelled = false;
KVAPI.treeAll(path, '/', 5000)
.then((r) => { if (!cancelled) setData({ loading: false, branches: r.branches, leaves: r.leaves }); })
.catch((e) => { if (!cancelled) setData({ loading: false, branches: [], leaves: [], error: e.message }); });
return () => { cancelled = true; };
}, [path]);
if (data.loading) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (data.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>{data.error}</div>;
}
return (
<>
{data.branches.map((b) => (
<LazyBranch
key={b}
path={b}
parentPath={path}
depth={depth}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
))}
{data.leaves.map((l) => (
<div
key={l}
className={`kv-tree-row kv-tree-leaf${l === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(l)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{l.slice(path.length)}</span>
</div>
))}
</>
);
}
interface LazyBranchProps {
path: string;
parentPath: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
function LazyBranch({ path, parentPath, depth, selectedKey, onSelectLeaf, onSelectBranch }: LazyBranchProps) {
const [expanded, setExpanded] = useState(false);
const label = path.slice(parentPath.length);
return (
<>
<div
className={`kv-tree-row kv-tree-branch${path === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => { setExpanded(!expanded); onSelectBranch(path); }}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{label}</span>
</div>
{expanded && (
<LazySubtree
path={path}
depth={depth + 1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</>
);
}
+896
View File
@@ -0,0 +1,896 @@
/* KV Tree Editor — fills the iframe viewport with a 2-pane layout. */
.kv-app {
display: flex;
flex-direction: column;
height: 100vh;
width: 100%;
overflow: hidden;
background: #fafbfc;
color: #1f2330;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
font-size: 13px;
}
/* ── Header ───────────────────────────────────────────────────────────── */
.kv-app-header {
border-bottom: 1px solid #e2e6ee;
padding: 8px 12px;
background: #fff;
flex: 0 0 auto;
}
.kv-app-search {
display: flex;
align-items: center;
gap: 8px;
}
.kv-search-input {
flex: 1;
padding: 7px 11px;
border: 1px solid #d1d6e0;
border-radius: 6px;
font-size: 13px;
background: #fff;
outline: none;
}
.kv-search-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-app-stats {
font-size: 12px;
color: #8089a0;
white-space: nowrap;
}
.kv-app-breadcrumb {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 2px;
font-size: 12px;
}
.kv-crumb-label {
color: #8089a0;
margin-right: 4px;
}
.kv-crumb {
border: none;
background: transparent;
padding: 1px 4px;
border-radius: 3px;
cursor: pointer;
color: #4a5170;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-crumb:hover { background: #eef0fb; color: #2a2f48; }
/* ── Body ─────────────────────────────────────────────────────────────── */
.kv-app-body {
flex: 1 1 auto;
display: flex;
min-height: 0;
}
.kv-app-left {
flex: 0 0 38%;
min-width: 280px;
max-width: 600px;
border-right: 1px solid #e2e6ee;
background: #fff;
overflow: auto;
}
.kv-app-right {
flex: 1 1 auto;
overflow: auto;
padding: 16px 20px;
background: #fafbfc;
}
/* ── Tree ─────────────────────────────────────────────────────────────── */
.kv-tree {
padding: 4px 0;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-tree-list {
list-style: none;
margin: 0;
padding: 0;
}
.kv-tree-row {
display: flex;
align-items: center;
gap: 4px;
padding: 2px 4px;
cursor: pointer;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-row:hover { background: #f3f5fb; }
.kv-tree-toggle {
display: inline-block;
width: 12px;
color: #8089a0;
font-size: 10px;
}
.kv-tree-leaf-dot {
display: inline-block;
width: 12px;
color: #c5cad6;
}
.kv-tree-label {
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-branch .kv-tree-label {
color: #2a3050;
font-weight: 500;
}
.kv-tree-branch .kv-tree-filename {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85em;
color: #1e3a8a;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
}
.kv-tree-hash-pill {
margin-left: 0.5rem;
padding: 0.05rem 0.4rem;
background: rgba(100, 108, 255, 0.12);
color: #4a50c8;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.7em;
border-radius: 999px;
flex-shrink: 0;
}
.kv-tree-leaf .kv-tree-label {
color: #4a5170;
}
.kv-tree-row.selected {
background: #e7eafd;
color: #2532a3;
}
.kv-tree-row.selected:hover { background: #dde2fc; }
.kv-tree-loading,
.kv-tree-empty,
.kv-tree-error {
font-size: 12px;
color: #8089a0;
padding: 4px 8px;
font-style: italic;
}
.kv-tree-error { color: #c0392b; }
/* ── Right pane (leaf / branch) ───────────────────────────────────────── */
.kv-leaf, .kv-branch {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-leaf-key {
display: flex;
align-items: center;
gap: 6px;
background: #f0f2f8;
border: 1px solid #e2e6ee;
border-radius: 6px;
padding: 8px 10px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
}
.kv-leaf-key-text {
flex: 1;
word-break: break-all;
color: #2a3050;
}
.kv-leaf-meta {
display: flex;
align-items: center;
gap: 8px;
}
.kv-leaf-label {
font-size: 12px;
color: #6e7790;
font-weight: 500;
}
.kv-leaf-type {
border: 1px solid #d1d6e0;
border-radius: 5px;
padding: 4px 8px;
font-size: 12px;
background: #fff;
}
.kv-leaf-hint {
font-size: 12px;
color: #6e7790;
font-style: italic;
}
.kv-leaf-input {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
box-sizing: border-box;
}
.kv-leaf-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-textarea {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
resize: vertical;
box-sizing: border-box;
font-family: inherit;
}
.kv-leaf-textarea:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-mono {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-url {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-url-open {
font-size: 12px;
color: #6c7afa;
text-decoration: none;
white-space: nowrap;
}
.kv-leaf-actions {
display: flex;
gap: 8px;
}
/* ── CID leaf ─────────────────────────────────────────────────────────── */
.kv-leaf-cid {
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-leaf-cid-controls {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-cid-actions {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
}
.kv-leaf-cid-meta {
font-size: 12px;
color: #6b7280;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-cid-preview {
max-width: 100%;
max-height: 320px;
border-radius: 6px;
border: 1px solid #e2e6ee;
background: #f5f6fa;
object-fit: contain;
align-self: flex-start;
}
.kv-leaf-cid-preview-media {
width: 100%;
max-width: 560px;
}
/* ── File-detail view ─────────────────────────────────────────────────── */
.kv-file {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-file-header {
padding: 14px 16px;
background: #fff;
border: 1px solid #e2e6ee;
border-radius: 8px;
display: grid;
grid-template-columns: 1fr auto;
grid-template-areas:
"name actions"
"meta actions";
gap: 6px 16px;
align-items: center;
}
.kv-file-name {
grid-area: name;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-weight: 600;
font-size: 14px;
color: #1e3a8a;
word-break: break-all;
}
.kv-file-name-fallback {
color: #9ca3af;
font-style: italic;
}
.kv-file-meta {
grid-area: meta;
display: flex;
gap: 12px;
align-items: baseline;
font-size: 12px;
color: #6b7280;
min-width: 0;
}
.kv-file-hash {
background: rgba(100, 122, 250, 0.1);
color: #4a50c8;
padding: 1px 6px;
border-radius: 999px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
flex-shrink: 0;
}
.kv-file-path {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
}
.kv-file-actions {
grid-area: actions;
display: flex;
gap: 8px;
align-items: center;
}
.kv-file-rows {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row {
display: grid;
grid-template-columns: minmax(140px, 220px) 1fr auto;
gap: 12px;
padding: 8px 12px;
background: #fff;
border: 1px solid #e2e6ee;
border-left: 3px solid #d6deed;
border-radius: 6px;
align-items: start;
}
.kv-file-row.pending-delete {
border-left-color: #dc2626;
background: #fef2f2;
}
.kv-file-row.pending-delete .kv-file-row-label {
text-decoration: line-through;
color: #9ca3af;
}
.kv-file-row-deleted-note {
color: #b91c1c;
font-style: italic;
font-size: 12px;
padding: 6px 0;
}
.kv-file-row-delete {
width: 24px;
height: 24px;
align-self: center;
display: inline-flex;
align-items: center;
justify-content: center;
background: transparent;
border: 1px solid transparent;
border-radius: 4px;
color: #dc2626;
font-size: 16px;
line-height: 1;
cursor: pointer;
opacity: 0.4;
transition: opacity 0.15s, background 0.15s, border-color 0.15s;
}
.kv-file-row:hover .kv-file-row-delete,
.kv-file-row.pending-delete .kv-file-row-delete {
opacity: 1;
}
.kv-file-row-delete:hover {
background: rgba(220, 38, 38, 0.1);
border-color: rgba(220, 38, 38, 0.3);
}
.kv-file-add {
display: flex;
gap: 8px;
align-items: center;
padding: 10px 12px;
background: rgba(100, 122, 250, 0.05);
border: 1px dashed rgba(100, 122, 250, 0.3);
border-radius: 6px;
}
.kv-file-add-prefix {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
color: #6b7280;
white-space: nowrap;
flex-shrink: 0;
max-width: 220px;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-file-add > input {
flex: 1;
}
.kv-file-add > input:first-of-type {
flex: 0 0 200px;
}
.kv-file-row.dirty {
border-left-color: #fbbf24;
background: #fffbeb;
}
.kv-file-row.pulsed {
animation: kv-row-pulse 1.4s ease-out;
}
@keyframes kv-row-pulse {
0% { background: rgba(100, 122, 250, 0.35); border-left-color: #4a50c8; }
100% { background: transparent; }
}
.kv-file-row-label,
.kv-file-row-label-static {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
background: none;
border: none;
text-align: left;
cursor: pointer;
padding: 6px 0;
word-break: break-all;
}
.kv-file-row-label-static {
cursor: default;
}
.kv-file-row-label:hover {
color: #4a50c8;
text-decoration: underline;
}
.kv-file-row-value {
display: flex;
min-width: 0;
}
.kv-file-row-value > * {
flex: 1;
min-width: 0;
}
.kv-file-group {
display: flex;
flex-direction: column;
border: 1px solid #e2e6ee;
border-radius: 8px;
background: rgba(100, 122, 250, 0.03);
overflow: hidden;
}
.kv-file-group-header {
display: flex;
align-items: center;
gap: 8px;
padding: 6px 12px;
border: none;
background: rgba(100, 122, 250, 0.1);
cursor: pointer;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
}
.kv-file-group-header:hover {
background: rgba(100, 122, 250, 0.2);
}
.kv-file-group-chevron {
font-size: 10px;
color: #6b7280;
}
.kv-file-group-name {
font-weight: 600;
color: #1e3a8a;
}
.kv-file-group-count {
margin-left: auto;
padding: 1px 8px;
background: rgba(100, 122, 250, 0.2);
color: #4a50c8;
border-radius: 999px;
font-size: 11px;
}
.kv-file-group-body {
padding: 8px;
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row-lang .kv-file-lang-entries {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-lang-entry {
display: flex;
gap: 6px;
align-items: center;
}
.kv-file-lang-entry > .kv-leaf-input {
flex: 1;
}
.kv-file-lang-remove {
color: #dc2626;
}
.kv-file-lang-add {
align-self: flex-start;
background: rgba(100, 122, 250, 0.1);
}
/* ── DateTime leaf ────────────────────────────────────────────────────── */
.kv-leaf-datetime {
display: flex;
gap: 8px;
align-items: center;
}
.kv-leaf-datetime-shape {
font-size: 11px;
color: #6b7280;
background: rgba(100, 122, 250, 0.1);
padding: 2px 6px;
border-radius: 999px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
white-space: nowrap;
}
/* ── Branch ───────────────────────────────────────────────────────────── */
.kv-branch-section {
border: 1px solid #e2e6ee;
border-radius: 6px;
background: #fff;
padding: 12px 14px;
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-branch-section h4 {
margin: 0;
font-size: 13px;
color: #2a3050;
}
.kv-branch-row {
display: flex;
align-items: center;
gap: 6px;
}
.kv-branch-prefix {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
color: #6e7790;
background: #f0f2f8;
padding: 6px 8px;
border-radius: 4px;
white-space: nowrap;
max-width: 50%;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-branch-note {
margin: 0;
font-size: 12px;
color: #6e7790;
line-height: 1.4;
}
/* ── Buttons ──────────────────────────────────────────────────────────── */
.kv-btn {
border: 1px solid #d1d6e0;
background: #fff;
color: #2a3050;
padding: 6px 14px;
border-radius: 5px;
font-size: 12.5px;
cursor: pointer;
}
.kv-btn:hover:not(:disabled) { background: #f3f5fb; border-color: #b8c0d0; }
.kv-btn:disabled { opacity: 0.45; cursor: not-allowed; }
.kv-btn-primary {
background: #6c7afa;
border-color: #6c7afa;
color: #fff;
}
.kv-btn-primary:hover:not(:disabled) {
background: #5868e8;
border-color: #5868e8;
}
.kv-btn-danger {
border-color: #e0788a;
color: #c0392b;
}
.kv-btn-danger:hover:not(:disabled) {
background: #fdecef;
border-color: #c0392b;
}
.kv-btn-icon {
border: 1px solid transparent;
background: transparent;
color: #6e7790;
padding: 2px 6px;
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.kv-btn-icon:hover { background: #eef0fb; color: #2a3050; }
/* ── Pane status ──────────────────────────────────────────────────────── */
.kv-pane-status {
color: #6e7790;
font-size: 12.5px;
padding: 8px 0;
}
.kv-pane-empty {
text-align: center;
padding: 80px 20px;
font-size: 14px;
}
.kv-pane-hint {
margin-top: 8px;
font-size: 12px;
color: #8089a0;
}
/* ── Field chips (configure search-in fields) ─────────────────────────── */
.kv-chips {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 4px;
}
.kv-chips-label {
font-size: 12px;
color: #8089a0;
margin-right: 4px;
}
.kv-chip {
display: inline-flex;
align-items: center;
gap: 4px;
background: #eef0fb;
border: 1px solid #d5dafa;
color: #2a3050;
border-radius: 12px;
padding: 1px 4px 1px 8px;
font-size: 11.5px;
}
.kv-chip code {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11.5px;
}
.kv-chip-x {
border: none;
background: transparent;
color: #6e7790;
cursor: pointer;
padding: 0 4px;
font-size: 14px;
line-height: 1;
}
.kv-chip-x:hover { color: #c0392b; }
.kv-chip-input {
border: 1px dashed #c5cad6;
background: transparent;
border-radius: 12px;
padding: 2px 8px;
font-size: 11.5px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
outline: none;
min-width: 90px;
}
.kv-chip-input:focus { border-style: solid; border-color: #6c7afa; }
/* ── Search header (shared across filter mode) ────────────────────────── */
.kv-search-header {
padding: 6px 10px;
font-size: 11px;
color: #6e7790;
background: #f3f5fb;
border-bottom: 1px solid #e2e6ee;
position: sticky;
top: 0;
z-index: 1;
}
/* ── Filtered tree (search-active state) ──────────────────────────────── */
.kv-filtered {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-filtered-entry {
border-bottom: 1px solid #f0f2f8;
}
.kv-filtered-count {
margin-left: auto;
background: #eef0fb;
color: #4a5170;
padding: 0 6px;
border-radius: 9px;
font-size: 10.5px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-weight: 600;
}
.kv-filtered-hits {
padding: 2px 4px 4px 24px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 12px;
}
.kv-filtered-hit {
display: flex;
gap: 6px;
align-items: baseline;
padding: 2px 6px;
border-radius: 3px;
cursor: pointer;
line-height: 1.35;
}
.kv-filtered-hit:hover { background: #f3f5fb; }
.kv-filtered-hit.selected {
background: #e7eafd;
}
.kv-filtered-hit-field {
background: #eef0fb;
color: #4a5170;
padding: 0 5px;
border-radius: 3px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 10.5px;
flex: 0 0 auto;
}
.kv-filtered-hit-value {
color: #2a3050;
word-break: break-word;
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
}
.kv-filtered-more {
padding: 2px 6px 0;
font-size: 11px;
font-style: italic;
color: #8089a0;
}
/* ── Toasts ───────────────────────────────────────────────────────────── */
.kv-toast-stack {
position: fixed;
bottom: 16px;
right: 16px;
display: flex;
flex-direction: column-reverse;
gap: 6px;
z-index: 1000;
max-width: 480px;
}
.kv-toast {
background: #2a3050;
color: #fff;
border-radius: 6px;
padding: 9px 12px;
box-shadow: 0 4px 14px rgba(0,0,0,0.15);
font-size: 12.5px;
}
.kv-toast-success { background: #2f9e44; }
.kv-toast-error { background: #c0392b; }
.kv-toast-row {
display: flex;
align-items: flex-start;
gap: 8px;
justify-content: space-between;
}
.kv-toast-text {
flex: 1;
word-break: break-word;
}
.kv-toast-close {
border: none;
background: transparent;
color: rgba(255,255,255,0.8);
cursor: pointer;
font-size: 16px;
line-height: 1;
}
.kv-toast-close:hover { color: #fff; }
.kv-toast-details {
margin: 6px 0 0;
padding-left: 18px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11px;
color: rgba(255,255,255,0.85);
max-height: 160px;
overflow: auto;
}
.kv-app-gear {
margin-left: 0.25rem;
font-size: 1.05rem;
opacity: 0.6;
}
.kv-app-gear:hover {
opacity: 1;
}
.kv-modal-backdrop {
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.45);
z-index: 1000;
display: flex;
align-items: flex-start;
justify-content: center;
padding: 4vh 1rem;
overflow-y: auto;
}
.kv-modal {
position: relative;
background: white;
color: inherit;
border-radius: 8px;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
width: min(960px, 100%);
max-height: 92vh;
overflow-y: auto;
}
.kv-modal-close {
position: absolute;
top: 0.5rem;
right: 0.75rem;
background: transparent;
border: none;
font-size: 1.6rem;
line-height: 1;
cursor: pointer;
color: inherit;
opacity: 0.6;
z-index: 1;
}
.kv-modal-close:hover {
opacity: 1;
}
+313
View File
@@ -0,0 +1,313 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI, KVFindMatch, KVInfoResponse } from '../api/kvApi';
import { ancestorBranches, KVTree } from './KVTree';
import { FilteredKVTree } from './FilteredKVTree';
import { KVLeafEditor } from './KVLeafEditor';
import { KVBranchEditor } from './KVBranchEditor';
import { KVFileEditor } from './KVFileEditor';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
import { SearchFieldChips } from './SearchFieldChips';
import { ToastStack, makeToast } from './Toast';
import { ToastEntry } from './types';
import { SnapshotPanel } from '../components/SnapshotPanel';
import './KVApp.css';
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
const DEFAULT_SEARCH_FIELDS = ['originalTitle', 'filePath', 'fileName', 'plot/eng'];
function loadSearchFields(): string[] {
try {
const raw = localStorage.getItem(SEARCH_FIELDS_LS_KEY);
if (!raw) return DEFAULT_SEARCH_FIELDS;
const parsed = JSON.parse(raw);
if (Array.isArray(parsed) && parsed.every((x) => typeof x === 'string')) return parsed;
} catch {}
return DEFAULT_SEARCH_FIELDS;
}
// Top-level shell. Two panes inside the iframe:
// left — lazy tree of the keyspace
// right — leaf editor (when a leaf is selected) or branch editor (branch).
//
// A sticky header carries the search bar, the breadcrumb, and basic stats.
export default function KVApp() {
const [snapshotOpen, setSnapshotOpen] = useState(false);
// Selected node. branchPath is set when a branch is selected; selectedKey
// when a leaf is. Mutually exclusive.
const [selectedKey, setSelectedKey] = useState<string | null>(null);
const [branchPath, setBranchPath] = useState<string | null>(null);
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set(['']));
const [searchInput, setSearchInput] = useState('');
const [searchFields, setSearchFields] = useState<string[]>(loadSearchFields);
const [searchResults, setSearchResults] = useState<KVFindMatch[] | null>(null);
const [searchTruncated, setSearchTruncated] = useState(false);
const [searching, setSearching] = useState(false);
const [info, setInfo] = useState<KVInfoResponse | null>(null);
const [reloadToken, setReloadToken] = useState(0);
const [toasts, setToasts] = useState<ToastEntry[]>([]);
const searchTimerRef = useRef<number | null>(null);
// When the right pane is a file-detail view and the user clicks a leaf
// under the same file branch in the tree, we route the click as a
// scroll-anchor inside the file detail instead of a leaf-editor switch.
// The token bumps each click so identical anchors still trigger a scroll.
const [anchor, setAnchor] = useState<{ field: string; token: number } | null>(null);
const isFileBranch = (p: string | null) => !!p && /^file:[^/]+\/$/.test(p);
const handleLeafClick = (key: string) => {
if (isFileBranch(branchPath) && key.startsWith(branchPath!)) {
setAnchor({ field: key.slice(branchPath!.length), token: (anchor?.token ?? 0) + 1 });
return;
}
setSelectedKey(key);
setBranchPath(null);
setAnchor(null);
};
const handleBranchClick = (path: string) => {
setBranchPath(path);
setSelectedKey(null);
setAnchor(null);
};
const pushToast = (t: ToastEntry) => setToasts((cur) => [...cur, t]);
const dismissToast = (id: number) => setToasts((cur) => cur.filter((t) => t.id !== id));
useEffect(() => {
KVAPI.info().then(setInfo).catch(() => {});
}, [reloadToken]);
// Deep-link: /editor/file/<hashId> opens the tree pre-expanded at that hash.
useEffect(() => {
const m = window.location.pathname.match(/\/file\/([a-z0-9]+)$/i);
if (m) {
const hashId = m[1];
const branch = `file:${hashId}/`;
setExpandedPaths((cur) => new Set([...cur, '', 'file:', branch]));
setBranchPath(branch);
}
}, []);
// Persist field list across reloads.
useEffect(() => {
try { localStorage.setItem(SEARCH_FIELDS_LS_KEY, JSON.stringify(searchFields)); } catch {}
}, [searchFields]);
// Debounced live value-search across configured fields. Empty input clears.
useEffect(() => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
if (!searchInput.trim() || searchFields.length === 0) {
setSearchResults(null);
setSearchTruncated(false);
setSearching(false);
return;
}
setSearching(true);
searchTimerRef.current = window.setTimeout(async () => {
try {
const r = await KVAPI.find(searchInput.trim(), searchFields, 200);
setSearchResults(r.matches);
setSearchTruncated(r.truncated);
} catch (e: any) {
pushToast(makeToast('error', `Search failed: ${e.message}`));
} finally {
setSearching(false);
}
}, 250);
return () => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
};
}, [searchInput, searchFields]);
// Jump to a key from a search hit: select it, expand all ancestor branches.
const jumpToKey = (key: string) => {
const ancestors = ancestorBranches(key);
setExpandedPaths((cur) => new Set([...cur, '', ...ancestors]));
if (key.endsWith('/') || key.endsWith(':')) {
setBranchPath(key);
setSelectedKey(null);
} else {
setSelectedKey(key);
setBranchPath(null);
}
};
const breadcrumb = useMemo(() => {
const target = selectedKey ?? branchPath ?? '';
if (!target) return null;
const parts: { label: string; path: string }[] = [];
let i = 0;
while (i < target.length) {
let next = -1;
for (const d of [':', '/']) {
const idx = target.indexOf(d, i);
if (idx >= 0 && (next < 0 || idx < next)) next = idx;
}
if (next < 0) {
parts.push({ label: target.slice(i), path: target });
break;
}
parts.push({ label: target.slice(i, next + 1), path: target.slice(0, next + 1) });
i = next + 1;
}
return parts;
}, [selectedKey, branchPath]);
return (
<div className="kv-app">
<header className="kv-app-header">
<div className="kv-app-search">
<input
type="search"
className="kv-search-input"
placeholder="filter by value across configured fields…"
value={searchInput}
onChange={(e) => setSearchInput(e.target.value)}
autoFocus
/>
{searchInput && (
<button className="kv-btn-icon" onClick={() => setSearchInput('')} title="clear">×</button>
)}
<span className="kv-app-stats">
{info ? `${info.keyCount} files · ${info.memoryUsage}` : ''}
</span>
<button
className="kv-btn"
onClick={() => setReloadToken((n) => n + 1)}
title="refresh tree"
>Refresh</button>
<button
className="kv-btn-icon kv-app-gear"
onClick={() => setSnapshotOpen(true)}
title="Snapshot — export / import / wipe"
aria-label="Open snapshot settings"
>
⚙
</button>
</div>
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
{breadcrumb && breadcrumb.length > 0 && (
<div className="kv-app-breadcrumb">
<span className="kv-crumb-label">path:</span>
{breadcrumb.map((b, i) => (
<button
key={i}
className="kv-crumb"
onClick={() => jumpToKey(b.path)}
>{b.label}</button>
))}
</div>
)}
</header>
<div className="kv-app-body">
<aside className="kv-app-left">
{searchResults !== null ? (
<FilteredKVTree
matches={searchResults}
truncated={searchTruncated}
searching={searching}
selectedKey={selectedKey}
expandedPaths={expandedPaths}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
) : (
<KVTree
selectedKey={selectedKey}
expandedPaths={expandedPaths}
reloadToken={reloadToken}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
)}
</aside>
<main className="kv-app-right">
{selectedKey ? (
<KVLeafEditor
selectedKey={selectedKey}
reloadToken={reloadToken}
onSaved={() => {
pushToast(makeToast('success', `Saved ${selectedKey}`));
setReloadToken((n) => n + 1);
}}
onDeleted={() => {
pushToast(makeToast('success', `Deleted ${selectedKey}`));
setSelectedKey(null);
setReloadToken((n) => n + 1);
}}
onError={(msg) => pushToast(makeToast('error', msg))}
/>
) : branchPath && FILE_BRANCH_RE.test(branchPath) ? (
<KVFileEditor
key={branchPath}
branchPath={branchPath}
reloadToken={reloadToken}
scrollToField={anchor}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
onSelectLeaf={(k) => { setSelectedKey(k); setBranchPath(null); setAnchor(null); }}
/>
) : branchPath ? (
<KVBranchEditor
branchPath={branchPath}
reloadToken={reloadToken}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
/>
) : (
<div className="kv-pane-status kv-pane-empty">
Select a key from the tree.
<div className="kv-pane-hint">
Click a branch to add / rename / delete a subtree, or a leaf to edit its value.
</div>
</div>
)}
</main>
</div>
{snapshotOpen && (
<div
className="kv-modal-backdrop"
onClick={() => setSnapshotOpen(false)}
>
<div
className="kv-modal"
onClick={(e) => e.stopPropagation()}
role="dialog"
aria-modal="true"
>
<button
className="kv-modal-close"
onClick={() => setSnapshotOpen(false)}
aria-label="Close"
title="Close"
>×</button>
<SnapshotPanel />
</div>
</div>
)}
<ToastStack toasts={toasts} onDismiss={dismissToast} />
</div>
);
}
+194
View File
@@ -0,0 +1,194 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
interface Props {
// Branch path, ends with the delimiter (":" or "/").
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
}
// Branch operations are non-atomic — each per-key write/delete goes one round
// trip to Redis and may fail individually. Failures collect into a list and
// surface via the parent's toast stack. Per the design, we accept partial
// failure and surface what failed.
export function KVBranchEditor({ branchPath, reloadToken, onChanged }: Props) {
const [count, setCount] = useState<number | null>(null);
const [busy, setBusy] = useState(false);
const [newChild, setNewChild] = useState('');
const [newValue, setNewValue] = useState('');
const [renameTo, setRenameTo] = useState('');
// Use the search endpoint with the branch path as the substring to count
// descendants. Truncated when there are more than the limit; we just show
// an approximate count.
useEffect(() => {
let cancelled = false;
KVAPI.search(branchPath, 5000)
.then((r) => {
if (cancelled) return;
// Filter to true descendants (those starting with branchPath).
const descendants = r.keys.filter((k) => k.startsWith(branchPath));
setCount(descendants.length);
})
.catch(() => { if (!cancelled) setCount(null); });
return () => { cancelled = true; };
}, [branchPath, reloadToken]);
// Walk all descendants and run an op against each, collecting failures.
const forEachDescendant = async (op: (key: string) => Promise<void>): Promise<string[]> => {
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const errors: string[] = [];
for (const k of keys) {
try { await op(k); } catch (e: any) { errors.push(`${k}: ${e.message}`); }
}
return errors;
};
const addChild = async () => {
if (!newChild) return;
const childKey = branchPath + newChild;
setBusy(true);
try {
await KVAPI.put(childKey, newValue);
setNewChild('');
setNewValue('');
onChanged(`Created ${childKey}`);
} catch (e: any) {
onChanged(`Failed to create ${childKey}`, [e.message]);
} finally {
setBusy(false);
}
};
const deleteSubtree = async () => {
if (!confirm(`Delete every key under ${branchPath} (${count ?? '?'} entries)?`)) return;
setBusy(true);
try {
const errors = await forEachDescendant((k) => KVAPI.del(k).then(() => undefined));
if (errors.length === 0) {
onChanged(`Deleted subtree ${branchPath}`);
} else {
onChanged(`Deleted subtree ${branchPath} with ${errors.length} failure(s)`, errors);
}
} finally {
setBusy(false);
}
};
const renameBranch = async () => {
if (!renameTo) return;
const target = renameTo.endsWith('/') || renameTo.endsWith(':') ? renameTo : renameTo + '/';
if (target === branchPath) return;
if (!confirm(`Rename ${branchPath} → ${target}? This copies ${count ?? '?'} keys then deletes the originals.`)) return;
setBusy(true);
try {
// Pass 1: copy
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const copyErrors: string[] = [];
for (const oldKey of keys) {
try {
const v = await KVAPI.get(oldKey);
if (!v.exists || v.type !== 'string') {
copyErrors.push(`${oldKey}: skipped (type ${v.type || 'missing'})`);
continue;
}
const newKey = target + oldKey.slice(branchPath.length);
await KVAPI.put(newKey, v.value);
} catch (e: any) {
copyErrors.push(`${oldKey}: ${e.message}`);
}
}
// Pass 2: delete originals (only the ones we successfully copied)
const failedOldKeys = new Set(copyErrors.map((e) => e.split(':')[0]));
const deleteErrors: string[] = [];
for (const oldKey of keys) {
if (failedOldKeys.has(oldKey)) continue;
try { await KVAPI.del(oldKey); } catch (e: any) { deleteErrors.push(`${oldKey}: ${e.message}`); }
}
const allErrors = [...copyErrors, ...deleteErrors];
if (allErrors.length === 0) {
onChanged(`Renamed ${branchPath} → ${target}`);
} else {
onChanged(`Renamed ${branchPath} → ${target} with ${allErrors.length} failure(s)`, allErrors);
}
setRenameTo('');
} finally {
setBusy(false);
}
};
return (
<div className="kv-branch">
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{branchPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(branchPath)}
title="copy path"
>⧉</button>
</div>
<div className="kv-pane-status">
{count == null ? 'counting…' : `${count} descendant key${count === 1 ? '' : 's'}`}
</div>
<section className="kv-branch-section">
<h4>Add child</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">{branchPath}</span>
<input
className="kv-leaf-input"
placeholder="child path (may contain /)"
value={newChild}
onChange={(e) => setNewChild(e.target.value)}
disabled={busy}
/>
</div>
<input
className="kv-leaf-input"
placeholder="value"
value={newValue}
onChange={(e) => setNewValue(e.target.value)}
disabled={busy}
/>
<button className="kv-btn kv-btn-primary" disabled={!newChild || busy} onClick={addChild}>
Add
</button>
</section>
<section className="kv-branch-section">
<h4>Rename branch</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">→</span>
<input
className="kv-leaf-input"
placeholder="new prefix (e.g. file:newhash/)"
value={renameTo}
onChange={(e) => setRenameTo(e.target.value)}
disabled={busy}
/>
</div>
<p className="kv-branch-note">
Non-atomic: copies each descendant to the new prefix then deletes the original.
Failures are surfaced individually.
</p>
<button className="kv-btn" disabled={!renameTo || busy} onClick={renameBranch}>
Rename
</button>
</section>
<section className="kv-branch-section">
<h4>Delete subtree</h4>
<p className="kv-branch-note">
Deletes every descendant key. Cannot be undone.
</p>
<button className="kv-btn kv-btn-danger" disabled={busy || count === 0} onClick={deleteSubtree}>
Delete {count ?? ''} key{count === 1 ? '' : 's'}
</button>
</section>
</div>
);
}
+675
View File
@@ -0,0 +1,675 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { SchemaFieldInfo, useSchema } from '../api/schemaApi';
import { LanguageCombobox, formatLanguageOption } from '../components/LanguageCombobox';
import { LeafValueEditor } from './leafRenderers';
import { schemaToFieldType } from './schemaLookup';
import { detectType } from './typeHeuristics';
import { FieldType } from './types';
interface Props {
/** Branch path of a file, e.g. "file:<hash>/". */
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
/** Navigate to a single leaf for the manual deep editor. */
onSelectLeaf: (key: string) => void;
/**
* When set, scroll to and pulse the row matching this logical field path
* (e.g. "releasedate", "plot", "stream/0"). The `token` lets the same field
* fire a fresh scroll across consecutive clicks.
*/
scrollToField?: { field: string; token: number } | null;
}
const LANG_CODE_RE = /^[a-z]{2,3}$/;
interface LeafRow {
kind: 'leaf';
fieldKey: string;
value: string;
fieldType: FieldType;
info?: SchemaFieldInfo;
}
interface LanguageRow {
kind: 'language-group';
fieldKey: string;
/** Current language → text map. */
languageMap: Record<string, string>;
info?: SchemaFieldInfo;
}
type Row = LeafRow | LanguageRow;
interface RowGroup {
prefix: string;
isCluster: boolean;
rows: Row[];
}
export function KVFileEditor({
branchPath,
reloadToken,
onChanged,
onSelectLeaf,
scrollToField,
}: Props) {
const [metadata, setMetadata] = useState<Record<string, string> | null>(null);
const [edits, setEdits] = useState<Map<string, string | null>>(new Map());
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
const [pulse, setPulse] = useState<string | null>(null);
const [newFieldKey, setNewFieldKey] = useState('');
const [newFieldValue, setNewFieldValue] = useState('');
const [addingField, setAddingField] = useState(false);
// Track whether we've seeded collapsedGroups for the current branch; resets
// when the branch changes so each file gets its defaults reapplied.
const defaultsAppliedRef = useRef<string | null>(null);
const { schema } = useSchema();
// When the parent passes a target field, expand its prefix group (if any)
// and scroll the row into view. We also briefly pulse the row to draw the
// eye — the pulse is reset after the animation duration.
useEffect(() => {
if (!scrollToField) return;
const { field } = scrollToField;
const prefix = field.split('/', 1)[0];
setCollapsedGroups((cur) => {
if (!cur.has(prefix)) return cur;
const next = new Set(cur);
next.delete(prefix);
return next;
});
requestAnimationFrame(() => {
const el = document.querySelector(
`[data-row-key="${cssEscape(field)}"]`,
) as HTMLElement | null;
if (el) {
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
setPulse(field);
window.setTimeout(() => setPulse((p) => (p === field ? null : p)), 1400);
}
});
}, [scrollToField?.field, scrollToField?.token]);
const hashId = branchPath.slice('file:'.length, -1);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.treeAll(branchPath, '/', 5000)
.then(async (resp) => {
// resp.leaves = full keys under branchPath. Recursive descent: also
// walk branch children (they're paths to sub-objects with their own
// leaves, e.g. file:<h>/plot/ → plot/eng).
const all: string[] = [...resp.leaves];
const queue = [...resp.branches];
while (queue.length > 0) {
const next = queue.shift()!;
try {
const sub = await KVAPI.treeAll(next, '/', 5000);
all.push(...sub.leaves);
queue.push(...sub.branches);
} catch {
// Ignore failures on a sub-branch; show what we can.
}
}
if (cancelled) return;
// MGET-equivalent: parallel fetches with a small cap to avoid burst.
const values = await Promise.all(
all.map((k) => KVAPI.get(k).then((r) => [k, r.exists ? r.value : null] as const).catch(() => [k, null] as const)),
);
if (cancelled) return;
const flat: Record<string, string> = {};
for (const [k, v] of values) {
if (v == null) continue;
// Strip the branch prefix so keys are the logical field path
// ("releasedate", "plot/eng", "stream/0").
const fieldKey = k.slice(branchPath.length);
if (fieldKey) flat[fieldKey] = v;
}
setMetadata(flat);
setEdits(new Map());
})
.catch(() => {
if (!cancelled) setMetadata({});
})
.finally(() => {
if (!cancelled) setLoading(false);
});
return () => {
cancelled = true;
};
}, [branchPath, reloadToken]);
const groups = useMemo(
() => groupRows(buildRows(metadata ?? {}, schema)),
[metadata, schema],
);
// Default state: every cluster group (e.g. stream/*, cid_*) starts collapsed.
// Reapplied when the user switches to a different file.
useEffect(() => {
if (groups.length === 0) return;
if (defaultsAppliedRef.current === branchPath) return;
defaultsAppliedRef.current = branchPath;
setCollapsedGroups(new Set(groups.filter((g) => g.isCluster).map((g) => g.prefix)));
}, [groups, branchPath]);
if (loading) {
return <div className="kv-pane-status">loading file…</div>;
}
if (!metadata) {
return <div className="kv-pane-status">no data for {branchPath}</div>;
}
const fileName = metadata['fileName'];
const filePath = metadata['filePath'];
const onLeafChange = (fieldKey: string, value: string) => {
setEdits((prev) => {
const next = new Map(prev);
const original = metadata[fieldKey] ?? null;
if (value === original) next.delete(fieldKey);
else next.set(fieldKey, value);
return next;
});
};
/**
* Toggle a field's pending-delete state. Re-clicking restores; if the field
* is brand-new (no original value) we just drop it from the edits map.
*/
const onLeafDeleteToggle = (fieldKey: string) => {
setEdits((prev) => {
const next = new Map(prev);
if (next.get(fieldKey) === null) {
// Currently pending delete → undo.
next.delete(fieldKey);
} else {
next.set(fieldKey, null);
}
return next;
});
};
const handleAddField = async () => {
const key = newFieldKey.trim();
if (!key) return;
const fullKey = branchPath + key;
setAddingField(true);
try {
await KVAPI.put(fullKey, newFieldValue);
setNewFieldKey('');
setNewFieldValue('');
onChanged(`Added ${fullKey}`);
// Refetch this file so the new field appears in the row list.
const r = await KVAPI.get(fullKey);
if (r.exists) {
setMetadata((cur) => ({ ...(cur ?? {}), [key]: r.value }));
}
} catch (e: any) {
onChanged(`Failed to add ${fullKey}`, [e.message]);
} finally {
setAddingField(false);
}
};
const onLanguageGroupChange = (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => {
setEdits((prev) => {
const next = new Map(prev);
const allLangs = new Set([...Object.keys(originalMap), ...Object.keys(nextMap)]);
for (const lang of allLangs) {
const fieldKey = `${groupKey}/${lang}`;
const orig = originalMap[lang] ?? null;
const cur = nextMap[lang] ?? null;
if (cur === orig) next.delete(fieldKey);
else if (cur == null) next.set(fieldKey, null);
else next.set(fieldKey, cur);
}
return next;
});
};
const handleSave = async () => {
if (edits.size === 0) return;
setSaving(true);
const errors: string[] = [];
for (const [fieldKey, newValue] of edits) {
const fullKey = branchPath + fieldKey;
try {
if (newValue === null) await KVAPI.del(fullKey);
else await KVAPI.put(fullKey, newValue);
} catch (e: any) {
errors.push(`${fieldKey}: ${e.message}`);
}
}
setSaving(false);
if (errors.length === 0) {
onChanged(`Saved ${edits.size} field${edits.size === 1 ? '' : 's'} on ${fileName ?? hashId}`);
} else {
onChanged(
`Saved with ${errors.length} failure(s) on ${fileName ?? hashId}`,
errors,
);
}
};
const handleDiscard = () => setEdits(new Map());
const toggleGroup = (prefix: string) => {
setCollapsedGroups((cur) => {
const next = new Set(cur);
if (next.has(prefix)) next.delete(prefix);
else next.add(prefix);
return next;
});
};
return (
<div className="kv-file">
<div className="kv-file-header">
<div className="kv-file-name">
{fileName ?? <span className="kv-file-name-fallback">(no fileName)</span>}
</div>
<div className="kv-file-meta">
<code className="kv-file-hash" title={hashId}>
{hashId.slice(0, 8)}…{hashId.slice(-6)}
</code>
{filePath && <span className="kv-file-path" title={filePath}>{filePath}</span>}
</div>
<div className="kv-file-actions">
<button
className="kv-btn kv-btn-primary"
disabled={edits.size === 0 || saving}
onClick={handleSave}
>
{saving ? 'saving…' : `Save (${edits.size})`}
</button>
<button
className="kv-btn"
disabled={edits.size === 0 || saving}
onClick={handleDiscard}
>
Discard
</button>
</div>
</div>
<div className="kv-file-rows">
{groups.length === 0 && (
<div className="kv-pane-status">No data yet for this file.</div>
)}
{groups.map((g) => (
<FieldGroupView
key={g.prefix}
group={g}
edits={edits}
collapsed={collapsedGroups.has(g.prefix)}
pulse={pulse}
onToggle={() => toggleGroup(g.prefix)}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageGroupChange}
onJumpToLeaf={(fieldKey) => onSelectLeaf(branchPath + fieldKey)}
/>
))}
</div>
<div className="kv-file-add">
<span className="kv-file-add-prefix">{branchPath}</span>
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
placeholder="field path (may contain /)"
value={newFieldKey}
onChange={(e) => setNewFieldKey(e.target.value)}
disabled={addingField}
/>
<input
type="text"
className="kv-leaf-input"
placeholder="value"
value={newFieldValue}
onChange={(e) => setNewFieldValue(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter' && newFieldKey.trim()) handleAddField();
}}
disabled={addingField}
/>
<button
className="kv-btn kv-btn-primary"
disabled={!newFieldKey.trim() || addingField}
onClick={handleAddField}
>
{addingField ? 'adding…' : '+ Add field'}
</button>
</div>
</div>
);
}
function FieldGroupView({
group,
edits,
collapsed,
pulse,
onToggle,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
group: RowGroup;
edits: Map<string, string | null>;
collapsed: boolean;
pulse: string | null;
onToggle: () => void;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
if (!group.isCluster) {
return (
<>
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</>
);
}
return (
<div className={`kv-file-group ${collapsed ? 'collapsed' : ''}`}>
<button
type="button"
className="kv-file-group-header"
onClick={onToggle}
title={collapsed ? 'expand' : 'collapse'}
>
<span className="kv-file-group-chevron">{collapsed ? '▸' : '▾'}</span>
<span className="kv-file-group-name">{group.prefix}</span>
<span className="kv-file-group-count">{group.rows.length}</span>
</button>
{!collapsed && (
<div className="kv-file-group-body">
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</div>
)}
</div>
);
}
function RowView({
row,
edits,
pulse,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
row: Row;
edits: Map<string, string | null>;
pulse: string | null;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
const pulsed = pulse === row.fieldKey;
if (row.kind === 'leaf') {
const pendingDelete = edits.get(row.fieldKey) === null;
const draft = edits.has(row.fieldKey) ? (edits.get(row.fieldKey) ?? '') : row.value;
const dirty = edits.has(row.fieldKey);
return (
<div
className={`kv-file-row ${dirty ? 'dirty' : ''} ${pendingDelete ? 'pending-delete' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<button
type="button"
className="kv-file-row-label"
onClick={() => onJumpToLeaf(row.fieldKey)}
title="open in single-leaf editor"
>
{row.fieldKey}
</button>
<div className="kv-file-row-value">
{pendingDelete ? (
<div className="kv-file-row-deleted-note">marked for deletion</div>
) : (
<LeafValueEditor
type={row.fieldType}
value={draft}
onChange={(v) => onLeafChange(row.fieldKey, v)}
/>
)}
</div>
<button
type="button"
className="kv-file-row-delete"
onClick={() => onLeafDeleteToggle(row.fieldKey)}
title={pendingDelete ? 'undo deletion' : 'delete this field'}
aria-label={pendingDelete ? 'undo deletion' : 'delete this field'}
>
{pendingDelete ? '↺' : '×'}
</button>
</div>
);
}
return (
<LanguageGroupRow
row={row}
edits={edits}
pulsed={pulsed}
onLanguageChange={onLanguageChange}
/>
);
}
function LanguageGroupRow({
row,
edits,
pulsed,
onLanguageChange,
}: {
row: LanguageRow;
edits: Map<string, string | null>;
pulsed: boolean;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
}) {
// Apply pending edits on top of the original language map.
const effectiveMap: Record<string, string> = { ...row.languageMap };
for (const [k, v] of edits) {
if (!k.startsWith(`${row.fieldKey}/`)) continue;
const lang = k.slice(row.fieldKey.length + 1);
if (v === null) delete effectiveMap[lang];
else effectiveMap[lang] = v;
}
const entries = Object.entries(effectiveMap);
const dirty = [...edits.keys()].some((k) => k.startsWith(`${row.fieldKey}/`));
const updateLang = (lang: string, text: string) => {
onLanguageChange(row.fieldKey, { ...effectiveMap, [lang]: text }, row.languageMap);
};
const renameLang = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const next = { ...effectiveMap };
const text = next[oldLang];
delete next[oldLang];
next[newLang] = text;
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const removeLang = (lang: string) => {
const next = { ...effectiveMap };
delete next[lang];
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const addLang = () => {
const used = new Set(entries.map(([k]) => k));
const defaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa', 'fre'];
const defaultLang = defaults.find((l) => !used.has(l)) || 'eng';
onLanguageChange(row.fieldKey, { ...effectiveMap, [defaultLang]: '' }, row.languageMap);
};
return (
<div
className={`kv-file-row kv-file-row-lang ${dirty ? 'dirty' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<div className="kv-file-row-label-static">{row.fieldKey}</div>
<div className="kv-file-row-value">
<div className="kv-file-lang-entries">
{entries.map(([lang, text]) => (
<div key={lang} className="kv-file-lang-entry">
<LanguageCombobox
value={lang}
onChange={(newLang) => renameLang(lang, newLang)}
/>
<input
type="text"
className="kv-leaf-input"
value={text}
title={formatLanguageOption(lang)}
onChange={(e) => updateLang(lang, e.target.value)}
/>
<button
type="button"
className="kv-btn-icon kv-file-lang-remove"
onClick={() => removeLang(lang)}
title="remove this language"
>
×
</button>
</div>
))}
<button
type="button"
className="kv-btn kv-file-lang-add"
onClick={addLang}
>
+ Add language
</button>
</div>
</div>
</div>
);
}
// ──────────────────────────────────────────────────────────────────────────
// Row model — kept local because it's only used here and is small.
// ──────────────────────────────────────────────────────────────────────────
function buildRows(
flat: Record<string, string>,
schema: ReturnType<typeof useSchema>['schema'],
): Row[] {
const languageGroupKeys = new Set<string>();
if (schema) {
for (const [k, info] of Object.entries(schema.fields)) {
if (info.key_hint === 'language-code') languageGroupKeys.add(k);
}
}
const langMaps: Record<string, Record<string, string>> = {};
const rows: Row[] = [];
for (const [key, value] of Object.entries(flat)) {
const slash = key.lastIndexOf('/');
if (slash > 0) {
const prefix = key.slice(0, slash);
const tail = key.slice(slash + 1);
if (languageGroupKeys.has(prefix) && LANG_CODE_RE.test(tail)) {
if (!langMaps[prefix]) langMaps[prefix] = {};
langMaps[prefix][tail] = value;
continue;
}
}
const info = schema?.fields[key];
rows.push({
kind: 'leaf',
fieldKey: key,
value,
// Schema decides when it can; otherwise heuristic on (path, value).
fieldType: schemaToFieldType(info) ?? detectType(key, value),
info,
});
}
for (const [groupKey, langMap] of Object.entries(langMaps)) {
rows.push({
kind: 'language-group',
fieldKey: groupKey,
languageMap: langMap,
info: schema?.fields[groupKey],
});
}
rows.sort((a, b) => a.fieldKey.localeCompare(b.fieldKey));
return rows;
}
/** Escape characters that would break a CSS attribute selector value. */
function cssEscape(s: string): string {
return s.replace(/["\\]/g, (c) => `\\${c}`);
}
function groupRows(rows: Row[]): RowGroup[] {
const out: RowGroup[] = [];
for (const row of rows) {
const prefix = row.fieldKey.split('/', 1)[0];
const last = out[out.length - 1];
if (last && last.prefix === prefix) {
last.rows.push(row);
} else {
out.push({ prefix, isCluster: false, rows: [row] });
}
}
for (const g of out) g.isCluster = g.rows.length > 1;
return out;
}
+160
View File
@@ -0,0 +1,160 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { useSchema } from '../api/schemaApi';
import { ALL_TYPES, describeType, detectType } from './typeHeuristics';
import { lookupSchemaForKey } from './schemaLookup';
import { LeafValueEditor } from './leafRenderers';
import { FieldType } from './types';
interface Props {
// Full Redis key path of the selected leaf.
selectedKey: string;
// Bumped after a delete or external change to force refetch.
reloadToken: number;
onSaved: () => void;
onDeleted: () => void;
onError: (msg: string) => void;
}
export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onError }: Props) {
const [original, setOriginal] = useState<string>('');
const [draft, setDraft] = useState<string>('');
const [exists, setExists] = useState<boolean>(true);
const [redisType, setRedisType] = useState<string>('');
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
// Manual override for the inferred type. "auto" defers to schema → heuristic.
const [typeOverride, setTypeOverride] = useState<FieldType>('auto');
const { schema } = useSchema();
const schemaLookup = lookupSchemaForKey(schema, selectedKey);
// Schema is authoritative when it has an opinion; heuristics fill the gap.
const detected = schemaLookup.fieldType ?? detectType(selectedKey, original);
const effectiveType = typeOverride === 'auto' ? detected : typeOverride;
// Reset override when the selected key changes — the previous override
// doesn't apply to a different leaf.
useEffect(() => { setTypeOverride('auto'); }, [selectedKey]);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.get(selectedKey)
.then((r) => {
if (cancelled) return;
setOriginal(r.value);
setDraft(r.value);
setExists(r.exists);
setRedisType(r.type);
})
.catch((e) => onError(e.message))
.finally(() => { if (!cancelled) setLoading(false); });
return () => { cancelled = true; };
}, [selectedKey, reloadToken, onError]);
const dirty = draft !== original;
const editable = exists ? redisType === 'string' || redisType === '' : true;
const save = async () => {
setSaving(true);
try {
await KVAPI.put(selectedKey, draft);
setOriginal(draft);
setExists(true);
setRedisType('string');
onSaved();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
const del = async () => {
if (!confirm(`Delete ${selectedKey}?`)) return;
setSaving(true);
try {
await KVAPI.del(selectedKey);
onDeleted();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
if (loading) {
return <div className="kv-pane-status">loading…</div>;
}
if (!editable) {
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-pane-status">
Key has Redis type <code>{redisType}</code> — only string-typed values are editable here.
</div>
</div>
);
}
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-leaf-meta">
<label className="kv-leaf-label">Type</label>
<select
className="kv-leaf-type"
value={typeOverride}
onChange={(e) => setTypeOverride(e.target.value as FieldType)}
>
{ALL_TYPES.map((t) => (
<option key={t} value={t}>
{t === 'auto' ? `auto (${describeType(detected)})` : describeType(t)}
</option>
))}
</select>
{!exists && <span className="kv-leaf-hint">new key — save to create</span>}
</div>
<LeafValueEditor type={effectiveType} value={draft} onChange={setDraft} />
<div className="kv-leaf-actions">
<button
className="kv-btn kv-btn-primary"
onClick={save}
disabled={!dirty || saving}
>{saving ? 'saving…' : exists ? 'Save' : 'Create'}</button>
<button
className="kv-btn"
onClick={() => setDraft(original)}
disabled={!dirty || saving}
>Reset</button>
{exists && (
<button
className="kv-btn kv-btn-danger"
onClick={del}
disabled={saving}
>Delete</button>
)}
</div>
</div>
);
}
function KeyHeader({ keyPath }: { keyPath: string }) {
return (
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{keyPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(keyPath)}
title="copy key"
>⧉</button>
</div>
);
}
// Leaf value rendering moved to leafRenderers.tsx (LeafValueEditor)
// so the file-detail view can reuse it per row.
+339
View File
@@ -0,0 +1,339 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
const fileNameCache = new Map<string, string | null>();
const fileNamePromises = new Map<string, Promise<void>>();
const fileNameSubscribers = new Set<() => void>();
function notifyFileNameSubscribers() {
for (const cb of fileNameSubscribers) cb();
}
function ensureFileName(branchPath: string): void {
if (fileNameCache.has(branchPath) || fileNamePromises.has(branchPath)) return;
const p = KVAPI.get(branchPath + 'fileName')
.then((r) => {
fileNameCache.set(branchPath, r.exists && r.value ? r.value : null);
})
.catch(() => {
fileNameCache.set(branchPath, null);
})
.finally(() => {
fileNamePromises.delete(branchPath);
notifyFileNameSubscribers();
});
fileNamePromises.set(branchPath, p);
}
function useFileName(branchPath: string | null): string | null {
const [, force] = useState(0);
useEffect(() => {
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return;
ensureFileName(branchPath);
const cb = () => force((n) => n + 1);
fileNameSubscribers.add(cb);
return () => {
fileNameSubscribers.delete(cb);
};
}, [branchPath]);
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return null;
return fileNameCache.get(branchPath) ?? null;
}
function isFileBranch(path: string): boolean {
return FILE_BRANCH_RE.test(path);
}
function shortHashFromFileBranch(path: string): string {
// path = "file:<hash>/" → last 6 chars of hash
const hash = path.slice('file:'.length, -1);
if (hash.length <= 12) return hash;
return `…${hash.slice(-6)}`;
}
// A node in the lazy tree. branch-typed nodes have an associated path that
// ends with the delimiter and lazy-load their children on first expand.
type NodeKind = 'branch' | 'leaf';
interface TreeNode {
kind: NodeKind;
// Display label: the segment of the path under the parent.
label: string;
// Full key path. For branches, ends with the delimiter.
path: string;
}
interface BranchState {
loaded: boolean;
loading: boolean;
error?: string;
children: TreeNode[];
}
// Sub-segment of a path used as the visible label for a branch / leaf.
// Splits on either ":" or "/" so file:abc/title/eng becomes file: > abc > title > eng.
function labelOf(path: string, parentPath: string): string {
const rest = path.slice(parentPath.length);
// The "rest" includes a trailing delimiter for branches; strip it for display.
return rest.replace(/[/:]+$/, '');
}
// Hide internal Redis structures from the tree. The editor is for managing
// file metadata; everything else is meta-core's own bookkeeping and noise
// for the user. Hidden:
// - `meta:` — event streams (file:events, meta:events, etc.)
// - `cid:` — reverse index (cid:<algo>:<value> → uuid), derived
// from each file's cid_* fields; the editor surfaces
// these per-file under their parent root
// - `meta-core:` — internal service state (mounts, watcher config,
// leader info), edited via dedicated UI pages, not
// the KV tree
// - bare top-level leaves under a colon-namespace like file:__index__
// and file:events (Redis sets / streams that aren't user data).
function isHiddenNode(node: TreeNode): boolean {
if (node.kind === 'branch') {
if (node.path === 'meta:' || node.path === 'cid:' || node.path === 'meta-core:') {
return true;
}
}
if (node.kind === 'leaf' && /^[^/]+:[^/]+$/.test(node.path)) return true;
return false;
}
interface Props {
// Selected leaf path (full key) — highlighted.
selectedKey: string | null;
// When set, the tree forces these branches open (used for auto-expand on
// search hit or deep-link).
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
// Bumping this triggers a full reload of root and any expanded branches.
reloadToken: number;
}
export function KVTree({
selectedKey,
expandedPaths,
onSelectLeaf,
onSelectBranch,
onToggleBranch,
reloadToken,
}: Props) {
// Map of branch path → load state. The empty key "" represents the root.
const [branches, setBranches] = useState<Record<string, BranchState>>({});
// Load (or reload) a branch's children from the API. The root is special-
// cased: we list at prefix="" with delimiter=":" to surface the namespaces.
const loadBranch = async (path: string) => {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: true, children: b[path]?.children ?? [] },
}));
try {
const isRoot = path === '';
const delim = isRoot ? ':' : '/';
const resp = await KVAPI.treeAll(path, delim, 5000);
const branchNodes: TreeNode[] = resp.branches.map((p) => ({
kind: 'branch',
label: labelOf(p, path) + delim,
path: p,
}));
const leafNodes: TreeNode[] = resp.leaves.map((p) => ({
kind: 'leaf',
label: labelOf(p, path),
path: p,
}));
const children = [...branchNodes, ...leafNodes];
setBranches((b) => ({
...b,
[path]: { loaded: true, loading: false, children },
}));
} catch (err: any) {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: false, children: [], error: err.message },
}));
}
};
// On reloadToken change, refetch root and every currently-expanded branch.
useEffect(() => {
loadBranch('');
for (const p of expandedPaths) {
if (p !== '') loadBranch(p);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [reloadToken]);
// Whenever expandedPaths gains a path we haven't loaded yet, lazy-load it.
useEffect(() => {
for (const p of expandedPaths) {
if (p === '') continue;
const state = branches[p];
if (!state || (!state.loaded && !state.loading)) {
loadBranch(p);
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [expandedPaths]);
return (
<div className="kv-tree">
<Branch
path=""
state={branches['']}
depth={0}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={(p, open) => {
onToggleBranch(p, open);
if (open && (!branches[p] || !branches[p].loaded)) loadBranch(p);
}}
/>
</div>
);
}
interface BranchProps {
path: string;
state: BranchState | undefined;
depth: number;
expandedPaths: Set<string>;
branches: Record<string, BranchState>;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function Branch({
state, depth, expandedPaths, branches,
selectedKey, onSelectLeaf, onSelectBranch, onToggleBranch,
}: BranchProps) {
if (!state) {
return null;
}
if (state.loading && state.children.length === 0) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (state.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>error: {state.error}</div>;
}
const visibleChildren = state.children.filter((c) => !isHiddenNode(c));
if (visibleChildren.length === 0) {
return <div className="kv-tree-empty" style={{ paddingLeft: depth * 14 + 18 }}>(empty)</div>;
}
return (
<ul className="kv-tree-list">
{visibleChildren.map((node) => {
if (node.kind === 'branch') {
const open = expandedPaths.has(node.path);
const childState = branches[node.path];
return (
<li key={node.path}>
<BranchRow
node={node}
open={open}
depth={depth}
onToggle={() => {
onToggleBranch(node.path, !open);
onSelectBranch(node.path);
}}
/>
{open && (
<Branch
path={node.path}
state={childState}
depth={depth + 1}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
)}
</li>
);
}
const isSelected = node.path === selectedKey;
return (
<li key={node.path}>
<div
className={`kv-tree-row kv-tree-leaf${isSelected ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(node.path)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{node.label}</span>
</div>
</li>
);
})}
</ul>
);
}
interface BranchRowProps {
node: TreeNode;
open: boolean;
depth: number;
onToggle: () => void;
}
function BranchRow({ node, open, depth, onToggle }: BranchRowProps) {
// Surface the actual filename next to file:<hash>/ branches so the tree
// reads like a file list at that level instead of a wall of hashes.
const fileName = useFileName(isFileBranch(node.path) ? node.path : null);
return (
<div
className={`kv-tree-row kv-tree-branch${open ? ' open' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={onToggle}
title={isFileBranch(node.path) ? node.path : undefined}
>
<span className="kv-tree-toggle">{open ? '▾' : '▸'}</span>
{fileName ? (
<>
<span className="kv-tree-label kv-tree-filename">{fileName}</span>
<span className="kv-tree-hash-pill">{shortHashFromFileBranch(node.path)}</span>
</>
) : (
<span className="kv-tree-label">{node.label}</span>
)}
</div>
);
}
// Helper: given a key path with both ":" and "/" separators, return the list
// of ancestor branch paths that need to be expanded to reveal it.
export function ancestorBranches(key: string): string[] {
const out: string[] = [];
let i = 0;
while (i < key.length) {
const next = nextSep(key, i);
if (next < 0) break;
out.push(key.slice(0, next + 1));
i = next + 1;
}
return out;
}
function nextSep(s: string, from: number): number {
let best = -1;
for (const d of [':', '/']) {
const idx = s.indexOf(d, from);
if (idx >= 0 && (best < 0 || idx < best)) best = idx;
}
return best;
}
// Re-export for convenience in the shell
export type { TreeNode };
+57
View File
@@ -0,0 +1,57 @@
import { useState } from 'react';
interface Props {
fields: string[];
onChange: (fields: string[]) => void;
}
// Pill list of field names that the value-search uses. Each pill is removable;
// the inline input adds a new field on Enter or comma. Persisting to
// localStorage is the parent's responsibility.
export function SearchFieldChips({ fields, onChange }: Props) {
const [input, setInput] = useState('');
const add = (raw: string) => {
const next = raw.trim().replace(/,+$/, '');
if (!next) return;
if (fields.includes(next)) return;
onChange([...fields, next]);
setInput('');
};
const remove = (f: string) => onChange(fields.filter((x) => x !== f));
return (
<div className="kv-chips">
<span className="kv-chips-label">search in:</span>
{fields.map((f) => (
<span key={f} className="kv-chip">
<code>{f}</code>
<button
className="kv-chip-x"
onClick={() => remove(f)}
title={`remove ${f}`}
aria-label={`remove ${f}`}
>×</button>
</span>
))}
<input
className="kv-chip-input"
type="text"
value={input}
placeholder="+ field"
onChange={(e) => {
const v = e.target.value;
if (v.endsWith(',')) add(v);
else setInput(v);
}}
onKeyDown={(e) => {
if (e.key === 'Enter') { e.preventDefault(); add(input); }
if (e.key === 'Backspace' && input === '' && fields.length > 0) {
remove(fields[fields.length - 1]);
}
}}
/>
</div>
);
}
+45
View File
@@ -0,0 +1,45 @@
import { useEffect } from 'react';
import { ToastEntry } from './types';
interface Props {
toasts: ToastEntry[];
onDismiss: (id: number) => void;
}
export function ToastStack({ toasts, onDismiss }: Props) {
return (
<div className="kv-toast-stack">
{toasts.map((t) => (
<Toast key={t.id} entry={t} onDismiss={() => onDismiss(t.id)} />
))}
</div>
);
}
function Toast({ entry, onDismiss }: { entry: ToastEntry; onDismiss: () => void }) {
useEffect(() => {
if (entry.kind === 'error') return;
const timer = setTimeout(onDismiss, 4500);
return () => clearTimeout(timer);
}, [entry.kind, onDismiss]);
return (
<div className={`kv-toast kv-toast-${entry.kind}`}>
<div className="kv-toast-row">
<span className="kv-toast-text">{entry.text}</span>
<button className="kv-toast-close" onClick={onDismiss} aria-label="dismiss">×</button>
</div>
{entry.details && entry.details.length > 0 && (
<ul className="kv-toast-details">
{entry.details.slice(0, 8).map((d, i) => <li key={i}>{d}</li>)}
{entry.details.length > 8 && <li>… {entry.details.length - 8} more</li>}
</ul>
)}
</div>
);
}
let nextId = 1;
export function makeToast(kind: ToastEntry['kind'], text: string, details?: string[]): ToastEntry {
return { id: nextId++, kind, text, details };
}
+295
View File
@@ -0,0 +1,295 @@
import { useEffect, useState } from 'react';
import { LanguageCombobox } from '../components/LanguageCombobox';
import { FieldType } from './types';
interface ValueRendererProps {
value: string;
onChange: (v: string) => void;
}
interface LeafValueEditorProps extends ValueRendererProps {
type: FieldType;
}
/**
* Type-dispatched leaf value editor. Used by both the single-leaf editor
* (KVLeafEditor) and the per-row editor in the file-detail view.
*/
export function LeafValueEditor({ type, value, onChange }: LeafValueEditorProps) {
switch (type) {
case 'bool':
return (
<select
className="kv-leaf-input"
value={value === 'true' ? 'true' : value === 'false' ? 'false' : ''}
onChange={(e) => onChange(e.target.value)}
>
<option value="">(unset)</option>
<option value="true">true</option>
<option value="false">false</option>
</select>
);
case 'number':
return (
<input
type="number"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'datetime':
return <DateTimeLeaf value={value} onChange={onChange} />;
case 'lang':
return <LanguageCombobox value={value} onChange={onChange} />;
case 'url':
return (
<div className="kv-leaf-url">
<input
type="url"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
{value && (
<a className="kv-leaf-url-open" href={value} target="_blank" rel="noreferrer">
open ↗
</a>
)}
</div>
);
case 'json':
return (
<textarea
className="kv-leaf-textarea kv-leaf-mono"
rows={12}
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
);
case 'text':
return (
<textarea
className="kv-leaf-textarea"
rows={8}
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'hash':
return <CidLeaf value={value} onChange={onChange} />;
case 'auto':
case 'string':
default:
return (
<input
type="text"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
}
}
interface CidFileInfo {
exists: boolean;
contentType?: string;
size?: number;
filePath?: string;
}
/**
* CID leaf editor. Shows the value as an editable monospace input plus a
* content-typed preview when meta-core can resolve the CID to a file on disk
* (`/api/file/{cid}/info` → image/video/audio rendering, else download-only).
*/
function CidLeaf({ value, onChange }: ValueRendererProps) {
const [info, setInfo] = useState<CidFileInfo | null>(null);
const cid = value.trim();
const fileUrl = cid ? `/file/${cid}` : '';
const downloadName = info?.filePath?.split('/').pop() || cid;
useEffect(() => {
if (!cid) {
setInfo(null);
return;
}
let alive = true;
fetch(`/api/file/${cid}/info`)
.then((r) => (r.ok ? r.json() : { exists: false }))
.then((data: CidFileInfo) => {
if (alive) setInfo(data);
})
.catch(() => {
if (alive) setInfo({ exists: false });
});
return () => {
alive = false;
};
}, [cid]);
const renderPreview = () => {
if (!info?.exists || !info.contentType || !cid) return null;
if (info.contentType.startsWith('image/')) {
return <img src={fileUrl} alt="" className="kv-leaf-cid-preview" loading="lazy" />;
}
if (info.contentType.startsWith('video/')) {
return (
<video
src={fileUrl}
className="kv-leaf-cid-preview kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
if (info.contentType.startsWith('audio/')) {
return (
<audio
src={fileUrl}
className="kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
return null;
};
return (
<div className="kv-leaf-cid">
{renderPreview()}
<div className="kv-leaf-cid-controls">
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
<button
type="button"
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(value)}
title="copy CID"
>
⧉
</button>
</div>
{info?.exists && cid && (
<div className="kv-leaf-cid-actions">
<a
href={fileUrl}
download={downloadName}
className="kv-btn kv-btn-primary"
title={`Download ${downloadName}${info.size ? ` (${formatBytes(info.size)})` : ''}`}
>
⬇ Download
</a>
<a href={fileUrl} className="kv-btn" target="_blank" rel="noreferrer">
open ↗
</a>
<span className="kv-leaf-cid-meta">
{info.contentType}
{info.size != null ? ` · ${formatBytes(info.size)}` : ''}
</span>
</div>
)}
</div>
);
}
/** Timestamp shapes we round-trip through the picker. */
type TimestampShape = 'iso-date' | 'iso-datetime' | 'unix-s' | 'unix-ms';
function detectShape(raw: string): TimestampShape | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) return 'iso-date';
if (/^\d+$/.test(trimmed)) {
const n = Number.parseInt(trimmed, 10);
if (n >= 1_000_000_000 && n <= 4_102_444_800) return 'unix-s';
if (n >= 1_000_000_000_000 && n <= 4_102_444_800_000) return 'unix-ms';
return null;
}
if (!Number.isNaN(Date.parse(trimmed))) return 'iso-datetime';
return null;
}
function rawToDate(raw: string, shape: TimestampShape): Date | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (shape === 'unix-s') return new Date(Number.parseInt(trimmed, 10) * 1000);
if (shape === 'unix-ms') return new Date(Number.parseInt(trimmed, 10));
const d = new Date(trimmed);
return Number.isNaN(d.getTime()) ? null : d;
}
function dateToRaw(d: Date, shape: TimestampShape): string {
switch (shape) {
case 'unix-s':
return String(Math.floor(d.getTime() / 1000));
case 'unix-ms':
return String(d.getTime());
case 'iso-date':
return d.toISOString().slice(0, 10);
case 'iso-datetime':
return d.toISOString();
}
}
/**
* Datetime leaf editor. Preserves the original storage shape — ISO date stays
* ISO date, unix seconds stay unix seconds, etc. Unknown shapes fall back to
* iso-datetime.
*/
function DateTimeLeaf({ value, onChange }: ValueRendererProps) {
const shape: TimestampShape = detectShape(value) ?? 'iso-datetime';
const date = rawToDate(value, shape);
const inputType = shape === 'iso-date' ? 'date' : 'datetime-local';
const inputValue = (() => {
if (!date || Number.isNaN(date.getTime())) return '';
if (inputType === 'date') return date.toISOString().slice(0, 10);
// datetime-local wants YYYY-MM-DDTHH:mm in the user's local timezone.
const local = new Date(date.getTime() - date.getTimezoneOffset() * 60000);
return local.toISOString().slice(0, 16);
})();
return (
<div className="kv-leaf-datetime">
<input
type={inputType}
className="kv-leaf-input"
value={inputValue}
onChange={(e) => {
const v = e.target.value;
if (!v) {
onChange('');
return;
}
const parsed = new Date(v);
if (Number.isNaN(parsed.getTime())) return;
onChange(dateToRaw(parsed, shape));
}}
/>
<span className="kv-leaf-datetime-shape" title="Storage format preserved on save">
{shape}
</span>
</div>
);
}
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
const units = ['KB', 'MB', 'GB', 'TB'];
let v = n / 1024;
let i = 0;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[i]}`;
}
+90
View File
@@ -0,0 +1,90 @@
import { SchemaFieldInfo, SchemaResponse } from '../api/schemaApi';
import { FieldType } from './types';
export interface SchemaLookup {
/** The matched schema entry, if any. */
info?: SchemaFieldInfo;
/** Suggested FieldType for the leaf editor, or null when schema can't decide. */
fieldType: FieldType | null;
/**
* True when the leaf is one variant under a language-keyed group
* (e.g. plot/eng). The schema entry returned is the group's, not the leaf's.
*/
isLanguageVariant: boolean;
}
/**
* Resolve the logical schema entry for a full Redis key.
*
* - "file:<hash>/releasedate" → schema.fields["releasedate"]
* - "file:<hash>/plot/eng" → schema.fields["plot"] (language-keyed group)
* - non-file keys → undefined (schema is per-file today)
*/
export function lookupSchemaForKey(
schema: SchemaResponse | null,
key: string,
): SchemaLookup {
if (!schema) {
return { fieldType: null, isLanguageVariant: false };
}
const propertyPath = stripFilePrefix(key);
if (!propertyPath) {
return { fieldType: null, isLanguageVariant: false };
}
const direct = schema.fields[propertyPath];
if (direct) {
return {
info: direct,
fieldType: schemaToFieldType(direct),
isLanguageVariant: false,
};
}
// Language-keyed collapse: tail is a 2/3-letter lang code under a known group.
const lastSlash = propertyPath.lastIndexOf('/');
if (lastSlash > 0) {
const tail = propertyPath.slice(lastSlash + 1);
if (/^[a-z]{2,3}$/.test(tail)) {
const group = schema.fields[propertyPath.slice(0, lastSlash)];
if (group?.key_hint === 'language-code') {
return {
info: group,
fieldType: schemaToFieldType(group),
isLanguageVariant: true,
};
}
}
}
return { fieldType: null, isLanguageVariant: false };
}
export function schemaToFieldType(info: SchemaFieldInfo | undefined): FieldType | null {
if (!info) return null;
if (info.hint === 'cid') return 'hash';
if (info.hint === 'timestamp') return 'datetime';
switch (info.type) {
case 'bool':
return 'bool';
case 'int':
case 'float':
return 'number';
case 'json-object':
case 'json-array':
return 'json';
case 'string':
case 'mixed':
case 'undefined':
default:
return null;
}
}
function stripFilePrefix(key: string): string {
if (!key.startsWith('file:')) return '';
const slash = key.indexOf('/');
if (slash < 0) return '';
return key.slice(slash + 1);
}
+59
View File
@@ -0,0 +1,59 @@
import { FieldType } from './types';
// Path-pattern table. The leftmost segment of the path that matches wins.
// Order matters: more specific patterns should appear first.
const PATH_PATTERNS: Array<[RegExp, FieldType]> = [
[/(^|\/)(language|lang|audioLang|subLang)$/i, 'lang'],
[/(^|\/)(cid_|hash$|sha\d|midhash|md5|crc32)/i, 'hash'],
[/(^|\/)(duration|bitrate|fps|width|height|count|size|sizeByte|increment|episode|season|year|runtime|trackId|index)$/i, 'number'],
[/(^|\/)(enabled|active|disabled|isHidden|hidden|favorite|watched)$/i, 'bool'],
[/(^|\/)(url|uri|href|link|posterUrl|backdropUrl|imageUrl)$/i, 'url'],
[/(^|\/)(.+_at|.+At|.+Date|.+Time|timestamp|createdAt|updatedAt)$/i, 'datetime'],
[/(^|\/)(description|overview|notes?|comment|summary|nfo)$/i, 'text'],
];
function detectFromPath(path: string): FieldType | null {
for (const [re, t] of PATH_PATTERNS) {
if (re.test(path)) return t;
}
return null;
}
function detectFromValue(v: string): FieldType {
if (v === '') return 'string';
if (v === 'true' || v === 'false') return 'bool';
if (/^-?\d+$/.test(v)) return 'number';
if (/^-?\d*\.\d+$/.test(v)) return 'number';
if (/^https?:\/\//i.test(v)) return 'url';
const trimmed = v.trim();
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
try { JSON.parse(trimmed); return 'json'; } catch {}
}
if (v.includes('\n') || v.length > 200) return 'text';
return 'string';
}
// Combine path and value heuristics. Path wins when it matches; otherwise
// fall back to value shape.
export function detectType(path: string, value: string): FieldType {
return detectFromPath(path) ?? detectFromValue(value);
}
export const ALL_TYPES: FieldType[] = [
'auto', 'string', 'text', 'number', 'bool', 'json', 'url', 'lang', 'datetime', 'hash',
];
export function describeType(t: FieldType): string {
switch (t) {
case 'auto': return 'auto';
case 'string': return 'string';
case 'text': return 'text (multiline)';
case 'number': return 'number';
case 'bool': return 'bool';
case 'json': return 'json';
case 'url': return 'url';
case 'lang': return 'language code';
case 'datetime': return 'datetime';
case 'hash': return 'hash / cid';
}
}
+19
View File
@@ -0,0 +1,19 @@
// Inferred type for a leaf value. "auto" defers to the heuristic detector.
export type FieldType =
| 'auto'
| 'string'
| 'number'
| 'bool'
| 'json'
| 'url'
| 'text'
| 'lang'
| 'datetime'
| 'hash';
export interface ToastEntry {
id: number;
kind: 'info' | 'success' | 'error';
text: string;
details?: string[];
}
+2 -2
View File
@@ -1,10 +1,10 @@
import { StrictMode } from 'react';
import { createRoot } from 'react-dom/client';
import './index.css';
import App from './App.tsx';
import KVApp from './kv/KVApp.tsx';
createRoot(document.getElementById('root')!).render(
<StrictMode>
<App />
<KVApp />
</StrictMode>,
);
-155
View File
@@ -1,155 +0,0 @@
// API Response Types
export interface FileMetadata {
// File info
fileName?: string;
extension?: string;
byteSize?: number;
mimeType?: string;
fileType?: string;
// Filename parsing
title?: string;
titles?: string[];
originaltitle?: string;
showtitle?: string;
season?: number;
episode?: number;
movieYear?: number;
videoType?: string;
// General media metadata
year?: number;
imdbid?: string;
tmdbid?: string;
anidbid?: string;
premiered?: string;
genre?: string[];
releasedate?: string;
plot?: string;
director?: string;
rating?: number;
runtime?: number;
studio?: string;
art?: any;
actor?: any[];
aired?: string;
customrating?: string;
dateadded?: string;
sorttitle?: string;
mpaa?: string;
aspectratio?: string;
collectionnumber?: number;
criticrating?: number;
// Hashes
cid_crc32?: string;
cid_md5?: string;
cid_sha1?: string;
'cid_sha2-256'?: string;
'cid_sha3-256'?: string;
cid_sha3_384?: string;
// Video info
video?: {
codec?: string;
width?: number;
height?: number;
bitrate?: number;
fps?: number;
};
// Audio info
audio?: Array<{
codec?: string;
channels?: number;
language?: string;
bitrate?: number;
}>;
// Subtitles
subtitles?: Array<{
language?: string;
codec?: string;
}>;
// Other
extra?: any;
anime?: any;
fileinfo?: any;
trackers?: string[];
// Allow any additional properties
[key: string]: any;
}
export interface SearchResult {
hashId: string;
metadata: FileMetadata;
score?: number;
}
export interface SearchResponse {
results: SearchResult[];
count: number;
total?: number;
}
export interface UpdateResult {
status: string;
hashId: string;
message?: string;
error?: string;
}
export interface BatchUpdateResponse {
status: string;
total: number;
success: number;
errors: number;
results: Array<{
hashId: string;
status: string;
error?: string;
}>;
}
// Plugin Schema Types
export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
export interface SelectOption {
value: string;
label: string;
}
export interface SchemaField {
label: string;
type?: SchemaFieldType;
readonly?: boolean;
indexed?: boolean;
options?: SelectOption[];
hint?: string;
link?: string;
pattern?: string;
min?: number;
max?: number;
placeholder?: string;
}
export interface PluginSchema {
id: string;
name: string;
version: string;
description?: string;
priority?: number;
color?: string;
dependencies: string[];
active: boolean;
metadataSchema?: Record<string, SchemaField>;
}
export interface PluginsResponse {
plugins: PluginSchema[];
executionOrder: string[];
activeCount: number;
totalCount: number;
}
-53
View File
@@ -1,53 +0,0 @@
/**
* Format bytes to human readable format
*/
export function formatBytes(bytes: number | undefined): string {
if (!bytes) return 'N/A';
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
if (bytes === 0) return '0 B';
const i = Math.floor(Math.log(bytes) / Math.log(1024));
return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i];
}
/**
* Format duration in seconds to HH:MM:SS
*/
export function formatDuration(seconds: number | undefined): string {
if (!seconds) return 'N/A';
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
if (hours > 0) {
return `${hours}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
}
return `${minutes}:${secs.toString().padStart(2, '0')}`;
}
/**
* Truncate hash to first and last 6 characters
*/
export function truncateHash(hash: string | undefined): string {
if (!hash) return 'N/A';
if (hash.length <= 16) return hash;
return `${hash.slice(0, 6)}...${hash.slice(-6)}`;
}
/**
* Get display name for a file from metadata
*/
export function getDisplayName(metadata: any): string {
if (metadata.title) {
if (metadata.season && metadata.episode) {
return `${metadata.title} S${String(metadata.season).padStart(2, '0')}E${String(metadata.episode).padStart(2, '0')}`;
}
if (metadata.movieYear) {
return `${metadata.title} (${metadata.movieYear})`;
}
return metadata.title;
}
return metadata.fileName || 'Unknown';
}
+1
View File
@@ -0,0 +1 @@
{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/schemaApi.ts","./src/api/snapshotApi.ts","./src/components/LanguageCombobox.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVFileEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/leafRenderers.tsx","./src/kv/schemaLookup.ts","./src/kv/typeHeuristics.ts","./src/kv/types.ts"],"version":"5.9.3"}
+8 -1
View File
@@ -3,13 +3,20 @@ module github.com/metazla/meta-core
go 1.21
require (
github.com/alicebob/miniredis/v2 v2.31.1
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1
github.com/mr-tron/base58 v1.3.0
github.com/redis/go-redis/v9 v9.4.0
golang.org/x/net v0.20.0
golang.org/x/crypto v0.31.0
golang.org/x/net v0.21.0
)
require (
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a // indirect
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/yuin/gopher-lua v1.1.0 // indirect
golang.org/x/sys v0.28.0
)
+24 -2
View File
@@ -1,16 +1,38 @@
github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a h1:HbKu58rmZpUGpz5+4FfNmIU+FmZg2P3Xaj2v2bfNWmk=
github.com/alicebob/gopher-json v0.0.0-20200520072559-a9ecdc9d1d3a/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc=
github.com/alicebob/miniredis/v2 v2.31.1 h1:7XAt0uUg3DtwEKW5ZAGa+K7FZV2DdKQo5K/6TTnfX8Y=
github.com/alicebob/miniredis/v2 v2.31.1/go.mod h1:UB/T2Uztp7MlFSDakaX1sTXUv5CASoprx0wulRT6HBg=
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI=
github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI=
github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU=
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
github.com/yuin/gopher-lua v1.1.0 h1:BojcDhfyDWgU2f2TOzYK/g5p2gxMrku8oupLDqlnSqE=
github.com/yuin/gopher-lua v1.1.0/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+131
View File
@@ -0,0 +1,131 @@
package api
import (
"net/http"
"strconv"
)
// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
// stranded midhash-rooted entries into their UUID roots. See
// storage.MigrateDualRoots for the data-flow detail and
// docs/api-mediated-access.md for the bug background.
//
// Returns {fixed: N} on success. Safe to call repeatedly — entries already
// migrated are skipped on the next pass.
func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.MigrateDualRootsWithTimeout()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
// handleMigrateDomainScreen triggers the one-shot sweep that rewrites the
// retired `domain=film|tv` values to `screen` and backfills `workForm` from
// `contentKind` (METADATA_KEYS.md §14.17). See
// storage.MigrateDomainScreen for why the vocabulary change is paid in the
// data rather than with a read-side alias.
//
// Returns {fixed: N} — records changed, not records examined. Safe to call
// repeatedly: a record already on the current vocabulary is skipped.
func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.MigrateDomainScreenWithTimeout()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
// `anilistid` on a site card, and `workcid` / position keys on any card
// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
// why chapters and anchor cards' ids are deliberately left alone.
//
// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
// keys, so a bare call only reports what it would delete. An unparseable
// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
// sweep that ran.
//
// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
// records, byKey counts fields per key. Safe to call repeatedly: a swept
// record plans nothing on the next pass.
func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
apply := false
if v := r.URL.Query().Get("apply"); v != "" {
parsed, err := strconv.ParseBool(v)
if err != nil {
writeError(w, http.StatusBadRequest, "apply must be true or false")
return
}
apply = parsed
}
res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"dryRun": !apply,
"planned": res.Planned,
"fixed": res.Fixed,
"byKey": res.ByKey,
})
}
// handleMigrateUserCids triggers the one-shot sweep that builds
// `udl:idx:user:<uid>:cids` for User Data Layer records written before that
// index existed. See storage.UDLBackfillUserCids.
//
// The index is what makes whole-profile export a set read instead of a
// keyspace scan, and it is maintained by the upsert script from now on — so
// this only ever matters once, for data already on disk. Until it has run, a
// profile that predates the index exports as **empty**, which means a
// cross-instance recovery reports success having restored nothing.
//
// Returns {fixed: N} — cids added, not cids examined. Safe to call repeatedly:
// a second pass adds nothing.
func (s *Server) handleMigrateUserCids(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.UDLBackfillUserCids()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
+93
View File
@@ -0,0 +1,93 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
// newSweepTestServer is a Server over miniredis with one literature card
// carrying a copied `workcid`.
func newSweepTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
for k, v := range map[string]string{
"domain": "literature",
"fileType": "card",
"workcid": "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
} {
if err := c.SetProperty("card-1", k, v); err != nil {
t.Fatalf("seed %s: %v", k, err)
}
}
return &Server{storage: c}
}
type sweepResponse struct {
Success bool `json:"success"`
DryRun bool `json:"dryRun"`
Planned int `json:"planned"`
Fixed int `json:"fixed"`
ByKey map[string]int `json:"byKey"`
}
func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
t.Helper()
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
s.handleSweepLiteratureEchoes(rr, req)
var body sweepResponse
if rr.Code == http.StatusOK {
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
t.Fatalf("decode %s: %v", rr.Body.String(), err)
}
}
return rr, body
}
// A bare call must never delete: the default is the dry run.
func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
s := newSweepTestServer(t)
rr, body := postSweep(t, s, "")
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
}
if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
t.Errorf("dry run = %+v", body)
}
if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
t.Errorf("dry run deleted workcid")
}
rr, body = postSweep(t, s, "?apply=true")
if rr.Code != http.StatusOK {
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
}
if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
t.Errorf("apply = %+v", body)
}
if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
t.Errorf("apply left workcid = %q", v)
}
}
// A typo must not pass for a sweep that ran.
func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
s := newSweepTestServer(t)
rr, _ := postSweep(t, s, "?apply=yes-please")
if rr.Code != http.StatusBadRequest {
t.Errorf("status = %d, want 400", rr.Code)
}
}
+70
View File
@@ -0,0 +1,70 @@
package api
import (
"net/http"
"github.com/gorilla/mux"
)
// handleGetMetaByCID handles GET /api/meta/{cid}.
//
// Resolves any known CID — midhash256, sha256, IPFS, btih, or anything else
// in the reverse index — to the full metadata document for that file.
// Returns:
// - 200 with the document on a hit
// - 404 if the CID isn't registered
// - 503 if storage is down
//
// CIDs in the URL are bare multibase CIDv1 strings ("bafk…") — the midhash,
// a sibling sha2-256, an IPFS CID, a btih, anything registered in the
// reverse index. A CID is self-describing (its algorithm is the multicodec),
// so there is no <algo>: prefix.
//
// This endpoint is the public read API for the CID-resolution layer and
// is auth-bypassed alongside /api/file/{cid} so that meta-share peers and
// IPFS-style clients can use it without going through Authelia.
func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
uuid, err := s.storage.GetByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if uuid == "" {
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
doc, err := s.storage.GetMetadataDocument(uuid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
// Reverse index pointed at a UUID with no remaining keys — stale
// alias, probably mid-delete race. Treat as not-found.
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
// cids / canonical_cid / duplicates), with cid added so the caller can
// correlate when they fan out N lookups. canonical_cid is derived by
// rank from the cids key-set on read — it is not a stored field.
writeJSON(w, http.StatusOK, map[string]interface{}{
"cid": cid,
"metadata": doc.Flat,
"cids": doc.CIDs,
"canonical_cid": doc.Canonical,
"duplicates": doc.Duplicates,
})
}
+86
View File
@@ -0,0 +1,86 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
)
// The per-algorithm CID fields (`cid_sha2-256`, `cid_midhash256`, …) and the
// stored `canonical_cid` scalar were removed (METADATA_KEYS.md §14.13). They
// must be rejected at the write boundary rather than stored.
//
// Why a hard 400 and not a silent rewrite: the reverse-index hook
// (maybeAddAliasFromFieldLocked) only registers an alias for a field named
// `cids/<cid>`. A `cid_*` field was therefore written verbatim and never
// indexed — the record silently became unresolvable by that CID. A loud
// failure is strictly better than a record that looks correct and cannot be
// found.
func TestRejectDeprecatedField(t *testing.T) {
rejected := []string{
"cid_midhash256",
"cid_sha2-256",
"cid_btih_v2",
"cid_crc32",
"midhash256",
"canonical_cid",
}
for _, f := range rejected {
if !rejectDeprecatedField(f) {
t.Errorf("rejectDeprecatedField(%q) = false, want true (deprecated shape)", f)
}
}
accepted := []string{
"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
"filePath",
"title",
"fileinfo/duration",
"tmdb/poster",
// Near-misses that must NOT be caught: the prefix is `cid_`, with an
// underscore. A field merely starting with "cid" is fine.
"cids",
"cidrank",
}
for _, f := range accepted {
if rejectDeprecatedField(f) {
t.Errorf("rejectDeprecatedField(%q) = true, want false (legitimate field)", f)
}
}
}
func TestValidateFields_Rejects400WithSlug(t *testing.T) {
rr := httptest.NewRecorder()
blocked := validateFields(rr, map[string]string{
"title": "Inception",
"cid_midhash256": "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4",
})
if !blocked {
t.Fatal("validateFields returned false for a payload carrying cid_midhash256; the write would have been stored unindexed")
}
if rr.Code != http.StatusBadRequest {
t.Errorf("status = %d, want %d", rr.Code, http.StatusBadRequest)
}
var body map[string]interface{}
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
t.Fatalf("response is not JSON: %v", err)
}
if got := body["error"]; got != ErrDeprecatedField {
t.Errorf("error slug = %v, want %q", got, ErrDeprecatedField)
}
}
func TestValidateFields_AllowsCidsKeyset(t *testing.T) {
rr := httptest.NewRecorder()
blocked := validateFields(rr, map[string]string{
"filePath": "/watch/Inception.mkv",
"cids/bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4": "true",
"cids/baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua": "true",
})
if blocked {
t.Fatalf("validateFields rejected a legitimate cids/ key-set write: %s", rr.Body.String())
}
}
+132
View File
@@ -0,0 +1,132 @@
package api
import (
"encoding/json"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/metazla/meta-core/internal/watcher"
)
// registerFileRequest is the body of POST /api/files/register.
type registerFileRequest struct {
// Path relative to FILES_PATH, e.g. "plugin/subtitles/ab12….eng.srt".
Path string `json:"path"`
}
type registerFileResponse struct {
Root string `json:"root"`
CID string `json:"cid"`
Created bool `json:"created"`
Path string `json:"path"`
Size int64 `json:"size"`
}
// pluginRelPath normalises a FILES_PATH-relative path and refuses anything
// outside plugin/. Plugin output is the only tree a service may register this
// way: /files/watch is the user's library and belongs to the watcher.
func pluginRelPath(raw string) (string, bool) {
p := strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(raw)), "/")
if p == "" {
return "", false
}
for _, seg := range strings.Split(p, "/") {
if seg == ".." {
return "", false
}
}
clean := path.Clean(p)
if !strings.HasPrefix(clean, "plugin/") || clean == "plugin/" {
return "", false
}
return clean, true
}
// handleRegisterFile handles POST /api/files/register.
//
// Does for ONE file what the watcher does for every file it scans: midhash256
// it, mint a root carrying filePath/sizeByte/mtimeNano, register the
// cid:<midhash> alias, and append a file:events entry so meta-share's
// ipfs_seed chunks, seeds and provides it.
//
// It exists for cores that run with ENABLE_FILE_WATCHER=false (a meta-watch
// client box). Without it, bytes written under /files/plugin over WebDAV are
// never resolvable by CID and never reach the swarm. Idempotent: the same
// content at the same path returns the existing root with created=false.
func (s *Server) handleRegisterFile(w http.ResponseWriter, r *http.Request) {
if s.storage == nil || !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req registerFileRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
rel, ok := pluginRelPath(req.Path)
if !ok {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
fullPath := filepath.Join(s.config.FilesPath, filepath.FromSlash(rel))
absFiles, _ := filepath.Abs(filepath.Join(s.config.FilesPath, "plugin"))
absFull, _ := filepath.Abs(fullPath)
if !strings.HasPrefix(absFull, absFiles+string(filepath.Separator)) {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
info, err := os.Stat(fullPath)
if err != nil {
writeError(w, http.StatusNotFound, "file not found")
return
}
if info.IsDir() {
writeError(w, http.StatusBadRequest, "path is a directory, not a file")
return
}
midhash, err := watcher.ComputeMidHash256(fullPath, info.Size())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to hash file")
return
}
root, created, err := s.storage.RegisterFileTuple(fullPath, info.Size(), info.ModTime().UnixNano(), midhash)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if created {
ev := watcher.FileEvent{
Type: watcher.EventTypeAdd,
Path: rel,
Size: info.Size(),
MidHash256: midhash,
Timestamp: watcher.NowMS(),
}
if err := watcher.PublishEvent(s.storage, ev); err != nil {
// The record is written and resolvable; only seeding is late.
writeError(w, http.StatusInternalServerError, "registered, but file event not published: "+err.Error())
return
}
}
status := http.StatusOK
if created {
status = http.StatusCreated
}
writeJSON(w, status, registerFileResponse{
Root: root,
CID: midhash,
Created: created,
Path: rel,
Size: info.Size(),
})
}
+144
View File
@@ -0,0 +1,144 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/storage"
)
// newRegisterTestServer wires handleRegisterFile to an in-memory Redis and a
// temp FILES_PATH. Built directly, not through NewServer, so no watcher runs —
// which is exactly the configuration this route exists for.
func newRegisterTestServer(t *testing.T) (*Server, string, *miniredis.Miniredis) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
files := t.TempDir()
return &Server{storage: c, config: &config.Config{FilesPath: files}}, files, mr
}
func writeFixture(t *testing.T, files, rel, body string) string {
t.Helper()
full := filepath.Join(files, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return full
}
func postRegister(t *testing.T, s *Server, path string) (int, registerFileResponse) {
t.Helper()
body, _ := json.Marshal(registerFileRequest{Path: path})
req := httptest.NewRequest(http.MethodPost, "/api/files/register", strings.NewReader(string(body)))
rr := httptest.NewRecorder()
s.handleRegisterFile(rr, req)
var resp registerFileResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func streamLen(t *testing.T, mr *miniredis.Miniredis) int {
t.Helper()
entries, err := mr.Stream("file:events")
if err != nil {
return 0
}
return len(entries)
}
// The whole point of the route: with the watcher off, a file written under
// plugin/ becomes a record resolvable by its midhash, and meta-share hears
// about it on file:events.
func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
rel := "plugin/subtitles/ab12cd34.eng.srt"
full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
code, resp := postRegister(t, s, rel)
if code != http.StatusCreated || !resp.Created {
t.Fatalf("want 201 created, got %d %+v", code, resp)
}
if !strings.HasPrefix(resp.CID, "b") || resp.Root == "" {
t.Fatalf("want a multibase cid and a root, got %+v", resp)
}
root, err := s.storage.GetByCID(resp.CID)
if err != nil || root != resp.Root {
t.Fatalf("cid alias must resolve to the root: got %q err=%v, want %q", root, err, resp.Root)
}
if fp, _ := s.storage.GetProperty(resp.Root, "filePath"); fp != full {
t.Fatalf("filePath = %q, want %q", fp, full)
}
entries, err := mr.Stream("file:events")
if err != nil || len(entries) != 1 {
t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
}
vals := strings.Join(entries[0].Values, "|")
if !strings.Contains(vals, "midhash256|"+resp.CID) || !strings.Contains(vals, "path|"+rel) {
t.Fatalf("event must carry the midhash and the relative path, got %q", vals)
}
}
// Re-uploading the same subtitle is a no-op: same root, no second event (which
// would make meta-share re-chunk and re-provide the file).
func TestRegisterFile_IsIdempotent(t *testing.T) {
s, files, mr := newRegisterTestServer(t)
rel := "plugin/subtitles/same.eng.srt"
writeFixture(t, files, rel, "same bytes")
_, first := postRegister(t, s, rel)
code, second := postRegister(t, s, rel)
if code != http.StatusOK || second.Created {
t.Fatalf("second register must be 200 created=false, got %d %+v", code, second)
}
if second.Root != first.Root || second.CID != first.CID {
t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
}
if n := streamLen(t, mr); n != 1 {
t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
}
}
// Only plugin output may be registered here. /files/watch is the user's
// library and belongs to the watcher; traversal must not escape either.
func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
writeFixture(t, files, "watch/movie.mkv", "x")
for _, p := range []string{
"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
"plugin/../watch/movie.mkv", "../etc/passwd", "plugin/../../etc/passwd",
} {
if code, _ := postRegister(t, s, p); code != http.StatusBadRequest {
t.Errorf("path %q: want 400, got %d", p, code)
}
}
}
func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
s, files, _ := newRegisterTestServer(t)
if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
t.Errorf("missing file: want 404, got %d", code)
}
if err := os.MkdirAll(filepath.Join(files, "plugin", "subtitles", "dir"), 0o755); err != nil {
t.Fatal(err)
}
if code, _ := postRegister(t, s, "plugin/subtitles/dir"); code != http.StatusBadRequest {
t.Errorf("directory: want 400, got %d", code)
}
}
+74
View File
@@ -0,0 +1,74 @@
package api
import "net/http"
// fileTupleJSON is one file-backed root on the wire.
type fileTupleJSON struct {
HashID string `json:"hashId"`
FilePath string `json:"filePath"`
SizeByte int64 `json:"sizeByte"`
MtimeNano int64 `json:"mtimeNano"`
}
// FileTuplesResponse is the body of GET /api/files/tuples.
//
// Files is a pointer so the summary form can omit it while the full form
// always carries it — an empty library is `"files": []`, never an absent key,
// which a client could mistake for a meta-core that lacks the rows.
type FileTuplesResponse struct {
Files *[]fileTupleJSON `json:"files,omitempty"`
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
}
// handleGetFileTuples handles GET /api/files/tuples.
//
// Lists the FILE-BACKED roots — records carrying filePath, sizeByte and
// mtimeNano, i.e. what the watcher or /api/files/register minted — with their
// sizes, plus the count and total. Gateway / cid-rooted records carry no file
// and are not included; neither is a locator record that only merged a cache
// file's metadata (it has no mtimeNano).
//
// It exists so a client never walks every hashId to answer "how many files,
// how big": GetTuplesForAllFiles reads the three fields for every root in
// chunked MGETs server-side (~250 round-trips for 82k roots, no SCAN), where
// the client-side equivalent was one HTTP request per record.
//
// ?summary=1 omits the rows and returns only {count, totalSize}.
func (s *Server) handleGetFileTuples(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
tuples, err := s.storage.GetTuplesForAllFiles()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
summary := r.URL.Query().Get("summary")
withRows := summary == "" || summary == "0" || summary == "false"
resp := FileTuplesResponse{Count: len(tuples)}
var rows []fileTupleJSON
if withRows {
rows = make([]fileTupleJSON, 0, len(tuples))
}
for _, t := range tuples {
resp.TotalSize += t.Size
if withRows {
rows = append(rows, fileTupleJSON{
HashID: t.HashID,
FilePath: t.FilePath,
SizeByte: t.Size,
MtimeNano: t.MtimeNano,
})
}
}
if withRows {
resp.Files = &rows
}
writeJSON(w, http.StatusOK, resp)
}
+114
View File
@@ -0,0 +1,114 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sort"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return &Server{storage: c}, c
}
func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesResponse) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/api/files/tuples"+query, nil)
rr := httptest.NewRecorder()
s.handleGetFileTuples(rr, req)
var resp FileTuplesResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
t.Helper()
id, err := c.Mint(path, size, mtime)
if err != nil {
t.Fatalf("mint %s: %v", path, err)
}
return id
}
// Only records that are actually files count: a gateway record and a record
// that merely carries a path + size (no mtimeNano) are both left out.
func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
s, c := newFileTuplesTestServer(t)
a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
if err := c.SetMetadataFlat("bagcsgatewayrecord", map[string]string{"title": "Some Release"}); err != nil {
t.Fatal(err)
}
if err := c.SetMetadataFlat("01partialroot", map[string]string{"filePath": "/files/cache/x.mkv", "sizeByte": "999"}); err != nil {
t.Fatal(err)
}
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
if resp.Files == nil || len(*resp.Files) != 2 {
t.Fatalf("want 2 rows, got %+v", resp.Files)
}
got := []string{(*resp.Files)[0].HashID, (*resp.Files)[1].HashID}
want := []string{a, b}
sort.Strings(got)
sort.Strings(want)
if got[0] != want[0] || got[1] != want[1] {
t.Fatalf("want roots %v, got %v", want, got)
}
}
func TestFileTuples_SummaryOmitsRows(t *testing.T) {
s, c := newFileTuplesTestServer(t)
mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
code, resp := getFileTuples(t, s, "?summary=1")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files != nil {
t.Fatalf("summary must omit rows, got %d", len(*resp.Files))
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
}
// An empty library still carries "files": [] — clients treat an absent key as
// "this meta-core cannot list files", so it must not disappear when empty.
func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
s, _ := newFileTuplesTestServer(t)
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files == nil || len(*resp.Files) != 0 || resp.Count != 0 {
t.Fatalf("want an empty files array and count 0, got %+v", resp)
}
}
func TestFileTuples_StorageNotConnected(t *testing.T) {
s := &Server{storage: storage.NewClient("")}
code, _ := getFileTuples(t, s, "")
if code != http.StatusServiceUnavailable {
t.Fatalf("want 503, got %d", code)
}
}
+237 -62
View File
@@ -4,6 +4,7 @@ import (
"crypto/sha256"
"encoding/base32"
"encoding/json"
"fmt"
"io"
"net/http"
"os"
@@ -13,6 +14,7 @@ import (
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
)
// contentTypeByExt maps file extensions to MIME types
@@ -63,10 +65,85 @@ type DataPathResponse struct {
Exists bool `json:"exists"`
}
// ErrorResponse is the response for errors
// ErrorResponse is the typed error envelope returned on every 4xx/5xx.
//
// Shape and slug vocabulary documented in docs/api-mediated-access.md
// ("Error envelope"). Stable slug strings let clients switch on the failure
// category without parsing the message; `retryable` tells callers whether
// a backoff-and-retry has any chance of succeeding.
type ErrorResponse struct {
Error string `json:"error"`
Message string `json:"message,omitempty"`
Error string `json:"error"`
Message string `json:"message,omitempty"`
Retryable bool `json:"retryable"`
}
// Stable error-slug strings. Pinned in the doc; consumers may match on
// these. Do not rename without a deprecation cycle.
const (
ErrAliasCollision = "alias_collision"
ErrUnknownRoot = "unknown_root"
ErrUnknownCID = "unknown_cid"
ErrSchemaViolation = "schema_violation"
ErrStorageUnavailable = "storage_unavailable"
ErrInternal = "internal"
ErrDeprecatedField = "deprecated_field"
)
// rejectDeprecatedField reports whether a field name is one of the removed
// per-algorithm CID shapes (`cid_sha2-256`, `cid_midhash256`, `midhash256`, …).
//
// This is a tripwire, not a compatibility shim. Such a field used to be
// accepted and written verbatim — but the reverse-index hook
// (`maybeAddAliasFromFieldLocked`) only registers aliases for fields named
// `cids/<cid>`, so a `cid_*` field was stored and then NEVER INDEXED: the
// record silently became unresolvable by that CID. Failing the write loudly is
// strictly better than a record that looks fine and cannot be found.
//
// Writers must emit the bare-CID key-set instead: `cids/<bareCid>` = "true".
// See METADATA_KEYS.md §2/§14.13.
func rejectDeprecatedField(field string) bool {
return strings.HasPrefix(field, "cid_") ||
field == "midhash256" ||
field == "canonical_cid"
}
// validateFields rejects a write carrying any deprecated CID field. Returns
// true if the request was rejected (the handler must return immediately).
func validateFields(w http.ResponseWriter, metadata map[string]string) bool {
for field := range metadata {
if rejectDeprecatedField(field) {
writeErrorSlug(w, http.StatusBadRequest, ErrDeprecatedField,
fmt.Sprintf("field %q is a removed per-algorithm CID shape; "+
"write the bare-CID key-set instead: \"cids/<bareCid>\": \"true\" "+
"(a cid_* field would be stored but never indexed, leaving the "+
"record unresolvable by that CID). See METADATA_KEYS.md §2/§14.13.",
field),
false)
return true
}
}
return false
}
// defaultSlugFor maps an HTTP status code to the most likely error slug.
// Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
// instead of "unknown_root" on 404) call writeErrorSlug directly.
func defaultSlugFor(status int) (slug string, retryable bool) {
switch status {
case http.StatusConflict:
return ErrAliasCollision, false
case http.StatusNotFound:
return ErrUnknownRoot, false
case http.StatusBadRequest:
return ErrSchemaViolation, false
case http.StatusServiceUnavailable:
return ErrStorageUnavailable, true
case http.StatusInternalServerError:
return ErrInternal, true
default:
// 4xx → not retryable, 5xx → retryable. Conservative default.
return ErrInternal, status >= 500
}
}
var startTime = time.Now()
@@ -169,6 +246,10 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
return
}
// Resolve CID → UUID via the reverse index so reads find data wherever
// the watcher minted the root, regardless of which CID the caller knows.
hashID = s.storage.ResolveRoot(hashID)
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -203,12 +284,20 @@ func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
return
}
// CID → UUID resolution so writes land on the canonical root, not on
// a parallel midhash-rooted entry the watcher's UUID never sees.
hashID = s.storage.ResolveRoot(hashID)
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if validateFields(w, metadata) {
return
}
if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
@@ -235,6 +324,8 @@ func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
return
}
hashID = s.storage.ResolveRoot(hashID)
deleted, err := s.storage.DeleteMetadata(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -340,6 +431,63 @@ func (s *Server) handleHeadData(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// handleGetFileByCIDInfo handles GET /api/file/{cid}/info.
// Returns lightweight metadata for the file (existence, content type, size,
// relative path) without streaming any bytes. Used by the editor to decide
// whether to render a preview (image/video) or a download-only button.
func (s *Server) handleGetFileByCIDInfo(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
cid := vars["cid"]
type infoResp struct {
Exists bool `json:"exists"`
ContentType string `json:"contentType,omitempty"`
Size int64 `json:"size,omitempty"`
FilePath string `json:"filePath,omitempty"`
}
w.Header().Set("Content-Type", "application/json")
if cid == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
relPath, err := s.storage.LookupPathByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if relPath == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
fullPath := filepath.Join(s.config.FilesPath, relPath)
fi, err := os.Stat(fullPath)
if err != nil {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false, FilePath: relPath})
return
}
ext := strings.ToLower(filepath.Ext(fullPath))
contentType, ok := contentTypeByExt[ext]
if !ok {
contentType = "application/octet-stream"
}
_ = json.NewEncoder(w).Encode(infoResp{
Exists: true,
ContentType: contentType,
Size: fi.Size(),
FilePath: relPath,
})
}
// handleGetFileByCID handles GET /file/{cid}
// Serves a file by looking up its CID in poster/backdrop metadata
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
@@ -364,7 +512,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
}
if relPath == "" {
writeError(w, http.StatusNotFound, "file not found for CID")
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file not found for CID", false)
return
}
@@ -374,7 +522,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
// Check if file exists
fileInfo, err := os.Stat(fullPath)
if err != nil {
writeError(w, http.StatusNotFound, "file does not exist on disk")
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file does not exist on disk", false)
return
}
@@ -400,64 +548,51 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
}
// handleListServices handles GET /services and GET /api/services
// Supports optional ?current=<service-name> query parameter
func (s *Server) handleListServices(w http.ResponseWriter, r *http.Request) {
services, err := s.discovery.DiscoverAll()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
//
// Beacon v2 (docs/project-architecture/beacon-v2.md). The list is this node's
// in-memory view of the UDP group, so it works across stacks that share a
// network but no volume. Instances of one name collapse to the most recent
// (one row per service, the nav menu's shape); `?all=1` returns every
// instance, `?cap=<pattern>` keeps only nodes with a matching capability.
func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
response := map[string]interface{}{
"services": services,
"count": len(services),
"current": s.config.ServiceName,
"neighbors": []interface{}{},
"count": 0,
}
// Add 'current' field if query parameter is provided
current := r.URL.Query().Get("current")
if current != "" {
response["current"] = current
if s.mesh == nil {
// Discovery off or failed to bind. Report empty rather than 503 —
// the nav is decoration and callers already treat [] as "hide".
response["enabled"] = false
writeJSON(w, http.StatusOK, response)
return
}
var list []meshdisco.Neighbor
if r.URL.Query().Get("all") != "" {
list = s.mesh.Neighbors()
} else {
list = s.mesh.NeighborsByName()
}
if pattern := r.URL.Query().Get("cap"); pattern != "" {
kept := list[:0]
for _, nb := range list {
if nb.Matches(pattern) {
kept = append(kept, nb)
}
}
list = kept
}
response["enabled"] = true
response["neighbors"] = list
response["count"] = len(list)
response["self"] = s.mesh.Self()
writeJSON(w, http.StatusOK, response)
}
// handleGetService handles GET /services/{name}
func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name := vars["name"]
if name == "" {
writeError(w, http.StatusBadRequest, "service name is required")
return
}
service, err := s.discovery.Discover(name)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if service == nil {
writeError(w, http.StatusNotFound, "service not found")
return
}
writeJSON(w, http.StatusOK, service)
}
// handleCleanupStats handles GET /services/cleanup/stats
func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
if s.cleaner == nil {
writeError(w, http.StatusServiceUnavailable, "cleaner not initialized")
return
}
stats := s.cleaner.Stats()
writeJSON(w, http.StatusOK, stats)
}
// writeJSON writes a JSON response
func writeJSON(w http.ResponseWriter, status int, data interface{}) {
w.Header().Set("Content-Type", "application/json")
@@ -465,11 +600,26 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
json.NewEncoder(w).Encode(data)
}
// writeError writes an error response
// writeError writes the typed error envelope. The slug + retryable bit
// are inferred from the status (see defaultSlugFor); call writeErrorSlug
// directly when a more specific slug applies.
func writeError(w http.ResponseWriter, status int, message string) {
slug, retryable := defaultSlugFor(status)
writeJSON(w, status, ErrorResponse{
Error: http.StatusText(status),
Message: message,
Error: slug,
Message: message,
Retryable: retryable,
})
}
// writeErrorSlug writes the typed envelope with an explicit slug and
// retryable bit. Use this for endpoint-specific slugs (alias_collision,
// unknown_cid) where the default mapping from HTTP status would be wrong.
func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, retryable bool) {
writeJSON(w, status, ErrorResponse{
Error: slug,
Message: message,
Retryable: retryable,
})
}
@@ -489,12 +639,21 @@ func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
return
}
// PATCH is the hot write path for meta-sort. Resolve so plugin output
// merges into the watcher's UUID root rather than minting a parallel
// midhash-rooted entry.
hashID = s.storage.ResolveRoot(hashID)
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if validateFields(w, metadata) {
return
}
updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -530,6 +689,8 @@ func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
return
}
hashID = s.storage.ResolveRoot(hashID)
value, err := s.storage.GetProperty(hashID, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -570,11 +731,17 @@ func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
return
}
if validateFields(w, map[string]string{key: ""}) {
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
hashID = s.storage.ResolveRoot(hashID)
// Read body
buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf)
@@ -629,6 +796,8 @@ func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
return
}
hashID = s.storage.ResolveRoot(hashID)
if err := s.storage.DeleteProperty(hashID, key); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
@@ -659,11 +828,17 @@ func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
return
}
if validateFields(w, map[string]string{key: ""}) {
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
hashID = s.storage.ResolveRoot(hashID)
// Read body
buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf)
@@ -772,10 +947,10 @@ func (s *Server) handleComputeFileCID(w http.ResponseWriter, r *http.Request) {
// CIDv1 format: version (0x01) + codec (0x55) + multihash
// Multihash format: hash-code (0x12) + length (0x20) + hash
cidBytes := make([]byte, 0, 2+2+32)
cidBytes = append(cidBytes, 0x01) // CIDv1
cidBytes = append(cidBytes, 0x55) // raw codec
cidBytes = append(cidBytes, 0x12) // sha256 code
cidBytes = append(cidBytes, 0x20) // 32 bytes
cidBytes = append(cidBytes, 0x01) // CIDv1
cidBytes = append(cidBytes, 0x55) // raw codec
cidBytes = append(cidBytes, 0x12) // sha256 code
cidBytes = append(cidBytes, 0x20) // 32 bytes
cidBytes = append(cidBytes, hashBytes...)
// Encode as base32lower with 'b' prefix (multibase)
+564
View File
@@ -0,0 +1,564 @@
package api
import (
"encoding/base64"
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"strings"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
// identityStatusResponse is the shape of GET /api/identity. The private key
// is intentionally absent — it is only ever returned once, by
// POST /api/identity/generate, immediately after creation.
type identityStatusResponse struct {
HasIdentity bool `json:"hasIdentity"`
UID string `json:"uid,omitempty"`
Curve string `json:"curve,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
// AccountCount is the total number of accounts in the keystore. Lets
// multi-account clients (meta-watch) tell "single identity" from "pick a
// profile" without a second call. Older single-identity clients ignore it.
AccountCount int `json:"accountCount,omitempty"`
}
// identityAccountInfo is one entry in GET /api/identity/accounts. Never carries
// the private key — that is only ever returned once, by generate.
type identityAccountInfo struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
}
type identityAccountsResponse struct {
Accounts []identityAccountInfo `json:"accounts"`
}
type identityGenerateResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportRequest struct {
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
// Created distinguishes "this key was new here" (201) from "this key was
// already in the keystore" (200). Import is the sign-in primitive for
// clients: presenting the secret key is the proof of ownership, so a
// re-import must succeed rather than conflict. Clients use the flag only
// to word the confirmation ("new profile" vs "welcome back").
Created bool `json:"created"`
}
// identityChallengeRequest asks for a one-shot challenge to sign.
type identityChallengeRequest struct {
UID string `json:"uid"`
Action string `json:"action"`
}
type identityChallengeResponse struct {
Challenge string `json:"challenge"`
ExpiresAt int64 `json:"expiresAt"`
}
// identityProof is the proof-of-possession carried by reveal and delete: the
// challenge text as issued, and a signature over it by the account's key.
//
// The signature is produced by whoever holds the key — the browser, locally.
// It is NOT obtainable from POST /api/identity/sign, which refuses any payload
// in the challenge domain; that refusal is what makes this proof mean
// something on a node that stores every private key in plaintext.
type identityProof struct {
Challenge string `json:"challenge"`
Signature string `json:"signature"`
}
// identityRevealRequest asks for an existing account's private key back.
//
// Gated on proof-of-possession, which makes it a "does this node hold the same
// key I do" check rather than the recovery path it once was. That is the
// deliberate consequence of the rule this file enforces: the private key, and
// nothing else, is authority over the account. An operator with root on the box
// can still read the file; an operator with only the API cannot.
type identityRevealRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
type identityRevealResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityDeleteRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
// identityDeleteResponse reports what the delete actually removed. The purge
// counts are the point: "account removed" alone leaves the caller unable to
// tell a clean removal from one that silently stranded a decade of ratings.
type identityDeleteResponse struct {
UID string `json:"uid"`
Purged storage.UDLUserStats `json:"purged"`
}
type identitySignRequest struct {
BytesB64 string `json:"bytesB64"`
// UID selects which account signs. Optional — when omitted and exactly one
// account exists, that account is used (single-identity back-compat).
UID string `json:"uid,omitempty"`
}
type identitySignResponse struct {
SigB64 string `json:"sigB64"`
}
// identitySignBatchRequest signs many payloads under one account in one call.
//
// Exists because a User Data Layer write signs exactly one record, and the bulk
// verbs above it ("mark this whole series seen") produce one record per episode:
// a 300-episode show is 300 sequential sign round trips through
// meta-watch -> meta-search -> here, which dominates the operation. The private
// key still never leaves this process; only the transport is batched.
type identitySignBatchRequest struct {
// PayloadsB64 are the canonical byte strings to sign, in order. The
// response's signatures line up index-for-index.
PayloadsB64 []string `json:"payloadsB64"`
// UID selects which account signs — same optional single-identity
// back-compat as identitySignRequest.
UID string `json:"uid,omitempty"`
}
type identitySignBatchResponse struct {
SigsB64 []string `json:"sigsB64"`
}
// maxSignBatch bounds one batch. A signature is cheap but not free, and the
// request holds the account lock for its duration; a caller with more than this
// should page. Sized well above the longest real series (Supernatural, 327).
const maxSignBatch = 1000
// reservedPayloadMessage explains the one refusal the signing endpoints make.
//
// This is the load-bearing half of proof-of-possession on a node that holds
// every private key: without it, a caller wanting to delete someone else's
// account just asks meta-core to sign the challenge for them, and the gate on
// reveal/delete becomes a formality. See identity.ChallengeDomain.
const reservedPayloadMessage = "payload is in the reserved account-authorisation domain (" +
identity.ChallengeDomain + "…); meta-core will not sign one. " +
"Signing a challenge is what proves you hold the key, so it must be done where the key is."
type identityAEADKeyResponse struct {
AEADKeyB64 string `json:"aeadKeyB64"`
}
// resolveAccount picks the account an operation targets. A non-empty uid must
// match an existing account. An empty uid falls back to the sole account
// (single-identity back-compat); with zero or many accounts and no uid it
// returns a non-zero status for the caller to surface. status == 0 means ok.
func (s *Server) resolveAccount(uid string) (id *identity.Identity, status int, slug, msg string) {
dir := s.config.IdentityAccountsDir()
uid = strings.TrimSpace(uid)
if uid != "" {
got, err := identity.LoadByUID(dir, uid)
if err != nil {
return nil, http.StatusBadRequest, "bad_uid", err.Error()
}
if got == nil {
return nil, http.StatusNotFound, "no_identity", "no identity for the given uid"
}
return got, 0, "", ""
}
list, err := identity.List(dir)
if err != nil {
return nil, http.StatusInternalServerError, "list_failed", err.Error()
}
switch len(list) {
case 0:
return nil, http.StatusNotFound, "no_identity", "no identity is configured"
case 1:
return list[0], 0, "", ""
default:
return nil, http.StatusBadRequest, "uid_required", "multiple accounts exist; specify uid"
}
}
func (s *Server) handleIdentityGet(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if len(list) == 0 {
writeJSON(w, http.StatusOK, identityStatusResponse{HasIdentity: false})
return
}
// Back-compat: report the oldest account as "the" identity. Multi-account
// clients enumerate via GET /api/identity/accounts instead.
def := list[0]
writeJSON(w, http.StatusOK, identityStatusResponse{
HasIdentity: true,
UID: def.UID,
Curve: def.Curve,
CreatedAt: def.CreatedAt,
AccountCount: len(list),
})
}
func (s *Server) handleIdentityAccounts(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
out := make([]identityAccountInfo, 0, len(list))
for _, id := range list {
out = append(out, identityAccountInfo{UID: id.UID, Curve: id.Curve, CreatedAt: id.CreatedAt})
}
writeJSON(w, http.StatusOK, identityAccountsResponse{Accounts: out})
}
func (s *Server) handleIdentityGenerate(w http.ResponseWriter, r *http.Request) {
id, err := identity.Generate()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if err := identity.SaveAccount(s.config.IdentityAccountsDir(), id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityGenerateResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
PrivateKeyHex: id.PrivateKeyHex,
})
}
func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
var body identityImportRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if strings.TrimSpace(body.PrivateKeyHex) == "" {
writeError(w, http.StatusBadRequest, "privateKeyHex is required")
return
}
id, err := identity.Import(strings.TrimSpace(body.PrivateKeyHex))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
dir := s.config.IdentityAccountsDir()
// Idempotent: importing a key that is already in the keystore is how a
// second device signs in to an existing account. Answer 200 with the
// stored account (preserving its original createdAt) instead of 409 —
// the caller proved ownership by presenting the key.
if existing, err := identity.LoadByUID(dir, id.UID); err == nil && existing != nil {
writeJSON(w, http.StatusOK, identityImportResponse{
UID: existing.UID,
Curve: existing.Curve,
CreatedAt: existing.CreatedAt,
Created: false,
})
return
}
if err := identity.SaveAccount(dir, id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityImportResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
Created: true,
})
}
// handleIdentityChallenge issues the one-shot text a client signs to prove it
// holds an account's private key.
//
// Unauthenticated by design: a challenge is useless without the key, and
// GET /api/identity/accounts already publishes every uid, so refusing to mint
// one for an unknown caller would protect nothing. Requiring the account to
// exist is for a clear error, not secrecy.
func (s *Server) handleIdentityChallenge(w http.ResponseWriter, r *http.Request) {
var body identityChallengeRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := strings.TrimSpace(body.UID)
if uid == "" {
writeErrorSlug(w, http.StatusBadRequest, "uid_required", "uid is required", false)
return
}
action := identity.Action(strings.TrimSpace(body.Action))
if !identity.ValidAction(action) {
writeErrorSlug(w, http.StatusBadRequest, "bad_action",
`action must be "reveal" or "delete"`, false)
return
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
writeErrorSlug(w, http.StatusNotFound, "no_identity", "no identity for the given uid", false)
return
}
text, expires, err := s.challenges.Issue(uid, action)
if err != nil {
if errors.Is(err, identity.ErrChallengeFull) {
writeErrorSlug(w, http.StatusServiceUnavailable, "challenges_full", err.Error(), true)
return
}
writeErrorSlug(w, http.StatusBadRequest, "bad_uid", err.Error(), false)
return
}
writeJSON(w, http.StatusOK, identityChallengeResponse{
Challenge: text,
ExpiresAt: expires.Unix(),
})
}
// requireProof resolves the target account and verifies the caller's
// proof-of-possession for `action`. status == 0 means the proof held.
//
// The uid is mandatory here, unlike elsewhere in this file: the old "sole
// account" fallback would have this pick a target the signature does not name,
// and a proof that authorises an unnamed account authorises the wrong one as
// soon as a second profile exists.
func (s *Server) requireProof(uid string, proof identityProof, action identity.Action) (
id *identity.Identity, status int, slug, msg string,
) {
uid = strings.TrimSpace(uid)
if uid == "" {
return nil, http.StatusBadRequest, "uid_required",
"uid is required: the signature authorises one specific account"
}
if proof.Challenge == "" || proof.Signature == "" {
return nil, http.StatusUnauthorized, "signature_required",
"this operation requires proof you hold the account's private key: " +
"POST /api/identity/challenge, sign the returned text, and resend it as {challenge, signature}"
}
sig, err := base64.StdEncoding.DecodeString(proof.Signature)
if err != nil {
return nil, http.StatusBadRequest, "bad_signature", "signature is not valid base64"
}
// Resolve before redeeming so a request naming a missing account does not
// burn a challenge it could not have used anyway.
id, status, slug, msg = s.resolveAccount(uid)
if status != 0 {
return nil, status, slug, msg
}
// Redeem first, verify second: the challenge is single-use either way, so a
// wrong signature costs the attacker a fresh round trip per guess rather
// than letting them grind one challenge offline against the same nonce.
if err := s.challenges.Redeem(proof.Challenge, uid, action); err != nil {
return nil, http.StatusUnauthorized, "bad_challenge", err.Error()
}
if err := identity.Verify(uid, []byte(proof.Challenge), sig); err != nil {
return nil, http.StatusUnauthorized, "bad_signature", err.Error()
}
return id, 0, "", ""
}
// handleIdentityReveal returns an existing account's private key to a caller
// that proves it already holds that key.
//
// Deliberately POST (never a bare GET) so it cannot be triggered by a link or
// an image tag. The proof is what actually guards it: before this gate existed,
// anything that could reach the port could read every private key on the node.
func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
var body identityRevealRequest
_ = json.NewDecoder(r.Body).Decode(&body)
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionReveal)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
writeJSON(w, http.StatusOK, identityRevealResponse{
UID: id.UID,
Curve: id.Curve,
PrivateKeyHex: id.PrivateKeyHex,
})
}
// handleIdentityDelete removes one account and everything it wrote.
//
// Two things happen, in this order and no other:
//
// 1. the account's User Data Layer rows are purged (cells plus every index
// that names the uid, including the cid-scoped ones shared with other
// accounts);
// 2. the keypair file is removed.
//
// Purge-then-delete so a failure never strands data: if the purge errors the
// key is still on disk, the account still exists, and the caller can retry.
// The reverse order would leave rows nobody can ever attribute, reach or
// rewrite — the orphaning this endpoint exists to prevent.
//
// What deletion still does NOT do, and cannot: records already signed under
// this uid and replicated to other peers keep verifying there. Removing a key
// is not a recall.
func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
var body identityDeleteRequest
// Tolerate an empty body so the uid may ride in the query — meta-watch's
// proxy puts it there. The proof itself must be in the body: a signature in
// a URL ends up in every access log between here and the browser.
_ = json.NewDecoder(r.Body).Decode(&body)
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionDelete)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
// A purge needs storage. Refusing here — rather than deleting the key and
// reporting the purge as "0 records" — is the difference between a retry
// and permanent orphaned data: once the key is gone the account cannot be
// named for a second attempt through this API.
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage is not connected; refusing to delete the key while its data cannot be purged", true)
return
}
purged, err := s.storage.UDLPurgeUser(id.UID)
if err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
"could not purge this account's data; the account was NOT deleted: "+err.Error(), true)
return
}
if err := identity.DeleteByUID(s.config.IdentityAccountsDir(), id.UID); err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
return
}
log.Printf("[API] Deleted account %s and purged %d User Data Layer records across %d cids",
id.UID, purged.Records, purged.Cids)
writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
}
func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
var body identitySignRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := body.UID
if strings.TrimSpace(uid) == "" {
uid = r.URL.Query().Get("uid")
}
id, status, slug, msg := s.resolveAccount(uid)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
raw, err := base64.StdEncoding.DecodeString(body.BytesB64)
if err != nil {
writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
return
}
sig, err := identity.Sign(id, raw)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, identitySignResponse{
SigB64: base64.StdEncoding.EncodeToString(sig),
})
}
// handleIdentitySignBatch signs every payload in the request under one account.
//
// All-or-nothing: a single bad base64 or signing failure fails the whole
// request rather than returning a short array. The caller pairs signatures with
// records by index, so a partial response would silently misalign them — and a
// misaligned signature verifies against the wrong record, which is worse than
// an error.
func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) {
var body identitySignBatchRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := body.UID
if strings.TrimSpace(uid) == "" {
uid = r.URL.Query().Get("uid")
}
if len(body.PayloadsB64) == 0 {
writeError(w, http.StatusBadRequest, "payloadsB64 is required")
return
}
if len(body.PayloadsB64) > maxSignBatch {
writeError(w, http.StatusBadRequest, "too many payloads in one batch")
return
}
id, status, slug, msg := s.resolveAccount(uid)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
sigs := make([]string, 0, len(body.PayloadsB64))
for i, p := range body.PayloadsB64 {
raw, err := base64.StdEncoding.DecodeString(p)
if err != nil {
writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload",
"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
return
}
sig, err := identity.Sign(id, raw)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
}
writeJSON(w, http.StatusOK, identitySignBatchResponse{SigsB64: sigs})
}
func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
key, err := identity.DeriveAEADKey(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, identityAEADKeyResponse{
AEADKeyB64: base64.StdEncoding.EncodeToString(key),
})
}
+257
View File
@@ -0,0 +1,257 @@
package api
// End-to-end exercise of the account gate against a live router + miniredis:
// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
// both the key and every row are gone. Deliberately drives the real HTTP
// handlers rather than the packages beneath them, because the bugs that matter
// here (a challenge accepted for the wrong action, a purge that runs after the
// key is gone) live in the wiring.
import (
"bytes"
"encoding/base64"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
func newTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
stor := storage.NewClient("")
if err := stor.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect: %v", err)
}
t.Cleanup(func() { _ = stor.Close() })
s := &Server{
config: &config.Config{MetaCorePath: t.TempDir()},
storage: stor,
challenges: identity.NewChallengeStore(),
router: mux.NewRouter(),
}
s.setupIdentityTestRoutes()
return s
}
// Only the routes under test, so this does not depend on every other subsystem
// booting.
func (s *Server) setupIdentityTestRoutes() {
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
}
func do(t *testing.T, s *Server, method, path string, body any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
var rdr *bytes.Reader
if body != nil {
b, _ := json.Marshal(body)
rdr = bytes.NewReader(b)
} else {
rdr = bytes.NewReader(nil)
}
req := httptest.NewRequest(method, path, rdr)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, req)
out := map[string]any{}
_ = json.Unmarshal(rec.Body.Bytes(), &out)
return rec, out
}
// signAs produces the proof a browser would, from the raw key.
func signAs(t *testing.T, privHex, challenge string) map[string]any {
t.Helper()
raw, err := hex.DecodeString(privHex)
if err != nil {
t.Fatalf("decode key: %v", err)
}
id, err := identity.Import(hex.EncodeToString(raw))
if err != nil {
t.Fatalf("import: %v", err)
}
sig, err := identity.Sign(id, []byte(challenge))
if err != nil {
t.Fatalf("sign: %v", err)
}
return map[string]any{
"challenge": challenge,
"signature": base64.StdEncoding.EncodeToString(sig),
}
}
func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
s := newTestServer(t)
rec, gen := do(t, s, "POST", "/api/identity/generate", nil)
if rec.Code != http.StatusCreated {
t.Fatalf("generate: %d %s", rec.Code, rec.Body)
}
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
// Some data to lose.
for _, cell := range [][2]string{{"cidA", "like"}, {"cidA", "rating"}, {"cidB", "seek"}} {
if ok, err := s.storage.UDLUpsertIfNewer(uid, cell[0], cell[1], 1, 100, "rec", "", false, false); err != nil || !ok {
t.Fatalf("seed: %v", err)
}
}
// The count the UI shows in the confirmation.
rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
if rec.Code != http.StatusOK {
t.Fatalf("stats: %d %s", rec.Code, rec.Body)
}
users := stats["users"].(map[string]any)
mine := users[uid].(map[string]any)
if mine["records"].(float64) != 3 || mine["cids"].(float64) != 2 {
t.Fatalf("stats = %v, want 3 records / 2 cids", mine)
}
// Unsigned delete must be refused, and must not have touched anything.
rec, body := do(t, s, "DELETE", "/api/identity", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned delete = %d %v, want 401 signature_required", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("a refused delete removed the key anyway")
}
// meta-core must refuse to sign the caller's authorisation for them —
// without this the gate is decoration on a node holding every key.
ch, _, err := s.challenges.Issue(uid, identity.ActionDelete)
if err != nil {
t.Fatalf("issue: %v", err)
}
rec, body = do(t, s, "POST", "/api/identity/sign", map[string]any{
"uid": uid,
"bytesB64": base64.StdEncoding.EncodeToString([]byte(ch)),
})
if rec.Code != http.StatusForbidden || body["error"] != "reserved_payload" {
t.Fatalf("sign of a challenge = %d %v, want 403 reserved_payload", rec.Code, body)
}
// A challenge minted for reveal must not authorise a delete.
rec, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("reveal challenge used for delete = %d %v, want 401", rec.Code, body)
}
// The real thing.
rec, chBody = do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof = signAs(t, priv, chBody["challenge"].(string))
req = map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed delete = %d %s", rec.Code, rec.Body)
}
purged := body["purged"].(map[string]any)
if purged["records"].(float64) != 3 || purged["cids"].(float64) != 2 {
t.Fatalf("purged = %v, want 3 records / 2 cids", purged)
}
// Key gone.
if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("the account file survived the delete")
}
// Data gone — the orphaning this whole change exists to prevent.
after, err := s.storage.UDLAllUserStats()
if err != nil {
t.Fatalf("stats after: %v", err)
}
if got, present := after[uid]; present {
t.Fatalf("deleted account still holds %+v", got)
}
for _, cid := range []string{"cidA", "cidB"} {
uids, err := s.storage.UDLCidUsers(cid)
if err != nil {
t.Fatalf("cid users: %v", err)
}
if len(uids) != 0 {
t.Fatalf("%s still names %v after the purge", cid, uids)
}
}
// The challenge is single-use, so replaying the same proof does nothing.
rec, _ = do(t, s, "DELETE", "/api/identity", req)
if rec.Code == http.StatusOK {
t.Fatal("a replayed proof must not succeed")
}
}
func TestE2E_RevealRequiresProof(t *testing.T) {
s := newTestServer(t)
_, gen := do(t, s, "POST", "/api/identity/generate", nil)
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
rec, body := do(t, s, "POST", "/api/identity/reveal", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned reveal = %d %v, want 401 signature_required", rec.Code, body)
}
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "POST", "/api/identity/reveal", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed reveal = %d %s", rec.Code, rec.Body)
}
if body["privateKeyHex"] != priv {
t.Fatal("reveal returned a different key than generate did")
}
}
// Another account's key must not authorise this one, even with a challenge
// minted for the right uid and action.
func TestE2E_AnotherAccountsKeyIsRefused(t *testing.T) {
s := newTestServer(t)
_, alice := do(t, s, "POST", "/api/identity/generate", nil)
_, mallory := do(t, s, "POST", "/api/identity/generate", nil)
uid := alice["uid"].(string)
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
proof := signAs(t, mallory["privateKeyHex"].(string), chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body := do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized || body["error"] != "bad_signature" {
t.Fatalf("mallory deleting alice = %d %v, want 401 bad_signature", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("alice's account was deleted by mallory's signature")
}
}
+286
View File
@@ -1,12 +1,14 @@
package api
import (
"encoding/json"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/storage"
)
// KVInfoResponse is the response for GET /api/kv/info
@@ -184,3 +186,287 @@ func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, response)
}
// KVTreeResponse is the response for GET /api/kv/tree
type KVTreeResponse struct {
Prefix string `json:"prefix"`
Delimiter string `json:"delimiter"`
Branches []string `json:"branches"`
Leaves []string `json:"leaves"`
Cursor string `json:"cursor"`
HasMore bool `json:"hasMore"`
}
// KVSearchResponse is the response for GET /api/kv/search
type KVSearchResponse struct {
Contains string `json:"contains"`
Keys []string `json:"keys"`
Truncated bool `json:"truncated"`
}
// KVFindResponse is the response for GET /api/kv/find
type KVFindResponse struct {
Contains string `json:"contains"`
Fields []string `json:"fields"`
Matches []storage.FindMatch `json:"matches"`
Truncated bool `json:"truncated"`
}
// KVValueResponse is the response for GET /api/kv/value
type KVValueResponse struct {
Key string `json:"key"`
Type string `json:"type"`
Value string `json:"value"`
Exists bool `json:"exists"`
}
// KVValuePutRequest is the body for PUT /api/kv/value
type KVValuePutRequest struct {
Key string `json:"key"`
Value string `json:"value"`
}
// KVValueDeleteResponse is the response for DELETE /api/kv/value
type KVValueDeleteResponse struct {
Key string `json:"key"`
Deleted bool `json:"deleted"`
}
// handleKVTree handles GET /api/kv/tree?prefix=&delimiter=/&cursor=0&max=500
//
// Lists the immediate children at prefix. When delimiter is set, keys with a
// further delimiter are collapsed into branches. Pass the returned cursor back
// to continue when hasMore is true.
func (s *Server) handleKVTree(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
prefix := q.Get("prefix")
delimiter := q.Get("delimiter")
cursor := q.Get("cursor")
maxResults := 500
if v := q.Get("max"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
maxResults = n
}
}
leaves, branches, nextCursor, err := s.storage.ScanKeys(prefix, delimiter, cursor, maxResults)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVTreeResponse{
Prefix: prefix,
Delimiter: delimiter,
Branches: branches,
Leaves: leaves,
Cursor: nextCursor,
HasMore: nextCursor != "0",
})
}
// handleKVFind handles GET /api/kv/find?contains=&fields=a,b,c&limit=200
//
// Value-search: scans every key whose path ends with /<field> for each
// configured field, then returns those whose value contains the substring
// (case-insensitive). Used as the editor's "filter the view" action.
func (s *Server) handleKVFind(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
fieldsParam := q.Get("fields")
if fieldsParam == "" {
writeError(w, http.StatusBadRequest, "fields is required (comma-separated)")
return
}
fields := strings.Split(fieldsParam, ",")
for i := range fields {
fields[i] = strings.TrimSpace(fields[i])
}
limit := 200
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
matches, truncated, err := s.storage.FindByValue(contains, fields, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if matches == nil {
matches = []storage.FindMatch{}
}
writeJSON(w, http.StatusOK, KVFindResponse{
Contains: contains,
Fields: fields,
Matches: matches,
Truncated: truncated,
})
}
// handleKVSearch handles GET /api/kv/search?contains=&limit=500
func (s *Server) handleKVSearch(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
limit := 500
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
keys, truncated, err := s.storage.SearchKeys(contains, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVSearchResponse{
Contains: contains,
Keys: keys,
Truncated: truncated,
})
}
// handleKVValueGet handles GET /api/kv/value?key=...
func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
kind, value, exists, err := s.storage.GetKeyInfo(key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: key,
Type: kind,
Value: value,
Exists: exists,
})
}
// handleKVValuePut handles PUT /api/kv/value with body {key, value}.
// Maintains the file index when the key matches file:<hashId>/...
func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req KVValuePutRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if err := s.storage.SetRawValue(req.Key, req.Value); err != nil {
writeError(w, http.StatusConflict, err.Error())
return
}
if hashID := extractFileHashID(req.Key); hashID != "" {
if err := s.storage.IndexAdd(hashID); err != nil {
// Non-fatal: the value was written; index entry is best-effort.
// Surface in the log path of writeJSON via response header.
w.Header().Set("X-KV-Index-Warning", err.Error())
}
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: req.Key,
Type: "string",
Value: req.Value,
Exists: true,
})
}
// handleKVValueDelete handles DELETE /api/kv/value?key=...
// When the key matches file:<hashId>/..., removes the file index entry if no
// sibling keys remain after the delete.
func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
deleted, err := s.storage.DeleteRawKey(key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if deleted {
if hashID := extractFileHashID(key); hashID != "" {
if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil {
w.Header().Set("X-KV-Index-Warning", err.Error())
}
}
}
writeJSON(w, http.StatusOK, KVValueDeleteResponse{
Key: key,
Deleted: deleted,
})
}
// extractFileHashID returns the hashID for a key shaped like file:<hashId>/...
// Returns "" when the key does not follow that convention.
func extractFileHashID(key string) string {
if !strings.HasPrefix(key, "file:") {
return ""
}
rest := strings.TrimPrefix(key, "file:")
slash := strings.Index(rest, "/")
if slash <= 0 {
return ""
}
return rest[:slash]
}
+62 -74
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
)
@@ -39,6 +38,36 @@ type MetadataSearchRequest struct {
Property string `json:"property,omitempty"`
PropertyValue string `json:"propertyValue,omitempty"`
Limit int `json:"limit,omitempty"`
// Filters is the structured form: AND across entries, OR within an
// entry's Values. Evaluated in memory against the search index alongside
// the free-text Query, so an id-anchored lookup costs a map probe per
// record rather than a Redis round-trip per record.
//
// Property/PropertyValue is the single-filter legacy shape and folds into
// this list (see normalizedFilters) with Exact=false, preserving its
// substring semantics byte-for-byte.
Filters []MetadataFilter `json:"filters,omitempty"`
}
// MetadataFilter is one structured constraint on a record.
//
// A record satisfies it when ANY of Values matches, and a search result must
// satisfy EVERY filter (OR within, AND across). An entry with no usable value
// constrains nothing — same rule the gateway's `genres_filter_matches` uses, so
// a caller that lowers an empty filter doesn't accidentally empty the result.
type MetadataFilter struct {
// Property is the field name, WITHOUT a key-set suffix: `genres`, not
// `genres/Action`. Both storage shapes are probed — see recordMatchesFilter.
Property string `json:"property"`
// Values are OR'd. Compared case-insensitively.
Values []string `json:"values,omitempty"`
// Exact selects CSV-exact comparison (the semantics meta-gateway's
// `any_csv_value_matches` applies at its own Layer-B re-filter) over
// substring containment. Callers doing identity lookups want this: with a
// result limit, a substring match on `tmdbid=1405` also admits `21405` and
// can crowd the real matches out of the page.
Exact bool `json:"exact,omitempty"`
}
// MetadataSearchResult is a single search result
@@ -193,6 +222,9 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
return
}
// Same resolution as the PUT below: callers hold a CID, not the root.
hashID = s.storage.ResolveRoot(hashID)
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -227,6 +259,17 @@ func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Req
return
}
// The {hashId} in the URL is whatever CID the caller knows the file by —
// the gateway's auto-store keys this PUT by the hash it just computed
// (dispatch.rs → store_metadata_only), which for a usenet→IPFS fetch is
// the dag-pb root, not the nzb-release cid the record was minted under.
// Without resolving, the write lands on file:<thatCid>/* and forks a
// second root for a file that already has one: a full duplicate record,
// cross-linked and separately indexed, which every downstream ingest then
// sees as two files. Unknown CIDs pass through unchanged, so a genuinely
// new gateway record still mints its root at the hash.
hashID = s.storage.ResolveRoot(hashID)
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
@@ -263,6 +306,12 @@ func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Req
return
}
// meta-gateway retracts a dropped hit by whichever CID it holds
// (meta_core.rs `delete`, which documents this resolution as a
// precondition). Without it, deleting by a non-root CID is a silent
// no-op and the retracted record lingers.
hashID = s.storage.ResolveRoot(hashID)
deletedCount, err := s.storage.DeleteMetadata(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -395,79 +444,18 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
return
}
// Get all hash IDs
allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
var results []MetadataSearchResult
total := len(allHashIDs)
// Search fields for query
searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
for _, hashID := range allHashIDs {
if len(results) >= req.Limit {
break
}
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil || metadata == nil {
continue
}
matches := false
// Property search
if req.Property != "" && req.PropertyValue != "" {
value, ok := metadata[req.Property]
if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) {
matches = true
}
}
// General query search
if req.Query != "" && req.Property == "" {
queryLower := strings.ToLower(req.Query)
// Check if query matches hashId
if strings.Contains(strings.ToLower(hashID), queryLower) {
matches = true
}
// Check search fields
if !matches {
for _, field := range searchFields {
if value, ok := metadata[field]; ok {
if strings.Contains(strings.ToLower(value), queryLower) {
matches = true
break
}
}
}
}
}
// If no query or property filter, return all
if req.Query == "" && req.Property == "" {
matches = true
}
if matches {
results = append(results, MetadataSearchResult{
HashID: hashID,
Metadata: metadata,
})
}
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: total,
})
// Everything else — free text, structured filters, both, or neither
// (which returns the first `limit`) — is served from the in-memory search
// index (search_index.go). Matching and the returned metadata both come
// from memory, so a query costs zero Redis round-trips.
//
// There is deliberately no second path. The property branch used to fall
// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which
// is what made an id-anchored lookup 3.63s over ~20k records (vs 10ms for
// the free-text branch on the same corpus) and kept meta-core out of the
// gateway's anchored search path entirely. The index already holds every
// field of every record, so that scan was re-reading data it had in hand.
s.serveIndexedSearch(w, req)
}
// handleBatchUpdate handles POST /api/metadata/batch
+45
View File
@@ -0,0 +1,45 @@
package api
import (
"encoding/json"
"net/http"
"time"
)
// handleSchemaGet returns the live per-field schema derived from meta:events.
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
snap := s.schemaIndexer.Snapshot()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(snap)
}
// handleSchemaRescan re-emits a set event for every existing property via the
// meta publisher, waits for the indexer to drain, and returns the refreshed
// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
count, err := s.RepublishMetadata()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.schemaIndexer.WaitForDrain(10 * time.Second)
snap := s.schemaIndexer.Snapshot()
snap.Source = "rescan"
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"published_events": count,
"schema": snap,
})
}
+211
View File
@@ -0,0 +1,211 @@
package api
import "testing"
// Tests for the structured-filter matcher that lets meta-core answer an
// id-anchored query from the in-memory search index (search_index.go).
//
// Why this matters beyond "filters work": meta-core is a *coarse pre-filter*
// for meta-gateway, which re-evaluates every record it emits through its own
// `query_eval::record_matches`. So the property that has to hold is that this
// matcher stays a SUPERSET of that one — it may admit records the gateway will
// drop, but it must never drop a record the gateway would have kept. Every
// case below is written from that angle:
//
// - key-set fields (`genres/Action`) must match a `genres:action` filter, or
// card-tier records vanish from a genre-scoped query;
// - a missing field fails CLOSED, matching record_matches;
// - an empty/unusable filter constrains NOTHING, so a caller that lowers a
// blank filter gets its results rather than an empty page.
func f(property string, exact bool, values ...string) MetadataFilter {
return MetadataFilter{Property: property, Values: values, Exact: exact}
}
func TestRecordMatchesFilter(t *testing.T) {
fields := map[string]string{
"fileType": "video",
"videoType": "tv,anime",
"tmdbid": "1405",
"genres/Action": "true",
"genres/Science Fiction": "true",
"source/gateway:nyaa.si": "true",
"categories/XXX": "true",
"retracted": "false",
}
cases := []struct {
name string
filter MetadataFilter
want bool
}{
// Flat scalar.
{"flat exact hit", f("fileType", true, "video"), true},
{"flat exact miss", f("fileType", true, "audio"), false},
{"flat exact is case-insensitive", f("fileType", true, "VIDEO"), true},
// csv-set field: each comma piece is compared independently.
{"csv member hit", f("videoType", true, "anime"), true},
{"csv member miss", f("videoType", true, "movie"), false},
// Exact vs contains — the reason Exact exists. A substring match on an
// identity field admits neighbours and, under a result limit, crowds
// the real matches off the page.
{"contains admits a superstring", f("tmdbid", false, "140"), true},
{"exact rejects a superstring", f("tmdbid", true, "140"), false},
{"exact accepts the whole value", f("tmdbid", true, "1405"), true},
// Key-set shape. The stored member is verbatim (`genres/Action`) while
// query values arrive lowercased, so the case-insensitive sweep is what
// makes a genre-scoped row work at all.
{"key-set exact casing", f("genres", true, "Action"), true},
{"key-set lowercased query", f("genres", true, "action"), true},
{"key-set with a space", f("genres", true, "science fiction"), true},
{"key-set miss", f("genres", true, "horror"), false},
{"key-set label containing a colon", f("source", true, "gateway:nyaa.si"), true},
// OR within one filter's values.
{"or hits on the second value", f("fileType", true, "audio", "video"), true},
{"or misses on all values", f("fileType", true, "audio", "image"), false},
// Fail-closed on an absent field — mirrors record_matches, and is what
// keeps a filtered query from returning unrelated records.
{"absent field fails closed", f("imdbid", true, "tt0903747"), false},
// A filter that constrains nothing must not empty the result.
{"no values constrains nothing", f("fileType", true), true},
{"blank value constrains nothing", f("fileType", true, " "), true},
{"no property constrains nothing", f("", true, "video"), true},
// A FLAT field whose value happens to be "false" is still a value, and
// an exact filter for it matches. Only the key-set arm treats "false"
// as non-membership (pinned in the test below).
{"flat field valued false still matches", f("retracted", true, "false"), true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := recordMatchesFilter(fields, &tc.filter); got != tc.want {
t.Fatalf("recordMatchesFilter(%+v) = %v, want %v", tc.filter, got, tc.want)
}
})
}
}
// `retracted = "false"` is a flat field, not a key-set member, so the flat arm
// legitimately matches the literal string "false". The key-set arm is what must
// treat it as non-membership — pinned separately so the two arms don't get
// conflated if either is refactored.
func TestFalseyKeySetMemberIsNotMembership(t *testing.T) {
fields := map[string]string{"genres/Action": "false", "genres/Drama": "0"}
for _, v := range []string{"Action", "Drama"} {
filter := f("genres", true, v)
if recordMatchesFilter(fields, &filter) {
t.Fatalf("genres/%s with a falsey value must not assert membership", v)
}
}
}
func TestRecordMatchesFiltersIsAndAcrossEntries(t *testing.T) {
fields := map[string]string{"fileType": "video", "season": "2"}
all := []MetadataFilter{f("fileType", true, "video"), f("season", true, "2")}
if !recordMatchesFilters(fields, all) {
t.Fatal("a record satisfying every filter must match")
}
one := []MetadataFilter{f("fileType", true, "video"), f("season", true, "3")}
if recordMatchesFilters(fields, one) {
t.Fatal("filters are AND across entries — one miss must reject the record")
}
if !recordMatchesFilters(fields, nil) {
t.Fatal("no filters must match everything")
}
}
// The legacy Property/PropertyValue pair keeps its substring semantics: it is
// the shape existing callers already use, and tightening it silently would
// change results for them. New callers pass Filters and opt into Exact.
func TestNormalizedFiltersFoldsTheLegacyPair(t *testing.T) {
got := normalizedFilters(MetadataSearchRequest{Property: "tmdbid", PropertyValue: "1405"})
if len(got) != 1 {
t.Fatalf("legacy pair must fold into exactly one filter, got %d", len(got))
}
if got[0].Property != "tmdbid" || len(got[0].Values) != 1 || got[0].Values[0] != "1405" {
t.Fatalf("legacy pair folded incorrectly: %+v", got[0])
}
if got[0].Exact {
t.Fatal("the legacy pair must stay substring (Exact=false) for compatibility")
}
// A half-specified pair is not a filter — it must not become one that
// matches nothing and empties the response.
if n := len(normalizedFilters(MetadataSearchRequest{Property: "tmdbid"})); n != 0 {
t.Fatalf("a property with no value must not fold into a filter, got %d", n)
}
if n := len(normalizedFilters(MetadataSearchRequest{PropertyValue: "1405"})); n != 0 {
t.Fatalf("a value with no property must not fold into a filter, got %d", n)
}
both := normalizedFilters(MetadataSearchRequest{
Property: "tmdbid",
PropertyValue: "1405",
Filters: []MetadataFilter{f("fileType", true, "video")},
})
if len(both) != 2 {
t.Fatalf("the legacy pair must be appended to explicit filters, got %d", len(both))
}
}
// The acceptance bar for routing the anchored path through the index: a
// full-miss filtered sweep over a realistic corpus must stay in the same class
// as the free-text branch (measured 10 ms over 19 622 live records), not the
// legacy per-record Redis scan it replaces (measured 3.63 s over the same
// corpus). Full-miss is the worst case — the loop cannot short-circuit on the
// result limit, so every record is evaluated.
//
// Run: go test ./internal/api/ -bench BenchmarkFilterSweep -benchtime 10x
func BenchmarkFilterSweep(b *testing.B) {
const records = 20000
corpus := make([]map[string]string, records)
for i := range corpus {
corpus[i] = map[string]string{
"title": "Some Release",
"fileType": "video",
"videoType": "tv",
"tmdbid": "1405",
"season": "2",
"genres/Action": "true",
"genres/Drama": "true",
"source/gateway:nyaa.si": "true",
"categories/TV/Anime": "true",
"indexer": "nyaa.si",
"fileName": "some.release.s02e05.mkv",
"description/eng": "text",
"poster": "bafy",
"sizeByte": "123456789",
"contentKind": "episode",
"anchoredBy/tmdb:1405": "true",
}
}
// A miss on the identity field: the shape an id-anchored gateway query
// takes, and the one that must not degrade.
filters := []MetadataFilter{
f("tmdbid", true, "999999"),
f("fileType", true, "video"),
f("genres", true, "action"),
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
hits := 0
for _, fields := range corpus {
if recordMatchesFilters(fields, filters) {
hits++
}
}
if hits != 0 {
b.Fatalf("expected a full miss, got %d hits", hits)
}
}
}
+342
View File
@@ -0,0 +1,342 @@
package api
import (
"log"
"net/http"
"sort"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/metazla/meta-core/internal/storage"
)
// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
// from Redis in the background. Records ingest continuously, so a newly stored
// record becomes searchable within one interval; the rebuild is a single
// batched bulk read (storage.GetSearchIndexTuples), off the request path.
const searchIndexRefreshInterval = 60 * time.Second
// searchIndexReconcileEvery is how many refresh ticks pass between full
// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full
// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all.
const searchIndexReconcileEvery = 30
// searchIndex is an in-memory snapshot of every record's search haystack
// (hashID + lowercased searchable fields), kept current from Redis. It exists
// so POST /api/metadata/search matches in memory (string ops over a slice)
// instead of doing a SCAN+MGET Redis round-trip per record — the latter took
// 40s+ over ~72k records and made a live parallel search path unusable.
//
// # Two update paths, and why
//
// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a
// complete fresh copy. That is O(corpus) regardless of how much changed, and it
// used to run every 60 s forever. Measured on a 19 622-record staging corpus
// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side
// held 783 MB for data Redis stores in 135 MB — largely the double copy.
//
// `refreshIncremental` re-reads only the records that changed, tracked by
// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady
// state; the full rebuild runs once at startup (there is nothing to be
// incremental against) and then only as an explicit repair.
//
// The distinction matters because meta-core is the platform's permanent,
// additive store — every search hit on every gateway is retained — so a rebuild
// priced against the whole corpus is a cost that only ever grows.
type searchIndex struct {
mu sync.RWMutex
// byID is the authoritative content; tuples is a stable-order view of it
// rebuilt on each update. Map iteration order in Go is randomised, so
// serving a limited query straight off the map would return a different
// subset each call — the slice is what keeps results stable.
byID map[string]storage.SearchTuple
tuples []storage.SearchTuple
ready atomic.Bool
}
// snapshot returns the current tuples slice and whether the index has built at
// least once. Every update swaps in a fresh slice under the write lock (never
// mutates in place), so the returned slice is safe to read lock-free.
func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.tuples, idx.ready.Load()
}
// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds
// the write lock.
func (idx *searchIndex) rebuildViewLocked() {
view := make([]storage.SearchTuple, 0, len(idx.byID))
for _, t := range idx.byID {
view = append(view, t)
}
// Stable across rebuilds so a limited query returns a consistent page
// rather than a fresh random subset every interval.
sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID })
idx.tuples = view
}
// refresh rebuilds the whole index from Redis (a batched bulk read). Used for
// the initial build and as an explicit repair — not the steady state.
func (idx *searchIndex) refresh(stor *storage.Client) {
if stor == nil || !stor.IsConnected() {
return
}
tuples, err := stor.GetSearchIndexTuples()
if err != nil {
log.Printf("[API] search index refresh failed: %v", err)
return
}
byID := make(map[string]storage.SearchTuple, len(tuples))
for _, t := range tuples {
byID[t.HashID] = t
}
idx.mu.Lock()
idx.byID = byID
idx.rebuildViewLocked()
idx.mu.Unlock()
idx.ready.Store(true)
}
// refreshIncremental drains the dirty set and patches only those records.
// Returns whether more dirty entries remain, so a burst is caught up over
// consecutive passes rather than one per interval.
//
// A record in the dirty set but absent from storage was deleted and is evicted
// here — otherwise it would keep answering searches until the next full
// rebuild. A batch that fails after the (destructive) drain is re-marked by
// storage so it is retried rather than silently lost.
func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) {
if stor == nil || !stor.IsConnected() {
return false
}
ids, more, err := stor.DrainDirty()
if err != nil {
log.Printf("[API] search index drain failed: %v", err)
return false
}
if len(ids) == 0 {
return false
}
tuples, gone, err := stor.GetSearchIndexTuplesFor(ids)
if err != nil {
stor.MarkDirty(ids...)
log.Printf("[API] incremental search index refresh failed: %v", err)
return false
}
idx.mu.Lock()
if idx.byID == nil {
idx.byID = make(map[string]storage.SearchTuple, len(tuples))
}
for _, t := range tuples {
idx.byID[t.HashID] = t
}
for _, id := range gone {
delete(idx.byID, id)
}
idx.rebuildViewLocked()
idx.mu.Unlock()
log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples))
return more
}
// runRefreshLoop builds the index once in full, then keeps it current from the
// dirty set, with a periodic full reconcile as a safety net.
//
// The reconcile exists because a *missed* dirty mark is silent: no error, just
// a record whose indexed copy quietly diverges from storage until something
// forces a rebuild. Marking is spread across eight write paths plus anything
// added later, and the cost of getting it wrong is invisible — so the index
// should not depend on that coverage being perfect forever. Every
// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs
// whatever drifted.
//
// That still drops the full rebuild from every 60 s to every 30 min — a ~30×
// reduction in the steady-state cost that measured at ~45 % of all Redis CPU —
// while keeping the self-healing property that made the naive loop safe.
func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
idx.refresh(stor)
ticker := time.NewTicker(searchIndexRefreshInterval)
defer ticker.Stop()
ticks := 0
for range ticker.C {
ticks++
if ticks%searchIndexReconcileEvery == 0 {
idx.refresh(stor)
continue
}
// Drain until caught up, so a burst larger than one batch does not take
// one interval per batch to land.
for idx.refreshIncremental(stor) {
}
}
}
// serveIndexedSearch answers a search from the in-memory index: a token-AND
// match against each record's haystack (or a whole-query hashID substring,
// preserving the legacy behaviour) AND every structured filter, then returns
// the full metadata for the <=limit survivors. No Redis round-trip per scanned
// record — matching and results both come from memory.
//
// This is the ONLY search path. Free text, filters, both, or neither all land
// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
tuples, ready := s.search.snapshot()
if !ready {
// First call before the background build finished: warm inline with the
// efficient batched read (seconds), never the per-record SCAN fallback.
s.search.refresh(s.storage)
tuples, _ = s.search.snapshot()
}
tokens := strings.Fields(strings.ToLower(req.Query))
queryLower := strings.ToLower(req.Query)
filters := normalizedFilters(req)
results := make([]MetadataSearchResult, 0, req.Limit)
for _, t := range tuples {
if len(results) >= req.Limit {
break
}
if req.Query != "" &&
!matchAllTokens(t.Haystack, tokens) &&
!strings.Contains(strings.ToLower(t.HashID), queryLower) {
continue
}
if !recordMatchesFilters(t.Fields, filters) {
continue
}
// Full metadata is cached in the index — return it straight from memory,
// no Redis round-trip per result.
results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: len(tuples),
})
}
// matchAllTokens reports whether haystack contains every token (AND semantics).
// An empty token list matches everything.
func matchAllTokens(haystack string, tokens []string) bool {
for _, t := range tokens {
if !strings.Contains(haystack, t) {
return false
}
}
return true
}
// normalizedFilters returns the request's structured filters with the legacy
// Property/PropertyValue pair folded in as one more entry.
//
// The legacy pair keeps Exact=false so its long-standing substring semantics
// are unchanged for existing callers; new callers pass Filters and choose.
func normalizedFilters(req MetadataSearchRequest) []MetadataFilter {
if req.Property == "" || req.PropertyValue == "" {
return req.Filters
}
out := make([]MetadataFilter, 0, len(req.Filters)+1)
out = append(out, req.Filters...)
return append(out, MetadataFilter{
Property: req.Property,
Values: []string{req.PropertyValue},
})
}
// recordMatchesFilters reports whether fields satisfies every filter (AND).
func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool {
for i := range filters {
if !recordMatchesFilter(fields, &filters[i]) {
return false
}
}
return true
}
// recordMatchesFilter reports whether fields satisfies one filter (OR within
// its values), probing BOTH storage shapes a field can take:
//
// - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`;
// - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"`
// (METADATA_KEYS "How the store is shaped").
//
// Probing both is what lets one filter shape serve the whole vocabulary. The
// gateway needs a `genres:action` filter to find a record whose genre lives in
// the key `genres/Action`, and a `fileType:video` filter to find one whose value
// lives in the field `fileType` — without meta-core knowing which fields are
// key-sets. It also means this stays a SUPERSET of the gateway's own
// `record_matches`, which is the contract: meta-core pre-filters, the gateway
// re-filters authoritatively.
//
// A filter with no usable value constrains nothing (matches). A filter whose
// field is absent fails closed, matching `record_matches`.
func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool {
values := make([]string, 0, len(f.Values))
for _, v := range f.Values {
if v = strings.TrimSpace(v); v != "" {
values = append(values, v)
}
}
if f.Property == "" || len(values) == 0 {
return true
}
// Key-set shape: `<property>/<value>` present and truthy. Try the exact
// key first — that hits whenever the caller's casing matches what was
// written, which is the common case and costs one map probe.
for _, v := range values {
if isTruthyMember(fields[f.Property+"/"+v]) {
return true
}
}
// Members are written verbatim (`genres/Action`) while query values arrive
// lowercased, so fall back to one case-insensitive sweep. Single pass over
// the record's fields for ALL values, not one pass per value.
wanted := make(map[string]struct{}, len(values))
for _, v := range values {
wanted[strings.ToLower(v)] = struct{}{}
}
prefix := strings.ToLower(f.Property) + "/"
for k, val := range fields {
if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) {
continue
}
if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) {
return true
}
}
// Flat shape: the field's own value, split on `,` for csv-set fields.
fieldValue, ok := fields[f.Property]
if !ok {
return false
}
for _, piece := range strings.Split(fieldValue, ",") {
piece = strings.TrimSpace(piece)
if piece == "" {
continue
}
for _, v := range values {
if f.Exact {
if strings.EqualFold(piece, v) {
return true
}
} else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
return true
}
}
}
return false
}
// isTruthyMember reports whether a key-set member's value asserts membership.
// Mirrors the gateway's `is_truthy_member`: presence with a non-empty,
// non-"false"/"0" value.
func isTruthyMember(v string) bool {
v = strings.TrimSpace(strings.ToLower(v))
return v != "" && v != "false" && v != "0"
}
+234 -25
View File
@@ -9,10 +9,12 @@ import (
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/events"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/mounts"
"github.com/metazla/meta-core/internal/schema"
"github.com/metazla/meta-core/internal/storage"
"github.com/metazla/meta-core/internal/watcher"
"github.com/metazla/meta-core/internal/watchers"
@@ -23,12 +25,11 @@ import (
type Server struct {
config *config.Config
leaderProvider *leader.LeaderInfoProvider
discovery *discovery.Service
cleaner *discovery.Cleaner
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
mountPoller *mounts.Poller
mountStatsPoller *mounts.StatsPoller
watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
@@ -36,26 +37,43 @@ type Server struct {
watchersPoller *watchers.Poller
watchersHandlers *watchers.Handlers
metaPublisher *events.MetaPublisher
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler
router *mux.Router
server *http.Server
search *searchIndex
// challenges backs proof-of-possession on reveal/delete. In-process, so a
// restart invalidates the ones in flight — see identity/challenge.go.
challenges *identity.ChallengeStore
// mesh is the UDP discovery node (beacon v2). nil when discovery
// is disabled or failed to bind, in which case /api/neighbors reports an
// empty list rather than erroring — a dashboard nav is decoration and
// must not take the API down with it.
mesh *meshdisco.Node
router *mux.Router
server *http.Server
// stats caches the keyspace-walk half of GET /api/stats so a polling
// dashboard never puts a multi-second Redis walk on the request path.
stats *statsCache
}
// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
// server's constructor signature (and its tests) stay untouched.
func (s *Server) SetMeshNode(n *meshdisco.Node) {
s.mesh = n
}
// NewServer creates a new API server
func NewServer(
cfg *config.Config,
leaderProvider *leader.LeaderInfoProvider,
disc *discovery.Service,
cleaner *discovery.Cleaner,
stor *storage.Client,
) *Server {
s := &Server{
config: cfg,
leaderProvider: leaderProvider,
discovery: disc,
cleaner: cleaner,
storage: stor,
search: &searchIndex{},
router: mux.NewRouter(),
stats: &statsCache{},
}
// Initialize file watcher/scanner and dispatcher
@@ -71,6 +89,11 @@ func NewServer(
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
} else {
s.fileWatcher = fileWatcher
// Wire storage so the watcher can persist (path,size,mtime,cid)
// tuples to Redis as it computes midhashes.
if stor != nil {
fileWatcher.SetStorageClient(stor)
}
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
// Create scan function from watcher for mounts
scanFunc = fileWatcher.ScanMountPath
@@ -88,6 +111,20 @@ func NewServer(
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
// Pass republish callback to handlers (handles nil metaPublisher case)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
// Hydrate the in-memory state registry from Redis tuples so
// the first scan tick after boot can skip ComputeMidHash256
// for files whose (size, mtime) match.
configs := watchersManager.List()
roots := make([]watcher.WatcherRoot, 0, len(configs))
for _, c := range configs {
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
}
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
} else {
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
}
}
}
}
@@ -103,11 +140,27 @@ func NewServer(
// Create and configure mount poller
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
mountsManager.SetPoller(s.mountPoller)
// IO stats poller — queries the local rclone RC API every few seconds
// and exposes live throughput per mount via the same status JSON.
// Stats are daemon-global (rclone limitation); see stats_poller.go.
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
mountsManager.SetStatsPoller(s.mountStatsPoller)
}
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
s.webdavHandler = webdav.NewHandler(cfg)
s.challenges = identity.NewChallengeStore()
// One-time migration: fold a pre-multi-account identity.json into the
// per-account keystore so existing single-identity hosts keep their uid.
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
log.Printf("[API] Warning: identity migration failed: %v", err)
} else if migrated {
log.Printf("[API] Migrated legacy identity.json into multi-account keystore")
}
s.setupRoutes()
return s
}
@@ -120,23 +173,68 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
// Dashboard headline counters in one request. Cheap half live, keyspace
// walks served from a background-refreshed cache — see stats.go.
s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
// Metadata Editor API routes (must be before /meta/{hash} routes)
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
// User Data Layer signing identity (see internal/identity). Auth-gated
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
// Snapshot (export / import / wipe)
s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
// Schema inference
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
// KV Browser API routes
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
s.router.HandleFunc("/api/kv/tree", s.handleKVTree).Methods("GET")
s.router.HandleFunc("/api/kv/search", s.handleKVSearch).Methods("GET")
s.router.HandleFunc("/api/kv/find", s.handleKVFind).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValueGet).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValuePut).Methods("PUT")
s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
// User Data Layer (UDL) routes — per-user signed records (My List,
// Continue Watching, ratings, profile). See internal/api/udl.go.
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordGet).Methods("GET")
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordPut).Methods("PUT")
s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
// Metadata operations - base endpoints
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
@@ -157,42 +255,66 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
// File operations (by CID)
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
// Register one file under /files/plugin by midhash — the watcher's
// mint + alias + file:events path for a single file. Registered
// unconditionally: it exists for boxes whose watcher is OFF.
s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST")
// File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched
// server-side, so clients answer "how many files, how big" without one
// read per hashId. ?summary=1 returns only {count, totalSize}.
s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET")
// Service discovery
// Both /services and /api/services are supported for consistency with other services
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
s.router.HandleFunc("/api/services", s.handleListServices).Methods("GET")
s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
// Public metadata-by-CID resolver. Reverse-index lookup → full document.
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
// without going through Authelia.
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
// Admin: reunify stranded midhash-rooted entries into their UUID roots
// (the dual-root pattern caused by historical writes that bypassed CID
// resolution).
s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
// (METADATA_KEYS.md §14.17). Idempotent.
s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST")
// One-shot sweep of query echoes on literature cards (meta-read
// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
// SSE event streams. Mediated mirror of the Redis Streams; consumers
// reach them over HTTP instead of touching Redis directly. Auth-bypass
// applies but exposure should be inside-only (do NOT proxy through
// Caddy publicly). See docs/api-mediated-access.md "Auth".
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
// beacon v2: neighbours heard over UDP. Supersedes /api/services,
// which is still served above while consumers migrate.
s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
// Mount management routes (if manager initialized)
if s.mountsHandlers != nil {
s.mountsHandlers.RegisterRoutes(s.router)
log.Println("[API] Mount management routes registered at /api/mounts/*")
}
log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
log.Println("[API] KV Browser routes registered at /api/kv/*")
// File watcher routes (if watcher initialized)
if s.watcherHandlers != nil {
s.watcherHandlers.RegisterRoutes(s.router)
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
}
// Watchers management routes (if watchers initialized)
if s.watchersHandlers != nil {
s.watchersHandlers.RegisterRoutes(s.router)
log.Println("[API] Watchers management routes registered at /api/watchers/*")
}
// WebDAV handler (caching is handled by nginx proxy_cache)
if s.webdavHandler != nil {
s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
log.Println("[API] WebDAV routes registered at /webdav/*")
}
// Add middleware
@@ -234,6 +356,20 @@ func (s *Server) Start() error {
}
}
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
// mounts), so always-on rather than gated by config.
if s.mountStatsPoller != nil {
s.mountStatsPoller.Start()
}
// Build + periodically refresh the in-memory search index so
// POST /api/metadata/search matches in memory instead of a Redis
// round-trip per record (see search_index.go).
if s.storage != nil {
go s.search.runRefreshLoop(s.storage)
}
// Start meta publisher (if storage is connected)
// Publishes keyspace notifications to meta:events stream
if s.storage != nil && s.storage.IsConnected() {
@@ -241,8 +377,32 @@ func (s *Server) Start() error {
if err := s.metaPublisher.Start(); err != nil {
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
} else {
// Schema indexer consumes the same stream to derive a live
// per-field schema (type/hint/breakdown). Start it before the
// republish flood so it captures the bootstrap events.
s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
if err := s.schemaIndexer.Start(); err != nil {
log.Printf("[API] Warning: failed to start schema indexer: %v", err)
s.schemaIndexer = nil
}
// Republish events for all existing metadata
go func() {
// Build any missing per-record field indexes first, in ONE
// keyspace pass. The republish below reads every record, and
// an un-indexed record read costs a full-keyspace SCAN — on a
// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
// paid again on every restart). Warming first turns the whole
// republish into index-served reads. Idempotent, so a
// fully-indexed store just pays the single scan.
// See storage/field_index.go.
started := time.Now()
if n, err := s.storage.WarmFieldIndexes(); err != nil {
log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
} else if n > 0 {
log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
}
count, err := s.RepublishMetadata()
if err != nil {
log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)
@@ -272,6 +432,18 @@ func (s *Server) Stop() error {
}
}
// Stop IO stats poller
if s.mountStatsPoller != nil {
s.mountStatsPoller.Stop()
}
// Stop schema indexer before the publisher so the consumer drains cleanly
if s.schemaIndexer != nil {
if err := s.schemaIndexer.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
}
}
// Stop meta publisher
if s.metaPublisher != nil {
if err := s.metaPublisher.Stop(); err != nil {
@@ -313,12 +485,49 @@ func (s *Server) RepublishMetadata() (int, error) {
return s.metaPublisher.RepublishAllMetadata(republishFunc)
}
// loggingMiddleware logs all requests
// statusRecorder captures the response status so loggingMiddleware can stay
// silent on success and only report failures. It forwards Flush/Unwrap so SSE
// and other streaming handlers keep working through the wrapper.
type statusRecorder struct {
http.ResponseWriter
status int
written bool
}
func (r *statusRecorder) WriteHeader(code int) {
if !r.written {
r.status = code
r.written = true
}
r.ResponseWriter.WriteHeader(code)
}
func (r *statusRecorder) Write(b []byte) (int, error) {
if !r.written {
r.status = http.StatusOK
r.written = true
}
return r.ResponseWriter.Write(b)
}
func (r *statusRecorder) Flush() {
if f, ok := r.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
// loggingMiddleware logs a request only when it fails (status >= 400), keeping
// the nominal case quiet. Successful requests are not logged.
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
next.ServeHTTP(w, r)
log.Printf("[API] %s %s %s", r.Method, r.URL.Path, time.Since(start))
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(rec, r)
if rec.status >= 400 {
log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
}
})
}
+167
View File
@@ -0,0 +1,167 @@
package api
import (
"bytes"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/metazla/meta-core/internal/snapshot"
)
// Bound the upload size so a malicious client can't OOM the server.
// 2 GiB is well above any realistic export.
const maxImportBytes = 2 << 30
// pluginOutputDir is where plugin-produced artifacts live on disk
// (subtitles, posters, etc). Mounted from the host into the meta-core
// container at $FILES_PATH/plugin.
func (s *Server) pluginOutputDir() string {
return filepath.Join(s.config.FilesPath, "plugin")
}
// handleSnapshotExport streams a ZIP snapshot of all metadata.
// GET /api/snapshot/export?include=hash-index,plugin-output
func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// The server has a 30s WriteTimeout (server.go:222). Long exports
// blow past that, leaving a truncated ZIP without a central directory.
// Clear the deadline for this handler so the export can run as long as
// needed. ResponseController is the standard way to do this.
rc := http.NewResponseController(w)
_ = rc.SetWriteDeadline(time.Time{})
include := splitCSVParam(r.URL.Query().Get("include"))
opts := snapshot.ExportOptions{
IncludePluginOutput: include["plugin-output"],
PluginOutputDir: s.pluginOutputDir(),
}
filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
// We stream — no Content-Length.
exporter := snapshot.NewExporter(s.storage)
if _, err := exporter.Export(w, opts); err != nil {
// Headers may already be sent; log and bail.
// Client will see a truncated zip, which is the best signal we can give.
fmt.Printf("[Snapshot] export failed: %v\n", err)
return
}
}
// handleSnapshotImport applies an uploaded ZIP snapshot.
// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
// when the snapshot is large. Clear both for this handler.
rc := http.NewResponseController(w)
_ = rc.SetReadDeadline(time.Time{})
_ = rc.SetWriteDeadline(time.Time{})
q := r.URL.Query()
opts := snapshot.ImportOptions{
Mode: snapshot.Mode(q.Get("mode")),
Conflict: snapshot.Conflict(q.Get("conflict")),
DryRun: strings.EqualFold(q.Get("dry_run"), "true"),
PluginOutputDir: s.pluginOutputDir(),
}
if opts.Mode == "" {
opts.Mode = snapshot.ModeMerge
}
if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
opts.Conflict = snapshot.ConflictSource
}
body, err := readImportBody(r)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
importer := snapshot.NewImporter(s.storage)
result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, result)
}
// handleSnapshotWipe clears Redis metadata.
// POST /api/snapshot/wipe?scope=metadata
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
scopes := splitCSVParam(r.URL.Query().Get("scope"))
if len(scopes) == 0 {
writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
return
}
scope := snapshot.WipeScope{
Metadata: scopes["metadata"],
}
if !scope.Metadata {
writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
return
}
wiper := snapshot.NewWiper(s.storage)
res, err := wiper.Wipe(scope)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, res)
}
// readImportBody accepts either a raw application/zip body or a multipart
// upload with a field named "snapshot". Buffers up to maxImportBytes.
func readImportBody(r *http.Request) ([]byte, error) {
r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
ct := r.Header.Get("Content-Type")
if strings.HasPrefix(ct, "multipart/form-data") {
if err := r.ParseMultipartForm(32 << 20); err != nil {
return nil, fmt.Errorf("parse multipart: %w", err)
}
f, _, err := r.FormFile("snapshot")
if err != nil {
return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
}
defer f.Close()
return io.ReadAll(f)
}
return io.ReadAll(r.Body)
}
// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
func splitCSVParam(v string) map[string]bool {
out := map[string]bool{}
for _, part := range strings.Split(v, ",") {
p := strings.TrimSpace(part)
if p != "" {
out[p] = true
}
}
return out
}
+280
View File
@@ -0,0 +1,280 @@
package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
// SSE event stream endpoints for the file:events and meta:events Redis Streams.
//
// Wire contract (see docs/api-mediated-access.md):
// - One SSE event per Redis Stream entry.
// - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
// MUST treat it as a black-box string they echo back via Last-Event-ID.
// - `event:` is the `type` field from the underlying entry.
// - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
// - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
// - On reconnect with Last-Event-ID, resume from the next entry. If the
// cursor has been trimmed by retention, emit one `event: gap` payload
// before resuming from the oldest available entry.
const (
sseHeartbeatInterval = 30 * time.Second
// sseReadBlock controls how long each XREAD blocks before returning.
// Much shorter than sseHeartbeatInterval so the handler stays responsive
// to client disconnects and the heartbeat check runs frequently enough
// that the keep-alive fires within a few seconds of its nominal cadence.
sseReadBlock = 2 * time.Second
// sseReadCount caps how many entries one XREAD returns. Small enough
// to keep per-flush memory low, large enough to drain bursts cheaply.
sseReadCount = 100
)
// handleEventsFiles streams the file:events Redis Stream as SSE.
func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "file:events")
}
// handleEventsMeta streams the meta:events Redis Stream as SSE.
func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "meta:events")
}
// streamSSE is the shared SSE pump shared by both endpoints. One handler per
// stream — there is no fan-out; each subscriber holds its own XREAD loop.
// That's fine at our scale (≤20 subscribers; one per consumer service).
func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
if !s.storage.IsConnected() {
w.Header().Set("Retry-After", "5")
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
flusher, ok := w.(http.Flusher)
if !ok {
writeError(w, http.StatusInternalServerError, "streaming unsupported")
return
}
// The server-wide WriteTimeout (30s on Server.server in server.go) is
// fine for ordinary handlers but lethal for SSE — it would close every
// connection at exactly 30s, racing the heartbeat. Disable it
// per-request via http.NewResponseController. SSE endpoints never need
// a write deadline; the client disconnect path already cleans up.
rc := http.NewResponseController(w)
if err := rc.SetWriteDeadline(time.Time{}); err != nil {
// Older transports (or nested response writers) may not support
// this. Log and continue — the 30s cap will still apply, but the
// handler is still useful for short-lived subscriptions.
log.Printf("[SSE %s] could not disable write deadline: %v", stream, err)
}
// SSE response headers. X-Accel-Buffering disables nginx response
// buffering; the hash-lock sidecar uses nginx, so this matters.
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher.Flush()
client := s.storage.GetRedisClient()
if client == nil {
// Storage was just disconnected — close cleanly.
return
}
ctx := r.Context()
cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
lastWrite := time.Now()
for {
if err := ctx.Err(); err != nil {
return
}
// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
// context's deadline isn't propagated into XREAD's BLOCK arg —
// that's a poll-style timeout on Redis itself — so we keep BLOCK
// short.
readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
result, err := client.XRead(readCtx, &redis.XReadArgs{
Streams: []string{stream, cursor},
Block: sseReadBlock,
Count: sseReadCount,
}).Result()
cancel()
if err != nil {
// redis.Nil = BLOCK timeout with no entries. Anything else
// while the client is still connected is a real failure.
if errors.Is(err, redis.Nil) || isContextTimeout(err) {
// fall through to heartbeat check
} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
return
} else {
log.Printf("[SSE %s] XREAD error: %v", stream, err)
return
}
}
for _, str := range result {
for _, msg := range str.Messages {
writeSSEEntry(w, flusher, msg)
cursor = msg.ID
lastWrite = time.Now()
}
}
// Heartbeat if quiet for >= the interval. Comment line; SSE
// clients ignore it per spec.
if time.Since(lastWrite) >= sseHeartbeatInterval {
if _, err := fmt.Fprint(w, ":keep-alive\n\n"); err != nil {
return
}
flusher.Flush()
lastWrite = time.Now()
}
}
}
// resolveStartCursor decides where to start reading.
// - No Last-Event-ID → start at "$" (only new entries).
// - Last-Event-ID provided AND still in stream → resume from it.
// - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
// and resume from the oldest available entry's predecessor "0", so the
// next XREAD picks up everything still in the stream.
//
// The "still in stream" check is cheap: one XRANGE for the first entry.
func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
if lastID == "" {
return "$"
}
// One-entry XRANGE to find the oldest ID currently in the stream.
xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
if err != nil {
// Treat any error as "stream not available right now" and resume
// from the cursor — XREAD will either find it or block until new
// entries arrive.
log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
return lastID
}
if len(entries) == 0 {
// Stream is empty. Cursor may be from a previous incarnation; safest
// is to resume from the cursor — XREAD against an empty stream
// returns nothing and blocks until new entries arrive.
return lastID
}
oldest := entries[0].ID
if compareStreamIDs(lastID, oldest) >= 0 {
// Cursor is at or past the oldest — fine to resume.
return lastID
}
// Cursor was trimmed. Emit a gap event so consumers that care can react.
gap := map[string]string{
"requested": lastID,
"resumeFrom": oldest,
"reason": "retention",
}
payload, _ := json.Marshal(gap)
fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
flusher.Flush()
// Return "0" so the next XREAD picks up entries starting from the
// stream's true minimum — including `oldest` itself.
return "0"
}
// writeSSEEntry serialises one Redis Stream entry as an SSE event.
//
// `event:` comes from the entry's `type` field (no prefix munging). `data:`
// is a JSON object containing all OTHER fields. Field values from the Redis
// stream are interface{} (typically strings); we encode whatever the stream
// holds.
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
eventType := "message"
data := make(map[string]interface{}, len(msg.Values))
for k, v := range msg.Values {
if k == "type" {
if s, ok := v.(string); ok {
eventType = s
continue
}
}
data[k] = v
}
payload, err := json.Marshal(data)
if err != nil {
log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
return
}
// Spec: each event ends with a blank line. id/event/data lines are
// terminated with single \n.
fmt.Fprintf(w, "id: %s\n", msg.ID)
fmt.Fprintf(w, "event: %s\n", eventType)
fmt.Fprintf(w, "data: %s\n\n", string(payload))
flusher.Flush()
}
// compareStreamIDs compares two Redis Stream IDs of the form "<ms>-<seq>".
// Returns -1/0/+1. Missing or malformed IDs sort as zero parts.
func compareStreamIDs(a, b string) int {
aMs, aSeq := parseStreamID(a)
bMs, bSeq := parseStreamID(b)
switch {
case aMs < bMs:
return -1
case aMs > bMs:
return 1
case aSeq < bSeq:
return -1
case aSeq > bSeq:
return 1
default:
return 0
}
}
func parseStreamID(id string) (int64, int64) {
parts := strings.SplitN(id, "-", 2)
ms, _ := strconv.ParseInt(parts[0], 10, 64)
var seq int64
if len(parts) == 2 {
seq, _ = strconv.ParseInt(parts[1], 10, 64)
}
return ms, seq
}
// isContextTimeout returns true for the context deadline error wrapped by
// go-redis when XREAD's read context fires. We treat it the same as
// redis.Nil — i.e. an empty poll cycle, not a fatal error.
func isContextTimeout(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
// go-redis sometimes wraps this in its own error type. String match is
// a safety net.
return strings.Contains(err.Error(), "context deadline exceeded")
}
+164
View File
@@ -0,0 +1,164 @@
package api
import (
"log"
"net/http"
"sync"
"time"
"github.com/metazla/meta-core/internal/identity"
)
// statsSweepTTL is how long a keyspace-walk result stays good. The walk is
// never on the request path (see statsCache), so this only controls how stale
// the library tiles may look, not how slow the endpoint is.
const statsSweepTTL = 60 * time.Second
// fileStatsJSON is the file-backed half of the library: the roots that carry
// filePath+sizeByte+mtimeNano. Gateway / cid-rooted records have no file on
// disk and are counted in Records, not here — the gap between the two numbers
// is exactly "metadata we hold for files we do not have".
type fileStatsJSON struct {
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
}
// StatsResponse is the body of GET /api/stats — every headline number the
// dashboard shows, in one request.
//
// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
// O(1) commands (SCARD, DBSIZE, INFO) plus a directory listing, recomputed on
// every call. Files and UDLUsers need a keyspace walk — ~0.5s for the file
// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
// from a background-refreshed cache and are null until the first sweep lands.
type StatsResponse struct {
Records int64 `json:"records"`
RedisKeys int64 `json:"redisKeys"`
RedisMemory string `json:"redisMemory"`
Identities int `json:"identities"`
Files *fileStatsJSON `json:"files"`
UDLUsers *int `json:"udlUsers"`
// SweptAt is when the cached half was computed (unix ms), 0 if never.
SweptAt int64 `json:"sweptAt"`
// Sweeping reports that a refresh is in flight, so the UI can show the
// cached numbers as provisional instead of pretending they are live.
Sweeping bool `json:"sweeping"`
}
// statsCache holds the expensive half of /api/stats.
//
// Why a cache and not just "compute it": the dashboard polls. A 3s Redis walk
// per tick, per open tab, is a self-inflicted outage on a box that is also
// serving playback — the same shape as meta-sort's /api/stats full sweep. So
// the handler NEVER waits on a walk: it returns whatever the last sweep
// produced and kicks off a new one if the old one is stale. `running` makes
// that single-flight, so ten tabs cost one walk.
type statsCache struct {
mu sync.Mutex
files *fileStatsJSON
udlUsers *int
sweptAt time.Time
running bool
}
// snapshot returns the cached values and whether a refresh is in flight.
func (c *statsCache) snapshot() (*fileStatsJSON, *int, time.Time, bool) {
c.mu.Lock()
defer c.mu.Unlock()
return c.files, c.udlUsers, c.sweptAt, c.running
}
// refreshIfStale starts a background sweep unless one is already running or
// the last result is still fresh. Returns true if it started one.
func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
c.mu.Lock()
if c.running || time.Since(c.sweptAt) < statsSweepTTL {
c.mu.Unlock()
return false
}
c.running = true
c.mu.Unlock()
go func() {
files, udl := sweep()
c.mu.Lock()
defer c.mu.Unlock()
c.running = false
// A failed sweep leaves the previous numbers in place rather than
// blanking the tiles; sweptAt still advances so a hard-failing walk
// (Redis down) does not spin once per poll.
c.sweptAt = time.Now()
if files != nil {
c.files = files
}
if udl != nil {
c.udlUsers = udl
}
}()
return true
}
// handleStats handles GET /api/stats.
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
resp := StatsResponse{}
if n, err := s.storage.CountRoots(); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
} else {
resp.Records = n
}
if n, err := s.storage.CountKeys(); err == nil {
resp.RedisKeys = n
}
if mem, err := s.storage.GetMemoryInfo(); err == nil {
resp.RedisMemory = mem
} else {
resp.RedisMemory = "N/A"
}
if list, err := identity.List(s.config.IdentityAccountsDir()); err == nil {
resp.Identities = len(list)
}
s.stats.refreshIfStale(s.sweepStats)
files, udl, sweptAt, sweeping := s.stats.snapshot()
resp.Files = files
resp.UDLUsers = udl
resp.Sweeping = sweeping
if !sweptAt.IsZero() {
resp.SweptAt = sweptAt.UnixMilli()
}
writeJSON(w, http.StatusOK, resp)
}
// sweepStats runs the two keyspace walks behind the cached tiles. Called off
// the request path only. A failing half returns nil so the other half still
// lands.
func (s *Server) sweepStats() (*fileStatsJSON, *int) {
var files *fileStatsJSON
if tuples, err := s.storage.GetTuplesForAllFiles(); err != nil {
log.Printf("[API] stats: file tuple sweep failed: %v", err)
} else {
fs := fileStatsJSON{Count: len(tuples)}
for _, t := range tuples {
fs.TotalSize += t.Size
}
files = &fs
}
var udl *int
if stats, err := s.storage.UDLAllUserStats(); err != nil {
log.Printf("[API] stats: UDL user sweep failed: %v", err)
} else {
n := len(stats)
udl = &n
}
return files, udl
}
+454
View File
@@ -0,0 +1,454 @@
package api
// User Data Layer (UDL) HTTP surface.
//
// Backs meta-watch's per-user store (My List, Continue Watching, ratings,
// profile) now that the authoritative store lives in meta-core instead of
// meta-watch's local op-log. Records arrive already signed (and, for
// private-tier keys, already encrypted) as opaque base64 CBOR blobs; meta-core
// version-gates and indexes them but never inspects the crypto. See
// internal/storage/udl.go for the Redis model.
import (
"encoding/json"
"math"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/storage"
)
// UDLPutRequest is the body of PUT /api/udl/record.
type UDLPutRequest struct {
Uid string `json:"uid"`
Cid string `json:"cid"`
Key string `json:"key"`
Version int64 `json:"version"`
Ts int64 `json:"ts"`
Record string `json:"record"` // base64 CBOR, opaque + signed
Value *json.RawMessage `json:"value,omitempty"`
// Tombstone marks this write as a deletion of (uid, cid, key). The record
// itself is opaque here — meta-core never decodes the CBOR, and a private-
// tier value and a tombstone are equally "no plaintext value" — so the
// writer has to say. Without it the cid is never removed from
// udl:idx:user:<uid>:key:<key>, which then grows for the life of the
// account and is returned in full on every My List / Continue Watching
// read. Omitted by older writers, which keeps the previous behaviour.
Tombstone bool `json:"tombstone,omitempty"`
}
// UDLPutResponse is the response of PUT /api/udl/record.
type UDLPutResponse struct {
Accepted bool `json:"accepted"`
}
// UDLPutBatchRequest is the body of PUT /api/udl/records — many cells, one
// request. Each entry is exactly a UDLPutRequest, so a client can build the
// same signed record for either endpoint.
type UDLPutBatchRequest struct {
Records []UDLPutRequest `json:"records"`
}
// UDLPutBatchResult is one cell's outcome. Accepted=false with an empty Error
// means the write was stale (the single-record endpoint's 409), which in a
// batch is an ordinary outcome rather than a failure.
type UDLPutBatchResult struct {
Key string `json:"key"`
Accepted bool `json:"accepted"`
Error string `json:"error,omitempty"`
}
// UDLPutBatchResponse mirrors the request order index-for-index.
type UDLPutBatchResponse struct {
Results []UDLPutBatchResult `json:"results"`
}
// maxUDLBatch bounds one batch. Each record is a Redis round trip while the
// request is held open; sized to match the sign batch on the other side of the
// same operation (see maxSignBatch).
const maxUDLBatch = 1000
// UDLRecordResponse is the response of GET /api/udl/record.
type UDLRecordResponse struct {
Record string `json:"record"`
Version int64 `json:"version"`
Ts int64 `json:"ts"`
Exists bool `json:"exists"`
// Value is the plaintext JSON scalar stored beside the opaque record, and
// is present only for PUBLIC-tier keys (like, rating, profile:name,
// profile:avatar). json.RawMessage rather than string because the field is
// generic over keys — a bool for `like`, a number for `rating`, a string
// for a profile name (see aggregatePublicValues) — and because it is the
// exact type UDLPutRequest.Value uses, so a value written through PUT reads
// back byte-identical.
//
// Absent, never null, when there is no plaintext: private-tier cells,
// tombstones and never-written cells must not read as an empty public
// value. Readers key off presence, not `exists`.
Value *json.RawMessage `json:"value,omitempty"`
}
// UDLAggregate mirrors meta-watch's Aggregate wire shape exactly (snake_case
// field names; `avg` is null for boolean keys like `like`), so the meta-watch
// client deserializes it into the same struct it serializes to the UI.
type UDLAggregate struct {
Key string `json:"key"`
Count int `json:"count"`
Avg *float64 `json:"avg"`
Trues int `json:"trues"`
SampleSize int `json:"sample_size"`
}
// handleUDLRecordGet handles GET /api/udl/record?uid&cid&key
func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
uid, cid, key := q.Get("uid"), q.Get("cid"), q.Get("key")
if uid == "" || cid == "" || key == "" {
writeError(w, http.StatusBadRequest, "uid, cid and key are required")
return
}
cell, exists, err := s.storage.UDLGetCell(uid, cid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
resp := UDLRecordResponse{
Record: cell.Record,
Version: cell.Version,
Ts: cell.Ts,
Exists: exists,
}
// json.Valid guards the encoder: a malformed `value` hash field (hand-edited
// Redis, a future writer bug) would otherwise fail mid-encode and emit a
// truncated body instead of a readable error.
if exists && cell.HasValue && json.Valid([]byte(cell.Value)) {
raw := json.RawMessage(cell.Value)
resp.Value = &raw
}
writeJSON(w, http.StatusOK, resp)
}
// handleUDLRecordPut handles PUT /api/udl/record. Version-gated: a stale write
// (version <= stored) returns 409 stale_version so the client can re-read and
// retry with a higher version.
func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req UDLPutRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Uid == "" || req.Cid == "" || req.Key == "" || req.Record == "" {
writeError(w, http.StatusBadRequest, "uid, cid, key and record are required")
return
}
publicValue := ""
hasValue := false
if req.Value != nil {
publicValue = string(*req.Value)
hasValue = true
}
accepted, err := s.storage.UDLUpsertIfNewer(req.Uid, req.Cid, req.Key, req.Version, req.Ts, req.Record, publicValue, hasValue, req.Tombstone)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if !accepted {
writeErrorSlug(w, http.StatusConflict, "stale_version", "a record with an equal or newer version already exists", true)
return
}
writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true})
}
// handleUDLRecordsPut handles PUT /api/udl/records — the batch form of
// handleUDLRecordPut.
//
// Exists for the bulk verbs above this layer ("mark this whole series seen"),
// where one user action produces one cell per episode. Each cell still goes
// through the same version-gated Lua upsert, so the CRDT semantics are
// identical — this batches the transport only.
//
// **Not transactional, and deliberately not 409.** A per-record `accepted` flag
// replaces the single-record conflict status: in a batch, one stale cell is a
// normal outcome (another device already marked that episode) and must not fail
// the 299 cells beside it. Callers read the array, not the status code.
func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req UDLPutBatchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if len(req.Records) == 0 {
writeError(w, http.StatusBadRequest, "records is required")
return
}
if len(req.Records) > maxUDLBatch {
writeError(w, http.StatusBadRequest, "too many records in one batch")
return
}
results := make([]UDLPutBatchResult, 0, len(req.Records))
for _, rec := range req.Records {
if rec.Uid == "" || rec.Cid == "" || rec.Key == "" || rec.Record == "" {
results = append(results, UDLPutBatchResult{
Key: rec.Key,
Error: "uid, cid, key and record are required",
})
continue
}
publicValue := ""
hasValue := false
if rec.Value != nil {
publicValue = string(*rec.Value)
hasValue = true
}
accepted, err := s.storage.UDLUpsertIfNewer(rec.Uid, rec.Cid, rec.Key, rec.Version, rec.Ts, rec.Record, publicValue, hasValue, rec.Tombstone)
if err != nil {
// A storage error is per-record here rather than fatal for the whole
// batch: the cells already written stay written, and reporting which
// ones failed lets the caller retry exactly those.
results = append(results, UDLPutBatchResult{Key: rec.Key, Error: err.Error()})
continue
}
results = append(results, UDLPutBatchResult{Key: rec.Key, Accepted: accepted})
}
writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: results})
}
// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}
// (active_for_user_key — My List, Continue Watching).
func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
vars := mux.Vars(r)
uid, key := vars["uid"], vars["key"]
if uid == "" || key == "" {
writeError(w, http.StatusBadRequest, "uid and key are required")
return
}
entries, err := s.storage.UDLListUserKey(uid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
}
// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}
// (list_for_user_cid).
func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
vars := mux.Vars(r)
uid, cid := vars["uid"], vars["cid"]
if uid == "" || cid == "" {
writeError(w, http.StatusBadRequest, "uid and cid are required")
return
}
entries, err := s.storage.UDLListUserCid(uid, cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
}
// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=<cid>&limit=<n>
// — every record a uid owns, for cross-instance profile recovery.
//
// The other user-scoped reads answer a question with one coordinate already
// known. Recovery knows only the uid: a box that has just imported a secret key
// has no idea which cids or keys the profile ever touched, so it needs the
// whole set or it gets nothing.
//
// Paged on a cid boundary — `nextCid` empty means the export is complete. The
// response is opaque signed records, exactly as stored; whoever consumes it is
// responsible for verifying each one before trusting it (meta-core has never
// held the account key and cannot verify on anyone's behalf).
func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
uid := mux.Vars(r)["uid"]
if uid == "" {
writeError(w, http.StatusBadRequest, "uid is required")
return
}
limit := maxUDLBatch
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n <= 0 {
writeError(w, http.StatusBadRequest, "limit must be a positive integer")
return
}
if n < limit {
limit = n
}
}
entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"entries": entries,
"nextCid": nextCid,
})
}
// handleUDLUserCount handles GET /api/udl/user/{uid}/count — how many cids a
// uid holds records for.
//
// Exists so a gateway mirror can enforce a per-profile quota without walking
// anything: 32 random bytes are a valid uid, so an unbounded mirror is a
// disk-fill DoS that needs no account. A quota check that itself cost a scan
// would just move the denial of service one layer up.
func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
uid := mux.Vars(r)["uid"]
if uid == "" {
writeError(w, http.StatusBadRequest, "uid is required")
return
}
n, err := s.storage.UDLCountUserCids(uid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
}
// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users (uids_for_cid).
func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
users, err := s.storage.UDLCidUsers(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
}
// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,rating.
// Computes per-key population stats from the stored public value fields
// (absorbs meta-watch's aggregate.rs). Reflects only records this core holds.
func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
keysParam := r.URL.Query().Get("keys")
if keysParam == "" {
keysParam = "like,rating"
}
aggregates := make([]UDLAggregate, 0)
for _, key := range strings.Split(keysParam, ",") {
key = strings.TrimSpace(key)
if key == "" {
continue
}
values, err := s.storage.UDLPublicValues(cid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
aggregates = append(aggregates, aggregatePublicValues(key, values))
}
writeJSON(w, http.StatusOK, map[string]interface{}{"aggregates": aggregates})
}
// aggregatePublicValues folds raw JSON scalar values into an UDLAggregate:
// counts booleans as trues and averages numeric values.
func aggregatePublicValues(key string, values []string) UDLAggregate {
agg := UDLAggregate{Key: key, SampleSize: len(values), Count: len(values)}
var sum float64
var numeric int
for _, raw := range values {
var v interface{}
if err := json.Unmarshal([]byte(raw), &v); err != nil {
continue
}
switch t := v.(type) {
case bool:
if t {
agg.Trues++
}
case float64:
sum += t
numeric++
}
}
if numeric > 0 {
avg := math.Round((sum/float64(numeric))*100) / 100
agg.Avg = &avg
}
return agg
}
// udlUserStatsResponse is the shape of GET /api/udl/users/stats.
type udlUserStatsResponse struct {
// Users maps uid -> counts. Accounts holding nothing are absent rather than
// present with zeroes; a caller rendering a list treats "missing" as 0.
Users map[string]storage.UDLUserStats `json:"users"`
}
// handleUDLUserStats handles GET /api/udl/users/stats — how much every account
// holds, in one keyspace walk.
//
// Bulk rather than per-uid because the only caller is the accounts page, which
// needs a number for every row: the counts come from a SCAN (there is no
// per-user cid index — see storage/udl_admin.go), and MATCH filters without
// indexing, so N per-uid calls would walk the whole keyspace N times.
//
// Deliberately not on the critical path of anything. It is an admin read whose
// cost grows with the keyspace, so callers should fetch it after rendering,
// never before.
func (s *Server) handleUDLUserStats(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage not connected", true)
return
}
stats, err := s.storage.UDLAllUserStats()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, udlUserStatsResponse{Users: stats})
}
+226
View File
@@ -0,0 +1,226 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/alicebob/miniredis/v2"
"github.com/metazla/meta-core/internal/storage"
)
// newUDLTestServer returns a Server wired to an in-memory Redis.
//
// The struct is built directly rather than through NewServer: NewServer boots
// the file watcher, mounts manager, watchers poller and WebDAV handler, none of
// which the UDL handlers touch — they use s.storage and nothing else.
func newUDLTestServer(t *testing.T) *Server {
t.Helper()
mr := miniredis.RunT(t)
c := storage.NewClient("")
if err := c.Connect("redis://" + mr.Addr()); err != nil {
t.Fatalf("connect to miniredis: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return &Server{storage: c}
}
// getRecord drives handleUDLRecordGet and decodes the body into raw JSON
// fields, so a test can assert on the *encoding* of `value` (is it a quoted
// string? is the key there at all?) rather than on what a typed struct would
// have coerced it into.
func getRecord(t *testing.T, s *Server, uid, cid, key string) (int, map[string]json.RawMessage) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
rr := httptest.NewRecorder()
s.handleUDLRecordGet(rr, req)
var body map[string]json.RawMessage
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
t.Fatalf("decode response %q: %v", rr.Body.String(), err)
}
return rr.Code, body
}
// The plaintext value of a PUBLIC-tier cell must come back over the wire, and
// come back *as it was written* — a JSON string stays a quoted string, a bool
// stays a bool.
//
// Both halves matter. Without the value at all, the meta-core dashboard can
// only ever show 44-character base58 uids: display names live in the User Data
// Layer as the public cell (uid, "self", "profile:name"), and the plaintext
// sits in Redis where the storage layer has always read it — the handler just
// used to drop it. And the type has to stay json.RawMessage: this endpoint is
// generic over keys, so a `*string` would mangle `like` (bool) and `rating`
// (number) while looking correct for names.
func TestUDLRecordGet_ReturnsPublicValue(t *testing.T) {
s := newUDLTestServer(t)
const uid = "zAlice"
if ok, err := s.storage.UDLUpsertIfNewer(uid, "self", "profile:name", 1, 100, "cmVj", `"Ada Lovelace"`, true, false); err != nil || !ok {
t.Fatalf("seed name: ok=%v err=%v", ok, err)
}
if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "like", 1, 100, "cmVj", `true`, true, false); err != nil || !ok {
t.Fatalf("seed like: ok=%v err=%v", ok, err)
}
code, body := getRecord(t, s, uid, "self", "profile:name")
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if got := string(body["value"]); got != `"Ada Lovelace"` {
t.Fatalf("value = %s, want a quoted JSON string", got)
}
var name string
if err := json.Unmarshal(body["value"], &name); err != nil || name != "Ada Lovelace" {
t.Fatalf("value should unmarshal to the plain name, got %q err=%v", name, err)
}
// A boolean key survives unquoted — the assertion a *string field fails.
if _, body = getRecord(t, s, uid, "bagcsaaa", "like"); string(body["value"]) != "true" {
t.Fatalf("like value = %s, want the bare JSON literal true", body["value"])
}
}
// A cell with no plaintext must omit `value` entirely rather than send null.
//
// That is the shape of every private-tier key (seek, watched, card, localpath —
// their real value rides encrypted inside the opaque record) and of every
// tombstone. A reader keys off presence: "there is no plaintext here" and "the
// plaintext is null" are different facts, and a name cleared by its owner
// leaves the cell existing with no value, which must not read as a name.
func TestUDLRecordGet_OmitsValueForPrivateCell(t *testing.T) {
s := newUDLTestServer(t)
const uid = "zBob"
if ok, err := s.storage.UDLUpsertIfNewer(uid, "bagcsaaa", "seek", 1, 100, "ZW5jcnlwdGVk", "", false, false); err != nil || !ok {
t.Fatalf("seed private cell: ok=%v err=%v", ok, err)
}
code, body := getRecord(t, s, uid, "bagcsaaa", "seek")
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if _, ok := body["value"]; ok {
t.Fatalf("private cell must omit `value`, got %s", body["value"])
}
if string(body["exists"]) != "true" {
t.Fatalf("exists = %s, want true", body["exists"])
}
}
// A cell that was never written answers 200 with exists:false and no value —
// not a 404, and not an empty-string value that would render as a blank name.
func TestUDLRecordGet_MissingCellHasNoValue(t *testing.T) {
s := newUDLTestServer(t)
code, body := getRecord(t, s, "zNobody", "self", "profile:name")
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if string(body["exists"]) != "false" {
t.Fatalf("exists = %s, want false", body["exists"])
}
if _, ok := body["value"]; ok {
t.Fatalf("missing cell must omit `value`, got %s", body["value"])
}
}
// putRecords drives handleUDLRecordsPut with a raw JSON body and decodes the
// per-record results.
func putRecords(t *testing.T, s *Server, body string) (int, UDLPutBatchResponse) {
t.Helper()
req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(body))
rr := httptest.NewRecorder()
s.handleUDLRecordsPut(rr, req)
var out UDLPutBatchResponse
if rr.Code == http.StatusOK {
if err := json.Unmarshal(rr.Body.Bytes(), &out); err != nil {
t.Fatalf("decode response %q: %v", rr.Body.String(), err)
}
}
return rr.Code, out
}
// **The load-bearing property of the batch endpoint: one stale cell does not
// fail the batch.**
//
// The single-record PUT answers a stale version with 409, which is right when
// the caller wrote one thing. In a batch it would be wrong: bulk verbs ("mark
// this whole series seen") routinely include episodes another device already
// marked, and 409-ing there would abandon every cell after it. So staleness is
// reported per record as accepted:false, with 200 on the request itself.
func TestUDLRecordsPut_StaleCellDoesNotFailTheBatch(t *testing.T) {
s := newUDLTestServer(t)
const uid = "zCarol"
const cid = "bagdsaaaseries"
// Seed s01e01 at version 4 — the batch below tries to write it at 2.
if ok, err := s.storage.UDLUpsertIfNewer(uid, cid, "seen:s01e01", 4, 100, "b2xk", "", false, false); err != nil || !ok {
t.Fatalf("seed: ok=%v err=%v", ok, err)
}
code, out := putRecords(t, s, `{"records":[
{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e01","version":2,"ts":200,"record":"c3RhbGU"},
{"uid":"`+uid+`","cid":"`+cid+`","key":"seen:s01e02","version":1,"ts":200,"record":"ZnJlc2g"}
]}`)
if code != http.StatusOK {
t.Fatalf("status = %d, want 200 — a stale cell is an outcome, not an error", code)
}
if len(out.Results) != 2 {
t.Fatalf("results = %d, want 2 (one per record, in order)", len(out.Results))
}
if out.Results[0].Accepted || out.Results[0].Error != "" {
t.Fatalf("stale cell = %+v, want accepted:false with no error", out.Results[0])
}
if !out.Results[1].Accepted {
t.Fatalf("fresh cell = %+v, want accepted:true", out.Results[1])
}
// The fresh cell really landed, and the stale one did not overwrite.
if _, body := getRecord(t, s, uid, cid, "seen:s01e02"); string(body["exists"]) != "true" {
t.Fatalf("s01e02 was not written")
}
if _, body := getRecord(t, s, uid, cid, "seen:s01e01"); string(body["version"]) != "4" {
t.Fatalf("s01e01 version = %s, want 4 (the stale write must not apply)", body["version"])
}
}
// A malformed record is reported in its own slot rather than rejecting the
// request, for the same reason: the caller retries that cell, not the batch.
func TestUDLRecordsPut_ReportsBadRecordPerSlot(t *testing.T) {
s := newUDLTestServer(t)
code, out := putRecords(t, s, `{"records":[
{"uid":"zDan","cid":"bagdsaaa","key":"","version":1,"ts":1,"record":"eA"},
{"uid":"zDan","cid":"bagdsaaa","key":"seen:s01e01","version":1,"ts":1,"record":"eA"}
]}`)
if code != http.StatusOK {
t.Fatalf("status = %d, want 200", code)
}
if out.Results[0].Error == "" {
t.Fatalf("record with no key must report an error, got %+v", out.Results[0])
}
if !out.Results[1].Accepted {
t.Fatalf("the valid record beside it must still land, got %+v", out.Results[1])
}
}
// An empty batch is a client bug, not a no-op: it almost always means the
// caller built its record list wrong, and answering 200 with an empty array
// would hide that.
func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
s := newUDLTestServer(t)
if code, _ := putRecords(t, s, `{"records":[]}`); code != http.StatusBadRequest {
t.Fatalf("status = %d, want 400", code)
}
}
+92
View File
@@ -0,0 +1,92 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"fmt"
"strings"
"unicode/utf8"
)
// DecodeCard recovers the (source, id) pair a card locator (CodeCard, 0x1007)
// wraps. Wire form:
//
// 0x01 ‖ varint(0x1007) ‖ 0x00 (identity mh) ‖ varint(len(digest)) ‖ digest
// digest = varint(len(source)) ‖ source ‖ id (both UTF-8)
//
// Decode stops at the multihash code because ranking needs nothing past it;
// reading *which work* a card addresses needs the digest too, so this is the
// one place in Go that walks the whole thing.
//
// ⚠ Mirror of meta-read `card_cid::decode` and of meta-feeder-sdk
// `compute_card_cid`. The vectors in card_test.go are pinned literals copied
// from meta-read, not round-trips, so a drift on either side fails a test
// instead of silently reading a different work out of the same CID.
//
// Identity multihash only: a 0x1007 CID carrying any other multihash is not
// something anyone mints, and reading its digest as (source, id) would be
// reading hash bytes as text. Bytes past the declared digest length are
// ignored, as meta-read does.
func DecodeCard(cidStr string) (source, id string, err error) {
if len(cidStr) < 2 || cidStr[0] != 'b' {
return "", "", fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
}
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
if err != nil {
return "", "", fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
}
next := func(buf []byte, what string) (uint64, []byte, error) {
v, n := binary.Uvarint(buf)
if n <= 0 {
return 0, nil, fmt.Errorf("cid %q: bad %s varint", cidStr, what)
}
return v, buf[n:], nil
}
version, raw, err := next(raw, "version")
if err != nil {
return "", "", err
}
if version != 1 {
return "", "", fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
}
codec, raw, err := next(raw, "codec")
if err != nil {
return "", "", err
}
if codec != CodeCard {
return "", "", fmt.Errorf("cid %q: codec 0x%x is not a card locator", cidStr, codec)
}
mh, raw, err := next(raw, "multihash-code")
if err != nil {
return "", "", err
}
if mh != 0 {
return "", "", fmt.Errorf("cid %q: card locator with non-identity multihash 0x%x", cidStr, mh)
}
digestLen, raw, err := next(raw, "digest-length")
if err != nil {
return "", "", err
}
if digestLen > uint64(len(raw)) {
return "", "", fmt.Errorf("cid %q: digest truncated", cidStr)
}
digest := raw[:digestLen]
sourceLen, rest, err := next(digest, "source-length")
if err != nil {
return "", "", err
}
if sourceLen > uint64(len(rest)) {
return "", "", fmt.Errorf("cid %q: source truncated", cidStr)
}
source, id = string(rest[:sourceLen]), string(rest[sourceLen:])
if source == "" || id == "" {
return "", "", fmt.Errorf("cid %q: empty source or id", cidStr)
}
if !utf8.ValidString(source) || !utf8.ValidString(id) {
return "", "", fmt.Errorf("cid %q: source or id is not UTF-8", cidStr)
}
return source, id, nil
}
+71
View File
@@ -0,0 +1,71 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"strings"
"testing"
)
// Pinned from meta-read `src/card_cid.rs` / meta-feeder-sdk `compute_card_cid`
// — literals, not round-trips (see DecodeCard).
const (
goldenAnilistCard = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy"
goldenMangadexCard = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
)
// wireCID builds a CIDv1 from raw parts so the negative cases below can vary
// one field at a time.
func wireCID(codec, mh uint64, digest []byte) string {
buf := []byte{0x01}
buf = binary.AppendUvarint(buf, codec)
buf = binary.AppendUvarint(buf, mh)
buf = binary.AppendUvarint(buf, uint64(len(digest)))
buf = append(buf, digest...)
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf))
}
func cardDigest(source, id string) []byte {
d := binary.AppendUvarint(nil, uint64(len(source)))
d = append(d, source...)
return append(d, id...)
}
func TestDecodeCard_GoldenVectors(t *testing.T) {
for _, tc := range []struct {
cid, source, id string
}{
{goldenAnilistCard, "anilist", "manga:105778"},
{goldenMangadexCard, "mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"},
} {
source, id, err := DecodeCard(tc.cid)
if err != nil {
t.Fatalf("DecodeCard(%s): %v", tc.cid, err)
}
if source != tc.source || id != tc.id {
t.Errorf("DecodeCard(%s) = (%q, %q), want (%q, %q)", tc.cid, source, id, tc.source, tc.id)
}
// The test encoder must agree with the pinned literal, or the negative
// cases below are testing a different wire format.
if got := wireCID(CodeCard, 0, cardDigest(tc.source, tc.id)); got != tc.cid {
t.Errorf("wireCID(%q, %q) = %s, want %s", tc.source, tc.id, got, tc.cid)
}
}
}
func TestDecodeCard_RejectsEverythingElse(t *testing.T) {
for name, s := range map[string]string{
"url locator codec": wireCID(CodeURL, 0, cardDigest("anilist", "1")),
"non-identity multihash": wireCID(CodeCard, CodeSha256, cardDigest("anilist", "1")),
"empty id": wireCID(CodeCard, 0, cardDigest("anilist", "")),
"source past digest": wireCID(CodeCard, 0, []byte{0x20, 'a'}),
"not a CID": "title:berserk",
"CIDv0": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
"truncated": goldenAnilistCard[:12],
"empty": "",
} {
if source, id, err := DecodeCard(s); err == nil {
t.Errorf("%s: DecodeCard(%q) = (%q, %q), want error", name, s, source, id)
}
}
}
+184
View File
@@ -0,0 +1,184 @@
{
"$schema": "cid-rank-vectors/v1",
"$comment": [
"CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
"The rank decides which of a record's several CIDs is elected canonical - the cid a",
"record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
"they cannot share code (independent submodules, no cargo path-dep). These vectors are",
"the only thing holding them in sync: every implementation's test suite loads this file.",
"",
"This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
"own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
"",
"Ranking is by UTILITY for external retrieval, not cryptographic strength.",
"Ties break on the lexicographically smaller cid, so the election is deterministic",
"across peers and does not churn.",
"",
"See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
],
"ladder": {
"40": "dag-pb (IPFS block)",
"30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
"25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
"20": "BitTorrent infohashes - swarm interop",
"10": "midhash256 - the record address",
"5": "opaque locators (nzb-release, url, card, yt-video, ext-play) - never outrank a real digest",
"0": "weak/unknown digests, unparseable input"
},
"tieBreak": "lexicographically smaller cid wins",
"vectors": [
{
"name": "ipfs-dagpb",
"cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 40,
"note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
},
{
"name": "sha2-256-raw-codec",
"cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 30,
"note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
},
{
"name": "sha2-256-codec-eq-mh",
"cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
"rank": 30,
"note": "REAL meta-hash fixture; meta-hash sets codec == mh"
},
{
"name": "sha3-256",
"cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
"rank": 30,
"note": "REAL meta-hash fixture"
},
{
"name": "sha3-384",
"cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
"rank": 30,
"note": "REAL meta-hash fixture"
},
{
"name": "nzb-posting",
"cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 25,
"note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
},
{
"name": "btih-v2-codec-eq-mh",
"cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "meta-hash encoder convention"
},
{
"name": "btih-v2-raw-codec",
"cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "fullhash encoder convention - SAME tier, both conventions must match"
},
{
"name": "btih-v1-file",
"cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "gateway per-file BitTorrent v1"
},
{
"name": "btih-v2-file",
"cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "gateway per-file BitTorrent v2"
},
{
"name": "midhash256-codec-eq-mh",
"cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 10,
"note": "the record address; internally meaningful, externally invisible"
},
{
"name": "midhash256-raw-codec",
"cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 10
},
{
"name": "nzb-release-locator",
"cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
"rank": 5,
"note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
},
{
"name": "url-locator",
"cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
"rank": 5,
"note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
},
{
"name": "card-locator",
"cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
"rank": 5,
"note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
},
{
"name": "yt-video-locator",
"cid": "bagecaaarav3gszdfn42ein3vgvfumn2tka4a",
"rank": 5,
"note": "codec 0x1008 + IDENTITY multihash 0x00, digest = varint(len(kind)) || kind || id — here (\"video\",\"4D7u5KF7SP8\"). DELEGATED PLAYBACK: the limit case beyond card. A card addresses a work with no bytes anywhere; this addresses a *rendition* whose bytes exist but are permanently someone else's — an external player renders it, nothing is transported, seeded or content-addressed. Framing is byte-identical to card (0x1007) on purpose, so each implementation extends the parser it already wrote. Same codec-slot rule: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5 is load-bearing for the on-ramp: a delegated reference and a real archived copy can sit on ONE record, and that sha2-256 must win the canonical election over this locator with no addressing change. NOTE the byte path must REFUSE this (400), not 404 — see docs/cid-formats.md §7.4."
},
{
"name": "ext-play-locator",
"cid": "bagesaabenb2hi4dthixs6zlymfwxa3dffzrgc3temnqw24bomnxw2l3bnrrhk3jppa",
"rank": 5,
"note": "codec 0x1009 + IDENTITY multihash 0x00, digest = the verbatim UTF-8 URL. WIRE-IDENTICAL to url (0x1006) apart from the codec, and that difference is the entire contract: 0x1006 means 'fetch these bytes once and seed them', 0x1009 means 'open this in a browser, there are no bytes for us'. A decoder that matched on SHAPE rather than on the codec slot would see an identity multihash wrapping an http(s) URL, take the 0x1006 path, fetch a watch/Bandcamp page, chunk it, seed the HTML and link that content CID onto the record AS THE FILE — the MSR1 poisoning shape, except permanent and replicated. Same tier/reasoning as the rest of the locator family."
},
{
"name": "provider-file-locator",
"cid": "bagfcaaa2bvxxazloon2we5djorwgk43gnfwgkorxga3dcobtgq",
"rank": 5,
"note": "codec 0x100A + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id - here (\"opensubtitles\",\"file:7061834\"). The GENERIC fetchable locator: one codec for every provider whose bytes are fetched the same way; the provider is inside the digest. Framing byte-identical to card (0x1007), but the byte path RESOLVES it (via source's fetch tier) instead of 404ing. Same codec-slot rule as the rest of the family: match on the CODEC, never the mh (identity 0x00 would fall through and score 0). Tier 5: once the fetched bytes' sha2-256 is linked onto the record it wins the canonical election with no addressing change."
},
{
"name": "sha1",
"cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "md5",
"cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "crc32",
"cid": "bagzafmqcathsbnrk",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "bt-pieces-root",
"cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 0,
"note": "torrent-internal merkle root, NOT a file-identity cid"
},
{
"name": "cidv0-rejected",
"cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
"rank": 0,
"note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
},
{
"name": "token-form-rejected",
"cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 0,
"note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
},
{
"name": "garbage",
"cid": "not-a-cid",
"rank": 0
},
{
"name": "empty",
"cid": "",
"rank": 0
}
]
}
+230
View File
@@ -0,0 +1,230 @@
// Package cid centralizes CID-format helpers: decoding a multibase-base32
// CIDv1 into its multicodec parts, ranking by utility for external
// presentation, and picking which CID is the canonical externally-visible
// identifier of a file.
//
// Sibling CIDs are stored as a bare-CID key-set (`cids/<cid> = "true"`); a
// CID is self-describing — its algorithm is recoverable from the CIDv1
// codec / multihash code, so there is no `<algorithm>:` prefix and no
// per-algorithm field name. Ranking and selection parse the multicodec
// rather than a string prefix.
package cid
import (
"encoding/base32"
"encoding/binary"
"fmt"
"strings"
)
// Multicodec / multihash codes used for ranking. Source of truth for the
// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
// METADATA_KEYS.md §2.
//
// The tier every code maps to is pinned by the golden fixture
// `cid-rank-vectors.json` (vendored next to this package). This is one of
// seven implementations of the same ladder across Go, Rust and TypeScript;
// the fixture is the only thing keeping them in agreement. If you add a code
// here, add a vector — an unranked code silently falls to tier 0, which is
// exactly how the nzb-release/url locators came to be ranked 20 in meta-share
// and 0 here and in meta-search.
const (
CodecRaw = 0x55 // raw — IPFS content-addressed block
CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block
CodeSha1 = 0x11 // multihash sha1
CodeSha256 = 0x12 // multihash sha2-256
CodeSha3_384 = 0x15 // multihash sha3-384
CodeSha3_256 = 0x16 // multihash sha3-256
CodeMd5 = 0xd5 // multihash md5
CodeCrc32 = 0x0132 // multihash crc32
CodeMidhash256 = 0x1000 // custom — SHA-256(size‖middle 1MB)
CodeBtPiecesRoot = 0xb702 // bittorrent v2 pieces root
CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52)
CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID
CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID
// Opaque locators. These are NOT digests: the payload rides in an identity
// multihash (0x00), so the code identifying them sits in the *content
// codec* slot, never the multihash slot. Rank must match on the codec for
// these — matching on mh would see 0x00 and rank them 0.
CodeNzbRelease = 0x1005 // custom — self-describing Newznab release locator
CodeURL = 0x1006 // custom — identity-multihash CID wrapping an http(s) URL
// CodeCard is the limit case of the family: the only CID with NO bytes
// behind it anywhere, ever. It addresses a *work* (a series, a film) via
// varint(len(source))‖source‖id — ("tmdb","tv:95479") — so `/api/file/{cid}`
// correctly 404s and bitswap never offers it. Ranking it here (rather than
// letting it fall to RankUnknown) is what makes the planned "one card, two
// CIDs" cross-source merge safe: a TMDB and a MyAnimeList locator can sit on
// one record, and a real digest added later still wins the election.
CodeCard = 0x1007 // custom — identity-multihash CID wrapping a card (work) id
// CodeYtVideo and CodeExtPlay are the DELEGATED-PLAYBACK locators — the
// limit case beyond CodeCard. A card addresses a work with no bytes
// anywhere; these address a *rendition* whose bytes exist but are
// permanently someone else's: an external player renders the media, nothing
// is transported, seeded or content-addressed.
//
// They rank on the CODEC like the rest of the family — their multihash is
// identity (0x00), so a rank that dispatched on the multihash would see 0x00,
// fall through and silently score 0. That is the exact drift that once made
// meta-share and meta-search elect different canonical CIDs for one record.
//
// Tier 5 is also what keeps the on-ramp safe: a delegated reference and a
// real CID can sit on one record, and if the media is ever archived to the
// mesh its sha2-256 wins the canonical election with no addressing change.
CodeYtVideo = 0x1008 // custom — identity-multihash CID wrapping a YouTube (kind, id)
CodeExtPlay = 0x1009 // custom — identity-multihash CID wrapping a URL to OPEN, never fetch
// CodeProviderFile is the FETCHABLE twin of CodeExtPlay: a provider-held
// file addressed as varint(len(source))‖source‖id — ("opensubtitles",
// "file:7061834"). Framing is byte-identical to CodeCard; only the byte
// path differs (it redeems the id through that source's feeder instead of
// 404ing). Same codec-slot rule and same tier: once the fetched bytes'
// sha2-256 is linked onto the record it wins with no addressing change.
CodeProviderFile = 0x100A // custom — identity-multihash CID wrapping a (source, id) provider file
// CodeNzbPosting is the INVERSE trap of the two above: its multihash is a
// REAL sha2-256 digest (over the normalised, sorted article Message-ID
// set of a Usenet posting we scanned ourselves — never a locator, no
// indexer host embedded), so it would silently fall into RankDigest via
// the mh switch unless matched on the codec FIRST. It is content-derived
// (so it outranks the opaque locators) but addresses no fetchable block —
// the digest is over an id set, not over any block's bytes, so it can
// never be bitswap-fetched under itself — so it must stay BELOW a real
// digest (once the media is materialised and re-seeded, that genuine
// sha2-256 cid has to win the canonical election, not this one).
CodeNzbPosting = 0x1003 // custom — self-scanned Usenet posting identity
)
// Rank tiers, named so the ladder reads as a ladder.
const (
RankIPFS = 40 // dag-pb — broadest network reach
RankDigest = 30 // sha2-256 / sha3-* — real content digests
RankNzbPosting = 25 // content-derived, but the digest addresses no fetchable block
RankBtih = 20 // BitTorrent infohashes — swarm interop
RankMidhash = 10 // the record address — internally meaningful, externally invisible
RankLocator = 5 // opaque locators — never outrank a real digest
RankUnknown = 0 // weak/unknown digests, unparseable input
)
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content
// codec and multihash function code. It is the inverse of the encoder in
// internal/watcher/midhash.go. Returns an error for anything that isn't a
// CIDv1 in base32lower (e.g. a CIDv0 "Qm…" or a non-CID string).
func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
if len(cidStr) < 2 || cidStr[0] != 'b' {
return 0, 0, fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
}
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
if err != nil {
return 0, 0, fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
}
version, n := binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad version varint", cidStr)
}
raw = raw[n:]
if version != 1 {
return 0, 0, fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
}
codec, n = binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad codec varint", cidStr)
}
raw = raw[n:]
mhCode, n = binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad multihash-code varint", cidStr)
}
return codec, mhCode, nil
}
// Rank returns a score for a bare CID. Higher = more preferred as the
// public-facing "canonical" identifier for a file. Unknown / undecodable
// CIDs rank 0 so a new digest type can't accidentally outrank an
// established one until it's classified here.
//
// We rank by the **multihash code** (the algorithm), with the custom
// BitTorrent CID codecs checked alongside it. This is deliberate: MetaMesh's
// two CID encoders disagree on the *content codec* — meta-hash/the watcher
// set codec == algorithm code, while the fullhash plugin and gateway wrap
// every digest with the raw codec (0x55) and put the algorithm in the
// multihash code. The multihash code is the algorithm in all of them, so it
// is the only reliable discriminator. (The BT info hash is given a distinct
// multihash code — btih-v2 0x10B7 — so it can't be confused with a plain
// sha2-256, see METADATA_KEYS.md §14.13.)
//
// Ordering rationale (utility-weighted, NOT cryptographic-strength):
// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
// block of the file IS its sha2-256, so they share this tier).
// - nzb-posting is content-derived (a digest over a Usenet posting's article
// Message-ID set) so it outranks the opaque locators below, but the
// digest addresses no fetchable block, so it stays below a real content
// digest — once the media is materialised and re-seeded under a genuine
// sha2-256 cid, that cid must win the election, not this one.
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
// swarm interop, not strength.
// - midhash256 is the floor among real *digests*: internally meaningful,
// externally invisible.
// - opaque locators (nzb-release, url, card) sit BELOW midhash. They are not
// content-derived — two URLs to the same bytes yield two different locator
// CIDs — so a locator must never displace a real digest as the identity a
// record is advertised under. Tier 5 means a locator wins the election only
// when it is the record's sole CID, which is exactly the guarantee
// METADATA_KEYS.md §2 states.
// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
func Rank(cidStr string) int {
codec, mh, err := Decode(cidStr)
if err != nil {
return RankUnknown
}
if codec == CodecDagPB {
return RankIPFS
}
// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
// multihash code, in either encoder convention.
if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
return RankBtih
}
// Opaque locators. Codec-only: their multihash is identity (0x00), so
// there is nothing to match on the mh side.
if codec == CodeNzbRelease || codec == CodeURL || codec == CodeCard ||
codec == CodeYtVideo || codec == CodeExtPlay || codec == CodeProviderFile {
return RankLocator
}
// nzb-posting: codec-only, checked BEFORE the mh switch below — its
// multihash is a real sha2-256 and would otherwise match CodeSha256 there
// and rank 30, tying with genuine content digests.
if codec == CodeNzbPosting {
return RankNzbPosting
}
switch mh {
case CodeSha256, CodeSha3_256, CodeSha3_384:
return RankDigest
case CodeMidhash256:
return RankMidhash
default:
// md5, crc32, plain sha1, pieces-root, unknown.
return RankUnknown
}
}
// Better picks the higher-ranked of two CIDs. On ties, returns the
// lexicographically smaller one — gives the reconciler a deterministic
// answer so the chosen canonical doesn't churn when peers add the same
// digest under a different ordering.
func Better(a, b string) string {
ra, rb := Rank(a), Rank(b)
if ra > rb {
return a
}
if rb > ra {
return b
}
if a <= b {
return a
}
return b
}
+195
View File
@@ -0,0 +1,195 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"encoding/json"
"os"
"strings"
"testing"
)
// goldenVectors is the CANONICAL cross-language fixture: the same file is
// vendored into meta-search, meta-share and meta-hash, and each of their test
// suites asserts against it. It is the only mechanism keeping the seven
// implementations of this ladder in agreement — they live in independent
// submodules and cannot share code. `scripts/check-cid-vectors.sh` at the
// meta-root fails if any vendored copy drifts from the canonical one.
const goldenVectors = "cid-rank-vectors.json"
type vector struct {
Name string `json:"name"`
CID string `json:"cid"`
Rank int `json:"rank"`
Note string `json:"note"`
}
func loadVectors(t *testing.T) []vector {
t.Helper()
raw, err := os.ReadFile(goldenVectors)
if err != nil {
t.Fatalf("read %s: %v (run ./scripts/check-cid-vectors.sh --fix from the meta-root)", goldenVectors, err)
}
var doc struct {
Vectors []vector `json:"vectors"`
}
if err := json.Unmarshal(raw, &doc); err != nil {
t.Fatalf("parse %s: %v", goldenVectors, err)
}
if len(doc.Vectors) == 0 {
t.Fatalf("%s has no vectors", goldenVectors)
}
return doc.Vectors
}
// TestRank_GoldenVectors is the load-bearing test. Every other test in this
// file is a convenience; this one is the contract with Rust and TypeScript.
func TestRank_GoldenVectors(t *testing.T) {
for _, v := range loadVectors(t) {
if got := Rank(v.CID); got != v.Rank {
t.Errorf("Rank(%s) = %d, want %d\n cid: %s\n note: %s",
v.Name, got, v.Rank, v.CID, v.Note)
}
}
}
// buildCID assembles a multibase-base32 CIDv1 from a content codec, a
// multihash code, and a digest length (digest bytes are zero — only the
// header is under test). Mirrors the encoder in watcher/midhash.go and the
// TS CID.createV1(code, create(code, digest)) path.
func buildCID(codec, mhCode uint64, digestLen int) string {
var b []byte
b = binary.AppendUvarint(b, 1) // CIDv1
b = binary.AppendUvarint(b, codec)
b = binary.AppendUvarint(b, mhCode)
b = binary.AppendUvarint(b, uint64(digestLen))
b = append(b, make([]byte, digestLen)...)
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
return "b" + strings.ToLower(enc)
}
// Real CIDs produced by @metazla/meta-hash (from its main.test.ts). For
// digest CIDs meta-hash sets codec == multihash code == algorithm code.
const (
cidCrc32 = "bagzafmqcathsbnrk"
cidMd5 = "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa"
cidSha1 = "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra"
cidSha256 = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
cidSha3256 = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
cidSha3384 = "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga"
)
func TestDecode_Fixtures(t *testing.T) {
cases := []struct {
name string
cid string
wantMh uint64
}{
{"crc32", cidCrc32, CodeCrc32},
{"md5", cidMd5, CodeMd5},
{"sha1", cidSha1, CodeSha1},
{"sha2-256", cidSha256, CodeSha256},
{"sha3-256", cidSha3256, CodeSha3_256},
{"sha3-384", cidSha3384, CodeSha3_384},
}
for _, tc := range cases {
codec, mh, err := Decode(tc.cid)
if err != nil {
t.Errorf("Decode(%s %q) error: %v", tc.name, tc.cid, err)
continue
}
// meta-hash digest CIDs use codec == multihash code.
if codec != tc.wantMh || mh != tc.wantMh {
t.Errorf("Decode(%s) = codec 0x%x mh 0x%x, want both 0x%x", tc.name, codec, mh, tc.wantMh)
}
}
}
func TestDecode_Rejects(t *testing.T) {
for _, in := range []string{"", "Qmabc", "x", "bzzzz!!"} {
if _, _, err := Decode(in); err == nil {
t.Errorf("Decode(%q) expected error, got nil", in)
}
}
}
func TestRank_Tiers(t *testing.T) {
ipfsDagPB := buildCID(CodecDagPB, CodeSha256, 32) // chunked IPFS file
rawSha256 := buildCID(CodecRaw, CodeSha256, 32) // fullhash-style sha2-256
btihV2meta := buildCID(CodeBtihV2, CodeBtihV2, 32) // meta-hash btih-v2 (codec==mh)
btihV2raw := buildCID(CodecRaw, CodeBtihV2, 32) // fullhash btih-v2 (raw codec)
btV1File := buildCID(CodeBtihV1File, CodeBtihV1File, 20) // gateway per-file btih v1
midhashRaw := buildCID(CodecRaw, CodeMidhash256, 32) // fullhash midhash (raw codec)
cases := []struct {
name string
cid string
want int
}{
{"ipfs-dagpb", ipfsDagPB, 40},
{"sha2-256(fixture, codec==mh)", cidSha256, 30},
{"sha2-256(raw codec)", rawSha256, 30},
{"sha3-256", cidSha3256, 30},
{"sha3-384", cidSha3384, 30},
{"btih-v2(codec==mh)", btihV2meta, 20},
{"btih-v2(raw codec)", btihV2raw, 20},
{"btih-v1-file", btV1File, 20},
{"midhash256(raw codec)", midhashRaw, 10},
{"sha1", cidSha1, 0},
{"md5", cidMd5, 0},
{"crc32", cidCrc32, 0},
{"garbage", "not-a-cid", 0},
}
for _, tc := range cases {
if got := Rank(tc.cid); got != tc.want {
t.Errorf("Rank(%s) = %d, want %d", tc.name, got, tc.want)
}
}
}
func TestRank_StrictOrdering(t *testing.T) {
ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
sha := Rank(cidSha256)
posting := Rank(buildCID(CodeNzbPosting, CodeSha256, 32))
btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
unknown := Rank(cidMd5)
if !(ipfs > sha && sha > posting && posting > btih && btih > mid && mid > unknown) {
t.Fatalf("rank ordering broken: ipfs=%d sha=%d posting=%d btih=%d mid=%d unknown=%d",
ipfs, sha, posting, btih, mid, unknown)
}
}
// TestRank_NzbPostingMatchesCodecNotMultihash pins the inverse trap:
// nzb-posting's multihash is a REAL sha2-256 (unlike nzb-release/url, whose mh
// is identity 0x00), so Rank must match on the codec BEFORE falling into the
// mh switch — otherwise this ties with a genuine content digest at 30 instead
// of sitting at 25.
func TestRank_NzbPostingMatchesCodecNotMultihash(t *testing.T) {
posting := buildCID(CodeNzbPosting, CodeSha256, 32)
if got := Rank(posting); got != RankNzbPosting {
t.Errorf("Rank(nzb-posting) = %d, want %d (RankNzbPosting) — codec check must precede the mh switch", got, RankNzbPosting)
}
// A real sha2-256 with the SAME digest bytes must still rank 30 — the
// codec, not the digest, is what distinguishes the two.
realDigest := buildCID(CodecRaw, CodeSha256, 32)
if got := Rank(realDigest); got != RankDigest {
t.Errorf("Rank(sha2-256) regressed to %d, want %d", got, RankDigest)
}
}
func TestBetter(t *testing.T) {
dagpb := buildCID(CodecDagPB, CodeSha256, 32)
// dag-pb (IPFS) beats a sha2-256 digest, regardless of arg order.
if got := Better(dagpb, cidSha256); got != dagpb {
t.Errorf("Better(dagpb, sha256) = %q, want dagpb", got)
}
if got := Better(cidSha256, dagpb); got != dagpb {
t.Errorf("Better(sha256, dagpb) = %q, want dagpb (order shouldn't matter)", got)
}
// Tie → lexicographically smaller wins, deterministic.
a, b := "bafyaaa", "bafyaab"
if Better(a, b) != a || Better(b, a) != a {
t.Errorf("tie-break not deterministic for %q vs %q", a, b)
}
}
+79 -35
View File
@@ -17,50 +17,64 @@ type Config struct {
ServiceVersion string // Service version (default: "1.0.0")
APIPort int // Service HTTP port (for leader info)
BaseURL string // Base URL for stable service discovery
PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
// Redis configuration
RedisPort int // Redis port (default: 6379)
// HTTP API configuration
HTTPPort int // HTTP API port (default: 9000)
HTTPHost string // HTTP API host (default: "127.0.0.1")
HTTPHost string // HTTP API host (default: "0.0.0.0" — meta-core always runs in a container and must answer siblings)
// Timing configuration
HealthCheckIntervalMS int // Health check interval in ms (default: 5000)
HeartbeatIntervalMS int // Service heartbeat interval in ms (default: 30000)
StaleThresholdMS int // Stale service threshold in ms (default: 60000)
CleanupIntervalMS int // Dead service cleanup interval in ms (default: 600000)
DeadServiceThresholdMS int // Dead service threshold in ms (default: 600000)
HealthCheckIntervalMS int // Health check interval in ms (default: 5000)
HeartbeatIntervalMS int // Service heartbeat interval in ms (default: 30000)
StaleThresholdMS int // Stale service threshold in ms (default: 60000)
CleanupIntervalMS int // Dead service cleanup interval in ms (default: 600000)
DeadServiceThresholdMS int // Dead service threshold in ms (default: 600000)
// File watcher configuration
WatchIntervalMS int // Polling interval for network mounts (default: 1000)
DebounceMS int // File change debounce time (default: 30000)
EnableFileWatcher bool // Enable file watcher (default: true)
WatchIntervalMS int // Polling interval for network mounts (default: 1000)
DebounceMS int // File change debounce time (default: 30000)
EnableFileWatcher bool // Enable file watcher (default: true)
// Mount configuration
MountsDir string // Path to mounts configuration (default: /meta-core/mounts)
// UDP discovery (beacon v2). Replaces the file-based registry that used
// to live under /meta-core/services; see
// docs/project-architecture/beacon-v2.md.
EnableUDPDiscovery bool // Advertise + listen on the multicast group (default: true)
DiscoveryGroup string // BEACON_GROUP: IPv4 multicast group (default: 239.255.99.1)
DiscoveryPort int // BEACON_PORT: UDP port (default: 9099)
DiscoveryIntervalMS int // BEACON_INTERVAL_MS: unsolicited advertise interval (default: 10000)
}
// Load creates a Config from environment variables
func Load() *Config {
cfg := &Config{
MetaCorePath: getEnv("META_CORE_PATH", "/meta-core"),
FilesPath: getEnv("FILES_PATH", "/files"),
ServiceName: getEnv("SERVICE_NAME", "meta-core"),
ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"),
APIPort: getEnvInt("API_PORT", 8180),
BaseURL: getEnv("BASE_URL", ""),
RedisPort: getEnvInt("REDIS_PORT", 6379),
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
HTTPHost: getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
MetaCorePath: getEnv("META_CORE_PATH", "/meta-core"),
FilesPath: getEnv("FILES_PATH", "/files"),
ServiceName: getEnv("SERVICE_NAME", "meta-core"),
ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"),
APIPort: getEnvInt("API_PORT", 8180),
BaseURL: getEnv("BASE_URL", ""),
PublicURL: getEnv("META_CORE_PUBLIC_URL", ""),
RedisPort: getEnvInt("REDIS_PORT", 6379),
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
HTTPHost: getEnv("META_CORE_HTTP_HOST", "0.0.0.0"),
HealthCheckIntervalMS: getEnvInt("HEALTH_CHECK_INTERVAL_MS", 5000),
HeartbeatIntervalMS: getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
StaleThresholdMS: getEnvInt("STALE_THRESHOLD_MS", 60000),
CleanupIntervalMS: getEnvInt("CLEANUP_INTERVAL_MS", 600000),
HeartbeatIntervalMS: getEnvInt("HEARTBEAT_INTERVAL_MS", 30000),
StaleThresholdMS: getEnvInt("STALE_THRESHOLD_MS", 60000),
CleanupIntervalMS: getEnvInt("CLEANUP_INTERVAL_MS", 600000),
DeadServiceThresholdMS: getEnvInt("DEAD_SERVICE_THRESHOLD_MS", 600000),
WatchIntervalMS: getEnvInt("WATCH_INTERVAL_MS", 1000),
DebounceMS: getEnvInt("DEBOUNCE_MS", 30000),
EnableFileWatcher: getEnvBool("ENABLE_FILE_WATCHER", true),
WatchIntervalMS: getEnvInt("WATCH_INTERVAL_MS", 1000),
DebounceMS: getEnvInt("DEBOUNCE_MS", 30000),
EnableFileWatcher: getEnvBool("ENABLE_FILE_WATCHER", true),
EnableUDPDiscovery: getEnvBool("ENABLE_UDP_DISCOVERY", true),
DiscoveryGroup: getEnv("BEACON_GROUP", "239.255.99.1"),
DiscoveryPort: getEnvInt("BEACON_PORT", 9099),
DiscoveryIntervalMS: getEnvInt("BEACON_INTERVAL_MS", 10000),
}
// Set mounts directory
@@ -69,26 +83,18 @@ func Load() *Config {
return cfg
}
// LockFilePath returns the path to the leader lock file
// LockFilePath returns the path to the Redis-volume mutex. Held by
// docker/leader-election.sh; nothing reads it, and it is NOT service
// discovery — see docs/project-architecture/beacon-v2.md.
func (c *Config) LockFilePath() string {
return c.MetaCorePath + "/locks/kv-leader.lock"
}
// InfoFilePath returns the path to the leader info file
func (c *Config) InfoFilePath() string {
return c.MetaCorePath + "/locks/kv-leader.info"
}
// RedisDataDir returns the path to Redis data directory
func (c *Config) RedisDataDir() string {
return c.MetaCorePath + "/db/redis"
}
// ServicesDir returns the path to services directory
func (c *Config) ServicesDir() string {
return c.MetaCorePath + "/services"
}
// MountsFilePath returns the path to the mounts configuration file
func (c *Config) MountsFilePath() string {
return c.MountsDir + "/mounts.json"
@@ -104,6 +110,27 @@ func (c *Config) WatchersFilePath() string {
return c.MetaCorePath + "/watchers.json"
}
// IdentityDir returns the directory holding the User Data Layer signing
// identity (created by the dashboard's Identity menu — never auto-generated
// at startup).
func (c *Config) IdentityDir() string {
return c.MetaCorePath + "/identity"
}
// IdentityFilePath returns the path to the legacy single-identity JSON. It is
// only read once, by the multi-account migration on startup; new accounts live
// under IdentityAccountsDir.
func (c *Config) IdentityFilePath() string {
return c.IdentityDir() + "/identity.json"
}
// IdentityAccountsDir returns the directory holding one JSON file per account
// in the multi-account keystore ({uid}.json). meta-core is a pure multi-key
// signer — display name / avatar live in the User Data Layer, not here.
func (c *Config) IdentityAccountsDir() string {
return c.IdentityDir() + "/accounts"
}
func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" {
return value
@@ -132,3 +159,20 @@ func getEnvBool(key string, defaultValue bool) bool {
func (c *Config) DefaultWatchPath() string {
return c.FilesPath + "/watch"
}
// DefaultWatcherPaths returns every directory that should be watched out of
// the box on a fresh install. The watcher seeds these on first start.
//
// - /files/watch — user-curated media. The main library.
// - /files/plugin — plugin output (TMDB posters/backdrops, extracted
// subtitles, etc.). Indexed so each artefact gets a
// midhash256 alias and CID-based references from
// other files (e.g. `backdrop=<cid>`) resolve through
// the standard reverse index instead of needing a
// path companion field.
func (c *Config) DefaultWatcherPaths() []string {
return []string{
c.FilesPath + "/watch",
c.FilesPath + "/plugin",
}
}
-196
View File
@@ -1,196 +0,0 @@
package discovery
import (
"encoding/json"
"log"
"os"
"path/filepath"
"sync"
"time"
"github.com/metazla/meta-core/internal/config"
)
// CleanupStats holds statistics about the cleanup process
type CleanupStats struct {
LastRunTime string `json:"lastRunTime,omitempty"`
ServicesRemoved int `json:"servicesRemoved"`
TotalRuns int `json:"totalRuns"`
Errors int `json:"errors"`
}
// Cleaner handles periodic cleanup of dead service registration files
type Cleaner struct {
config *config.Config
servicesDir string
ownFile string // Our own service file (never delete)
stats CleanupStats
stopChan chan struct{}
wg sync.WaitGroup
mu sync.RWMutex
}
// NewCleaner creates a new service cleaner
func NewCleaner(cfg *config.Config) *Cleaner {
hostname, _ := os.Hostname()
return &Cleaner{
config: cfg,
servicesDir: cfg.ServicesDir(),
ownFile: filepath.Join(cfg.ServicesDir(), cfg.ServiceName+"-"+hostname+".json"),
stopChan: make(chan struct{}),
}
}
// Start begins the cleanup loop
func (c *Cleaner) Start() error {
log.Printf("[Cleaner] Starting dead service cleanup (interval: %dms, threshold: %dms)",
c.config.CleanupIntervalMS, c.config.DeadServiceThresholdMS)
c.wg.Add(1)
go c.cleanupLoop()
return nil
}
// Stop stops the cleanup loop
func (c *Cleaner) Stop() error {
log.Println("[Cleaner] Stopping dead service cleanup...")
close(c.stopChan)
c.wg.Wait()
return nil
}
// cleanupLoop runs the periodic cleanup
func (c *Cleaner) cleanupLoop() {
defer c.wg.Done()
// Run an initial cleanup shortly after starting
initialDelay := time.NewTimer(30 * time.Second)
select {
case <-c.stopChan:
initialDelay.Stop()
return
case <-initialDelay.C:
c.runCleanup()
}
ticker := time.NewTicker(time.Duration(c.config.CleanupIntervalMS) * time.Millisecond)
defer ticker.Stop()
for {
select {
case <-c.stopChan:
return
case <-ticker.C:
c.runCleanup()
}
}
}
// runCleanup scans the services directory and removes dead services
func (c *Cleaner) runCleanup() {
c.mu.Lock()
c.stats.TotalRuns++
c.stats.LastRunTime = time.Now().UTC().Format(time.RFC3339)
c.mu.Unlock()
entries, err := os.ReadDir(c.servicesDir)
if err != nil {
if !os.IsNotExist(err) {
log.Printf("[Cleaner] Failed to read services directory: %v", err)
c.mu.Lock()
c.stats.Errors++
c.mu.Unlock()
}
return
}
threshold := time.Duration(c.config.DeadServiceThresholdMS) * time.Millisecond
removedCount := 0
for _, entry := range entries {
if entry.IsDir() || filepath.Ext(entry.Name()) != ".json" {
continue
}
filePath := filepath.Join(c.servicesDir, entry.Name())
// Never delete our own service file
if filePath == c.ownFile {
continue
}
// Check if service is dead
isDead, serviceName, age := c.isServiceDead(filePath, threshold)
if isDead {
if err := os.Remove(filePath); err != nil {
log.Printf("[Cleaner] Failed to remove dead service file %s: %v", entry.Name(), err)
c.mu.Lock()
c.stats.Errors++
c.mu.Unlock()
} else {
ageMinutes := age.Minutes()
log.Printf("[Cleaner] Removed dead service: %s (last heartbeat: %.1f minutes ago)", serviceName, ageMinutes)
removedCount++
}
}
}
if removedCount > 0 {
c.mu.Lock()
c.stats.ServicesRemoved += removedCount
c.mu.Unlock()
log.Printf("[Cleaner] Cleanup complete: removed %d dead service(s)", removedCount)
}
}
// isServiceDead checks if a service file represents a dead service
// Returns: isDead, serviceName, age since last heartbeat
func (c *Cleaner) isServiceDead(filePath string, threshold time.Duration) (bool, string, time.Duration) {
data, err := os.ReadFile(filePath)
if err != nil {
// Can't read file - treat as dead to clean up corrupted files
log.Printf("[Cleaner] Cannot read service file %s: %v (treating as dead)", filepath.Base(filePath), err)
return true, filepath.Base(filePath), threshold
}
var info ServiceInfo
if err := json.Unmarshal(data, &info); err != nil {
// Corrupted JSON - skip rather than delete (might be recoverable)
log.Printf("[Cleaner] Skipping corrupted service file %s: %v", filepath.Base(filePath), err)
return false, "", 0
}
// Parse last heartbeat timestamp
if info.LastHeartbeat == "" {
// No heartbeat timestamp - treat as dead
return true, info.Name, threshold
}
lastHeartbeat, err := time.Parse(time.RFC3339, info.LastHeartbeat)
if err != nil {
// Invalid timestamp - treat as dead
log.Printf("[Cleaner] Invalid timestamp in service file %s: %v (treating as dead)", filepath.Base(filePath), err)
return true, info.Name, threshold
}
age := time.Since(lastHeartbeat)
if age > threshold {
return true, info.Name, age
}
return false, info.Name, age
}
// Stats returns a copy of the current cleanup statistics
func (c *Cleaner) Stats() CleanupStats {
c.mu.RLock()
defer c.mu.RUnlock()
return CleanupStats{
LastRunTime: c.stats.LastRunTime,
ServicesRemoved: c.stats.ServicesRemoved,
TotalRuns: c.stats.TotalRuns,
Errors: c.stats.Errors,
}
}

Some files were not shown because too many files have changed in this diff Show More