Compare commits

...
52 Commits
Author SHA1 Message Date
Worph f367ed037f Report inbox: /api/reports in its own bounded reports.db, dashboard Reports page (list, filter, triage, copy as text)
Build and Push Docker Image / build (push) Failing after 1m20s
2026-11-04 11:43:01 +00:00
Worph cef9dba59e migrate
Build and Push Docker Image / build (push) Failing after 2m28s
2026-11-04 11:55:00 +01:00
Worph 9ccaff90d8 CI: publish a Gitea release naming the IPCR image and digest on v* tags
Build and Push Docker Image / build (push) Successful in 1m7s
2026-10-25 19:02:36 +00:00
Worph 84af298792 v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410
Build and Push Docker Image / build (push) Successful in 5m1s
2026-10-25 17:36:00 +00:00
Manielle b0adefb97b CI accepts four-part tags (v1.2.3.1) for fixes
Build and Push Docker Image / build (push) Successful in 4m20s
2026-10-25 14:22:01 +01:00
Worph 5369dc4b9c docs: search plan + counters for v1.2.0; tools/search-parity (parity + latency vs a brute-force reference)
Build and Push Docker Image / build (push) Successful in 2m50s
2026-10-19 21:59:36 +00:00
Worph fb4bdad61b v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver
Build and Push Docker Image / build (push) Successful in 7m38s
2026-10-19 21:41:55 +00:00
Worph dc5cbdfb9b replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
2026-10-19 20:51:00 +00:00
Worph c53146e514 rank vectors: provider-file-tachiyomi
Build and Push Docker Image / build (push) Successful in 2m28s
2026-10-19 18:45:19 +00:00
Worph bb04a3f4dc release (first build on gitea-watch / IPCR)
Build and Push Docker Image / build (push) Successful in 1m0s
2026-10-13 11:02:23 +00:00
Worph 36eb3543b9 gitea workflow: push to IPCR's direct host (Cloudflare 100 MB cap) [skip ci] 2026-10-13 10:02:01 +00:00
Worph 155fd83392 gitea workflow: pull path via metadec.eth
Build and Push Docker Image / build (push) Successful in 56s
2026-10-12 18:30:11 +00:00
Worph cf63a8da68 gitea workflow: publish to IPCR (metadec.eth) instead of ghcr
Build and Push Docker Image / build (push) Successful in 55s
2026-10-12 18:26:08 +00:00
Worph 27ed45999b beacon v2 discovery: advertise metamesh.core + service resources on 239.255.99.1:9099, /api/neighbors ?cap= filter
Build and Push Docker Image / build (push) Failing after 14s
2026-10-04 11:51:03 +00:00
Worph 9254f043a4 README: sync with current code 2026-10-03 11:51:55 +00:00
Worph ebc894fa42 deleting a cids/ member also drops its reverse alias
Build and Push Docker Image / build (push) Failing after 14s
2026-09-16 17:10:00 +00:00
Worph 515ec04369 bugfix and dashboard improvement
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 19:04:23 +02:00
Worph 0403a6f3bc replace file discovery by udp discovery
Build and Push Docker Image / build (push) Failing after 13s
2026-09-14 11:24:46 +02:00
Worph 355c666183 chnage service discovery and identity 2026-09-13 19:48:52 +02:00
Worph 0cccceccd9 align with metadata standard
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 15:57:30 +02:00
Worph 40b553016c cid vector
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 14:36:50 +02:00
Worph 4894810fd4 new file tuple API
Build and Push Docker Image / build (push) Failing after 13s
2026-09-13 12:01:33 +02:00
Worph ac21315255 improved cid resolution
Build and Push Docker Image / build (push) Failing after 13s
2026-09-12 21:37:38 +02:00
Worph ec1b227631 improved search 2026-09-12 16:37:51 +02:00
Worph 8f4e08b6f1 rework domain field
Build and Push Docker Image / build (push) Failing after 13s
2026-09-02 00:07:02 +02:00
Worph 0c4e3a2227 align to metadata keys doc
Build and Push Docker Image / build (push) Failing after 13s
2026-08-30 19:52:36 +02:00
Worph 1f41ea0166 bugfix scroll
Build and Push Docker Image / build (push) Failing after 14s
2026-08-25 18:09:44 +02:00
Worph 5efdd2c07d improved identity management 2026-08-25 17:48:23 +02:00
Worph 6ae87a2cdb audio support widening 2026-08-21 18:23:20 +02:00
Worph 604e718cdc add identity management
Build and Push Docker Image / build (push) Failing after 13s
2026-08-06 11:03:50 +02:00
Worph 0bbc656263 add identity page
Build and Push Docker Image / build (push) Failing after 13s
2026-08-05 23:37:12 +02:00
Worph a98734171c bugfix indexer field
Build and Push Docker Image / build (push) Failing after 13s
2026-08-03 19:44:22 +02:00
Worph a61057cd81 optimisation on assets retreive 2026-08-03 18:42:05 +02:00
Worph 55d08e77ed improvement on user management
Build and Push Docker Image / build (push) Failing after 14s
2026-08-02 21:35:39 +02:00
Worph 1f084b817c improve identity handling
Build and Push Docker Image / build (push) Failing after 14s
2026-08-01 23:31:49 +02:00
Worph 80050cd2a8 update vector with card locator
Build and Push Docker Image / build (push) Failing after 13s
2026-07-26 18:49:27 +02:00
Worph 75dda1e7d6 new NZB cid support
Build and Push Docker Image / build (push) Failing after 13s
2026-07-25 21:30:46 +02:00
Worph 76169920d6 improve meta core user management
Build and Push Docker Image / build (push) Failing after 13s
2026-07-18 17:30:17 +02:00
Worph df6a2d78f8 bug fix vector
Build and Push Docker Image / build (push) Failing after 14s
2026-07-14 16:02:26 +02:00
Worph 0ca03ac95a deprecate old cids fields 2026-07-13 22:02:29 +02:00
Worph 1798821c53 update metacore default 2026-07-12 14:47:21 +02:00
Worph d268bb83e6 search optimisation
Build and Push Docker Image / build (push) Failing after 13s
2026-07-10 22:56:59 +02:00
Worph 075d6f9bcb cleanup logs 2026-07-10 13:25:16 +02:00
Worph f1cd94b17e bugfix advertise url
Build and Push Docker Image / build (push) Failing after 13s
2026-06-14 18:47:28 +02:00
Pierre 52f5b5cedc multi identity support for metacore 2026-06-06 18:44:52 +02:00
Pierre f8a10091b8 remove in metadata cid_* to put in the cids/ set 2026-05-31 11:41:32 +02:00
Pierre 94ad0c240b bugfix and new internal API 2026-05-25 23:47:30 +02:00
Pierre 415595c1c6 new meta-core api, now redis is not exposed as an "API" 2026-05-24 17:43:17 +02:00
Pierre aa45f4f32e improve editor and add type detection 2026-05-14 23:31:07 +02:00
Pierre 3d65e7e951 doc improvement 2026-05-10 19:47:05 +02:00
Pierre bd833b7be4 improved kv metacore 2026-04-27 10:31:11 +02:00
Pierre 98da535335 improve editor for KV 2026-04-26 15:28:50 +02:00
172 changed files with 28082 additions and 8894 deletions
+6
View File
@@ -0,0 +1,6 @@
# FIX CRLF always when developer has not set
# Linux/Mac: git config --global core.autocrlf input
# Windows: git config --global core.autocrlf true
# Auto detect text files and perform LF normalization
* text=auto
* eol=lf
+129
View File
@@ -0,0 +1,129 @@
# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
# folder exists, and GitHub never reads it.
#
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
name: Build and Push Docker Image
on:
push:
branches:
- main
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Tag for the image (optional, defaults to sha)'
required: false
type: string
env:
REGISTRY: ipcr-push-85-17-246-67.nip.io
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: write # the release step below
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Docker CLI
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
- name: Trust the Mesh Router CA
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
env:
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
buildkitd-config-inline: |
[registry."${{ env.REGISTRY }}"]
ca = ["/tmp/ipcr-ca.pem"]
- name: Log in to IPCR
env:
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
run: |
mkdir -p ~/.docker
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
- name: Build and push Docker image
id: build
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
# Gitea's Packages tab only lists images in Gitea's own registry, so a v* tag gets a
# release instead, naming the image it shipped. The digest line is the one to trust:
# ipcr 1.7.1 can drop some of the tags pushed together (see the tag list).
- name: Publish the Gitea release
if: startsWith(github.ref, 'refs/tags/v')
env:
TOKEN: ${{ secrets.GITEA_TOKEN }}
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
TAG: ${{ github.ref_name }}
SHA: ${{ github.sha }}
DIGEST: ${{ steps.build.outputs.digest }}
TAGS: ${{ steps.meta.outputs.tags }}
PULL: ipcr-watch.nsl.sh/ipns/metadec.eth/${{ env.IMAGE_NAME }}
run: |
node -e '
const e = process.env, ver = e.TAG.replace(/^v/, "");
const tags = e.TAGS.split("\n").filter(Boolean).map(t => "`" + t.split(":").pop() + "`");
const body = [
"**Image** (anonymous pull):",
"```", `docker pull ${e.PULL}:${ver}`, "```",
"Pinned by digest:",
"```", `${e.PULL}@${e.DIGEST}`, "```",
`Tags pushed: ${tags.join(", ")} · commit ${e.SHA.slice(0, 7)}`,
].join("\n");
console.log(JSON.stringify({ tag_name: e.TAG, name: e.TAG, body }));
' > /tmp/release.json
auth="Authorization: token $TOKEN"
code=$(curl -s -o /tmp/existing.json -w '%{http_code}' -H "$auth" "$API/releases/tags/$TAG")
if [ "$code" = 200 ]; then
id=$(node -p 'require("/tmp/existing.json").id')
method=PATCH url="$API/releases/$id"
else
method=POST url="$API/releases"
fi
code=$(curl -s -o /tmp/out.json -w '%{http_code}' -X "$method" -H "$auth" \
-H 'Content-Type: application/json' --data @/tmp/release.json "$url")
case "$code" in 2*) echo "release $TAG: $method $code" ;; *) cat /tmp/out.json; exit 1 ;; esac
+3
View File
@@ -45,6 +45,9 @@ jobs:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}}
+2
View File
@@ -23,3 +23,5 @@ go.work
dashboard/node_modules/*
dist/*
dashboard/dist/*
editor/dist/
editor/node_modules/
+28 -18
View File
@@ -6,20 +6,21 @@
# - Bash-based leader election script (flock on shared volume)
# - meta-core Go binary (HTTP API, Mount API, Watcher)
# - nginx (dashboard, WebDAV, API proxy)
# - Redis (managed by supervisord on leader)
# - rclone (remote mount daemon)
# - SQLite metadata store, embedded in the Go binary: truth /meta-core/db/meta-core.db
# (signed claims, backed up) + cache /meta-core/db-cache/cache.db (rebuildable)
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
# - supervisord (process management - runs on leader only)
#
# Architecture:
# - leader-election.sh handles flock, starts supervisord on leader
# - Followers remain dormant, retry flock every 5s
# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
# - Only leader runs meta-core, nginx, rclone, mount-watcher
# =============================================================================
# -----------------------------------------------------------------------------
# Stage 1: Build Go binary
# -----------------------------------------------------------------------------
FROM golang:1.21-alpine AS go-builder
FROM golang:1.26-alpine AS go-builder
RUN apk add --no-cache git
@@ -67,27 +68,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
LABEL org.opencontainers.image.licenses=MIT
# Install runtime dependencies
# Pinned rclone release. Alpine's package can lag the latest by several
# minor versions and we depend on VFS cache fixes shipped post-1.65.
ARG RCLONE_VERSION=1.73.5
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
# all remote mounts go through rclone now.
RUN apk add --no-cache \
# Core services
redis \
sqlite \
nginx \
nginx-mod-http-dav-ext \
supervisor \
rclone \
fuse3 \
# Mount utilities
nfs-utils \
cifs-utils \
# Tools
bash \
curl \
python3 \
ca-certificates \
apache2-utils \
# Process utilities
procps
# Install rclone from the official binary release (pinned).
RUN set -eux; \
apk add --no-cache --virtual .rclone-deps unzip; \
cd /tmp; \
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
unzip -q rclone.zip; \
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
apk del .rclone-deps; \
/usr/local/bin/rclone version | head -2
# Copy Go binary from builder
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
RUN chmod +x /usr/local/bin/meta-core
@@ -95,7 +105,8 @@ RUN chmod +x /usr/local/bin/meta-core
# Create directories
RUN mkdir -p \
/meta-core/locks \
/meta-core/db/redis \
/meta-core/db \
/meta-core/db-cache \
/meta-core/services \
/meta-core/mounts/errors \
/files \
@@ -131,13 +142,12 @@ ENV META_CORE_PATH=/meta-core \
SERVICE_VERSION=1.0.0 \
META_CORE_HTTP_PORT=9000 \
META_CORE_HTTP_HOST=0.0.0.0 \
REDIS_PORT=6379 \
WATCH_INTERVAL_MS=1000
# Expose ports
# 80: nginx (dashboard + API proxy + WebDAV)
# 6379: Redis (for other services)
EXPOSE 80 6379
# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port:
# it is a file inside this container, reachable only through the HTTP API.
EXPOSE 80
# Health check
HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
+15 -1
View File
@@ -2,7 +2,7 @@
# Variables
BINARY := meta-core
VERSION ?= 1.0.0
VERSION ?= 2.0.0
BUILD_DIR := bin
DOCKER_IMAGE := meta-core
@@ -64,10 +64,24 @@ clean:
rm -f coverage.out coverage.html
# Build Docker image
#
# We tag THREE names:
# - meta-core:$(VERSION) — the canonical versioned tag
# - meta-core:latest — convenience for `docker run meta-core`
# - meta-core:local — the tag every sub-stack docker-compose.yml uses
# (meta-share federation, meta-gateway dev stacks).
# Without this, `make docker` produces an image
# the dev stacks can't see by name, and previously
# -tagged stale `:local` images linger, silently
# breaking downstream API fields (e.g. /urls
# missing webdavUrlInternal → meta-gateway's
# poster auto-store fails). See README.md →
# Quick start → Build for the audit recipe.
docker:
@echo "Building Docker image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) .
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local
# Run locally (for development)
run: build
+420 -454
View File
@@ -1,520 +1,486 @@
# meta-core
A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services.
Standalone Go service that owns the metadata store (an embedded SQLite file),
the HTTP+SSE metadata surface, the
User Data Layer store and signing keystore, and a WebDAV file server for the
MetaMesh dev/prod stack.
## Overview
meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul.
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
its `metamesh.core` advertise carries the full `/urls` payload) and then call its
HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded
in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API
(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
### Key Capabilities
### Key capabilities
| Feature | Description |
|---------|-------------|
| **Leader Election** | flock-based leader election (Go binary only runs as leader) |
| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash |
| **HTTP API** | Language-agnostic REST interface for metadata and service discovery |
| **Service Discovery** | File-based registry with heartbeats and stale detection |
| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations |
| **File Watching** | Directory scanning, MidHash256 computation, event dispatch |
| **Mount Management** | SMB/rclone configuration, health monitoring |
| **WebDAV Server** | File access with nginx proxy_cache |
| **Event Publishing** | Redis keyspace notifications to meta:events stream |
| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. |
| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). |
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field_v` over (field name, value) of every record key, plus an FTS5 trigram table; the smallest part drives, the other filters are probed per candidate — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). |
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. |
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
| **Report inbox** | `/api/reports` stores the bug reports / feedback the client apps' 🐛 buttons file (forwarded server-to-server by meta-watch and meta-read), in its own bounded `db/reports.db`; the dashboard's **Reports** page lists, reads and triages them. See [architecture: Report inbox](docs/meta-core-architecture.md#report-inbox). |
| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
### Design Characteristics
### Design characteristics
- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes)
- **Automatic failover** - Lock releases on process death, followers detect and re-elect
- **Thread-safe** - RWMutex protection on all shared state
- **Graceful shutdown** - Signal handling with reverse startup order
- **Static binary** - CGO disabled, single binary deployment
- **Single writer** — flock on the `/meta-core` volume; no external consensus.
- **HTTP-only externally** — there is no database port; debug read-only with
`docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …`
(dev stacks: `/meta-core-db/meta-core.db`).
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
on-disk store layout predates the current build (alpha clean-wipe policy;
see `docs/uuid-rooted-metadata.md`).
- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a
thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI
(for debugging).
## Architecture
meta-core runs as a **standalone container** that other services connect to via HTTP API:
```
┌───────────────────────────────────────────────────────────────────────┐
│ meta-core container (standalone) │
┌──────────────────────────────────────────────────────────────────────────┐
│ metacore-app (standalone container) │
│ │
│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │
│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │
│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │
│ │ │ │ │ │ │ │ /health, /services │ │
│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │
│ │
│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │
└──────────────────────────────────────────────┬────────────────────────┘
│ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
│ then exec supervisord) │
│ supervisord │
│ ├─ nginx :80 (dashboard, /editor, API proxy, │
│ │ proxy_cache for /webdav, rclone GUI /rclone) │
│ ├─ rclone rcd (mount daemon, RC API on :5572) │
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
│ ─► embedded SQLite (db/meta-core.db) │
│ ─► UDP advertise/listen 239.255.99.1:9099 │
└──────────────────────────────────────────────────────────────────────────┘
│
┌──────────────────────────────────────┴──────────────────────────┐
│ /meta-core (shared volume) │
│ ├── locks/kv-leader.info (leader URLs) │
│ ├── db/redis/ (Redis data) │
│ ├── services/ (hostname-based .json) │
│ └── mounts/ (mount configurations) │
└─────────────────────────────────────────────────────────────────┘
▲ ▲ ▲
┌────────────┘ │ └────────────┐
│ │ │
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
│ meta-sort │ │ meta-fuse │ │ meta-stremio │
│ (reads info, │ │ (reads info, │ │ (reads info, │
│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │
└───────────────┘ └───────────────┘ └───────────────┘
┌──────────────────────┴──────────────────────┐
│ /meta-core (meta-core's own volume) │
│ locks/kv-leader.lock │
│ db/meta-core.db (+ -wal, -shm) │
│ identity/accounts/*.json │
│ mounts/{mounts.json, errors/} │
│ watchers.json │
│ cache/ (nginx proxy_cache) │
└─────────────────────────────────────────────┘
meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
│ 1. hear the metamesh.core advertise on UDP multicast (carries /urls)
└─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume
```
## Quick Start
## Quick start
### Build
```bash
# Build binary
make build
# Build Docker image
make docker
# Run tests
make test
make build # build static binary to bin/meta-core
make docker # build container image as meta-core:1.0.0 + :latest + :local
make test # run Go unit tests
```
### Run
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
`:local` tag is the one the sub-stack docker-compose files reference
(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
sub-stacks can't see a freshly-built image by name.
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
> image, but **already-running containers stay on the old image SHA** —
> `docker compose up -d` won't recreate them unless the compose file
> itself changed. After a meta-core source change, force-recreate the
> downstream containers so they pick up the new binary:
>
> ```bash
> make docker
> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
> up -d --force-recreate metagateway-hub-metacore
> docker compose -f packages/meta-share/dev/docker-compose.yml \
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
> docker compose -f packages/meta-read/dev/docker-compose.yml \
> up -d --force-recreate metaread-dev-core
> docker compose -f packages/meta-listen/dev/docker-compose.yml \
> up -d --force-recreate metalisten-dev-core
> ```
>
> The symptom of a missed recreate is missing API fields on `/urls`
> (e.g. `webdavUrlInternal` absent → meta-gateway's
> `store_poster_for_record` silently fails and torznab posters never
> land). Audit recipe:
>
> ```bash
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}' # repeat for :latest
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
> ```
### Dev container
In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`:
```bash
# With environment variables
META_CORE_PATH=/meta-core \
FILES_PATH=/files \
SERVICE_NAME=meta-sort \
META_CORE_HTTP_PORT=9000 \
./bin/meta-core
cd dev && ./scripts/start.sh
```
### Docker
```bash
docker build -t meta-core .
docker run -v meta-core:/meta-core -v files:/files meta-core
```
The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend
HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
## Configuration
| Environment Variable | Default | Description |
|---------------------|---------|-------------|
| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db |
| `FILES_PATH` | `/files` | Shared volume for media files |
| `SERVICE_NAME` | `meta-core` | Service identifier |
| `SERVICE_VERSION` | `1.0.0` | Service version |
| `API_PORT` | `8180` | Main service HTTP port |
| `BASE_URL` | - | Stable service URL |
| `REDIS_PORT` | `6379` | Redis port (leader only) |
| `META_CORE_HTTP_PORT` | `9000` | HTTP API port |
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval |
| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold |
| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning |
Environment variables (from `internal/config/config.go` unless noted):
## API Reference
| Variable | Default | Description |
|---|---|---|
| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. |
| `FILES_PATH` | `/files` | Files volume root. |
| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). |
| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). |
| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
| `META_CORE_REPORTS_DB_PATH` | `<META_CORE_PATH>/db/reports.db` | Report inbox database. |
| `META_CORE_REPORTS_MAX` / `META_CORE_REPORTS_MAX_MB` | `5000` / `256` | Inbox bounds; closed reports are evicted first, then the oldest. |
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
### Health & Status
`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
them (leftovers of the file-based registry); setting them has no effect. There
is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES`
(gone with Redis and the in-memory search index; search results now include
`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` /
`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
configs created via `/api/watchers`.
## API reference
All endpoints below are registered in `internal/api/server.go` (mounts,
watcher and watchers routes in their own packages' `handlers.go`). Examples assume
the in-container Go port `:9000`; from the host use
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
(debug-direct).
### Health, leader, URLs
```bash
# Health check
curl http://localhost:9000/health
# {"status":"ok","redis":true,"timestamp":"..."}
# Detailed status
curl http://localhost:9000/status
# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...}
# Current leader
curl http://localhost:9000/leader
# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...}
GET /health # storage + role (`redis: bool` kept for compatibility = store open)
GET /status # version, uptime, etc.
GET /leader # current leader info (hostname, PID, timestamps)
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility)
GET /neighbors # neighbours heard over UDP (beacon v2)
GET /api/neighbors # same, under /api
GET /api/stats # dashboard counters (see below)
```
### Metadata Operations
`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
are **gone** — the registration-and-heartbeat registry they served was replaced
by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
than a 404; that is what made the dashboard fail with
`SyntaxError: Unexpected token '<'` until 1.0.21.
```bash
# List all file hashes
curl http://localhost:9000/meta
# {"hashIds":["midhash256:abc123",...],"count":42}
#### `GET /api/stats`
# Get metadata for a file
curl http://localhost:9000/meta/{hash}
# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}}
Every headline counter in one request, split by cost:
# Update metadata
curl -X PUT http://localhost:9000/meta/{hash} \
-H "Content-Type: application/json" \
-d '{"title":"New Title"}'
# Delete metadata
curl -X DELETE http://localhost:9000/meta/{hash}
```
### Data Operations
```bash
# Get file path
curl http://localhost:9000/data/{hash}/path
# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true}
# Check if file exists
curl -I http://localhost:9000/data/{hash}
# HTTP/1.1 200 OK (or 404)
```
### File Operations (by CID)
```bash
# Serve file by CID (looks up poster/backdrop CIDs in metadata)
curl http://localhost:9000/file/{cid} --output poster.jpg
# Returns raw file bytes with appropriate Content-Type header
# Example: fetch a poster image
curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4
# Content-Type: image/jpeg
# [binary image data]
```
The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval.
### Service Discovery
```bash
# List all services
curl http://localhost:9000/services
# {"services":[{"name":"meta-sort",...}],"count":3}
# Get specific service
curl http://localhost:9000/services/meta-sort
# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...}
```
### Mount Management
```bash
# List all mounts
curl http://localhost:9000/api/mounts/list
# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]}
# Add a new mount
curl -X POST http://localhost:9000/api/mounts/add \
-H "Content-Type: application/json" \
-d '{"type":"smb","path":"/files/corn/nas","config":{...}}'
# Remove a mount
curl -X DELETE http://localhost:9000/api/mounts/smb-nas
```
### File Watching
```bash
# Get scan status
curl http://localhost:9000/api/scan/status
# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847}
# Trigger rescan
curl -X POST http://localhost:9000/api/scan/trigger
# Stream file events (SSE)
curl http://localhost:9000/api/events/stream
# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."}
```
### KV Browser
```bash
# Storage statistics
curl http://localhost:9000/api/kv/info
# {"totalKeys":50000,"memoryUsed":"256MB",...}
# List keys with cursor pagination
curl "http://localhost:9000/api/kv/keys?cursor=0&count=100"
# Get specific key value
curl http://localhost:9000/api/kv/key/file:bafkr.../title
```
### Metadata Editor
```bash
# Get all hash IDs
curl http://localhost:9000/api/metadata/hash-ids
# {"hashIds":["bafkr...","bafks..."]}
# Paginated metadata listing
curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50"
# Search metadata
curl -X POST http://localhost:9000/api/metadata/search \
-H "Content-Type: application/json" \
-d '{"query":"inception","fields":["title"]}'
# Batch update
curl -X POST http://localhost:9000/api/metadata/batch \
-H "Content-Type: application/json" \
-d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}'
# Clear all metadata
curl -X POST http://localhost:9000/api/metadata/clear
```
### WebDAV Access
```bash
# List directory (returns JSON)
curl http://localhost:9000/webdav/
# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}]
# Get file
curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv
# Upload file
curl -X PUT http://localhost:9000/webdav/plugin/output.txt \
-H "Content-Type: application/octet-stream" \
--data-binary @output.txt
# Create directory
curl -X MKCOL http://localhost:9000/webdav/newdir/
# Delete file/directory
curl -X DELETE http://localhost:9000/webdav/oldfile.txt
```
## Leader Election
meta-core uses POSIX file locking (flock) for distributed leader election:
1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock`
2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info`
3. Losers become **followers**, read leader info, connect to leader's Redis
4. Lock automatically releases when process dies (no stale locks)
### Election Flow
```
Instance 1 Instance 2 Instance 3
│ │ │
├── Try flock ──────────┼───────────────────────┤
│ │ │
[WIN] [LOSE] [LOSE]
LEADER FOLLOWER FOLLOWER
│ │ │
Start Redis Read leader.info Read leader.info
│ │ │
Write leader.info Connect to Redis Connect to Redis
│ │ │
[Health loop 5s] [Health loop 5s] [Health loop 5s]
- Update timestamp - Re-read leader - Re-read leader
- Check Redis alive - Detect changes - Detect changes
```
### Role Callbacks
The election system provides callbacks for role transitions:
```go
election.OnBecomeLeader(func(info LeaderLockInfo) {
// Start Redis, initialize resources
})
election.OnBecomeFollower(func(info LeaderLockInfo) {
// Connect to leader's Redis
})
election.OnLeaderLost(func() {
// Handle leader failure, attempt re-election
})
```
### Lock Info Format
The `kv-leader.info` file contains a plain text API URL:
```
http://10.0.1.50:9000
```
Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.).
## Redis Management
The leader is responsible for spawning and managing the Redis server.
### Redis Configuration
```bash
redis-server --port 6379 --bind 0.0.0.0 \
--dir /meta-core/db/redis \
--appendonly yes --appendfilename appendonly.aof \
--dbfilename dump.rdb --save 60 1 \
--loglevel warning
```
### Persistence Strategy
| Method | Configuration | Purpose |
|--------|--------------|---------|
| **AOF** | `appendonly yes` | Write-ahead log for durability |
| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed |
### Failure Handling
- **Redis crash**: Auto-detected via health check, automatically restarted
- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs
- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed
## Metadata Storage
meta-core uses a flat key-value schema in Redis for storing file metadata.
### Storage Format
```
Key: /file/{hashId}/{property}
Value: string
Examples:
/file/midhash256:abc123/title → "Movie Title"
/file/midhash256:abc123/year → "2024"
/file/midhash256:abc123/genres → "action,drama"
/file/midhash256:abc123/filePath → "movies/Movie.mkv"
```
### Storage Client Features
| Feature | Details |
|---------|---------|
| Connection Pool | 10 connections |
| Timeouts | 5s dial, 30s read/write |
| Batch Operations | Pipelined writes for performance |
| Key Scanning | SCAN-based iteration (non-blocking) |
| CID Lookup | Find file paths by poster/backdrop CID |
## Service Discovery
Services register by writing JSON to `/meta-core/services/{name}.json`.
### Registration Format
```json
```jsonc
{
"name": "meta-sort",
"version": "2.0.0",
"api": "http://10.0.1.50:8180",
"status": "running",
"pid": 12345,
"hostname": "meta-sort-dev",
"startedAt": "2024-01-01T00:00:00Z",
"lastHeartbeat": "2024-01-01T00:01:00Z",
"capabilities": ["meta-core"],
"endpoints": {
"health": "http://10.0.1.50:9000/health",
"meta": "http://10.0.1.50:9000/meta"
}
"records": 88709, // count of idx:roots/* — metadata roots, file or not
"redisKeys": 3115447, // kv row count (name kept for compatibility; cached 60 s)
"redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility)
"identities": 3, // signing accounts on disk
"files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep
"udlUsers": 5, // accounts holding User Data Layer records
"sweptAt": 1789402401970,
"sweeping": false
}
```
### Heartbeat & Stale Detection
`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and
`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
box), so they are served from a single-flight cache refreshed in the background
with a 60s TTL — the walk is **never** on the request path, because the
dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this
over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
across records that share a file.
| Parameter | Default | Purpose |
|-----------|---------|---------|
| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` |
| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this |
### Metadata — primary surface
Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file.
## Development
These are the routes other services use. `{hash}` may be a UUID root or
any registered CID alias; CID aliases resolve to the underlying UUID
before reads/writes (`internal/storage/cid_resolution.go`).
```bash
# Format code
make fmt
# Run linter
make lint
# Run tests with coverage
make test-cover
# Clean build artifacts
make clean
GET /meta # list of root IDs
GET /meta/{hash} # full document (flat keys → nested JSON)
PUT /meta/{hash} # replace document
PATCH /meta/{hash} # merge into document
DELETE /meta/{hash} # delete root + all CID aliases
GET /meta/{hash}/{key...} # single property (key may contain "/")
PUT /meta/{hash}/{key...} # set single property
DELETE /meta/{hash}/{key...} # delete single property
POST /meta/{hash}/_add/{key...} # add value to set-valued property
```
## Internal Packages
CID-addressed access (public; auth-bypassed in the Caddy perimeter):
```bash
GET /api/meta/{cid} # resolve CID → metadata document
GET /api/file/{cid}/info # CID resolution metadata
GET /file/{cid} # serve file bytes (range-aware)
POST /file/cid # compute CID for an uploaded file
POST /api/files/register # mint + alias one file under /files/plugin by midhash
# (the watcher's path for one file; works with the watcher OFF)
HEAD /data/{hash} # existence + size
GET /data/{hash}/path # absolute path on /files
```
(There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or
the `/file/{cid}` reverse-lookup path.)
### Editor / KV browser surface
```bash
GET /api/metadata/hash-ids
GET /api/files/tuples
GET /api/metadata/list
POST /api/metadata/search
POST /api/metadata/batch
POST /api/metadata/clear
GET /api/metadata/{hashId}
PUT /api/metadata/{hashId}
DELETE /api/metadata/{hashId}
GET /api/metadata/{hashId}/property
PUT /api/metadata/{hashId}/property
GET /api/kv/info
GET /api/kv/keys
GET /api/kv/tree # cursor is an opaque next-key ("0" when exhausted)
GET /api/kv/search
GET /api/kv/find
GET /api/kv/value # type is always "string" for an existing key
PUT /api/kv/value
DELETE /api/kv/value
GET /api/kv/key/{key...}
GET /api/schema
POST /api/schema/rescan
GET /api/snapshot/export
POST /api/snapshot/import
POST /api/snapshot/wipe
```
### SSE event streams
The HTTP-mirror of the store's event streams (Redis Streams until 2026-10;
IDs keep the `<ms>-<seq>` form). External services never read the store
directly — they subscribe here with `Last-Event-ID` for resume. See
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
contract (one event per stream entry, opaque IDs, heartbeats, gap
signalling).
```bash
GET /api/events/files # file:events stream (watcher events)
GET /api/events/meta # meta:events stream (metadata mutations)
GET /api/events/poll # DEPRECATED long-poll over file:events — use SSE
```
### Identity (signing keystore)
Reached through meta-search's identity proxy (and directly by meta-gateway,
meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
```bash
GET /api/identity # status (never the private key)
GET /api/identity/accounts
POST /api/identity/generate # the only call that returns a new private key
POST /api/identity/import
POST /api/identity/challenge # one-shot challenge for proof-of-possession
POST /api/identity/reveal # needs proof-of-possession
DELETE /api/identity # needs proof-of-possession; reports purged UDL counts
POST /api/identity/sign
POST /api/identity/sign-batch
GET /api/identity/aead-key
```
### User Data Layer
Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec:<uid>/<cid>/<key>` → JSON `{record,version,ts,value?}`; index sets as one key per member).
```bash
GET /api/udl/record?uid&cid&key
PUT /api/udl/record
PUT /api/udl/records # batch
GET /api/udl/user/{uid}/key/{key}
GET /api/udl/user/{uid}/cid/{cid}
GET /api/udl/user/{uid}/all
GET /api/udl/user/{uid}/count
GET /api/udl/cid/{cid}/users
GET /api/udl/cid/{cid}/aggregate # meta-watch Aggregate wire shape
GET /api/udl/users/stats
```
### Reports (bug-report / feedback inbox)
```
POST /api/reports # {app, note?, page?, route?, cid?, title?, uid?, build?, filedAt?, log?, …blocks} → 201 {id}
GET /api/reports # ?app=&status=open|triaged|closed&q=&before=<id>&limit= → {reports, apps, counts, more}
GET /api/reports/{id} # full report incl. body
PATCH /api/reports/{id} # {status?, triage_note?}
DELETE /api/reports/{id}
```
### Mounts
```bash
GET /api/mounts
POST /api/mounts
GET /api/mounts/{id}
PUT /api/mounts/{id} # also PATCH
DELETE /api/mounts/{id}
POST /api/mounts/{id}/mount
POST /api/mounts/{id}/unmount
POST /api/mounts/{id}/safe-unmount
POST /api/mounts/{id}/scan
GET /api/mounts/rclone/remotes
```
### Watchers
```bash
GET /api/watchers
POST /api/watchers
GET /api/watchers/{id}
PUT /api/watchers/{id}
DELETE /api/watchers/{id}
POST /api/watchers/{id}/scan
POST /api/watchers/{id}/reset
POST /api/watchers/scan-all
POST /api/watchers/reset-all
```
The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
return a deprecation pointer at the `/api/watchers/*` routes
(`internal/watcher/handlers.go:32-33`).
### WebDAV
```bash
GET/PUT/DELETE /webdav/... # mounted on /files
```
The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`,
`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were
removed with the 2026-10 SQLite switch; they had been run once on
watch.nsl.sh before the migration.
No `/metrics` endpoint is exposed; observability is via container logs and
the `/health` / `/status` JSON.
## DB-file mutex
`docker/leader-election.sh` is a bash flock loop, not a Go-level state
machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
script `exec`s supervisord; a second container mounting the same volume
retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
dies, the kernel releases the lock. There is no follower path inside the Go
binary.
The lock is a **mutex, not service discovery** — it only stops two
writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info`
file it used to publish is gone: siblings locate meta-core over UDP
(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
`internal/leader` keeps its name for history, but only builds the `/urls`
payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
and the UDP announce share.
## Metadata storage shape (one-line summary)
Roots are opaque UUIDv7 strings; every property is a separate row
of the `kv` table at `file:<uuid>/<property>`; every known CID is a key-set member
`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
(the canonical CID is derived by rank on read, never stored). Full details in
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
The authoritative registry of field semantics and value formats lives in
the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
## Internal packages
| Package | Purpose |
|---------|---------|
| `cmd/meta-core` | Main entry point, startup/shutdown orchestration |
| `internal/config` | Environment-based configuration with defaults |
| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) |
| `internal/storage` | Redis client wrapper with flat key-value operations |
| `internal/discovery` | Service registration, heartbeat loop, discovery |
| `internal/api` | HTTP server, router, and all endpoint handlers |
| `internal/events` | Redis keyspace notification publisher to meta:events |
| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch |
| `internal/watchers` | Polling-based watcher configuration management |
| `internal/mounts` | SMB/rclone mount configuration and health monitoring |
| `internal/webdav` | WebDAV protocol handler with caching integration |
| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. |
| `internal/config` | Env-driven configuration, path helpers. |
| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). |
| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field_v` + FTS (`search.go`), derived counters (`onChange`), UDL store (`udl.go`, `udl_admin.go`), stats. |
| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
| `internal/reports` | Report inbox: its own SQLite file, normalisation/bounds, list/filter/paging, prune (closed first). |
| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. |
| `internal/schema` | Per-field schema, folded from counters the write path keeps exact (`idx:schema/…`). |
| `internal/snapshot` | Snapshot export / import / wipe. |
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. |
| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). |
### Startup Sequence
### Startup sequence
```
1. Load configuration from environment
2. Create storage client (Redis wrapper)
3. Initialize leader election
4. Register role transition callbacks
5. Start election (acquire lock or become follower)
6. Start service discovery (register + heartbeat loop)
7. Initialize dead service cleaner
8. Initialize file watcher (if enabled)
9. Initialize mount manager
10. Start HTTP API server (WebDAV caching handled by nginx)
11. Wait for SIGINT/SIGTERM
12. Shutdown in reverse order
```
## Integration
### As Sidecar in Docker
Add to existing service's supervisord config:
```ini
[program:meta-core]
command=/usr/local/bin/meta-core
priority=10
autostart=true
autorestart=true
```
### Client Usage (TypeScript)
```typescript
const response = await fetch('http://localhost:9000/meta/' + hashId);
const { metadata } = await response.json();
```
### Client Usage (Python)
```python
import requests
response = requests.get(f'http://localhost:9000/meta/{hash_id}')
metadata = response.json()['metadata']
2. Build LeaderInfoProvider (no election — the bash gate already won)
3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
4. EnsureSchemaVersion — abort if the existing store layout is stale
5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
6. Construct API server (identity keystore migration, watcher, watchers,
mounts, WebDAV)
7. EnsureCounters — one-time build of the derived counters (record count,
per-field schema) on a store that predates them; blocking, ~1 min at 169k
records, then never again
8. Start API server + pollers (no republish: consumers that want the whole
state ask /api/events/meta for `0-0` and get a snapshot)
9. Wait for SIGINT/SIGTERM, shutdown in reverse order
```
## Dependencies
| Dependency | Version | Purpose |
|------------|---------|---------|
| `github.com/gorilla/mux` | v1.8.1 | HTTP router |
| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client |
| `golang.org/x/net` | v0.20.0 | WebDAV support |
| `github.com/google/uuid` | v1.6.0 | UUID generation |
| Go stdlib | - | context, encoding/json, net, os, sync, syscall |
| Dependency | Purpose |
|------------|---------|
| `github.com/gorilla/mux` | HTTP router |
| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) |
| `github.com/google/uuid` | UUIDv7 minting |
| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
| `golang.org/x/net` | WebDAV support |
### Runtime Requirements
- **Go 1.21+** for building
- **Redis** binary available in PATH (leader spawns it)
- **Shared filesystem** accessible by all instances (for flock)
Runtime requirements: Go 1.21+ at build time; the container ships
rclone, nginx, and supervisord alongside the binary, plus the React dashboard
and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
## License
+120 -35
View File
@@ -1,22 +1,25 @@
package main
import (
"encoding/json"
"fmt"
"log"
"os"
"os/signal"
"path/filepath"
"syscall"
"time"
"github.com/metazla/meta-core/internal/api"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/reports"
"github.com/metazla/meta-core/internal/storage"
)
// Version is set at build time
var Version = "1.0.0"
var Version = "2.0.0"
func main() {
log.SetFlags(log.LstdFlags | log.Lshortfile)
@@ -29,42 +32,106 @@ func main() {
// Create leader info provider (Go binary only runs as leader - followers loop in bash)
leaderProvider := leader.NewLeaderInfoProvider(cfg)
// Connect to local Redis (supervisord starts Redis before meta-core)
// Retry connection since Redis may still be loading its dataset
redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
// Open the two embedded SQLite files: truth (signed claims, backed up)
// and the cache (everything derived, rebuildable). No database process.
if err := os.MkdirAll(filepath.Dir(cfg.CachePath()), 0o755); err != nil {
log.Fatalf("[meta-core] Failed to create the cache folder: %v", err)
}
storageClient := storage.NewClient()
if err := storageClient.Connect(cfg.DBPath(), cfg.CachePath()); err != nil {
log.Fatalf("[meta-core] Failed to open the metadata store: %v", err)
}
storageClient := storage.NewClient("")
maxRetries := 30
retryDelay := 1 // seconds
var err error
for i := 0; i < maxRetries; i++ {
err = storageClient.Connect(redisURL)
if err == nil {
break
// Schema-version gate: refuses a v1.x store (which the default truth path
// still points at) and any unknown layout. No migration code here — see
// tools/claims-migrate.
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
log.Fatalf("[meta-core] %v", err)
}
log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err)
time.Sleep(time.Duration(retryDelay) * time.Second)
log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion)
// Bring the cache level with truth (replay after a crash, full rebuild
// when the cache folder is empty), then start the apply loop.
started := time.Now()
if err := storageClient.Start(); err != nil {
log.Fatalf("[meta-core] %v", err)
}
log.Printf("[meta-core] Cache ready in %s", time.Since(started).Round(time.Millisecond))
// Derived counters (record count, per-field schema) are kept exact by the
// write path. A store that predates them gets a one-time build here,
// before anything else writes.
started = time.Now()
if built, err := storageClient.EnsureCounters(); err != nil {
log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err)
} else if built {
log.Printf("[meta-core] Built derived counters in %s", time.Since(started).Round(time.Millisecond))
}
// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
// as metamesh.core — carrying the /urls block services locate us by — plus
// its own metamesh.service/<name> nav entry. The payload is rebuilt from
// leaderProvider on every tick, which keeps it and GET /urls from drifting.
var mesh *meshdisco.Node
if cfg.EnableUDPDiscovery {
hostname, _ := os.Hostname()
mesh = meshdisco.New(meshdisco.Config{
Name: cfg.ServiceName,
Instance: hostname,
Version: Version,
Group: cfg.DiscoveryGroup,
Port: cfg.DiscoveryPort,
Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
Payload: func() (string, []meshdisco.Resource) {
service := meshdisco.Resource{
ID: "service",
Caps: []string{meshdisco.CapServicePrefix + cfg.ServiceName},
Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
}
info := leaderProvider.LeaderInfo()
if info == nil {
return "starting", []meshdisco.Resource{service}
}
data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
Hostname: info.Hostname,
BaseUrl: info.BaseUrl,
ApiUrl: info.ApiUrl,
WebdavUrl: info.WebdavUrl,
WebdavUrlInternal: info.WebdavUrlInternal,
}})
core := meshdisco.Resource{
ID: "core",
Caps: []string{meshdisco.CapCore},
Endpoints: map[string]string{"http": info.ApiUrl},
Data: data,
}
return "running", []meshdisco.Resource{core, service}
},
})
if err := mesh.Start(); err != nil {
log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
mesh = nil
}
} else {
log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
}
// Report inbox: its own SQLite file beside truth. Not load-bearing — a
// failure to open it leaves /api/reports answering 503, never the core down.
reportStore, err := reports.Open(cfg.ReportsDBPath(), cfg.ReportsMax, int64(cfg.ReportsMaxMB)<<20)
if err != nil {
log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err)
}
log.Println("[meta-core] Connected to Redis")
// Create and start service discovery
disc := discovery.NewService(cfg)
if err := disc.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
}
// Create and start dead service cleaner
cleaner := discovery.NewCleaner(cfg)
if err := cleaner.Start(); err != nil {
log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
log.Printf("[meta-core] Warning: report inbox disabled: %v", err)
reportStore = nil
} else {
log.Printf("[meta-core] Report inbox %s (max %d reports, %d MiB)", reportStore.Path(), cfg.ReportsMax, cfg.ReportsMaxMB)
}
// Create and start API server
apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
apiServer := api.NewServer(cfg, leaderProvider, storageClient)
apiServer.SetMeshNode(mesh)
if reportStore != nil {
apiServer.SetReports(reportStore)
}
if err := apiServer.Start(); err != nil {
log.Fatalf("[meta-core] Failed to start API server: %v", err)
}
@@ -81,12 +148,16 @@ func main() {
log.Printf("[meta-core] Error stopping API server: %v", err)
}
if err := cleaner.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service cleaner: %v", err)
if mesh != nil {
if err := mesh.Stop(); err != nil {
log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
}
}
if err := disc.Stop(); err != nil {
log.Printf("[meta-core] Error stopping service discovery: %v", err)
if reportStore != nil {
if err := reportStore.Close(); err != nil {
log.Printf("[meta-core] Error closing report inbox: %v", err)
}
}
if err := storageClient.Close(); err != nil {
@@ -103,3 +174,17 @@ func waitForShutdown() {
<-sigChan
}
// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
// the three-way fallback the file registry used (discovery/service.go):
// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
// the container IP.
func coreBaseURL(cfg *config.Config) string {
if cfg.PublicURL != "" {
return cfg.PublicURL
}
if cfg.BaseURL != "" {
return cfg.BaseURL
}
return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
}
+1 -1
View File
@@ -21,7 +21,7 @@
min-height: 100vh;
}
</style>
<script type="module" crossorigin src="/assets/index-DJlbGMsa.js"></script>
<script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
</head>
<body>
+10
View File
@@ -8,6 +8,7 @@
"name": "@meta-core/dashboard",
"version": "1.0.0",
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
@@ -744,6 +745,15 @@
"@jridgewell/sourcemap-codec": "^1.4.14"
}
},
"node_modules/@noble/secp256k1": {
"version": "3.1.0",
"resolved": "https://registry.npmjs.org/@noble/secp256k1/-/secp256k1-3.1.0.tgz",
"integrity": "sha512-+F7iS7tUMaNGXcc9X3PjmjvuQnXEuSjCRNzVVA2xAcKXgCaP0dHYz4SFyt4FKNHef7sOP//xihowcySSS7PK9g==",
"license": "MIT",
"funding": {
"url": "https://paulmillr.com/funding/"
}
},
"node_modules/@remix-run/router": {
"version": "1.23.2",
"resolved": "https://registry.npmjs.org/@remix-run/router/-/router-1.23.2.tgz",
+3 -2
View File
@@ -9,10 +9,11 @@
"preview": "vite preview"
},
"dependencies": {
"@noble/secp256k1": "^3.1.0",
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0",
"react-router-dom": "^6.20.0",
"iso-639-3": "^3.0.1"
"react-router-dom": "^6.20.0"
},
"devDependencies": {
"@types/react": "^18.2.38",
+16
View File
@@ -3,6 +3,8 @@ import Dashboard from './pages/Dashboard';
import Mounts from './pages/Mounts';
import Watchers from './pages/Watchers';
import Files from './pages/Files';
import Identity from './pages/Identity';
import Reports from './pages/Reports';
import ServiceNav from './components/ServiceNav';
const navStyle: React.CSSProperties = {
@@ -74,6 +76,18 @@ export default function App() {
>
Files
</NavLink>
<NavLink
to="/identity"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
>
Identity
</NavLink>
<NavLink
to="/reports"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
>
Reports
</NavLink>
<NavLink
to="/editor"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
@@ -89,6 +103,8 @@ export default function App() {
<Route path="/mounts" element={<Mounts />} />
<Route path="/watchers" element={<Watchers />} />
<Route path="/files" element={<Files />} />
<Route path="/identity" element={<Identity />} />
<Route path="/reports" element={<Reports />} />
<Route path="/editor" element={<EditorPage />} />
</Routes>
</div>
+37 -139
View File
@@ -1,155 +1,53 @@
import { useState, useEffect } from 'react';
import { useEffect } from 'react';
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
// of defining the custom element.
import '../meta-service-menu.js';
interface ServiceInfo {
name: string;
hostname: string;
baseUrl: string;
status: string;
lastHeartbeat: string;
/**
* ServiceNav — the neighbouring meta-* services, as a burger menu.
*
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
* a framework-free custom element shared by every meta-* UI (see
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
* that existed in four near-identical copies and polled meta-core's
* /api/services; the element polls this service's own /api/neighbors instead,
* so the nav still renders when meta-core is down.
*
* The element lives in shadow DOM and cannot see this app's CSS variables, so
* the four --mm-nav-* tokens below map our palette onto it.
*/
declare global {
// eslint-disable-next-line @typescript-eslint/no-namespace
namespace JSX {
interface IntrinsicElements {
'meta-service-menu': React.DetailedHTMLProps<
React.HTMLAttributes<HTMLElement>,
HTMLElement
> & { current?: string; endpoint?: string; label?: string };
}
interface ServicesResponse {
services: ServiceInfo[];
current: string;
}
const serviceIcons: Record<string, string> = {
'meta-core': '🗄️',
'meta-sort': '📁',
'meta-fuse': '🗂️',
'meta-stremio': '🎬',
'meta-orbit': '🌐',
'default': '📦'
};
function formatServiceName(name: string): string {
return name.split('-').map(word =>
word.charAt(0).toUpperCase() + word.slice(1)
).join(' ');
}
function ServiceNav() {
const [services, setServices] = useState<ServiceInfo[]>([]);
const [loading, setLoading] = useState(true);
const currentService = 'meta-core';
useEffect(() => {
const fetchServices = async () => {
try {
const response = await fetch('/services');
if (response.ok) {
const data: ServicesResponse = await response.json();
setServices(data.services || []);
}
} catch (error) {
console.error('Failed to fetch services:', error);
} finally {
setLoading(false);
}
};
fetchServices();
const interval = setInterval(fetchServices, 30000);
return () => clearInterval(interval);
// Import above is for the side effect of defining the element; this
// keeps bundlers from tree-shaking it away in production builds.
}, []);
if (loading) return null;
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
if (sortedServices.length === 0) return null;
return (
<nav className="services-nav">
<span className="services-nav-label">Services:</span>
<div className="services-nav-items">
{sortedServices.map(service => {
const icon = serviceIcons[service.name] || serviceIcons.default;
const isActive = service.name === currentService;
return (
<a
key={service.name}
href={isActive ? '#' : service.baseUrl}
className={`service-link${isActive ? ' active' : ''}`}
onClick={(e) => {
e.preventDefault();
if (!isActive) {
window.location.href = service.baseUrl;
}
}}
>
<span className="service-icon">{icon}</span>
<span>{formatServiceName(service.name)}</span>
<span className="service-status"></span>
</a>
);
})}
</div>
<>
<meta-service-menu current="meta-core" />
<style>{`
.services-nav {
display: flex;
align-items: center;
gap: 0.5rem;
}
.services-nav-label {
color: #888;
font-size: 0.8rem;
white-space: nowrap;
}
.services-nav-items {
display: flex;
gap: 0.35rem;
}
.service-link {
display: inline-flex;
align-items: center;
gap: 0.3rem;
padding: 0.35rem 0.6rem;
border-radius: 6px;
text-decoration: none;
font-size: 0.8rem;
font-weight: 500;
transition: all 0.2s;
color: #ccc;
background: #0f3460;
border: 1px solid #1a4a7a;
}
.service-link:hover {
background: #1a4a7a;
border-color: rgba(78, 205, 196, 0.5);
text-decoration: none;
color: #fff;
}
.service-link.active {
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
border-color: rgba(78, 205, 196, 0.5);
color: #fff;
}
.service-link .service-icon {
font-size: 0.9rem;
}
.service-link .service-status {
width: 5px;
height: 5px;
border-radius: 50%;
background: #4ecdc4;
}
@media (max-width: 800px) {
.services-nav-label {
display: none;
}
meta-service-menu {
--mm-nav-fg: var(--text-primary, #e0e0e0);
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
--mm-nav-accent: var(--accent-primary, #4ecdc4);
}
`}</style>
</nav>
</>
);
}
+28
View File
@@ -0,0 +1,28 @@
// JSON fetch that fails with a sentence instead of a parser error.
//
// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
// path it does not have a proxy rule for answers 200 text/html with the SPA
// shell. A caller that does `res.ok && res.json()` on such a path gets
// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
// no clue which request produced it — which is exactly how a route deleted
// from the Go router (`/services`, removed in favour of /api/neighbors) sat
// broken on the dashboard.
//
// So: check the content type, and name the URL in every error.
export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
const res = await fetch(url, init);
const ct = res.headers.get('content-type') || '';
if (!res.ok) {
throw new Error(`${url} → HTTP ${res.status}`);
}
if (!ct.includes('application/json')) {
// The SPA fallback is the overwhelmingly likely cause; say so.
throw new Error(
`${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
`missing and nginx served the dashboard shell instead)`
);
}
return (await res.json()) as T;
}
+271
View File
@@ -0,0 +1,271 @@
// How a report reads at a glance (the inbox row headline) and in full (the
// copy-as-text dump pasted into the Deck board).
//
// meta-core stores a report's body verbatim and knows nothing about it, so
// the meaning lives here, per app. The meta-watch headline is ported from the
// /info page that used to own reports (meta-watch ui/info.html,
// reportHeadline) — its comments explain why each chip earned its place; the
// short version is that every chip there is a stall or desync that once read
// as "healthy" until it was measured. Anything an app adds that this file does
// not know still renders: in the dump, and as a JSON section in the detail.
/* eslint-disable @typescript-eslint/no-explicit-any */
export type Body = Record<string, any>;
export interface ReportSummary {
id: number;
ts: number;
received: number;
app: string;
build?: string;
uid?: string;
page?: string;
route?: string;
cid?: string;
title?: string;
note?: string;
status: 'open' | 'triaged' | 'closed';
triage_note?: string;
updated?: number;
log_lines: number;
bytes: number;
}
export interface Report extends ReportSummary {
body: Body;
}
/** One headline segment; `strong` ones are what a human wrote or a failure. */
export interface Bit {
text: string;
strong?: boolean;
}
export function fmtDur(ms: number): string {
if (ms == null || !isFinite(ms)) return '?';
if (ms < 1000) return `${Math.round(ms)}ms`;
const s = ms / 1000;
if (s < 60) return `${s.toFixed(s < 10 ? 1 : 0)}s`;
const m = Math.floor(s / 60);
const r = Math.round(s % 60);
if (m < 60) return `${m}m${r ? ` ${r}s` : ''}`;
return `${Math.floor(m / 60)}h ${m % 60}m`;
}
export function shortCid(cid: string): string {
return cid && cid.length > 16 ? `${cid.slice(0, 8)}…${cid.slice(-6)}` : cid || '';
}
const clip = (s: string, n: number) => (s.length > n ? s.slice(0, n - 1) + '…' : s);
/** First line of the build blob, minus the app prefix: which build filed it. */
export function buildLabel(r: ReportSummary): string {
const first = (r.build || '').split('\n')[0];
return first.replace(/^meta-(watch|read|listen)( page| server)?\s+/, '').trim();
}
/** The device a report came from, in one line. */
export function deviceLine(d: any): string {
if (!d || typeof d !== 'object') return '';
const bits: string[] = [];
if (d.model) bits.push(d.model);
else if (d.platform) bits.push(d.platform);
if (d.isTv) bits.push(d.tvForced ? 'TV (forced)' : 'TV');
if (d.screen) bits.push(d.screen);
if (d.cores) bits.push(`${d.cores} cores`);
if (d.memoryGb) bits.push(`${d.memoryGb} GB`);
if (d.gpu) bits.push(d.gpu);
return bits.join(' · ');
}
/** What the row is about: a title, a page + route, or a cid. */
export function subject(r: ReportSummary, body?: Body): string {
const c = (body && body.context) || {};
if (r.page === 'search') return c.query ? `search “${c.query}”` : 'search';
if (r.title) return r.title;
if (r.page && r.page !== 'watch' && r.page !== 'read') return r.page;
if (r.cid) return shortCid(r.cid);
return r.page || '—';
}
// ── Headlines ─────────────────────────────────────────────────────────────
function failureBit(c: any): Bit | null {
const f = c && (c.failure || c.lastFailure);
if (!f) return null;
const who = c.lastFailure ? ` ${f.title || shortCid(f.cid || '')}` : '';
const tries = Array.isArray(f.attempts) && f.attempts.length > 1 ? ` · ${f.attempts.length} tries` : '';
return {
text: `FAILED${who}${f.elapsedMs != null ? ' after ' + fmtDur(f.elapsedMs) : ''}: ${clip(f.message || '', 90)}${tries}`,
strong: true,
};
}
function watchBits(r: ReportSummary, b: Body): string[] {
const bits: string[] = [];
const c = b.context || {};
if (r.page === 'search') {
if (c.state) bits.push(c.state === 'errored' ? 'STREAM ERRORED' : c.state);
if (c.anchored != null) {
bits.push(
`${c.anchored} anchored · ${c.raw || 0} raw` +
(c.shown != null && c.shown !== c.anchored ? ` · ${c.shown} after language filter` : '')
);
}
if (c.elapsedMs != null) bits.push(`stream ${fmtDur(c.elapsedMs)}`);
if (c.cacheSeeded) bits.push('painted from cache');
const ups = Array.isArray(c.upstreams) ? c.upstreams : [];
if (c.state !== 'idle' && c.state !== 'youtube-paste') {
if (ups.length) {
const recs = ups.reduce((n: number, u: any) => n + (u.records || 0), 0);
bits.push(`${ups.length} upstream${ups.length === 1 ? '' : 's'} (${recs} records)`);
} else bits.push('no upstream seen');
}
} else if (r.page === 'title') {
if (c.episodes != null) {
bits.push(`${c.episodes} release${c.episodes === 1 ? '' : 's'}` + (c.rawSources ? ` · ${c.rawSources} raw` : ''));
}
if (c.streamsOpen) bits.push(`${c.streamsOpen} stream${c.streamsOpen === 1 ? '' : 's'} open`);
else if (c.settled != null) bits.push(c.settled ? 'settled' : 'not settled');
if (c.placeholderCard) bits.push('placeholder hero');
}
if (c.lastPlayback) bits.push(`after playing ${c.lastPlayback.title || shortCid(c.lastPlayback.cid || '')}`);
const d = b.decode || {};
const p = b.pipeline || {};
if (d.underruns != null) bits.push(`${d.underruns} underrun${d.underruns === 1 ? '' : 's'}`);
if (d.minAhead != null) bits.push(`min ${(+d.minAhead).toFixed(1)}s ahead`);
if (d.decoded) {
const pct = (d.dropped / d.decoded) * 100;
bits.push(`${d.dropped} dropped (${pct.toFixed(pct >= 1 ? 1 : 2)}%)`);
}
if (d.capability) bits.push(d.capability);
if (p.pctDemuxCpu != null) bits.push(`demux CPU ${p.pctDemuxCpu}%`);
// Subtitle verdict: a stalled overlay is invisible in every other number.
const sub = b.subtitles;
if (sub && sub.up) {
const ago = sub.lastDrawAgoMs;
if (ago == null) bits.push('subs: NO DRAWS');
else if (ago > 4000) {
const why =
sub.paused === true ? ' while PAUSED'
: sub.advancing === true ? ' while playing'
: sub.paused === false ? ' (playing, not advancing)'
: '';
bits.push(`subs: STALLED ${fmtDur(ago)}${why}`);
} else bits.push(`subs ok (${sub.draws} draws)`);
if (sub.skipped > 0 && sub.draws > 0) {
const share = Math.round((100 * sub.skipped) / (sub.skipped + sub.draws));
if (share >= 15) bits.push(`subs: ${share}% of frames dropped`);
}
if (sub.commitMaxMs != null && sub.commitMaxMs > 250) {
bits.push(
`subs: commit ${sub.commitMs}/${sub.commitMaxMs}ms` +
(sub.pruned ? ` (${sub.lines}+${sub.pruned}p lines)` : ` (${sub.lines} lines)`)
);
}
if (sub.lastEventAgoMs != null && sub.lastEventAgoMs > 30000 && sub.advancing) {
bits.push(`subs: no packets for ${fmtDur(sub.lastEventAgoMs)}`);
}
if (sub.canvas && sub.canvas !== 'ok') bits.push(`subs canvas ${sub.canvas}`);
if (sub.videoW === 0 || sub.resizeDeferred) bits.push('subs: overlay resize owed (video has no size)');
else if (sub.resizesBlocked > 0) {
bits.push(`subs: ${sub.resizesBlocked} zero-size resize${sub.resizesBlocked > 1 ? 's' : ''} blocked`);
}
} else if (sub && sub.failed) {
const dd = sub.death;
if (!dd) bits.push('subs: fell back to WebVTT (build too old to say why)');
else {
bits.push(`subs: fell back to WebVTT at ${fmtDur(dd.mediaTime)} — ${dd.why}`);
if (dd.commitFailures) bits.push(`after ${dd.commitFailures} rejected commits`);
if (dd.strikes) bits.push(`after ${dd.strikes} stuck-loop strikes`);
bits.push(dd.draws ? `had drawn ${dd.draws} frames (${dd.lines}+${dd.pruned}p lines)` : 'never drew a frame');
if (dd.canvas && dd.canvas !== 'ok') bits.push(`canvas ${dd.canvas} at death`);
if (dd.resizesBlocked) bits.push(`${dd.resizesBlocked} zero-size resizes blocked`);
if (dd.videoW === 0) bits.push('video had no size at death');
}
}
// A/V sync: only the peak earns a chip.
const au = b.audio;
if (au && au.driftMaxMs != null && Math.abs(au.driftMaxMs) >= 40) {
const at = au.driftMaxAtSec != null ? ` @ ${fmtDur(au.driftMaxAtSec * 1000)}` : '';
bits.push(`A/V DESYNC ${au.driftMaxMs > 0 ? '+' : ''}${au.driftMaxMs}ms${at}`);
}
if (au && au.path === 'polyfill') bits.push(`audio ${au.codec || '?'} via polyfill`);
const rs = b.restart;
if (rs && rs.audioMinusVideoSec != null && rs.audioMinusVideoSec < -0.001) {
bits.push(`restart base from AUDIO (${rs.audioMinusVideoSec.toFixed(3)}s)`);
}
if (b.ttmvpMs != null) bits.push(`TTMVP ${fmtDur(b.ttmvpMs)}`);
if (b.capturedMsAgo != null && b.capturedMsAgo > 2000) bits.push(`measured ${fmtDur(b.capturedMsAgo)} before filing`);
return bits;
}
function readBits(r: ReportSummary, b: Body): string[] {
const bits: string[] = [];
const rd = b.reader;
if (rd && typeof rd === 'object') {
if (rd.page != null) bits.push(`page ${rd.page + 1}${rd.pages ? `/${rd.pages}` : ''}`);
if (rd.flow) bits.push(rd.flow);
if (rd.direction) bits.push(rd.direction);
if (rd.spread) bits.push('spread');
if (rd.strip && rd.strip.segments) bits.push(`strip ${rd.strip.active ?? '?'}/${rd.strip.segments}`);
if (rd.via) bits.push(`via ${rd.via}`);
if (rd.atEnd) bits.push('at end');
}
const c = b.context || {};
if (r.page === 'search') {
if (c.state) bits.push(c.state);
if (c.count != null) bits.push(`${c.count} shown`);
}
return bits;
}
/** The row headline: the note first (the only part a human wrote), then a
* failure if there was one, then the app's own measurements. */
export function headline(r: ReportSummary, body?: Body): Bit[] {
const out: Bit[] = [];
if (r.note) out.push({ text: `“${clip(r.note, 80)}”`, strong: true });
if (!body) return out;
const f = failureBit(body.context);
if (f) out.push(f);
const bits = r.app === 'meta-watch' ? watchBits(r, body) : r.app === 'meta-read' ? readBits(r, body) : [];
for (const t of bits) out.push({ text: t });
if (body.flags && typeof body.flags === 'object' && Object.keys(body.flags).length) {
out.push({ text: 'overrides: ' + Object.entries(body.flags).map(([k, v]) => `${k}=${v}`).join(' ') });
}
return out;
}
// ── Full text dump ────────────────────────────────────────────────────────
const LIFTED = new Set(['app', 'build', 'uid', 'page', 'route', 'cid', 'title', 'note', 'log']);
/** Every block of a report that is not one of the lifted fields or the log,
* in a stable order: context first, device second, then alphabetical. */
export function blocks(body: Body): [string, any][] {
const keys = Object.keys(body).filter((k) => !LIFTED.has(k));
const rank = (k: string) => (k === 'context' ? 0 : k === 'device' ? 1 : 2);
keys.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
return keys.map((k) => [k, body[k]]);
}
/** The whole report as text, for pasting into a ticket. */
export function dump(r: Report): string {
const parts: string[] = [];
parts.push(`REPORT #${r.id} · ${r.app} · ${new Date(r.ts).toISOString()} · ${r.status}` +
(r.uid ? `\nUID ${r.uid}` : ''));
if (r.note) parts.push('NOTE\n' + r.note);
if (r.page || r.route) parts.push('PAGE ' + (r.page || '—') + (r.route ? '\nROUTE ' + r.route : ''));
if (r.cid || r.title) parts.push(`CONTENT ${r.title || ''}${r.cid ? ` (${r.cid})` : ''}`.trim());
if (r.build) parts.push(r.build);
for (const [k, v] of blocks(r.body || {})) {
parts.push(k.toUpperCase() + '\n' + (typeof v === 'string' ? v : JSON.stringify(v, null, 2)));
}
const log: string[] = Array.isArray(r.body?.log) ? r.body.log : [];
if (log.length) parts.push(`LOG (${log.length} lines)\n` + log.join('\n'));
if (r.triage_note) parts.push('TRIAGE\n' + r.triage_note);
return parts.join('\n\n');
}
+127
View File
@@ -0,0 +1,127 @@
// Client-side proof of possession for account operations.
//
// meta-core stores every private key in plaintext, so it can sign as anybody —
// which means a signature it produced proves nothing about who asked. The only
// signature that carries authority is one made where the key actually is, and
// for a browser-driven admin console that is here. meta-core enforces the other
// half by refusing to sign anything in the challenge domain
// (identity.ChallengeDomain), so this path cannot be short-circuited.
//
// The secret key entered on this page is used for exactly one signature and is
// never sent anywhere: `/api/identity/challenge` receives only the uid, and the
// request that follows carries only the challenge and its signature.
//
// Same scheme, same curve and same encodings as meta-watch's sign-in
// (packages/meta-watch/ui/js/auth.js) — deliberately, so one implementation on
// the server verifies both.
import * as secp from '@noble/secp256k1';
export type IdentityAction = 'reveal' | 'delete';
function hexToBytes(hex: string): Uint8Array | null {
let h = hex.trim();
if (h.startsWith('0x') || h.startsWith('0X')) h = h.slice(2);
if (h.length !== 64 || /[^0-9a-fA-F]/.test(h)) return null;
const out = new Uint8Array(32);
for (let i = 0; i < 32; i++) out[i] = parseInt(h.slice(i * 2, i * 2 + 2), 16);
return out;
}
function bytesToBase64(bytes: Uint8Array): string {
let s = '';
for (const b of bytes) s += String.fromCharCode(b);
return btoa(s);
}
const B58 = '123456789ABCDEFGHJKLMNPQRSTUVWXYZabcdefghijkmnopqrstuvwxyz';
/// base58btc, the multibase "z" alphabet. Spelled out rather than pulled in:
/// the alternative is a second dependency for thirty lines of arithmetic.
function base58Encode(bytes: Uint8Array): string {
const digits: number[] = [0];
for (const byte of bytes) {
let carry = byte;
for (let i = 0; i < digits.length; i++) {
carry += digits[i] << 8;
digits[i] = carry % 58;
carry = (carry / 58) | 0;
}
while (carry) {
digits.push(carry % 58);
carry = (carry / 58) | 0;
}
}
// Leading zero bytes encode as leading '1's rather than being dropped.
let out = '';
for (let i = 0; i < bytes.length && bytes[i] === 0; i++) out += B58[0];
for (let i = digits.length - 1; i >= 0; i--) out += B58[digits[i]];
return out;
}
/// The uid a secret key belongs to: "z" + base58btc(33-byte compressed pubkey).
/// Derived locally so a mistyped or wrong-account key is caught before any
/// request goes out. Mirrors `identityFromPrivKey` in meta-core's identity.go.
export function uidFromSecretKey(secretKeyHex: string): string | null {
const raw = hexToBytes(secretKeyHex);
if (!raw) return null;
try {
return 'z' + base58Encode(secp.getPublicKey(raw, true));
} catch {
return null; // not a valid scalar
}
}
/// Whether this page can sign at all.
///
/// noble's signAsync takes its sha256 from crypto.subtle, which browsers expose
/// only in a secure context. The dashboard behind Caddy is https and fine; the
/// debug-direct port (18083) is plain http, where every signature would fail
/// with a cryptic error unless we say so first.
export function canSign(): boolean {
return Boolean(globalThis.crypto?.subtle);
}
export const INSECURE_CONTEXT_MESSAGE =
'This page cannot sign over a plain-http connection — browsers only expose the ' +
'needed crypto on https (or localhost). Reach the dashboard through Caddy ' +
'(https://metacore-dev.localhost:8083) rather than the debug-direct port.';
/// Ask meta-core for a challenge, sign it locally, and hand back the proof.
///
/// Throws with a message fit to show the user. The caller passes the proof
/// straight into the reveal/delete request body.
export async function proveOwnership(
uid: string,
action: IdentityAction,
secretKeyHex: string
): Promise<{ challenge: string; signature: string }> {
if (!canSign()) throw new Error(INSECURE_CONTEXT_MESSAGE);
const raw = hexToBytes(secretKeyHex);
if (!raw) {
throw new Error('That is not a valid secret key: expected 64 hex characters.');
}
const derived = uidFromSecretKey(secretKeyHex);
if (!derived) throw new Error('That is not a valid secp256k1 secret key.');
if (derived !== uid) {
// Caught here rather than by the server, so the message can say which
// account the key actually is — the useful half of the answer.
throw new Error(`That key belongs to a different account (${derived}), not this one.`);
}
const cRes = await fetch('/api/identity/challenge', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ uid, action }),
});
const cBody = await cRes.json().catch(() => ({}));
if (!cRes.ok || !cBody.challenge) {
throw new Error(cBody.message ?? `Could not start verification (HTTP ${cRes.status}).`);
}
// signAsync prehashes with sha256 and returns a 64-byte compact (r‖s)
// signature — noble v3 dropped DER. meta-core's identity.Verify accepts both.
const sig = await secp.signAsync(new TextEncoder().encode(cBody.challenge), raw);
return { challenge: cBody.challenge, signature: bytesToBase64(sig) };
}
+308
View File
@@ -0,0 +1,308 @@
/**
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
*
* Replaces the eight divergent "services bar" implementations (four copies of
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
* ones) with a single framework-free custom element. It drops into the React
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
* as an ES module.
*
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
* make a fetch to a neighbour a dead end. Links are plain navigations.
*
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
* `scripts/check-mirrors.sh`. Edit one → edit all.
*
* Shadow DOM is not optional here. The eight UIs use four incompatible token
* vocabularies, and two of them define *different colours under the same
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
* #161b22 in meta-dup). meta-search and meta-share are also the only
* light-mode-capable UIs, so a hardcoded dark palette would break them.
* The element therefore styles itself from four tokens with dark defaults:
*
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
*
* Each host maps its own variables onto those, e.g.
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
*
* Attributes:
* current name of the host service, so it can be marked and not linked
* endpoint override the default "/api/neighbors"
* label button tooltip (default "Services")
*
* Property:
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
* translate internal hostnames into browser-reachable ones
* (meta-gateway does this for its dev/prod split).
*/
const ICONS = {
"meta-core": "⚙️",
"meta-sort": "📁",
"meta-fuse": "🗂️",
"meta-stremio": "🎬",
"meta-dup": "🔍",
"meta-search": "🔎",
"meta-share": "🔗",
"meta-gateway": "🌉",
"meta-watch": "📺",
"meta-listen": "🎧",
"meta-read": "📖",
};
const DEFAULT_ICON = "📦";
const POLL_MS = 30000;
function titleCase(name) {
return String(name || "")
.split("-")
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
.join(" ");
}
const TEMPLATE = `
<style>
:host {
--_fg: var(--mm-nav-fg, #e8eaed);
--_bg: var(--mm-nav-bg, #1a1a2e);
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
--_accent: var(--mm-nav-accent, #4ecdc4);
position: relative;
display: inline-block;
font-family: inherit;
font-size: 14px;
line-height: 1.4;
}
button.trigger {
display: inline-flex;
align-items: center;
justify-content: center;
gap: 0.4rem;
width: 2.1rem;
height: 2.1rem;
padding: 0;
cursor: pointer;
color: var(--_fg);
background: transparent;
border: 1px solid var(--_border);
border-radius: 8px;
font: inherit;
transition: border-color 0.15s, background 0.15s;
}
button.trigger:hover { border-color: var(--_accent); }
button.trigger:focus-visible {
outline: 2px solid var(--_accent);
outline-offset: 2px;
}
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
.bars { display: block; width: 15px; }
.bars span {
display: block;
height: 2px;
margin: 3px 0;
border-radius: 2px;
background: currentColor;
}
.panel {
position: absolute;
top: calc(100% + 6px);
right: 0;
z-index: 1000;
min-width: 13rem;
max-height: 70vh;
overflow-y: auto;
padding: 0.3rem;
background: var(--_bg);
border: 1px solid var(--_border);
border-radius: 10px;
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
}
.panel[hidden] { display: none; }
a.item, .item {
display: flex;
align-items: center;
gap: 0.55rem;
padding: 0.45rem 0.6rem;
border-radius: 6px;
color: var(--_fg);
text-decoration: none;
white-space: nowrap;
}
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
.item.dead { opacity: 0.55; cursor: not-allowed; }
.icon { font-size: 1rem; line-height: 1; }
.dot {
width: 6px;
height: 6px;
margin-left: auto;
border-radius: 50%;
background: var(--_accent);
}
.empty {
padding: 0.6rem;
opacity: 0.7;
font-size: 0.85em;
white-space: nowrap;
}
/* Respect a host that opts out of motion. */
@media (prefers-reduced-motion: reduce) {
button.trigger { transition: none; }
}
</style>
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
</button>
<div class="panel" role="menu" hidden></div>
`;
class MetaServiceMenu extends HTMLElement {
constructor() {
super();
this.attachShadow({ mode: "open" });
this.shadowRoot.innerHTML = TEMPLATE;
this._btn = this.shadowRoot.querySelector("button.trigger");
this._panel = this.shadowRoot.querySelector(".panel");
this._neighbors = [];
this._timer = null;
this._open = false;
this._onDocClick = (e) => {
if (!this._open) return;
if (e.composedPath().includes(this)) return;
this._setOpen(false);
};
this._onKey = (e) => {
if (e.key === "Escape" && this._open) {
this._setOpen(false);
this._btn.focus();
}
};
}
connectedCallback() {
this._btn.title = this.getAttribute("label") || "Services";
this._btn.setAttribute("aria-label", this._btn.title);
this._btn.addEventListener("click", () => this._setOpen(!this._open));
document.addEventListener("click", this._onDocClick);
document.addEventListener("keydown", this._onKey);
this.refresh();
this._timer = setInterval(() => this.refresh(), POLL_MS);
}
disconnectedCallback() {
if (this._timer) clearInterval(this._timer);
this._timer = null;
document.removeEventListener("click", this._onDocClick);
document.removeEventListener("keydown", this._onKey);
}
get endpoint() {
return this.getAttribute("endpoint") || "/api/neighbors";
}
get current() {
return this.getAttribute("current") || "";
}
async refresh() {
try {
const res = await fetch(this.endpoint, {
headers: { Accept: "application/json" },
});
if (!res.ok) throw new Error(String(res.status));
const data = await res.json();
// `services` is the legacy alias some backends still emit.
this._neighbors = data.neighbors || data.services || [];
} catch {
// Silent: the menu is navigation, not a health indicator. Keep whatever
// we last had rather than blanking the list on one failed poll.
}
this._render();
}
_setOpen(open) {
this._open = open;
this._panel.hidden = !open;
this._btn.setAttribute("aria-expanded", String(open));
if (open) this.refresh();
}
_render() {
const panel = this._panel;
panel.textContent = "";
const seen = new Set();
const rows = [];
for (const n of this._neighbors) {
const name = n.name || "";
if (!name || seen.has(name)) continue;
// Beacon v2 neighbours include plugins; the nav lists services only.
// (A row without `caps` comes from a pre-v2 backend — keep it.)
const isService = (c) => String(c).startsWith("metamesh.service/");
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
seen.add(name);
rows.push(n);
}
// Always show the host itself, even before its own echo comes back.
if (this.current && !seen.has(this.current)) {
rows.push({ name: this.current, baseUrl: "" });
}
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
if (rows.length === 0) {
const empty = document.createElement("div");
empty.className = "empty";
empty.textContent = "No services found";
panel.appendChild(empty);
return;
}
for (const n of rows) {
const isActive = n.name === this.current;
let url = n.baseUrl || "";
if (typeof this.rewriteUrl === "function") {
try {
url = this.rewriteUrl(n.name, url, n) || "";
} catch {
/* a bad hook must not blank the menu */
}
}
// A neighbour with no reachable URL is shown but not linked — better
// than a link that goes nowhere.
const el = document.createElement(isActive || !url ? "span" : "a");
el.className =
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
if (el.tagName === "A") {
el.href = url;
el.setAttribute("role", "menuitem");
}
if (!isActive && !url) el.title = "No reachable URL advertised";
const icon = document.createElement("span");
icon.className = "icon";
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
const label = document.createElement("span");
label.textContent = titleCase(n.name);
el.append(icon, label);
// Beacon v2 omits `status` when running.
if ((n.status || "running") === "running" || isActive) {
const dot = document.createElement("span");
dot.className = "dot";
dot.title = n.status || "running";
el.appendChild(dot);
}
panel.appendChild(el);
}
}
}
if (!customElements.get("meta-service-menu")) {
customElements.define("meta-service-menu", MetaServiceMenu);
}
export default MetaServiceMenu;
+171 -71
View File
@@ -1,4 +1,5 @@
import { useEffect, useState } from 'react';
import { getJSON } from '../lib/api';
interface HealthStatus {
status: string;
@@ -6,6 +7,22 @@ interface HealthStatus {
timestamp: string;
}
interface FileStats {
count: number;
totalSize: number;
}
interface Stats {
records: number;
redisKeys: number;
redisMemory: string;
identities: number;
files: FileStats | null;
udlUsers: number | null;
sweptAt: number;
sweeping: boolean;
}
interface WatcherStatus {
id: string;
path: string;
@@ -22,13 +39,6 @@ interface WatchersResponse {
count: number;
}
interface ServiceInfo {
name: string;
api: string;
capabilities: string[];
timestamp: number;
}
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
const gridStyle: React.CSSProperties = {
display: 'grid',
gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
// 200px, not 220: five tiles must fit one row inside the 1200px container
// (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
gap: '1rem',
marginBottom: '1rem',
};
const statStyle: React.CSSProperties = {
fontSize: '2rem',
fontWeight: 'bold',
color: '#4ade80',
lineHeight: 1.2,
};
const labelStyle: React.CSSProperties = {
marginBottom: '0.75rem',
color: '#888',
fontSize: '0.85rem',
textTransform: 'uppercase',
letterSpacing: '0.04em',
};
const noteStyle: React.CSSProperties = {
color: '#6b7a99',
fontSize: '0.8rem',
marginTop: '0.4rem',
};
const numberFmt = new Intl.NumberFormat();
function formatCount(n: number | null | undefined): string {
return n === null || n === undefined ? '—' : numberFmt.format(n);
}
// Binary units: this counts bytes on disk, and every other tool the operator
// compares against (du, docker, the mount page) reports GiB.
function formatBytes(bytes: number | null | undefined): string {
if (bytes === null || bytes === undefined) return '—';
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
let value = bytes;
let unit = 0;
while (value >= 1024 && unit < units.length - 1) {
value /= 1024;
unit++;
}
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
}
function Tile({
label,
value,
note,
pending,
}: {
label: string;
value: string;
note?: string;
pending?: boolean;
}) {
return (
<div style={cardStyle}>
<div style={labelStyle}>{label}</div>
<div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
{value}
</div>
{note && <div style={noteStyle}>{note}</div>}
</div>
);
}
export default function Dashboard() {
const [health, setHealth] = useState<HealthStatus | null>(null);
const [stats, setStats] = useState<Stats | null>(null);
const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
const [services, setServices] = useState<ServiceInfo[]>([]);
const [error, setError] = useState<string | null>(null);
useEffect(() => {
let cancelled = false;
const fetchData = async () => {
try {
// Fetch health
const healthRes = await fetch('/health');
if (healthRes.ok) {
setHealth(await healthRes.json());
}
// Settled, not all: one dead endpoint must not blank every tile.
const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
getJSON<HealthStatus>('/health'),
getJSON<Stats>('/api/stats'),
getJSON<WatchersResponse>('/api/watchers'),
]);
if (cancelled) return;
// Fetch watchers status
const watchersRes = await fetch('/api/watchers');
if (watchersRes.ok) {
setWatchersData(await watchersRes.json());
}
if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
if (statsRes.status === 'fulfilled') setStats(statsRes.value);
if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
// Fetch services
const servicesRes = await fetch('/services');
if (servicesRes.ok) {
const data = await servicesRes.json();
setServices(data.services || []);
}
setError(null);
} catch (err) {
setError(String(err));
}
const failures = [healthRes, statsRes, watchersRes]
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
.map((r) => String(r.reason?.message ?? r.reason));
setError(failures.length ? failures.join(' · ') : null);
};
fetchData();
const interval = setInterval(fetchData, 5000);
return () => clearInterval(interval);
return () => {
cancelled = true;
clearInterval(interval);
};
}, []);
const handleTriggerScan = async () => {
@@ -95,49 +162,102 @@ export default function Dashboard() {
}
};
// Aggregate watcher stats
const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
const lastScan =
watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
// Records we hold metadata for but have no local file: gateway/cid-rooted
// entries, plus anything whose file went away. The interesting number on a
// client box, where it should be most of them.
const fileless =
stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
const sweepPending = !!stats && !stats.files;
const sweptNote = (() => {
if (!stats) return undefined;
if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
if (stats.sweeping) return 'refreshing…';
if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
return undefined;
})();
return (
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
<h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
{error && (
<div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
Error: {error}
{error}
</div>
)}
<div style={gridStyle}>
{/* Health Card */}
<Tile
label="Files discovered"
value={formatCount(stats?.files?.count)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Library size"
value={formatBytes(stats?.files?.totalSize)}
note={sweptNote}
pending={sweepPending}
/>
<Tile
label="Metadata entries"
value={formatCount(stats?.records)}
note={
fileless === null
? 'records in the index'
: `${formatCount(fileless)} with no local file`
}
/>
<Tile
label="Store keys"
value={formatCount(stats?.redisKeys)}
note={stats?.redisMemory ? `${stats.redisMemory} on disk` : undefined}
/>
<Tile
label="Identities"
value={formatCount(stats?.identities)}
note={
stats?.udlUsers !== null && stats?.udlUsers !== undefined
? `${formatCount(stats.udlUsers)} with user data`
: 'signing accounts'
}
/>
</div>
<div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
<div style={labelStyle}>Service health</div>
{health ? (
<>
<div style={statStyle}>
{health.status === 'ok' ? 'Healthy' : health.status}
</div>
<p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
<p style={noteStyle}>
Metadata store: {health.redis ? 'Open' : 'Unavailable'}
</p>
</>
) : (
<p>Loading...</p>
<p style={noteStyle}>Loading…</p>
)}
</div>
{/* Watchers Status Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
<div style={labelStyle}>File watchers</div>
{watchersData ? (
<>
<div style={statStyle}>{totalFiles}</div>
<p>Files discovered</p>
<p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
<p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
<div style={statStyle}>
{activeWatchers} / {watchersData.count}
</div>
<p style={noteStyle}>active watchers</p>
<p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
{lastScan > 0 && (
<p>Last scan: {new Date(lastScan).toLocaleString()}</p>
<p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
)}
<button
onClick={handleTriggerScan}
@@ -155,27 +275,7 @@ export default function Dashboard() {
</button>
</>
) : (
<p>Loading...</p>
)}
</div>
{/* Services Card */}
<div style={cardStyle}>
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
<div style={statStyle}>{services.length}</div>
<p>Services registered</p>
{services.length > 0 && (
<ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
{services.map((svc) => (
<li key={svc.name} style={{ marginBottom: '0.5rem' }}>
<strong>{svc.name}</strong>
<br />
<span style={{ color: '#888', fontSize: '0.9rem' }}>
{svc.capabilities?.join(', ') || 'No capabilities'}
</span>
</li>
))}
</ul>
<p style={noteStyle}>Loading…</p>
)}
</div>
</div>
+857
View File
@@ -0,0 +1,857 @@
import { useEffect, useRef, useState } from 'react';
import {
INSECURE_CONTEXT_MESSAGE,
canSign,
proveOwnership,
uidFromSecretKey,
type IdentityAction,
} from '../lib/sign';
// Admin console for meta-core's account keystore.
//
// meta-core stores one secp256k1 keypair per file under
// {META_CORE_PATH}/identity/accounts/<uid>.json and has NO notion of a current
// user — which account is "you" is decided by whichever client signs in
// (meta-watch). So this page lists accounts; it never speaks of an active one.
//
// The private key is returned by exactly two calls: POST /api/identity/generate
// (once, at creation) and POST /api/identity/reveal (on demand, for an account
// that already exists). Both land in the same exclusive SecretKeyPanel.
//
// Reveal and remove both require the account's own secret key, entered here and
// used to sign a one-shot challenge locally — the key itself is never sent. So
// this page can enumerate and create, but it holds no authority over an
// existing account that its operator does not already have the key for. There
// is deliberately no admin override: on this node the private key *is* the
// account.
interface Account {
uid: string;
curve: string;
createdAt: number;
}
// What one account holds in the User Data Layer. Absent from
// /api/udl/users/stats means zero, not unknown.
interface Stats {
records: number;
cids: number;
keys: number;
}
// A secret on screen. Carries no createdAt: /api/identity/reveal answers with
// {uid, curve, privateKeyHex} only.
interface SecretKeyView {
uid: string;
privateKeyHex: string;
kind: 'generated' | 'revealed';
}
// Which row has an open confirmation, and for what. One discriminated value
// rather than a pair of ids, so at most one confirm panel can ever be open.
interface Pending {
uid: string;
kind: 'remove' | 'reveal';
}
// Both confirmable operations map to a server-side action name, and the
// challenge is bound to it: a challenge minted to reveal a key cannot delete
// the account. Keeping the mapping in one place stops the two drifting.
const ACTION_FOR: Record<Pending['kind'], IdentityAction> = {
reveal: 'reveal',
remove: 'delete',
};
// meta-watch's sentinel cell for profile records: display name and avatar are
// about the *user*, not a content file, so they live at (uid, "self", ...).
// meta-core stores them opaquely and knows nothing of the convention.
const PROFILE_CID = 'self';
const PROFILE_NAME_KEY = 'profile:name';
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
padding: '1.5rem',
marginBottom: '1rem',
};
const buttonStyle: React.CSSProperties = {
padding: '0.5rem 1rem',
background: '#0f3460',
color: '#fff',
border: 'none',
borderRadius: '4px',
cursor: 'pointer',
marginRight: '0.5rem',
};
const dangerButtonStyle: React.CSSProperties = {
...buttonStyle,
background: '#7f1d1d',
};
const inputStyle: React.CSSProperties = {
width: '100%',
padding: '0.5rem',
marginBottom: '0.5rem',
background: '#1a1a2e',
border: '1px solid #0f3460',
borderRadius: '4px',
color: '#fff',
fontFamily: 'monospace',
};
const monoStyle: React.CSSProperties = {
fontFamily: 'monospace',
fontSize: '0.85rem',
wordBreak: 'break-all',
background: '#0a0a18',
border: '1px solid #0f3460',
padding: '0.75rem',
borderRadius: '4px',
};
const dangerPanelStyle: React.CSSProperties = {
background: '#1f0a0a',
border: '1px solid #7f1d1d',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginTop: '0.75rem',
};
const errorTextStyle: React.CSSProperties = {
color: '#f87171',
background: '#1f0a0a',
padding: '0.5rem 0.75rem',
borderRadius: '4px',
};
function formatCreatedAt(unixSec?: number): string {
if (!unixSec) return '';
try {
return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC');
} catch {
return String(unixSec);
}
}
// Read the JSON body of a non-2xx response if present, fall back to status text.
async function errorMessage(res: Response, fallback: string): Promise<string> {
try {
const j = await res.json();
if (j && (j.message || j.error)) return j.message ?? j.error;
} catch {
// not JSON
}
return `${fallback} (HTTP ${res.status})`;
}
// Copy text, falling back to selecting the element that displays it.
//
// The fallback is load-bearing, not politeness: the dashboard is served over
// plain HTTP, where navigator.clipboard is undefined — without it the copy
// buttons do nothing at all on a real deployment.
async function copyToClipboard(text: string, fallbackElementId: string): Promise<void> {
try {
await navigator.clipboard.writeText(text);
} catch {
const sel = window.getSelection();
const el = document.getElementById(fallbackElementId);
if (sel && el) {
const range = document.createRange();
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
}
}
}
async function fetchAccounts(): Promise<Account[]> {
const res = await fetch('/api/identity/accounts');
if (!res.ok) throw new Error(await errorMessage(res, 'Failed to load accounts'));
const data = (await res.json()) as { accounts?: Account[] };
// Backend order is CreatedAt ascending then uid (identity.List) — stable, so
// render it as given rather than re-sorting here.
return data.accounts ?? [];
}
// Resolve an account's display name from the User Data Layer, or null.
//
// Never throws and never reports an error upward: a name is decoration on top
// of the uid, and the account list must render identically whether the store is up
// (identity itself is file-backed, the UDL is not). The name is the user's
// signed MMUR2 claim (meta-core verified it on write); the uid stays on screen
// as the real identity, and the length cap bounds the layout.
async function fetchDisplayName(uid: string): Promise<string | null> {
const url =
`/api/udl/record?uid=${encodeURIComponent(uid)}` +
`&cid=${encodeURIComponent(PROFILE_CID)}` +
`&key=${encodeURIComponent(PROFILE_NAME_KEY)}`;
try {
const res = await fetch(url);
if (!res.ok) return null; // 503 when storage is down, 500, …
const data = (await res.json()) as { exists?: boolean; claim?: { value?: unknown } | null };
// A retraction (null) or ciphertext ({b64}) has nothing to show.
const value = data.claim?.value;
if (typeof value !== 'string') return null;
const name = value.trim();
return name ? name.slice(0, 64) : null;
} catch {
return null;
}
}
// How much each account holds. One call for the whole page, not one per row:
// meta-core answers it from a keyspace SCAN (there is no per-user cid index),
// so per-row calls would walk the keyspace once per account.
//
// Never throws, for the same reason fetchDisplayName does not: the counts are
// context on top of the uid, and the list must render identically when the store is
// down — accounts are file-backed, their data is not.
//
// Returns null on failure rather than an empty map. The difference is
// load-bearing: an empty map renders as "no stored data" against every account,
// which inside a delete confirmation is a false statement about what is at
// stake. Unknown has to look like unknown.
async function fetchStats(): Promise<Record<string, Stats> | null> {
try {
const res = await fetch('/api/udl/users/stats');
if (!res.ok) return null;
const data = (await res.json()) as { users?: Record<string, Stats> };
return data.users ?? {};
} catch {
return null;
}
}
// The secret-key box shared by the reveal and remove confirmations.
//
// Separate from the surrounding panel so both operations ask the same way, and
// so the key lives in this component's state rather than the page's — it is
// discarded with the panel instead of lingering in a parent that outlives it.
function SecretKeyConfirm({
label,
actionLabel,
busy,
onConfirm,
onCancel,
}: {
label: string;
actionLabel: string;
busy: boolean;
onConfirm: (secretKeyHex: string) => void;
onCancel: () => void;
}) {
const [secret, setSecret] = useState('');
const signable = canSign();
// Derived locally, purely to tell the operator they have the wrong key before
// they hit the button. The server would refuse either way.
const derived = secret.trim() ? uidFromSecretKey(secret) : null;
const mismatch = Boolean(secret.trim() && derived && derived !== label);
return (
<form
onSubmit={(e) => {
e.preventDefault();
onConfirm(secret.trim());
}}
>
<p style={{ color: '#fecaca', margin: '0 0 0.5rem' }}>
Paste this account's secret key to prove it is yours. It is used to sign a one-time
challenge <em>in this browser</em> and is never sent to the server.
</p>
{!signable && <p style={errorTextStyle}>{INSECURE_CONTEXT_MESSAGE}</p>}
<input
style={inputStyle}
type="password"
autoComplete="off"
placeholder="64 hex characters"
value={secret}
onChange={(e) => setSecret(e.target.value)}
disabled={busy || !signable}
autoFocus
/>
{mismatch && (
<p style={{ ...errorTextStyle, marginTop: 0 }}>
That key belongs to a different account ({derived}).
</p>
)}
<button
type="submit"
style={dangerButtonStyle}
disabled={busy || !signable || !secret.trim() || mismatch}
>
{busy ? 'Verifying…' : actionLabel}
</button>
<button type="button" style={buttonStyle} onClick={onCancel} disabled={busy}>
Cancel
</button>
</form>
);
}
// The one place a private key is ever displayed. Rendered exclusively (the
// account list is hidden while it is open) so the "save it now" contract of a
// freshly generated key is not competing with anything, and so the hardcoded
// element id below stays unique.
function SecretKeyPanel({ data, onDismiss }: { data: SecretKeyView; onDismiss: () => void }) {
const [copied, setCopied] = useState(false);
const copy = async () => {
await copyToClipboard(data.privateKeyHex, 'mm-privkey-box');
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
<div
style={{
background: '#7f1d1d',
color: '#fee2e2',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginBottom: '1rem',
}}
>
{data.kind === 'generated' ? (
<>
<strong>This private key is shown ONCE.</strong> Save it now in a password manager.
You can ask for it again later with <em>Reveal private key</em>, but only for as long
as this account exists on this node — delete it and the key is gone for good.
</>
) : (
<>
<strong>This is the live private key for an existing account.</strong> Anyone holding
it can sign as this uid forever, on any machine. There is no revocation and no
rotation. Do not paste it into chat, tickets or screenshots.
</>
)}
</div>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
<div style={monoStyle}>{data.uid}</div>
<p style={{ color: '#9ca3af', margin: '1rem 0 0.5rem' }}>privateKeyHex (secret):</p>
<div id="mm-privkey-box" style={{ ...monoStyle, background: '#1f0a0a', borderColor: '#7f1d1d' }}>
{data.privateKeyHex}
</div>
<div style={{ marginTop: '1rem' }}>
<button style={buttonStyle} onClick={copy}>
{copied ? 'Copied' : 'Copy private key'}
</button>
<button style={dangerButtonStyle} onClick={onDismiss}>
{data.kind === 'generated' ? 'I have saved it' : 'Done — hide it'}
</button>
</div>
</div>
);
}
// Create / import. Always available, not just on an empty keystore: this is an
// admin console, so adding an account to a populated node has to be possible.
function NewAccountPanel({
onGenerated,
onImported,
}: {
onGenerated: (s: SecretKeyView) => void;
onImported: () => void;
}) {
const [busy, setBusy] = useState(false);
const [importing, setImporting] = useState(false);
const [importHex, setImportHex] = useState('');
const [error, setError] = useState<string | null>(null);
const [notice, setNotice] = useState<string | null>(null);
const generate = async () => {
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/generate', { method: 'POST' });
if (!res.ok) {
setError(await errorMessage(res, 'Generate failed'));
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
onGenerated({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'generated' });
} catch (e) {
setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
const importKey = async (e: React.FormEvent) => {
e.preventDefault();
setBusy(true);
setError(null);
setNotice(null);
try {
const res = await fetch('/api/identity/import', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ privateKeyHex: importHex.trim() }),
});
if (!res.ok) {
setError(await errorMessage(res, 'Import failed'));
return;
}
// Import is idempotent — presenting the key IS the proof of ownership, so
// a key this node already holds is a success, not a conflict. `created`
// only decides the wording.
const data = (await res.json()) as { created?: boolean };
setNotice(
data.created
? 'Imported — a new account was added to this keystore.'
: 'That key was already in this keystore; nothing changed.'
);
setImportHex('');
setImporting(false);
onImported();
} catch (e) {
setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
return (
<div style={cardStyle}>
<h3 style={{ marginTop: 0 }}>Add an account</h3>
<p style={{ color: '#9ca3af', marginTop: 0 }}>
Generating mints a fresh secp256k1 keypair here. Importing adds a key created elsewhere —
it is also how an existing account is moved onto this node, so importing a key the
keystore already holds succeeds and changes nothing.
</p>
{error && <p style={errorTextStyle}>{error}</p>}
{notice && (
<p style={{ color: '#9ca3af', background: '#0a0a18', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
{notice}
</p>
)}
<div>
<button style={buttonStyle} disabled={busy} onClick={generate}>
{busy ? 'Working…' : 'Generate new keypair'}
</button>
<button
style={buttonStyle}
disabled={busy}
onClick={() => {
setImporting((x) => !x);
setError(null);
}}
>
{importing ? 'Cancel import' : 'Import existing private key'}
</button>
</div>
{importing && (
<form onSubmit={importKey} style={{ marginTop: '1rem' }}>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>
64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine.
</p>
<input
style={inputStyle}
placeholder="e.g. a3f1…"
value={importHex}
onChange={(e) => setImportHex(e.target.value)}
required
autoFocus
/>
<button type="submit" style={buttonStyle} disabled={busy || !importHex.trim()}>
{busy ? 'Importing…' : 'Import'}
</button>
</form>
)}
</div>
);
}
function AccountCard({
account,
name,
stats,
busy,
disabled,
pending,
error,
onAsk,
onCancel,
onReveal,
onRemove,
}: {
account: Account;
name?: string;
// undefined means "counted, and it holds nothing"; null means "could not be
// counted". The two must not read the same on a confirmation screen.
stats?: Stats | null;
busy: boolean;
disabled: boolean;
pending: Pending | null;
error: string | null;
onAsk: (p: Pending) => void;
onCancel: () => void;
onReveal: (uid: string, secretKeyHex: string) => void;
onRemove: (uid: string, secretKeyHex: string) => void;
}) {
const [copied, setCopied] = useState(false);
const uidBoxId = `mm-uid-${account.uid}`;
const label = name ?? account.uid;
const counted = stats !== null;
const records = stats?.records ?? 0;
const titles = stats?.cids ?? 0;
const copyUid = async () => {
await copyToClipboard(account.uid, uidBoxId);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
};
return (
<div style={cardStyle}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start', gap: '1rem' }}>
<div style={{ minWidth: 0, flex: 1 }}>
<h3 style={{ margin: '0 0 0.5rem', color: name ? '#fff' : '#888' }}>
{name ?? 'Unnamed account'}
</h3>
<div id={uidBoxId} style={monoStyle}>
{account.uid}
</div>
<p style={{ color: '#888', margin: '0.75rem 0 0', fontSize: '0.85rem' }}>
Curve: {account.curve} · Created: {formatCreatedAt(account.createdAt)}
</p>
<p style={{ color: '#888', margin: '0.25rem 0 0', fontSize: '0.85rem' }}>
{/* An account that has never been used legitimately holds nothing, so
zero is a real answer — but only once the count actually ran. */}
{!counted
? 'Stored data: could not be counted (storage unavailable)'
: records === 0
? 'No stored data'
: `${records.toLocaleString()} stored ${records === 1 ? 'record' : 'records'}` +
` across ${titles.toLocaleString()} ${titles === 1 ? 'title' : 'titles'}`}
</p>
</div>
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem', flexShrink: 0 }}>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={copyUid} disabled={disabled}>
{copied ? 'Copied' : 'Copy uid'}
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'reveal' })}
disabled={disabled}
>
Reveal private key
</button>
<button
style={{ ...dangerButtonStyle, marginRight: 0 }}
onClick={() => onAsk({ uid: account.uid, kind: 'remove' })}
disabled={disabled}
>
Remove
</button>
</div>
</div>
{error && <p style={{ ...errorTextStyle, marginBottom: 0 }}>{error}</p>}
{pending?.uid === account.uid && pending.kind === 'reveal' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Show the private key for <strong>{label}</strong>?
</p>
<p style={{ color: '#fecaca', marginTop: 0 }}>
The key <em>is</em> the account. Anyone who copies it can sign records as this uid on
any machine, forever — there is no way to revoke or rotate it afterwards. Because
proving ownership already requires the key, this only confirms that the copy this node
holds matches yours.
</p>
<SecretKeyConfirm
label={account.uid}
actionLabel="Show the private key"
busy={busy}
onConfirm={(secret) => onReveal(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
{pending?.uid === account.uid && pending.kind === 'remove' && (
<div style={dangerPanelStyle}>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Remove <strong>{label}</strong> from this node?
</p>
<ul style={{ color: '#fecaca', paddingLeft: '1.2rem', margin: '0 0 0.75rem' }}>
<li>
<strong>The key is gone for good.</strong> The uid <em>is</em> the public key, and
the private half exists only in this node's keystore. If nobody saved it, nothing —
not you, not meta-core — can ever sign as this uid again. Reveal and copy it first
if there is any doubt.
</li>
<li>
<strong>Nothing is retracted.</strong> Records already signed under this uid keep
verifying on every peer that replicated them. Deleting a key is not a recall.
</li>
<li>
<strong>Its data is destroyed, not orphaned.</strong>{' '}
{!counted
? 'This account\u2019s records could not be counted because storage is unavailable — ' +
'so meta-core will refuse this delete rather than remove the key and strand them.'
: records === 0
? 'This account holds no User Data Layer records, so there is nothing to purge.'
: `All ${records.toLocaleString()} User Data Layer ${
records === 1 ? 'record' : 'records'
} — likes, ratings, watch progress across ${titles.toLocaleString()} ${
titles === 1 ? 'title' : 'titles'
} — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '}
This cannot be undone and there is no backup.
</li>
<li>
<strong>This signs nobody out elsewhere.</strong> Any device that holds this key
still holds it, and can import it back here at any time.
</li>
</ul>
<SecretKeyConfirm
label={account.uid}
actionLabel="Remove this account and all its data"
busy={busy}
onConfirm={(secret) => onRemove(account.uid, secret)}
onCancel={onCancel}
/>
</div>
)}
</div>
);
}
export default function Identity() {
const [accounts, setAccounts] = useState<Account[] | null>(null);
const [names, setNames] = useState<Record<string, string>>({});
const [stats, setStats] = useState<Record<string, Stats> | null>({});
const [notice, setNotice] = useState<string | null>(null);
const [loading, setLoading] = useState(true);
const [fetchError, setFetchError] = useState<string | null>(null);
const [secret, setSecret] = useState<SecretKeyView | null>(null);
const [pending, setPending] = useState<Pending | null>(null);
const [busyUid, setBusyUid] = useState<string | null>(null);
const [rowError, setRowError] = useState<{ uid: string; message: string } | null>(null);
// Bumped per refresh so a slow response from a superseded load cannot
// overwrite a newer one. The page does not poll (unlike the rest of the
// dashboard) — a key list has no business changing under the cursor.
const reqGen = useRef(0);
const refresh = async () => {
const gen = ++reqGen.current;
setLoading(true);
setFetchError(null);
let list: Account[];
try {
list = await fetchAccounts();
} catch (e) {
if (gen !== reqGen.current) return;
setFetchError(e instanceof Error ? e.message : String(e));
setLoading(false);
return;
}
if (gen !== reqGen.current) return;
// Render the list before resolving names, so a down store delays nothing.
setAccounts(list);
setLoading(false);
// Names and counts both come after the list is on screen, and neither can
// fail the page: a down store costs decoration, not the account list.
const [settled, counts] = await Promise.all([
Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
fetchStats(),
]);
if (gen !== reqGen.current) return;
const next: Record<string, string> = {};
settled.forEach((r, i) => {
if (r.status === 'fulfilled' && r.value) next[list[i].uid] = r.value;
});
// Replace rather than merge, so a removed account's name and counts go with
// it instead of lingering against a uid that no longer exists.
setNames(next);
setStats(counts);
};
useEffect(() => {
refresh();
}, []);
const reveal = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.reveal, secretKeyHex);
// POST with a body rather than ?uid=, so neither the target nor the
// signature reaches the proxy's access log. The handler accepts the uid
// either way; the proof must be in the body.
const res = await fetch('/api/identity/reveal', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Reveal failed') });
return;
}
const data = (await res.json()) as { uid: string; privateKeyHex: string };
setPending(null);
setSecret({ uid: data.uid, privateKeyHex: data.privateKeyHex, kind: 'revealed' });
} catch (e) {
setRowError({ uid, message: `Reveal failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const remove = async (uid: string, secretKeyHex: string) => {
setBusyUid(uid);
setRowError(null);
setNotice(null);
try {
const proof = await proveOwnership(uid, ACTION_FOR.remove, secretKeyHex);
// The uid is mandatory: the signature authorises one specific account, so
// the backend refuses an unqualified delete rather than guess a target.
const res = await fetch('/api/identity', {
method: 'DELETE',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ confirm: true, uid, ...proof }),
});
if (!res.ok) {
setRowError({ uid, message: await errorMessage(res, 'Remove failed') });
return;
}
// Report what the purge actually removed. "Account removed" on its own
// would not distinguish a clean delete from one that stranded the data.
const data = (await res.json().catch(() => ({}))) as { purged?: Stats };
const purged = data.purged;
setNotice(
purged && purged.records > 0
? `Account removed. Purged ${purged.records.toLocaleString()} User Data Layer ` +
`${purged.records === 1 ? 'record' : 'records'} across ` +
`${purged.cids.toLocaleString()} ${purged.cids === 1 ? 'title' : 'titles'}.`
: 'Account removed. It held no User Data Layer records.'
);
setPending(null);
await refresh();
} catch (e) {
setRowError({ uid, message: `Remove failed: ${e instanceof Error ? e.message : String(e)}` });
} finally {
setBusyUid(null);
}
};
const dismissSecret = () => {
setSecret(null);
refresh();
};
const ask = (p: Pending) => {
setRowError(null);
setNotice(null);
setPending(p);
};
const showList = !loading && !fetchError && !secret;
return (
// App.tsx's content area is `overflow: hidden`, so every page owns its own
// scrolling (Files.tsx does the same). Without this the account list is
// simply clipped at the viewport with no scrollbar — invisible on a node
// with two accounts, and a dead end on one with fifteen.
//
// The scroller is the outer element and the padding lives inside it, so the
// scrollbar sits at the edge of the content area rather than floating in
// 2rem of whitespace, and `margin: 0 auto` still centres the column.
<div style={{ height: '100%', overflowY: 'auto', minHeight: 0 }}>
<div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
<h1 style={{ marginBottom: 0 }}>Identity</h1>
<button style={{ ...buttonStyle, marginRight: 0 }} onClick={refresh} disabled={loading}>
{loading ? 'Loading…' : 'Refresh'}
</button>
</div>
<p style={{ color: '#9ca3af' }}>
Every signing key in this node's keystore. meta-core has no "current user" — it stores
accounts, and which one is <em>you</em> is decided by whichever client signs in (e.g.
meta-watch). Keys live one JSON file per account under{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/&lt;uid&gt;.json</code>, and are used to sign
User Data Layer records (ratings, watch progress, "liked" flags).
</p>
<div style={{ ...dangerPanelStyle, marginTop: 0, marginBottom: '1rem' }}>
<p style={{ color: '#fecaca', margin: 0 }}>
<strong>The key is the only authority.</strong> Revealing or removing an account requires
that account's secret key, entered here and used to sign a one-time challenge in your
browser — there is no admin override, and holding this page is not one. Listing and
creating accounts are still open to anything that can reach this API, so the perimeter in
front of this port is what keeps the roster private.
</p>
<p style={{ color: '#fecaca', margin: '0.5rem 0 0' }}>
What this does <em>not</em> defend against: anyone with filesystem access to{' '}
<code>{'{META_CORE_PATH}'}/identity/accounts/</code> reads every private key directly.
The keys are stored in plaintext, matching the trust model of the rest of{' '}
<code>/meta-core</code>.
</p>
</div>
{notice && (
<div style={{ ...cardStyle, border: '1px solid #0f3460' }}>
<p style={{ color: '#9ca3af', margin: 0 }}>{notice}</p>
</div>
)}
{loading && <div style={cardStyle}>Loading…</div>}
{fetchError && (
<div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
<p style={{ color: '#f87171', margin: '0 0 1rem' }}>{fetchError}</p>
<button style={buttonStyle} onClick={refresh}>
Retry
</button>
</div>
)}
{!loading && !fetchError && secret && <SecretKeyPanel data={secret} onDismiss={dismissSecret} />}
{showList && (
<>
<NewAccountPanel onGenerated={(s) => setSecret(s)} onImported={refresh} />
<h2 style={{ fontSize: '1.1rem', color: '#9ca3af', margin: '1.5rem 0 1rem' }}>
Accounts ({accounts?.length ?? 0})
</h2>
{accounts && accounts.length === 0 && (
<div style={cardStyle}>
<p style={{ color: '#9ca3af', margin: 0 }}>
No accounts in this keystore. Accounts are normally created by a client when
someone signs up; use <em>Add an account</em> above if you are bootstrapping this
node.
</p>
</div>
)}
{accounts?.map((a) => (
<AccountCard
key={a.uid}
account={a}
name={names[a.uid]}
stats={stats === null ? null : stats[a.uid]}
busy={busyUid === a.uid}
disabled={busyUid !== null}
pending={pending}
error={rowError?.uid === a.uid ? rowError.message : null}
onAsk={ask}
onCancel={() => setPending(null)}
onReveal={reveal}
onRemove={remove}
/>
))}
</>
)}
</div>
</div>
);
}
+220 -42
View File
@@ -1,19 +1,47 @@
import { useEffect, useState } from 'react';
interface MountIOStats {
source: 'rclone';
bytesRead: number;
bytesWritten?: number;
readBps: number;
writeBps?: number;
readOps?: number;
writeOps?: number;
readIops?: number;
writeIops?: number;
transfersPerSec?: number;
lastSampleAt: number;
intervalMs?: number;
}
interface Mount {
id: string;
name: string;
type: 'nfs' | 'smb' | 'rclone';
type: 'smb' | 'rclone';
enabled: boolean;
desiredMounted: boolean;
mountPath: string;
mounted: boolean;
error?: string;
lastChecked: number;
// VFS cache configuration applied at mount time. Empty string means "use
// server default" (50G / 24h / 5m).
cacheMaxSize?: string;
cacheMaxAge?: string;
dirCacheTime?: string;
pollingEnabled?: boolean;
pollingIntervalMs?: number;
pollingActive?: boolean;
lastPolledScan?: number;
// Adaptive scan progress (set by the meta-core poller)
scanning?: boolean;
currentScanStartedAt?: number; // ms; only while scanning
lastScanDurationMs?: number;
nextScanAt?: number; // ms; planned start of next adaptive scan
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
// so the same numbers appear on every mount card.
ioStats?: MountIOStats;
}
interface RcloneRemote {
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
background: '#7f1d1d',
};
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
function formatBytes(n: number): string {
if (!isFinite(n) || n < 0) return '—';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let v = n;
while (v >= 1000 && i < units.length - 1) {
v /= 1000;
i++;
}
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
}
function formatRate(bps: number): string {
return `${formatBytes(bps)}/s`;
}
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
// is just noise.
function formatDuration(ms: number): string {
if (ms < 0 || !isFinite(ms)) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
const remS = s % 60;
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
const h = Math.floor(m / 60);
const remM = m % 60;
return remM ? `${h}h ${remM}m` : `${h}h`;
}
// IOStatsLine renders a single line of live throughput info beneath each
// mount card. Source is always rclone after the rclone-only consolidation,
// so we report file-level transfers/s rather than kernel IOPS.
function IOStatsLine({ stats }: { stats: MountIOStats }) {
const opsLabel = 'transfers/s';
const opsValue = stats.transfersPerSec;
const ageMs = Date.now() - stats.lastSampleAt;
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
return (
<p
style={{
color: stale ? '#6b7280' : '#9ca3af',
fontSize: '0.85rem',
margin: '0.25rem 0 0',
}}
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
>
↓ {formatRate(stats.readBps)}
{opsValue !== undefined && opsValue > 0 && (
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
)}
<> · {formatBytes(stats.bytesRead)} read total</>
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
<> · {formatBytes(stats.bytesWritten)} written</>
)}
</p>
);
}
interface ScanProgressProps {
mount: Mount;
now: number;
}
// Progress bar shows scan ETA when running, an idle row with a countdown to
// the next adaptive scan when waiting. We can't show true % complete (we
// don't know the file count up-front), so the running bar is filled by
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
// it pins at 95% to signal "taking longer than last time".
function ScanProgress({ mount, now }: ScanProgressProps) {
if (!mount.pollingEnabled) return null;
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
const lastDuration = mount.lastScanDurationMs ?? 0;
let fill = 0;
let label = '';
if (isScanning) {
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
if (lastDuration > 0) {
fill = Math.min(0.95, elapsed / lastDuration);
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
} else {
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
}
} else {
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
if (lastDuration > 0 && mount.lastPolledScan) {
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
} else if (mount.nextScanAt) {
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
} else {
label = 'Polling enabled — waiting for first scan';
}
}
return (
<div style={{ marginTop: '0.5rem' }}>
<div
style={{
height: '6px',
background: '#0f3460',
borderRadius: '3px',
overflow: 'hidden',
position: 'relative',
}}
>
<div
style={{
width: `${Math.round(fill * 100)}%`,
height: '100%',
background: isScanning ? '#4ade80' : '#374151',
transition: 'width 0.5s linear',
}}
/>
</div>
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
</div>
);
}
export default function Mounts() {
const [mounts, setMounts] = useState<Mount[]>([]);
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
@@ -50,19 +209,23 @@ export default function Mounts() {
const [showForm, setShowForm] = useState(false);
const [formData, setFormData] = useState({
name: '',
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
nfsServer: '',
nfsPath: '',
type: 'smb' as 'smb' | 'rclone',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
const [scanning, setScanning] = useState<string | null>(null);
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
// refetches (which run every 5s).
const [now, setNow] = useState<number>(() => Date.now());
const fetchMounts = async () => {
try {
@@ -94,7 +257,11 @@ export default function Mounts() {
fetchMounts();
fetchRemotes();
const interval = setInterval(fetchMounts, 5000);
return () => clearInterval(interval);
const tick = setInterval(() => setNow(Date.now()), 1000);
return () => {
clearInterval(interval);
clearInterval(tick);
};
}, []);
const handleMount = async (id: string) => {
@@ -143,10 +310,7 @@ export default function Mounts() {
type: formData.type,
};
if (formData.type === 'nfs') {
body.nfsServer = formData.nfsServer;
body.nfsPath = formData.nfsPath;
} else if (formData.type === 'smb') {
if (formData.type === 'smb') {
body.smbServer = formData.smbServer;
body.smbShare = formData.smbShare;
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
@@ -156,6 +320,12 @@ export default function Mounts() {
body.rclonePath = formData.rclonePath;
}
// Cache knobs — only forward non-empty values; the server applies defaults
// when fields are omitted.
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
// Add polling configuration
body.pollingEnabled = formData.pollingEnabled;
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
@@ -171,16 +341,17 @@ export default function Mounts() {
setShowForm(false);
setFormData({
name: '',
type: 'nfs',
nfsServer: '',
nfsPath: '',
type: 'smb',
smbServer: '',
smbShare: '',
smbUsername: '',
smbPassword: '',
rcloneRemote: '',
rclonePath: '',
pollingEnabled: false,
cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000,
});
fetchMounts();
@@ -231,32 +402,15 @@ export default function Mounts() {
<select
style={inputStyle}
value={formData.type}
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
onChange={(e) => {
const type = e.target.value as 'smb' | 'rclone';
setFormData({ ...formData, type });
}}
>
<option value="nfs">NFS</option>
<option value="smb">SMB/CIFS</option>
<option value="rclone">rclone</option>
<option value="smb">SMB (via rclone)</option>
<option value="rclone">rclone (pre-defined remote)</option>
</select>
{formData.type === 'nfs' && (
<>
<input
style={inputStyle}
placeholder="NFS Server (e.g., 192.168.1.100)"
value={formData.nfsServer}
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
required
/>
<input
style={inputStyle}
placeholder="NFS Path (e.g., /export/media)"
value={formData.nfsPath}
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
required
/>
</>
)}
{formData.type === 'smb' && (
<>
<input
@@ -313,6 +467,30 @@ export default function Mounts() {
</>
)}
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
VFS cache (leave blank for defaults: 50G / 24h / 5m)
</p>
<input
style={inputStyle}
placeholder="Cache max size (e.g., 50G)"
value={formData.cacheMaxSize}
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Cache max age (e.g., 24h)"
value={formData.cacheMaxAge}
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Dir cache time (e.g., 5m)"
value={formData.dirCacheTime}
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
/>
</div>
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
<input
@@ -362,17 +540,17 @@ export default function Mounts() {
</span>
</p>
{mount.pollingEnabled && (
<p style={{ color: '#888', fontSize: '0.9rem' }}>
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
Polling:{' '}
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
{mount.pollingActive ? 'Active' : 'Inactive'}
</span>
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{mount.lastPolledScan && (
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
)}
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{' • Adaptive 10× idle'}
</p>
)}
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
<ScanProgress mount={mount} now={now} />
{mount.error && (
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
)}
+406
View File
@@ -0,0 +1,406 @@
import { useCallback, useEffect, useRef, useState } from 'react';
import { getJSON } from '../lib/api';
import {
blocks,
buildLabel,
deviceLine,
dump,
headline,
type Report,
type ReportSummary,
shortCid,
subject,
} from '../lib/reportText';
// The bug-report / feedback inbox. Client apps (meta-watch, meta-read, …)
// forward what their 🐛 buttons file to POST /api/reports; this page lists,
// reads and triages them. See internal/reports for storage and bounds.
type Status = 'open' | 'triaged' | 'closed';
const STATUSES: Status[] = ['open', 'triaged', 'closed'];
interface ListPage {
reports: ReportSummary[];
apps: Record<string, number>;
counts: Record<string, number>;
more: boolean;
}
const card: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
padding: '1rem',
display: 'flex',
flexDirection: 'column',
overflow: 'hidden',
minHeight: 0,
};
const chip = (active: boolean): React.CSSProperties => ({
padding: '0.25rem 0.7rem',
borderRadius: '999px',
border: '1px solid #0f3460',
background: active ? '#0f3460' : 'transparent',
color: active ? '#fff' : '#aaa',
cursor: 'pointer',
fontSize: '0.85rem',
});
const button: React.CSSProperties = {
padding: '0.35rem 0.8rem',
borderRadius: '4px',
border: '1px solid #0f3460',
background: '#0f3460',
color: '#fff',
cursor: 'pointer',
fontSize: '0.85rem',
};
const mono: React.CSSProperties = { fontFamily: 'ui-monospace, Menlo, Consolas, monospace', fontSize: '0.8rem' };
const statusColor: Record<Status, string> = { open: '#b45309', triaged: '#1d4ed8', closed: '#374151' };
function StatusBadge({ s }: { s: Status }) {
return (
<span style={{ background: statusColor[s], color: '#fff', borderRadius: '3px', padding: '0.05rem 0.4rem', fontSize: '0.72rem' }}>
{s}
</span>
);
}
const fmtTime = (ms: number) => new Date(ms).toLocaleString();
export default function Reports() {
const [app, setApp] = useState('');
const [status, setStatus] = useState<Status | ''>('open');
const [q, setQ] = useState('');
const [page, setPage] = useState<ListPage | null>(null);
const [older, setOlder] = useState<ReportSummary[]>([]);
const [more, setMore] = useState(false);
const [error, setError] = useState<string | null>(null);
const [selected, setSelected] = useState<number | null>(null);
const query = useCallback(
(before?: number) => {
const p = new URLSearchParams();
if (app) p.set('app', app);
if (status) p.set('status', status);
if (q.trim()) p.set('q', q.trim());
if (before) p.set('before', String(before));
return `/api/reports?${p}`;
},
[app, status, q]
);
// The newest page is polled; pages loaded with "older" are kept beneath it
// until a filter changes, and then they own the "more" flag.
const olderCount = useRef(0);
olderCount.current = older.length;
const refresh = useCallback(async () => {
try {
const data = await getJSON<ListPage>(query());
setPage(data);
if (olderCount.current === 0) setMore(data.more);
setError(null);
} catch (e) {
setError(String((e as Error).message || e));
}
}, [query]);
useEffect(() => {
setOlder([]);
setMore(false);
}, [app, status, q]);
useEffect(() => {
refresh();
const t = setInterval(refresh, 10000);
return () => clearInterval(t);
}, [refresh]);
const loadOlder = async () => {
const all = [...(page?.reports || []), ...older];
const last = all[all.length - 1];
if (!last) return;
try {
const data = await getJSON<ListPage>(query(last.id));
setOlder((prev) => [...prev, ...data.reports]);
setMore(data.more);
} catch (e) {
setError(String((e as Error).message || e));
}
};
const rows = [...(page?.reports || []), ...older.filter((o) => !(page?.reports || []).some((r) => r.id === o.id))];
const apps = Object.entries(page?.apps || {}).sort((a, b) => a[0].localeCompare(b[0]));
const total = Object.values(page?.counts || {}).reduce((n, c) => n + c, 0);
return (
<div style={{ padding: '1rem', height: '100%', display: 'flex', flexDirection: 'column', overflow: 'hidden', boxSizing: 'border-box' }}>
<div style={{ display: 'flex', alignItems: 'baseline', gap: '1rem', flexShrink: 0, flexWrap: 'wrap' }}>
<h1 style={{ margin: '0 0 0.75rem 0' }}>Reports</h1>
<span style={{ color: '#888', fontSize: '0.9rem' }}>bug reports and feedback filed from the apps' 🐛 buttons</span>
</div>
<div style={{ display: 'flex', gap: '0.5rem', flexWrap: 'wrap', alignItems: 'center', marginBottom: '0.75rem', flexShrink: 0 }}>
<span style={chip(status === '')} onClick={() => setStatus('')}>all {total}</span>
{STATUSES.map((s) => (
<span key={s} style={chip(status === s)} onClick={() => setStatus(s)}>
{s} {page?.counts?.[s] ?? 0}
</span>
))}
<span style={{ width: '1rem' }} />
<span style={chip(app === '')} onClick={() => setApp('')}>every app</span>
{apps.map(([a, n]) => (
<span key={a} style={chip(app === a)} onClick={() => setApp(a)}>
{a} {n}
</span>
))}
<input
value={q}
onChange={(e) => setQ(e.target.value)}
placeholder="search note, title, cid, uid"
style={{ marginLeft: 'auto', padding: '0.35rem 0.6rem', borderRadius: '4px', border: '1px solid #0f3460', background: '#0b1630', color: '#eee', minWidth: '16rem' }}
/>
</div>
{error && <div style={{ color: '#f87171', marginBottom: '0.5rem', flexShrink: 0 }}>Could not load reports: {error}</div>}
<div style={{ display: 'grid', gridTemplateColumns: 'minmax(22rem, 2fr) 3fr', gap: '1rem', flex: 1, minHeight: 0 }}>
<div style={card}>
{rows.length === 0 ? (
<p style={{ color: '#888', margin: 0 }}>{page ? 'No reports match.' : 'Loading…'}</p>
) : (
<div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
{rows.map((r) => (
<div
key={r.id}
onClick={() => setSelected(r.id)}
style={{
padding: '0.55rem 0.5rem',
borderBottom: '1px solid #0f3460',
cursor: 'pointer',
background: selected === r.id ? '#0f3460' : 'transparent',
}}
>
<div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', fontSize: '0.8rem', color: '#888' }}>
<span style={mono}>#{r.id}</span>
<StatusBadge s={r.status} />
<span>{r.app}</span>
<span style={{ marginLeft: 'auto' }}>{fmtTime(r.ts)}</span>
</div>
<div style={{ margin: '0.2rem 0', color: '#eee' }}>{subject(r)}</div>
<div style={{ fontSize: '0.85rem', color: r.note ? '#ddd' : '#666' }}>
{r.note ? `“${r.note.length > 120 ? r.note.slice(0, 119) + '…' : r.note}”` : 'no note'}
</div>
<div style={{ ...mono, color: '#666', marginTop: '0.15rem' }}>
{buildLabel(r) || 'build unknown'}
{r.uid ? ` · ${shortCid(r.uid)}` : ''}
{r.log_lines ? ` · ${r.log_lines} log lines` : ''}
</div>
</div>
))}
{more && (
<div style={{ padding: '0.75rem', textAlign: 'center' }}>
<button style={button} onClick={loadOlder}>Load older</button>
</div>
)}
</div>
)}
</div>
<div style={card}>
{selected == null ? (
<p style={{ color: '#888', margin: 0 }}>Pick a report to read it.</p>
) : (
<ReportDetail
key={selected}
id={selected}
onChanged={refresh}
onDeleted={() => {
setSelected(null);
setOlder((prev) => prev.filter((r) => r.id !== selected));
refresh();
}}
/>
)}
</div>
</div>
</div>
);
}
function ReportDetail({ id, onChanged, onDeleted }: { id: number; onChanged: () => void; onDeleted: () => void }) {
const [r, setR] = useState<Report | null>(null);
const [error, setError] = useState<string | null>(null);
const [note, setNote] = useState('');
const [busy, setBusy] = useState(false);
const [copied, setCopied] = useState(false);
const copyTimer = useRef<number>();
useEffect(() => {
getJSON<Report>(`/api/reports/${id}`)
.then((rep) => {
setR(rep);
setNote(rep.triage_note || '');
})
.catch((e) => setError(String(e.message || e)));
return () => window.clearTimeout(copyTimer.current);
}, [id]);
const patch = async (change: { status?: Status; triage_note?: string }) => {
setBusy(true);
try {
const rep = await getJSON<Report>(`/api/reports/${id}`, {
method: 'PATCH',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(change),
});
setR(rep);
setNote(rep.triage_note || '');
setError(null);
onChanged();
} catch (e) {
setError(String((e as Error).message || e));
} finally {
setBusy(false);
}
};
const remove = async () => {
if (!window.confirm(`Delete report #${id}? This cannot be undone.`)) return;
setBusy(true);
try {
const res = await fetch(`/api/reports/${id}`, { method: 'DELETE' });
if (!res.ok && res.status !== 404) throw new Error(`/api/reports/${id} → HTTP ${res.status}`);
onDeleted();
} catch (e) {
setError(String((e as Error).message || e));
setBusy(false);
}
};
const copy = async () => {
if (!r) return;
try {
await navigator.clipboard.writeText(dump(r));
setCopied(true);
copyTimer.current = window.setTimeout(() => setCopied(false), 1800);
} catch {
setError('Clipboard unavailable here — select the text below and copy it by hand.');
}
};
if (error && !r) return <p style={{ color: '#f87171', margin: 0 }}>Could not load report #{id}: {error}</p>;
if (!r) return <p style={{ color: '#888', margin: 0 }}>Loading #{id}…</p>;
const body = r.body || {};
const bits = headline(r, body);
const device = deviceLine(body.device);
const log: string[] = Array.isArray(body.log) ? body.log : [];
return (
<div style={{ flex: 1, overflow: 'auto', minHeight: 0 }}>
<div style={{ display: 'flex', gap: '0.5rem', alignItems: 'center', flexWrap: 'wrap' }}>
<h2 style={{ margin: 0, fontSize: '1.15rem' }}>#{r.id} · {subject(r, body)}</h2>
<StatusBadge s={r.status} />
<span style={{ marginLeft: 'auto', display: 'flex', gap: '0.4rem' }}>
{STATUSES.filter((s) => s !== r.status).map((s) => (
<button key={s} style={button} disabled={busy} onClick={() => patch({ status: s })}>
Mark {s}
</button>
))}
<button style={button} onClick={copy}>{copied ? 'Copied' : 'Copy as text'}</button>
<button style={{ ...button, background: '#7f1d1d', borderColor: '#7f1d1d' }} disabled={busy} onClick={remove}>
Delete
</button>
</span>
</div>
{error && <div style={{ color: '#f87171', marginTop: '0.5rem' }}>{error}</div>}
<div style={{ color: '#888', fontSize: '0.85rem', margin: '0.4rem 0 0.75rem' }}>
{r.app} · filed {fmtTime(r.ts)}
{r.received - r.ts > 60000 ? ` · delivered ${fmtTime(r.received)}` : ''}
{r.uid ? ` · uid ${r.uid}` : ''}
</div>
{bits.length > 0 && (
<div style={{ marginBottom: '0.75rem', lineHeight: 1.6 }}>
{bits.map((b, i) => (
<span key={i}>
{i > 0 && <span style={{ color: '#555' }}> · </span>}
{b.strong ? <b style={{ color: '#fff' }}>{b.text}</b> : <span style={{ color: '#ccc' }}>{b.text}</span>}
</span>
))}
</div>
)}
<Field label="Page">
{r.page || '—'}
{r.route && <span style={{ ...mono, color: '#888' }}> {r.route}</span>}
</Field>
{(r.title || r.cid) && (
<Field label="Content">
{r.title}
{r.cid && <span style={{ ...mono, color: '#888' }}> {r.cid}</span>}
</Field>
)}
{device && <Field label="Device">{device}</Field>}
{r.build && (
<Field label="Build">
<pre style={{ ...mono, margin: 0, whiteSpace: 'pre-wrap', color: '#ccc' }}>{r.build}</pre>
</Field>
)}
<Field label="Triage">
<div style={{ display: 'flex', gap: '0.5rem' }}>
<input
value={note}
onChange={(e) => setNote(e.target.value)}
placeholder="e.g. Deck #812, duplicate of #40, fixed in 1.0.95"
style={{ flex: 1, padding: '0.35rem 0.6rem', borderRadius: '4px', border: '1px solid #0f3460', background: '#0b1630', color: '#eee' }}
/>
<button style={button} disabled={busy || note === (r.triage_note || '')} onClick={() => patch({ triage_note: note })}>
Save
</button>
</div>
</Field>
{blocks(body)
.filter(([, v]) => v === null || typeof v !== 'object')
.map(([k, v]) => (
<Field key={k} label={k}>
<span style={mono}>{String(v)}</span>
</Field>
))}
{blocks(body).filter(([, v]) => v !== null && typeof v === 'object').map(([k, v]) => (
<details key={k} open={k === 'context' || k === 'reader'} style={{ marginTop: '0.6rem' }}>
<summary style={{ cursor: 'pointer', color: '#9ab' }}>{k}</summary>
<pre style={{ ...mono, background: '#0b1630', padding: '0.6rem', borderRadius: '4px', whiteSpace: 'pre-wrap', wordBreak: 'break-word', margin: '0.3rem 0 0' }}>
{typeof v === 'string' ? v : JSON.stringify(v, null, 2)}
</pre>
</details>
))}
{log.length > 0 && (
<details open style={{ marginTop: '0.6rem' }}>
<summary style={{ cursor: 'pointer', color: '#9ab' }}>log ({log.length} lines)</summary>
<pre style={{ ...mono, background: '#0b1630', padding: '0.6rem', borderRadius: '4px', whiteSpace: 'pre-wrap', wordBreak: 'break-word', margin: '0.3rem 0 0', maxHeight: '28rem', overflow: 'auto' }}>
{log.join('\n')}
</pre>
</details>
)}
</div>
);
}
function Field({ label, children }: { label: string; children: React.ReactNode }) {
return (
<div style={{ display: 'grid', gridTemplateColumns: '5.5rem 1fr', gap: '0.5rem', margin: '0.3rem 0', alignItems: 'baseline' }}>
<span style={{ color: '#888', fontSize: '0.85rem' }}>{label}</span>
<div>{children}</div>
</div>
);
}
+2 -2
View File
@@ -9,8 +9,8 @@ set -e
# Create required directories
mkdir -p /meta-core/locks
mkdir -p /meta-core/db/redis
mkdir -p /meta-core/services
mkdir -p /meta-core/db
mkdir -p /meta-core/db-cache
mkdir -p /meta-core/mounts/errors
mkdir -p /files
mkdir -p /var/log/supervisor
+8 -29
View File
@@ -2,9 +2,14 @@
# =============================================================================
# Leader Election Script for meta-core
#
# Uses flock for leader election. Only the leader runs the full application
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
# Followers remain dormant, periodically retrying to acquire the lock.
# The flock here is a MUTEX, not service discovery. It is the only thing
# stopping two containers that mount the same /meta-core volume from putting
# two writers on one database file (/meta-core/db/meta-core.db). Sibling
# services no longer read anything from this directory — they locate meta-core
# over UDP (beacon v2, see docs/project-architecture/beacon-v2.md), so the
# kv-leader.info file
# this script used to publish is gone. Keep the lock; it costs nothing and the
# corruption it prevents is unrecoverable.
#
# Environment variables:
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
@@ -20,7 +25,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
# Get the container's IP address
@@ -40,35 +44,13 @@ get_local_ip() {
echo "$ip"
}
# Write leader info file (plain text API URL)
write_leader_info() {
local ip=$(get_local_ip)
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
# Write atomically using temp file + rename
local temp_file="${INFO_FILE}.tmp"
echo "$api_url" > "$temp_file"
mv "$temp_file" "$INFO_FILE"
echo "[election] Wrote leader info: $api_url"
}
# Clean up leader info file on shutdown
cleanup() {
echo "[election] Cleaning up..."
rm -f "$INFO_FILE"
exit 0
}
# Set up signal handlers for graceful shutdown
trap cleanup SIGTERM SIGINT
# Ensure lock directory exists
mkdir -p "$(dirname "$LOCK_FILE")"
echo "[election] Starting leader election..."
echo "[election] Lock file: $LOCK_FILE"
echo "[election] Info file: $INFO_FILE"
# Open lock file for flock (file descriptor 200)
exec 200>"$LOCK_FILE"
@@ -79,9 +61,6 @@ while true; do
if flock -n 200; then
echo "[election] Acquired flock - transitioning to LEADER"
# Write leader info before starting services
write_leader_info
# exec supervisord - replaces this process
# When supervisord exits, the flock is released automatically
echo "[election] Starting supervisord..."
+135 -167
View File
@@ -2,11 +2,17 @@
# =============================================================================
# Mount Watcher Script
#
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
# Runs as a supervisord program, polling every 5 seconds.
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
# rclone RC daemon. After the rclone-only consolidation, this script no
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
# through rclone's mount/mount endpoint.
#
# Uses findmnt to check actual mount status (source of truth).
# Writes errors to /meta-core/mounts/errors/{id}.error
# Mount types:
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
# - "rclone" → references a pre-defined remote in rclone.conf
#
# All mounts are read-only by construction (no API surface to override).
# Errors surface in /meta-core/mounts/errors/{id}.error
# =============================================================================
set -u
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
FILES_PATH="${FILES_PATH:-/files}"
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
# Fall-back VFS cache settings — applied when a mount entry leaves the field
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
# packages/meta-core/internal/mounts/types.go.
DEFAULT_CACHE_MAX_SIZE="50G"
DEFAULT_CACHE_MAX_AGE="24h"
DEFAULT_DIR_CACHE_TIME="5m"
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
# ANSI colors
RED='\033[0;31m'
GREEN='\033[0;32m'
@@ -54,9 +70,7 @@ init_dirs() {
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
is_mounted() {
local mount_path="$1"
# Remove trailing slash for consistent matching
local clean_path="${mount_path%/}"
# Check if the path appears as a mount point in /proc/mounts (field 2)
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
}
@@ -74,145 +88,102 @@ write_error() {
echo "$error" >> "$ERROR_DIR/$id.error"
}
# Mount NFS share (always read-only)
do_mount_nfs() {
local id="$1"
local server="$2"
local path="$3"
local mount_path="$4"
local options="${5:-}"
# Build the rclone remote source string for a mount entry. Echoes the result.
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
# the optional remote path.
build_rclone_fs() {
local type="$1"
local smb_server="$2"
local smb_share="$3"
local smb_username="$4"
local smb_password_obscured="$5"
local smb_domain="$6"
local rclone_remote="$7"
local rclone_path="$8"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
local rc=$?
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "NFS mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "NFS mount failed: $output"
case "$type" in
smb)
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
# embed without quoting.
local opts="host=${smb_server}"
if [ -n "$smb_username" ]; then
opts="${opts},user=${smb_username}"
fi
return $rc
if [ -n "$smb_password_obscured" ]; then
opts="${opts},pass=${smb_password_obscured}"
fi
if [ -n "$smb_domain" ]; then
opts="${opts},domain=${smb_domain}"
fi
echo ":smb,${opts}:${smb_share}"
;;
rclone)
echo "${rclone_remote}:${rclone_path}"
;;
*)
echo ""
;;
esac
}
# Mount SMB/CIFS share (always read-only)
do_mount_smb() {
local id="$1"
local server="$2"
local share="$3"
local mount_path="$4"
local username="${5:-}"
local password_obscured="${6:-}"
local domain="${7:-}"
local options="${8:-}"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
# If we have credentials, reveal password and build options
if [ -n "$username" ]; then
mount_opts="${mount_opts},username=${username}"
if [ -n "$password_obscured" ]; then
# Reveal password using rclone (only exists in memory briefly)
local password
password=$(rclone reveal "$password_obscured" 2>/dev/null)
if [ -n "$password" ]; then
mount_opts="${mount_opts},password=${password}"
fi
fi
if [ -n "$domain" ]; then
mount_opts="${mount_opts},domain=${domain}"
fi
fi
# Add any additional options
if [ -n "$options" ]; then
if [ -n "$mount_opts" ]; then
mount_opts="${mount_opts},${options}"
else
mount_opts="$options"
fi
fi
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
# Execute mount directly (not via eval) to handle special chars in password
local output
local rc
if [ -n "$mount_opts" ]; then
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
rc=$?
else
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
rc=$?
fi
# Clear password from any variables (best effort)
password=""
mount_opts=""
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "SMB mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "SMB mount failed: $output"
fi
return $rc
}
# Mount rclone remote using RC API (always read-only)
# Mount via rclone RC API (always read-only).
do_mount_rclone() {
local id="$1"
local remote="$2"
local remote_path="$3"
local mount_path="$4"
local options="${5:-}"
local fs="$2"
local mount_path="$3"
local cache_max_size="$4"
local cache_max_age="$5"
local dir_cache_time="$6"
mkdir -p "$mount_path"
# Build the remote source string
local fs="${remote}:${remote_path}"
# Apply defaults for any blank cache fields.
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
# Use rclone RC API to mount (always read-only for safety)
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
# Build JSON body for rclone mount with read-only option
# Build JSON body via python3 — embedding the obscured password and
# display fs string into a hand-rolled heredoc is asking for an injection
# bug the next time we add a field with a quote in it.
local json_body
json_body=$(cat <<EOF
{
"fs": "${fs}",
"mountPoint": "${mount_path}",
json_body=$(python3 -c '
import json, sys
print(json.dumps({
"fs": sys.argv[1],
"mountPoint": sys.argv[2],
"mountOpt": {
"AllowOther": true,
"ReadOnly": true
"AllowOther": True,
"ReadOnly": True,
},
"vfsOpt": {
"CacheMode": 2,
"ReadOnly": true
}
}
EOF
)
# CacheMode 3 = "full" — cache both reads and writes on local disk.
# CacheMode 2 ("writes") would silently bypass the cache for reads,
# which is the opposite of what we want for a read-only mount.
"CacheMode": 3,
"CacheMaxSize": sys.argv[3],
"CacheMaxAge": sys.argv[4],
"DirCacheTime": sys.argv[5],
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
# throughput in isolation but, under the 4+ concurrent reads from
# the processing pipeline, multiplies into 16+ simultaneous SMB
# requests and tanks aggregate throughput. Left at default — rely on
# file-level concurrency from the pipeline for parallelism instead.
"ReadOnly": True,
},
}))
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
local output
output=$(curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-u "$RCLONE_RC_AUTH" \
-d "$json_body" \
"http://127.0.0.1:5572/mount/mount" 2>&1)
"${RCLONE_RC_URL}/mount/mount" 2>&1)
# Wait a moment for mount to establish
sleep 2
if is_mounted "$mount_path"; then
@@ -221,7 +192,6 @@ EOF
return 0
else
local error_msg="rclone mount failed"
# Try to extract error from response
if echo "$output" | grep -q "error"; then
error_msg="$output"
fi
@@ -231,36 +201,28 @@ EOF
fi
}
# Unmount a path
# Unmount a path. Always goes through the rclone RC API now — every mount we
# create lives behind rclone's FUSE driver.
do_unmount() {
local id="$1"
local mount_path="$2"
local type="$3"
if ! is_mounted "$mount_path"; then
log "Already unmounted: $mount_path"
return 0
fi
if [ "$type" = "rclone" ]; then
# Use rclone RC API to unmount
log "Unmounting rclone: $mount_path"
log "Unmounting: $mount_path"
curl -s -X POST \
-H "Content-Type: application/json" \
-u admin:admin \
-u "$RCLONE_RC_AUTH" \
-d "{\"mountPoint\":\"${mount_path}\"}" \
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
else
log "Unmounting: $mount_path"
umount "$mount_path" 2>&1
fi
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
# Wait a moment
sleep 1
# Check if still mounted
# Fallback to lazy umount in case the FUSE process is wedged.
if is_mounted "$mount_path"; then
# Force unmount with lazy option
log_warn "Force unmounting (lazy): $mount_path"
umount -l "$mount_path" 2>&1
sleep 1
@@ -277,12 +239,11 @@ do_unmount() {
# Process all mounts from config
process_mounts() {
# Check if config file exists
if [ ! -f "$CONFIG_FILE" ]; then
return
fi
# Parse JSON using python3
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
local mounts
mounts=$(python3 -c '
import sys, json
@@ -294,15 +255,12 @@ except:
sys.exit(0)
for m in data.get("mounts", []):
# Output pipe-separated values
fields = [
m.get("id", ""),
m.get("type", ""),
str(m.get("enabled", True)),
str(m.get("desiredMounted", False)),
m.get("mountPath", ""),
m.get("nfsServer", ""),
m.get("nfsPath", ""),
m.get("smbServer", ""),
m.get("smbShare", ""),
m.get("smbUsername", ""),
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
m.get("smbDomain", ""),
m.get("rcloneRemote", ""),
m.get("rclonePath", ""),
m.get("options", "")
m.get("cacheMaxSize", ""),
m.get("cacheMaxAge", ""),
m.get("dirCacheTime", ""),
]
print("|".join(fields))
' 2>/dev/null)
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
return
fi
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
while IFS='|' read -r id type enabled desired mount_path \
smb_server smb_share smb_username smb_password_obscured smb_domain \
rclone_remote rclone_path \
cache_max_size cache_max_age dir_cache_time; do
# Skip empty lines
[ -z "$id" ] && continue
# Check actual mount status
# Skip retired/legacy types — the Go-side migration disables NFS but a
# stale entry could still arrive here on a downgrade/upgrade race.
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
continue
fi
local is_mounted_now=false
is_mounted "$mount_path" && is_mounted_now=true
# Handle disabled mounts - unmount if currently mounted
# Disabled → unmount if currently up
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) disabled, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
continue
fi
# Handle desired state
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
# Should be mounted
if [ "$is_mounted_now" = false ]; then
local fs
fs=$(build_rclone_fs "$type" \
"$smb_server" "$smb_share" "$smb_username" \
"$smb_password_obscured" "$smb_domain" \
"$rclone_remote" "$rclone_path")
if [ -z "$fs" ]; then
log_error "Could not build rclone fs spec for mount $id (type=$type)"
write_error "$id" "Invalid mount config: missing required fields for type=$type"
continue
fi
log "Mount $id ($mount_path) desired but not mounted, mounting..."
case "$type" in
nfs)
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
;;
smb)
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
;;
rclone)
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
;;
*)
log_error "Unknown mount type: $type"
;;
esac
do_mount_rclone "$id" "$fs" "$mount_path" \
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
fi
else
# Should be unmounted
if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) not desired, unmounting..."
do_unmount "$id" "$mount_path" "$type"
do_unmount "$id" "$mount_path"
fi
fi
done <<< "$mounts"
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
# Main loop
main() {
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
init_dirs
+19 -8
View File
@@ -70,6 +70,9 @@ http {
proxy_buffering off;
proxy_cache off;
proxy_read_timeout 86400s;
# Snapshot import accepts multi-MB ZIP uploads.
client_max_body_size 0;
}
# =========================================================================
@@ -120,14 +123,10 @@ http {
# =========================================================================
# Service discovery
# =========================================================================
location /services {
proxy_pass http://meta-core-api/services;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
location /services/ {
proxy_pass http://meta-core-api/services/;
# meta-discovery v1 neighbours (supersedes /services, which stays
# while consumers migrate).
location /neighbors {
proxy_pass http://meta-core-api/neighbors;
proxy_http_version 1.1;
proxy_set_header Host $host;
}
@@ -192,6 +191,18 @@ http {
# WebDAV HTML browser (for iframe embedding in dashboard)
# Returns HTML directory listings instead of JSON
# =========================================================================
# At the browser root, strip the "../" parent-dir row that autoindex
# otherwise emits — clicking it inside the dashboard iframe would
# navigate to "/" (the dashboard itself), nesting the page inside itself.
location = /webdav-browser/ {
alias /files/;
autoindex on;
charset utf-8;
sub_filter '<a href="../">../</a>' '';
sub_filter_once on;
}
location /webdav-browser/ {
alias /files/;
autoindex on;
+5 -19
View File
@@ -16,26 +16,13 @@ serverurl=unix:///var/run/supervisor.sock
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
# =============================================================================
# Redis: Key-value store (managed by supervisord, started before meta-core)
# Previously spawned by Go code, now managed by bash-based leader election
# =============================================================================
[program:redis]
command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
priority=1
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
stdout_logfile_maxbytes=0
stderr_logfile=/dev/stderr
stderr_logfile_maxbytes=0
startsecs=2
# =============================================================================
# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
# meta-core: HTTP API, Mount API, File Watcher, metadata store (embedded SQLite)
# =============================================================================
[program:meta-core]
command=/usr/local/bin/meta-core
priority=10
; Room to finish in-flight writes and checkpoint the WAL on shutdown.
stopwaitsecs=60
autostart=true
autorestart=true
stdout_logfile=/dev/stdout
@@ -50,7 +37,6 @@ environment=
SERVICE_VERSION="%(ENV_SERVICE_VERSION)s",
META_CORE_HTTP_PORT="%(ENV_META_CORE_HTTP_PORT)s",
META_CORE_HTTP_HOST="%(ENV_META_CORE_HTTP_HOST)s",
REDIS_PORT="%(ENV_REDIS_PORT)s",
WATCH_INTERVAL_MS="%(ENV_WATCH_INTERVAL_MS)s",
API_PORT="80",
BASE_URL="%(ENV_BASE_URL)s"
@@ -71,7 +57,7 @@ startsecs=2
# rclone: Remote storage mount daemon (RC API on port 5572)
# =============================================================================
[program:rclone]
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
priority=30
autostart=true
autorestart=true
@@ -80,7 +66,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
startsecs=3
# =============================================================================
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
# =============================================================================
[program:mount-watcher]
command=/app/scripts/mount-watcher.sh
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+290 -559
View File
@@ -1,595 +1,326 @@
# Redis Metadata Storage Architecture
# Metadata Storage Structure
This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling.
How meta-core lays out metadata on disk since **v2.0.0 (claims model)**: the
two SQLite databases, their tables, how a record is assembled from signed
votes, and the schema-version gate. Implementation basis:
[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md).
This document does **not** enumerate field semantics — that is the job of the
repo-root [`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative
key registry for everyone who writes or reads metadata.
## Overview
Related: why rids are UUIDs and hidden —
[`uuid-rooted-metadata.md`](uuid-rooted-metadata.md); why the engine is
shaped this way, its guarantees and measurements —
[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md);
the external-access contract (HTTP-only, SSE for events) —
[`api-mediated-access.md`](api-mediated-access.md).
MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables:
> **History.** Until 2026-10 metadata lived in Redis; v1.1.0–v1.2.x kept the
> same flat keys (`file:<rid>/<field>`, `cid:<cid>`, `idx:*`, `dup:*`,
> `udl:*`, `ev:*`) in one SQLite string→string `kv` table. v2.0.0 replaced
> that keyspace entirely; the old layout is summarised at the end
> ([Previous layout](#previous-layout-v11v12-kv-table)).
- **Redis keyspace notifications** for field-level change detection
- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates)
- **meta:events stream** integration for real-time updates across services
- Individual field updates without affecting other properties
- Fast VFS lookups using content-based hash IDs
## Quick summary
## Key Structure
- **Every metadata value is a signed claim**: `claim(uid, cid, key) → value`
+ `ts` + `sig` (MMUR2, secp256k1). One row per signer per field. Writers
sign; meta-core never signs.
- **Two databases.** *Truth* `/meta-core/db/meta-core.db`
(`META_CORE_DB_PATH`, backed up, primary keys only): `principal`, `claim`,
`cid_group`, `meta`. *Cache* `/meta-core/db-cache/cache.db`
(`META_CORE_CACHE_PATH`, never backed up, rebuildable): the v1 `kv` table
(merged view as `file:<rid>/<field>` rows, lookup, counters, event streams,
bookkeeping, service scratch keys) with its `kv_field_v` index and `fts`,
plus two real tables, `claim_by_pid` and `local_file`.
- **Schema-free.** A field is a `key` string in a row; a new field never
needs DDL.
- **Records are cid groups.** A vote names a cid, never a rid. `cid_group`
maps each cid to a **rid** — a hidden, locally minted UUIDv7 (26-char
Crockford) naming the group. Examples use `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
- **What readers get is the merged view**: one precomputed value per
`(rid, key)`, stored as the cache row `file:<rid>/<key>`. `nsrc` (how many
signers support it) is computed by the merge and returned by
`POST /api/merge`, not stored. `canonical_cid` is still **derived on read**
by rank, never stored.
- **Watcher facts are not claims**: `filePath`, size, `mtimeNano`, midhash
live in `local_file` and are joined into the record view unsigned.
- **A schema-version sentinel** (`schema-version = "3"` in truth `meta`).
meta-core explicitly refuses a truth file holding the v1 `kv` table, and any
store with claims but no sentinel or another version.
### Flat Key Storage
## Truth database — `/meta-core/db/meta-core.db`
Each file property is stored as a separate Redis STRING key:
Backed up; only primary keys (plus `principal.uid`'s implicit UNIQUE index).
| Table | Columns (PK first) | Notes |
|---|---|---|
| `principal` | `pid` INTEGER PK, `uid` TEXT UNIQUE | `uid` = `"z" + base58btc(33-byte compressed pubkey)`. `pid` is a local surrogate: never signed, never on the wire |
| `claim` | `(cid, key, pid)` PK, `value` ANY, `ts`, `sig`, `seq` | `STRICT, WITHOUT ROWID`. `value`: TEXT = plaintext, BLOB = ciphertext envelope, NULL = retraction (tombstone, kept). `ts` (ms) is the version. `seq` = global change counter, not indexed |
| `cid_group` | `cid` PK, `rid` | the cid → rid map: the record identity layer |
| `meta` | `name` PK, `value` | `schema-version = "3"`, `last_seq` |
`claim` is written **only** by `POST /api/claims`. A claim replaces the
stored row for its `(cid, key, pid)` only if its `ts` is strictly greater;
identical `(ts, sig)` is `unchanged`; the **same value** from the same signer
(any `ts`) is also `unchanged` and keeps the stored row with its older `ts`;
otherwise older or equal is `stale`, and a bad signature is refused. Why `STRICT` with an `ANY` column: a non-STRICT `ANY`
column gets NUMERIC affinity and turns the text `"007"` into the integer 7;
STRICT keeps the written type, which is the cipher flag.
## Cache database — `/meta-core/db-cache/cache.db`
Never backed up (the store app lists the folder in `backup.exclude`).
Deleting it is the recovery procedure: meta-core rebuilds it from truth at
boot (no `meta:applied_seq` → full rebuild), and `local_file` refills from a
rescan.
It keeps the **v1 key/value layout** (`internal/storage/client.go`): one
`kv(k TEXT PRIMARY KEY, v TEXT NOT NULL) WITHOUT ROWID` table, whose rows are
all derived — no client writes `file:` rows any more.
| `kv` rows | Holds | Written by | Read by |
|---|---|---|---|
| `file:<rid>/<field>` | the **merged view**, one row per `(rid, key)` — the only rows under `file:`. Includes the synthesised `cids/<cid> = "true"` members (from `cid_group`) and the box-local `filePath` / `sizeByte` / `mtimeNano` (from `local_file`) | apply loop | every record read; search |
| `cid:<cid>` | cid → rid (copy of truth `cid_group`) | apply loop, watcher | every cid-addressed request |
| `idx:roots/<rid>` | the set of every record | apply loop | listing, counts |
| `idx:fts/<rid>` | rowid of the record's `fts` row | apply loop | FTS maintenance |
| `idx:count/*`, `idx:schema/*` | record count, per-field schema counters | the cache engine's change hook, from `file:` deltas | search `total`, `/api/stats`, `/api/schema` |
| `ev:<stream>/<ms13>-<seq10>`, `evhwm:<stream>` | `meta:events` (every `file:` change, same transaction) and `file:events` (watcher) | cache engine; watcher | `/api/events/meta`, `/api/events/files` (SSE) |
| `meta:applied_seq`, `meta:rebuilt_at` | how far the cache has applied truth; last rebuild (older SSE cursors get `gap`) | apply loop, rebuild | boot catch-up; SSE |
| anything else | service scratch KV (`btmagnet:*`, `searchcov:*`, `nzb:*`) | other services via `/api/kv/value` (reserved prefixes refused) | same |
| Index / table | Shape | Written by | Read by |
|---|---|---|---|
| `kv_field_v` (index) | `kv(lower(field parsed from k), lower(v), v)` over `file:` rows only | — | filter search |
| `fts` | FTS5 trigram `(rid UNINDEXED, hay)`, one row per record | apply loop, when a haystack field's merged value changes | free-text search |
| `claim_by_pid` | `(pid, cid, key)` PK | apply loop | `?as=<uid>` overlay, `/api/udl/*` facade, identity retraction, `/api/udl/users/stats` |
| `local_file` | `path` PK → `midhash`, `size`, `mtime`; index `(midhash, path)` | file watcher, `POST /api/files/register` | record view (`filePath`, `sizeByte`, `mtimeNano`) |
The cache is **behind, never wrong**: after an unclean shutdown, boot finds
`meta:applied_seq < last_seq` and re-applies every truth claim with `seq >
applied_seq` (or rebuilds in full when one of them is a `cids/*` link). Write
path, apply loop and durability are detailed in
[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §4.
## Keys
The `key` column carries the same field paths METADATA_KEYS.md defines —
what used to follow `file:<rid>/`:
```
Key Pattern: file:{hashId}/{property/path}
Key Type: STRING
Value: String value
claim(cid=bagacbabaec…, key=title, uid=<tmdb plugin>) = "Inception"
claim(cid=bagacbabaec…, key=fileinfo/duration, uid=<ffmpeg plugin>) = "8878.5"
claim(cid=bagacbabaec…, key=fileinfo/streamdetails/video/0/codec, uid=<ffmpeg plugin>) = "h264"
claim(cid=bagacbabaec…, key=genres/Drama, uid=<tmdb plugin>) = "true"
claim(cid=bagacbabaec…, key=cids/bafybeigd…, uid=<full-hash plugin>) = "true"
```
**Example Redis commands:**
```bash
# Store file metadata (each property is a separate key)
SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..."
SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv"
SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5"
SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm"
- `/` is the path separator; nested objects are flattened on write and
reconstructed on read.
- **Key-sets** put the member inside the key (`genres/<g>`, `cids/<cid>`,
`titles/eng/<name>`, `anchoredBy/tmdb:123`): one claim per member, union by
the ordinary merge, removal by a signed retraction.
- Plugins keep writing under their prefixes (`{plugin-id}/<field>`,
`{plugin-id}/<group>/<n>/<field>`), each signing with its own key. Two
sources compete on one key only when they make the same kind of statement;
a different fact (e.g. a filename guess vs a TMDB lookup) gets a different
key.
# Retrieve single field
GET "file:bafkr4ih5kapbjzq.../fileinfo/duration"
# Retrieve all metadata for a file (scan by prefix)
SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000
```
### Why Flat Keys?
The flat key structure (vs. Redis Hashes) enables:
1. **Keyspace Notifications**: Redis publishes events when individual keys change
2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*`
3. **Selective Updates**: Update one field without touching others
4. **Stream Integration**: The `meta:events` stream captures field-level changes
### File Index
A Redis Set tracks all known file hash IDs:
## Records: `cid_group` and the hidden rid
```
Key: file:__index__
Type: Set
Value: {hashId1, hashId2, hashId3, ...}
votes (truth, signed) cid_group (truth) cache kv rows (merged)
(A, M, title) = "The Matrix" M → R1 file:R1/title = "The Matrix"
(A, M, cids/Q) = "true" Q → R1 file:R1/cids/M = "true" (synthesised)
(B, Q, title) = "Matrix, The" file:R1/cids/Q = "true" (synthesised)
file:R1/sizeByte = "734003200"
(B, Q, sizeByte) = "734003200"
```
This enables fast bulk queries like "get all files" without scanning keys.
- A text `cids/<other>` vote on cid X links X and `<other>` into one
group. Retracting it does **not** split the group (phase 1). In the merged
view, membership is synthesised from `cid_group` — `cids/<cid> = "true"` for
every cid of the group — and `cids/*` claims are never merged. Record R = every vote on every cid of R's group, merged per key,
each signer counted once per key.
- First-seen cid with no link → new group with a fresh rid
(`storage.NewUUID()`, UUIDv7 → 26-char Crockford Base32, ULID layout).
- Linking two groups → the **older rid survives**; the other group's cids
move to it; `/api/events/meta` emits `del` for each key of the losing rid
and `set` for the survivor. There is no rid alias table.
- `GET /api/meta/<cid>` is a `cid:<cid>` lookup in the cache, then a range
read `[file:<rid>/, file:<rid>0)` — any known cid, any hash algorithm, no
primary.
- Rids are local names: never signed, never on the swarm or gateway wire.
Local consumers (meta-sort, meta-fuse, meta-stremio) do hold them, which is
why `cid_group` lives in truth.
```bash
# Get all hash IDs
SMEMBERS "file:__index__"
There is **no stored `canonical_cid`**. `GetMetadataDocument` ranks the
group's cids through `internal/cid/rank.go` (`cid.Better`) and returns the
winner as `canonical_cid` in the JSON response only.
# Count total files
SCARD "file:__index__"
## Merge
# Check if a file exists
SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..."
```
The cache row `file:<rid>/<key>` holds the winner over the group's votes
(each signer's newest vote counts once; retractions and ciphertext drop out):
most distinct signers (values compared after **trim + case-fold only**, no
NFC) → oldest `ts` → byte-wise smallest. The stored value is the winner's
exact text; `nsrc` is not stored (`POST /api/merge` returns it). With
`?as=<uid>`, the viewer's own live text votes (found via `claim_by_pid`)
override at read time; they are never written to the cache. Box-local
fields: when several local paths hold the same content, `filePath` is the
lexicographically first (size and mtime from the same `local_file` row); a
signed `sizeByte` claim wins over the on-disk size. Full rules:
implementation basis §7 (`internal/storage/merge.go`).
### Keyspace Notifications
## Reads and writes
Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration:
| Operation | Endpoint | Store access |
|---|---|---|
| write metadata | `POST /api/claims` (batch of signed claims) | truth upsert, then the apply loop updates the cache |
| removed writes | `PUT/PATCH/DELETE /meta/*`, `POST /meta/{h}/_add/{k}`, `PUT/DELETE /api/metadata/{id}`, `PUT /api/metadata/{id}/property`, `POST /api/metadata/batch` | `410 Gone`, pointing at `POST /api/claims` |
| read a record | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{h}` | `cid:` lookup → `file:<rid>/*` rows, optional `?as=` |
| list / search | `GET /api/metadata/hash-ids`, `/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `/api/schema`, `/api/stats` | `file:` rows, `kv_field_v`, `fts`, `idx:count/*` / `idx:schema/*` |
| raw votes | `GET /api/meta/{cid}/claims`, `GET /api/metadata/{rid}/claims` (`?key=`, `?uid=`) | `{rid, cids, claims}`: every live signed claim of the group, all signers, tombstones excluded; 404 for an unknown cid / rid |
| merge without storing | `POST /api/merge` `{claims, as?}` → `{fields, nsrc, rejected}` | none — verify + merge in memory (plus `as`'s votes this core holds) |
| watcher facts | the watcher, `POST /api/files/register` | `local_file` + `cid_group` |
| service scratch | `PUT/GET/DELETE /api/kv/value` | non-reserved cache `kv` rows |
| wipe | `POST /api/metadata/clear` | both databases |
```bash
# Required Redis config (set by meta-core)
CONFIG SET notify-keyspace-events "K$"
Truth writes go through **one writer goroutine** with group commit; reads use
a separate pool and see committed state (WAL). Read paths and shapes are
unchanged from v1.x — they are now served from the derived `file:` rows.
# K = Keyspace events (published on __keyspace@<db>__:<key>)
# $ = String commands (SET, etc.)
```
## Reconstruction (flat → nested)
Services can subscribe to keyspace events for real-time updates:
```bash
# Subscribe to all file key changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Subscribe to specific field pattern (e.g., TMDB updates)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### meta:events Stream
The `meta:events` stream provides reliable event delivery for file system events from meta-core:
The HTTP API reconstructs the merged flat keys into a nested JSON object:
```
Stream Key: meta:events
Entry Fields:
- type: add | change | delete | rename | reset
- path: File path relative to /files
- midhash256: Content hash (for add/change events)
- oldPath: Previous path (for rename events)
- timestamp: Event timestamp
flat nested
───────────────────────────────────── ───────────────────────────
title = "Inception" title: "Inception"
fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5",
fileinfo/streamdetails/ streamdetails: { video: { 0: {
video/0/codec = "h264" codec: "h264", ... } }, ... } }
streams/0/language = "eng" streams: [ { language: "eng" }, ... ]
```
Consumer groups enable reliable processing with acknowledgment:
```bash
# Create consumer group
XGROUP CREATE meta:events mygroup 0 MKSTREAM
# Read new events (blocking)
XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events >
# Acknowledge processed events
XACK meta:events mygroup <entry-id>
```
## Primary Hash ID
The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from:
1. SHA-256 hash of the middle 1MB of the file
2. File size in bytes
3. IPFS CID v1 encoding (multicodec 0x1000)
**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...`
This provides:
- Content-based addressing (same file = same ID)
- Fast computation (only reads 1MB)
- IPFS compatibility for future distributed storage
### Collision Handling
If two different files produce the same midhash (rare), the system falls back to full SHA-256:
```typescript
// Normal case
hashId = midHash256; // "bafkr4ih5kapbjzq..."
// Collision detected (different filePath with same midhash)
hashId = metadata['cid_sha2-256']; // Full hash fallback
```
## Metadata Flattening
Nested objects are flattened into path-based keys for efficient Redis storage.
### Flattening Rules
```typescript
// Input (nested object)
{
title: "Inception",
video: {
codec: "h265",
width: "1920"
},
streams: [
{ language: "eng" },
{ language: "jpn" }
]
}
// Output (flattened for Redis)
title = "Inception"
video/codec = "h265"
video/width = "1920"
streams/0/language = "eng"
streams/1/language = "jpn"
```
**Key rules:**
- Nested objects use `/` as path separator
- Arrays use numeric indices (0, 1, 2...)
- All values stored as strings
- `null`/`undefined` stored as empty strings
- Numbers and booleans converted via `String()`
### Reconstruction
When reading, the flattened structure is reconstructed into nested objects:
```typescript
const metadata = await kvClient.getMetadataFlat(hashId);
// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... }
```
## Plugin Metadata Structure
Each plugin writes metadata using a prefixed key pattern:
```
{plugin-id}/{field} # Simple field
{plugin-id}/{group}/{field} # Grouped field
{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays)
```
### Plugin Contribution Flow
```
File Discovered
↓
[FileProcessorPiscina]
├─ Compute midhash256
├─ Store preliminary metadata
└─ Dispatch plugin tasks
↓
[Plugin Processing]
├─ file-info → fileType, mimeType, sizeByte
├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/...
├─ filename-parser → titles/eng, season, episode, videoType
├─ anime-detector → anime, titles/jpn, titles/jpl
├─ tmdb → tmdbid, plot/eng, genres
└─ full-hash → cid_sha2-256, cid_sha1
↓
[Plugin Callbacks]
└─ Store metadata via flat keys (SET per property)
```
## Core Metadata Types
### HashMeta - Content Identification
```
cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132)
cid_md5 = "bafkr..." # MD5 (multicodec 0xd5)
cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11)
cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12)
cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000)
```
### FileStatMeta - File Properties
```
fileType = "video" # video, audio, image, text
mimeType = "video/x-matroska" # MIME type
sizeByte = "4294967296" # File size in bytes
```
### VideoMeta - Classification
```
videoType = "movie" # movie, episode, unknown
originalTitle = "Inception"
titles/eng = "Inception" # Localized titles
titles/jpn = "インセプション"
season = "1"
episode = "5"
movieYear = "2010"
```
### AnimeMeta - Anime Detection
```
anime = "true" # "true" or "false"
titles/jpn = "進撃の巨人" # Japanese title
titles/jpl = "Shingeki no Kyojin" # Romaji
```
### SubtitleMeta - Subtitle References
```
subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID
subtitles/eng/1 = "bafks..."
subtitles/jpn/0 = "bafkt..."
```
### JellyfinMeta - Comprehensive Metadata
```
imdbid = "tt1375666"
tmdbid = "27205"
anidbid = "12345"
plot/eng = "A movie about..."
genre/0 = "Action"
genre/1 = "Sci-Fi"
rating = "8.8"
mpaa = "PG-13"
releasedate = "2010-07-16"
art/poster = "bafkr..." # Reference to poster image CID
art/fanart = "bafks..." # Reference to backdrop image CID
```
## FFmpeg Plugin - Special Case
The FFmpeg plugin extracts detailed stream information with a specific nested structure.
### Stream Details Structure
```
fileinfo/duration = "7200.5"
fileinfo/formatName = "matroska,webm"
fileinfo/streamdetails/video/0/codec = "h264"
fileinfo/streamdetails/video/0/width = "1920"
fileinfo/streamdetails/video/0/height = "1080"
fileinfo/streamdetails/video/0/aspect = "16:9"
fileinfo/streamdetails/video/0/bitrate = "8000000"
fileinfo/streamdetails/audio/0/codec = "aac"
fileinfo/streamdetails/audio/0/language = "eng"
fileinfo/streamdetails/audio/0/channels = "6"
fileinfo/streamdetails/audio/1/codec = "dts"
fileinfo/streamdetails/audio/1/language = "fra"
fileinfo/streamdetails/audio/1/channels = "6"
fileinfo/streamdetails/subtitle/0/language = "eng"
fileinfo/streamdetails/subtitle/0/format = "subrip"
fileinfo/streamdetails/subtitle/1/language = "jpn"
fileinfo/streamdetails/subtitle/1/format = "ass"
fileinfo/streamdetails/embeddedimage/0/type = "cover"
```
### Stream Type Keys
| Stream Type | Key Pattern | Description |
|------------|-------------|-------------|
| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) |
| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks |
| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles |
| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails |
### Common Stream Fields
**Video streams:**
- `codec` - h264, hevc, av1, vp9, etc.
- `width`, `height` - Resolution
- `aspect` - Aspect ratio
- `bitrate` - Video bitrate
- `profile` - Codec profile (main, high, etc.)
- `level` - Codec level
**Audio streams:**
- `codec` - aac, dts, ac3, flac, etc.
- `language` - ISO 639-2 code (eng, jpn, fra)
- `channels` - Channel count (2, 6, 8)
- `bitrate` - Audio bitrate
- `samplerate` - Sample rate
**Subtitle streams:**
- `language` - ISO 639-2 code
- `format` - subrip, ass, pgs, vobsub
## Real-Time Notifications
MetaMesh provides multiple mechanisms for real-time metadata change notifications:
### 1. meta:events Stream (Recommended)
The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups:
```bash
# File system events from meta-core watcher
XREAD STREAMS meta:events 0
# Event types: add, change, delete, rename, reset
```
Benefits:
- **Reliable delivery** via consumer groups and acknowledgment
- **Replay capability** for catching up after restarts
- **Ordered processing** with stream IDs
### 2. Keyspace Notifications
For field-level change detection, subscribe to Redis keyspace notifications:
```bash
# All file metadata changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Specific plugin updates (e.g., TMDB enrichment)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### Cache Invalidation
The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation:
```
┌─────────────────────────────────────────────────────────────────┐
│ Cache Invalidation Flow │
│ │
│ Redis Keyspace Cache │
│ ┌──────────────┐ Invalidator │
│ │ SET file:*/ │ ──────────► ┌──────────────┐ │
│ │ DEL file:*/ │ subscribe │ Listens to │ │
│ └──────────────┘ │ __keyspace@ │ │
│ │ 0__:file:* │ │
│ └──────┬───────┘ │
│ │ │
│ ▼ │
│ ┌──────────────┐ │
│ │ Remove from │ │
│ │ LRU cache │ │
│ └──────────────┘ │
└─────────────────────────────────────────────────────────────────┘
```
Flow:
1. Cache invalidator subscribes to `__keyspace@0__:file:*`
2. When metadata changes (SET/DEL), invalidator receives notification
3. Invalidator removes the corresponding file from the WebDAV cache
4. Next access fetches fresh copy from source filesystem
This ensures cached files stay synchronized with metadata changes.
### 3. Pub/Sub Channels (Legacy)
Batch notification channels are still supported:
```
meta-sort:file:batch # Batch file updates (every 5 seconds)
meta-sort:scan:reset # Full rescan notification
```
```json
{
"timestamp": 1704067200000,
"changes": [
{ "action": "add", "hashId": "bafkr4ih5..." },
{ "action": "update", "hashId": "bafkr4ih5..." },
{ "action": "remove", "hashId": "bafkr4ih5..." }
]
}
```
## Complete Example
Here's a complete metadata example for a movie file:
```bash
# Query all keys for a file
redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*'
# Example flat keys stored in Redis:
# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..."
# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv"
# file:bafkr4ih5kapbjzq.../fileType = "video"
# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5"
# file:bafkr4ih5kapbjzq.../titles/eng = "Inception"
# ...
# When reconstructed by the API (nested object representation)
{
# Content identification
"cid_midhash256": "bafkr4ih5kapbjzq...",
"cid_sha2-256": "bafks7ij2kbpck...",
"filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv",
# File properties
"fileType": "video",
"mimeType": "video/x-matroska",
"sizeByte": "4294967296",
# Classification
"videoType": "movie",
"originalTitle": "Inception",
"movieYear": "2010",
# Localized titles
"titles": {
"eng": "Inception",
"fra": "Inception",
"jpn": "インセプション"
},
# FFmpeg stream details
"fileinfo": {
"duration": "8878.5",
"formatName": "matroska,webm",
"streamdetails": {
"video": {
"0": {
"codec": "h264",
"width": "1920",
"height": "1080",
"aspect": "16:9",
"bitrate": "8000000"
}
},
"audio": {
"0": { "codec": "dts", "language": "eng", "channels": "6" },
"1": { "codec": "ac3", "language": "fra", "channels": "6" }
},
"subtitle": {
"0": { "language": "eng", "format": "subrip" },
"1": { "language": "fra", "format": "subrip" }
}
}
},
# External metadata (TMDB)
"tmdbid": "27205",
"imdbid": "tt1375666",
"plot": {
"eng": "A thief who steals corporate secrets through dream-sharing technology..."
},
"genre": ["Action", "Science Fiction", "Adventure"],
"rating": "8.4",
"releasedate": "2010-07-16",
# Media assets
"art": {
"poster": "bafkr4poster...",
"fanart": "bafkr4fanart..."
},
# Detection flags
"anime": "false"
}
```
## Key Implementation Files
| Component | File Path |
|-----------|-----------|
| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` |
| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` |
| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` |
| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` |
| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` |
| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` |
| meta-core Storage | `packages/meta-core/internal/storage/client.go` |
| meta-core Events | `packages/meta-core/internal/events/publisher.go` |
| meta-core Cache | `packages/meta-core/internal/cache/` |
- Numeric path segments become array indices on read.
- All values are strings on the wire; type interpretation is the caller's
job (documented per field in `METADATA_KEYS.md`).
- Box-local fields from `local_file` are joined in unsigned; they never
appear in `GET /api/meta/{cid}/claims` or on the swarm.
## Search structures
Both live in the cache and are maintained in the same transaction as the
merged-row change, so results are fresh as soon as a change is applied:
- **`kv_field_v`** — an expression index over the field name parsed out of
the key, `lower(v)` and `v`, restricted to `file:` rows. Answers
"which records have field F with a value like V" for any field, including
ones invented tomorrow; covering for the substring test.
- **`fts`** — FTS5 trigram `(rid UNINDEXED, hay)`, one row per record: the
haystack of its searchable merged fields plus the rid. Every 3+ character substring is
indexable.
`POST /api/metadata/search` builds a candidate superset in SQL and verifies
each hydrated merged record with the Go matchers (`recordMatchesFilter`,
`matchAllTokens`); results come in rid order. Plan, rejected shapes and the
measurement status are in
[`metacore-storage-engine.md`](../../../docs/project-architecture/metacore-storage-engine.md) §8.
## Schema-version sentinel
Truth `meta` row `schema-version` = `"3"`, set at first boot and checked on
every boot (`internal/storage/schema_version.go`), after both files are
opened and before the cache catch-up. A truth file holding the v1 `kv` table
(v1.x, `meta-core:schema-version = "2"`) is **explicitly refused**, naming the
migration tool; claims without a sentinel, or another version, are refused
too; a fresh store is stamped. Alpha policy: no migration code in the
service binary; operators wipe and restart. watch.nsl.sh is converted once by
the separate one-off tool `packages/meta-core/tools/claims-migrate` (not in
the image; rids keep their UUIDs; legacy fields re-signed by a throwaway key;
UDL cells re-signed by their account keys — implementation basis §11).
## Event streams
| Stream | Producer | Contents |
|---|---|---|
| `/api/events/meta` | the cache engine, as `ev:meta:events/…` rows of the cache `kv` table | `{type: set\|del, key, ts}` for every change of a **merged** value, in the same cache transaction as the change |
| `/api/events/files` | `internal/watcher` dispatcher (`ev:file:events/…` rows) | `add` / `change` / `delete` / `rename` / `reset` for files on `/files` (unchanged) |
- A vote that does not change the merged value emits nothing.
- A group merge emits `del` for every key of the losing rid and `set` for the
survivor's keys.
- IDs keep the `<ms>-<seq>` form and stay time-based, so persisted cursors
keep comparing, including across a cache rebuild.
- `Last-Event-ID: 0` / `0-0` returns a snapshot (one `set` per merged key)
then the live tail; only the last snapshot event carries an `id:`.
- A trimmed cursor gets one `event: gap`. A cache rebuild emits no per-key
events; consumers get `gap` and take a snapshot.
- Heartbeats are SSE comments (`:keep-alive\n\n`) every 30 s of silence.
Contract: `internal/api/sse_events.go`, [`api-mediated-access.md`](api-mediated-access.md).
## Debugging
```bash
# List all file hash IDs
docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__'
# Count total files
docker exec meta-core-dev redis-cli SCARD 'file:__index__'
# Get a sample hash ID
docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__'
# List all keys for a specific file (flat key scan)
docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*'
# Get specific field (flat key)
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration'
# Get filePath for a file
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath'
# Verify keys are STRING type (not HASH)
docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath'
# Should return: string
# Monitor metadata changes via stream
docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0
# Monitor keyspace notifications (requires notify-keyspace-events enabled)
docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*'
# meta-core API debugging
# Cache status
curl http://localhost:9000/api/cache/status
# Clear cache
curl -X POST http://localhost:9000/api/cache/clear
# Watch events stream (SSE)
curl http://localhost:9000/api/events/stream
# KV browser - storage info
curl http://localhost:9000/api/kv/info
# Scan status
curl http://localhost:9000/api/scan/status
```
## meta:events Stream
The `meta:events` stream captures metadata change events for real-time notifications:
Paths: truth `/meta-core/db/meta-core.db`, cache `/meta-core/db-cache/cache.db`
(image default and store app); dev stacks keep both on named volumes (the
`/meta-core` bind is 9p on Windows dev boxes, where SQLite WAL is unsafe).
Always open **read-only**; writes go through `POST /api/claims`.
```bash
# View recent events
docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10
T=/meta-core/db/meta-core.db; C=/meta-core/db-cache/cache.db # dev: the named-volume paths
QT() { docker exec metacore-app sqlite3 -readonly "$T" "$1"; }
QC() { docker exec metacore-app sqlite3 -readonly "$C" "$1"; }
# Read events from a consumer group
docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' >
QT "SELECT rid FROM cid_group WHERE cid = 'bafkreih...'" # cid → rid
QT "SELECT cid FROM cid_group WHERE rid = '01JKR8XW5T4QMV7AHNJ2DEFGHK'" # the group (full scan: no rid index)
QC "SELECT k, v FROM kv WHERE k >= 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/' AND k < 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK0'" # merged record
QT "SELECT c.key, typeof(c.value), c.value, p.uid, c.ts, c.seq
FROM claim c JOIN principal p USING(pid) WHERE c.cid = 'bafkreih...'" # every vote on one cid
QT "SELECT name, value FROM meta"; QC "SELECT k, v FROM kv WHERE k LIKE 'meta:%'" # last_seq vs applied_seq
QC "SELECT path, midhash FROM local_file WHERE path LIKE '/files/watch/%' LIMIT 10"
QC "SELECT k, v FROM kv WHERE k >= 'ev:meta:events/' AND k < 'ev:meta:events0' ORDER BY k DESC LIMIT 10"
# HTTP equivalents — preferred
curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>' # merged document (+ canonical_cid)
curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>/claims' # signed votes
curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE
```
Stream events contain:
- `type`: Event type (add, change, delete, rename, reset)
- `path`: File path
- `midhash256`: Content hash (for add/change)
- `timestamp`: Event timestamp
## Key implementation files
| Component | Path |
|---|---|
| Single-writer, group-commit SQLite engine | `internal/kv/` |
| Truth, apply loop, merge, local files, counters | `internal/storage/{truth,apply,merge,localfile,client}.go` |
| Search (`kv_field_v` + FTS) | `internal/storage/search.go` |
| MMUR2 claim encoding + verification | `internal/claim/` (secp256k1 in `internal/identity/`) |
| UUIDv7 minting | `internal/storage/uuid.go` |
| Canonical-CID rank | `internal/cid/rank.go` |
| Schema-version gate | `internal/storage/schema_version.go` |
| SSE event mirror + snapshot | `internal/api/sse_events.go` |
| Watcher → `local_file`, `/api/events/files` | `internal/watcher/` |
| Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
## Previous layout (v1.1–v1.2 `kv` table)
Kept so old notes and scripts can be read. In v2.0.0 the **truth** database
has no `kv` table at all; the **cache** keeps the same key layout for derived
rows (see above), but nothing in it is written by clients.
| Key | Value | Replaced by |
|---|---|---|
| `file:<rid>/<field>` | one value per field, last writer wins | `claim` (one row per signer) → derived cache `file:` rows |
| `cid:<cid>` | `<rid>` | `cid_group` (truth), copied to cache `cid:` rows |
| `idx:roots/<rid>`, `idx:count/*`, `idx:schema/*`, `idx:fts/<rid>` | root set, counters, FTS rowid | same keys, derived, in the cache |
| `dup:<rid>/<path>` | duplicate path | `local_file` rows |
| `udl:rec:<uid>/<cid>/<key>`, `udl:idx:*` | User Data Layer cells (MMUR1, `version`) | `claim` (MMUR2, `ts`) + `claim_by_pid` |
| `ev:<stream>/<ms13>-<seq10>`, `evhwm:<stream>` | stream rows | same keys, in the cache |
| `btmagnet:*`, `searchcov:*`, `nzb:*` | service scratch | same keys, in the cache |
| `meta-core:schema-version` = `"2"` | gate | truth `meta` `schema-version` = `"3"` |
The flat-key idea survived — a field is still one string key, a key-set
member is still inside the key — but the record moved from "one value per
field" to "one value per signer per field".
+276
View File
@@ -0,0 +1,276 @@
# UUID-Rooted Metadata with CID Aliasing
## Status
**Status:** Design record. Implemented in 2026 alpha (clean-wipe gate);
**reframed by meta-core v2.0.0 (claims model)** — see
[`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md)
§4. This doc keeps the *why* behind UUID rids and CID symmetry; it is not
the schema reference. For what is stored, see
[metadata-storage-structure.md](./metadata-storage-structure.md) and
`METADATA_KEYS.md` §2/§14.13.
What still holds, unchanged since the original proposal:
- **No CID is privileged.** Any known CID resolves to its record; all hash
types are symmetric.
- **Rids are UUIDv7**, 26-char Crockford Base32, **minted locally** with no
central allocator.
- **`canonical_cid` is derived on read** by rank (`internal/cid/rank.go` →
`cid.Better`), never stored.
- **`cids` is a key-set of bare CIDs** (`cids/<bareCid>` = `"true"`), no
`<algo>:` prefix — a CIDv1's multicodec already names the hash function.
What changed in v2.0.0:
> 1. **The rid is a hidden local name for a group of cids.** Metadata is no
> longer stored under `file:<uuid>/<field>`. Every value is a signed claim
> about a **cid** (`claim(uid, cid, key) → value`); `cid_group(cid → rid)`
> says which cids form one record. The rid is **derived from cid groups,
> never signed and never on the wire** (swarm, gateway); each box mints its
> own.
> 2. **The `cid:<cid>` reverse index is gone**, replaced by the `cid_group`
> table (still a primary-key lookup, so still O(1)-class).
> 3. **meta-core no longer writes `cids/*` members.** They are signed votes
> from the writers that learn a cid (e.g. the full-hash plugin); a non-null
> `cids/<other>` vote links two groups. Only the watcher's midhash enters
> `cid_group` directly.
> 4. **`duplicates` / `dup:<uuid>/<path>` are gone.** Box-local paths live in
> the cache's `local_file(path → midhash, size, mtime)`; several paths with
> the same midhash are the duplicates.
>
> Earlier superseded points, kept for the record: the proposed
> `reconcileCanonicalCIDLocked` never shipped; `cids` never was a Redis SET
> of `<algo>:<value>` tokens; Redis itself went in v1.1.0 (2026-10). Text
> below marked *(proposal)* or *(v1.x)* describes those shapes, not v2.0.0.
## Motivation (history)
Before UUID rids, every metadata entry in Redis was keyed by a content hash,
specifically midhash256:
```
file:midhash256:abc.../filePath
file:midhash256:abc.../title
file:midhash256:abc.../tmdb/poster
```
Other digests (sha256, IPFS CIDs, BitTorrent infohashes) lived as fields
*inside* that entry (`cid_sha256`, `cid_ipfs`). Two problems:
1. **midhash256 was privileged by accident.** It *was* the key, while every
other CID was a secondary field. Hash-algorithm changes, or new digest
types added by plugins, forced rewrites of every root. The privilege
itself was the issue, not which hash held it.
2. **CID lookup was O(n).** "Given any CID, return the metadata" scanned
every entry (`storage/client.go::LookupPathByCID`). Unacceptable for a
public API, especially when peers query by any CID variant.
The requirement: **any CID a client knows must resolve to that content's
metadata in O(1)**, and the storage layout must not bias toward any digest
type. The claims model adds a second one: **what is signed must name
something every peer agrees on** — a rid minted at random on one box does
not qualify, a cid does.
## Goal
```
GET /api/meta/<cid> → full (merged) metadata document for that content
```
`<cid>` may be any supported type (midhash, sha256, IPFS, btih, sha3, future
additions), all treated symmetrically.
## Design (v2.0.0)
### Votes name cids; a rid names a group
```
votes (signed, travel) derived (local, persistent: truth cid_group)
(A, M, title) = "The Matrix" M → R1
(A, M, cids/Q) = "true" ← alias vote Q → R1
(B, Q, title) = "Matrix, The"
(B, Q, sizeByte)= "734003200" record R1 = every vote on M or Q, merged per key
```
- **First-seen cid, no link** → new group, fresh UUIDv7 rid.
- **`cids/<other>` vote on X** → X and `<other>` join one group.
- **Link between two existing groups** → the group with the **older rid
survives**; the other's cids move to it; the losing rid gets a `del` event
for each of its keys, the survivor `set` events. No `rid_alias`: consumers
already handle deletes, and cids re-resolve.
- **Reading record R** = every vote on every cid of R's group, merged per key
(one signer counts once per key), precomputed into the cache's
`file:<rid>/<field>` rows (the v1 key layout, now derived).
- **Retracting a `cids/<other>` link does not split the group** (phase 1):
the tombstone drops out of the merge, `cid_group` keeps both cids under one
rid. In the record view, membership is synthesised from `cid_group`
(`cids/<cid> = "true"` for every cid of the group).
Why keep a rid at all, if votes never mention it: local consumers (meta-sort,
meta-fuse, meta-stremio) and the merged view need one stable name per record,
and a group's cid set grows over time. Why `cid_group` lives in **truth**
although it could be recomputed from `cids/*` votes: the rid names are minted
at random and consumers hold them, so a recomputation would rename every
record.
### Why UUIDv7 (still valid)
- **128 bits**: collision-free under distributed minting; no allocator. Each
box minting its own rids for the same content is now the normal case, not a
corner case — which is exactly why rids never travel.
- **Time-sortable prefix**: rids minted in a window sort together
(`k >= 'file:01JKR8' AND k < 'file:01JKR9'` on the cache's merged rows),
and "older rid
survives" on a group merge is a plain string comparison.
- **Monotonic-ish**: adjacent rids cluster in the cache's `file:<rid>/…`
key range.
Encoded as 26-char Crockford Base32 (ULID-compatible) for compactness and
case-insensitive matching. Migrated records kept their v1.x UUIDs as rids.
### The `cids` key-set
`cids/<bareCid>` = `"true"`, one claim per member, by whichever signer knows
the link. A key-set merges conflict-free: a member is present while at least
one vote supports it, and removal is a signed retraction. That is the reason
the proposal's Redis SET was reversed before shipping, and it is what lets
the claims model treat `cids` with no special merge code.
Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are **not**
claims — meta-core cannot sign and the disk can always be rescanned. The
watcher records them in `local_file` and puts the midhash into `cid_group`;
the record view joins them in unsigned. When several local paths hold the
same content, `filePath` is the lexicographically first one.
### Canonical CID selection (unchanged)
The canonical CID is a label, not a key, so nothing external breaks when it
changes. It is computed on every read from the group's cids:
1. IPFS CID (any variant)
2. SHA-256 / SHA-3 family
3. BitTorrent infohash (v2 preferred over v1)
4. midhash256
Ranked by **utility for external retrieval**, not cryptographic strength:
midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
externally meaningful; btih ranks above midhash because it enables swarm
interop. The full ladder (including the custom locator codecs) lives in
`internal/cid/rank.go`, mirrored in meta-search / meta-share
(`scripts/check-cid-vectors.sh`).
## Lifecycles (v2.0.0)
### File arrival
```
1. Watcher sees /files/watch/foo.mkv, computes its midhash M.
2. local_file: /files/watch/foo.mkv → (M, size, mtime).
3. cid_group: M already grouped → nothing; otherwise M → fresh rid.
4. Plugins process the file by cid M (the rid is never disclosed to them)
and sign claims about M with their own keys.
5. The full-hash plugin computes sha256 S and signs (plugin, M, cids/S) = "true".
→ S joins M's group (or the two groups merge, older rid surviving).
→ The next read derives canonical_cid = S, because it outranks the midhash.
```
A second path with the same midhash is just another `local_file` row: same
cid, same group, same record; the record's `filePath` is the
lexicographically first of the paths.
### File deletion
The watcher removes the path's `local_file` row; the record view loses that
`filePath`. Claims are signed truth from their writers and are not deleted
by meta-core (the store is permanent and additive,
`docs/project-architecture/metacore-as-permanent-store.md`).
*(v1.x)* Deletion promoted a duplicate to `filePath`, or called `DeleteRoot`,
which removed every `cid:*` alias and every `file:<uuid>/*` key.
## Public API
```
GET /api/meta/<cid> → 200 merged document { "filePath", "canonical_cid", "cids": [...], ... }
(optional ?as=<uid>: the viewer's own votes win)
→ 404 if no such cid
GET /api/meta/<cid>/claims → {rid, cids, claims}: every live signed claim of the cid's
group, all signers (?key=, ?uid=); 404 if no such cid
GET /api/metadata/<rid>/claims → the same, addressed by rid (local consumers)
```
The rid is not part of the cid-addressed contract and never appears in a
claim. It is visible only to local consumers that list or address records by
rid (`/api/metadata/{rid}`, `/api/metadata/{rid}/claims`, `hash-ids`, SSE event
keys; the claims routes also echo it as `rid`), and may change on a
group merge (delete event, then re-resolve by cid).
Auth: `/api/meta/<cid>` bypasses the perimeter like `/api/file/<cid>` — it is
the metadata twin of the content endpoint.
## Cross-service impact (v2.0.0)
- **Writers** (plugins, self-publishing feeders, gateway core, meta-share,
meta-search): address claims by cid only and sign them; they never resolve
a rid.
- **meta-fuse, meta-stremio**: unchanged — merged reads and SSE keep their
shape; a group merge reaches them as ordinary `del` + `set` events.
- **meta-search / meta-share**: announce the derived canonical cid (each
mirrors the Go rank) and relay raw claims; every cid of the group travels as
`cids/*` claims, so a peer querying by any digest gets a hit.
## Migration (history)
> **The v1.0 sweep below was never run.** The alpha shipped behind a
> clean-wipe gate. The v1.x → v2.0.0 conversion of watch.nsl.sh is a
> one-off tool outside the service image, `packages/meta-core/tools/claims-migrate`
> (canonical cid by rank per record; `cid:` aliases → `cid_group`, rids keep
> their UUIDs, and the `cid:` index wins when a record's `cids/*` member
> disagrees; fields re-signed by a throwaway legacy key); every other box
> starts empty. Implementation basis §11.
*(proposal)* One-time sweep over `file:__index__`: mint a uuid, `AddAlias`
the old hashID and every `cid_*` field, `RENAME` every property under the new
root, reconcile canonical, swap the index entry; crash-recoverable by
detecting which step completed.
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| When is the rid minted? | When a cid is first seen with no link to a known group | v1.x: at first hash of a watched file. Now any first-seen cid (watcher midhash or a claim about a new cid) mints one |
| Rid format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe |
| Rid visible externally? | No — never signed, never on the swarm/gateway wire | Local HTTP consumers may hold it |
| What do signed votes name? | A cid | Every peer agrees on a cid; rids differ per box |
| Record identity | A group of cids (`cid_group`, in truth) | Linked by `cids/<other>` votes |
| Group merge | Older rid survives; losing rid → `del` events | No `rid_alias` (out of scope for phase 1) |
| Duplicate paths | `local_file` rows sharing a midhash (cache) | v1.x: `duplicates` SET, then `dup:<uuid>/<path>` |
| Canonical CID rank | IPFS > sha-family > btih > midhash, derived on read | Utility-weighted; never stored |
| `cids` shape | Key-set of bare CIDs, one claim per member | Merges with the generic rule; a SET would not |
| Reverse index | `cid_group(cid PK → rid)` in truth, copied to cache `cid:<cid>` rows for reads | v1.x: flat `cid:<cid>` keys were the only copy |
| Unlinking | A retracted `cids/*` link does not split a group | Phase 1 |
## Open questions
- **Discovery-time rid minting.** Minting at discovery (before hashing)
would give two same-content paths two rids until hashing links them. The
claims model already handles that case — it is an ordinary group merge —
so this is now a policy choice, not a missing mechanism.
- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` for
external IPFS clients? Adjacent, not part of this design.
- ~~**meta-share announce strategy.**~~ Settled: the canonical cid addresses
the record, every group cid travels as a `cids/*` claim.
- ~~**Keyspace-notification consumers.**~~ Obsolete since Redis left.
## Non-goals
- **Not** a content-replication or pinning system. A `cids/<ipfs-cid>` vote
pins nothing; it states "hashing these bytes with IPFS's chunker gives this
CID."
- **Not** a history. `ts` versions each `(uid, cid, key)` and the newest vote
replaces the older one; only tombstones are kept, to block replay.
- **Not** strongly consistent across the two databases: the cache is applied
after the truth commit. `POST /api/claims` waits for the apply before
answering, so the writer reads its own write; another reader may briefly
miss a just-accepted claim. It is behind, never wrong.
+57 -121
View File
@@ -1,140 +1,76 @@
# @metazla/meta-editor
# @meta-core/editor
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface.
React + Vite metadata browser/editor that ships inside the meta-core
container. Walks the UUID-rooted Redis keyspace, edits leaf values,
imports/exports snapshots, and consults the live schema indexer.
## Features
## What it talks to
- **Search**: Find files by title, filename, or hash ID
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more
- **Batch Operations**: Update multiple files simultaneously
- **Real-time**: Direct integration with etcd via REST API
- **User-friendly**: Clean, intuitive interface with form validation
The editor is a pure SPA — it has no server of its own. It calls the
meta-core HTTP API directly under three prefixes (see `src/api/`):
## Development
- `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`,
`value`, `key/{key...}`). This is the UUID-exposing surface — keys look
like `file:<uuid>/<property>` and `cid:<algo>:<value>`.
- `/api/schema` — live per-field schema (type hints, value breakdowns),
produced by meta-core's schema indexer that consumes `meta:events`.
- `/api/snapshot/*` — snapshot export / import / wipe.
### Prerequisites
In production the editor is served by meta-core's nginx (the Vite build
output is copied into the image at the same path) under
`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to
`/editor/` accordingly.
- Node.js 21.6.2+
- pnpm 10.19.0+
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
### Running Locally
```bash
# From the root directory
pnpm run start:editor
# Or from the package directory
cd packages/meta-editor
pnpm run dev
```
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
### Building
```bash
# From the root directory
pnpm run build:editor
# Or from the package directory
cd packages/meta-editor
pnpm run build
```
The built files will be in `dist/` directory.
## Docker Deployment
In the Docker development environment, the editor is automatically available at:
## Project layout
```
http://localhost/editor
```
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
## API Integration
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
- `POST /api/metadata/search` - Search for files
- `GET /api/metadata/:hashId` - Get file metadata
- `PUT /api/metadata/:hashId` - Update complete metadata
- `PUT /api/metadata/:hashId/property` - Update specific property
- `POST /api/metadata/batch` - Batch update multiple files
See `src/api/metadataApi.ts` for the API client implementation.
## Usage
### Searching for Files
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
2. Enter your search query
3. Click "Search"
4. Results will appear in the left sidebar
### Editing Metadata
1. Select a file from the search results
2. Modify any fields in the editor form
3. Click "Save Changes" to persist updates
4. Use "Reset" to discard changes
### Supported Metadata Fields
**Basic Information:**
- Title, Original Title, Show Title
- Video Type (Movie, TV, Anime, Other)
- Year, Season, Episode
**Jellyfin Metadata:**
- Plot, IMDb ID, TMDb ID, AniDB ID
- Director, Studio, Rating, Runtime
- Genres, Premiered Date, Release Date
**File Information:**
- Filename, File Size, MIME Type
- Hash values (SHA-256, MD5, etc.) - read-only
## Project Structure
```
packages/meta-editor/
├── src/
│ ├── api/ # API client for metadata operations
│ ├── components/ # React components
│ │ ├── SearchBar.tsx # Search interface
│ │ ├── FileList.tsx # Search results list
│ │ └── MetadataEditor.tsx # Main editing form
│ ├── types/ # TypeScript type definitions
│ ├── utils/ # Utility functions
│ ├── App.tsx # Main application component
│ ├── main.tsx # Entry point
│ └── index.css # Global styles
packages/meta-core/editor/
├── index.html
├── vite.config.ts
├── tsconfig.json
└── package.json
├── package.json @meta-core/editor
└── src/
├── main.tsx entry
├── api/
│ ├── kvApi.ts /api/kv client
│ ├── schemaApi.ts /api/schema client + cache
│ └── snapshotApi.ts /api/snapshot client
├── kv/ KV browser + leaf/branch editors
├── components/ shared widgets (language combobox, snapshot panel, ...)
└── index.css
```
## Technology Stack
## Local development
- **React 18** - UI framework
- **TypeScript 5.7** - Type safety
- **Vite 6** - Build tool and dev server
- **Native CSS** - Styling (no CSS framework)
Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`.
## Contributing
```bash
cd packages/meta-core/editor
pnpm install
pnpm dev # vite dev server on http://localhost:5173
```
When adding new features:
The Vite dev server proxies `/api/*` to `http://localhost:3000` (see
`vite.config.ts`). For local development against a running meta-core
container, change that target to wherever meta-core is reachable on your
host — e.g. `http://localhost:18083` for the dev compose's direct-backend
port.
1. Add type definitions to `src/types/index.ts`
2. Update API client in `src/api/metadataApi.ts`
3. Create/update components as needed
4. Update this README with new features
No root-level `start:editor` or `build:editor` scripts exist; run pnpm
commands from this directory.
## License
## Build
Part of the MetaMesh project.
```bash
pnpm build # tsc -b && vite build → dist/
```
The container build copies `dist/` into the nginx web root so the editor
is served at `/editor/`.
## Tech
- React 18, TypeScript 5
- Vite 5 (`base: '/editor/'`)
- Native CSS — no UI framework
- `iso-639-3` for language code rendering
+13 -2
View File
@@ -1,13 +1,14 @@
{
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "@meta-sort/editor",
"name": "@meta-core/editor",
"version": "0.1.0",
"dependencies": {
"iso-639-3": "^3.0.1",
"react": "^18.2.0",
"react-dom": "^18.2.0"
},
@@ -1246,6 +1247,16 @@
"node": ">=6.9.0"
}
},
"node_modules/iso-639-3": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/iso-639-3/-/iso-639-3-3.0.1.tgz",
"integrity": "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/js-tokens": {
"version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-185
View File
@@ -1,185 +0,0 @@
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.app-header {
padding: 2rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.app-header h1 {
margin: 0 0 0.5rem 0;
color: #646cff;
}
.app-description {
margin: 0;
color: rgba(255, 255, 255, 0.6);
}
.app-main {
flex: 1;
padding: 2rem;
max-width: 1400px;
width: 100%;
margin: 0 auto;
}
.app-error {
padding: 1rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
border-radius: 8px;
color: #ff6b6b;
margin-bottom: 1rem;
}
.app-loading {
text-align: center;
padding: 3rem 2rem;
color: rgba(255, 255, 255, 0.7);
font-size: 1.1rem;
}
.app-welcome {
text-align: center;
padding: 4rem 2rem;
color: rgba(255, 255, 255, 0.6);
}
.app-welcome h2 {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.8);
}
.app-welcome-features {
margin-top: 1rem;
margin-bottom: 1.5rem;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
}
.app-kv-browser {
margin-top: 1.5rem;
text-align: left;
max-width: 900px;
margin-left: auto;
margin-right: auto;
}
.app-content {
display: grid;
grid-template-columns: 350px 1fr;
gap: 1.5rem;
margin-top: 1rem;
}
.app-sidebar {
min-height: 400px;
}
.app-editor {
min-height: 400px;
}
.app-no-selection {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
min-height: 400px;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
text-align: center;
padding: 2rem;
}
.app-footer {
padding: 1.5rem 2rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-size: 0.9rem;
}
.app-footer a {
color: #646cff;
text-decoration: none;
}
.app-footer a:hover {
text-decoration: underline;
}
@media (max-width: 1024px) {
.app-content {
grid-template-columns: 1fr;
}
.app-sidebar {
order: 2;
}
.app-editor {
order: 1;
}
}
@media (max-width: 768px) {
.app-header,
.app-main,
.app-footer {
padding: 1rem;
}
.app-content {
gap: 1rem;
}
}
@media (prefers-color-scheme: light) {
.app-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.app-description {
color: rgba(0, 0, 0, 0.6);
}
.app-error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.app-loading {
color: rgba(0, 0, 0, 0.7);
}
.app-welcome {
color: rgba(0, 0, 0, 0.6);
}
.app-welcome h2 {
color: rgba(0, 0, 0, 0.8);
}
.app-welcome-features {
color: rgba(0, 0, 0, 0.5);
}
.app-no-selection {
background-color: rgba(0, 0, 0, 0.02);
color: rgba(0, 0, 0, 0.5);
}
.app-footer {
border-top-color: rgba(0, 0, 0, 0.1);
color: rgba(0, 0, 0, 0.5);
}
}
-246
View File
@@ -1,246 +0,0 @@
import { useState, useEffect } from 'react';
import { SearchBar } from './components/SearchBar';
import { FileList } from './components/FileList';
import { MetadataEditor } from './components/MetadataEditor';
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
import { KVTreeView } from './components/KVTreeView';
import { MetadataAPI } from './api/metadataApi';
import { SearchResult } from './types';
import './App.css';
function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
const [bulkEditMode, setBulkEditMode] = useState(false);
const [isSearching, setIsSearching] = useState(false);
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
const [error, setError] = useState<string | null>(null);
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
const getHashFromUrl = (): string | null => {
const path = window.location.pathname;
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
if (fileMatch) {
return fileMatch[1];
}
// Fallback to query param for backwards compatibility
const params = new URLSearchParams(window.location.search);
return params.get('hash');
};
// Update URL when file is selected
const updateUrlForFile = (hashId: string | undefined) => {
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
if (hashId) {
const newPath = `${basePath}/file/${hashId}`;
window.history.pushState({ hashId }, '', newPath);
} else {
window.history.pushState({}, '', basePath || '/editor/');
}
};
// Check for hash parameter in URL on component mount
useEffect(() => {
const loadHashFromUrl = async () => {
const hashParam = getHashFromUrl();
if (hashParam) {
try {
setIsLoadingDirectHash(true);
setError(null);
// Try to get metadata for this hash
const metadata = await MetadataAPI.getMetadata(hashParam);
// Create a search result from the metadata
const searchResult: SearchResult = {
hashId: hashParam,
metadata: metadata,
score: 1.0,
};
// Set the search results and select this file
setSearchResults([searchResult]);
setSelectedHashId(hashParam);
} catch (err: any) {
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsLoadingDirectHash(false);
}
}
};
loadHashFromUrl();
// Handle browser back/forward
const handlePopState = (event: PopStateEvent) => {
const hashId = event.state?.hashId || getHashFromUrl();
if (hashId) {
setSelectedHashId(hashId);
} else {
setSelectedHashId(undefined);
}
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []); // Empty dependency array means this runs once on mount
const handleSearch = async (query: string) => {
try {
setIsSearching(true);
setError(null);
// Unified search - searches across all fields
const response = await MetadataAPI.search({ query });
setSearchResults(response.results);
// Auto-select first result if available
if (response.results.length > 0) {
setSelectedHashId(response.results[0].hashId);
} else {
setSelectedHashId(undefined);
}
} catch (err: any) {
setError(err.message);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsSearching(false);
}
};
const handleSelectFile = (hashId: string) => {
if (!bulkEditMode) {
setSelectedHashId(hashId);
setSelectedHashIds(new Set());
updateUrlForFile(hashId);
}
};
const handleToggleSelection = (hashId: string) => {
const newSelection = new Set(selectedHashIds);
if (newSelection.has(hashId)) {
newSelection.delete(hashId);
} else {
newSelection.add(hashId);
}
setSelectedHashIds(newSelection);
};
const handleSelectAll = () => {
const allHashIds = new Set(searchResults.map(r => r.hashId));
setSelectedHashIds(allHashIds);
};
const handleClearSelection = () => {
setSelectedHashIds(new Set());
setBulkEditMode(false);
};
const handleBulkEditMode = (enabled: boolean) => {
setBulkEditMode(enabled);
if (enabled) {
setSelectedHashId(undefined);
} else {
setSelectedHashIds(new Set());
}
};
const handleMetadataSaved = () => {
// Optionally refresh the search results to reflect changes
console.log('Metadata saved successfully');
};
const handleBulkMetadataSaved = () => {
console.log('Bulk metadata saved successfully');
handleClearSelection();
};
return (
<div className="app">
<header className="app-header">
<h1>MetaMesh Metadata Editor</h1>
<p className="app-description">
Search and edit metadata for your media files
</p>
</header>
<main className="app-main">
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
{error && <div className="app-error">{error}</div>}
{isLoadingDirectHash && (
<div className="app-loading">Loading file metadata...</div>
)}
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
<div className="app-welcome">
<h2>Welcome to MetaMesh Metadata Editor</h2>
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
<p className="app-welcome-features">
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
</p>
<div className="app-kv-browser">
<KVTreeView />
</div>
</div>
)}
{searchResults.length > 0 && (
<div className="app-content">
<div className="app-sidebar">
<FileList
results={searchResults}
selectedHashId={selectedHashId}
selectedHashIds={selectedHashIds}
bulkEditMode={bulkEditMode}
onSelectFile={handleSelectFile}
onToggleSelection={handleToggleSelection}
onSelectAll={handleSelectAll}
onClearSelection={handleClearSelection}
onBulkEditMode={handleBulkEditMode}
/>
</div>
<div className="app-editor">
{bulkEditMode && selectedHashIds.size > 0 ? (
<BulkMetadataEditor
hashIds={Array.from(selectedHashIds)}
onSaved={handleBulkMetadataSaved}
onCancel={handleClearSelection}
/>
) : selectedHashId ? (
<MetadataEditor
key={selectedHashId}
hashId={selectedHashId}
onSaved={handleMetadataSaved}
/>
) : (
<div className="app-no-selection">
{bulkEditMode
? 'Select files from the list to bulk edit metadata'
: 'Select a file from the list to edit its metadata'}
</div>
)}
</div>
</div>
)}
</main>
<footer className="app-footer">
<p>
MetaMesh Metadata Editor | Part of{' '}
<a href="/" target="_blank">
MetaMesh
</a>
</p>
</footer>
</div>
);
}
export default App;
+135
View File
@@ -0,0 +1,135 @@
// Signed claims (meta-core v2.0.0). The editor no longer writes keys: an edit
// is a claim signed as the chosen identity account — through meta-core's
// account-key service (/api/identity/sign, compact low-S) — and posted to
// POST /api/claims. That vote wins in that account's view; the shared merged
// value only moves if enough signers agree.
//
// MMUR2 bytes (must match internal/claim and its golden vectors):
// "MMUR2\n" || canonical CBOR [cid, uid, key, value, ts]
export interface Claim {
cid: string;
uid: string;
key: string;
value: string | { b64: string } | null;
ts: number;
sig: string;
}
export interface ClaimsResponse {
rid: string;
cids: string[];
claims: Claim[];
}
export interface Account {
uid: string;
curve: string;
createdAt: number;
}
const SIGNER_LS_KEY = 'kv-editor.signer.v1';
async function jsonOrThrow<T>(res: Response): Promise<T> {
if (!res.ok) {
let msg = `HTTP ${res.status}`;
try {
const body = await res.json();
if (body?.message) msg = body.message;
else if (body?.error) msg = body.error;
} catch {}
throw new Error(msg);
}
return res.json();
}
// --- canonical CBOR for the five MMUR2 shapes -------------------------------
function head(out: number[], major: number, n: number) {
const m = major << 5;
if (n < 24) out.push(m | n);
else if (n <= 0xff) out.push(m | 24, n);
else if (n <= 0xffff) out.push(m | 25, n >> 8, n & 0xff);
else if (n <= 0xffffffff) out.push(m | 26, (n >>> 24) & 0xff, (n >>> 16) & 0xff, (n >>> 8) & 0xff, n & 0xff);
else {
const hi = Math.floor(n / 2 ** 32);
const lo = n >>> 0;
out.push(m | 27, (hi >>> 24) & 0xff, (hi >>> 16) & 0xff, (hi >>> 8) & 0xff, hi & 0xff,
(lo >>> 24) & 0xff, (lo >>> 16) & 0xff, (lo >>> 8) & 0xff, lo & 0xff);
}
}
function text(out: number[], s: string) {
const b = new TextEncoder().encode(s);
head(out, 3, b.length);
for (const x of b) out.push(x);
}
export function signedBytes(cid: string, uid: string, key: string, value: string | null, ts: number): Uint8Array {
const out: number[] = [...new TextEncoder().encode('MMUR2\n')];
head(out, 4, 5);
text(out, cid);
text(out, uid);
text(out, key);
if (value === null) out.push(0xf6);
else text(out, value);
head(out, 0, ts);
return new Uint8Array(out);
}
function b64(bytes: Uint8Array): string {
let s = '';
for (const x of bytes) s += String.fromCharCode(x);
return btoa(s);
}
// --- signer -----------------------------------------------------------------
export function getSigner(): string {
try { return localStorage.getItem(SIGNER_LS_KEY) ?? ''; } catch { return ''; }
}
export function setSigner(uid: string) {
try { localStorage.setItem(SIGNER_LS_KEY, uid); } catch {}
}
export const ClaimsAPI = {
async accounts(): Promise<Account[]> {
const r = await jsonOrThrow<{ accounts: Account[] }>(await fetch('/api/identity/accounts'));
return r.accounts ?? [];
},
async forRecord(rid: string, key?: string): Promise<ClaimsResponse> {
const q = key ? `?key=${encodeURIComponent(key)}` : '';
return jsonOrThrow(await fetch(`/api/metadata/${encodeURIComponent(rid)}/claims${q}`));
},
// vote signs value (null = retraction) for key on the record's subject cid
// as the chosen account and posts it.
async vote(rid: string, key: string, value: string | null): Promise<void> {
const uid = getSigner();
if (!uid) throw new Error('choose a signing account first (top right)');
const rec = await this.forRecord(rid);
const cid = rec.cids[0];
if (!cid) throw new Error(`record ${rid} has no cid`);
// ts must beat this account's previous vote on the key.
const prev = rec.claims.filter((c) => c.uid === uid && c.key === key).map((c) => c.ts);
const ts = Math.max(Date.now(), ...prev.map((t) => t + 1));
const bytes = signedBytes(cid, uid, key, value, ts);
const signed = await jsonOrThrow<{ sigB64: string }>(await fetch('/api/identity/sign', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ uid, bytesB64: b64(bytes), format: 'compact' }),
}));
const claim: Claim = { cid, uid, key, value, ts, sig: signed.sigB64 };
const res = await jsonOrThrow<{ status: string; reason?: string }[]>(await fetch('/api/claims', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify([claim]),
}));
const st = res[0]?.status;
if (st !== 'accepted' && st !== 'unchanged') {
throw new Error(`claim ${st}${res[0]?.reason ? ': ' + res[0].reason : ''}`);
}
},
};
+259
View File
@@ -0,0 +1,259 @@
// The editor's key tree. Since meta-core v2.0.0 the records are no longer raw
// keys: `file:<rid>/<field>` is the MERGED view, derived from signed claims.
// This module keeps the slash-tree the components were built on, as a view:
//
// - reads of `file:` keys come from the merged record API
// (/api/metadata/hash-ids, /api/metadata/{rid});
// - writes of `file:` keys are signed claims (claimsApi.ts): put = a vote,
// del = a retraction of the chosen account's vote;
// - every other key is service scratch KV, through /api/kv/*.
import { ClaimsAPI } from './claimsApi';
const API_BASE = '/api/kv';
const RECORD_NS = 'file:';
export interface KVTreeResponse {
prefix: string;
delimiter: string;
branches: string[];
leaves: string[];
cursor: string;
hasMore: boolean;
}
export interface KVSearchResponse {
contains: string;
keys: string[];
truncated: boolean;
}
export interface KVFindMatch {
key: string;
value: string;
entryPath: string;
field: string;
}
export interface KVFindResponse {
contains: string;
fields: string[];
matches: KVFindMatch[];
truncated: boolean;
}
export interface KVValueResponse {
key: string;
type: string;
value: string;
exists: boolean;
}
export interface KVInfoResponse {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage: string;
}
async function jsonOrThrow<T>(res: Response): Promise<T> {
if (!res.ok) {
let msg = `HTTP ${res.status}`;
try {
const body = await res.json();
if (body?.message) msg = body.message;
else if (body?.error) msg = body.error;
} catch {}
throw new Error(msg);
}
return res.json();
}
// splitRecordKey decomposes file:<rid>/<field> (field may be empty).
export function splitRecordKey(key: string): { rid: string; field: string } | null {
if (!key.startsWith(RECORD_NS)) return null;
const rest = key.slice(RECORD_NS.length);
const slash = rest.indexOf('/');
if (slash <= 0) return null;
return { rid: rest.slice(0, slash), field: rest.slice(slash + 1) };
}
let hashIdsCache: { at: number; ids: string[] } | null = null;
async function hashIds(): Promise<string[]> {
if (hashIdsCache && Date.now() - hashIdsCache.at < 5000) return hashIdsCache.ids;
const r = await jsonOrThrow<{ hashIds: string[] }>(await fetch('/api/metadata/hash-ids'));
hashIdsCache = { at: Date.now(), ids: (r.hashIds ?? []).slice().sort() };
return hashIdsCache.ids;
}
async function recordFields(rid: string): Promise<Record<string, string>> {
const res = await fetch(`/api/metadata/${encodeURIComponent(rid)}`);
if (res.status === 404) return {};
const r = await jsonOrThrow<{ metadata: Record<string, string> }>(res);
return r.metadata ?? {};
}
// listUnder lists the children of prefix among keys, S3-style.
function listUnder(prefix: string, delimiter: string, keys: string[]) {
const branches = new Set<string>();
const leaves: string[] = [];
for (const k of keys) {
if (!k.startsWith(prefix)) continue;
const rest = k.slice(prefix.length);
const i = delimiter ? rest.indexOf(delimiter) : -1;
if (i >= 0) branches.add(prefix + rest.slice(0, i + delimiter.length));
else leaves.push(k);
}
return { branches: [...branches].sort(), leaves: leaves.sort() };
}
export const KVAPI = {
async info(): Promise<KVInfoResponse> {
return jsonOrThrow(await fetch(`${API_BASE}/info`));
},
async tree(params: {
prefix?: string;
delimiter?: string;
cursor?: string;
max?: number;
}): Promise<KVTreeResponse> {
const prefix = params.prefix ?? '';
const delimiter = params.delimiter ?? '';
const done = (branches: string[], leaves: string[]): KVTreeResponse =>
({ prefix, delimiter, branches, leaves, cursor: '0', hasMore: false });
if (prefix.startsWith(RECORD_NS)) {
const parts = splitRecordKey(prefix);
if (!parts) {
const ids = await hashIds();
return done(ids.map((id) => `${RECORD_NS}${id}/`), []);
}
const fields = await recordFields(parts.rid);
const keys = Object.keys(fields).map((f) => `${RECORD_NS}${parts.rid}/${f}`);
const l = listUnder(prefix, delimiter, keys);
return done(l.branches, l.leaves);
}
const q = new URLSearchParams();
q.set('prefix', prefix);
q.set('delimiter', delimiter);
if (params.cursor) q.set('cursor', params.cursor);
if (params.max) q.set('max', String(params.max));
const svc = await jsonOrThrow<KVTreeResponse>(await fetch(`${API_BASE}/tree?${q}`));
if (prefix === '' && delimiter) {
svc.branches = [RECORD_NS, ...svc.branches.filter((b) => b !== RECORD_NS)];
}
return svc;
},
async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
const seenLeaves = new Set<string>();
const seenBranches = new Set<string>();
let cursor = '';
while (true) {
const page = await this.tree({ prefix, delimiter, cursor, max: 1000 });
for (const k of page.leaves) seenLeaves.add(k);
for (const k of page.branches) seenBranches.add(k);
cursor = page.cursor;
if (!page.hasMore) break;
if (seenLeaves.size + seenBranches.size >= softCap) break;
}
return {
prefix,
delimiter,
branches: [...seenBranches].sort(),
leaves: [...seenLeaves].sort(),
cursor,
hasMore: cursor !== '0' && cursor !== '',
};
},
// search finds keys whose path contains the substring: records matched by
// free text (each matching field key), plus service keys.
async search(contains: string, limit = 500): Promise<KVSearchResponse> {
const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record<string, string> }[] }>(
await fetch('/api/metadata/search', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ query: contains, limit: 100 }),
}));
const needle = contains.toLowerCase();
const keys: string[] = [];
for (const hit of r.results ?? []) {
for (const [f, v] of Object.entries(hit.metadata)) {
const k = `${RECORD_NS}${hit.hashId}/${f}`;
if (k.toLowerCase().includes(needle) || v.toLowerCase().includes(needle)) keys.push(k);
}
}
const q = new URLSearchParams({ contains, limit: String(limit) });
const svc = await jsonOrThrow<KVSearchResponse>(await fetch(`${API_BASE}/search?${q}`));
const all = [...keys, ...svc.keys].sort();
return { contains, keys: all.slice(0, limit), truncated: all.length > limit || svc.truncated };
},
// find returns record fields (one of `fields`) whose value contains the
// substring, via the merged-record search.
async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
const matches: KVFindMatch[] = [];
let truncated = false;
for (const field of fields) {
const r = await jsonOrThrow<{ results: { hashId: string; metadata: Record<string, string> }[] }>(
await fetch('/api/metadata/search', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ filters: [{ property: field, values: [contains] }], limit }),
}));
for (const hit of r.results ?? []) {
const v = hit.metadata[field];
if (v === undefined) continue;
const entryPath = `${RECORD_NS}${hit.hashId}/`;
matches.push({ key: entryPath + field, value: v, entryPath, field });
if (matches.length >= limit) { truncated = true; break; }
}
if (truncated) break;
}
matches.sort((a, b) => a.key.localeCompare(b.key));
return { contains, fields, matches, truncated };
},
async get(key: string): Promise<KVValueResponse> {
const parts = splitRecordKey(key);
if (parts) {
const fields = await recordFields(parts.rid);
const value = fields[parts.field];
return { key, type: 'string', value: value ?? '', exists: value !== undefined };
}
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
},
// put on a record key is the chosen account's signed vote.
async put(key: string, value: string): Promise<KVValueResponse> {
const parts = splitRecordKey(key);
if (parts) {
await ClaimsAPI.vote(parts.rid, parts.field, value);
hashIdsCache = null;
return { key, type: 'string', value, exists: true };
}
const res = await fetch(`${API_BASE}/value`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ key, value }),
});
return jsonOrThrow(res);
},
// del on a record key retracts the chosen account's vote; the merged value
// stays while other signers still support it.
async del(key: string): Promise<{ key: string; deleted: boolean }> {
const parts = splitRecordKey(key);
if (parts) {
await ClaimsAPI.vote(parts.rid, parts.field, null);
return { key, deleted: true };
}
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
},
};
-151
View File
@@ -1,151 +0,0 @@
import {
FileMetadata,
SearchResponse,
UpdateResult,
BatchUpdateResponse,
} from '../types';
const API_BASE = '/api/metadata';
export class MetadataAPI {
/**
* Search for files by query, hashId, or property
*/
static async search(params: {
query?: string;
hashId?: string;
property?: string;
propertyValue?: string;
limit?: number;
}): Promise<SearchResponse> {
const response = await fetch(`${API_BASE}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Search failed');
}
return response.json();
}
/**
* Get metadata for a specific file by hash ID
*/
static async getMetadata(hashId: string): Promise<FileMetadata> {
const response = await fetch(`${API_BASE}/${hashId}`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get metadata');
}
return response.json();
}
/**
* Update complete metadata for a file
*/
static async updateMetadata(
hashId: string,
metadata: FileMetadata
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(metadata),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update metadata');
}
return response.json();
}
/**
* Update a specific property for a file
*/
static async updateProperty(
hashId: string,
property: string,
value: string
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}/property`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ property, value }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update property');
}
return response.json();
}
/**
* Batch update multiple files
*/
static async batchUpdate(
updates: Array<{
hashId: string;
metadata?: FileMetadata;
properties?: Record<string, string>;
}>
): Promise<BatchUpdateResponse> {
const response = await fetch(`${API_BASE}/batch`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ updates }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Batch update failed');
}
return response.json();
}
/**
* Delete metadata for a file
*/
static async deleteMetadata(hashId: string): Promise<void> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'DELETE',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to delete metadata');
}
}
/**
* Get all hash IDs
*/
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
const response = await fetch(`${API_BASE}/hash-ids`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get hash IDs');
}
return response.json();
}
}
+74
View File
@@ -0,0 +1,74 @@
import { useEffect, useState } from 'react';
export type SchemaPrimitive =
| 'string'
| 'int'
| 'float'
| 'bool'
| 'json-object'
| 'json-array'
| 'undefined'
| 'mixed';
export type SchemaHint = 'cid' | 'timestamp';
export type SchemaKeyHint = 'language-code';
export interface SchemaFieldInfo {
type: SchemaPrimitive;
hint?: SchemaHint;
key_hint?: SchemaKeyHint;
breakdown: Record<string, number>;
}
export interface SchemaResponse {
fields: Record<string, SchemaFieldInfo>;
generated_at: string;
source: 'live' | 'rescan';
}
let cached: Promise<SchemaResponse> | null = null;
export async function fetchSchema(): Promise<SchemaResponse> {
if (!cached) {
cached = fetch('/api/schema').then((r) => {
if (!r.ok) {
cached = null;
throw new Error(`schema fetch failed: ${r.status}`);
}
return r.json();
});
}
return cached;
}
/**
* Returns the live schema entry for a field, or undefined if not yet observed.
* Loads /api/schema lazily on first mount and caches the result process-wide.
*/
export function useSchema(): {
schema: SchemaResponse | null;
getFieldHint: (key: string) => SchemaFieldInfo | undefined;
} {
const [schema, setSchema] = useState<SchemaResponse | null>(null);
useEffect(() => {
let alive = true;
fetchSchema()
.then((s) => {
if (alive) setSchema(s);
})
.catch(() => {
// Schema endpoint missing or errored — editor falls back to plugin schemas.
});
return () => {
alive = false;
};
}, []);
const getFieldHint = (key: string): SchemaFieldInfo | undefined => {
if (!schema) return undefined;
return schema.fields[key];
};
return { schema, getFieldHint };
}
@@ -1,382 +0,0 @@
.bulk-metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.bulk-metadata-editor-header {
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-editor-header h2 {
margin: 0 0 0.5rem 0;
font-size: 1.5rem;
}
.bulk-info {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.6);
font-size: 0.95rem;
}
.bulk-metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.bulk-metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.bulk-metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.bulk-metadata-editor-form {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
.bulk-metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.bulk-metadata-section-header {
margin-bottom: 1.5rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.section-hint {
margin: 0.25rem 0 0 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.section-hint-bulk {
margin: 0.5rem 0 0 0;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
font-weight: 500;
}
/* Plugin-based bulk fields */
.bulk-fields-container {
display: flex;
flex-direction: column;
gap: 1rem;
}
.bulk-field-wrapper {
border: 1px solid rgba(255, 255, 255, 0.05);
border-radius: 4px;
padding: 0.75rem;
background-color: rgba(0, 0, 0, 0.1);
transition: all 0.2s ease;
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.bulk-field-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.field-key {
font-size: 0.85rem;
font-weight: 600;
color: rgba(255, 255, 255, 0.6);
text-transform: lowercase;
font-family: 'Courier New', monospace;
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.bulk-field-input {
/* Plugin component will render inside this */
}
/* Legacy bulk fields (for compatibility) */
.bulk-metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.bulk-metadata-field:last-child {
margin-bottom: 0;
}
.field-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.field-header label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.clear-field-btn {
width: 20px;
height: 20px;
padding: 0;
border: 1px solid rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
border-radius: 50%;
cursor: pointer;
font-size: 16px;
line-height: 1;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
.bulk-metadata-field input,
.bulk-metadata-field textarea,
.bulk-metadata-field select {
width: 100%;
}
.bulk-metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.bulk-metadata-field-row:last-child {
margin-bottom: 0;
}
/* Modified fields summary */
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-fields h3 {
color: #646cff;
}
.modified-fields-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.modified-field-item {
display: flex;
gap: 0.75rem;
align-items: center;
padding: 0.75rem;
background-color: rgba(100, 108, 255, 0.05);
border: 1px solid rgba(100, 108, 255, 0.2);
border-radius: 4px;
}
.field-name {
font-weight: 600;
color: #646cff;
min-width: 150px;
font-family: 'Courier New', monospace;
font-size: 0.9rem;
}
.field-value {
flex: 1;
word-break: break-word;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
}
.remove-field-btn {
padding: 0.25rem 0.75rem;
font-size: 0.85rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
white-space: nowrap;
}
.remove-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
@media (max-width: 768px) {
.bulk-metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.bulk-metadata-editor-actions {
width: 100%;
flex-direction: column;
}
.bulk-metadata-editor-actions button {
width: 100%;
}
.bulk-metadata-field-row {
grid-template-columns: 1fr;
}
.modified-field-item {
flex-direction: column;
align-items: flex-start;
}
.field-name {
min-width: auto;
}
.remove-field-btn {
align-self: flex-end;
}
}
@media (prefers-color-scheme: light) {
.bulk-metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.bulk-info {
color: rgba(0, 0, 0, 0.6);
}
.bulk-metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.section-hint {
color: rgba(0, 0, 0, 0.5);
}
.section-hint-bulk {
color: rgba(0, 0, 0, 0.4);
}
.bulk-field-wrapper {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.field-key {
color: rgba(0, 0, 0, 0.6);
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.field-header label {
color: rgba(0, 0, 0, 0.7);
}
.field-value {
color: rgba(0, 0, 0, 0.8);
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-field-item {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.2);
}
.clear-field-btn {
border-color: rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #cc0000;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
}
@@ -1,280 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './BulkMetadataEditor.css';
interface PluginsResponse {
plugins: PluginSchema[];
}
interface BulkMetadataEditorProps {
hashIds: string[];
onSaved?: () => void;
onCancel?: () => void;
}
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
// Load plugins and metadata on mount
useEffect(() => {
const loadData = async () => {
try {
setIsLoading(true);
setError(null);
// Fetch plugins
const pluginsResponse = await fetch('/api/plugins');
if (pluginsResponse.ok) {
const data: PluginsResponse = await pluginsResponse.json();
// Sort by priority
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
// Load metadata for all selected files
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
const metadata = await Promise.all(metadataPromises);
// Calculate common values
const commons: Record<string, any> = {};
// Get all editable fields from all plugins
const allFields: string[] = [];
plugins.forEach((plugin) => {
if (plugin.metadataSchema) {
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
if (!field.readonly) {
allFields.push(key);
}
});
}
});
allFields.forEach((field) => {
const values = metadata.map((m) => m[field as keyof FileMetadata]);
// Filter out undefined and null values
const definedValues = values.filter((v) => v !== undefined && v !== null);
if (definedValues.length === 0) {
return;
}
// Safely stringify values
const uniqueValues = Array.from(
new Set(
definedValues.map((v) => {
try {
return JSON.stringify(v);
} catch {
return String(v);
}
})
)
);
if (uniqueValues.length === 1) {
try {
commons[field] = JSON.parse(uniqueValues[0]);
} catch {
commons[field] = uniqueValues[0];
}
} else {
commons[field] = '*';
}
});
setCommonValues(commons);
} catch (err: any) {
setError(`Failed to load data: ${err.message}`);
} finally {
setIsLoading(false);
}
};
loadData();
}, [hashIds]);
const handleFieldChange = (field: string, value: any) => {
setEditedFields({
...editedFields,
[field]: value,
});
};
const handleClearField = (field: string) => {
const newFields = { ...editedFields };
delete newFields[field];
setEditedFields(newFields);
};
const handleSave = async () => {
if (Object.keys(editedFields).length === 0) {
setError('No fields have been edited');
return;
}
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
// Build batch update payload
const updates = hashIds.map((hashId) => ({
hashId,
metadata: editedFields,
}));
const result = await MetadataAPI.batchUpdate(updates);
if (result.errors > 0) {
setError(
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
);
console.error('Batch update errors:', result.results.filter((r) => r.error));
} else {
setSuccessMessage(`Successfully updated ${result.success} files!`);
}
if (onSaved) {
onSaved();
}
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const hasChanges = Object.keys(editedFields).length > 0;
// Filter to only editable plugins (those with non-readonly fields)
const editablePlugins = plugins.filter((plugin) => {
if (!plugin.metadataSchema) return false;
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
});
if (isLoading) {
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-loading">
Loading metadata for {hashIds.length} files...
</div>
</div>
);
}
// Build display metadata (edited values or common values)
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-header">
<h2>Bulk Edit Metadata</h2>
<p className="bulk-info">
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
be updated. Fields showing "*" have different values across files.
</p>
<div className="bulk-metadata-editor-actions">
{onCancel && (
<button onClick={onCancel} disabled={isSaving}>
Cancel
</button>
)}
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
</button>
</div>
</div>
{successMessage && (
<div className="bulk-metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
<div className="bulk-metadata-editor-form">
{editablePlugins.map((plugin) => {
if (!plugin.metadataSchema) return null;
// Get editable fields for this plugin
const editableFields = Object.entries(plugin.metadataSchema).filter(
([, field]) => !field.readonly
);
if (editableFields.length === 0) return null;
return (
<section
key={plugin.id}
className="bulk-metadata-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="bulk-metadata-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="section-hint">{plugin.description}</p>
)}
<p className="section-hint-bulk">
Common values shown. "*" indicates different values. Edit to update all files.
</p>
</div>
<div className="plugin-fields">
{editableFields.map(([key, field]) => (
<DynamicFieldRenderer
key={key}
fieldKey={key}
schema={field}
metadata={displayMetadata as FileMetadata}
onChange={handleFieldChange}
readonly={false}
/>
))}
</div>
</section>
);
})}
{/* Modified Fields Summary */}
{hasChanges && (
<section className="bulk-metadata-section modified-fields">
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
<div className="modified-fields-list">
{Object.entries(editedFields).map(([key, value]) => (
<div key={key} className="modified-field-item">
<span className="field-name">{key}:</span>
<span className="field-value">
{Array.isArray(value)
? value.join(', ')
: typeof value === 'object'
? JSON.stringify(value)
: String(value || '(empty)')}
</span>
<button
className="remove-field-btn"
onClick={() => handleClearField(key)}
title="Remove this field"
>
Remove
</button>
</div>
))}
</div>
</section>
)}
</div>
</div>
);
}
@@ -1,297 +0,0 @@
.dynamic-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.dynamic-field:last-child {
margin-bottom: 0;
}
.dynamic-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
width: 100%;
padding: 0.5rem 0.75rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-size: 0.9rem;
background: rgba(0, 0, 0, 0.3);
color: #fff;
transition: border-color 0.2s;
}
.dynamic-field input:focus,
.dynamic-field select:focus,
.dynamic-field textarea:focus {
outline: none;
border-color: #646cff;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: rgba(0, 0, 0, 0.2);
color: rgba(255, 255, 255, 0.5);
cursor: not-allowed;
}
.dynamic-field textarea {
resize: vertical;
min-height: 80px;
font-family: inherit;
}
.field-hint {
display: block;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
cursor: pointer;
font-weight: normal;
color: rgba(255, 255, 255, 0.7);
}
.checkbox-label input[type="checkbox"] {
width: 1.1rem;
height: 1.1rem;
cursor: pointer;
accent-color: #646cff;
}
.checkbox-label input:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.readonly-value {
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.7);
font-family: 'Courier New', monospace;
font-size: 0.9rem;
word-break: break-all;
}
.field-link {
color: #646cff;
text-decoration: none;
}
.field-link:hover {
text-decoration: underline;
}
/* CID field styling */
.cid-field .cid-value-container {
display: flex;
align-items: center;
gap: 0.5rem;
}
.cid-field .cid-value {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: #a0d8ef;
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cid-field .cid-value:hover {
background: rgba(0, 0, 0, 0.4);
border-color: #646cff;
}
.cid-field .cid-open-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 0.75rem;
background: #646cff;
color: #fff;
border-radius: 4px;
text-decoration: none;
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
transition: background 0.2s;
}
.cid-field .cid-open-button:hover {
background: #535bf2;
}
/* JSON field styling */
.json-field .json-viewer {
background: rgba(0, 0, 0, 0.4);
color: #d4d4d4;
padding: 1rem;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
overflow-x: auto;
max-height: 300px;
overflow-y: auto;
margin: 0;
white-space: pre-wrap;
word-break: break-all;
border: 1px solid rgba(255, 255, 255, 0.1);
}
/* Language string field styling */
.language-string-field .language-entries {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.language-string-field .language-entry {
display: flex;
align-items: center;
gap: 0.5rem;
}
.language-string-field .lang-input {
flex: 1;
min-width: 0;
}
.language-string-field .lang-code {
display: inline-block;
width: 50px;
padding: 0.4rem 0.5rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.8rem;
color: #a0d8ef;
text-align: center;
flex-shrink: 0;
}
.language-string-field .lang-text {
flex: 1;
padding: 0.4rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.language-string-field .lang-remove-btn {
width: 32px;
height: 32px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 100, 100, 0.2);
border: 1px solid rgba(255, 100, 100, 0.3);
border-radius: 4px;
color: #ff6b6b;
font-size: 1.2rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-remove-btn:hover {
background: rgba(255, 100, 100, 0.3);
border-color: rgba(255, 100, 100, 0.5);
}
.language-string-field .lang-add-btn {
align-self: flex-start;
padding: 0.4rem 0.75rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
color: #646cff;
font-size: 0.85rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-add-btn:hover {
background: rgba(100, 108, 255, 0.3);
border-color: rgba(100, 108, 255, 0.5);
}
/* Light theme */
@media (prefers-color-scheme: light) {
.dynamic-field label {
color: rgba(0, 0, 0, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
border-color: rgba(0, 0, 0, 0.1);
background: #fff;
color: #333;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: #f5f5f5;
color: rgba(0, 0, 0, 0.5);
}
.field-hint {
color: rgba(0, 0, 0, 0.5);
}
.checkbox-label {
color: rgba(0, 0, 0, 0.7);
}
.readonly-value {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
.json-field .json-viewer {
background: #1e1e1e;
border-color: rgba(0, 0, 0, 0.2);
}
.language-string-field .lang-code {
background: rgba(100, 108, 255, 0.1);
color: #4a50c8;
}
.language-string-field .lang-text {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
}
@@ -1,378 +0,0 @@
import { SchemaField, FileMetadata } from '../types';
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
import './DynamicFieldRenderer.css';
interface DynamicFieldRendererProps {
fieldKey: string;
schema: SchemaField;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
readonly?: boolean;
}
/**
* Renders a single metadata field based on its schema definition
*/
export function DynamicFieldRenderer({
fieldKey,
schema,
metadata,
onChange,
readonly = false,
}: DynamicFieldRendererProps) {
const isReadonly = readonly || schema.readonly;
const fieldType = schema.type || 'string';
// Get value from metadata using path-style key
const getValue = (): any => {
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = fieldKey.split('/');
let value: any = metadata;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
};
// Set value in metadata using path-style key
const setValue = (newValue: any) => {
onChange(fieldKey, newValue);
};
const value = getValue();
// Render link if configured
const renderValueWithLink = (displayValue: string) => {
if (schema.link && displayValue) {
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
return (
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
{displayValue}
</a>
);
}
return displayValue;
};
// Render based on field type
switch (fieldType) {
case 'boolean':
return (
<div className="dynamic-field">
<label className="checkbox-label">
<input
type="checkbox"
checked={Boolean(value)}
onChange={(e) => setValue(e.target.checked)}
disabled={isReadonly}
/>
<span>{schema.label}</span>
</label>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'number':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="number"
value={value ?? ''}
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
disabled={isReadonly}
min={schema.min}
max={schema.max}
placeholder={schema.placeholder}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'select':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<select
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
>
<option value="">Select...</option>
{schema.options?.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'text':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<textarea
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
placeholder={schema.placeholder}
rows={4}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'array':
// Convert RecordSet to comma-separated string for display
const arrayValue = (() => {
if (!value) return '';
if (typeof value === 'string') return value;
if (typeof value === 'object') {
return Object.values(value).join(', ');
}
return '';
})();
// Convert comma-separated string back to RecordSet
const handleArrayChange = (str: string) => {
const items = str.split(',').map(s => s.trim()).filter(Boolean);
if (items.length === 0) {
setValue(undefined);
return;
}
const recordSet: Record<string, string> = {};
items.forEach((item, idx) => {
recordSet[idx.toString()] = item;
});
setValue(recordSet);
};
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="text"
value={arrayValue}
onChange={(e) => handleArrayChange(e.target.value)}
disabled={isReadonly}
placeholder={schema.placeholder || 'Comma-separated values'}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'json':
// Aggregate matching keys based on pattern
const jsonData = (() => {
if (!schema.pattern) {
return value;
}
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
const result: Record<string, any> = {};
// Recursively find all matching keys
const findMatching = (obj: any, currentPath: string = '') => {
if (!obj || typeof obj !== 'object') return;
for (const [key, val] of Object.entries(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
// Store in result with relative path
const relativePath = newPath.substring(patternPrefix.length - 1);
if (relativePath) {
setNestedValue(result, relativePath, val);
}
}
if (typeof val === 'object' && val !== null) {
findMatching(val, newPath);
}
}
};
findMatching(metadata);
return Object.keys(result).length > 0 ? result : undefined;
})();
if (!jsonData) {
return null;
}
return (
<div className="dynamic-field json-field">
<label>{schema.label}</label>
<pre className="json-viewer">
{JSON.stringify(jsonData, null, 2)}
</pre>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'cid':
// CID field type - displays CID with "Open in Editor" button
if (!value) {
return null;
}
const cidValue = String(value);
const editorUrl = `/editor/file/${cidValue}`;
return (
<div className="dynamic-field cid-field">
<label>{schema.label}</label>
<div className="cid-value-container">
<code
className="cid-value"
onClick={() => navigator.clipboard.writeText(cidValue)}
title="Click to copy"
>
{cidValue}
</code>
<a
href={editorUrl}
className="cid-open-button"
title="Open in Editor"
target="_blank"
rel="noopener noreferrer"
>
↗ Open
</a>
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'languageString':
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
const langEntries: Array<[string, string]> = value && typeof value === 'object'
? Object.entries(value).map(([k, v]) => [k, String(v)])
: [];
const handleLangCodeChange = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const newValue = { ...value };
const text = newValue[oldLang];
delete newValue[oldLang];
newValue[newLang] = text;
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleLangTextChange = (lang: string, text: string) => {
const newValue = { ...value, [lang]: text };
setValue(newValue);
};
const handleRemoveLang = (lang: string) => {
const newValue = { ...value };
delete newValue[lang];
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleAddLang = () => {
// Find a language code not already used, default to 'eng'
const usedLangs = new Set(langEntries.map(([k]) => k));
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
setValue({ ...value, [defaultLang]: '' });
};
if (isReadonly && langEntries.length === 0) {
return null;
}
return (
<div className="dynamic-field language-string-field">
<label>{schema.label}</label>
<div className="language-entries">
{langEntries.map(([lang, text]) => (
<div key={lang} className="language-entry">
{isReadonly ? (
<>
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
<span className="lang-text">{text}</span>
</>
) : (
<>
<LanguageCombobox
value={lang}
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
/>
<input
type="text"
className="lang-input"
value={text}
onChange={(e) => handleLangTextChange(lang, e.target.value)}
placeholder={schema.placeholder || 'Text...'}
/>
<button
type="button"
className="lang-remove-btn"
onClick={() => handleRemoveLang(lang)}
title="Remove"
>
×
</button>
</>
)}
</div>
))}
{!isReadonly && (
<button
type="button"
className="lang-add-btn"
onClick={handleAddLang}
>
+ Add Language
</button>
)}
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'string':
default:
// Show nothing if no value and readonly
if (isReadonly && !value) {
return null;
}
return (
<div className="dynamic-field">
<label>{schema.label}</label>
{isReadonly ? (
<div className="readonly-value">
{renderValueWithLink(String(value ?? ''))}
</div>
) : (
<input
type="text"
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
placeholder={schema.placeholder}
/>
)}
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
}
}
// Helper to set nested value in object
function setNestedValue(obj: any, path: string, value: any) {
const parts = path.split('/').filter(Boolean);
let current = obj;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
}
current = current[parts[i]];
}
if (parts.length > 0) {
current[parts[parts.length - 1]] = value;
}
}
@@ -1,63 +0,0 @@
.plugin-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.plugin-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-section-header .plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.plugin-fields {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Responsive grid for fields */
@media (min-width: 768px) {
.plugin-fields {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1rem;
}
/* Make text and json fields span full width */
.plugin-fields .dynamic-field:has(textarea),
.plugin-fields .json-field {
grid-column: 1 / -1;
}
}
/* Light theme */
@media (prefers-color-scheme: light) {
.plugin-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-section-header .plugin-description {
color: rgba(0, 0, 0, 0.5);
}
}
@@ -1,126 +0,0 @@
import { PluginSchema, FileMetadata, SchemaField } from '../types';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './DynamicPluginSection.css';
interface DynamicPluginSectionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
}
/**
* Renders a plugin section with all its fields based on schema
*/
export function DynamicPluginSection({
plugin,
metadata,
onChange,
}: DynamicPluginSectionProps) {
const schema = plugin.metadataSchema;
if (!schema || Object.keys(schema).length === 0) {
return null;
}
// Check if plugin has any data in metadata
const hasData = checkPluginHasData(schema, metadata);
if (!hasData) {
return null;
}
// Sort fields: non-readonly first, then by key
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
// Non-readonly fields first
if (a.readonly !== b.readonly) {
return a.readonly ? 1 : -1;
}
// Then alphabetically
return keyA.localeCompare(keyB);
});
return (
<section
className="plugin-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="plugin-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="plugin-description">{plugin.description}</p>
)}
</div>
<div className="plugin-fields">
{sortedFields.map(([fieldKey, fieldSchema]) => (
<DynamicFieldRenderer
key={fieldKey}
fieldKey={fieldKey}
schema={fieldSchema}
metadata={metadata}
onChange={onChange}
/>
))}
</div>
</section>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function checkPluginHasData(
schema: Record<string, SchemaField>,
metadata: FileMetadata
): boolean {
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
/**
* Get nested value from object using path-style key
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
/**
* Check if object has any key starting with prefix
*/
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
-174
View File
@@ -1,174 +0,0 @@
.file-list {
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
overflow: hidden;
}
.file-list-header {
padding: 0.75rem 1rem;
background-color: rgba(255, 255, 255, 0.05);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
justify-content: space-between;
align-items: center;
}
.file-count {
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
}
.bulk-edit-toggle {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(100, 108, 255, 0.1);
border: 1px solid rgba(100, 108, 255, 0.3);
color: #646cff;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.bulk-edit-toggle:hover {
background-color: rgba(100, 108, 255, 0.2);
border-color: rgba(100, 108, 255, 0.5);
}
.bulk-edit-toggle.active {
background-color: rgba(100, 108, 255, 0.3);
border-color: #646cff;
font-weight: 500;
}
.file-list-bulk-actions {
padding: 0.75rem 1rem;
background-color: rgba(100, 108, 255, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
gap: 1rem;
align-items: center;
}
.select-all-btn {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(255, 255, 255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.2);
color: rgba(255, 255, 255, 0.9);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.select-all-btn:hover {
background-color: rgba(255, 255, 255, 0.2);
border-color: rgba(255, 255, 255, 0.3);
}
.selection-count {
font-size: 0.9em;
color: #646cff;
font-weight: 500;
}
.file-list-items {
max-height: 500px;
overflow-y: auto;
}
.file-list-item {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
cursor: pointer;
transition: background-color 0.2s;
display: flex;
gap: 0.75rem;
align-items: flex-start;
}
.file-list-item-checkbox {
padding-top: 0.2rem;
}
.file-list-item-checkbox input[type='checkbox'] {
width: 18px;
height: 18px;
cursor: pointer;
}
.file-list-item-content {
flex: 1;
}
.file-list-item:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.2);
border-left: 3px solid #646cff;
}
.file-list-item-title {
font-weight: 500;
margin-bottom: 0.5rem;
}
.file-list-item-details {
display: flex;
gap: 1rem;
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
margin-bottom: 0.25rem;
}
.file-list-item-hash {
font-size: 0.8em;
color: rgba(255, 255, 255, 0.4);
font-family: 'Courier New', monospace;
}
.file-list-empty {
padding: 3rem 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
@media (prefers-color-scheme: light) {
.file-list {
border-color: rgba(0, 0, 0, 0.1);
}
.file-list-header {
background-color: rgba(0, 0, 0, 0.02);
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.file-count {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.file-list-item:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.1);
}
.file-list-item-details {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item-hash {
color: rgba(0, 0, 0, 0.4);
}
.file-list-empty {
color: rgba(0, 0, 0, 0.5);
}
}
-123
View File
@@ -1,123 +0,0 @@
import { SearchResult } from '../types';
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
import './FileList.css';
interface FileListProps {
results: SearchResult[];
selectedHashId?: string;
selectedHashIds: Set<string>;
bulkEditMode: boolean;
onSelectFile: (hashId: string) => void;
onToggleSelection: (hashId: string) => void;
onSelectAll: () => void;
onClearSelection: () => void;
onBulkEditMode: (enabled: boolean) => void;
}
export function FileList({
results,
selectedHashId,
selectedHashIds,
bulkEditMode,
onSelectFile,
onToggleSelection,
onSelectAll,
onClearSelection,
onBulkEditMode,
}: FileListProps) {
if (results.length === 0) {
return (
<div className="file-list-empty">
<p>No files found. Try searching for a title, filename, or hash ID.</p>
</div>
);
}
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
// Sort results by file path
const sortedResults = [...results].sort((a, b) => {
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
return pathA.localeCompare(pathB);
});
return (
<div className="file-list">
<div className="file-list-header">
<span className="file-count">{results.length} files found</span>
<button
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
onClick={() => onBulkEditMode(!bulkEditMode)}
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
>
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
</button>
</div>
{bulkEditMode && (
<div className="file-list-bulk-actions">
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
{allSelected ? 'Deselect All' : 'Select All'}
</button>
{selectedHashIds.size > 0 && (
<span className="selection-count">
{selectedHashIds.size} selected
</span>
)}
</div>
)}
<div className="file-list-items">
{sortedResults.map((result) => {
const isSelected = bulkEditMode
? selectedHashIds.has(result.hashId)
: selectedHashId === result.hashId;
return (
<div
key={result.hashId}
className={`file-list-item ${isSelected ? 'selected' : ''}`}
onClick={() => {
if (bulkEditMode) {
onToggleSelection(result.hashId);
} else {
onSelectFile(result.hashId);
}
}}
>
{bulkEditMode && (
<div className="file-list-item-checkbox">
<input
type="checkbox"
checked={selectedHashIds.has(result.hashId)}
onChange={(e) => {
e.stopPropagation();
onToggleSelection(result.hashId);
}}
/>
</div>
)}
<div className="file-list-item-content">
<div className="file-list-item-title">
{getDisplayName(result.metadata)}
</div>
<div className="file-list-item-details">
<span className="file-list-item-type">
{result.metadata.videoType || 'Unknown'}
</span>
<span className="file-list-item-size">
{formatBytes(result.metadata.byteSize)}
</span>
</div>
<div className="file-list-item-hash">
Hash: {truncateHash(result.hashId)}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}
-247
View File
@@ -1,247 +0,0 @@
.kv-tree {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.1);
overflow: hidden;
}
.kv-tree-header {
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.kv-tree-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.8);
}
.kv-tree-info {
display: flex;
gap: 1rem;
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-refresh {
margin-left: auto;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
background-color: rgba(100, 108, 255, 0.2);
color: #646cff;
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.kv-tree-refresh:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.3);
}
.kv-tree-refresh:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-error {
padding: 0.5rem 1rem;
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
font-size: 0.8rem;
}
.kv-tree-content {
max-height: 400px;
overflow-y: auto;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 0.8rem;
}
.kv-tree-node {
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-node:last-child {
border-bottom: none;
}
.kv-tree-key {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-key:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-toggle {
width: 1rem;
color: rgba(255, 255, 255, 0.4);
font-size: 0.65rem;
}
.kv-tree-key-name {
flex: 1;
color: #88c0d0;
word-break: break-all;
}
.kv-tree-type {
padding: 0.1rem 0.4rem;
font-size: 0.65rem;
background-color: rgba(255, 255, 255, 0.1);
border-radius: 3px;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-value {
padding: 0.5rem 1rem 0.5rem 2.5rem;
background-color: rgba(0, 0, 0, 0.2);
border-top: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-fields {
width: 100%;
border-collapse: collapse;
}
.kv-tree-fields tr:hover {
background-color: rgba(255, 255, 255, 0.03);
}
.kv-field-name {
padding: 0.25rem 0.5rem 0.25rem 0;
color: #a3be8c;
white-space: nowrap;
vertical-align: top;
width: 1%;
}
.kv-field-value {
padding: 0.25rem 0;
color: rgba(255, 255, 255, 0.7);
word-break: break-all;
}
.kv-field-value pre {
margin: 0;
font-family: inherit;
white-space: pre-wrap;
font-size: 0.75rem;
}
.kv-tree-load-more {
display: block;
width: 100%;
padding: 0.75rem;
font-size: 0.8rem;
background-color: transparent;
color: #646cff;
border: none;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-empty {
padding: 2rem;
text-align: center;
color: rgba(255, 255, 255, 0.4);
}
/* Light mode */
@media (prefers-color-scheme: light) {
.kv-tree {
background-color: rgba(0, 0, 0, 0.02);
border-color: rgba(0, 0, 0, 0.1);
}
.kv-tree-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-header h3 {
color: rgba(0, 0, 0, 0.8);
}
.kv-tree-info {
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-error {
background-color: rgba(255, 0, 0, 0.08);
color: #cc0000;
}
.kv-tree-node {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-key:hover {
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-toggle {
color: rgba(0, 0, 0, 0.4);
}
.kv-tree-key-name {
color: #0077aa;
}
.kv-tree-type {
background-color: rgba(0, 0, 0, 0.08);
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-value {
background-color: rgba(0, 0, 0, 0.02);
border-top-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-fields tr:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.kv-field-name {
color: #228822;
}
.kv-field-value {
color: rgba(0, 0, 0, 0.7);
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-empty {
color: rgba(0, 0, 0, 0.4);
}
}
-216
View File
@@ -1,216 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import './KVTreeView.css';
interface KeyValue {
key: string;
type?: string;
value?: Record<string, any> | string;
expanded?: boolean;
loading?: boolean;
}
interface KVInfo {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage?: string;
}
const API_BASE = '/api/kv';
export function KVTreeView() {
const [keys, setKeys] = useState<KeyValue[]>([]);
const [cursor, setCursor] = useState<string>('0');
const [hasMore, setHasMore] = useState(true);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [info, setInfo] = useState<KVInfo | null>(null);
// Load KV info on mount
useEffect(() => {
fetch(`${API_BASE}/info`)
.then(res => res.json())
.then(data => setInfo(data))
.catch(err => console.error('Failed to load KV info:', err));
}, []);
// Load initial keys
const loadKeys = useCallback(async (reset = false) => {
if (loading) return;
const currentCursor = reset ? '0' : cursor;
if (!reset && currentCursor === '0' && keys.length > 0) return;
setLoading(true);
setError(null);
try {
const res = await fetch(`${API_BASE}/keys?cursor=${currentCursor}&count=50`);
if (!res.ok) throw new Error('Failed to load keys');
const data = await res.json();
const newKeys: KeyValue[] = data.keys.map((key: string) => ({
key,
expanded: false,
loading: false
}));
if (reset) {
setKeys(newKeys);
} else {
setKeys(prev => [...prev, ...newKeys]);
}
setCursor(data.cursor);
setHasMore(data.hasMore);
} catch (err: any) {
setError(err.message);
} finally {
setLoading(false);
}
}, [cursor, keys.length, loading]);
// Load on mount
useEffect(() => {
loadKeys(true);
}, []);
// Toggle key expansion and load value
const toggleKey = async (index: number) => {
const key = keys[index];
if (key.expanded) {
// Collapse
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: false } : k
));
return;
}
// Expand and load value if not already loaded
if (key.value === undefined) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: true } : k
));
try {
const res = await fetch(`${API_BASE}/key/${encodeURIComponent(key.key)}`);
if (!res.ok) throw new Error('Failed to load value');
const data = await res.json();
setKeys(prev => prev.map((k, i) =>
i === index ? {
...k,
type: data.type,
value: data.value,
expanded: true,
loading: false
} : k
));
} catch (err) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: false, value: 'Error loading value' } : k
));
}
} else {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: true } : k
));
}
};
const formatValue = (value: any): string => {
if (value === null) return 'null';
if (value === undefined) return 'undefined';
if (typeof value === 'object') {
return JSON.stringify(value, null, 2);
}
return String(value);
};
const formatSize = (bytes: number): string => {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
};
return (
<div className="kv-tree">
<div className="kv-tree-header">
<h3>KV Database Browser</h3>
{info && (
<div className="kv-tree-info">
<span>Keys: {info.keyCount}</span>
<span>Files: {formatSize(info.totalSize)}</span>
{info.memoryUsage && <span>Memory: {info.memoryUsage}</span>}
</div>
)}
<button
className="kv-tree-refresh"
onClick={() => loadKeys(true)}
disabled={loading}
>
{loading ? 'Loading...' : 'Refresh'}
</button>
</div>
{error && <div className="kv-tree-error">{error}</div>}
<div className="kv-tree-content">
{keys.map((item, index) => (
<div key={item.key} className="kv-tree-node">
<div
className={`kv-tree-key ${item.expanded ? 'expanded' : ''}`}
onClick={() => toggleKey(index)}
>
<span className="kv-tree-toggle">
{item.loading ? '...' : (item.expanded ? '▼' : '▶')}
</span>
<span className="kv-tree-key-name">{item.key}</span>
{item.type && <span className="kv-tree-type">{item.type}</span>}
</div>
{item.expanded && item.value && (
<div className="kv-tree-value">
{typeof item.value === 'object' ? (
<table className="kv-tree-fields">
<tbody>
{Object.entries(item.value).map(([field, val]) => (
<tr key={field}>
<td className="kv-field-name">{field}</td>
<td className="kv-field-value">
<pre>{formatValue(val)}</pre>
</td>
</tr>
))}
</tbody>
</table>
) : (
<pre>{formatValue(item.value)}</pre>
)}
</div>
)}
</div>
))}
{hasMore && (
<button
className="kv-tree-load-more"
onClick={() => loadKeys()}
disabled={loading}
>
{loading ? 'Loading...' : 'Load More'}
</button>
)}
{!loading && keys.length === 0 && (
<div className="kv-tree-empty">No keys found</div>
)}
</div>
</div>
);
}
@@ -8,15 +8,6 @@ for (const lang of iso6393) {
languagesByCode.set(lang.iso6393, lang);
}
// Get display name for a language code
export function getLanguageName(code: string): string {
const lang = languagesByCode.get(code);
if (lang) {
return lang.name;
}
return code.toUpperCase();
}
// Format for display: "English (eng)" or "eng" for unknown
export function formatLanguageOption(code: string): string {
const lang = languagesByCode.get(code);
-438
View File
@@ -1,438 +0,0 @@
.metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.metadata-editor-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-header h2 {
margin: 0;
font-size: 1.5rem;
}
.metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.metadata-editor-actions .download-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 1rem;
background-color: #28a745;
color: white;
border-radius: 4px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: background-color 0.2s;
}
.metadata-editor-actions .download-button:hover {
background-color: #218838;
}
.metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.metadata-editor-hashes {
margin-bottom: 1.5rem;
padding: 1rem;
background: rgba(0, 0, 0, 0.3);
border-radius: 8px;
border-left: 4px solid #888;
}
.hashes-header {
margin-bottom: 0.75rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hashes-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.7);
text-transform: uppercase;
letter-spacing: 0.05em;
}
.hashes-grid {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.hash-value {
display: flex;
flex-direction: column;
gap: 0.15rem;
}
.hash-value-primary {
margin-bottom: 0.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hash-value-primary .hash-label {
color: #4ecdc4;
}
.hash-value-primary .hash-code {
color: #4ecdc4;
font-size: 1rem;
}
.hash-label {
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.4);
text-transform: uppercase;
letter-spacing: 0.03em;
}
.hash-code {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.6);
word-break: break-all;
cursor: pointer;
transition: color 0.2s;
}
.hash-code:hover {
color: #fff;
}
/* Plugin Chips */
.metadata-editor-chips {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-bottom: 1.5rem;
}
.plugin-chip {
padding: 0.4rem 0.8rem;
border-radius: 16px;
font-size: 0.85rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
border: 2px solid var(--chip-color);
color: var(--chip-color);
background: transparent;
}
.plugin-chip:hover {
background-color: var(--chip-color);
color: white;
}
.plugin-chip.active {
color: white;
}
/* Plugin Accordion */
.plugin-accordion {
margin-bottom: 0.5rem;
border-radius: 8px;
overflow: hidden;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-accordion-header {
width: 100%;
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem 1.25rem;
border: none;
border-left: 4px solid #646cff;
background-color: rgba(255, 255, 255, 0.05);
color: inherit;
font-size: 1rem;
font-weight: 500;
cursor: pointer;
transition: background-color 0.2s ease;
}
.plugin-accordion-header:hover {
background-color: rgba(255, 255, 255, 0.1);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(255, 255, 255, 0.08);
}
.plugin-accordion-title {
color: inherit;
}
.plugin-accordion-icon {
font-size: 0.75rem;
opacity: 0.6;
transition: transform 0.2s ease;
}
.plugin-accordion-content {
padding: 1rem 1.25rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-form {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.metadata-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.metadata-field:last-child {
margin-bottom: 0;
}
.metadata-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.metadata-field input,
.metadata-field textarea,
.metadata-field select {
width: 100%;
}
.metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.metadata-field-row:last-child {
margin-bottom: 0;
}
.hash-field {
font-family: 'Courier New', monospace;
font-size: 0.9em;
}
.no-hash-info {
margin: 0;
padding: 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
padding: 3rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
.metadata-editor-error button {
margin-top: 1rem;
}
@media (max-width: 768px) {
.metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.metadata-editor-actions {
width: 100%;
}
.metadata-editor-actions button {
flex: 1;
}
.metadata-field-row {
grid-template-columns: 1fr;
}
}
@media (prefers-color-scheme: light) {
.metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-description {
color: rgba(0, 0, 0, 0.5);
}
.metadata-field label {
color: rgba(0, 0, 0, 0.7);
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
color: rgba(0, 0, 0, 0.5);
}
.no-hash-info {
color: rgba(0, 0, 0, 0.5);
}
.metadata-editor-hashes {
background: rgba(0, 0, 0, 0.05);
}
.hashes-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hashes-header h3 {
color: rgba(0, 0, 0, 0.7);
}
.hash-value-primary {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hash-value-primary .hash-label,
.hash-value-primary .hash-code {
color: #0a9e94;
}
.hash-label {
color: rgba(0, 0, 0, 0.4);
}
.hash-code {
color: rgba(0, 0, 0, 0.6);
}
.hash-code:hover {
color: #000;
}
/* Plugin Chips - Light theme */
.plugin-chip {
background: transparent;
}
.plugin-chip:hover,
.plugin-chip.active {
color: white;
}
/* Plugin Accordion - Light theme */
.plugin-accordion {
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.03);
}
.plugin-accordion-header:hover {
background-color: rgba(0, 0, 0, 0.06);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.05);
}
.plugin-accordion-content {
border-top-color: rgba(0, 0, 0, 0.1);
}
}
-392
View File
@@ -1,392 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema, PluginsResponse } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicPluginSection } from './DynamicPluginSection';
import './MetadataEditor.css';
interface PluginAccordionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (field: string, value: any) => void;
isExpanded: boolean;
onToggle: () => void;
}
function PluginAccordion({ plugin, metadata, onChange, isExpanded, onToggle }: PluginAccordionProps) {
return (
<div className={`plugin-accordion ${isExpanded ? 'expanded' : ''}`}>
<button
className="plugin-accordion-header"
onClick={onToggle}
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<span className="plugin-accordion-title">{plugin.name}</span>
<span className="plugin-accordion-icon">{isExpanded ? '▼' : '▶'}</span>
</button>
{isExpanded && (
<div className="plugin-accordion-content">
<DynamicPluginSection
plugin={plugin}
metadata={metadata}
onChange={onChange}
/>
</div>
)}
</div>
);
}
interface MetadataEditorProps {
hashId: string;
onSaved?: () => void;
}
export function MetadataEditor({ hashId, onSaved }: MetadataEditorProps) {
const [metadata, setMetadata] = useState<FileMetadata | null>(null);
const [editedMetadata, setEditedMetadata] = useState<FileMetadata | null>(null);
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
const [expandedPlugins, setExpandedPlugins] = useState<Set<string>>(new Set());
// Fetch plugins schema on mount
useEffect(() => {
fetchPlugins();
}, []);
// Fetch metadata when hashId changes
useEffect(() => {
loadMetadata();
}, [hashId]);
const fetchPlugins = async () => {
try {
const response = await fetch('/api/plugins');
if (response.ok) {
const data: PluginsResponse = await response.json();
// Sort by priority (lower = higher priority)
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
} catch (err) {
console.error('Failed to fetch plugin schemas:', err);
}
};
const loadMetadata = async () => {
try {
setIsLoading(true);
setError(null);
const data = await MetadataAPI.getMetadata(hashId);
setMetadata(data);
setEditedMetadata(data);
} catch (err: any) {
setError(err.message);
} finally {
setIsLoading(false);
}
};
const handleFieldChange = (field: string, value: any) => {
if (!editedMetadata) return;
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = field.split('/');
if (parts.length === 1) {
// Simple key
setEditedMetadata({
...editedMetadata,
[field]: value,
});
} else {
// Nested key - need to update nested structure
const newMetadata = { ...editedMetadata };
let current: any = newMetadata;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
} else {
current[parts[i]] = { ...current[parts[i]] };
}
current = current[parts[i]];
}
current[parts[parts.length - 1]] = value;
setEditedMetadata(newMetadata);
}
};
const handleSave = async () => {
if (!editedMetadata) return;
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
await MetadataAPI.updateMetadata(hashId, editedMetadata);
setMetadata(editedMetadata);
setSuccessMessage('Metadata saved successfully!');
if (onSaved) {
onSaved();
}
// Clear success message after 3 seconds
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleReset = () => {
setEditedMetadata(metadata);
setError(null);
setSuccessMessage(null);
};
const togglePlugin = (pluginId: string) => {
setExpandedPlugins(prev => {
const next = new Set(prev);
if (next.has(pluginId)) {
next.delete(pluginId);
} else {
next.add(pluginId);
}
return next;
});
};
const scrollToPlugin = (pluginId: string) => {
// Expand the plugin if not already expanded
if (!expandedPlugins.has(pluginId)) {
setExpandedPlugins(prev => new Set(prev).add(pluginId));
}
// Scroll to the plugin after a short delay to allow expansion
setTimeout(() => {
const element = document.getElementById(`plugin-${pluginId}`);
if (element) {
element.scrollIntoView({ behavior: 'smooth', block: 'start' });
}
}, 100);
};
if (isLoading) {
return <div className="metadata-editor-loading">Loading metadata...</div>;
}
if (error && !metadata) {
return (
<div className="metadata-editor-error">
<p>Error loading metadata: {error}</p>
<button onClick={loadMetadata}>Retry</button>
</div>
);
}
if (!metadata || !editedMetadata) {
return <div className="metadata-editor-empty">No metadata available</div>;
}
const hasChanges = JSON.stringify(metadata) !== JSON.stringify(editedMetadata);
// Filter plugins that have data
const pluginsWithData = plugins.filter(plugin => {
if (!plugin.metadataSchema) return false;
return hasPluginData(plugin, editedMetadata);
});
// Get file path for download
const filePath = editedMetadata?.filePath as string | undefined;
const downloadUrl = filePath ? `/api/file/download?path=${encodeURIComponent(filePath)}` : null;
return (
<div className="metadata-editor">
<div className="metadata-editor-header">
<h2>Edit Metadata</h2>
<div className="metadata-editor-actions">
{downloadUrl && (
<a
href={downloadUrl}
className="download-button"
title={`Download: ${filePath}`}
download
>
⬇ Download
</a>
)}
{hasChanges && (
<button onClick={handleReset} disabled={isSaving}>
Reset
</button>
)}
<button
onClick={handleSave}
disabled={isSaving || !hasChanges}
className="primary"
>
{isSaving ? 'Saving...' : 'Save Changes'}
</button>
</div>
</div>
{successMessage && (
<div className="metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="metadata-editor-message error">{error}</div>}
<div className="metadata-editor-hashes">
<div className="hashes-header">
<h3>Content Identifiers</h3>
</div>
<div className="hashes-grid">
<HashValue label="midhash256" value={hashId} primary />
{getHashValues(editedMetadata).map(({ key, label, value }) => (
<HashValue key={key} label={label} value={value} />
))}
</div>
</div>
{pluginsWithData.length > 0 && (
<div className="metadata-editor-chips">
{pluginsWithData.map((plugin) => (
<button
key={plugin.id}
className={`plugin-chip ${expandedPlugins.has(plugin.id) ? 'active' : ''}`}
style={{
'--chip-color': plugin.color || '#646cff',
backgroundColor: expandedPlugins.has(plugin.id) ? plugin.color || '#646cff' : 'transparent'
} as React.CSSProperties}
onClick={() => scrollToPlugin(plugin.id)}
>
{plugin.name}
</button>
))}
</div>
)}
<div className="metadata-editor-form">
{pluginsWithData.length === 0 ? (
<div className="metadata-editor-empty">
No metadata found for this file
</div>
) : (
pluginsWithData.map((plugin) => (
<div key={plugin.id} id={`plugin-${plugin.id}`}>
<PluginAccordion
plugin={plugin}
metadata={editedMetadata}
onChange={handleFieldChange}
isExpanded={expandedPlugins.has(plugin.id)}
onToggle={() => togglePlugin(plugin.id)}
/>
</div>
))
)}
</div>
</div>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function hasPluginData(plugin: PluginSchema, metadata: FileMetadata): boolean {
const schema = plugin.metadataSchema;
if (!schema) return false;
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
/**
* Extract all hash values (cid_*) from metadata
*/
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
const hashOrder = [
{ key: 'cid_sha2-256', label: 'SHA-256' },
{ key: 'cid_sha3-256', label: 'SHA3-256' },
{ key: 'cid_sha3-384', label: 'SHA3-384' },
{ key: 'cid_sha1', label: 'SHA-1' },
{ key: 'cid_md5', label: 'MD5' },
{ key: 'cid_crc32', label: 'CRC32' },
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
];
return hashOrder
.filter(({ key }) => (metadata as any)[key])
.map(({ key, label }) => ({
key,
label,
value: (metadata as any)[key],
}));
}
/**
* Display a single hash value with copy functionality
*/
function HashValue({ label, value, primary }: { label: string; value: string; primary?: boolean }) {
const handleCopy = () => {
navigator.clipboard.writeText(value);
};
return (
<div className={`hash-value ${primary ? 'hash-value-primary' : ''}`}>
<span className="hash-label">{label}</span>
<code className="hash-code" onClick={handleCopy} title="Click to copy">
{value}
</code>
</div>
);
}
-52
View File
@@ -1,52 +0,0 @@
.search-bar {
width: 100%;
padding: 1.5rem;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
margin-bottom: 1rem;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
}
.search-input {
flex: 1;
min-width: 200px;
}
.search-input.unified-search {
font-size: 1rem;
}
.search-bar button {
flex: 0 0 auto;
min-width: 100px;
}
.search-hint {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
@media (max-width: 768px) {
.search-controls {
flex-direction: column;
}
.search-input,
.search-bar button {
width: 100%;
}
}
@media (prefers-color-scheme: light) {
.search-hint {
color: rgba(0, 0, 0, 0.5);
}
}
-40
View File
@@ -1,40 +0,0 @@
import { useState } from 'react';
import './SearchBar.css';
interface SearchBarProps {
onSearch: (query: string) => void;
isLoading: boolean;
}
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
const [query, setQuery] = useState('');
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (query.trim()) {
onSearch(query.trim());
}
};
return (
<form className="search-bar" onSubmit={handleSubmit}>
<div className="search-controls">
<input
type="text"
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="Search by title, filename, filepath, or hash ID..."
className="search-input unified-search"
disabled={isLoading}
/>
<button type="submit" disabled={isLoading || !query.trim()}>
{isLoading ? 'Searching...' : 'Search'}
</button>
</div>
<p className="search-hint">
Search across all fields: title, filename, file path, and hash ID
</p>
</form>
);
}
+45
View File
@@ -0,0 +1,45 @@
import { useEffect, useState } from 'react';
import { Claim, ClaimsAPI, getSigner } from '../api/claimsApi';
import { splitRecordKey } from '../api/kvApi';
// ClaimsPanel lists the signed claims behind one merged field: who said what,
// and when. The merged value is what most signers agree on.
export function ClaimsPanel({ recordKey, reloadToken }: { recordKey: string; reloadToken: number }) {
const [claims, setClaims] = useState<Claim[] | null>(null);
const [error, setError] = useState('');
const parts = splitRecordKey(recordKey);
useEffect(() => {
if (!parts || !parts.field) return;
let cancelled = false;
ClaimsAPI.forRecord(parts.rid, parts.field)
.then((r) => { if (!cancelled) setClaims(r.claims); })
.catch((e) => { if (!cancelled) setError(e.message); });
return () => { cancelled = true; };
}, [recordKey, reloadToken]); // eslint-disable-line react-hooks/exhaustive-deps
if (!parts || !parts.field) return null;
if (error) return <div className="kv-claims kv-pane-status">claims: {error}</div>;
if (!claims) return <div className="kv-claims kv-pane-status">loading claims…</div>;
const me = getSigner();
return (
<div className="kv-claims">
<div className="kv-leaf-label">Claims ({claims.length})</div>
{claims.length === 0 ? (
<div className="kv-pane-hint">No signed claim — a box-local fact (file path, size, mtime) or a group member.</div>
) : (
<table className="kv-claims-table">
<tbody>
{claims.map((c) => (
<tr key={c.uid + c.cid}>
<td title={c.uid}><code>{c.uid.slice(0, 10)}…</code>{c.uid === me ? ' (you)' : ''}</td>
<td>{c.value === null ? <em>retracted</em> : typeof c.value === 'string' ? c.value : <em>ciphertext</em>}</td>
<td title={new Date(c.ts).toISOString()}>{new Date(c.ts).toLocaleString()}</td>
</tr>
))}
</tbody>
</table>
)}
</div>
);
}
+233
View File
@@ -0,0 +1,233 @@
import { useEffect, useMemo, useState } from 'react';
import { KVAPI, KVFindMatch } from '../api/kvApi';
interface Props {
matches: KVFindMatch[];
truncated: boolean;
searching: boolean;
selectedKey: string | null;
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
// FilteredKVTree is the search-active counterpart to KVTree. Instead of
// replacing the tree with a flat results list, it shows only the file
// branches that contain matches, with the matched (field, value) pairs
// rendered inline as sub-lines under each branch.
//
// Branches remain expandable: clicking one fetches the full subtree via
// KVAPI.treeAll just like the main tree, so the user can drill into every
// key of a matching file without clearing the search first.
export function FilteredKVTree({
matches, truncated, searching,
selectedKey, expandedPaths,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: Props) {
// Group matches by their entryPath (file:<hash>/) so each entry is one row
// with potentially multiple hit previews.
const groups = useMemo(() => {
const m = new Map<string, KVFindMatch[]>();
for (const x of matches) {
const arr = m.get(x.entryPath);
if (arr) arr.push(x);
else m.set(x.entryPath, [x]);
}
return m;
}, [matches]);
return (
<div className="kv-filtered">
<div className="kv-search-header">
{searching
? 'searching…'
: `${matches.length} match${matches.length === 1 ? '' : 'es'} in ${groups.size} file${groups.size === 1 ? '' : 's'}${truncated ? ' (truncated)' : ''}`}
</div>
{[...groups.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([entryPath, hits]) => (
<FilteredEntry
key={entryPath}
entryPath={entryPath}
hits={hits}
selectedKey={selectedKey}
expanded={expandedPaths.has(entryPath)}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
))}
</div>
);
}
interface EntryProps {
entryPath: string;
hits: KVFindMatch[];
selectedKey: string | null;
expanded: boolean;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function FilteredEntry({
entryPath, hits, selectedKey, expanded,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: EntryProps) {
// We deliberately do NOT mark the matched-leaf rows as "selected" via the
// tree row's selected style — the inline hit lines are themselves the
// selected indicator (they're highlighted when their key is current).
const isBranchSelected = entryPath === selectedKey;
const HIT_PREVIEW = 4;
return (
<div className="kv-filtered-entry">
<div
className={`kv-tree-row kv-tree-branch${isBranchSelected ? ' selected' : ''}`}
style={{ paddingLeft: 4 }}
onClick={() => {
onToggleBranch(entryPath, !expanded);
onSelectBranch(entryPath);
}}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{entryPath}</span>
<span className="kv-filtered-count" title={`${hits.length} match${hits.length === 1 ? '' : 'es'}`}>
{hits.length}
</span>
</div>
<div className="kv-filtered-hits">
{hits.slice(0, HIT_PREVIEW).map((h) => (
<div
key={h.key}
className={`kv-filtered-hit${h.key === selectedKey ? ' selected' : ''}`}
onClick={(e) => { e.stopPropagation(); onSelectLeaf(h.key); }}
title={h.value}
>
<span className="kv-filtered-hit-field">{h.field}</span>
<span className="kv-filtered-hit-value">{h.value}</span>
</div>
))}
{hits.length > HIT_PREVIEW && (
<div className="kv-filtered-more">
+{hits.length - HIT_PREVIEW} more match{hits.length - HIT_PREVIEW === 1 ? '' : 'es'}
{!expanded && ' · expand to see all keys'}
</div>
)}
</div>
{expanded && (
<LazySubtree
path={entryPath}
depth={1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</div>
);
}
interface SubtreeProps {
path: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
interface SubtreeData {
loading: boolean;
branches: string[];
leaves: string[];
error?: string;
}
// LazySubtree fetches its children once on mount and renders them. Sub-
// branches are themselves LazyBranch components that fetch their own
// children on first expand — recursion all the way down without the global
// expandedPaths state of the main KVTree.
function LazySubtree({ path, depth, selectedKey, onSelectLeaf, onSelectBranch }: SubtreeProps) {
const [data, setData] = useState<SubtreeData>({ loading: true, branches: [], leaves: [] });
useEffect(() => {
let cancelled = false;
KVAPI.treeAll(path, '/', 5000)
.then((r) => { if (!cancelled) setData({ loading: false, branches: r.branches, leaves: r.leaves }); })
.catch((e) => { if (!cancelled) setData({ loading: false, branches: [], leaves: [], error: e.message }); });
return () => { cancelled = true; };
}, [path]);
if (data.loading) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (data.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>{data.error}</div>;
}
return (
<>
{data.branches.map((b) => (
<LazyBranch
key={b}
path={b}
parentPath={path}
depth={depth}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
))}
{data.leaves.map((l) => (
<div
key={l}
className={`kv-tree-row kv-tree-leaf${l === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(l)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{l.slice(path.length)}</span>
</div>
))}
</>
);
}
interface LazyBranchProps {
path: string;
parentPath: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
function LazyBranch({ path, parentPath, depth, selectedKey, onSelectLeaf, onSelectBranch }: LazyBranchProps) {
const [expanded, setExpanded] = useState(false);
const label = path.slice(parentPath.length);
return (
<>
<div
className={`kv-tree-row kv-tree-branch${path === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => { setExpanded(!expanded); onSelectBranch(path); }}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{label}</span>
</div>
{expanded && (
<LazySubtree
path={path}
depth={depth + 1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</>
);
}
+902
View File
@@ -0,0 +1,902 @@
/* KV Tree Editor — fills the iframe viewport with a 2-pane layout. */
.kv-app {
display: flex;
flex-direction: column;
height: 100vh;
width: 100%;
overflow: hidden;
background: #fafbfc;
color: #1f2330;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
font-size: 13px;
}
/* ── Header ───────────────────────────────────────────────────────────── */
.kv-app-header {
border-bottom: 1px solid #e2e6ee;
padding: 8px 12px;
background: #fff;
flex: 0 0 auto;
}
.kv-app-search {
display: flex;
align-items: center;
gap: 8px;
}
.kv-search-input {
flex: 1;
padding: 7px 11px;
border: 1px solid #d1d6e0;
border-radius: 6px;
font-size: 13px;
background: #fff;
outline: none;
}
.kv-search-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-app-stats {
font-size: 12px;
color: #8089a0;
white-space: nowrap;
}
.kv-app-breadcrumb {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 2px;
font-size: 12px;
}
.kv-crumb-label {
color: #8089a0;
margin-right: 4px;
}
.kv-crumb {
border: none;
background: transparent;
padding: 1px 4px;
border-radius: 3px;
cursor: pointer;
color: #4a5170;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-crumb:hover { background: #eef0fb; color: #2a2f48; }
/* ── Body ─────────────────────────────────────────────────────────────── */
.kv-app-body {
flex: 1 1 auto;
display: flex;
min-height: 0;
}
.kv-app-left {
flex: 0 0 38%;
min-width: 280px;
max-width: 600px;
border-right: 1px solid #e2e6ee;
background: #fff;
overflow: auto;
}
.kv-app-right {
flex: 1 1 auto;
overflow: auto;
padding: 16px 20px;
background: #fafbfc;
}
/* ── Tree ─────────────────────────────────────────────────────────────── */
.kv-tree {
padding: 4px 0;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-tree-list {
list-style: none;
margin: 0;
padding: 0;
}
.kv-tree-row {
display: flex;
align-items: center;
gap: 4px;
padding: 2px 4px;
cursor: pointer;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-row:hover { background: #f3f5fb; }
.kv-tree-toggle {
display: inline-block;
width: 12px;
color: #8089a0;
font-size: 10px;
}
.kv-tree-leaf-dot {
display: inline-block;
width: 12px;
color: #c5cad6;
}
.kv-tree-label {
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-branch .kv-tree-label {
color: #2a3050;
font-weight: 500;
}
.kv-tree-branch .kv-tree-filename {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85em;
color: #1e3a8a;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
}
.kv-tree-hash-pill {
margin-left: 0.5rem;
padding: 0.05rem 0.4rem;
background: rgba(100, 108, 255, 0.12);
color: #4a50c8;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.7em;
border-radius: 999px;
flex-shrink: 0;
}
.kv-tree-leaf .kv-tree-label {
color: #4a5170;
}
.kv-tree-row.selected {
background: #e7eafd;
color: #2532a3;
}
.kv-tree-row.selected:hover { background: #dde2fc; }
.kv-tree-loading,
.kv-tree-empty,
.kv-tree-error {
font-size: 12px;
color: #8089a0;
padding: 4px 8px;
font-style: italic;
}
.kv-tree-error { color: #c0392b; }
/* ── Right pane (leaf / branch) ───────────────────────────────────────── */
.kv-leaf, .kv-branch {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-leaf-key {
display: flex;
align-items: center;
gap: 6px;
background: #f0f2f8;
border: 1px solid #e2e6ee;
border-radius: 6px;
padding: 8px 10px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
}
.kv-leaf-key-text {
flex: 1;
word-break: break-all;
color: #2a3050;
}
.kv-leaf-meta {
display: flex;
align-items: center;
gap: 8px;
}
.kv-leaf-label {
font-size: 12px;
color: #6e7790;
font-weight: 500;
}
.kv-leaf-type {
border: 1px solid #d1d6e0;
border-radius: 5px;
padding: 4px 8px;
font-size: 12px;
background: #fff;
}
.kv-leaf-hint {
font-size: 12px;
color: #6e7790;
font-style: italic;
}
.kv-leaf-input {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
box-sizing: border-box;
}
.kv-leaf-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-textarea {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
resize: vertical;
box-sizing: border-box;
font-family: inherit;
}
.kv-leaf-textarea:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-mono {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-url {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-url-open {
font-size: 12px;
color: #6c7afa;
text-decoration: none;
white-space: nowrap;
}
.kv-leaf-actions {
display: flex;
gap: 8px;
}
/* ── CID leaf ─────────────────────────────────────────────────────────── */
.kv-leaf-cid {
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-leaf-cid-controls {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-cid-actions {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
}
.kv-leaf-cid-meta {
font-size: 12px;
color: #6b7280;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-cid-preview {
max-width: 100%;
max-height: 320px;
border-radius: 6px;
border: 1px solid #e2e6ee;
background: #f5f6fa;
object-fit: contain;
align-self: flex-start;
}
.kv-leaf-cid-preview-media {
width: 100%;
max-width: 560px;
}
/* ── File-detail view ─────────────────────────────────────────────────── */
.kv-file {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-file-header {
padding: 14px 16px;
background: #fff;
border: 1px solid #e2e6ee;
border-radius: 8px;
display: grid;
grid-template-columns: 1fr auto;
grid-template-areas:
"name actions"
"meta actions";
gap: 6px 16px;
align-items: center;
}
.kv-file-name {
grid-area: name;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-weight: 600;
font-size: 14px;
color: #1e3a8a;
word-break: break-all;
}
.kv-file-name-fallback {
color: #9ca3af;
font-style: italic;
}
.kv-file-meta {
grid-area: meta;
display: flex;
gap: 12px;
align-items: baseline;
font-size: 12px;
color: #6b7280;
min-width: 0;
}
.kv-file-hash {
background: rgba(100, 122, 250, 0.1);
color: #4a50c8;
padding: 1px 6px;
border-radius: 999px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
flex-shrink: 0;
}
.kv-file-path {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
}
.kv-file-actions {
grid-area: actions;
display: flex;
gap: 8px;
align-items: center;
}
.kv-file-rows {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row {
display: grid;
grid-template-columns: minmax(140px, 220px) 1fr auto;
gap: 12px;
padding: 8px 12px;
background: #fff;
border: 1px solid #e2e6ee;
border-left: 3px solid #d6deed;
border-radius: 6px;
align-items: start;
}
.kv-file-row.pending-delete {
border-left-color: #dc2626;
background: #fef2f2;
}
.kv-file-row.pending-delete .kv-file-row-label {
text-decoration: line-through;
color: #9ca3af;
}
.kv-file-row-deleted-note {
color: #b91c1c;
font-style: italic;
font-size: 12px;
padding: 6px 0;
}
.kv-file-row-delete {
width: 24px;
height: 24px;
align-self: center;
display: inline-flex;
align-items: center;
justify-content: center;
background: transparent;
border: 1px solid transparent;
border-radius: 4px;
color: #dc2626;
font-size: 16px;
line-height: 1;
cursor: pointer;
opacity: 0.4;
transition: opacity 0.15s, background 0.15s, border-color 0.15s;
}
.kv-file-row:hover .kv-file-row-delete,
.kv-file-row.pending-delete .kv-file-row-delete {
opacity: 1;
}
.kv-file-row-delete:hover {
background: rgba(220, 38, 38, 0.1);
border-color: rgba(220, 38, 38, 0.3);
}
.kv-file-add {
display: flex;
gap: 8px;
align-items: center;
padding: 10px 12px;
background: rgba(100, 122, 250, 0.05);
border: 1px dashed rgba(100, 122, 250, 0.3);
border-radius: 6px;
}
.kv-file-add-prefix {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
color: #6b7280;
white-space: nowrap;
flex-shrink: 0;
max-width: 220px;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-file-add > input {
flex: 1;
}
.kv-file-add > input:first-of-type {
flex: 0 0 200px;
}
.kv-file-row.dirty {
border-left-color: #fbbf24;
background: #fffbeb;
}
.kv-file-row.pulsed {
animation: kv-row-pulse 1.4s ease-out;
}
@keyframes kv-row-pulse {
0% { background: rgba(100, 122, 250, 0.35); border-left-color: #4a50c8; }
100% { background: transparent; }
}
.kv-file-row-label,
.kv-file-row-label-static {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
background: none;
border: none;
text-align: left;
cursor: pointer;
padding: 6px 0;
word-break: break-all;
}
.kv-file-row-label-static {
cursor: default;
}
.kv-file-row-label:hover {
color: #4a50c8;
text-decoration: underline;
}
.kv-file-row-value {
display: flex;
min-width: 0;
}
.kv-file-row-value > * {
flex: 1;
min-width: 0;
}
.kv-file-group {
display: flex;
flex-direction: column;
border: 1px solid #e2e6ee;
border-radius: 8px;
background: rgba(100, 122, 250, 0.03);
overflow: hidden;
}
.kv-file-group-header {
display: flex;
align-items: center;
gap: 8px;
padding: 6px 12px;
border: none;
background: rgba(100, 122, 250, 0.1);
cursor: pointer;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
}
.kv-file-group-header:hover {
background: rgba(100, 122, 250, 0.2);
}
.kv-file-group-chevron {
font-size: 10px;
color: #6b7280;
}
.kv-file-group-name {
font-weight: 600;
color: #1e3a8a;
}
.kv-file-group-count {
margin-left: auto;
padding: 1px 8px;
background: rgba(100, 122, 250, 0.2);
color: #4a50c8;
border-radius: 999px;
font-size: 11px;
}
.kv-file-group-body {
padding: 8px;
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row-lang .kv-file-lang-entries {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-lang-entry {
display: flex;
gap: 6px;
align-items: center;
}
.kv-file-lang-entry > .kv-leaf-input {
flex: 1;
}
.kv-file-lang-remove {
color: #dc2626;
}
.kv-file-lang-add {
align-self: flex-start;
background: rgba(100, 122, 250, 0.1);
}
/* ── DateTime leaf ────────────────────────────────────────────────────── */
.kv-leaf-datetime {
display: flex;
gap: 8px;
align-items: center;
}
.kv-leaf-datetime-shape {
font-size: 11px;
color: #6b7280;
background: rgba(100, 122, 250, 0.1);
padding: 2px 6px;
border-radius: 999px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
white-space: nowrap;
}
/* ── Branch ───────────────────────────────────────────────────────────── */
.kv-branch-section {
border: 1px solid #e2e6ee;
border-radius: 6px;
background: #fff;
padding: 12px 14px;
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-branch-section h4 {
margin: 0;
font-size: 13px;
color: #2a3050;
}
.kv-branch-row {
display: flex;
align-items: center;
gap: 6px;
}
.kv-branch-prefix {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
color: #6e7790;
background: #f0f2f8;
padding: 6px 8px;
border-radius: 4px;
white-space: nowrap;
max-width: 50%;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-branch-note {
margin: 0;
font-size: 12px;
color: #6e7790;
line-height: 1.4;
}
/* ── Buttons ──────────────────────────────────────────────────────────── */
.kv-btn {
border: 1px solid #d1d6e0;
background: #fff;
color: #2a3050;
padding: 6px 14px;
border-radius: 5px;
font-size: 12.5px;
cursor: pointer;
}
.kv-btn:hover:not(:disabled) { background: #f3f5fb; border-color: #b8c0d0; }
.kv-btn:disabled { opacity: 0.45; cursor: not-allowed; }
.kv-btn-primary {
background: #6c7afa;
border-color: #6c7afa;
color: #fff;
}
.kv-btn-primary:hover:not(:disabled) {
background: #5868e8;
border-color: #5868e8;
}
.kv-btn-danger {
border-color: #e0788a;
color: #c0392b;
}
.kv-btn-danger:hover:not(:disabled) {
background: #fdecef;
border-color: #c0392b;
}
.kv-btn-icon {
border: 1px solid transparent;
background: transparent;
color: #6e7790;
padding: 2px 6px;
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.kv-btn-icon:hover { background: #eef0fb; color: #2a3050; }
/* ── Pane status ──────────────────────────────────────────────────────── */
.kv-pane-status {
color: #6e7790;
font-size: 12.5px;
padding: 8px 0;
}
.kv-pane-empty {
text-align: center;
padding: 80px 20px;
font-size: 14px;
}
.kv-pane-hint {
margin-top: 8px;
font-size: 12px;
color: #8089a0;
}
/* ── Field chips (configure search-in fields) ─────────────────────────── */
.kv-chips {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 4px;
}
.kv-chips-label {
font-size: 12px;
color: #8089a0;
margin-right: 4px;
}
.kv-chip {
display: inline-flex;
align-items: center;
gap: 4px;
background: #eef0fb;
border: 1px solid #d5dafa;
color: #2a3050;
border-radius: 12px;
padding: 1px 4px 1px 8px;
font-size: 11.5px;
}
.kv-chip code {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11.5px;
}
.kv-chip-x {
border: none;
background: transparent;
color: #6e7790;
cursor: pointer;
padding: 0 4px;
font-size: 14px;
line-height: 1;
}
.kv-chip-x:hover { color: #c0392b; }
.kv-chip-input {
border: 1px dashed #c5cad6;
background: transparent;
border-radius: 12px;
padding: 2px 8px;
font-size: 11.5px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
outline: none;
min-width: 90px;
}
.kv-chip-input:focus { border-style: solid; border-color: #6c7afa; }
/* ── Search header (shared across filter mode) ────────────────────────── */
.kv-search-header {
padding: 6px 10px;
font-size: 11px;
color: #6e7790;
background: #f3f5fb;
border-bottom: 1px solid #e2e6ee;
position: sticky;
top: 0;
z-index: 1;
}
/* ── Filtered tree (search-active state) ──────────────────────────────── */
.kv-filtered {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-filtered-entry {
border-bottom: 1px solid #f0f2f8;
}
.kv-filtered-count {
margin-left: auto;
background: #eef0fb;
color: #4a5170;
padding: 0 6px;
border-radius: 9px;
font-size: 10.5px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-weight: 600;
}
.kv-filtered-hits {
padding: 2px 4px 4px 24px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 12px;
}
.kv-filtered-hit {
display: flex;
gap: 6px;
align-items: baseline;
padding: 2px 6px;
border-radius: 3px;
cursor: pointer;
line-height: 1.35;
}
.kv-filtered-hit:hover { background: #f3f5fb; }
.kv-filtered-hit.selected {
background: #e7eafd;
}
.kv-filtered-hit-field {
background: #eef0fb;
color: #4a5170;
padding: 0 5px;
border-radius: 3px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 10.5px;
flex: 0 0 auto;
}
.kv-filtered-hit-value {
color: #2a3050;
word-break: break-word;
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
}
.kv-filtered-more {
padding: 2px 6px 0;
font-size: 11px;
font-style: italic;
color: #8089a0;
}
/* ── Toasts ───────────────────────────────────────────────────────────── */
.kv-toast-stack {
position: fixed;
bottom: 16px;
right: 16px;
display: flex;
flex-direction: column-reverse;
gap: 6px;
z-index: 1000;
max-width: 480px;
}
.kv-toast {
background: #2a3050;
color: #fff;
border-radius: 6px;
padding: 9px 12px;
box-shadow: 0 4px 14px rgba(0,0,0,0.15);
font-size: 12.5px;
}
.kv-toast-success { background: #2f9e44; }
.kv-toast-error { background: #c0392b; }
.kv-toast-row {
display: flex;
align-items: flex-start;
gap: 8px;
justify-content: space-between;
}
.kv-toast-text {
flex: 1;
word-break: break-word;
}
.kv-toast-close {
border: none;
background: transparent;
color: rgba(255,255,255,0.8);
cursor: pointer;
font-size: 16px;
line-height: 1;
}
.kv-toast-close:hover { color: #fff; }
.kv-toast-details {
margin: 6px 0 0;
padding-left: 18px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11px;
color: rgba(255,255,255,0.85);
max-height: 160px;
overflow: auto;
}
.kv-app-gear {
margin-left: 0.25rem;
font-size: 1.05rem;
opacity: 0.6;
}
.kv-app-gear:hover {
opacity: 1;
}
.kv-modal-backdrop {
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.45);
z-index: 1000;
display: flex;
align-items: flex-start;
justify-content: center;
padding: 4vh 1rem;
overflow-y: auto;
}
.kv-modal {
position: relative;
background: white;
color: inherit;
border-radius: 8px;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
width: min(960px, 100%);
max-height: 92vh;
overflow-y: auto;
}
.kv-modal-close {
position: absolute;
top: 0.5rem;
right: 0.75rem;
background: transparent;
border: none;
font-size: 1.6rem;
line-height: 1;
cursor: pointer;
color: inherit;
opacity: 0.6;
z-index: 1;
}
.kv-modal-close:hover {
opacity: 1;
}
.kv-signer { font-size: 12px; display: inline-flex; align-items: center; gap: 4px; }
.kv-signer select { font-size: 12px; }
.kv-claims { margin-top: 16px; }
.kv-claims-table { width: 100%; border-collapse: collapse; font-size: 12px; }
.kv-claims-table td { padding: 3px 6px; border-top: 1px solid rgba(127, 127, 127, 0.25); vertical-align: top; }
+283
View File
@@ -0,0 +1,283 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI, KVFindMatch, KVInfoResponse } from '../api/kvApi';
import { ancestorBranches, KVTree } from './KVTree';
import { FilteredKVTree } from './FilteredKVTree';
import { KVLeafEditor } from './KVLeafEditor';
import { KVBranchEditor } from './KVBranchEditor';
import { KVFileEditor } from './KVFileEditor';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
import { SearchFieldChips } from './SearchFieldChips';
import { ToastStack, makeToast } from './Toast';
import { ToastEntry } from './types';
import { SignerPicker } from './SignerPicker';
import './KVApp.css';
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
const DEFAULT_SEARCH_FIELDS = ['originalTitle', 'filePath', 'fileName', 'plot/eng'];
function loadSearchFields(): string[] {
try {
const raw = localStorage.getItem(SEARCH_FIELDS_LS_KEY);
if (!raw) return DEFAULT_SEARCH_FIELDS;
const parsed = JSON.parse(raw);
if (Array.isArray(parsed) && parsed.every((x) => typeof x === 'string')) return parsed;
} catch {}
return DEFAULT_SEARCH_FIELDS;
}
// Top-level shell. Two panes inside the iframe:
// left — lazy tree of the keyspace
// right — leaf editor (when a leaf is selected) or branch editor (branch).
//
// A sticky header carries the search bar, the breadcrumb, and basic stats.
export default function KVApp() {
// Selected node. branchPath is set when a branch is selected; selectedKey
// when a leaf is. Mutually exclusive.
const [selectedKey, setSelectedKey] = useState<string | null>(null);
const [branchPath, setBranchPath] = useState<string | null>(null);
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set(['']));
const [searchInput, setSearchInput] = useState('');
const [searchFields, setSearchFields] = useState<string[]>(loadSearchFields);
const [searchResults, setSearchResults] = useState<KVFindMatch[] | null>(null);
const [searchTruncated, setSearchTruncated] = useState(false);
const [searching, setSearching] = useState(false);
const [info, setInfo] = useState<KVInfoResponse | null>(null);
const [reloadToken, setReloadToken] = useState(0);
const [toasts, setToasts] = useState<ToastEntry[]>([]);
const searchTimerRef = useRef<number | null>(null);
// When the right pane is a file-detail view and the user clicks a leaf
// under the same file branch in the tree, we route the click as a
// scroll-anchor inside the file detail instead of a leaf-editor switch.
// The token bumps each click so identical anchors still trigger a scroll.
const [anchor, setAnchor] = useState<{ field: string; token: number } | null>(null);
const isFileBranch = (p: string | null) => !!p && /^file:[^/]+\/$/.test(p);
const handleLeafClick = (key: string) => {
if (isFileBranch(branchPath) && key.startsWith(branchPath!)) {
setAnchor({ field: key.slice(branchPath!.length), token: (anchor?.token ?? 0) + 1 });
return;
}
setSelectedKey(key);
setBranchPath(null);
setAnchor(null);
};
const handleBranchClick = (path: string) => {
setBranchPath(path);
setSelectedKey(null);
setAnchor(null);
};
const pushToast = (t: ToastEntry) => setToasts((cur) => [...cur, t]);
const dismissToast = (id: number) => setToasts((cur) => cur.filter((t) => t.id !== id));
useEffect(() => {
KVAPI.info().then(setInfo).catch(() => {});
}, [reloadToken]);
// Deep-link: /editor/file/<hashId> opens the tree pre-expanded at that hash.
useEffect(() => {
const m = window.location.pathname.match(/\/file\/([a-z0-9]+)$/i);
if (m) {
const hashId = m[1];
const branch = `file:${hashId}/`;
setExpandedPaths((cur) => new Set([...cur, '', 'file:', branch]));
setBranchPath(branch);
}
}, []);
// Persist field list across reloads.
useEffect(() => {
try { localStorage.setItem(SEARCH_FIELDS_LS_KEY, JSON.stringify(searchFields)); } catch {}
}, [searchFields]);
// Debounced live value-search across configured fields. Empty input clears.
useEffect(() => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
if (!searchInput.trim() || searchFields.length === 0) {
setSearchResults(null);
setSearchTruncated(false);
setSearching(false);
return;
}
setSearching(true);
searchTimerRef.current = window.setTimeout(async () => {
try {
const r = await KVAPI.find(searchInput.trim(), searchFields, 200);
setSearchResults(r.matches);
setSearchTruncated(r.truncated);
} catch (e: any) {
pushToast(makeToast('error', `Search failed: ${e.message}`));
} finally {
setSearching(false);
}
}, 250);
return () => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
};
}, [searchInput, searchFields]);
// Jump to a key from a search hit: select it, expand all ancestor branches.
const jumpToKey = (key: string) => {
const ancestors = ancestorBranches(key);
setExpandedPaths((cur) => new Set([...cur, '', ...ancestors]));
if (key.endsWith('/') || key.endsWith(':')) {
setBranchPath(key);
setSelectedKey(null);
} else {
setSelectedKey(key);
setBranchPath(null);
}
};
const breadcrumb = useMemo(() => {
const target = selectedKey ?? branchPath ?? '';
if (!target) return null;
const parts: { label: string; path: string }[] = [];
let i = 0;
while (i < target.length) {
let next = -1;
for (const d of [':', '/']) {
const idx = target.indexOf(d, i);
if (idx >= 0 && (next < 0 || idx < next)) next = idx;
}
if (next < 0) {
parts.push({ label: target.slice(i), path: target });
break;
}
parts.push({ label: target.slice(i, next + 1), path: target.slice(0, next + 1) });
i = next + 1;
}
return parts;
}, [selectedKey, branchPath]);
return (
<div className="kv-app">
<header className="kv-app-header">
<div className="kv-app-search">
<input
type="search"
className="kv-search-input"
placeholder="filter by value across configured fields…"
value={searchInput}
onChange={(e) => setSearchInput(e.target.value)}
autoFocus
/>
{searchInput && (
<button className="kv-btn-icon" onClick={() => setSearchInput('')} title="clear">×</button>
)}
<span className="kv-app-stats">
{info ? `${info.keyCount} files · ${info.memoryUsage}` : ''}
</span>
<button
className="kv-btn"
onClick={() => setReloadToken((n) => n + 1)}
title="refresh tree"
>Refresh</button>
<SignerPicker onError={(msg) => pushToast(makeToast('error', msg))} />
</div>
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
{breadcrumb && breadcrumb.length > 0 && (
<div className="kv-app-breadcrumb">
<span className="kv-crumb-label">path:</span>
{breadcrumb.map((b, i) => (
<button
key={i}
className="kv-crumb"
onClick={() => jumpToKey(b.path)}
>{b.label}</button>
))}
</div>
)}
</header>
<div className="kv-app-body">
<aside className="kv-app-left">
{searchResults !== null ? (
<FilteredKVTree
matches={searchResults}
truncated={searchTruncated}
searching={searching}
selectedKey={selectedKey}
expandedPaths={expandedPaths}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
) : (
<KVTree
selectedKey={selectedKey}
expandedPaths={expandedPaths}
reloadToken={reloadToken}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
)}
</aside>
<main className="kv-app-right">
{selectedKey ? (
<KVLeafEditor
selectedKey={selectedKey}
reloadToken={reloadToken}
onSaved={() => {
pushToast(makeToast('success', `Saved ${selectedKey}`));
setReloadToken((n) => n + 1);
}}
onDeleted={() => {
pushToast(makeToast('success', `Deleted ${selectedKey}`));
setSelectedKey(null);
setReloadToken((n) => n + 1);
}}
onError={(msg) => pushToast(makeToast('error', msg))}
/>
) : branchPath && FILE_BRANCH_RE.test(branchPath) ? (
<KVFileEditor
key={branchPath}
branchPath={branchPath}
reloadToken={reloadToken}
scrollToField={anchor}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
onSelectLeaf={(k) => { setSelectedKey(k); setBranchPath(null); setAnchor(null); }}
/>
) : branchPath ? (
<KVBranchEditor
branchPath={branchPath}
reloadToken={reloadToken}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
/>
) : (
<div className="kv-pane-status kv-pane-empty">
Select a key from the tree.
<div className="kv-pane-hint">
Click a branch to add / rename / delete a subtree, or a leaf to edit its value.
</div>
</div>
)}
</main>
</div>
<ToastStack toasts={toasts} onDismiss={dismissToast} />
</div>
);
}
+194
View File
@@ -0,0 +1,194 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
interface Props {
// Branch path, ends with the delimiter (":" or "/").
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
}
// Branch operations are non-atomic — each per-key write/delete goes one round
// trip to Redis and may fail individually. Failures collect into a list and
// surface via the parent's toast stack. Per the design, we accept partial
// failure and surface what failed.
export function KVBranchEditor({ branchPath, reloadToken, onChanged }: Props) {
const [count, setCount] = useState<number | null>(null);
const [busy, setBusy] = useState(false);
const [newChild, setNewChild] = useState('');
const [newValue, setNewValue] = useState('');
const [renameTo, setRenameTo] = useState('');
// Use the search endpoint with the branch path as the substring to count
// descendants. Truncated when there are more than the limit; we just show
// an approximate count.
useEffect(() => {
let cancelled = false;
KVAPI.search(branchPath, 5000)
.then((r) => {
if (cancelled) return;
// Filter to true descendants (those starting with branchPath).
const descendants = r.keys.filter((k) => k.startsWith(branchPath));
setCount(descendants.length);
})
.catch(() => { if (!cancelled) setCount(null); });
return () => { cancelled = true; };
}, [branchPath, reloadToken]);
// Walk all descendants and run an op against each, collecting failures.
const forEachDescendant = async (op: (key: string) => Promise<void>): Promise<string[]> => {
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const errors: string[] = [];
for (const k of keys) {
try { await op(k); } catch (e: any) { errors.push(`${k}: ${e.message}`); }
}
return errors;
};
const addChild = async () => {
if (!newChild) return;
const childKey = branchPath + newChild;
setBusy(true);
try {
await KVAPI.put(childKey, newValue);
setNewChild('');
setNewValue('');
onChanged(`Created ${childKey}`);
} catch (e: any) {
onChanged(`Failed to create ${childKey}`, [e.message]);
} finally {
setBusy(false);
}
};
const deleteSubtree = async () => {
if (!confirm(`Delete every key under ${branchPath} (${count ?? '?'} entries)?`)) return;
setBusy(true);
try {
const errors = await forEachDescendant((k) => KVAPI.del(k).then(() => undefined));
if (errors.length === 0) {
onChanged(`Deleted subtree ${branchPath}`);
} else {
onChanged(`Deleted subtree ${branchPath} with ${errors.length} failure(s)`, errors);
}
} finally {
setBusy(false);
}
};
const renameBranch = async () => {
if (!renameTo) return;
const target = renameTo.endsWith('/') || renameTo.endsWith(':') ? renameTo : renameTo + '/';
if (target === branchPath) return;
if (!confirm(`Rename ${branchPath} → ${target}? This copies ${count ?? '?'} keys then deletes the originals.`)) return;
setBusy(true);
try {
// Pass 1: copy
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const copyErrors: string[] = [];
for (const oldKey of keys) {
try {
const v = await KVAPI.get(oldKey);
if (!v.exists || v.type !== 'string') {
copyErrors.push(`${oldKey}: skipped (type ${v.type || 'missing'})`);
continue;
}
const newKey = target + oldKey.slice(branchPath.length);
await KVAPI.put(newKey, v.value);
} catch (e: any) {
copyErrors.push(`${oldKey}: ${e.message}`);
}
}
// Pass 2: delete originals (only the ones we successfully copied)
const failedOldKeys = new Set(copyErrors.map((e) => e.split(':')[0]));
const deleteErrors: string[] = [];
for (const oldKey of keys) {
if (failedOldKeys.has(oldKey)) continue;
try { await KVAPI.del(oldKey); } catch (e: any) { deleteErrors.push(`${oldKey}: ${e.message}`); }
}
const allErrors = [...copyErrors, ...deleteErrors];
if (allErrors.length === 0) {
onChanged(`Renamed ${branchPath} → ${target}`);
} else {
onChanged(`Renamed ${branchPath} → ${target} with ${allErrors.length} failure(s)`, allErrors);
}
setRenameTo('');
} finally {
setBusy(false);
}
};
return (
<div className="kv-branch">
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{branchPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(branchPath)}
title="copy path"
>⧉</button>
</div>
<div className="kv-pane-status">
{count == null ? 'counting…' : `${count} descendant key${count === 1 ? '' : 's'}`}
</div>
<section className="kv-branch-section">
<h4>Add child</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">{branchPath}</span>
<input
className="kv-leaf-input"
placeholder="child path (may contain /)"
value={newChild}
onChange={(e) => setNewChild(e.target.value)}
disabled={busy}
/>
</div>
<input
className="kv-leaf-input"
placeholder="value"
value={newValue}
onChange={(e) => setNewValue(e.target.value)}
disabled={busy}
/>
<button className="kv-btn kv-btn-primary" disabled={!newChild || busy} onClick={addChild}>
Add
</button>
</section>
<section className="kv-branch-section">
<h4>Rename branch</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">→</span>
<input
className="kv-leaf-input"
placeholder="new prefix (e.g. file:newhash/)"
value={renameTo}
onChange={(e) => setRenameTo(e.target.value)}
disabled={busy}
/>
</div>
<p className="kv-branch-note">
Non-atomic: copies each descendant to the new prefix then deletes the original.
Failures are surfaced individually.
</p>
<button className="kv-btn" disabled={!renameTo || busy} onClick={renameBranch}>
Rename
</button>
</section>
<section className="kv-branch-section">
<h4>Delete subtree</h4>
<p className="kv-branch-note">
Deletes every descendant key. Cannot be undone.
</p>
<button className="kv-btn kv-btn-danger" disabled={busy || count === 0} onClick={deleteSubtree}>
Delete {count ?? ''} key{count === 1 ? '' : 's'}
</button>
</section>
</div>
);
}
+675
View File
@@ -0,0 +1,675 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { SchemaFieldInfo, useSchema } from '../api/schemaApi';
import { LanguageCombobox, formatLanguageOption } from '../components/LanguageCombobox';
import { LeafValueEditor } from './leafRenderers';
import { schemaToFieldType } from './schemaLookup';
import { detectType } from './typeHeuristics';
import { FieldType } from './types';
interface Props {
/** Branch path of a file, e.g. "file:<hash>/". */
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
/** Navigate to a single leaf for the manual deep editor. */
onSelectLeaf: (key: string) => void;
/**
* When set, scroll to and pulse the row matching this logical field path
* (e.g. "releasedate", "plot", "stream/0"). The `token` lets the same field
* fire a fresh scroll across consecutive clicks.
*/
scrollToField?: { field: string; token: number } | null;
}
const LANG_CODE_RE = /^[a-z]{2,3}$/;
interface LeafRow {
kind: 'leaf';
fieldKey: string;
value: string;
fieldType: FieldType;
info?: SchemaFieldInfo;
}
interface LanguageRow {
kind: 'language-group';
fieldKey: string;
/** Current language → text map. */
languageMap: Record<string, string>;
info?: SchemaFieldInfo;
}
type Row = LeafRow | LanguageRow;
interface RowGroup {
prefix: string;
isCluster: boolean;
rows: Row[];
}
export function KVFileEditor({
branchPath,
reloadToken,
onChanged,
onSelectLeaf,
scrollToField,
}: Props) {
const [metadata, setMetadata] = useState<Record<string, string> | null>(null);
const [edits, setEdits] = useState<Map<string, string | null>>(new Map());
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
const [pulse, setPulse] = useState<string | null>(null);
const [newFieldKey, setNewFieldKey] = useState('');
const [newFieldValue, setNewFieldValue] = useState('');
const [addingField, setAddingField] = useState(false);
// Track whether we've seeded collapsedGroups for the current branch; resets
// when the branch changes so each file gets its defaults reapplied.
const defaultsAppliedRef = useRef<string | null>(null);
const { schema } = useSchema();
// When the parent passes a target field, expand its prefix group (if any)
// and scroll the row into view. We also briefly pulse the row to draw the
// eye — the pulse is reset after the animation duration.
useEffect(() => {
if (!scrollToField) return;
const { field } = scrollToField;
const prefix = field.split('/', 1)[0];
setCollapsedGroups((cur) => {
if (!cur.has(prefix)) return cur;
const next = new Set(cur);
next.delete(prefix);
return next;
});
requestAnimationFrame(() => {
const el = document.querySelector(
`[data-row-key="${cssEscape(field)}"]`,
) as HTMLElement | null;
if (el) {
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
setPulse(field);
window.setTimeout(() => setPulse((p) => (p === field ? null : p)), 1400);
}
});
}, [scrollToField?.field, scrollToField?.token]);
const hashId = branchPath.slice('file:'.length, -1);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.treeAll(branchPath, '/', 5000)
.then(async (resp) => {
// resp.leaves = full keys under branchPath. Recursive descent: also
// walk branch children (they're paths to sub-objects with their own
// leaves, e.g. file:<h>/plot/ → plot/eng).
const all: string[] = [...resp.leaves];
const queue = [...resp.branches];
while (queue.length > 0) {
const next = queue.shift()!;
try {
const sub = await KVAPI.treeAll(next, '/', 5000);
all.push(...sub.leaves);
queue.push(...sub.branches);
} catch {
// Ignore failures on a sub-branch; show what we can.
}
}
if (cancelled) return;
// MGET-equivalent: parallel fetches with a small cap to avoid burst.
const values = await Promise.all(
all.map((k) => KVAPI.get(k).then((r) => [k, r.exists ? r.value : null] as const).catch(() => [k, null] as const)),
);
if (cancelled) return;
const flat: Record<string, string> = {};
for (const [k, v] of values) {
if (v == null) continue;
// Strip the branch prefix so keys are the logical field path
// ("releasedate", "plot/eng", "stream/0").
const fieldKey = k.slice(branchPath.length);
if (fieldKey) flat[fieldKey] = v;
}
setMetadata(flat);
setEdits(new Map());
})
.catch(() => {
if (!cancelled) setMetadata({});
})
.finally(() => {
if (!cancelled) setLoading(false);
});
return () => {
cancelled = true;
};
}, [branchPath, reloadToken]);
const groups = useMemo(
() => groupRows(buildRows(metadata ?? {}, schema)),
[metadata, schema],
);
// Default state: every cluster group (e.g. stream/*, cid_*) starts collapsed.
// Reapplied when the user switches to a different file.
useEffect(() => {
if (groups.length === 0) return;
if (defaultsAppliedRef.current === branchPath) return;
defaultsAppliedRef.current = branchPath;
setCollapsedGroups(new Set(groups.filter((g) => g.isCluster).map((g) => g.prefix)));
}, [groups, branchPath]);
if (loading) {
return <div className="kv-pane-status">loading file…</div>;
}
if (!metadata) {
return <div className="kv-pane-status">no data for {branchPath}</div>;
}
const fileName = metadata['fileName'];
const filePath = metadata['filePath'];
const onLeafChange = (fieldKey: string, value: string) => {
setEdits((prev) => {
const next = new Map(prev);
const original = metadata[fieldKey] ?? null;
if (value === original) next.delete(fieldKey);
else next.set(fieldKey, value);
return next;
});
};
/**
* Toggle a field's pending-delete state. Re-clicking restores; if the field
* is brand-new (no original value) we just drop it from the edits map.
*/
const onLeafDeleteToggle = (fieldKey: string) => {
setEdits((prev) => {
const next = new Map(prev);
if (next.get(fieldKey) === null) {
// Currently pending delete → undo.
next.delete(fieldKey);
} else {
next.set(fieldKey, null);
}
return next;
});
};
const handleAddField = async () => {
const key = newFieldKey.trim();
if (!key) return;
const fullKey = branchPath + key;
setAddingField(true);
try {
await KVAPI.put(fullKey, newFieldValue);
setNewFieldKey('');
setNewFieldValue('');
onChanged(`Added ${fullKey}`);
// Refetch this file so the new field appears in the row list.
const r = await KVAPI.get(fullKey);
if (r.exists) {
setMetadata((cur) => ({ ...(cur ?? {}), [key]: r.value }));
}
} catch (e: any) {
onChanged(`Failed to add ${fullKey}`, [e.message]);
} finally {
setAddingField(false);
}
};
const onLanguageGroupChange = (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => {
setEdits((prev) => {
const next = new Map(prev);
const allLangs = new Set([...Object.keys(originalMap), ...Object.keys(nextMap)]);
for (const lang of allLangs) {
const fieldKey = `${groupKey}/${lang}`;
const orig = originalMap[lang] ?? null;
const cur = nextMap[lang] ?? null;
if (cur === orig) next.delete(fieldKey);
else if (cur == null) next.set(fieldKey, null);
else next.set(fieldKey, cur);
}
return next;
});
};
const handleSave = async () => {
if (edits.size === 0) return;
setSaving(true);
const errors: string[] = [];
for (const [fieldKey, newValue] of edits) {
const fullKey = branchPath + fieldKey;
try {
if (newValue === null) await KVAPI.del(fullKey);
else await KVAPI.put(fullKey, newValue);
} catch (e: any) {
errors.push(`${fieldKey}: ${e.message}`);
}
}
setSaving(false);
if (errors.length === 0) {
onChanged(`Saved ${edits.size} field${edits.size === 1 ? '' : 's'} on ${fileName ?? hashId}`);
} else {
onChanged(
`Saved with ${errors.length} failure(s) on ${fileName ?? hashId}`,
errors,
);
}
};
const handleDiscard = () => setEdits(new Map());
const toggleGroup = (prefix: string) => {
setCollapsedGroups((cur) => {
const next = new Set(cur);
if (next.has(prefix)) next.delete(prefix);
else next.add(prefix);
return next;
});
};
return (
<div className="kv-file">
<div className="kv-file-header">
<div className="kv-file-name">
{fileName ?? <span className="kv-file-name-fallback">(no fileName)</span>}
</div>
<div className="kv-file-meta">
<code className="kv-file-hash" title={hashId}>
{hashId.slice(0, 8)}…{hashId.slice(-6)}
</code>
{filePath && <span className="kv-file-path" title={filePath}>{filePath}</span>}
</div>
<div className="kv-file-actions">
<button
className="kv-btn kv-btn-primary"
disabled={edits.size === 0 || saving}
onClick={handleSave}
>
{saving ? 'saving…' : `Save (${edits.size})`}
</button>
<button
className="kv-btn"
disabled={edits.size === 0 || saving}
onClick={handleDiscard}
>
Discard
</button>
</div>
</div>
<div className="kv-file-rows">
{groups.length === 0 && (
<div className="kv-pane-status">No data yet for this file.</div>
)}
{groups.map((g) => (
<FieldGroupView
key={g.prefix}
group={g}
edits={edits}
collapsed={collapsedGroups.has(g.prefix)}
pulse={pulse}
onToggle={() => toggleGroup(g.prefix)}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageGroupChange}
onJumpToLeaf={(fieldKey) => onSelectLeaf(branchPath + fieldKey)}
/>
))}
</div>
<div className="kv-file-add">
<span className="kv-file-add-prefix">{branchPath}</span>
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
placeholder="field path (may contain /)"
value={newFieldKey}
onChange={(e) => setNewFieldKey(e.target.value)}
disabled={addingField}
/>
<input
type="text"
className="kv-leaf-input"
placeholder="value"
value={newFieldValue}
onChange={(e) => setNewFieldValue(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter' && newFieldKey.trim()) handleAddField();
}}
disabled={addingField}
/>
<button
className="kv-btn kv-btn-primary"
disabled={!newFieldKey.trim() || addingField}
onClick={handleAddField}
>
{addingField ? 'adding…' : '+ Add field'}
</button>
</div>
</div>
);
}
function FieldGroupView({
group,
edits,
collapsed,
pulse,
onToggle,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
group: RowGroup;
edits: Map<string, string | null>;
collapsed: boolean;
pulse: string | null;
onToggle: () => void;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
if (!group.isCluster) {
return (
<>
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</>
);
}
return (
<div className={`kv-file-group ${collapsed ? 'collapsed' : ''}`}>
<button
type="button"
className="kv-file-group-header"
onClick={onToggle}
title={collapsed ? 'expand' : 'collapse'}
>
<span className="kv-file-group-chevron">{collapsed ? '▸' : '▾'}</span>
<span className="kv-file-group-name">{group.prefix}</span>
<span className="kv-file-group-count">{group.rows.length}</span>
</button>
{!collapsed && (
<div className="kv-file-group-body">
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</div>
)}
</div>
);
}
function RowView({
row,
edits,
pulse,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
row: Row;
edits: Map<string, string | null>;
pulse: string | null;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
const pulsed = pulse === row.fieldKey;
if (row.kind === 'leaf') {
const pendingDelete = edits.get(row.fieldKey) === null;
const draft = edits.has(row.fieldKey) ? (edits.get(row.fieldKey) ?? '') : row.value;
const dirty = edits.has(row.fieldKey);
return (
<div
className={`kv-file-row ${dirty ? 'dirty' : ''} ${pendingDelete ? 'pending-delete' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<button
type="button"
className="kv-file-row-label"
onClick={() => onJumpToLeaf(row.fieldKey)}
title="open in single-leaf editor"
>
{row.fieldKey}
</button>
<div className="kv-file-row-value">
{pendingDelete ? (
<div className="kv-file-row-deleted-note">marked for deletion</div>
) : (
<LeafValueEditor
type={row.fieldType}
value={draft}
onChange={(v) => onLeafChange(row.fieldKey, v)}
/>
)}
</div>
<button
type="button"
className="kv-file-row-delete"
onClick={() => onLeafDeleteToggle(row.fieldKey)}
title={pendingDelete ? 'undo deletion' : 'delete this field'}
aria-label={pendingDelete ? 'undo deletion' : 'delete this field'}
>
{pendingDelete ? '↺' : '×'}
</button>
</div>
);
}
return (
<LanguageGroupRow
row={row}
edits={edits}
pulsed={pulsed}
onLanguageChange={onLanguageChange}
/>
);
}
function LanguageGroupRow({
row,
edits,
pulsed,
onLanguageChange,
}: {
row: LanguageRow;
edits: Map<string, string | null>;
pulsed: boolean;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
}) {
// Apply pending edits on top of the original language map.
const effectiveMap: Record<string, string> = { ...row.languageMap };
for (const [k, v] of edits) {
if (!k.startsWith(`${row.fieldKey}/`)) continue;
const lang = k.slice(row.fieldKey.length + 1);
if (v === null) delete effectiveMap[lang];
else effectiveMap[lang] = v;
}
const entries = Object.entries(effectiveMap);
const dirty = [...edits.keys()].some((k) => k.startsWith(`${row.fieldKey}/`));
const updateLang = (lang: string, text: string) => {
onLanguageChange(row.fieldKey, { ...effectiveMap, [lang]: text }, row.languageMap);
};
const renameLang = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const next = { ...effectiveMap };
const text = next[oldLang];
delete next[oldLang];
next[newLang] = text;
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const removeLang = (lang: string) => {
const next = { ...effectiveMap };
delete next[lang];
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const addLang = () => {
const used = new Set(entries.map(([k]) => k));
const defaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa', 'fre'];
const defaultLang = defaults.find((l) => !used.has(l)) || 'eng';
onLanguageChange(row.fieldKey, { ...effectiveMap, [defaultLang]: '' }, row.languageMap);
};
return (
<div
className={`kv-file-row kv-file-row-lang ${dirty ? 'dirty' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<div className="kv-file-row-label-static">{row.fieldKey}</div>
<div className="kv-file-row-value">
<div className="kv-file-lang-entries">
{entries.map(([lang, text]) => (
<div key={lang} className="kv-file-lang-entry">
<LanguageCombobox
value={lang}
onChange={(newLang) => renameLang(lang, newLang)}
/>
<input
type="text"
className="kv-leaf-input"
value={text}
title={formatLanguageOption(lang)}
onChange={(e) => updateLang(lang, e.target.value)}
/>
<button
type="button"
className="kv-btn-icon kv-file-lang-remove"
onClick={() => removeLang(lang)}
title="remove this language"
>
×
</button>
</div>
))}
<button
type="button"
className="kv-btn kv-file-lang-add"
onClick={addLang}
>
+ Add language
</button>
</div>
</div>
</div>
);
}
// ──────────────────────────────────────────────────────────────────────────
// Row model — kept local because it's only used here and is small.
// ──────────────────────────────────────────────────────────────────────────
function buildRows(
flat: Record<string, string>,
schema: ReturnType<typeof useSchema>['schema'],
): Row[] {
const languageGroupKeys = new Set<string>();
if (schema) {
for (const [k, info] of Object.entries(schema.fields)) {
if (info.key_hint === 'language-code') languageGroupKeys.add(k);
}
}
const langMaps: Record<string, Record<string, string>> = {};
const rows: Row[] = [];
for (const [key, value] of Object.entries(flat)) {
const slash = key.lastIndexOf('/');
if (slash > 0) {
const prefix = key.slice(0, slash);
const tail = key.slice(slash + 1);
if (languageGroupKeys.has(prefix) && LANG_CODE_RE.test(tail)) {
if (!langMaps[prefix]) langMaps[prefix] = {};
langMaps[prefix][tail] = value;
continue;
}
}
const info = schema?.fields[key];
rows.push({
kind: 'leaf',
fieldKey: key,
value,
// Schema decides when it can; otherwise heuristic on (path, value).
fieldType: schemaToFieldType(info) ?? detectType(key, value),
info,
});
}
for (const [groupKey, langMap] of Object.entries(langMaps)) {
rows.push({
kind: 'language-group',
fieldKey: groupKey,
languageMap: langMap,
info: schema?.fields[groupKey],
});
}
rows.sort((a, b) => a.fieldKey.localeCompare(b.fieldKey));
return rows;
}
/** Escape characters that would break a CSS attribute selector value. */
function cssEscape(s: string): string {
return s.replace(/["\\]/g, (c) => `\\${c}`);
}
function groupRows(rows: Row[]): RowGroup[] {
const out: RowGroup[] = [];
for (const row of rows) {
const prefix = row.fieldKey.split('/', 1)[0];
const last = out[out.length - 1];
if (last && last.prefix === prefix) {
last.rows.push(row);
} else {
out.push({ prefix, isCluster: false, rows: [row] });
}
}
for (const g of out) g.isCluster = g.rows.length > 1;
return out;
}
+163
View File
@@ -0,0 +1,163 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { useSchema } from '../api/schemaApi';
import { ALL_TYPES, describeType, detectType } from './typeHeuristics';
import { lookupSchemaForKey } from './schemaLookup';
import { LeafValueEditor } from './leafRenderers';
import { FieldType } from './types';
import { ClaimsPanel } from './ClaimsPanel';
interface Props {
// Full key path of the selected leaf (file:<rid>/<field> = merged view).
selectedKey: string;
// Bumped after a delete or external change to force refetch.
reloadToken: number;
onSaved: () => void;
onDeleted: () => void;
onError: (msg: string) => void;
}
export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onError }: Props) {
const [original, setOriginal] = useState<string>('');
const [draft, setDraft] = useState<string>('');
const [exists, setExists] = useState<boolean>(true);
const [redisType, setRedisType] = useState<string>('');
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
// Manual override for the inferred type. "auto" defers to schema → heuristic.
const [typeOverride, setTypeOverride] = useState<FieldType>('auto');
const { schema } = useSchema();
const schemaLookup = lookupSchemaForKey(schema, selectedKey);
// Schema is authoritative when it has an opinion; heuristics fill the gap.
const detected = schemaLookup.fieldType ?? detectType(selectedKey, original);
const effectiveType = typeOverride === 'auto' ? detected : typeOverride;
// Reset override when the selected key changes — the previous override
// doesn't apply to a different leaf.
useEffect(() => { setTypeOverride('auto'); }, [selectedKey]);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.get(selectedKey)
.then((r) => {
if (cancelled) return;
setOriginal(r.value);
setDraft(r.value);
setExists(r.exists);
setRedisType(r.type);
})
.catch((e) => onError(e.message))
.finally(() => { if (!cancelled) setLoading(false); });
return () => { cancelled = true; };
}, [selectedKey, reloadToken, onError]);
const dirty = draft !== original;
const editable = exists ? redisType === 'string' || redisType === '' : true;
const save = async () => {
setSaving(true);
try {
await KVAPI.put(selectedKey, draft);
setOriginal(draft);
setExists(true);
setRedisType('string');
onSaved();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
const del = async () => {
if (!confirm(`Retract your vote on ${selectedKey}? Other signers' claims keep the merged value.`)) return;
setSaving(true);
try {
await KVAPI.del(selectedKey);
onDeleted();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
if (loading) {
return <div className="kv-pane-status">loading…</div>;
}
if (!editable) {
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-pane-status">
Key has Redis type <code>{redisType}</code> — only string-typed values are editable here.
</div>
</div>
);
}
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-leaf-meta">
<label className="kv-leaf-label">Type</label>
<select
className="kv-leaf-type"
value={typeOverride}
onChange={(e) => setTypeOverride(e.target.value as FieldType)}
>
{ALL_TYPES.map((t) => (
<option key={t} value={t}>
{t === 'auto' ? `auto (${describeType(detected)})` : describeType(t)}
</option>
))}
</select>
{!exists && <span className="kv-leaf-hint">new key — save to create</span>}
</div>
<LeafValueEditor type={effectiveType} value={draft} onChange={setDraft} />
<div className="kv-leaf-actions">
<button
className="kv-btn kv-btn-primary"
onClick={save}
disabled={!dirty || saving}
>{saving ? 'saving…' : exists ? 'Save' : 'Create'}</button>
<button
className="kv-btn"
onClick={() => setDraft(original)}
disabled={!dirty || saving}
>Reset</button>
{exists && (
<button
className="kv-btn kv-btn-danger"
onClick={del}
disabled={saving}
>Retract</button>
)}
</div>
<ClaimsPanel recordKey={selectedKey} reloadToken={reloadToken} />
</div>
);
}
function KeyHeader({ keyPath }: { keyPath: string }) {
return (
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{keyPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(keyPath)}
title="copy key"
>⧉</button>
</div>
);
}
// Leaf value rendering moved to leafRenderers.tsx (LeafValueEditor)
// so the file-detail view can reuse it per row.
+339
View File
@@ -0,0 +1,339 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
const fileNameCache = new Map<string, string | null>();
const fileNamePromises = new Map<string, Promise<void>>();
const fileNameSubscribers = new Set<() => void>();
function notifyFileNameSubscribers() {
for (const cb of fileNameSubscribers) cb();
}
function ensureFileName(branchPath: string): void {
if (fileNameCache.has(branchPath) || fileNamePromises.has(branchPath)) return;
const p = KVAPI.get(branchPath + 'fileName')
.then((r) => {
fileNameCache.set(branchPath, r.exists && r.value ? r.value : null);
})
.catch(() => {
fileNameCache.set(branchPath, null);
})
.finally(() => {
fileNamePromises.delete(branchPath);
notifyFileNameSubscribers();
});
fileNamePromises.set(branchPath, p);
}
function useFileName(branchPath: string | null): string | null {
const [, force] = useState(0);
useEffect(() => {
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return;
ensureFileName(branchPath);
const cb = () => force((n) => n + 1);
fileNameSubscribers.add(cb);
return () => {
fileNameSubscribers.delete(cb);
};
}, [branchPath]);
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return null;
return fileNameCache.get(branchPath) ?? null;
}
function isFileBranch(path: string): boolean {
return FILE_BRANCH_RE.test(path);
}
function shortHashFromFileBranch(path: string): string {
// path = "file:<hash>/" → last 6 chars of hash
const hash = path.slice('file:'.length, -1);
if (hash.length <= 12) return hash;
return `…${hash.slice(-6)}`;
}
// A node in the lazy tree. branch-typed nodes have an associated path that
// ends with the delimiter and lazy-load their children on first expand.
type NodeKind = 'branch' | 'leaf';
interface TreeNode {
kind: NodeKind;
// Display label: the segment of the path under the parent.
label: string;
// Full key path. For branches, ends with the delimiter.
path: string;
}
interface BranchState {
loaded: boolean;
loading: boolean;
error?: string;
children: TreeNode[];
}
// Sub-segment of a path used as the visible label for a branch / leaf.
// Splits on either ":" or "/" so file:abc/title/eng becomes file: > abc > title > eng.
function labelOf(path: string, parentPath: string): string {
const rest = path.slice(parentPath.length);
// The "rest" includes a trailing delimiter for branches; strip it for display.
return rest.replace(/[/:]+$/, '');
}
// Hide internal Redis structures from the tree. The editor is for managing
// file metadata; everything else is meta-core's own bookkeeping and noise
// for the user. Hidden:
// - `meta:` — event streams (file:events, meta:events, etc.)
// - `cid:` — reverse index (cid:<algo>:<value> → uuid), derived
// from each file's cid_* fields; the editor surfaces
// these per-file under their parent root
// - `meta-core:` — internal service state (mounts, watcher config,
// leader info), edited via dedicated UI pages, not
// the KV tree
// - bare top-level leaves under a colon-namespace like file:__index__
// and file:events (Redis sets / streams that aren't user data).
function isHiddenNode(node: TreeNode): boolean {
if (node.kind === 'branch') {
if (node.path === 'meta:' || node.path === 'cid:' || node.path === 'meta-core:') {
return true;
}
}
if (node.kind === 'leaf' && /^[^/]+:[^/]+$/.test(node.path)) return true;
return false;
}
interface Props {
// Selected leaf path (full key) — highlighted.
selectedKey: string | null;
// When set, the tree forces these branches open (used for auto-expand on
// search hit or deep-link).
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
// Bumping this triggers a full reload of root and any expanded branches.
reloadToken: number;
}
export function KVTree({
selectedKey,
expandedPaths,
onSelectLeaf,
onSelectBranch,
onToggleBranch,
reloadToken,
}: Props) {
// Map of branch path → load state. The empty key "" represents the root.
const [branches, setBranches] = useState<Record<string, BranchState>>({});
// Load (or reload) a branch's children from the API. The root is special-
// cased: we list at prefix="" with delimiter=":" to surface the namespaces.
const loadBranch = async (path: string) => {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: true, children: b[path]?.children ?? [] },
}));
try {
const isRoot = path === '';
const delim = isRoot ? ':' : '/';
const resp = await KVAPI.treeAll(path, delim, 5000);
const branchNodes: TreeNode[] = resp.branches.map((p) => ({
kind: 'branch',
label: labelOf(p, path) + delim,
path: p,
}));
const leafNodes: TreeNode[] = resp.leaves.map((p) => ({
kind: 'leaf',
label: labelOf(p, path),
path: p,
}));
const children = [...branchNodes, ...leafNodes];
setBranches((b) => ({
...b,
[path]: { loaded: true, loading: false, children },
}));
} catch (err: any) {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: false, children: [], error: err.message },
}));
}
};
// On reloadToken change, refetch root and every currently-expanded branch.
useEffect(() => {
loadBranch('');
for (const p of expandedPaths) {
if (p !== '') loadBranch(p);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [reloadToken]);
// Whenever expandedPaths gains a path we haven't loaded yet, lazy-load it.
useEffect(() => {
for (const p of expandedPaths) {
if (p === '') continue;
const state = branches[p];
if (!state || (!state.loaded && !state.loading)) {
loadBranch(p);
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [expandedPaths]);
return (
<div className="kv-tree">
<Branch
path=""
state={branches['']}
depth={0}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={(p, open) => {
onToggleBranch(p, open);
if (open && (!branches[p] || !branches[p].loaded)) loadBranch(p);
}}
/>
</div>
);
}
interface BranchProps {
path: string;
state: BranchState | undefined;
depth: number;
expandedPaths: Set<string>;
branches: Record<string, BranchState>;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function Branch({
state, depth, expandedPaths, branches,
selectedKey, onSelectLeaf, onSelectBranch, onToggleBranch,
}: BranchProps) {
if (!state) {
return null;
}
if (state.loading && state.children.length === 0) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (state.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>error: {state.error}</div>;
}
const visibleChildren = state.children.filter((c) => !isHiddenNode(c));
if (visibleChildren.length === 0) {
return <div className="kv-tree-empty" style={{ paddingLeft: depth * 14 + 18 }}>(empty)</div>;
}
return (
<ul className="kv-tree-list">
{visibleChildren.map((node) => {
if (node.kind === 'branch') {
const open = expandedPaths.has(node.path);
const childState = branches[node.path];
return (
<li key={node.path}>
<BranchRow
node={node}
open={open}
depth={depth}
onToggle={() => {
onToggleBranch(node.path, !open);
onSelectBranch(node.path);
}}
/>
{open && (
<Branch
path={node.path}
state={childState}
depth={depth + 1}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
)}
</li>
);
}
const isSelected = node.path === selectedKey;
return (
<li key={node.path}>
<div
className={`kv-tree-row kv-tree-leaf${isSelected ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(node.path)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{node.label}</span>
</div>
</li>
);
})}
</ul>
);
}
interface BranchRowProps {
node: TreeNode;
open: boolean;
depth: number;
onToggle: () => void;
}
function BranchRow({ node, open, depth, onToggle }: BranchRowProps) {
// Surface the actual filename next to file:<hash>/ branches so the tree
// reads like a file list at that level instead of a wall of hashes.
const fileName = useFileName(isFileBranch(node.path) ? node.path : null);
return (
<div
className={`kv-tree-row kv-tree-branch${open ? ' open' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={onToggle}
title={isFileBranch(node.path) ? node.path : undefined}
>
<span className="kv-tree-toggle">{open ? '▾' : '▸'}</span>
{fileName ? (
<>
<span className="kv-tree-label kv-tree-filename">{fileName}</span>
<span className="kv-tree-hash-pill">{shortHashFromFileBranch(node.path)}</span>
</>
) : (
<span className="kv-tree-label">{node.label}</span>
)}
</div>
);
}
// Helper: given a key path with both ":" and "/" separators, return the list
// of ancestor branch paths that need to be expanded to reveal it.
export function ancestorBranches(key: string): string[] {
const out: string[] = [];
let i = 0;
while (i < key.length) {
const next = nextSep(key, i);
if (next < 0) break;
out.push(key.slice(0, next + 1));
i = next + 1;
}
return out;
}
function nextSep(s: string, from: number): number {
let best = -1;
for (const d of [':', '/']) {
const idx = s.indexOf(d, from);
if (idx >= 0 && (best < 0 || idx < best)) best = idx;
}
return best;
}
// Re-export for convenience in the shell
export type { TreeNode };
+57
View File
@@ -0,0 +1,57 @@
import { useState } from 'react';
interface Props {
fields: string[];
onChange: (fields: string[]) => void;
}
// Pill list of field names that the value-search uses. Each pill is removable;
// the inline input adds a new field on Enter or comma. Persisting to
// localStorage is the parent's responsibility.
export function SearchFieldChips({ fields, onChange }: Props) {
const [input, setInput] = useState('');
const add = (raw: string) => {
const next = raw.trim().replace(/,+$/, '');
if (!next) return;
if (fields.includes(next)) return;
onChange([...fields, next]);
setInput('');
};
const remove = (f: string) => onChange(fields.filter((x) => x !== f));
return (
<div className="kv-chips">
<span className="kv-chips-label">search in:</span>
{fields.map((f) => (
<span key={f} className="kv-chip">
<code>{f}</code>
<button
className="kv-chip-x"
onClick={() => remove(f)}
title={`remove ${f}`}
aria-label={`remove ${f}`}
>×</button>
</span>
))}
<input
className="kv-chip-input"
type="text"
value={input}
placeholder="+ field"
onChange={(e) => {
const v = e.target.value;
if (v.endsWith(',')) add(v);
else setInput(v);
}}
onKeyDown={(e) => {
if (e.key === 'Enter') { e.preventDefault(); add(input); }
if (e.key === 'Backspace' && input === '' && fields.length > 0) {
remove(fields[fields.length - 1]);
}
}}
/>
</div>
);
}
+36
View File
@@ -0,0 +1,36 @@
import { useEffect, useState } from 'react';
import { Account, ClaimsAPI, getSigner, setSigner } from '../api/claimsApi';
// SignerPicker chooses the identity account whose key signs the editor's
// edits. Every save is that account's claim — it wins in that account's view.
export function SignerPicker({ onError }: { onError: (msg: string) => void }) {
const [accounts, setAccounts] = useState<Account[]>([]);
const [uid, setUid] = useState(getSigner());
useEffect(() => {
ClaimsAPI.accounts()
.then((list) => {
setAccounts(list);
if (!uid && list.length === 1) {
setSigner(list[0].uid);
setUid(list[0].uid);
}
})
.catch((e) => onError(`accounts: ${e.message}`));
}, []); // eslint-disable-line react-hooks/exhaustive-deps
return (
<label className="kv-signer" title="Edits are claims signed by this account">
sign as{' '}
<select
value={uid}
onChange={(e) => { setSigner(e.target.value); setUid(e.target.value); }}
>
<option value="">— choose an account —</option>
{accounts.map((a) => (
<option key={a.uid} value={a.uid}>{a.uid.slice(0, 10)}…</option>
))}
</select>
</label>
);
}
+45
View File
@@ -0,0 +1,45 @@
import { useEffect } from 'react';
import { ToastEntry } from './types';
interface Props {
toasts: ToastEntry[];
onDismiss: (id: number) => void;
}
export function ToastStack({ toasts, onDismiss }: Props) {
return (
<div className="kv-toast-stack">
{toasts.map((t) => (
<Toast key={t.id} entry={t} onDismiss={() => onDismiss(t.id)} />
))}
</div>
);
}
function Toast({ entry, onDismiss }: { entry: ToastEntry; onDismiss: () => void }) {
useEffect(() => {
if (entry.kind === 'error') return;
const timer = setTimeout(onDismiss, 4500);
return () => clearTimeout(timer);
}, [entry.kind, onDismiss]);
return (
<div className={`kv-toast kv-toast-${entry.kind}`}>
<div className="kv-toast-row">
<span className="kv-toast-text">{entry.text}</span>
<button className="kv-toast-close" onClick={onDismiss} aria-label="dismiss">×</button>
</div>
{entry.details && entry.details.length > 0 && (
<ul className="kv-toast-details">
{entry.details.slice(0, 8).map((d, i) => <li key={i}>{d}</li>)}
{entry.details.length > 8 && <li>… {entry.details.length - 8} more</li>}
</ul>
)}
</div>
);
}
let nextId = 1;
export function makeToast(kind: ToastEntry['kind'], text: string, details?: string[]): ToastEntry {
return { id: nextId++, kind, text, details };
}
+295
View File
@@ -0,0 +1,295 @@
import { useEffect, useState } from 'react';
import { LanguageCombobox } from '../components/LanguageCombobox';
import { FieldType } from './types';
interface ValueRendererProps {
value: string;
onChange: (v: string) => void;
}
interface LeafValueEditorProps extends ValueRendererProps {
type: FieldType;
}
/**
* Type-dispatched leaf value editor. Used by both the single-leaf editor
* (KVLeafEditor) and the per-row editor in the file-detail view.
*/
export function LeafValueEditor({ type, value, onChange }: LeafValueEditorProps) {
switch (type) {
case 'bool':
return (
<select
className="kv-leaf-input"
value={value === 'true' ? 'true' : value === 'false' ? 'false' : ''}
onChange={(e) => onChange(e.target.value)}
>
<option value="">(unset)</option>
<option value="true">true</option>
<option value="false">false</option>
</select>
);
case 'number':
return (
<input
type="number"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'datetime':
return <DateTimeLeaf value={value} onChange={onChange} />;
case 'lang':
return <LanguageCombobox value={value} onChange={onChange} />;
case 'url':
return (
<div className="kv-leaf-url">
<input
type="url"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
{value && (
<a className="kv-leaf-url-open" href={value} target="_blank" rel="noreferrer">
open ↗
</a>
)}
</div>
);
case 'json':
return (
<textarea
className="kv-leaf-textarea kv-leaf-mono"
rows={12}
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
);
case 'text':
return (
<textarea
className="kv-leaf-textarea"
rows={8}
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'hash':
return <CidLeaf value={value} onChange={onChange} />;
case 'auto':
case 'string':
default:
return (
<input
type="text"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
}
}
interface CidFileInfo {
exists: boolean;
contentType?: string;
size?: number;
filePath?: string;
}
/**
* CID leaf editor. Shows the value as an editable monospace input plus a
* content-typed preview when meta-core can resolve the CID to a file on disk
* (`/api/file/{cid}/info` → image/video/audio rendering, else download-only).
*/
function CidLeaf({ value, onChange }: ValueRendererProps) {
const [info, setInfo] = useState<CidFileInfo | null>(null);
const cid = value.trim();
const fileUrl = cid ? `/file/${cid}` : '';
const downloadName = info?.filePath?.split('/').pop() || cid;
useEffect(() => {
if (!cid) {
setInfo(null);
return;
}
let alive = true;
fetch(`/api/file/${cid}/info`)
.then((r) => (r.ok ? r.json() : { exists: false }))
.then((data: CidFileInfo) => {
if (alive) setInfo(data);
})
.catch(() => {
if (alive) setInfo({ exists: false });
});
return () => {
alive = false;
};
}, [cid]);
const renderPreview = () => {
if (!info?.exists || !info.contentType || !cid) return null;
if (info.contentType.startsWith('image/')) {
return <img src={fileUrl} alt="" className="kv-leaf-cid-preview" loading="lazy" />;
}
if (info.contentType.startsWith('video/')) {
return (
<video
src={fileUrl}
className="kv-leaf-cid-preview kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
if (info.contentType.startsWith('audio/')) {
return (
<audio
src={fileUrl}
className="kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
return null;
};
return (
<div className="kv-leaf-cid">
{renderPreview()}
<div className="kv-leaf-cid-controls">
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
<button
type="button"
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(value)}
title="copy CID"
>
⧉
</button>
</div>
{info?.exists && cid && (
<div className="kv-leaf-cid-actions">
<a
href={fileUrl}
download={downloadName}
className="kv-btn kv-btn-primary"
title={`Download ${downloadName}${info.size ? ` (${formatBytes(info.size)})` : ''}`}
>
⬇ Download
</a>
<a href={fileUrl} className="kv-btn" target="_blank" rel="noreferrer">
open ↗
</a>
<span className="kv-leaf-cid-meta">
{info.contentType}
{info.size != null ? ` · ${formatBytes(info.size)}` : ''}
</span>
</div>
)}
</div>
);
}
/** Timestamp shapes we round-trip through the picker. */
type TimestampShape = 'iso-date' | 'iso-datetime' | 'unix-s' | 'unix-ms';
function detectShape(raw: string): TimestampShape | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) return 'iso-date';
if (/^\d+$/.test(trimmed)) {
const n = Number.parseInt(trimmed, 10);
if (n >= 1_000_000_000 && n <= 4_102_444_800) return 'unix-s';
if (n >= 1_000_000_000_000 && n <= 4_102_444_800_000) return 'unix-ms';
return null;
}
if (!Number.isNaN(Date.parse(trimmed))) return 'iso-datetime';
return null;
}
function rawToDate(raw: string, shape: TimestampShape): Date | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (shape === 'unix-s') return new Date(Number.parseInt(trimmed, 10) * 1000);
if (shape === 'unix-ms') return new Date(Number.parseInt(trimmed, 10));
const d = new Date(trimmed);
return Number.isNaN(d.getTime()) ? null : d;
}
function dateToRaw(d: Date, shape: TimestampShape): string {
switch (shape) {
case 'unix-s':
return String(Math.floor(d.getTime() / 1000));
case 'unix-ms':
return String(d.getTime());
case 'iso-date':
return d.toISOString().slice(0, 10);
case 'iso-datetime':
return d.toISOString();
}
}
/**
* Datetime leaf editor. Preserves the original storage shape — ISO date stays
* ISO date, unix seconds stay unix seconds, etc. Unknown shapes fall back to
* iso-datetime.
*/
function DateTimeLeaf({ value, onChange }: ValueRendererProps) {
const shape: TimestampShape = detectShape(value) ?? 'iso-datetime';
const date = rawToDate(value, shape);
const inputType = shape === 'iso-date' ? 'date' : 'datetime-local';
const inputValue = (() => {
if (!date || Number.isNaN(date.getTime())) return '';
if (inputType === 'date') return date.toISOString().slice(0, 10);
// datetime-local wants YYYY-MM-DDTHH:mm in the user's local timezone.
const local = new Date(date.getTime() - date.getTimezoneOffset() * 60000);
return local.toISOString().slice(0, 16);
})();
return (
<div className="kv-leaf-datetime">
<input
type={inputType}
className="kv-leaf-input"
value={inputValue}
onChange={(e) => {
const v = e.target.value;
if (!v) {
onChange('');
return;
}
const parsed = new Date(v);
if (Number.isNaN(parsed.getTime())) return;
onChange(dateToRaw(parsed, shape));
}}
/>
<span className="kv-leaf-datetime-shape" title="Storage format preserved on save">
{shape}
</span>
</div>
);
}
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
const units = ['KB', 'MB', 'GB', 'TB'];
let v = n / 1024;
let i = 0;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[i]}`;
}
+90
View File
@@ -0,0 +1,90 @@
import { SchemaFieldInfo, SchemaResponse } from '../api/schemaApi';
import { FieldType } from './types';
export interface SchemaLookup {
/** The matched schema entry, if any. */
info?: SchemaFieldInfo;
/** Suggested FieldType for the leaf editor, or null when schema can't decide. */
fieldType: FieldType | null;
/**
* True when the leaf is one variant under a language-keyed group
* (e.g. plot/eng). The schema entry returned is the group's, not the leaf's.
*/
isLanguageVariant: boolean;
}
/**
* Resolve the logical schema entry for a full Redis key.
*
* - "file:<hash>/releasedate" → schema.fields["releasedate"]
* - "file:<hash>/plot/eng" → schema.fields["plot"] (language-keyed group)
* - non-file keys → undefined (schema is per-file today)
*/
export function lookupSchemaForKey(
schema: SchemaResponse | null,
key: string,
): SchemaLookup {
if (!schema) {
return { fieldType: null, isLanguageVariant: false };
}
const propertyPath = stripFilePrefix(key);
if (!propertyPath) {
return { fieldType: null, isLanguageVariant: false };
}
const direct = schema.fields[propertyPath];
if (direct) {
return {
info: direct,
fieldType: schemaToFieldType(direct),
isLanguageVariant: false,
};
}
// Language-keyed collapse: tail is a 2/3-letter lang code under a known group.
const lastSlash = propertyPath.lastIndexOf('/');
if (lastSlash > 0) {
const tail = propertyPath.slice(lastSlash + 1);
if (/^[a-z]{2,3}$/.test(tail)) {
const group = schema.fields[propertyPath.slice(0, lastSlash)];
if (group?.key_hint === 'language-code') {
return {
info: group,
fieldType: schemaToFieldType(group),
isLanguageVariant: true,
};
}
}
}
return { fieldType: null, isLanguageVariant: false };
}
export function schemaToFieldType(info: SchemaFieldInfo | undefined): FieldType | null {
if (!info) return null;
if (info.hint === 'cid') return 'hash';
if (info.hint === 'timestamp') return 'datetime';
switch (info.type) {
case 'bool':
return 'bool';
case 'int':
case 'float':
return 'number';
case 'json-object':
case 'json-array':
return 'json';
case 'string':
case 'mixed':
case 'undefined':
default:
return null;
}
}
function stripFilePrefix(key: string): string {
if (!key.startsWith('file:')) return '';
const slash = key.indexOf('/');
if (slash < 0) return '';
return key.slice(slash + 1);
}
+59
View File
@@ -0,0 +1,59 @@
import { FieldType } from './types';
// Path-pattern table. The leftmost segment of the path that matches wins.
// Order matters: more specific patterns should appear first.
const PATH_PATTERNS: Array<[RegExp, FieldType]> = [
[/(^|\/)(language|lang|audioLang|subLang)$/i, 'lang'],
[/(^|\/)(cid_|hash$|sha\d|midhash|md5|crc32)/i, 'hash'],
[/(^|\/)(duration|bitrate|fps|width|height|count|size|sizeByte|increment|episode|season|year|runtime|trackId|index)$/i, 'number'],
[/(^|\/)(enabled|active|disabled|isHidden|hidden|favorite|watched)$/i, 'bool'],
[/(^|\/)(url|uri|href|link|posterUrl|backdropUrl|imageUrl)$/i, 'url'],
[/(^|\/)(.+_at|.+At|.+Date|.+Time|timestamp|createdAt|updatedAt)$/i, 'datetime'],
[/(^|\/)(description|overview|notes?|comment|summary|nfo)$/i, 'text'],
];
function detectFromPath(path: string): FieldType | null {
for (const [re, t] of PATH_PATTERNS) {
if (re.test(path)) return t;
}
return null;
}
function detectFromValue(v: string): FieldType {
if (v === '') return 'string';
if (v === 'true' || v === 'false') return 'bool';
if (/^-?\d+$/.test(v)) return 'number';
if (/^-?\d*\.\d+$/.test(v)) return 'number';
if (/^https?:\/\//i.test(v)) return 'url';
const trimmed = v.trim();
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
try { JSON.parse(trimmed); return 'json'; } catch {}
}
if (v.includes('\n') || v.length > 200) return 'text';
return 'string';
}
// Combine path and value heuristics. Path wins when it matches; otherwise
// fall back to value shape.
export function detectType(path: string, value: string): FieldType {
return detectFromPath(path) ?? detectFromValue(value);
}
export const ALL_TYPES: FieldType[] = [
'auto', 'string', 'text', 'number', 'bool', 'json', 'url', 'lang', 'datetime', 'hash',
];
export function describeType(t: FieldType): string {
switch (t) {
case 'auto': return 'auto';
case 'string': return 'string';
case 'text': return 'text (multiline)';
case 'number': return 'number';
case 'bool': return 'bool';
case 'json': return 'json';
case 'url': return 'url';
case 'lang': return 'language code';
case 'datetime': return 'datetime';
case 'hash': return 'hash / cid';
}
}
+19
View File
@@ -0,0 +1,19 @@
// Inferred type for a leaf value. "auto" defers to the heuristic detector.
export type FieldType =
| 'auto'
| 'string'
| 'number'
| 'bool'
| 'json'
| 'url'
| 'text'
| 'lang'
| 'datetime'
| 'hash';
export interface ToastEntry {
id: number;
kind: 'info' | 'success' | 'error';
text: string;
details?: string[];
}
+2 -2
View File
@@ -1,10 +1,10 @@
import { StrictMode } from 'react';
import { createRoot } from 'react-dom/client';
import './index.css';
import App from './App.tsx';
import KVApp from './kv/KVApp.tsx';
createRoot(document.getElementById('root')!).render(
<StrictMode>
<App />
<KVApp />
</StrictMode>,
);
-155
View File
@@ -1,155 +0,0 @@
// API Response Types
export interface FileMetadata {
// File info
fileName?: string;
extension?: string;
byteSize?: number;
mimeType?: string;
fileType?: string;
// Filename parsing
title?: string;
titles?: string[];
originaltitle?: string;
showtitle?: string;
season?: number;
episode?: number;
movieYear?: number;
videoType?: string;
// General media metadata
year?: number;
imdbid?: string;
tmdbid?: string;
anidbid?: string;
premiered?: string;
genre?: string[];
releasedate?: string;
plot?: string;
director?: string;
rating?: number;
runtime?: number;
studio?: string;
art?: any;
actor?: any[];
aired?: string;
customrating?: string;
dateadded?: string;
sorttitle?: string;
mpaa?: string;
aspectratio?: string;
collectionnumber?: number;
criticrating?: number;
// Hashes
cid_crc32?: string;
cid_md5?: string;
cid_sha1?: string;
'cid_sha2-256'?: string;
'cid_sha3-256'?: string;
cid_sha3_384?: string;
// Video info
video?: {
codec?: string;
width?: number;
height?: number;
bitrate?: number;
fps?: number;
};
// Audio info
audio?: Array<{
codec?: string;
channels?: number;
language?: string;
bitrate?: number;
}>;
// Subtitles
subtitles?: Array<{
language?: string;
codec?: string;
}>;
// Other
extra?: any;
anime?: any;
fileinfo?: any;
trackers?: string[];
// Allow any additional properties
[key: string]: any;
}
export interface SearchResult {
hashId: string;
metadata: FileMetadata;
score?: number;
}
export interface SearchResponse {
results: SearchResult[];
count: number;
total?: number;
}
export interface UpdateResult {
status: string;
hashId: string;
message?: string;
error?: string;
}
export interface BatchUpdateResponse {
status: string;
total: number;
success: number;
errors: number;
results: Array<{
hashId: string;
status: string;
error?: string;
}>;
}
// Plugin Schema Types
export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
export interface SelectOption {
value: string;
label: string;
}
export interface SchemaField {
label: string;
type?: SchemaFieldType;
readonly?: boolean;
indexed?: boolean;
options?: SelectOption[];
hint?: string;
link?: string;
pattern?: string;
min?: number;
max?: number;
placeholder?: string;
}
export interface PluginSchema {
id: string;
name: string;
version: string;
description?: string;
priority?: number;
color?: string;
dependencies: string[];
active: boolean;
metadataSchema?: Record<string, SchemaField>;
}
export interface PluginsResponse {
plugins: PluginSchema[];
executionOrder: string[];
activeCount: number;
totalCount: number;
}
-53
View File
@@ -1,53 +0,0 @@
/**
* Format bytes to human readable format
*/
export function formatBytes(bytes: number | undefined): string {
if (!bytes) return 'N/A';
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
if (bytes === 0) return '0 B';
const i = Math.floor(Math.log(bytes) / Math.log(1024));
return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i];
}
/**
* Format duration in seconds to HH:MM:SS
*/
export function formatDuration(seconds: number | undefined): string {
if (!seconds) return 'N/A';
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
if (hours > 0) {
return `${hours}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
}
return `${minutes}:${secs.toString().padStart(2, '0')}`;
}
/**
* Truncate hash to first and last 6 characters
*/
export function truncateHash(hash: string | undefined): string {
if (!hash) return 'N/A';
if (hash.length <= 16) return hash;
return `${hash.slice(0, 6)}...${hash.slice(-6)}`;
}
/**
* Get display name for a file from metadata
*/
export function getDisplayName(metadata: any): string {
if (metadata.title) {
if (metadata.season && metadata.episode) {
return `${metadata.title} S${String(metadata.season).padStart(2, '0')}E${String(metadata.episode).padStart(2, '0')}`;
}
if (metadata.movieYear) {
return `${metadata.title} (${metadata.movieYear})`;
}
return metadata.title;
}
return metadata.fileName || 'Unknown';
}
+1
View File
@@ -0,0 +1 @@
{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/schemaApi.ts","./src/api/snapshotApi.ts","./src/components/LanguageCombobox.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVFileEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/leafRenderers.tsx","./src/kv/schemaLookup.ts","./src/kv/typeHeuristics.ts","./src/kv/types.ts"],"version":"5.9.3"}
+15 -5
View File
@@ -1,15 +1,25 @@
module github.com/metazla/meta-core
go 1.21
go 1.26.0
require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1
github.com/redis/go-redis/v9 v9.4.0
golang.org/x/net v0.20.0
github.com/mr-tron/base58 v1.3.0
golang.org/x/crypto v0.31.0
golang.org/x/net v0.21.0
modernc.org/sqlite v1.60.1
)
require (
github.com/cespare/xxhash/v2 v2.2.0 // indirect
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
github.com/dustin/go-humanize v1.0.1 // indirect
github.com/mattn/go-isatty v0.0.24 // indirect
github.com/ncruces/go-strftime v1.0.0 // indirect
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec // indirect
modernc.org/libc v1.77.1 // indirect
modernc.org/mathutil v1.7.1 // indirect
modernc.org/memory v1.12.1 // indirect
)
require golang.org/x/sys v0.48.0
+58 -12
View File
@@ -1,16 +1,62 @@
github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3 h1:LMLX+LgTNWpfvCBdFebv6EsYotImrt/Ppc5cXIriCSo=
github.com/google/pprof v0.0.0-20260802141513-ef3492d7dac3/go.mod h1:jl5iWTm0/hd5PjEYEOuwAJ57L/CibdZfrqZ5XA5GrCk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k=
github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM=
github.com/mattn/go-isatty v0.0.24 h1:tGZZoVgT/KiqK1c8ocVLeDS8BSWMRd47J3Lbz7vsReI=
github.com/mattn/go-isatty v0.0.24/go.mod h1:nMCL3Zebbrt45jsMDgnfIwz6ydEQApk5oEI3HqDio6A=
github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
github.com/ncruces/go-strftime v1.0.0 h1:HMFp8mLCTPp341M/ZnA4qaf7ZlsbTc+miZjCLOFAw7w=
github.com/ncruces/go-strftime v1.0.0/go.mod h1:Fwc5htZGVVkseilnfgOVb9mKy6w1naJmn9CehxcKcls=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec h1:W09IVJc94icq4NjY3clb7Lk8O1qJ8BdBEF8z0ibU0rE=
github.com/remyoudompheng/bigfft v0.0.0-20230129092748-24d4a6f8daec/go.mod h1:qqbHyh8v60DhA7CoWK5oRCqLrMHRGoxYCSS9EjAz6Eo=
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/mod v0.41.0 h1:qJmnOUb4YB+FsEuM3HcWucdZASCPGhsX6uljO6pog0c=
golang.org/x/mod v0.41.0/go.mod h1:Ek9pY8RKWXwsWvd3rQiHYtMqkjSUV+s1Rj7j4H5Ur6o=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
golang.org/x/sync v0.23.0 h1:KameEIfc1IkluZyXWLn39Wd4tURc6GbCiISGiZm2bQk=
golang.org/x/sync v0.23.0/go.mod h1:sUUOizhqBxiL6pEWpqNLUiaJn1ShEbZ6BBqskPbjZm0=
golang.org/x/sys v0.48.0 h1:bbX/i/6MgT9BVLM9RT1thmxL04yeTAhbEz4SyadbXoo=
golang.org/x/sys v0.48.0/go.mod h1:hNLxWAXmnKAxqDtdwIYC4bM9oQPEecfsnNMuSxOs3og=
golang.org/x/tools v0.50.0 h1:c2ifzfcuY7L90lZ2aKd8S4K2NpASF08SZx9ZuJkHmSU=
golang.org/x/tools v0.50.0/go.mod h1:7ulVMw3831Mwi5EZD6RomGyffr4VFjuNYXf2BbCEAV0=
modernc.org/cc/v4 v4.29.7 h1:q+NXGJ0bK3b4TXFYQQVr9pYETGnmwFWkrUzJnMya/Tg=
modernc.org/cc/v4 v4.29.7/go.mod h1:OnovgIhbbMXMu1aISnJ0wvVD1KnW+cAUJkIrAWh+kVI=
modernc.org/ccgo/v4 v4.36.1 h1:ZNIUZAryN0UgnJwtyxrdEzcFc3yD4Cu4AzjfPXsLsIE=
modernc.org/ccgo/v4 v4.36.1/go.mod h1:rrtGc2QkS239nYb/mQNuBMyjq3/y3ZXWbBjPoV3wqzA=
modernc.org/fileutil v1.4.0 h1:j6ZzNTftVS054gi281TyLjHPp6CPHr2KCxEXjEbD6SM=
modernc.org/fileutil v1.4.0/go.mod h1:EqdKFDxiByqxLk8ozOxObDSfcVOv/54xDs/DUHdvCUU=
modernc.org/gc/v2 v2.6.5 h1:nyqdV8q46KvTpZlsw66kWqwXRHdjIlJOhG6kxiV/9xI=
modernc.org/gc/v2 v2.6.5/go.mod h1:YgIahr1ypgfe7chRuJi2gD7DBQiKSLMPgBQe9oIiito=
modernc.org/gc/v3 v3.1.5 h1:21ldfPfRYE31Tb7B3mwAK8gy1AxP4+dKjrOQPfqakoc=
modernc.org/gc/v3 v3.1.5/go.mod h1:HFK/6AGESC7Ex+EZJhJ2Gni6cTaYpSMmU/cT9RmlfYY=
modernc.org/goabi0 v0.2.0 h1:HvEowk7LxcPd0eq6mVOAEMai46V+i7Jrj13t4AzuNks=
modernc.org/goabi0 v0.2.0/go.mod h1:CEFRnnJhKvWT1c1JTI3Avm+tgOWbkOu5oPA8eH8LnMI=
modernc.org/libc v1.77.1 h1:Ct8j47QtiZ1Enj2DtFXQtUqrPCAjdCmPjtCuvrYQ0Hs=
modernc.org/libc v1.77.1/go.mod h1:87/pZ4L6nD1zqW4nItuS12YO7hN1igAah34xjnQo/W0=
modernc.org/mathutil v1.7.1 h1:GCZVGXdaN8gTqB1Mf/usp1Y/hSqgI2vAGGP4jZMCxOU=
modernc.org/mathutil v1.7.1/go.mod h1:4p5IwJITfppl0G4sUEDtCr4DthTaT47/N3aT6MhfgJg=
modernc.org/memory v1.12.1 h1:nFMiWrpStgZczNl6XI9GnIk/rWhYIyHGUaR04pGbp9g=
modernc.org/memory v1.12.1/go.mod h1:/JP4VbVC+K5sU2wZi9bHoq2MAkCnrt2r98UGeSK7Mjw=
modernc.org/opt v0.2.0 h1:tGyef5ApycA7FSEOMraay9SaTk5zmbx7Tu+cJs4QKZg=
modernc.org/opt v0.2.0/go.mod h1:03fq9lsNfvkYSfxrfUhZCWPk1lm4cq4N+Bh//bEtgns=
modernc.org/sortutil v1.2.1 h1:+xyoGf15mM3NMlPDnFqrteY07klSFxLElE2PVuWIJ7w=
modernc.org/sortutil v1.2.1/go.mod h1:7ZI3a3REbai7gzCLcotuw9AC4VZVpYMjDzETGsSMqJE=
modernc.org/sqlite v1.60.1 h1:/blz53O951KWFOso4QQvEs/Fq6cDBKLtMVrYNSeJVKw=
modernc.org/sqlite v1.60.1/go.mod h1:1dIoEagfDE72QytD5scH1lxARtaUgKgHC/NuApA27r0=
modernc.org/strutil v1.2.1 h1:UneZBkQA+DX2Rp35KcM69cSsNES9ly8mQWD71HKlOA0=
modernc.org/strutil v1.2.1/go.mod h1:EHkiggD70koQxjVdSBM3JKM7k6L0FbGE5eymy9i3B9A=
modernc.org/token v1.1.0 h1:Xl7Ap9dKaEs5kLoOQeQmPWevfnk/DM5qcLcYlA8ys6Y=
modernc.org/token v1.1.0/go.mod h1:UGzOrNV1mAFSEB63lOFHIpNRUVMvYTc6yu1SMY/XTDM=
+76
View File
@@ -0,0 +1,76 @@
package api
import (
"net/http"
"github.com/gorilla/mux"
)
// handleGetMetaByCID handles GET /api/meta/{cid}.
//
// Resolves any known CID — midhash256, sha256, IPFS, btih, or anything else
// in the reverse index — to the full metadata document for that file.
// Returns:
// - 200 with the document on a hit
// - 404 if the CID isn't registered
// - 503 if storage is down
//
// CIDs in the URL are bare multibase CIDv1 strings ("bafk…") — the midhash,
// a sibling sha2-256, an IPFS CID, a btih, anything registered in the
// reverse index. A CID is self-describing (its algorithm is the multicodec),
// so there is no <algo>: prefix.
//
// This endpoint is the public read API for the CID-resolution layer and
// is auth-bypassed alongside /api/file/{cid} so that meta-share peers and
// IPFS-style clients can use it without going through Authelia.
func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
uuid, err := s.storage.GetByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if uuid == "" {
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
doc, err := s.storage.GetMetadataDocument(uuid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
// Reverse index pointed at a UUID with no remaining keys — stale
// alias, probably mid-delete race. Treat as not-found.
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
if as := r.URL.Query().Get("as"); as != "" {
if flat, err := s.storage.GetMetadataFlatAs(uuid, as); err == nil && flat != nil {
doc.Flat = flat
}
}
// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
// cids / canonical_cid / duplicates), with cid added so the caller can
// correlate when they fan out N lookups. canonical_cid is derived by
// rank from the cids key-set on read — it is not a stored field.
writeJSON(w, http.StatusOK, map[string]interface{}{
"cid": cid,
"metadata": doc.Flat,
"cids": doc.CIDs,
"canonical_cid": doc.Canonical,
"duplicates": doc.Duplicates,
})
}
+172
View File
@@ -0,0 +1,172 @@
package api
// Signed claims (meta-core v2.0.0, docs/implementation/metacore-claims-model.md).
//
// POST /api/claims the only metadata write path
// GET /api/meta/{cid}/claims every live claim of the cid's group
// GET /api/metadata/{rid}/claims the same, addressed by rid
// POST /api/merge stateless merge of claims the caller holds
//
// meta-core never signs a claim here: it verifies each one (MMUR2, low-S) and
// the signature is the write authentication.
import (
"encoding/json"
"net/http"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/claim"
"github.com/metazla/meta-core/internal/storage"
)
// GoneResponse is the body of every removed unsigned write route.
type GoneResponse struct {
Error string `json:"error"`
Use string `json:"use"`
}
// handleGone answers the unsigned write routes removed in v2.0.0.
func (s *Server) handleGone(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusGone, GoneResponse{Error: "gone", Use: "POST /api/claims"})
}
// decodeClaims accepts a JSON array of claims or {"claims": [...]}.
func decodeClaims(r *http.Request) ([]claim.Claim, error) {
var raw json.RawMessage
if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&raw); err != nil {
return nil, err
}
var claims []claim.Claim
if len(raw) > 0 && raw[0] == '{' {
var env struct {
Claims []claim.Claim `json:"claims"`
}
if err := json.Unmarshal(raw, &env); err != nil {
return nil, err
}
return env.Claims, nil
}
if err := json.Unmarshal(raw, &claims); err != nil {
return nil, err
}
return claims, nil
}
// handlePostClaims handles POST /api/claims: one status per claim, in input
// order. 200 even when some claims are refused — the statuses say which.
func (s *Server) handlePostClaims(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
claims, err := decodeClaims(r)
if err != nil {
writeErrorSlug(w, http.StatusBadRequest, "invalid_json", "body must be a JSON array of claims: "+err.Error(), false)
return
}
if len(claims) > storage.MaxClaimsPerBatch {
writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per batch", false)
return
}
results, err := s.storage.WriteClaims(r.Context(), claims)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, results)
}
// ClaimsResponse is the body of the claims read routes: the record's rid, the
// cids of its group, and every live signed claim on them.
type ClaimsResponse struct {
RID string `json:"rid"`
CIDs []string `json:"cids"`
Claims []claim.Claim `json:"claims"`
}
// handleClaimsByCID handles GET /api/meta/{cid}/claims[?key=…].
func (s *Server) handleClaimsByCID(w http.ResponseWriter, r *http.Request) {
cid := mux.Vars(r)["cid"]
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
rid, err := s.storage.GetByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if rid == "" {
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this CID", false)
return
}
s.writeClaims(w, r, rid)
}
// handleClaimsByRID handles GET /api/metadata/{rid}/claims[?key=…]. The rid
// may also be a cid (resolved like every /api/metadata read).
func (s *Server) handleClaimsByRID(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
rid := s.storage.ResolveRoot(mux.Vars(r)["hashId"])
s.writeClaims(w, r, rid)
}
// writeClaims answers both claims routes: ?key= keeps one exact key, ?uid=
// one signer; 404 when the record does not exist.
func (s *Server) writeClaims(w http.ResponseWriter, r *http.Request, rid string) {
cids, claims, err := s.storage.RecordClaims(rid, r.URL.Query().Get("key"))
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if len(cids) == 0 {
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no record for this rid or cid", false)
return
}
out := make([]claim.Claim, 0, len(claims))
uid := r.URL.Query().Get("uid")
for _, cl := range claims {
if uid == "" || cl.UID == uid {
out = append(out, cl)
}
}
writeJSON(w, http.StatusOK, ClaimsResponse{RID: rid, CIDs: cids, Claims: out})
}
// MergeRequest is the body of POST /api/merge.
type MergeRequest struct {
Claims []claim.Claim `json:"claims"`
As string `json:"as,omitempty"`
}
// MergeResponse is the merged field map, the number of agreeing signers per
// key, and how many claims failed verification (dropped, never an error).
type MergeResponse struct {
Fields map[string]string `json:"fields"`
Nsrc map[string]int `json:"nsrc"`
Rejected int `json:"rejected"`
}
// handleMerge handles POST /api/merge: verify, merge per key with the same
// rule the cache uses, overlay `as`'s own votes held by this core. No write.
func (s *Server) handleMerge(w http.ResponseWriter, r *http.Request) {
var req MergeRequest
if err := json.NewDecoder(http.MaxBytesReader(nil, r.Body, 64<<20)).Decode(&req); err != nil {
writeErrorSlug(w, http.StatusBadRequest, "invalid_json", err.Error(), false)
return
}
if len(req.Claims) > storage.MaxClaimsPerBatch {
writeErrorSlug(w, http.StatusRequestEntityTooLarge, "too_many_claims", "at most 10000 claims per merge", false)
return
}
fields, nsrc, rejected, err := s.storage.MergeClaims(req.Claims, req.As)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, MergeResponse{Fields: fields, Nsrc: nsrc, Rejected: rejected})
}
+182
View File
@@ -0,0 +1,182 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/claim"
"github.com/metazla/meta-core/internal/storage"
)
func withVars(r *http.Request, vars map[string]string) *http.Request { return mux.SetURLVars(r, vars) }
// routedServer wires the real route table, so the test sees what clients see.
func routedServer(t *testing.T) *Server {
t.Helper()
s := &Server{storage: newTestStorage(t), router: mux.NewRouter()}
s.setupRoutes()
return s
}
func call(t *testing.T, s *Server, method, path, body string) *httptest.ResponseRecorder {
t.Helper()
req := httptest.NewRequest(method, path, strings.NewReader(body))
rr := httptest.NewRecorder()
s.router.ServeHTTP(rr, req)
return rr
}
// Every unsigned write route of v1.x answers 410 with a pointer to the claims.
func TestUnsignedWritesAreGone(t *testing.T) {
s := routedServer(t)
for _, c := range []struct{ method, path string }{
{"PUT", "/meta/bafy"}, {"PATCH", "/meta/bafy"}, {"DELETE", "/meta/bafy"},
{"POST", "/meta/bafy/_add/genres"}, {"PUT", "/meta/bafy/title"}, {"DELETE", "/meta/bafy/title"},
{"PUT", "/api/metadata/bafy"}, {"DELETE", "/api/metadata/bafy"}, {"PUT", "/api/metadata/bafy/property"},
{"POST", "/api/metadata/batch"},
} {
rr := call(t, s, c.method, c.path, `{"title":"x"}`)
if rr.Code != http.StatusGone || !strings.Contains(rr.Body.String(), `"use":"POST /api/claims"`) {
t.Errorf("%s %s = %d %s", c.method, c.path, rr.Code, rr.Body)
}
}
if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"file:R/title","value":"x"}`); rr.Code != http.StatusConflict {
t.Errorf("raw record write via /api/kv/value = %d", rr.Code)
}
if rr := call(t, s, "PUT", "/api/kv/value", `{"key":"btmagnet:abc","value":"magnet:"}`); rr.Code != http.StatusOK {
t.Errorf("service key write = %d %s", rr.Code, rr.Body)
}
}
func TestPostClaimsAndReadBack(t *testing.T) {
s := routedServer(t)
a, b := testSigner(t, 1), testSigner(t, 2)
bad := signed(t, b, "bafyA", "year", "1999", 5)
bad.Value = claim.Text("2001")
body, _ := json.Marshal([]claim.Claim{
signed(t, a, "bafyA", "title", "Matrix", 5),
signed(t, b, "bafyA", "title", "Matrix", 6),
bad,
})
rr := call(t, s, "POST", "/api/claims", string(body))
if rr.Code != http.StatusOK {
t.Fatalf("POST /api/claims = %d %s", rr.Code, rr.Body)
}
var res []storage.ClaimResult
_ = json.Unmarshal(rr.Body.Bytes(), &res)
if len(res) != 3 || res[0].Status != "accepted" || res[1].Status != "accepted" || res[2].Status != "bad_signature" {
t.Fatalf("statuses %+v", res)
}
rr = call(t, s, "GET", "/api/meta/bafyA", "")
if !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
t.Fatalf("merged read = %s", rr.Body)
}
rr = call(t, s, "GET", "/api/meta/bafyA/claims", "")
var cr ClaimsResponse
_ = json.Unmarshal(rr.Body.Bytes(), &cr)
if rr.Code != http.StatusOK || len(cr.Claims) != 2 {
t.Fatalf("claims by cid = %d %s", rr.Code, rr.Body)
}
if cr.RID == "" || len(cr.CIDs) != 1 || cr.CIDs[0] != "bafyA" {
t.Fatalf("claims envelope rid/cids = %q %v", cr.RID, cr.CIDs)
}
rr = call(t, s, "GET", "/api/metadata/"+cr.RID+"/claims?key=title&uid="+a.UID, "")
var byRid ClaimsResponse
_ = json.Unmarshal(rr.Body.Bytes(), &byRid)
if rr.Code != http.StatusOK || len(byRid.Claims) != 1 || byRid.Claims[0].UID != a.UID || byRid.RID != cr.RID {
t.Fatalf("claims by rid with key+uid = %d %s", rr.Code, rr.Body)
}
if rr := call(t, s, "GET", "/api/meta/bafyA/claims?key=nope", ""); !strings.Contains(rr.Body.String(), `"claims":[]`) {
t.Fatalf("key filter = %s", rr.Body)
}
if rr := call(t, s, "GET", "/api/meta/bafyUnknown/claims", ""); rr.Code != http.StatusNotFound {
t.Fatalf("unknown cid claims = %d", rr.Code)
}
if rr := call(t, s, "GET", "/api/metadata/01NOSUCHRID/claims", ""); rr.Code != http.StatusNotFound {
t.Fatalf("unknown rid claims = %d", rr.Code)
}
// ?as= overlays the viewer's own vote.
me := testSigner(t, 9)
storeClaims(t, s.storage, signed(t, me, "bafyA", "title", "My title", 7))
if rr := call(t, s, "GET", "/api/meta/bafyA?as="+me.UID, ""); !strings.Contains(rr.Body.String(), `"title":"My title"`) {
t.Fatalf("?as overlay = %s", rr.Body)
}
if rr := call(t, s, "GET", "/api/meta/bafyA", ""); !strings.Contains(rr.Body.String(), `"title":"Matrix"`) {
t.Fatalf("shared view changed by a single viewer: %s", rr.Body)
}
}
func TestMergeEndpoint(t *testing.T) {
s := routedServer(t)
a, b, c := testSigner(t, 1), testSigner(t, 2), testSigner(t, 3)
bad := signed(t, c, "bafyX", "title", "Forged", 1)
bad.UID = a.UID
body, _ := json.Marshal(MergeRequest{Claims: []claim.Claim{
signed(t, a, "bafyX", "title", "Dune", 2),
signed(t, b, "bafyX", "title", "Dune", 3),
signed(t, c, "bafyX", "title", "Dune Part One", 1),
bad,
}})
rr := call(t, s, "POST", "/api/merge", string(body))
var mr MergeResponse
_ = json.Unmarshal(rr.Body.Bytes(), &mr)
if rr.Code != http.StatusOK || mr.Fields["title"] != "Dune" || mr.Nsrc["title"] != 2 || mr.Rejected != 1 {
t.Fatalf("merge = %d %s", rr.Code, rr.Body)
}
if !strings.Contains(rr.Body.String(), `"rejected":1`) || !strings.Contains(rr.Body.String(), `"nsrc":{"title":2}`) {
t.Fatalf("merge wire shape = %s", rr.Body)
}
// `as`: the viewer's own vote in the request wins…
me := testSigner(t, 9)
body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
signed(t, a, "bafyX", "title", "Dune", 2),
signed(t, b, "bafyX", "title", "Dune", 3),
signed(t, me, "bafyX", "title", "Dune (my cut)", 4),
}})
_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
if mr.Fields["title"] != "Dune (my cut)" || mr.Nsrc["title"] != 1 {
t.Fatalf("request overlay = %+v", mr)
}
// …and so do its votes held by this core for the same cids.
storeClaims(t, s.storage, signed(t, me, "bafyX", "year", "2021", 5))
body, _ = json.Marshal(MergeRequest{As: me.UID, Claims: []claim.Claim{
signed(t, a, "bafyX", "title", "Dune", 2),
signed(t, a, "bafyX", "year", "1984", 2),
}})
_ = json.Unmarshal(call(t, s, "POST", "/api/merge", string(body)).Body.Bytes(), &mr)
if mr.Fields["year"] != "2021" || mr.Fields["title"] != "Dune" {
t.Fatalf("local overlay = %+v", mr)
}
// An all-invalid batch is still 200 with everything rejected.
bad2 := signed(t, a, "bafyX", "title", "x", 9)
bad2.Sig = bad2.Sig[:10]
body, _ = json.Marshal(MergeRequest{Claims: []claim.Claim{bad2}})
rr = call(t, s, "POST", "/api/merge", string(body))
if rr.Code != http.StatusOK || !strings.Contains(rr.Body.String(), `"rejected":1`) {
t.Fatalf("all-invalid merge = %d %s", rr.Code, rr.Body)
}
}
func TestTooManyClaimsIs413(t *testing.T) {
s := routedServer(t)
var sb strings.Builder
sb.WriteString("[")
for i := 0; i <= storage.MaxClaimsPerBatch; i++ {
if i > 0 {
sb.WriteString(",")
}
sb.WriteString(`{"cid":"c","uid":"u","key":"k","value":null,"ts":1,"sig":""}`)
}
sb.WriteString("]")
if rr := call(t, s, "POST", "/api/claims", sb.String()); rr.Code != http.StatusRequestEntityTooLarge {
t.Fatalf("oversized batch = %d", rr.Code)
}
}
+132
View File
@@ -0,0 +1,132 @@
package api
import (
"encoding/json"
"net/http"
"os"
"path"
"path/filepath"
"strings"
"github.com/metazla/meta-core/internal/watcher"
)
// registerFileRequest is the body of POST /api/files/register.
type registerFileRequest struct {
// Path relative to FILES_PATH, e.g. "plugin/subtitles/ab12….eng.srt".
Path string `json:"path"`
}
type registerFileResponse struct {
Root string `json:"root"`
CID string `json:"cid"`
Created bool `json:"created"`
Path string `json:"path"`
Size int64 `json:"size"`
}
// pluginRelPath normalises a FILES_PATH-relative path and refuses anything
// outside plugin/. Plugin output is the only tree a service may register this
// way: /files/watch is the user's library and belongs to the watcher.
func pluginRelPath(raw string) (string, bool) {
p := strings.TrimPrefix(filepath.ToSlash(strings.TrimSpace(raw)), "/")
if p == "" {
return "", false
}
for _, seg := range strings.Split(p, "/") {
if seg == ".." {
return "", false
}
}
clean := path.Clean(p)
if !strings.HasPrefix(clean, "plugin/") || clean == "plugin/" {
return "", false
}
return clean, true
}
// handleRegisterFile handles POST /api/files/register.
//
// Does for ONE file what the watcher does for every file it scans: midhash256
// it, mint a root carrying filePath/sizeByte/mtimeNano, register the
// cid:<midhash> alias, and append a file:events entry so meta-share's
// ipfs_seed chunks, seeds and provides it.
//
// It exists for cores that run with ENABLE_FILE_WATCHER=false (a meta-watch
// client box). Without it, bytes written under /files/plugin over WebDAV are
// never resolvable by CID and never reach the swarm. Idempotent: the same
// content at the same path returns the existing root with created=false.
func (s *Server) handleRegisterFile(w http.ResponseWriter, r *http.Request) {
if s.storage == nil || !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req registerFileRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
rel, ok := pluginRelPath(req.Path)
if !ok {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
fullPath := filepath.Join(s.config.FilesPath, filepath.FromSlash(rel))
absFiles, _ := filepath.Abs(filepath.Join(s.config.FilesPath, "plugin"))
absFull, _ := filepath.Abs(fullPath)
if !strings.HasPrefix(absFull, absFiles+string(filepath.Separator)) {
writeError(w, http.StatusBadRequest, "path must be a file under plugin/")
return
}
info, err := os.Stat(fullPath)
if err != nil {
writeError(w, http.StatusNotFound, "file not found")
return
}
if info.IsDir() {
writeError(w, http.StatusBadRequest, "path is a directory, not a file")
return
}
midhash, err := watcher.ComputeMidHash256(fullPath, info.Size())
if err != nil {
writeError(w, http.StatusInternalServerError, "failed to hash file")
return
}
root, created, err := s.storage.RegisterFileTuple(fullPath, info.Size(), info.ModTime().UnixNano(), midhash)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if created {
ev := watcher.FileEvent{
Type: watcher.EventTypeAdd,
Path: rel,
Size: info.Size(),
MidHash256: midhash,
Timestamp: watcher.NowMS(),
}
if err := watcher.PublishEvent(s.storage, ev); err != nil {
// The record is written and resolvable; only seeding is late.
writeError(w, http.StatusInternalServerError, "registered, but file event not published: "+err.Error())
return
}
}
status := http.StatusOK
if created {
status = http.StatusCreated
}
writeJSON(w, status, registerFileResponse{
Root: root,
CID: midhash,
Created: created,
Path: rel,
Size: info.Size(),
})
}
+135
View File
@@ -0,0 +1,135 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"os"
"path/filepath"
"strings"
"testing"
"github.com/metazla/meta-core/internal/config"
)
// newRegisterTestServer wires handleRegisterFile to a temp store and a temp
// FILES_PATH. Built directly, not through NewServer, so no watcher runs —
// which is exactly the configuration this route exists for.
func newRegisterTestServer(t *testing.T) (*Server, string) {
t.Helper()
files := t.TempDir()
return &Server{storage: newTestStorage(t), config: &config.Config{FilesPath: files}}, files
}
func writeFixture(t *testing.T, files, rel, body string) string {
t.Helper()
full := filepath.Join(files, filepath.FromSlash(rel))
if err := os.MkdirAll(filepath.Dir(full), 0o755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(full, []byte(body), 0o644); err != nil {
t.Fatal(err)
}
return full
}
func postRegister(t *testing.T, s *Server, path string) (int, registerFileResponse) {
t.Helper()
body, _ := json.Marshal(registerFileRequest{Path: path})
req := httptest.NewRequest(http.MethodPost, "/api/files/register", strings.NewReader(string(body)))
rr := httptest.NewRecorder()
s.handleRegisterFile(rr, req)
var resp registerFileResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func streamLen(t *testing.T, s *Server) int {
t.Helper()
n, err := s.storage.GetStreamLength("file:events")
if err != nil {
t.Fatalf("stream length: %v", err)
}
return int(n)
}
// The whole point of the route: with the watcher off, a file written under
// plugin/ becomes a record resolvable by its midhash, and meta-share hears
// about it on file:events.
func TestRegisterFile_MintsAliasAndPublishesEvent(t *testing.T) {
s, files := newRegisterTestServer(t)
rel := "plugin/subtitles/ab12cd34.eng.srt"
full := writeFixture(t, files, rel, "1\n00:00:01,000 --> 00:00:02,000\nHello\n")
code, resp := postRegister(t, s, rel)
if code != http.StatusCreated || !resp.Created {
t.Fatalf("want 201 created, got %d %+v", code, resp)
}
if !strings.HasPrefix(resp.CID, "b") || resp.Root == "" {
t.Fatalf("want a multibase cid and a root, got %+v", resp)
}
root, err := s.storage.GetByCID(resp.CID)
if err != nil || root != resp.Root {
t.Fatalf("cid alias must resolve to the root: got %q err=%v, want %q", root, err, resp.Root)
}
if fp, _ := s.storage.GetProperty(resp.Root, "filePath"); fp != full {
t.Fatalf("filePath = %q, want %q", fp, full)
}
entries, err := s.storage.XRevRange("file:events", "+", "-", 0)
if err != nil || len(entries) != 1 {
t.Fatalf("want exactly one file:events entry, got %d err=%v", len(entries), err)
}
if vals := entries[0].Values; vals["midhash256"] != resp.CID || vals["path"] != rel {
t.Fatalf("event must carry the midhash and the relative path, got %v", vals)
}
}
// Re-uploading the same subtitle is a no-op: same root, no second event (which
// would make meta-share re-chunk and re-provide the file).
func TestRegisterFile_IsIdempotent(t *testing.T) {
s, files := newRegisterTestServer(t)
rel := "plugin/subtitles/same.eng.srt"
writeFixture(t, files, rel, "same bytes")
_, first := postRegister(t, s, rel)
code, second := postRegister(t, s, rel)
if code != http.StatusOK || second.Created {
t.Fatalf("second register must be 200 created=false, got %d %+v", code, second)
}
if second.Root != first.Root || second.CID != first.CID {
t.Fatalf("second register must return the same record: %+v vs %+v", second, first)
}
if n := streamLen(t, s); n != 1 {
t.Fatalf("an idempotent re-register must not publish again, stream has %d", n)
}
}
// Only plugin output may be registered here. /files/watch is the user's
// library and belongs to the watcher; traversal must not escape either.
func TestRegisterFile_RejectsPathsOutsidePlugin(t *testing.T) {
s, files := newRegisterTestServer(t)
writeFixture(t, files, "watch/movie.mkv", "x")
for _, p := range []string{
"", "plugin", "plugin/", "watch/movie.mkv", "/watch/movie.mkv",
"plugin/../watch/movie.mkv", "../etc/passwd", "plugin/../../etc/passwd",
} {
if code, _ := postRegister(t, s, p); code != http.StatusBadRequest {
t.Errorf("path %q: want 400, got %d", p, code)
}
}
}
func TestRegisterFile_MissingFileAndDirectory(t *testing.T) {
s, files := newRegisterTestServer(t)
if code, _ := postRegister(t, s, "plugin/subtitles/nope.srt"); code != http.StatusNotFound {
t.Errorf("missing file: want 404, got %d", code)
}
if err := os.MkdirAll(filepath.Join(files, "plugin", "subtitles", "dir"), 0o755); err != nil {
t.Fatal(err)
}
if code, _ := postRegister(t, s, "plugin/subtitles/dir"); code != http.StatusBadRequest {
t.Errorf("directory: want 400, got %d", code)
}
}
+74
View File
@@ -0,0 +1,74 @@
package api
import "net/http"
// fileTupleJSON is one file-backed root on the wire.
type fileTupleJSON struct {
HashID string `json:"hashId"`
FilePath string `json:"filePath"`
SizeByte int64 `json:"sizeByte"`
MtimeNano int64 `json:"mtimeNano"`
}
// FileTuplesResponse is the body of GET /api/files/tuples.
//
// Files is a pointer so the summary form can omit it while the full form
// always carries it — an empty library is `"files": []`, never an absent key,
// which a client could mistake for a meta-core that lacks the rows.
type FileTuplesResponse struct {
Files *[]fileTupleJSON `json:"files,omitempty"`
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
}
// handleGetFileTuples handles GET /api/files/tuples.
//
// Lists the FILE-BACKED roots — records carrying filePath, sizeByte and
// mtimeNano, i.e. what the watcher or /api/files/register minted — with their
// sizes, plus the count and total. Gateway / cid-rooted records carry no file
// and are not included; neither is a locator record that only merged a cache
// file's metadata (it has no mtimeNano).
//
// It exists so a client never walks every hashId to answer "how many files,
// how big": GetTuplesForAllFiles reads the three fields for every root in
// chunked MGETs server-side (~250 round-trips for 82k roots, no SCAN), where
// the client-side equivalent was one HTTP request per record.
//
// ?summary=1 omits the rows and returns only {count, totalSize}.
func (s *Server) handleGetFileTuples(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
tuples, err := s.storage.GetTuplesForAllFiles()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
summary := r.URL.Query().Get("summary")
withRows := summary == "" || summary == "0" || summary == "false"
resp := FileTuplesResponse{Count: len(tuples)}
var rows []fileTupleJSON
if withRows {
rows = make([]fileTupleJSON, 0, len(tuples))
}
for _, t := range tuples {
resp.TotalSize += t.Size
if withRows {
rows = append(rows, fileTupleJSON{
HashID: t.HashID,
FilePath: t.FilePath,
SizeByte: t.Size,
MtimeNano: t.MtimeNano,
})
}
}
if withRows {
resp.Files = &rows
}
writeJSON(w, http.StatusOK, resp)
}
+108
View File
@@ -0,0 +1,108 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sort"
"testing"
"github.com/metazla/meta-core/internal/storage"
)
func newFileTuplesTestServer(t *testing.T) (*Server, *storage.Client) {
t.Helper()
c := newTestStorage(t)
return &Server{storage: c}, c
}
func getFileTuples(t *testing.T, s *Server, query string) (int, FileTuplesResponse) {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/api/files/tuples"+query, nil)
rr := httptest.NewRecorder()
s.handleGetFileTuples(rr, req)
var resp FileTuplesResponse
_ = json.Unmarshal(rr.Body.Bytes(), &resp)
return rr.Code, resp
}
func mustMint(t *testing.T, c *storage.Client, path string, size, mtime int64) string {
t.Helper()
id, _, err := c.RegisterFileTuple(path, size, mtime, "bafkmid"+path)
if err != nil {
t.Fatalf("register %s: %v", path, err)
}
return id
}
// Only local files count: a gateway record — even one whose claims carry a
// filePath and size — is left out.
func TestFileTuples_OnlyFileBackedRoots(t *testing.T) {
s, c := newFileTuplesTestServer(t)
a := mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
b := mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
gw := testSigner(t, 1)
storeClaims(t, c,
signed(t, gw, "bagcsgatewayrecord", "title", "Some Release", 1),
signed(t, gw, "bafkpartial", "filePath", "/files/cache/x.mkv", 1),
signed(t, gw, "bafkpartial", "sizeByte", "999", 1),
)
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
if resp.Files == nil || len(*resp.Files) != 2 {
t.Fatalf("want 2 rows, got %+v", resp.Files)
}
got := []string{(*resp.Files)[0].HashID, (*resp.Files)[1].HashID}
want := []string{a, b}
sort.Strings(got)
sort.Strings(want)
if got[0] != want[0] || got[1] != want[1] {
t.Fatalf("want roots %v, got %v", want, got)
}
}
func TestFileTuples_SummaryOmitsRows(t *testing.T) {
s, c := newFileTuplesTestServer(t)
mustMint(t, c, "/files/watch/a.mkv", 1000, 1)
mustMint(t, c, "/files/watch/b.mkv", 2500, 2)
code, resp := getFileTuples(t, s, "?summary=1")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files != nil {
t.Fatalf("summary must omit rows, got %d", len(*resp.Files))
}
if resp.Count != 2 || resp.TotalSize != 3500 {
t.Fatalf("want count 2 / totalSize 3500, got %d / %d", resp.Count, resp.TotalSize)
}
}
// An empty library still carries "files": [] — clients treat an absent key as
// "this meta-core cannot list files", so it must not disappear when empty.
func TestFileTuples_EmptyLibraryKeepsFilesKey(t *testing.T) {
s, _ := newFileTuplesTestServer(t)
code, resp := getFileTuples(t, s, "")
if code != http.StatusOK {
t.Fatalf("want 200, got %d", code)
}
if resp.Files == nil || len(*resp.Files) != 0 || resp.Count != 0 {
t.Fatalf("want an empty files array and count 0, got %+v", resp)
}
}
func TestFileTuples_StorageNotConnected(t *testing.T) {
s := &Server{storage: storage.NewClient()}
code, _ := getFileTuples(t, s, "")
if code != http.StatusServiceUnavailable {
t.Fatalf("want 503, got %d", code)
}
}
+155 -296
View File
@@ -13,6 +13,7 @@ import (
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
)
// contentTypeByExt maps file extensions to MIME types
@@ -63,10 +64,49 @@ type DataPathResponse struct {
Exists bool `json:"exists"`
}
// ErrorResponse is the response for errors
// ErrorResponse is the typed error envelope returned on every 4xx/5xx.
//
// Shape and slug vocabulary documented in docs/api-mediated-access.md
// ("Error envelope"). Stable slug strings let clients switch on the failure
// category without parsing the message; `retryable` tells callers whether
// a backoff-and-retry has any chance of succeeding.
type ErrorResponse struct {
Error string `json:"error"`
Message string `json:"message,omitempty"`
Retryable bool `json:"retryable"`
}
// Stable error-slug strings. Pinned in the doc; consumers may match on
// these. Do not rename without a deprecation cycle.
const (
ErrAliasCollision = "alias_collision"
ErrUnknownRoot = "unknown_root"
ErrUnknownCID = "unknown_cid"
ErrSchemaViolation = "schema_violation"
ErrStorageUnavailable = "storage_unavailable"
ErrInternal = "internal"
ErrDeprecatedField = "deprecated_field"
)
// defaultSlugFor maps an HTTP status code to the most likely error slug.
// Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
// instead of "unknown_root" on 404) call writeErrorSlug directly.
func defaultSlugFor(status int) (slug string, retryable bool) {
switch status {
case http.StatusConflict:
return ErrAliasCollision, false
case http.StatusNotFound:
return ErrUnknownRoot, false
case http.StatusBadRequest:
return ErrSchemaViolation, false
case http.StatusServiceUnavailable:
return ErrStorageUnavailable, true
case http.StatusInternalServerError:
return ErrInternal, true
default:
// 4xx → not retryable, 5xx → retryable. Conservative default.
return ErrInternal, status >= 500
}
}
var startTime = time.Now()
@@ -169,7 +209,11 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
return
}
metadata, err := s.storage.GetMetadataFlat(hashID)
// Resolve CID → UUID via the reverse index so reads find data wherever
// the watcher minted the root, regardless of which CID the caller knows.
hashID = s.storage.ResolveRoot(hashID)
metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
@@ -188,66 +232,6 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, response)
}
// handlePutMeta handles PUT /meta/{hash}
func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
})
}
// handleDeleteMeta handles DELETE /meta/{hash}
func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
deleted, err := s.storage.DeleteMetadata(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
"deleted": deleted,
})
}
// handleListMeta handles GET /meta
func (s *Server) handleListMeta(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
@@ -340,6 +324,63 @@ func (s *Server) handleHeadData(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}
// handleGetFileByCIDInfo handles GET /api/file/{cid}/info.
// Returns lightweight metadata for the file (existence, content type, size,
// relative path) without streaming any bytes. Used by the editor to decide
// whether to render a preview (image/video) or a download-only button.
func (s *Server) handleGetFileByCIDInfo(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
cid := vars["cid"]
type infoResp struct {
Exists bool `json:"exists"`
ContentType string `json:"contentType,omitempty"`
Size int64 `json:"size,omitempty"`
FilePath string `json:"filePath,omitempty"`
}
w.Header().Set("Content-Type", "application/json")
if cid == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
relPath, err := s.storage.LookupPathByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if relPath == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
fullPath := filepath.Join(s.config.FilesPath, relPath)
fi, err := os.Stat(fullPath)
if err != nil {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false, FilePath: relPath})
return
}
ext := strings.ToLower(filepath.Ext(fullPath))
contentType, ok := contentTypeByExt[ext]
if !ok {
contentType = "application/octet-stream"
}
_ = json.NewEncoder(w).Encode(infoResp{
Exists: true,
ContentType: contentType,
Size: fi.Size(),
FilePath: relPath,
})
}
// handleGetFileByCID handles GET /file/{cid}
// Serves a file by looking up its CID in poster/backdrop metadata
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
@@ -364,7 +405,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
}
if relPath == "" {
writeError(w, http.StatusNotFound, "file not found for CID")
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file not found for CID", false)
return
}
@@ -374,7 +415,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
// Check if file exists
fileInfo, err := os.Stat(fullPath)
if err != nil {
writeError(w, http.StatusNotFound, "file does not exist on disk")
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file does not exist on disk", false)
return
}
@@ -400,62 +441,49 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
http.ServeContent(w, r, fileInfo.Name(), fileInfo.ModTime(), file)
}
// handleListServices handles GET /services and GET /api/services
// Supports optional ?current=<service-name> query parameter
func (s *Server) handleListServices(w http.ResponseWriter, r *http.Request) {
services, err := s.discovery.DiscoverAll()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
// handleListNeighbors handles GET /neighbors and GET /api/neighbors.
//
// Beacon v2 (docs/project-architecture/beacon-v2.md). The list is this node's
// in-memory view of the UDP group, so it works across stacks that share a
// network but no volume. Instances of one name collapse to the most recent
// (one row per service, the nav menu's shape); `?all=1` returns every
// instance, `?cap=<pattern>` keeps only nodes with a matching capability.
func (s *Server) handleListNeighbors(w http.ResponseWriter, r *http.Request) {
response := map[string]interface{}{
"services": services,
"count": len(services),
}
// Add 'current' field if query parameter is provided
current := r.URL.Query().Get("current")
if current != "" {
response["current"] = current
"current": s.config.ServiceName,
"neighbors": []interface{}{},
"count": 0,
}
if s.mesh == nil {
// Discovery off or failed to bind. Report empty rather than 503 —
// the nav is decoration and callers already treat [] as "hide".
response["enabled"] = false
writeJSON(w, http.StatusOK, response)
}
// handleGetService handles GET /services/{name}
func (s *Server) handleGetService(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
name := vars["name"]
if name == "" {
writeError(w, http.StatusBadRequest, "service name is required")
return
}
service, err := s.discovery.Discover(name)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
var list []meshdisco.Neighbor
if r.URL.Query().Get("all") != "" {
list = s.mesh.Neighbors()
} else {
list = s.mesh.NeighborsByName()
}
if pattern := r.URL.Query().Get("cap"); pattern != "" {
kept := list[:0]
for _, nb := range list {
if nb.Matches(pattern) {
kept = append(kept, nb)
}
}
list = kept
}
if service == nil {
writeError(w, http.StatusNotFound, "service not found")
return
}
writeJSON(w, http.StatusOK, service)
}
// handleCleanupStats handles GET /services/cleanup/stats
func (s *Server) handleCleanupStats(w http.ResponseWriter, r *http.Request) {
if s.cleaner == nil {
writeError(w, http.StatusServiceUnavailable, "cleaner not initialized")
return
}
stats := s.cleaner.Stats()
writeJSON(w, http.StatusOK, stats)
response["enabled"] = true
response["neighbors"] = list
response["count"] = len(list)
response["self"] = s.mesh.Self()
writeJSON(w, http.StatusOK, response)
}
// writeJSON writes a JSON response
@@ -465,46 +493,26 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
json.NewEncoder(w).Encode(data)
}
// writeError writes an error response
// writeError writes the typed error envelope. The slug + retryable bit
// are inferred from the status (see defaultSlugFor); call writeErrorSlug
// directly when a more specific slug applies.
func writeError(w http.ResponseWriter, status int, message string) {
slug, retryable := defaultSlugFor(status)
writeJSON(w, status, ErrorResponse{
Error: http.StatusText(status),
Error: slug,
Message: message,
Retryable: retryable,
})
}
// handlePatchMeta handles PATCH /meta/{hash}
// Merges the provided metadata into existing (does not delete missing keys)
func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
updated, err := s.storage.MergeMetadataFlat(hashID, metadata)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
"updated": updated,
// writeErrorSlug writes the typed envelope with an explicit slug and
// retryable bit. Use this for endpoint-specific slugs (alias_collision,
// unknown_cid) where the default mapping from HTTP status would be wrong.
func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, retryable bool) {
writeJSON(w, status, ErrorResponse{
Error: slug,
Message: message,
Retryable: retryable,
})
}
@@ -530,6 +538,8 @@ func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
return
}
hashID = s.storage.ResolveRoot(hashID)
value, err := s.storage.GetProperty(hashID, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
@@ -552,157 +562,6 @@ type PropertyBody struct {
Value string `json:"value"`
}
// handlePutProperty handles PUT /meta/{hash}/{key}
// Sets a single property value
// Accepts JSON body with {"value": "..."} or plain text
func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
key := vars["key"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Read body
buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf)
if err != nil && err.Error() != "EOF" {
writeError(w, http.StatusBadRequest, "failed to read body")
return
}
body := buf[:n]
// Try to parse as JSON {"value": "..."} first
var propBody PropertyBody
var value string
if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
// JSON body with value field
value = propBody.Value
} else {
// Plain text body
value = string(body)
}
if err := s.storage.SetProperty(hashID, key, value); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
"property": key,
})
}
// handleDeleteProperty handles DELETE /meta/{hash}/{key}
// Deletes a single property
func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
key := vars["key"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
if err := s.storage.DeleteProperty(hashID, key); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
"property": key,
})
}
// handleAddToSet handles POST /meta/{hash}/_add/{key}
// Adds a value to a set-type field
// Accepts JSON body with {"value": "..."} or plain text
func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hash"]
key := vars["key"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash is required")
return
}
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Read body
buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf)
if err != nil && err.Error() != "EOF" {
writeError(w, http.StatusBadRequest, "failed to read body")
return
}
body := buf[:n]
// Try to parse as JSON {"value": "..."} first
var propBody PropertyBody
var value string
if err := json.Unmarshal(body, &propBody); err == nil && propBody.Value != "" {
// JSON body with value field
value = propBody.Value
} else {
// Plain text body
value = string(body)
}
if value == "" {
writeError(w, http.StatusBadRequest, "value is required")
return
}
added, err := s.storage.AddToSet(hashID, key, value)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"hashId": hashID,
"property": key,
"added": added,
})
}
// CIDRequest is the request body for POST /file/cid
type CIDRequest struct {
Path string `json:"path"`
+630
View File
@@ -0,0 +1,630 @@
package api
import (
"encoding/base64"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/claim"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
// identityStatusResponse is the shape of GET /api/identity. The private key
// is intentionally absent — it is only ever returned once, by
// POST /api/identity/generate, immediately after creation.
type identityStatusResponse struct {
HasIdentity bool `json:"hasIdentity"`
UID string `json:"uid,omitempty"`
Curve string `json:"curve,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
// AccountCount is the total number of accounts in the keystore. Lets
// multi-account clients (meta-watch) tell "single identity" from "pick a
// profile" without a second call. Older single-identity clients ignore it.
AccountCount int `json:"accountCount,omitempty"`
}
// identityAccountInfo is one entry in GET /api/identity/accounts. Never carries
// the private key — that is only ever returned once, by generate.
type identityAccountInfo struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
}
type identityAccountsResponse struct {
Accounts []identityAccountInfo `json:"accounts"`
}
type identityGenerateResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportRequest struct {
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
// Created distinguishes "this key was new here" (201) from "this key was
// already in the keystore" (200). Import is the sign-in primitive for
// clients: presenting the secret key is the proof of ownership, so a
// re-import must succeed rather than conflict. Clients use the flag only
// to word the confirmation ("new profile" vs "welcome back").
Created bool `json:"created"`
}
// identityChallengeRequest asks for a one-shot challenge to sign.
type identityChallengeRequest struct {
UID string `json:"uid"`
Action string `json:"action"`
}
type identityChallengeResponse struct {
Challenge string `json:"challenge"`
ExpiresAt int64 `json:"expiresAt"`
}
// identityProof is the proof-of-possession carried by reveal and delete: the
// challenge text as issued, and a signature over it by the account's key.
//
// The signature is produced by whoever holds the key — the browser, locally.
// It is NOT obtainable from POST /api/identity/sign, which refuses any payload
// in the challenge domain; that refusal is what makes this proof mean
// something on a node that stores every private key in plaintext.
type identityProof struct {
Challenge string `json:"challenge"`
Signature string `json:"signature"`
}
// identityRevealRequest asks for an existing account's private key back.
//
// Gated on proof-of-possession, which makes it a "does this node hold the same
// key I do" check rather than the recovery path it once was. That is the
// deliberate consequence of the rule this file enforces: the private key, and
// nothing else, is authority over the account. An operator with root on the box
// can still read the file; an operator with only the API cannot.
type identityRevealRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
type identityRevealResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityDeleteRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
identityProof
}
// identityDeleteResponse reports what the delete actually removed. The purge
// counts are the point: "account removed" alone leaves the caller unable to
// tell a clean removal from one that silently stranded a decade of ratings.
type identityDeleteResponse struct {
UID string `json:"uid"`
Purged storage.UDLUserStats `json:"purged"`
}
type identitySignRequest struct {
BytesB64 string `json:"bytesB64"`
// Format is "der" or "compact" (64-byte r||s, low-S). Default: compact for
// MMUR2 claim payloads ("MMUR2\n" prefix), DER otherwise (MMUR1 back-compat).
Format string `json:"format,omitempty"`
// UID selects which account signs. Optional — when omitted and exactly one
// account exists, that account is used (single-identity back-compat).
UID string `json:"uid,omitempty"`
}
type identitySignResponse struct {
SigB64 string `json:"sigB64"`
}
// identitySignBatchRequest signs many payloads under one account in one call.
//
// Exists because a User Data Layer write signs exactly one record, and the bulk
// verbs above it ("mark this whole series seen") produce one record per episode:
// a 300-episode show is 300 sequential sign round trips through
// meta-watch -> meta-search -> here, which dominates the operation. The private
// key still never leaves this process; only the transport is batched.
type identitySignBatchRequest struct {
// PayloadsB64 are the canonical byte strings to sign, in order. The
// response's signatures line up index-for-index.
PayloadsB64 []string `json:"payloadsB64"`
// UID selects which account signs — same optional single-identity
// back-compat as identitySignRequest.
UID string `json:"uid,omitempty"`
// Format: see identitySignRequest.Format; applies to every payload.
Format string `json:"format,omitempty"`
}
type identitySignBatchResponse struct {
SigsB64 []string `json:"sigsB64"`
}
// maxSignBatch bounds one batch. A signature is cheap but not free, and the
// request holds the account lock for its duration; a caller with more than this
// should page. Sized well above the longest real series (Supernatural, 327).
const maxSignBatch = 1000
// reservedPayloadMessage explains the one refusal the signing endpoints make.
//
// This is the load-bearing half of proof-of-possession on a node that holds
// every private key: without it, a caller wanting to delete someone else's
// account just asks meta-core to sign the challenge for them, and the gate on
// reveal/delete becomes a formality. See identity.ChallengeDomain.
const reservedPayloadMessage = "payload is in the reserved account-authorisation domain (" +
identity.ChallengeDomain + "…); meta-core will not sign one. " +
"Signing a challenge is what proves you hold the key, so it must be done where the key is."
type identityAEADKeyResponse struct {
AEADKeyB64 string `json:"aeadKeyB64"`
}
// resolveAccount picks the account an operation targets. A non-empty uid must
// match an existing account. An empty uid falls back to the sole account
// (single-identity back-compat); with zero or many accounts and no uid it
// returns a non-zero status for the caller to surface. status == 0 means ok.
func (s *Server) resolveAccount(uid string) (id *identity.Identity, status int, slug, msg string) {
dir := s.config.IdentityAccountsDir()
uid = strings.TrimSpace(uid)
if uid != "" {
got, err := identity.LoadByUID(dir, uid)
if err != nil {
return nil, http.StatusBadRequest, "bad_uid", err.Error()
}
if got == nil {
return nil, http.StatusNotFound, "no_identity", "no identity for the given uid"
}
return got, 0, "", ""
}
list, err := identity.List(dir)
if err != nil {
return nil, http.StatusInternalServerError, "list_failed", err.Error()
}
switch len(list) {
case 0:
return nil, http.StatusNotFound, "no_identity", "no identity is configured"
case 1:
return list[0], 0, "", ""
default:
return nil, http.StatusBadRequest, "uid_required", "multiple accounts exist; specify uid"
}
}
func (s *Server) handleIdentityGet(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if len(list) == 0 {
writeJSON(w, http.StatusOK, identityStatusResponse{HasIdentity: false})
return
}
// Back-compat: report the oldest account as "the" identity. Multi-account
// clients enumerate via GET /api/identity/accounts instead.
def := list[0]
writeJSON(w, http.StatusOK, identityStatusResponse{
HasIdentity: true,
UID: def.UID,
Curve: def.Curve,
CreatedAt: def.CreatedAt,
AccountCount: len(list),
})
}
func (s *Server) handleIdentityAccounts(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
out := make([]identityAccountInfo, 0, len(list))
for _, id := range list {
out = append(out, identityAccountInfo{UID: id.UID, Curve: id.Curve, CreatedAt: id.CreatedAt})
}
writeJSON(w, http.StatusOK, identityAccountsResponse{Accounts: out})
}
func (s *Server) handleIdentityGenerate(w http.ResponseWriter, r *http.Request) {
id, err := identity.Generate()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if err := identity.SaveAccount(s.config.IdentityAccountsDir(), id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityGenerateResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
PrivateKeyHex: id.PrivateKeyHex,
})
}
func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
var body identityImportRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if strings.TrimSpace(body.PrivateKeyHex) == "" {
writeError(w, http.StatusBadRequest, "privateKeyHex is required")
return
}
id, err := identity.Import(strings.TrimSpace(body.PrivateKeyHex))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
dir := s.config.IdentityAccountsDir()
// Idempotent: importing a key that is already in the keystore is how a
// second device signs in to an existing account. Answer 200 with the
// stored account (preserving its original createdAt) instead of 409 —
// the caller proved ownership by presenting the key.
if existing, err := identity.LoadByUID(dir, id.UID); err == nil && existing != nil {
writeJSON(w, http.StatusOK, identityImportResponse{
UID: existing.UID,
Curve: existing.Curve,
CreatedAt: existing.CreatedAt,
Created: false,
})
return
}
if err := identity.SaveAccount(dir, id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityImportResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
Created: true,
})
}
// handleIdentityChallenge issues the one-shot text a client signs to prove it
// holds an account's private key.
//
// Unauthenticated by design: a challenge is useless without the key, and
// GET /api/identity/accounts already publishes every uid, so refusing to mint
// one for an unknown caller would protect nothing. Requiring the account to
// exist is for a clear error, not secrecy.
func (s *Server) handleIdentityChallenge(w http.ResponseWriter, r *http.Request) {
var body identityChallengeRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := strings.TrimSpace(body.UID)
if uid == "" {
writeErrorSlug(w, http.StatusBadRequest, "uid_required", "uid is required", false)
return
}
action := identity.Action(strings.TrimSpace(body.Action))
if !identity.ValidAction(action) {
writeErrorSlug(w, http.StatusBadRequest, "bad_action",
`action must be "reveal" or "delete"`, false)
return
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
writeErrorSlug(w, http.StatusNotFound, "no_identity", "no identity for the given uid", false)
return
}
text, expires, err := s.challenges.Issue(uid, action)
if err != nil {
if errors.Is(err, identity.ErrChallengeFull) {
writeErrorSlug(w, http.StatusServiceUnavailable, "challenges_full", err.Error(), true)
return
}
writeErrorSlug(w, http.StatusBadRequest, "bad_uid", err.Error(), false)
return
}
writeJSON(w, http.StatusOK, identityChallengeResponse{
Challenge: text,
ExpiresAt: expires.Unix(),
})
}
// requireProof resolves the target account and verifies the caller's
// proof-of-possession for `action`. status == 0 means the proof held.
//
// The uid is mandatory here, unlike elsewhere in this file: the old "sole
// account" fallback would have this pick a target the signature does not name,
// and a proof that authorises an unnamed account authorises the wrong one as
// soon as a second profile exists.
func (s *Server) requireProof(uid string, proof identityProof, action identity.Action) (
id *identity.Identity, status int, slug, msg string,
) {
uid = strings.TrimSpace(uid)
if uid == "" {
return nil, http.StatusBadRequest, "uid_required",
"uid is required: the signature authorises one specific account"
}
if proof.Challenge == "" || proof.Signature == "" {
return nil, http.StatusUnauthorized, "signature_required",
"this operation requires proof you hold the account's private key: " +
"POST /api/identity/challenge, sign the returned text, and resend it as {challenge, signature}"
}
sig, err := base64.StdEncoding.DecodeString(proof.Signature)
if err != nil {
return nil, http.StatusBadRequest, "bad_signature", "signature is not valid base64"
}
// Resolve before redeeming so a request naming a missing account does not
// burn a challenge it could not have used anyway.
id, status, slug, msg = s.resolveAccount(uid)
if status != 0 {
return nil, status, slug, msg
}
// Redeem first, verify second: the challenge is single-use either way, so a
// wrong signature costs the attacker a fresh round trip per guess rather
// than letting them grind one challenge offline against the same nonce.
if err := s.challenges.Redeem(proof.Challenge, uid, action); err != nil {
return nil, http.StatusUnauthorized, "bad_challenge", err.Error()
}
if err := identity.Verify(uid, []byte(proof.Challenge), sig); err != nil {
return nil, http.StatusUnauthorized, "bad_signature", err.Error()
}
return id, 0, "", ""
}
// handleIdentityReveal returns an existing account's private key to a caller
// that proves it already holds that key.
//
// Deliberately POST (never a bare GET) so it cannot be triggered by a link or
// an image tag. The proof is what actually guards it: before this gate existed,
// anything that could reach the port could read every private key on the node.
func (s *Server) handleIdentityReveal(w http.ResponseWriter, r *http.Request) {
var body identityRevealRequest
_ = json.NewDecoder(r.Body).Decode(&body)
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionReveal)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
writeJSON(w, http.StatusOK, identityRevealResponse{
UID: id.UID,
Curve: id.Curve,
PrivateKeyHex: id.PrivateKeyHex,
})
}
// handleIdentityDelete removes one account and everything it wrote.
//
// Two things happen, in this order and no other:
//
// 1. the account's User Data Layer rows are purged (cells plus every index
// that names the uid, including the cid-scoped ones shared with other
// accounts);
// 2. the keypair file is removed.
//
// Purge-then-delete so a failure never strands data: if the purge errors the
// key is still on disk, the account still exists, and the caller can retry.
// The reverse order would leave rows nobody can ever attribute, reach or
// rewrite — the orphaning this endpoint exists to prevent.
//
// What deletion still does NOT do, and cannot: records already signed under
// this uid and replicated to other peers keep verifying there. Removing a key
// is not a recall.
func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
var body identityDeleteRequest
// Tolerate an empty body so the uid may ride in the query — meta-watch's
// proxy puts it there. The proof itself must be in the body: a signature in
// a URL ends up in every access log between here and the browser.
_ = json.NewDecoder(r.Body).Decode(&body)
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
id, status, slug, msg := s.requireProof(uid, body.identityProof, identity.ActionDelete)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
// A purge needs storage. Refusing here — rather than deleting the key and
// reporting the purge as "0 records" — is the difference between a retry
// and permanent orphaned data: once the key is gone the account cannot be
// named for a second attempt through this API.
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage is not connected; refusing to delete the key while its data cannot be purged", true)
return
}
purged, err := s.retractAllClaims(r, id)
if err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "purge_failed",
"could not retract this account's claims; the account was NOT deleted: "+err.Error(), true)
return
}
if err := identity.DeleteByUID(s.config.IdentityAccountsDir(), id.UID); err != nil {
writeErrorSlug(w, http.StatusInternalServerError, "delete_failed", err.Error(), false)
return
}
log.Printf("[API] Deleted account %s after retracting %d claims across %d cids",
id.UID, purged.Records, purged.Cids)
writeJSON(w, http.StatusOK, identityDeleteResponse{UID: id.UID, Purged: purged})
}
func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
var body identitySignRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := body.UID
if strings.TrimSpace(uid) == "" {
uid = r.URL.Query().Get("uid")
}
id, status, slug, msg := s.resolveAccount(uid)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
raw, err := base64.StdEncoding.DecodeString(body.BytesB64)
if err != nil {
writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload", reservedPayloadMessage, false)
return
}
sig, err := signPayload(id, raw, body.Format)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, identitySignResponse{
SigB64: base64.StdEncoding.EncodeToString(sig),
})
}
// handleIdentitySignBatch signs every payload in the request under one account.
//
// All-or-nothing: a single bad base64 or signing failure fails the whole
// request rather than returning a short array. The caller pairs signatures with
// records by index, so a partial response would silently misalign them — and a
// misaligned signature verifies against the wrong record, which is worse than
// an error.
func (s *Server) handleIdentitySignBatch(w http.ResponseWriter, r *http.Request) {
var body identitySignBatchRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := body.UID
if strings.TrimSpace(uid) == "" {
uid = r.URL.Query().Get("uid")
}
if len(body.PayloadsB64) == 0 {
writeError(w, http.StatusBadRequest, "payloadsB64 is required")
return
}
if len(body.PayloadsB64) > maxSignBatch {
writeError(w, http.StatusBadRequest, "too many payloads in one batch")
return
}
id, status, slug, msg := s.resolveAccount(uid)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
sigs := make([]string, 0, len(body.PayloadsB64))
for i, p := range body.PayloadsB64 {
raw, err := base64.StdEncoding.DecodeString(p)
if err != nil {
writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"] is not valid base64")
return
}
if identity.IsChallengePayload(raw) {
writeErrorSlug(w, http.StatusForbidden, "reserved_payload",
"payloadsB64["+strconv.Itoa(i)+"]: "+reservedPayloadMessage, false)
return
}
sig, err := signPayload(id, raw, body.Format)
if err != nil {
writeError(w, http.StatusBadRequest, "payloadsB64["+strconv.Itoa(i)+"]: "+err.Error())
return
}
sigs = append(sigs, base64.StdEncoding.EncodeToString(sig))
}
writeJSON(w, http.StatusOK, identitySignBatchResponse{SigsB64: sigs})
}
func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
key, err := identity.DeriveAEADKey(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, identityAEADKeyResponse{
AEADKeyB64: base64.StdEncoding.EncodeToString(key),
})
}
// signPayload signs raw with the account key in the requested encoding.
func signPayload(id *identity.Identity, raw []byte, format string) ([]byte, error) {
switch format {
case "compact":
return identity.SignCompact(id, raw)
case "der":
return identity.Sign(id, raw)
case "":
if strings.HasPrefix(string(raw), claim.Domain) {
return identity.SignCompact(id, raw)
}
return identity.Sign(id, raw)
}
return nil, fmt.Errorf("format must be \"compact\" or \"der\"")
}
// retractAllClaims signs a retraction for every live claim the account holds
// and stores them (G11): the tombstones stay, so the votes cannot be replayed.
func (s *Server) retractAllClaims(r *http.Request, id *identity.Identity) (storage.UDLUserStats, error) {
refs, err := s.storage.LiveClaimsBySigner(id.UID)
if err != nil {
return storage.UDLUserStats{}, err
}
cids, keys := map[string]bool{}, map[string]bool{}
now := uint64(time.Now().UnixMilli())
batch := make([]claim.Claim, 0, len(refs))
for _, ref := range refs {
cur, ok, err := s.storage.UDLGetClaim(id.UID, ref[0], ref[1])
if err != nil {
return storage.UDLUserStats{}, err
}
ts := now
if ok && cur.TS >= ts {
ts = cur.TS + 1
}
tomb := claim.Claim{CID: ref[0], UID: id.UID, Key: ref[1], Value: claim.Null(), TS: ts}
if err := tomb.Sign(id); err != nil {
return storage.UDLUserStats{}, err
}
batch = append(batch, tomb)
cids[ref[0]], keys[ref[1]] = true, true
}
for start := 0; start < len(batch); start += storage.MaxClaimsPerBatch {
end := min(start+storage.MaxClaimsPerBatch, len(batch))
results, err := s.storage.WriteClaims(r.Context(), batch[start:end])
if err != nil {
return storage.UDLUserStats{}, err
}
for i, res := range results {
if res.Status != storage.StatusAccepted && res.Status != storage.StatusUnchanged {
return storage.UDLUserStats{}, fmt.Errorf("retraction of %s/%s refused: %s %s",
batch[start+i].CID, batch[start+i].Key, res.Status, res.Reason)
}
}
}
return storage.UDLUserStats{Records: len(batch), Cids: len(cids), Keys: len(keys)}, nil
}
+254
View File
@@ -0,0 +1,254 @@
package api
// End-to-end exercise of the account gate against a live router + a temp store:
// generate -> write UDL rows -> stats -> challenge -> sign -> delete -> verify
// both the key and every row are gone. Deliberately drives the real HTTP
// handlers rather than the packages beneath them, because the bugs that matter
// here (a challenge accepted for the wrong action, a purge that runs after the
// key is gone) live in the wiring.
import (
"bytes"
"encoding/base64"
"encoding/hex"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/identity"
)
func newTestServer(t *testing.T) *Server {
t.Helper()
stor := newTestStorage(t)
s := &Server{
config: &config.Config{MetaCorePath: t.TempDir()},
storage: stor,
challenges: identity.NewChallengeStore(),
router: mux.NewRouter(),
}
s.setupIdentityTestRoutes()
return s
}
// Only the routes under test, so this does not depend on every other subsystem
// booting.
func (s *Server) setupIdentityTestRoutes() {
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
}
func do(t *testing.T, s *Server, method, path string, body any) (*httptest.ResponseRecorder, map[string]any) {
t.Helper()
var rdr *bytes.Reader
if body != nil {
b, _ := json.Marshal(body)
rdr = bytes.NewReader(b)
} else {
rdr = bytes.NewReader(nil)
}
req := httptest.NewRequest(method, path, rdr)
req.Header.Set("Content-Type", "application/json")
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, req)
out := map[string]any{}
_ = json.Unmarshal(rec.Body.Bytes(), &out)
return rec, out
}
// signAs produces the proof a browser would, from the raw key.
func signAs(t *testing.T, privHex, challenge string) map[string]any {
t.Helper()
raw, err := hex.DecodeString(privHex)
if err != nil {
t.Fatalf("decode key: %v", err)
}
id, err := identity.Import(hex.EncodeToString(raw))
if err != nil {
t.Fatalf("import: %v", err)
}
sig, err := identity.Sign(id, []byte(challenge))
if err != nil {
t.Fatalf("sign: %v", err)
}
return map[string]any{
"challenge": challenge,
"signature": base64.StdEncoding.EncodeToString(sig),
}
}
func TestE2E_DeleteRequiresProofAndPurgesEverything(t *testing.T) {
s := newTestServer(t)
rec, gen := do(t, s, "POST", "/api/identity/generate", nil)
if rec.Code != http.StatusCreated {
t.Fatalf("generate: %d %s", rec.Code, rec.Body)
}
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
// Some data to lose: claims signed by the account.
acct, err := identity.Import(priv)
if err != nil {
t.Fatal(err)
}
storeClaims(t, s.storage,
signed(t, acct, "cidA", "like", "true", 100),
signed(t, acct, "cidA", "userRating", "5", 100),
signed(t, acct, "cidB", "seek", []byte("enc"), 100),
)
// The count the UI shows in the confirmation.
rec, stats := do(t, s, "GET", "/api/udl/users/stats", nil)
if rec.Code != http.StatusOK {
t.Fatalf("stats: %d %s", rec.Code, rec.Body)
}
users := stats["users"].(map[string]any)
mine := users[uid].(map[string]any)
if mine["records"].(float64) != 3 || mine["cids"].(float64) != 2 {
t.Fatalf("stats = %v, want 3 records / 2 cids", mine)
}
// Unsigned delete must be refused, and must not have touched anything.
rec, body := do(t, s, "DELETE", "/api/identity", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned delete = %d %v, want 401 signature_required", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("a refused delete removed the key anyway")
}
// meta-core must refuse to sign the caller's authorisation for them —
// without this the gate is decoration on a node holding every key.
ch, _, err := s.challenges.Issue(uid, identity.ActionDelete)
if err != nil {
t.Fatalf("issue: %v", err)
}
rec, body = do(t, s, "POST", "/api/identity/sign", map[string]any{
"uid": uid,
"bytesB64": base64.StdEncoding.EncodeToString([]byte(ch)),
})
if rec.Code != http.StatusForbidden || body["error"] != "reserved_payload" {
t.Fatalf("sign of a challenge = %d %v, want 403 reserved_payload", rec.Code, body)
}
// A challenge minted for reveal must not authorise a delete.
rec, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized {
t.Fatalf("reveal challenge used for delete = %d %v, want 401", rec.Code, body)
}
// The real thing.
rec, chBody = do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
if rec.Code != http.StatusOK {
t.Fatalf("challenge: %d %s", rec.Code, rec.Body)
}
proof = signAs(t, priv, chBody["challenge"].(string))
req = map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed delete = %d %s", rec.Code, rec.Body)
}
purged := body["purged"].(map[string]any)
if purged["records"].(float64) != 3 || purged["cids"].(float64) != 2 {
t.Fatalf("purged = %v, want 3 records / 2 cids", purged)
}
// Key gone.
if identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("the account file survived the delete")
}
// Every claim retracted (tombstones kept) — nothing left to orphan.
after, err := s.storage.UDLAllUserStats()
if err != nil {
t.Fatalf("stats after: %v", err)
}
if got, present := after[uid]; present {
t.Fatalf("deleted account still holds %+v", got)
}
for _, cid := range []string{"cidA", "cidB"} {
uids, err := s.storage.UDLCidUsers(cid)
if err != nil {
t.Fatalf("cid users: %v", err)
}
if len(uids) != 0 {
t.Fatalf("%s still names %v after the purge", cid, uids)
}
}
// The challenge is single-use, so replaying the same proof does nothing.
rec, _ = do(t, s, "DELETE", "/api/identity", req)
if rec.Code == http.StatusOK {
t.Fatal("a replayed proof must not succeed")
}
}
func TestE2E_RevealRequiresProof(t *testing.T) {
s := newTestServer(t)
_, gen := do(t, s, "POST", "/api/identity/generate", nil)
uid := gen["uid"].(string)
priv := gen["privateKeyHex"].(string)
rec, body := do(t, s, "POST", "/api/identity/reveal", map[string]any{"uid": uid, "confirm": true})
if rec.Code != http.StatusUnauthorized || body["error"] != "signature_required" {
t.Fatalf("unsigned reveal = %d %v, want 401 signature_required", rec.Code, body)
}
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "reveal"})
proof := signAs(t, priv, chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body = do(t, s, "POST", "/api/identity/reveal", req)
if rec.Code != http.StatusOK {
t.Fatalf("signed reveal = %d %s", rec.Code, rec.Body)
}
if body["privateKeyHex"] != priv {
t.Fatal("reveal returned a different key than generate did")
}
}
// Another account's key must not authorise this one, even with a challenge
// minted for the right uid and action.
func TestE2E_AnotherAccountsKeyIsRefused(t *testing.T) {
s := newTestServer(t)
_, alice := do(t, s, "POST", "/api/identity/generate", nil)
_, mallory := do(t, s, "POST", "/api/identity/generate", nil)
uid := alice["uid"].(string)
_, chBody := do(t, s, "POST", "/api/identity/challenge", map[string]any{"uid": uid, "action": "delete"})
proof := signAs(t, mallory["privateKeyHex"].(string), chBody["challenge"].(string))
req := map[string]any{"uid": uid, "confirm": true}
for k, v := range proof {
req[k] = v
}
rec, body := do(t, s, "DELETE", "/api/identity", req)
if rec.Code != http.StatusUnauthorized || body["error"] != "bad_signature" {
t.Fatalf("mallory deleting alice = %d %v, want 401 bad_signature", rec.Code, body)
}
if !identity.ExistsByUID(s.config.IdentityAccountsDir(), uid) {
t.Fatal("alice's account was deleted by mallory's signature")
}
}
+255
View File
@@ -1,12 +1,14 @@
package api
import (
"encoding/json"
"net/http"
"net/url"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/storage"
)
// KVInfoResponse is the response for GET /api/kv/info
@@ -184,3 +186,256 @@ func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, response)
}
// KVTreeResponse is the response for GET /api/kv/tree
type KVTreeResponse struct {
Prefix string `json:"prefix"`
Delimiter string `json:"delimiter"`
Branches []string `json:"branches"`
Leaves []string `json:"leaves"`
Cursor string `json:"cursor"`
HasMore bool `json:"hasMore"`
}
// KVSearchResponse is the response for GET /api/kv/search
type KVSearchResponse struct {
Contains string `json:"contains"`
Keys []string `json:"keys"`
Truncated bool `json:"truncated"`
}
// KVFindResponse is the response for GET /api/kv/find
type KVFindResponse struct {
Contains string `json:"contains"`
Fields []string `json:"fields"`
Matches []storage.FindMatch `json:"matches"`
Truncated bool `json:"truncated"`
}
// KVValueResponse is the response for GET /api/kv/value
type KVValueResponse struct {
Key string `json:"key"`
Type string `json:"type"`
Value string `json:"value"`
Exists bool `json:"exists"`
}
// KVValuePutRequest is the body for PUT /api/kv/value
type KVValuePutRequest struct {
Key string `json:"key"`
Value string `json:"value"`
}
// KVValueDeleteResponse is the response for DELETE /api/kv/value
type KVValueDeleteResponse struct {
Key string `json:"key"`
Deleted bool `json:"deleted"`
}
// handleKVTree handles GET /api/kv/tree?prefix=&delimiter=/&cursor=0&max=500
//
// Lists the immediate children at prefix. When delimiter is set, keys with a
// further delimiter are collapsed into branches. Pass the returned cursor back
// to continue when hasMore is true.
func (s *Server) handleKVTree(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
prefix := q.Get("prefix")
delimiter := q.Get("delimiter")
cursor := q.Get("cursor")
maxResults := 500
if v := q.Get("max"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
maxResults = n
}
}
leaves, branches, nextCursor, err := s.storage.ScanKeys(prefix, delimiter, cursor, maxResults)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVTreeResponse{
Prefix: prefix,
Delimiter: delimiter,
Branches: branches,
Leaves: leaves,
Cursor: nextCursor,
HasMore: nextCursor != "0",
})
}
// handleKVFind handles GET /api/kv/find?contains=&fields=a,b,c&limit=200
//
// Value-search: scans every key whose path ends with /<field> for each
// configured field, then returns those whose value contains the substring
// (case-insensitive). Used as the editor's "filter the view" action.
func (s *Server) handleKVFind(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
fieldsParam := q.Get("fields")
if fieldsParam == "" {
writeError(w, http.StatusBadRequest, "fields is required (comma-separated)")
return
}
fields := strings.Split(fieldsParam, ",")
for i := range fields {
fields[i] = strings.TrimSpace(fields[i])
}
limit := 200
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
matches, truncated, err := s.storage.FindByValue(contains, fields, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if matches == nil {
matches = []storage.FindMatch{}
}
writeJSON(w, http.StatusOK, KVFindResponse{
Contains: contains,
Fields: fields,
Matches: matches,
Truncated: truncated,
})
}
// handleKVSearch handles GET /api/kv/search?contains=&limit=500
func (s *Server) handleKVSearch(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
limit := 500
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
keys, truncated, err := s.storage.SearchKeys(contains, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVSearchResponse{
Contains: contains,
Keys: keys,
Truncated: truncated,
})
}
// handleKVValueGet handles GET /api/kv/value?key=...
func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
kind, value, exists, err := s.storage.GetKeyInfo(key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: key,
Type: kind,
Value: value,
Exists: exists,
})
}
// handleKVValuePut handles PUT /api/kv/value with body {key, value}: service
// scratch KV only (btmagnet:*, searchcov:*), kept in the cache. Merged records
// and bookkeeping are refused — metadata is written as signed claims.
func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req KVValuePutRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if err := s.storage.SetServiceValue(req.Key, req.Value); err != nil {
writeError(w, http.StatusConflict, err.Error())
return
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: req.Key,
Type: "string",
Value: req.Value,
Exists: true,
})
}
// handleKVValueDelete handles DELETE /api/kv/value?key=... (service keys only).
func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
deleted, err := s.storage.DeleteServiceValue(key)
if err != nil {
writeError(w, http.StatusConflict, err.Error())
return
}
writeJSON(w, http.StatusOK, KVValueDeleteResponse{
Key: key,
Deleted: deleted,
})
}
+51 -256
View File
@@ -4,7 +4,6 @@ import (
"encoding/json"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
)
@@ -39,6 +38,39 @@ type MetadataSearchRequest struct {
Property string `json:"property,omitempty"`
PropertyValue string `json:"propertyValue,omitempty"`
Limit int `json:"limit,omitempty"`
// As is a viewer uid: the search sees that signer's own claims overlaid
// (and their records as candidates). Also accepted as ?as=.
As string `json:"as,omitempty"`
// Filters is the structured form: AND across entries, OR within an
// entry's Values. Narrowed in SQL through the field index alongside the
// free-text Query, then verified per record (see serveIndexedSearch), so an
// id-anchored lookup costs an index probe rather than a read per record.
//
// Property/PropertyValue is the single-filter legacy shape and folds into
// this list (see normalizedFilters) with Exact=false, preserving its
// substring semantics byte-for-byte.
Filters []MetadataFilter `json:"filters,omitempty"`
}
// MetadataFilter is one structured constraint on a record.
//
// A record satisfies it when ANY of Values matches, and a search result must
// satisfy EVERY filter (OR within, AND across). An entry with no usable value
// constrains nothing — same rule the gateway's `genres_filter_matches` uses, so
// a caller that lowers an empty filter doesn't accidentally empty the result.
type MetadataFilter struct {
// Property is the field name, WITHOUT a key-set suffix: `genres`, not
// `genres/Action`. Both storage shapes are probed — see recordMatchesFilter.
Property string `json:"property"`
// Values are OR'd. Compared case-insensitively.
Values []string `json:"values,omitempty"`
// Exact selects CSV-exact comparison (the semantics meta-gateway's
// `any_csv_value_matches` applies at its own Layer-B re-filter) over
// substring containment. Callers doing identity lookups want this: with a
// result limit, a substring match on `tmdbid=1405` also admits `21405` and
// can crowd the real matches out of the page.
Exact bool `json:"exact,omitempty"`
}
// MetadataSearchResult is a single search result
@@ -193,7 +225,10 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
return
}
metadata, err := s.storage.GetMetadataFlat(hashID)
// Callers may hold a CID, not the root.
hashID = s.storage.ResolveRoot(hashID)
metadata, err := s.storage.GetMetadataFlatAs(hashID, r.URL.Query().Get("as"))
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
@@ -212,70 +247,6 @@ func (s *Server) handleGetMetadataByHashId(w http.ResponseWriter, r *http.Reques
writeJSON(w, http.StatusOK, response)
}
// handleUpdateMetadataByHashId handles PUT /api/metadata/{hashId}
func (s *Server) handleUpdateMetadataByHashId(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
// Exclude internal processing status field
delete(metadata, "processingStatus")
if err := s.storage.SetMetadataFlat(hashID, metadata); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"message": "Metadata updated successfully",
})
}
// handleDeleteMetadataByHashId handles DELETE /api/metadata/{hashId}
func (s *Server) handleDeleteMetadataByHashId(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
deletedCount, err := s.storage.DeleteMetadata(hashID)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"deletedKeys": deletedCount,
})
}
// handleMetadataGetProperty handles GET /api/metadata/{hashId}/property?property=X
func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
@@ -315,46 +286,6 @@ func (s *Server) handleMetadataGetProperty(w http.ResponseWriter, r *http.Reques
})
}
// handleMetadataUpdateProperty handles PUT /api/metadata/{hashId}/property
func (s *Server) handleMetadataUpdateProperty(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
hashID := vars["hashId"]
if hashID == "" {
writeError(w, http.StatusBadRequest, "hash ID is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req PropertyUpdateRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Property == "" {
writeError(w, http.StatusBadRequest, "property is required")
return
}
if err := s.storage.SetProperty(hashID, req.Property, req.Value); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"status": "ok",
"hashId": hashID,
"property": req.Property,
"value": req.Value,
"message": "Property updated successfully",
})
}
// handleSearchMetadata handles POST /api/metadata/search
func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
@@ -371,6 +302,9 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
if req.Limit == 0 {
req.Limit = 100
}
if req.As == "" {
req.As = r.URL.Query().Get("as")
}
// If hashId is provided, do exact match
if req.HashID != "" {
@@ -395,156 +329,17 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
return
}
// Get all hash IDs
allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
var results []MetadataSearchResult
total := len(allHashIDs)
// Search fields for query
searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
for _, hashID := range allHashIDs {
if len(results) >= req.Limit {
break
}
metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil || metadata == nil {
continue
}
matches := false
// Property search
if req.Property != "" && req.PropertyValue != "" {
value, ok := metadata[req.Property]
if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) {
matches = true
}
}
// General query search
if req.Query != "" && req.Property == "" {
queryLower := strings.ToLower(req.Query)
// Check if query matches hashId
if strings.Contains(strings.ToLower(hashID), queryLower) {
matches = true
}
// Check search fields
if !matches {
for _, field := range searchFields {
if value, ok := metadata[field]; ok {
if strings.Contains(strings.ToLower(value), queryLower) {
matches = true
break
}
}
}
}
}
// If no query or property filter, return all
if req.Query == "" && req.Property == "" {
matches = true
}
if matches {
results = append(results, MetadataSearchResult{
HashID: hashID,
Metadata: metadata,
})
}
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: total,
})
}
// handleBatchUpdate handles POST /api/metadata/batch
func (s *Server) handleBatchUpdate(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req MetadataBatchUpdateRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if len(req.Updates) == 0 {
writeError(w, http.StatusBadRequest, "updates array is required")
return
}
var results []MetadataBatchResult
successCount := 0
errorCount := 0
for _, update := range req.Updates {
if update.HashID == "" {
results = append(results, MetadataBatchResult{
HashID: "unknown",
Status: "error",
Error: "Missing hash ID",
})
errorCount++
continue
}
var updateErr error
// Update complete metadata
if update.Metadata != nil {
// Exclude internal processing status
delete(update.Metadata, "processingStatus")
updateErr = s.storage.SetMetadataFlat(update.HashID, update.Metadata)
}
// Update individual properties
if updateErr == nil && update.Properties != nil {
for property, value := range update.Properties {
if err := s.storage.SetProperty(update.HashID, property, value); err != nil {
updateErr = err
break
}
}
}
if updateErr != nil {
results = append(results, MetadataBatchResult{
HashID: update.HashID,
Status: "error",
Error: updateErr.Error(),
})
errorCount++
} else {
results = append(results, MetadataBatchResult{
HashID: update.HashID,
Status: "ok",
})
successCount++
}
}
writeJSON(w, http.StatusOK, MetadataBatchResponse{
Status: "completed",
Total: len(results),
Success: successCount,
Errors: errorCount,
Results: results,
})
// Everything else — free text, structured filters, both, or neither
// (which returns the first `limit`) — is served from the store's derived
// search indexes (search_index.go → storage.Search).
//
// There is deliberately no second path. The property branch used to fall
// through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which
// is what made an id-anchored lookup 3.63s over ~20k records (vs 10ms for
// the free-text branch on the same corpus) and kept meta-core out of the
// gateway's anchored search path entirely. The index already holds every
// field of every record, so that scan was re-reading data it had in hand.
s.serveIndexedSearch(w, req)
}
// handleClearMetadata handles POST /api/metadata/clear
+204
View File
@@ -0,0 +1,204 @@
package api
// Report inbox (/api/reports): bug reports and feedback filed from the client
// apps. An app's browser builds the report, the app's server forwards it here
// server-to-server, and the dashboard's Reports page reads it back. Storage
// and bounds live in internal/reports.
//
// Auth: like the rest of /api, gated at the perimeter — do NOT add to the
// nginx-hash-lock unauth bypass. Apps reach meta-core on the internal network.
import (
"encoding/json"
"errors"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/reports"
)
// maxReportBody bounds one POST. A report is at most a few blocks of 32 KiB
// plus a 200-line log tail; anything larger is not a report.
const maxReportBody = 512 << 10
// SetReports attaches the report inbox. Separate from NewServer so its
// signature (and its tests) stay untouched; nil leaves the routes answering
// 503.
func (s *Server) SetReports(r *reports.Store) {
s.reports = r
}
func (s *Server) setupReportRoutes() {
s.router.HandleFunc("/api/reports", s.handlePostReport).Methods("POST")
s.router.HandleFunc("/api/reports", s.handleListReports).Methods("GET")
s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handleGetReport).Methods("GET")
s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handlePatchReport).Methods("PATCH")
s.router.HandleFunc("/api/reports/{id:[0-9]+}", s.handleDeleteReport).Methods("DELETE")
}
func (s *Server) reportsReady(w http.ResponseWriter) bool {
if s.reports == nil {
writeError(w, http.StatusServiceUnavailable, "report inbox not available")
return false
}
return true
}
func reportID(r *http.Request) int64 {
id, _ := strconv.ParseInt(mux.Vars(r)["id"], 10, 64)
return id
}
// handlePostReport handles POST /api/reports → 201 {id}.
func (s *Server) handlePostReport(w http.ResponseWriter, r *http.Request) {
if !s.reportsReady(w) {
return
}
var in map[string]json.RawMessage
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxReportBody)).Decode(&in); err != nil {
var tooBig *http.MaxBytesError
if errors.As(err, &tooBig) {
writeError(w, http.StatusRequestEntityTooLarge, "report larger than 512 KiB")
return
}
writeError(w, http.StatusBadRequest, "body must be a JSON object")
return
}
rep, err := reports.Normalize(in, time.Now())
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
id, err := s.reports.Insert(rep)
if err != nil {
log.Printf("[reports] insert failed: %v", err)
writeError(w, http.StatusInternalServerError, "could not store the report")
return
}
log.Printf("[reports] #%d filed by %s (%s)", id, rep.App, firstLine(rep.Note))
writeJSON(w, http.StatusCreated, map[string]int64{"id": id})
}
// handleListReports handles GET /api/reports?app=&status=&q=&before=&limit=.
func (s *Server) handleListReports(w http.ResponseWriter, r *http.Request) {
if !s.reportsReady(w) {
return
}
q := r.URL.Query()
f := reports.Filter{App: q.Get("app"), Status: q.Get("status"), Q: q.Get("q")}
f.Before, _ = strconv.ParseInt(q.Get("before"), 10, 64)
f.Limit, _ = strconv.Atoi(q.Get("limit"))
if f.Status != "" && !reports.ValidStatus(f.Status) {
writeError(w, http.StatusBadRequest, "status must be open, triaged or closed")
return
}
page, err := s.reports.List(f)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, page)
}
// handleGetReport handles GET /api/reports/{id}.
func (s *Server) handleGetReport(w http.ResponseWriter, r *http.Request) {
if !s.reportsReady(w) {
return
}
rep, err := s.reports.Get(reportID(r))
if errors.Is(err, reports.ErrNotFound) {
writeError(w, http.StatusNotFound, "no such report")
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, rep)
}
// handlePatchReport handles PATCH /api/reports/{id} {status?, triage_note?}.
func (s *Server) handlePatchReport(w http.ResponseWriter, r *http.Request) {
if !s.reportsReady(w) {
return
}
var in struct {
Status *string `json:"status"`
TriageNote *string `json:"triage_note"`
}
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, 64<<10)).Decode(&in); err != nil {
writeError(w, http.StatusBadRequest, "body must be a JSON object")
return
}
if in.Status == nil && in.TriageNote == nil {
writeError(w, http.StatusBadRequest, "nothing to change: send status and/or triage_note")
return
}
if in.Status != nil && !reports.ValidStatus(*in.Status) {
writeError(w, http.StatusBadRequest, "status must be open, triaged or closed")
return
}
rep, err := s.reports.Update(reportID(r), in.Status, in.TriageNote)
if errors.Is(err, reports.ErrNotFound) {
writeError(w, http.StatusNotFound, "no such report")
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, rep)
}
// handleDeleteReport handles DELETE /api/reports/{id}.
func (s *Server) handleDeleteReport(w http.ResponseWriter, r *http.Request) {
if !s.reportsReady(w) {
return
}
err := s.reports.Delete(reportID(r))
if errors.Is(err, reports.ErrNotFound) {
writeError(w, http.StatusNotFound, "no such report")
return
}
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
// pruneReports re-applies the inbox bounds hourly. Insert already prunes, so
// this only matters when a bound was lowered between restarts.
func (s *Server) pruneReports() {
run := func() {
if n, err := s.reports.Prune(); err != nil {
log.Printf("[reports] prune failed: %v", err)
} else if n > 0 {
log.Printf("[reports] pruned %d report(s) over the inbox bounds", n)
}
}
run()
ticker := time.NewTicker(time.Hour)
defer ticker.Stop()
for range ticker.C {
run()
}
}
func firstLine(s string) string {
if i := strings.IndexByte(s, '\n'); i >= 0 {
s = s[:i]
}
if r := []rune(s); len(r) > 80 {
s = string(r[:80]) + "…"
}
if s == "" {
return "no note"
}
return strconv.Quote(s)
}
+121
View File
@@ -0,0 +1,121 @@
package api
import (
"bytes"
"net/http"
"net/http/httptest"
"path/filepath"
"strconv"
"strings"
"testing"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/reports"
)
func newReportServer(t *testing.T) *Server {
t.Helper()
store, err := reports.Open(filepath.Join(t.TempDir(), "reports.db"), 0, 0)
if err != nil {
t.Fatal(err)
}
t.Cleanup(func() { _ = store.Close() })
s := &Server{router: mux.NewRouter(), reports: store}
s.setupReportRoutes()
return s
}
func TestReportsRoundTrip(t *testing.T) {
s := newReportServer(t)
rec, out := do(t, s, "POST", "/api/reports", map[string]any{
"app": "meta-read", "note": "blank page", "uid": "u1",
"reader": map[string]any{"page": 3}, "log": []string{"a", "b"},
})
if rec.Code != http.StatusCreated || out["id"] == nil {
t.Fatalf("post: %d %s", rec.Code, rec.Body)
}
id := int64(out["id"].(float64))
rec, out = do(t, s, "GET", "/api/reports", nil)
list, _ := out["reports"].([]any)
if rec.Code != 200 || len(list) != 1 {
t.Fatalf("list: %d %s", rec.Code, rec.Body)
}
first := list[0].(map[string]any)
if _, hasBody := first["body"]; hasBody || first["log_lines"].(float64) != 2 {
t.Fatalf("summary should carry log_lines, not the body: %v", first)
}
path := "/api/reports/" + itoa(id)
rec, out = do(t, s, "GET", path, nil)
body, _ := out["body"].(map[string]any)
if rec.Code != 200 || body["reader"] == nil || out["uid"] != "u1" {
t.Fatalf("get: %d %s", rec.Code, rec.Body)
}
rec, out = do(t, s, "PATCH", path, map[string]any{"status": "triaged", "triage_note": "Deck #1"})
if rec.Code != 200 || out["status"] != "triaged" || out["triage_note"] != "Deck #1" {
t.Fatalf("patch: %d %s", rec.Code, rec.Body)
}
rec, _ = do(t, s, "GET", "/api/reports?status=triaged&app=meta-read", nil)
if rec.Code != 200 || !strings.Contains(rec.Body.String(), `"triaged":1`) {
t.Fatalf("filtered list: %d %s", rec.Code, rec.Body)
}
if rec, _ = do(t, s, "DELETE", path, nil); rec.Code != http.StatusNoContent {
t.Fatalf("delete: %d", rec.Code)
}
if rec, _ = do(t, s, "GET", path, nil); rec.Code != http.StatusNotFound {
t.Fatalf("get after delete: %d", rec.Code)
}
}
func TestReportsRejects(t *testing.T) {
s := newReportServer(t)
cases := []struct {
name, method, path string
body any
want int
}{
{"no app", "POST", "/api/reports", map[string]any{"note": "x"}, 400},
{"bad status filter", "GET", "/api/reports?status=done", nil, 400},
{"patch nothing", "PATCH", "/api/reports/1", map[string]any{}, 400},
{"patch bad status", "PATCH", "/api/reports/1", map[string]any{"status": "done"}, 400},
{"patch missing", "PATCH", "/api/reports/1", map[string]any{"status": "closed"}, 404},
{"delete missing", "DELETE", "/api/reports/1", nil, 404},
}
for _, c := range cases {
if rec, _ := do(t, s, c.method, c.path, c.body); rec.Code != c.want {
t.Errorf("%s: got %d want %d (%s)", c.name, rec.Code, c.want, rec.Body)
}
}
// Not a JSON object.
req := httptest.NewRequest("POST", "/api/reports", strings.NewReader(`[1,2]`))
rec := httptest.NewRecorder()
s.router.ServeHTTP(rec, req)
if rec.Code != 400 {
t.Errorf("array body: %d", rec.Code)
}
// Over the body cap.
big := bytes.Repeat([]byte("x"), maxReportBody+1)
req = httptest.NewRequest("POST", "/api/reports", strings.NewReader(`{"app":"a","note":"`+string(big)+`"}`))
rec = httptest.NewRecorder()
s.router.ServeHTTP(rec, req)
if rec.Code != http.StatusRequestEntityTooLarge {
t.Errorf("oversized body: %d", rec.Code)
}
}
func TestReportsUnavailableWithoutStore(t *testing.T) {
s := &Server{router: mux.NewRouter()}
s.setupReportRoutes()
if rec, _ := do(t, s, "GET", "/api/reports", nil); rec.Code != http.StatusServiceUnavailable {
t.Fatalf("no store: %d", rec.Code)
}
}
func itoa(n int64) string { return strconv.FormatInt(n, 10) }
+48
View File
@@ -0,0 +1,48 @@
package api
import (
"encoding/json"
"net/http"
)
// handleSchemaGet returns the per-field schema, folded from counters the write
// path keeps exact (see schema.Indexer).
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
snap, err := s.schemaIndexer.Snapshot(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(snap)
}
// handleSchemaRescan rebuilds the schema counters from a full read of the
// records and returns the result. The counters are exact by construction, so
// this is a repair tool, not routine: it runs as one write transaction, and
// every other write waits for it (minutes on a large store).
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
if err := s.storage.RebuildCounters(); err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
snap, err := s.schemaIndexer.Snapshot(r.Context())
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
snap.Source = "rescan"
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"published_events": 0,
"schema": snap,
})
}
+211
View File
@@ -0,0 +1,211 @@
package api
import "testing"
// Tests for the structured-filter matcher that lets meta-core answer an
// id-anchored query from the in-memory search index (search_index.go).
//
// Why this matters beyond "filters work": meta-core is a *coarse pre-filter*
// for meta-gateway, which re-evaluates every record it emits through its own
// `query_eval::record_matches`. So the property that has to hold is that this
// matcher stays a SUPERSET of that one — it may admit records the gateway will
// drop, but it must never drop a record the gateway would have kept. Every
// case below is written from that angle:
//
// - key-set fields (`genres/Action`) must match a `genres:action` filter, or
// card-tier records vanish from a genre-scoped query;
// - a missing field fails CLOSED, matching record_matches;
// - an empty/unusable filter constrains NOTHING, so a caller that lowers a
// blank filter gets its results rather than an empty page.
func f(property string, exact bool, values ...string) MetadataFilter {
return MetadataFilter{Property: property, Values: values, Exact: exact}
}
func TestRecordMatchesFilter(t *testing.T) {
fields := map[string]string{
"fileType": "video",
"videoType": "tv,anime",
"tmdbid": "1405",
"genres/Action": "true",
"genres/Science Fiction": "true",
"source/gateway:nyaa.si": "true",
"categories/XXX": "true",
"retracted": "false",
}
cases := []struct {
name string
filter MetadataFilter
want bool
}{
// Flat scalar.
{"flat exact hit", f("fileType", true, "video"), true},
{"flat exact miss", f("fileType", true, "audio"), false},
{"flat exact is case-insensitive", f("fileType", true, "VIDEO"), true},
// csv-set field: each comma piece is compared independently.
{"csv member hit", f("videoType", true, "anime"), true},
{"csv member miss", f("videoType", true, "movie"), false},
// Exact vs contains — the reason Exact exists. A substring match on an
// identity field admits neighbours and, under a result limit, crowds
// the real matches off the page.
{"contains admits a superstring", f("tmdbid", false, "140"), true},
{"exact rejects a superstring", f("tmdbid", true, "140"), false},
{"exact accepts the whole value", f("tmdbid", true, "1405"), true},
// Key-set shape. The stored member is verbatim (`genres/Action`) while
// query values arrive lowercased, so the case-insensitive sweep is what
// makes a genre-scoped row work at all.
{"key-set exact casing", f("genres", true, "Action"), true},
{"key-set lowercased query", f("genres", true, "action"), true},
{"key-set with a space", f("genres", true, "science fiction"), true},
{"key-set miss", f("genres", true, "horror"), false},
{"key-set label containing a colon", f("source", true, "gateway:nyaa.si"), true},
// OR within one filter's values.
{"or hits on the second value", f("fileType", true, "audio", "video"), true},
{"or misses on all values", f("fileType", true, "audio", "image"), false},
// Fail-closed on an absent field — mirrors record_matches, and is what
// keeps a filtered query from returning unrelated records.
{"absent field fails closed", f("imdbid", true, "tt0903747"), false},
// A filter that constrains nothing must not empty the result.
{"no values constrains nothing", f("fileType", true), true},
{"blank value constrains nothing", f("fileType", true, " "), true},
{"no property constrains nothing", f("", true, "video"), true},
// A FLAT field whose value happens to be "false" is still a value, and
// an exact filter for it matches. Only the key-set arm treats "false"
// as non-membership (pinned in the test below).
{"flat field valued false still matches", f("retracted", true, "false"), true},
}
for _, tc := range cases {
t.Run(tc.name, func(t *testing.T) {
if got := recordMatchesFilter(fields, &tc.filter); got != tc.want {
t.Fatalf("recordMatchesFilter(%+v) = %v, want %v", tc.filter, got, tc.want)
}
})
}
}
// `retracted = "false"` is a flat field, not a key-set member, so the flat arm
// legitimately matches the literal string "false". The key-set arm is what must
// treat it as non-membership — pinned separately so the two arms don't get
// conflated if either is refactored.
func TestFalseyKeySetMemberIsNotMembership(t *testing.T) {
fields := map[string]string{"genres/Action": "false", "genres/Drama": "0"}
for _, v := range []string{"Action", "Drama"} {
filter := f("genres", true, v)
if recordMatchesFilter(fields, &filter) {
t.Fatalf("genres/%s with a falsey value must not assert membership", v)
}
}
}
func TestRecordMatchesFiltersIsAndAcrossEntries(t *testing.T) {
fields := map[string]string{"fileType": "video", "season": "2"}
all := []MetadataFilter{f("fileType", true, "video"), f("season", true, "2")}
if !recordMatchesFilters(fields, all) {
t.Fatal("a record satisfying every filter must match")
}
one := []MetadataFilter{f("fileType", true, "video"), f("season", true, "3")}
if recordMatchesFilters(fields, one) {
t.Fatal("filters are AND across entries — one miss must reject the record")
}
if !recordMatchesFilters(fields, nil) {
t.Fatal("no filters must match everything")
}
}
// The legacy Property/PropertyValue pair keeps its substring semantics: it is
// the shape existing callers already use, and tightening it silently would
// change results for them. New callers pass Filters and opt into Exact.
func TestNormalizedFiltersFoldsTheLegacyPair(t *testing.T) {
got := normalizedFilters(MetadataSearchRequest{Property: "tmdbid", PropertyValue: "1405"})
if len(got) != 1 {
t.Fatalf("legacy pair must fold into exactly one filter, got %d", len(got))
}
if got[0].Property != "tmdbid" || len(got[0].Values) != 1 || got[0].Values[0] != "1405" {
t.Fatalf("legacy pair folded incorrectly: %+v", got[0])
}
if got[0].Exact {
t.Fatal("the legacy pair must stay substring (Exact=false) for compatibility")
}
// A half-specified pair is not a filter — it must not become one that
// matches nothing and empties the response.
if n := len(normalizedFilters(MetadataSearchRequest{Property: "tmdbid"})); n != 0 {
t.Fatalf("a property with no value must not fold into a filter, got %d", n)
}
if n := len(normalizedFilters(MetadataSearchRequest{PropertyValue: "1405"})); n != 0 {
t.Fatalf("a value with no property must not fold into a filter, got %d", n)
}
both := normalizedFilters(MetadataSearchRequest{
Property: "tmdbid",
PropertyValue: "1405",
Filters: []MetadataFilter{f("fileType", true, "video")},
})
if len(both) != 2 {
t.Fatalf("the legacy pair must be appended to explicit filters, got %d", len(both))
}
}
// The acceptance bar for routing the anchored path through the index: a
// full-miss filtered sweep over a realistic corpus must stay in the same class
// as the free-text branch (measured 10 ms over 19 622 live records), not the
// legacy per-record Redis scan it replaces (measured 3.63 s over the same
// corpus). Full-miss is the worst case — the loop cannot short-circuit on the
// result limit, so every record is evaluated.
//
// Run: go test ./internal/api/ -bench BenchmarkFilterSweep -benchtime 10x
func BenchmarkFilterSweep(b *testing.B) {
const records = 20000
corpus := make([]map[string]string, records)
for i := range corpus {
corpus[i] = map[string]string{
"title": "Some Release",
"fileType": "video",
"videoType": "tv",
"tmdbid": "1405",
"season": "2",
"genres/Action": "true",
"genres/Drama": "true",
"source/gateway:nyaa.si": "true",
"categories/TV/Anime": "true",
"indexer": "nyaa.si",
"fileName": "some.release.s02e05.mkv",
"description/eng": "text",
"poster": "bafy",
"sizeByte": "123456789",
"contentKind": "episode",
"anchoredBy/tmdb:1405": "true",
}
}
// A miss on the identity field: the shape an id-anchored gateway query
// takes, and the one that must not degrade.
filters := []MetadataFilter{
f("tmdbid", true, "999999"),
f("fileType", true, "video"),
f("genres", true, "action"),
}
b.ResetTimer()
for n := 0; n < b.N; n++ {
hits := 0
for _, fields := range corpus {
if recordMatchesFilters(fields, filters) {
hits++
}
}
if hits != 0 {
b.Fatalf("expected a full miss, got %d hits", hits)
}
}
}
+180
View File
@@ -0,0 +1,180 @@
package api
import (
"net/http"
"strings"
"github.com/metazla/meta-core/internal/storage"
)
// serveIndexedSearch answers a search from the database's derived indexes
// (storage.Search): SQL narrows to a superset of candidates through the field
// expression index and the FTS5 trigram table, then every candidate is checked
// here with the exact matchers below — a token-AND match against the record's
// haystack (or a whole-query hashID substring, preserving the legacy
// behaviour) AND every structured filter.
//
// Until 2026-10 this ran against an in-memory copy of every record, rebuilt
// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh)
// and is gone. The matchers did not change, so neither did the semantics.
//
// This is the ONLY search path. Free text, filters, both, or neither all land
// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
tokens := strings.Fields(strings.ToLower(req.Query))
queryLower := strings.ToLower(req.Query)
filters := normalizedFilters(req)
narrow := make([]storage.SearchFilter, 0, len(filters))
for _, f := range filters {
narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
}
hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool {
if req.Query != "" &&
!matchAllTokens(storage.Haystack(fields), tokens) &&
!strings.Contains(strings.ToLower(hashID), queryLower) {
return false
}
return recordMatchesFilters(fields, filters)
})
if err != nil {
writeError(w, http.StatusInternalServerError, "search failed: "+err.Error())
return
}
total, err := s.storage.CountRoots()
if err != nil {
total = 0
}
results := make([]MetadataSearchResult, 0, len(hits))
for _, h := range hits {
results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields})
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: int(total),
})
}
// matchAllTokens reports whether haystack contains every token (AND semantics).
// An empty token list matches everything.
func matchAllTokens(haystack string, tokens []string) bool {
for _, t := range tokens {
if !strings.Contains(haystack, t) {
return false
}
}
return true
}
// normalizedFilters returns the request's structured filters with the legacy
// Property/PropertyValue pair folded in as one more entry.
//
// The legacy pair keeps Exact=false so its long-standing substring semantics
// are unchanged for existing callers; new callers pass Filters and choose.
func normalizedFilters(req MetadataSearchRequest) []MetadataFilter {
if req.Property == "" || req.PropertyValue == "" {
return req.Filters
}
out := make([]MetadataFilter, 0, len(req.Filters)+1)
out = append(out, req.Filters...)
return append(out, MetadataFilter{
Property: req.Property,
Values: []string{req.PropertyValue},
})
}
// recordMatchesFilters reports whether fields satisfies every filter (AND).
func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool {
for i := range filters {
if !recordMatchesFilter(fields, &filters[i]) {
return false
}
}
return true
}
// recordMatchesFilter reports whether fields satisfies one filter (OR within
// its values), probing BOTH storage shapes a field can take:
//
// - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`;
// - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"`
// (METADATA_KEYS "How the store is shaped").
//
// Probing both is what lets one filter shape serve the whole vocabulary. The
// gateway needs a `genres:action` filter to find a record whose genre lives in
// the key `genres/Action`, and a `fileType:video` filter to find one whose value
// lives in the field `fileType` — without meta-core knowing which fields are
// key-sets. It also means this stays a SUPERSET of the gateway's own
// `record_matches`, which is the contract: meta-core pre-filters, the gateway
// re-filters authoritatively.
//
// A filter with no usable value constrains nothing (matches). A filter whose
// field is absent fails closed, matching `record_matches`.
func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool {
values := make([]string, 0, len(f.Values))
for _, v := range f.Values {
if v = strings.TrimSpace(v); v != "" {
values = append(values, v)
}
}
if f.Property == "" || len(values) == 0 {
return true
}
// Key-set shape: `<property>/<value>` present and truthy. Try the exact
// key first — that hits whenever the caller's casing matches what was
// written, which is the common case and costs one map probe.
for _, v := range values {
if isTruthyMember(fields[f.Property+"/"+v]) {
return true
}
}
// Members are written verbatim (`genres/Action`) while query values arrive
// lowercased, so fall back to one case-insensitive sweep. Single pass over
// the record's fields for ALL values, not one pass per value.
wanted := make(map[string]struct{}, len(values))
for _, v := range values {
wanted[strings.ToLower(v)] = struct{}{}
}
prefix := strings.ToLower(f.Property) + "/"
for k, val := range fields {
if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) {
continue
}
if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) {
return true
}
}
// Flat shape: the field's own value, split on `,` for csv-set fields.
fieldValue, ok := fields[f.Property]
if !ok {
return false
}
for _, piece := range strings.Split(fieldValue, ",") {
piece = strings.TrimSpace(piece)
if piece == "" {
continue
}
for _, v := range values {
if f.Exact {
if strings.EqualFold(piece, v) {
return true
}
} else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
return true
}
}
}
return false
}
// isTruthyMember reports whether a key-set member's value asserts membership.
// Mirrors the gateway's `is_truthy_member`: presence with a non-empty,
// non-"false"/"0" value.
func isTruthyMember(v string) bool {
v = strings.TrimSpace(strings.ToLower(v))
return v != "" && v != "false" && v != "0"
}
+223 -71
View File
@@ -9,10 +9,12 @@ import (
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/events"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/meshdisco"
"github.com/metazla/meta-core/internal/mounts"
"github.com/metazla/meta-core/internal/reports"
"github.com/metazla/meta-core/internal/schema"
"github.com/metazla/meta-core/internal/storage"
"github.com/metazla/meta-core/internal/watcher"
"github.com/metazla/meta-core/internal/watchers"
@@ -23,46 +25,61 @@ import (
type Server struct {
config *config.Config
leaderProvider *leader.LeaderInfoProvider
discovery *discovery.Service
cleaner *discovery.Cleaner
storage *storage.Client
mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers
mountPoller *mounts.Poller
mountStatsPoller *mounts.StatsPoller
watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers
watchersManager *watchers.Manager
watchersPoller *watchers.Poller
watchersHandlers *watchers.Handlers
metaPublisher *events.MetaPublisher
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler
// challenges backs proof-of-possession on reveal/delete. In-process, so a
// restart invalidates the ones in flight — see identity/challenge.go.
challenges *identity.ChallengeStore
// mesh is the UDP discovery node (beacon v2). nil when discovery
// is disabled or failed to bind, in which case /api/neighbors reports an
// empty list rather than erroring — a dashboard nav is decoration and
// must not take the API down with it.
mesh *meshdisco.Node
router *mux.Router
server *http.Server
// reports is the bug-report inbox (reports.go); nil = routes answer 503.
reports *reports.Store
// stats caches the keyspace-walk half of GET /api/stats so a polling
// dashboard never puts a multi-second store walk on the request path.
stats *statsCache
}
// SetMeshNode attaches the UDP discovery node. Separate from NewServer so the
// server's constructor signature (and its tests) stay untouched.
func (s *Server) SetMeshNode(n *meshdisco.Node) {
s.mesh = n
}
// NewServer creates a new API server
func NewServer(
cfg *config.Config,
leaderProvider *leader.LeaderInfoProvider,
disc *discovery.Service,
cleaner *discovery.Cleaner,
stor *storage.Client,
) *Server {
s := &Server{
config: cfg,
leaderProvider: leaderProvider,
discovery: disc,
cleaner: cleaner,
storage: stor,
router: mux.NewRouter(),
stats: &statsCache{},
}
// Initialize file watcher/scanner and dispatcher
var scanFunc mounts.ScanFunc
if cfg.EnableFileWatcher {
s.watcherDispatcher = watcher.NewDispatcher()
// Wire storage client for Redis Stream publishing
// Wire storage client for file:events publishing
if stor != nil {
s.watcherDispatcher.SetStorageClient(stor)
}
@@ -71,6 +88,11 @@ func NewServer(
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
} else {
s.fileWatcher = fileWatcher
// Wire storage so the watcher can persist (path,size,mtime,cid)
// tuples to the store as it computes midhashes.
if stor != nil {
fileWatcher.SetStorageClient(stor)
}
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
// Create scan function from watcher for mounts
scanFunc = fileWatcher.ScanMountPath
@@ -86,8 +108,21 @@ func NewServer(
// Create watchers poller
if s.fileWatcher != nil {
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
// Pass republish callback to handlers (handles nil metaPublisher case)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller)
// Hydrate the in-memory state registry from stored tuples so
// the first scan tick after boot can skip ComputeMidHash256
// for files whose (size, mtime) match.
configs := watchersManager.List()
roots := make([]watcher.WatcherRoot, 0, len(configs))
for _, c := range configs {
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
}
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
} else {
log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
}
}
}
}
@@ -103,11 +138,27 @@ func NewServer(
// Create and configure mount poller
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
mountsManager.SetPoller(s.mountPoller)
// IO stats poller — queries the local rclone RC API every few seconds
// and exposes live throughput per mount via the same status JSON.
// Stats are daemon-global (rclone limitation); see stats_poller.go.
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
mountsManager.SetStatsPoller(s.mountStatsPoller)
}
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
s.webdavHandler = webdav.NewHandler(cfg)
s.challenges = identity.NewChallengeStore()
// One-time migration: fold a pre-multi-account identity.json into the
// per-account keystore so existing single-identity hosts keep their uid.
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
log.Printf("[API] Warning: identity migration failed: %v", err)
} else if migrated {
log.Printf("[API] Migrated legacy identity.json into multi-account keystore")
}
s.setupRoutes()
return s
}
@@ -120,79 +171,140 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/leader", s.handleLeader).Methods("GET")
s.router.HandleFunc("/urls", s.handleURLs).Methods("GET")
// Dashboard headline counters in one request. Cheap half live, keyspace
// walks served from a background-refreshed cache — see stats.go.
s.router.HandleFunc("/api/stats", s.handleStats).Methods("GET")
// Metadata Editor API routes (must be before /meta/{hash} routes)
s.router.HandleFunc("/api/metadata/hash-ids", s.handleGetHashIds).Methods("GET")
s.router.HandleFunc("/api/metadata/list", s.handleListMetadata).Methods("GET")
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleGone).Methods("POST")
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
// User Data Layer signing identity (see internal/identity). Auth-gated
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/challenge", s.handleIdentityChallenge).Methods("POST")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
s.router.HandleFunc("/api/identity/reveal", s.handleIdentityReveal).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/identity/sign-batch", s.handleIdentitySignBatch).Methods("POST")
s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
// Signed claims (v2.0.0): the only metadata write path, the raw claims
// behind a record, and the stateless merge clients use for search hits.
s.router.HandleFunc("/api/claims", s.handlePostClaims).Methods("POST")
s.router.HandleFunc("/api/merge", s.handleMerge).Methods("POST")
s.router.HandleFunc("/api/meta/{cid}/claims", s.handleClaimsByCID).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/claims", s.handleClaimsByRID).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleGone).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGone).Methods("DELETE")
// Schema inference
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
// KV Browser API routes
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
s.router.HandleFunc("/api/kv/tree", s.handleKVTree).Methods("GET")
s.router.HandleFunc("/api/kv/search", s.handleKVSearch).Methods("GET")
s.router.HandleFunc("/api/kv/find", s.handleKVFind).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValueGet).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValuePut).Methods("PUT")
s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
// User Data Layer (UDL) routes — per-user signed records (My List,
// Continue Watching, ratings, profile). See internal/api/udl.go.
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordGet).Methods("GET")
s.router.HandleFunc("/api/udl/record", s.handleUDLRecordPut).Methods("PUT")
s.router.HandleFunc("/api/udl/records", s.handleUDLRecordsPut).Methods("PUT")
s.router.HandleFunc("/api/udl/user/{uid}/key/{key}", s.handleUDLUserKey).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/cid/{cid}", s.handleUDLUserCid).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/all", s.handleUDLUserAll).Methods("GET")
s.router.HandleFunc("/api/udl/user/{uid}/count", s.handleUDLUserCount).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/users", s.handleUDLCidUsers).Methods("GET")
s.router.HandleFunc("/api/udl/cid/{cid}/aggregate", s.handleUDLAggregate).Methods("GET")
s.router.HandleFunc("/api/udl/users/stats", s.handleUDLUserStats).Methods("GET")
// Metadata operations - base endpoints
s.router.HandleFunc("/meta/{hash}", s.handleGetMeta).Methods("GET")
s.router.HandleFunc("/meta/{hash}", s.handlePutMeta).Methods("PUT")
s.router.HandleFunc("/meta/{hash}", s.handlePatchMeta).Methods("PATCH")
s.router.HandleFunc("/meta/{hash}", s.handleDeleteMeta).Methods("DELETE")
s.router.HandleFunc("/meta/{hash}", s.handleGone).Methods("PUT", "PATCH", "DELETE")
s.router.HandleFunc("/meta", s.handleListMeta).Methods("GET")
// Metadata operations - set operations (must be before property routes)
s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleAddToSet).Methods("POST")
s.router.HandleFunc("/meta/{hash}/_add/{key:.*}", s.handleGone).Methods("POST")
// Metadata operations - property-level (key can contain slashes)
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGetProperty).Methods("GET")
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handlePutProperty).Methods("PUT")
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleDeleteProperty).Methods("DELETE")
s.router.HandleFunc("/meta/{hash}/{key:.*}", s.handleGone).Methods("PUT", "DELETE")
// Data operations
s.router.HandleFunc("/data/{hash}/path", s.handleGetDataPath).Methods("GET")
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
// File operations (by CID)
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
// Register one file under /files/plugin by midhash — the watcher's
// mint + alias + file:events path for a single file. Registered
// unconditionally: it exists for boxes whose watcher is OFF.
s.router.HandleFunc("/api/files/register", s.handleRegisterFile).Methods("POST")
// File-backed roots (filePath+sizeByte+mtimeNano) with sizes, batched
// server-side, so clients answer "how many files, how big" without one
// read per hashId. ?summary=1 returns only {count, totalSize}.
s.router.HandleFunc("/api/files/tuples", s.handleGetFileTuples).Methods("GET")
// Service discovery
// Both /services and /api/services are supported for consistency with other services
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
s.router.HandleFunc("/api/services", s.handleListServices).Methods("GET")
s.router.HandleFunc("/services/cleanup/stats", s.handleCleanupStats).Methods("GET")
s.router.HandleFunc("/services/{name}", s.handleGetService).Methods("GET")
// Public metadata-by-CID resolver. Reverse-index lookup → full document.
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
// without going through Authelia.
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
// SSE event streams. The only way consumers see meta-core's change
// streams — they never touch the database. Auth-bypass
// applies but exposure should be inside-only (do NOT proxy through
// Caddy publicly). See docs/api-mediated-access.md "Auth".
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
// beacon v2: neighbours heard over UDP. Supersedes /api/services,
// which is still served above while consumers migrate.
s.router.HandleFunc("/neighbors", s.handleListNeighbors).Methods("GET")
s.router.HandleFunc("/api/neighbors", s.handleListNeighbors).Methods("GET")
// Report inbox: bug reports / feedback forwarded by the client apps,
// read by the dashboard's Reports page. Perimeter-gated (reports.go).
s.setupReportRoutes()
// Mount management routes (if manager initialized)
if s.mountsHandlers != nil {
s.mountsHandlers.RegisterRoutes(s.router)
log.Println("[API] Mount management routes registered at /api/mounts/*")
}
log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
log.Println("[API] KV Browser routes registered at /api/kv/*")
// File watcher routes (if watcher initialized)
if s.watcherHandlers != nil {
s.watcherHandlers.RegisterRoutes(s.router)
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
}
// Watchers management routes (if watchers initialized)
if s.watchersHandlers != nil {
s.watchersHandlers.RegisterRoutes(s.router)
log.Println("[API] Watchers management routes registered at /api/watchers/*")
}
// WebDAV handler (caching is handled by nginx proxy_cache)
if s.webdavHandler != nil {
s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
log.Println("[API] WebDAV routes registered at /webdav/*")
}
// Add middleware
@@ -234,23 +346,32 @@ func (s *Server) Start() error {
}
}
// Start meta publisher (if storage is connected)
// Publishes keyspace notifications to meta:events stream
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
// mounts), so always-on rather than gated by config.
if s.mountStatsPoller != nil {
s.mountStatsPoller.Start()
}
// Schema: served from counters the write path keeps exact; nothing runs.
// Change events need no publisher either — the storage engine appends them
// in the same transaction as each change, and a consumer that wants the
// whole state asks /api/events/meta for "0" and gets a snapshot (see
// streamSnapshot). The boot republish of every field is gone with it.
if s.storage != nil && s.storage.IsConnected() {
s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient())
if err := s.metaPublisher.Start(); err != nil {
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
} else {
// Republish events for all existing metadata
s.schemaIndexer = schema.NewIndexer(s.storage.Store())
go func() {
count, err := s.RepublishMetadata()
if err != nil {
log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)
} else {
log.Printf("[API] Republished %d metadata events on startup", count)
started := time.Now()
if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
log.Printf("[API] Warning: search row rebuild failed: %v", err)
} else if n > 0 {
log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
}
}()
go s.logWriteStats()
}
if s.reports != nil {
go s.pruneReports()
}
return nil
@@ -272,11 +393,9 @@ func (s *Server) Stop() error {
}
}
// Stop meta publisher
if s.metaPublisher != nil {
if err := s.metaPublisher.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop meta publisher: %v", err)
}
// Stop IO stats poller
if s.mountStatsPoller != nil {
s.mountStatsPoller.Stop()
}
if s.server == nil {
@@ -291,34 +410,67 @@ func (s *Server) Stop() error {
return s.server.Shutdown(ctx)
}
// RepublishMetadata republishes all existing metadata to the meta:events stream
// This is called on startup and after reset-all
func (s *Server) RepublishMetadata() (int, error) {
if s.metaPublisher == nil {
return 0, fmt.Errorf("meta publisher not initialized")
// logWriteStats reports, every 15 minutes, how many writes changed a value
// and how many found it already in place (kv skips those). The ratio is what
// the no-op skip buys on a permanent store that re-persists repeated hits.
func (s *Server) logWriteStats() {
ticker := time.NewTicker(15 * time.Minute)
defer ticker.Stop()
for range ticker.C {
store := s.storage.Store()
if store == nil {
return
}
changed, unchanged := store.WriteStats()
if total := changed + unchanged; total > 0 {
log.Printf("[Storage] writes since start: %d changed, %d unchanged skipped (%.0f%% no-op)", changed, unchanged, 100*float64(unchanged)/float64(total))
}
}
if s.storage == nil || !s.storage.IsConnected() {
return 0, fmt.Errorf("storage not connected")
}
// Create the republish function that wraps storage methods
republishFunc := func() ([]string, func(string) (map[string]string, error), error) {
hashIDs, err := s.storage.GetAllHashIDs()
if err != nil {
return nil, nil, err
}
return hashIDs, s.storage.GetMetadataFlat, nil
// statusRecorder captures the response status so loggingMiddleware can stay
// silent on success and only report failures. It forwards Flush/Unwrap so SSE
// and other streaming handlers keep working through the wrapper.
type statusRecorder struct {
http.ResponseWriter
status int
written bool
}
return s.metaPublisher.RepublishAllMetadata(republishFunc)
func (r *statusRecorder) WriteHeader(code int) {
if !r.written {
r.status = code
r.written = true
}
r.ResponseWriter.WriteHeader(code)
}
// loggingMiddleware logs all requests
func (r *statusRecorder) Write(b []byte) (int, error) {
if !r.written {
r.status = http.StatusOK
r.written = true
}
return r.ResponseWriter.Write(b)
}
func (r *statusRecorder) Flush() {
if f, ok := r.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
// loggingMiddleware logs a request only when it fails (status >= 400), keeping
// the nominal case quiet. Successful requests are not logged.
func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now()
next.ServeHTTP(w, r)
log.Printf("[API] %s %s %s", r.Method, r.URL.Path, time.Since(start))
rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
next.ServeHTTP(rec, r)
if rec.status >= 400 {
log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
}
})
}
@@ -326,7 +478,7 @@ func loggingMiddleware(next http.Handler) http.Handler {
func corsMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Access-Control-Allow-Origin", "*")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Methods", "GET, POST, PUT, PATCH, DELETE, OPTIONS")
w.Header().Set("Access-Control-Allow-Headers", "Content-Type")
if r.Method == "OPTIONS" {
+342
View File
@@ -0,0 +1,342 @@
package api
import (
"context"
"encoding/json"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/kv"
"github.com/metazla/meta-core/internal/storage"
)
// SSE event stream endpoints for the file:events and meta:events streams.
//
// Wire contract (see docs/api-mediated-access.md):
// - One SSE event per stream entry.
// - `id:` is the opaque stream entry ID (`<ms>-<seq>`, the Redis Streams
// form, kept across the move to SQLite so cursors consumers persisted to
// disk still compare correctly); consumers MUST treat it as a black-box
// string they echo back via Last-Event-ID.
// - `event:` is the `type` field from the underlying entry.
// - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
// - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
// - On reconnect with Last-Event-ID, resume from the next entry. If the
// cursor has been trimmed by retention, emit one `event: gap` payload
// before resuming from the oldest available entry.
const (
sseHeartbeatInterval = 30 * time.Second
// sseReadBlock controls how long each read waits for new entries.
// Much shorter than sseHeartbeatInterval so the handler stays responsive
// to client disconnects and the heartbeat check runs frequently enough
// that the keep-alive fires within a few seconds of its nominal cadence.
sseReadBlock = 2 * time.Second
// sseReadCount caps how many entries one read returns. Small enough
// to keep per-flush memory low, large enough to drain bursts cheaply.
sseReadCount = 100
)
// handleEventsFiles streams the file:events stream as SSE.
func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "file:events")
}
// handleEventsMeta streams the meta:events stream as SSE.
func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "meta:events")
}
// streamSSE is the SSE pump shared by both endpoints. Each subscriber reads
// the stream itself and sleeps on the store's commit broadcast between reads —
// fine at our scale (≤20 subscribers; one per consumer service).
func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
if !s.storage.IsConnected() {
w.Header().Set("Retry-After", "5")
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
flusher, ok := w.(http.Flusher)
if !ok {
writeError(w, http.StatusInternalServerError, "streaming unsupported")
return
}
// The server-wide WriteTimeout (30s on Server.server in server.go) is
// fine for ordinary handlers but lethal for SSE — it would close every
// connection at exactly 30s, racing the heartbeat. Disable it
// per-request via http.NewResponseController. SSE endpoints never need
// a write deadline; the client disconnect path already cleans up.
rc := http.NewResponseController(w)
if err := rc.SetWriteDeadline(time.Time{}); err != nil {
// Older transports (or nested response writers) may not support
// this. Log and continue — the 30s cap will still apply, but the
// handler is still useful for short-lived subscriptions.
log.Printf("[SSE %s] could not disable write deadline: %v", stream, err)
}
// SSE response headers. X-Accel-Buffering disables nginx response
// buffering; the hash-lock sidecar uses nginx, so this matters.
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher.Flush()
store := s.storage.Store()
if store == nil {
// Storage was just disconnected — close cleanly.
return
}
ctx := r.Context()
lastID := r.Header.Get("Last-Event-ID")
var cursor string
if stream == storage.MetaEventsStream && (lastID == "0" || lastID == "0-0") {
// "From the beginning" on meta:events means "the whole current
// state": stream it from the records themselves, then the live tail.
head, err := streamSnapshot(ctx, store, w, flusher)
if err != nil {
if ctx.Err() == nil {
log.Printf("[SSE %s] snapshot failed: %v", stream, err)
}
return
}
cursor = head
} else {
if rebuilt := s.storage.RebuiltAt(); stream == storage.MetaEventsStream && lastID != "" && rebuilt != "" &&
compareStreamIDs(lastID, rebuilt) < 0 {
// The cache was rebuilt after this cursor: the rebuild published no
// per-key events, so the consumer must take a snapshot.
writeGap(w, flusher, lastID, rebuilt, "rebuild")
cursor = "0"
} else {
cursor = resolveStartCursor(ctx, store, stream, lastID, w, flusher)
}
}
lastWrite := time.Now()
for {
if err := ctx.Err(); err != nil {
return
}
// Take the wake-up channel BEFORE reading, so an append landing
// between the read and the wait still wakes us.
wake := store.Wait(stream)
entries, err := store.ReadAfter(ctx, stream, cursor, sseReadCount)
if err != nil {
if ctx.Err() != nil {
return
}
log.Printf("[SSE %s] read error: %v", stream, err)
return
}
for _, e := range entries {
writeSSEEntry(w, flusher, e)
cursor = e.ID
lastWrite = time.Now()
}
if len(entries) == 0 {
timer := time.NewTimer(sseReadBlock)
select {
case <-ctx.Done():
timer.Stop()
return
case <-wake:
case <-timer.C:
}
timer.Stop()
}
// Heartbeat if quiet for >= the interval. Comment line; SSE
// clients ignore it per spec.
if time.Since(lastWrite) >= sseHeartbeatInterval {
if _, err := fmt.Fprint(w, ":keep-alive\n\n"); err != nil {
return
}
flusher.Flush()
lastWrite = time.Now()
}
}
}
// streamSnapshot sends one `set` event per existing record key, then returns
// the stream position the live tail continues from.
//
// This is how a consumer that rebuilds its state from events (meta-fuse sends
// Last-Event-ID 0-0 on every start) gets ALL of it. The stream itself only
// retains ~100k entries, so replaying it never could; meta-core used to
// republish every field into it on boot instead, which took minutes and still
// lost everything past the retention window.
//
// The head is captured BEFORE the snapshot is read, so a change racing the
// snapshot appears in the tail too — re-applying a `set` is idempotent. Only
// the final snapshot event carries an `id:`, so a consumer that disconnects
// mid-snapshot still holds its old cursor and gets a fresh snapshot.
func streamSnapshot(ctx context.Context, store *kv.Store, w http.ResponseWriter, flusher http.Flusher) (string, error) {
head, err := store.LastID(ctx, storage.MetaEventsStream)
if err != nil {
return "", err
}
ts := strconv.FormatInt(time.Now().UnixMilli(), 10)
pending := ""
err = store.ScanPrefix(ctx, "file:", 5000, func(rows []kv.KV) error {
for _, row := range rows {
if pending != "" {
writeSnapshotEvent(w, pending, ts, "")
}
pending = row.K
}
flusher.Flush()
return ctx.Err()
})
if err != nil {
return "", err
}
if pending != "" {
writeSnapshotEvent(w, pending, ts, head)
flusher.Flush()
}
return head, nil
}
func writeSnapshotEvent(w http.ResponseWriter, key, ts, id string) {
payload, _ := json.Marshal(map[string]string{"key": key, "ts": ts})
if id != "" {
fmt.Fprintf(w, "id: %s\n", id)
}
fmt.Fprintf(w, "event: set\ndata: %s\n\n", payload)
}
// resolveStartCursor decides where to start reading.
// - No Last-Event-ID → the stream's last issued ID, resolved ONCE here (only
// entries appended from now on). Redis' "$" was re-resolved on every
// XREAD, which dropped anything appended between two reads.
// - Last-Event-ID provided AND still in stream → resume from it.
// - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
// and resume from the oldest available entry's predecessor "0", so the
// next read picks up everything still in the stream.
//
// The "still in stream" check is cheap: one read of the first entry.
func resolveStartCursor(ctx context.Context, store *kv.Store, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
if lastID == "" {
head, err := store.LastID(xCtx, stream)
if err != nil {
log.Printf("[SSE %s] head lookup failed: %v (starting from the beginning)", stream, err)
return "0"
}
return head
}
oldestEntry, ok, err := store.Oldest(xCtx, stream)
if err != nil {
// Treat any error as "stream not available right now" and resume
// from the cursor — the read either finds it or waits for new entries.
log.Printf("[SSE %s] oldest-entry probe failed: %v (resuming from cursor)", stream, err)
return lastID
}
if !ok {
// Stream is empty. Cursor may be from a previous incarnation; safest
// is to resume from the cursor — reading an empty stream returns
// nothing and waits until new entries arrive.
return lastID
}
oldest := oldestEntry.ID
if compareStreamIDs(lastID, oldest) >= 0 {
// Cursor is at or past the oldest — fine to resume.
return lastID
}
// Cursor was trimmed. Emit a gap event so consumers that care can react.
writeGap(w, flusher, lastID, oldest, "retention")
// Return "0" so the next read picks up entries starting from the
// stream's true minimum — including `oldest` itself.
return "0"
}
// writeSSEEntry serialises one stream entry as an SSE event.
//
// `event:` comes from the entry's `type` field (no prefix munging). `data:`
// is a JSON object containing all OTHER fields.
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, e kv.Entry) {
eventType := "message"
data := make(map[string]string, len(e.Values))
for k, v := range e.Values {
if k == "type" {
eventType = v
continue
}
data[k] = v
}
payload, err := json.Marshal(data)
if err != nil {
log.Printf("[SSE] failed to marshal entry %s: %v", e.ID, err)
return
}
// Spec: each event ends with a blank line. id/event/data lines are
// terminated with single \n.
fmt.Fprintf(w, "id: %s\n", e.ID)
fmt.Fprintf(w, "event: %s\n", eventType)
fmt.Fprintf(w, "data: %s\n\n", string(payload))
flusher.Flush()
}
// compareStreamIDs compares two Redis Stream IDs of the form "<ms>-<seq>".
// Returns -1/0/+1. Missing or malformed IDs sort as zero parts.
func compareStreamIDs(a, b string) int {
aMs, aSeq := parseStreamID(a)
bMs, bSeq := parseStreamID(b)
switch {
case aMs < bMs:
return -1
case aMs > bMs:
return 1
case aSeq < bSeq:
return -1
case aSeq > bSeq:
return 1
default:
return 0
}
}
func parseStreamID(id string) (int64, int64) {
parts := strings.SplitN(id, "-", 2)
ms, _ := strconv.ParseInt(parts[0], 10, 64)
var seq int64
if len(parts) == 2 {
seq, _ = strconv.ParseInt(parts[1], 10, 64)
}
return ms, seq
}
// writeGap emits the synthetic `gap` event: the consumer's cursor cannot be
// resumed losslessly and it should take a snapshot (Last-Event-ID: 0).
func writeGap(w http.ResponseWriter, flusher http.Flusher, requested, resumeFrom, reason string) {
payload, _ := json.Marshal(map[string]string{
"requested": requested,
"resumeFrom": resumeFrom,
"reason": reason,
})
fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
flusher.Flush()
}
+118
View File
@@ -0,0 +1,118 @@
package api
import (
"context"
"net/http"
"net/http/httptest"
"strings"
"sync"
"testing"
"time"
)
// syncRecorder is a ResponseRecorder safe to read while the handler writes.
type syncRecorder struct {
*httptest.ResponseRecorder
mu sync.Mutex
}
func (r *syncRecorder) Write(b []byte) (int, error) {
r.mu.Lock()
defer r.mu.Unlock()
return r.ResponseRecorder.Write(b)
}
func (r *syncRecorder) body() string {
r.mu.Lock()
defer r.mu.Unlock()
return r.Body.String()
}
// Last-Event-ID 0-0 on meta:events returns the whole current state — one set
// per record key, only the last carrying an id — then the live tail.
func TestMetaEventsSnapshotThenTail(t *testing.T) {
stor := newTestStorage(t)
s := &Server{storage: stor}
a := testSigner(t, 1)
for i, cid := range []string{"bafyR1", "bafyR2"} {
storeClaims(t, stor, signed(t, a, cid, "title", "T", 1), signed(t, a, cid, "n", string(rune('a'+i)), 1))
}
r1, r2 := stor.ResolveRoot("bafyR1"), stor.ResolveRoot("bafyR2")
ctx, cancel := context.WithCancel(context.Background())
req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
req.Header.Set("Last-Event-ID", "0-0")
rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()}
done := make(chan struct{})
go func() { s.handleEventsMeta(rec, req); close(done) }()
waitFor := func(substr string) {
t.Helper()
deadline := time.Now().Add(5 * time.Second)
for !strings.Contains(rec.body(), substr) {
if time.Now().After(deadline) {
t.Fatalf("timed out waiting for %q in:\n%s", substr, rec.body())
}
time.Sleep(20 * time.Millisecond)
}
}
waitFor(`"key":"file:` + r2 + `/title"`)
storeClaims(t, stor, signed(t, a, "bafyR1", "title", "changed", 2))
waitFor(`event: set`)
time.Sleep(300 * time.Millisecond)
cancel()
<-done
body := rec.body()
keys := []string{"file:" + r1 + "/title", "file:" + r1 + "/n", "file:" + r2 + "/title", "file:" + r2 + "/n",
"file:" + r1 + "/cids/bafyR1", "file:" + r2 + "/cids/bafyR2"}
for _, k := range keys {
if !strings.Contains(body, `"key":"`+k+`"`) {
t.Errorf("snapshot is missing %s", k)
}
}
events := strings.Split(strings.TrimSpace(body), "\n\n")
if len(events) != 7 {
t.Fatalf("want 6 snapshot events + 1 live event, got %d:\n%s", len(events), body)
}
for i, e := range events[:5] {
if strings.Contains(e, "id: ") {
t.Errorf("snapshot event %d carries an id: %q", i, e)
}
}
if !strings.HasPrefix(events[5], "id: ") || !strings.HasPrefix(events[6], "id: ") {
t.Fatalf("the last snapshot event and the live event must carry ids:\n%s", body)
}
if !strings.Contains(events[6], `"key":"file:`+r1+`/title"`) {
t.Fatalf("live tail did not follow the snapshot: %q", events[6])
}
}
// A cursor older than the last cache rebuild gets `gap` (reason rebuild): the
// rebuild published no per-key events, so the consumer must re-snapshot.
func TestMetaEventsGapAfterRebuild(t *testing.T) {
stor := newTestStorage(t)
s := &Server{storage: stor}
storeClaims(t, stor, signed(t, testSigner(t, 1), "bafyA", "title", "T", 1))
if err := stor.Rebuild(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithCancel(context.Background())
req := httptest.NewRequest(http.MethodGet, "/api/events/meta", nil).WithContext(ctx)
req.Header.Set("Last-Event-ID", "1000-0")
rec := &syncRecorder{ResponseRecorder: httptest.NewRecorder()}
done := make(chan struct{})
go func() { s.handleEventsMeta(rec, req); close(done) }()
deadline := time.Now().Add(5 * time.Second)
for !strings.Contains(rec.body(), "event: gap") {
if time.Now().After(deadline) {
t.Fatalf("no gap after rebuild:\n%s", rec.body())
}
time.Sleep(20 * time.Millisecond)
}
cancel()
<-done
if !strings.Contains(rec.body(), `"reason":"rebuild"`) {
t.Fatalf("gap reason: %s", rec.body())
}
}
+166
View File
@@ -0,0 +1,166 @@
package api
import (
"log"
"net/http"
"sync"
"time"
"github.com/metazla/meta-core/internal/identity"
)
// statsSweepTTL is how long a keyspace-walk result stays good. The walk is
// never on the request path (see statsCache), so this only controls how stale
// the library tiles may look, not how slow the endpoint is.
const statsSweepTTL = 60 * time.Second
// fileStatsJSON is the file-backed half of the library: the roots that carry
// filePath+sizeByte+mtimeNano. Gateway / cid-rooted records have no file on
// disk and are counted in Records, not here — the gap between the two numbers
// is exactly "metadata we hold for files we do not have".
type fileStatsJSON struct {
Count int `json:"count"`
TotalSize int64 `json:"totalSize"`
}
// StatsResponse is the body of GET /api/stats — every headline number the
// dashboard shows, in one request.
//
// Deliberately split by cost. Records/RedisKeys/RedisMemory/Identities are
// cheap (an index range count, a cached row count, a file stat) plus a
// directory listing, recomputed on every call. The `redis*` JSON names
// predate the move to SQLite and are kept for dashboard compatibility:
// redisKeys is the number of claim rows (truth), redisMemory both databases' on-disk size. Files and UDLUsers need a keyspace walk — ~0.5s for the file
// tuples and ~3s for the UDL SCAN on an 88k-record box — so they are served
// from a background-refreshed cache and are null until the first sweep lands.
type StatsResponse struct {
Records int64 `json:"records"`
RedisKeys int64 `json:"redisKeys"`
RedisMemory string `json:"redisMemory"`
Identities int `json:"identities"`
Files *fileStatsJSON `json:"files"`
UDLUsers *int `json:"udlUsers"`
// SweptAt is when the cached half was computed (unix ms), 0 if never.
SweptAt int64 `json:"sweptAt"`
// Sweeping reports that a refresh is in flight, so the UI can show the
// cached numbers as provisional instead of pretending they are live.
Sweeping bool `json:"sweeping"`
}
// statsCache holds the expensive half of /api/stats.
//
// Why a cache and not just "compute it": the dashboard polls. A multi-second store walk
// per tick, per open tab, is a self-inflicted outage on a box that is also
// serving playback — the same shape as meta-sort's /api/stats full sweep. So
// the handler NEVER waits on a walk: it returns whatever the last sweep
// produced and kicks off a new one if the old one is stale. `running` makes
// that single-flight, so ten tabs cost one walk.
type statsCache struct {
mu sync.Mutex
files *fileStatsJSON
udlUsers *int
sweptAt time.Time
running bool
}
// snapshot returns the cached values and whether a refresh is in flight.
func (c *statsCache) snapshot() (*fileStatsJSON, *int, time.Time, bool) {
c.mu.Lock()
defer c.mu.Unlock()
return c.files, c.udlUsers, c.sweptAt, c.running
}
// refreshIfStale starts a background sweep unless one is already running or
// the last result is still fresh. Returns true if it started one.
func (c *statsCache) refreshIfStale(sweep func() (*fileStatsJSON, *int)) bool {
c.mu.Lock()
if c.running || time.Since(c.sweptAt) < statsSweepTTL {
c.mu.Unlock()
return false
}
c.running = true
c.mu.Unlock()
go func() {
files, udl := sweep()
c.mu.Lock()
defer c.mu.Unlock()
c.running = false
// A failed sweep leaves the previous numbers in place rather than
// blanking the tiles; sweptAt still advances so a hard-failing walk
// (store down) does not spin once per poll.
c.sweptAt = time.Now()
if files != nil {
c.files = files
}
if udl != nil {
c.udlUsers = udl
}
}()
return true
}
// handleStats handles GET /api/stats.
func (s *Server) handleStats(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
resp := StatsResponse{}
if n, err := s.storage.CountRoots(); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
} else {
resp.Records = n
}
if n, err := s.storage.CountClaims(); err == nil {
resp.RedisKeys = n
}
if mem, err := s.storage.GetMemoryInfo(); err == nil {
resp.RedisMemory = mem
} else {
resp.RedisMemory = "N/A"
}
if list, err := identity.List(s.config.IdentityAccountsDir()); err == nil {
resp.Identities = len(list)
}
s.stats.refreshIfStale(s.sweepStats)
files, udl, sweptAt, sweeping := s.stats.snapshot()
resp.Files = files
resp.UDLUsers = udl
resp.Sweeping = sweeping
if !sweptAt.IsZero() {
resp.SweptAt = sweptAt.UnixMilli()
}
writeJSON(w, http.StatusOK, resp)
}
// sweepStats runs the two keyspace walks behind the cached tiles. Called off
// the request path only. A failing half returns nil so the other half still
// lands.
func (s *Server) sweepStats() (*fileStatsJSON, *int) {
var files *fileStatsJSON
if tuples, err := s.storage.GetTuplesForAllFiles(); err != nil {
log.Printf("[API] stats: file tuple sweep failed: %v", err)
} else {
fs := fileStatsJSON{Count: len(tuples)}
for _, t := range tuples {
fs.TotalSize += t.Size
}
files = &fs
}
var udl *int
if stats, err := s.storage.UDLAllUserStats(); err != nil {
log.Printf("[API] stats: UDL user sweep failed: %v", err)
} else {
n := len(stats)
udl = &n
}
return files, udl
}
+68
View File
@@ -0,0 +1,68 @@
package api
import (
"context"
"fmt"
"path/filepath"
"testing"
"github.com/metazla/meta-core/internal/claim"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/storage"
)
// newTestStorage opens (and starts) a fresh store in the test's temp dir.
func newTestStorage(t *testing.T) *storage.Client {
t.Helper()
dir := t.TempDir()
c := storage.NewClient()
if err := c.Connect(filepath.Join(dir, "meta-core.db"), filepath.Join(dir, "cache.db")); err != nil {
t.Fatalf("open store: %v", err)
}
if err := c.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
t.Fatalf("schema: %v", err)
}
if err := c.Start(); err != nil {
t.Fatalf("start: %v", err)
}
t.Cleanup(func() { _ = c.Close() })
return c
}
func testSigner(t *testing.T, n byte) *identity.Identity {
t.Helper()
id, err := identity.Import(fmt.Sprintf("%02x%062x", n, 1))
if err != nil {
t.Fatal(err)
}
return id
}
// signed builds a claim signed by id; v is string, []byte or nil.
func signed(t *testing.T, id *identity.Identity, cid, key string, v any, ts uint64) claim.Claim {
t.Helper()
cl := claim.Claim{CID: cid, UID: id.UID, Key: key, TS: ts}
switch x := v.(type) {
case string:
cl.Value = claim.Text(x)
case []byte:
cl.Value = claim.Bytes(x)
}
if err := cl.Sign(id); err != nil {
t.Fatal(err)
}
return cl
}
func storeClaims(t *testing.T, c *storage.Client, claims ...claim.Claim) {
t.Helper()
res, err := c.WriteClaims(context.Background(), claims)
if err != nil {
t.Fatal(err)
}
for i, r := range res {
if r.Status != storage.StatusAccepted {
t.Fatalf("claim %d: %s %s", i, r.Status, r.Reason)
}
}
}
+357
View File
@@ -0,0 +1,357 @@
package api
// User Data Layer facade (/api/udl/*). Since meta-core v2.0.0 a user's data are
// ordinary MMUR2 claims signed by the user's key (plaintext for public-social
// keys, ciphertext for private ones); these routes keep their paths for the
// clients (meta-watch, meta-read, meta-listen, the gateway mirror) and speak
// claims in the §3.4 JSON form:
//
// {"cid","uid","key","value": "…" | {"b64": "…"} | null,"ts","sig"}
//
// A write is the same as POST /api/claims; newest ts wins per (uid, cid, key).
import (
"encoding/json"
"math"
"net/http"
"strconv"
"strings"
"github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/claim"
"github.com/metazla/meta-core/internal/storage"
)
// UDLPutResponse answers PUT /api/udl/record.
type UDLPutResponse struct {
Accepted bool `json:"accepted"`
Status storage.ClaimStatus `json:"status"`
}
// UDLPutBatchRequest is PUT /api/udl/records.
type UDLPutBatchRequest struct {
Records []claim.Claim `json:"records"`
}
// UDLPutBatchResult is one entry of the batch response.
type UDLPutBatchResult struct {
Key string `json:"key"`
Accepted bool `json:"accepted"`
Status storage.ClaimStatus `json:"status"`
Error string `json:"error,omitempty"`
}
// UDLPutBatchResponse answers PUT /api/udl/records.
type UDLPutBatchResponse struct {
Results []UDLPutBatchResult `json:"results"`
}
const maxUDLBatch = 1000
// UDLRecordResponse answers GET /api/udl/record. Claim is the stored claim,
// tombstone included (its ts is what a newer write must exceed); Exists is
// false when it is absent or a tombstone.
type UDLRecordResponse struct {
Exists bool `json:"exists"`
Claim *claim.Claim `json:"claim"`
}
// UDLAggregate is one key's public aggregate on a cid.
type UDLAggregate struct {
Key string `json:"key"`
Count int `json:"count"`
Avg *float64 `json:"avg"`
Trues int `json:"trues"`
SampleSize int `json:"sample_size"`
}
func (s *Server) udlReady(w http.ResponseWriter) bool {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return false
}
return true
}
// handleUDLRecordGet handles GET /api/udl/record?uid=&cid=&key=.
func (s *Server) handleUDLRecordGet(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
q := r.URL.Query()
uid, cid, key := q.Get("uid"), q.Get("cid"), q.Get("key")
if uid == "" || cid == "" || key == "" {
writeError(w, http.StatusBadRequest, "uid, cid and key are required")
return
}
cl, ok, err := s.storage.UDLGetClaim(uid, cid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
resp := UDLRecordResponse{}
if ok {
resp.Claim = &cl
resp.Exists = !cl.Value.IsNull()
}
writeJSON(w, http.StatusOK, resp)
}
// handleUDLRecordPut handles PUT /api/udl/record with one claim as the body.
func (s *Server) handleUDLRecordPut(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
var cl claim.Claim
if err := json.NewDecoder(r.Body).Decode(&cl); err != nil {
writeError(w, http.StatusBadRequest, "body must be one claim: "+err.Error())
return
}
results, err := s.storage.WriteClaims(r.Context(), []claim.Claim{cl})
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
res := results[0]
switch res.Status {
case storage.StatusAccepted, storage.StatusUnchanged:
writeJSON(w, http.StatusOK, UDLPutResponse{Accepted: true, Status: res.Status})
case storage.StatusStale:
writeErrorSlug(w, http.StatusConflict, "stale_version", "a claim with an equal or newer ts already exists", true)
default:
writeErrorSlug(w, http.StatusBadRequest, string(res.Status), res.Reason, false)
}
}
// handleUDLRecordsPut handles PUT /api/udl/records {records: [claim, …]}.
func (s *Server) handleUDLRecordsPut(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
var req UDLPutBatchRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body: "+err.Error())
return
}
if len(req.Records) == 0 {
writeError(w, http.StatusBadRequest, "records is required")
return
}
if len(req.Records) > maxUDLBatch {
writeError(w, http.StatusBadRequest, "too many records in one batch")
return
}
results, err := s.storage.WriteClaims(r.Context(), req.Records)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
out := make([]UDLPutBatchResult, len(results))
for i, res := range results {
out[i] = UDLPutBatchResult{
Key: req.Records[i].Key,
Status: res.Status,
Accepted: res.Status == storage.StatusAccepted || res.Status == storage.StatusUnchanged,
Error: res.Reason,
}
}
writeJSON(w, http.StatusOK, UDLPutBatchResponse{Results: out})
}
func (s *Server) writeUDLEntries(w http.ResponseWriter, uid, cid, key string) {
entries, err := s.storage.UDLClaimsForUser(uid, cid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if entries == nil {
entries = []claim.Claim{}
}
writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries})
}
// handleUDLUserKey handles GET /api/udl/user/{uid}/key/{key}: every live claim
// of uid on key (My List, Continue Watching).
func (s *Server) handleUDLUserKey(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
vars := mux.Vars(r)
if vars["uid"] == "" || vars["key"] == "" {
writeError(w, http.StatusBadRequest, "uid and key are required")
return
}
s.writeUDLEntries(w, vars["uid"], "", vars["key"])
}
// handleUDLUserCid handles GET /api/udl/user/{uid}/cid/{cid}.
func (s *Server) handleUDLUserCid(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
vars := mux.Vars(r)
if vars["uid"] == "" || vars["cid"] == "" {
writeError(w, http.StatusBadRequest, "uid and cid are required")
return
}
s.writeUDLEntries(w, vars["uid"], vars["cid"], "")
}
// handleUDLUserAll handles GET /api/udl/user/{uid}/all?after=&limit= — the
// whole profile, paged on cid boundaries (recovery / export).
func (s *Server) handleUDLUserAll(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
uid := mux.Vars(r)["uid"]
if uid == "" {
writeError(w, http.StatusBadRequest, "uid is required")
return
}
limit := maxUDLBatch
if v := r.URL.Query().Get("limit"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n <= 0 {
writeError(w, http.StatusBadRequest, "limit must be a positive integer")
return
}
limit = min(n, limit)
}
entries, nextCid, err := s.storage.UDLAllForUser(uid, r.URL.Query().Get("after"), limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if entries == nil {
entries = []claim.Claim{}
}
writeJSON(w, http.StatusOK, map[string]interface{}{"entries": entries, "nextCid": nextCid})
}
// handleUDLUserCount handles GET /api/udl/user/{uid}/count.
func (s *Server) handleUDLUserCount(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
uid := mux.Vars(r)["uid"]
if uid == "" {
writeError(w, http.StatusBadRequest, "uid is required")
return
}
n, err := s.storage.UDLCountUserCids(uid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"cids": n})
}
// handleUDLCidUsers handles GET /api/udl/cid/{cid}/users: signers holding live
// claims on the cid.
func (s *Server) handleUDLCidUsers(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
users, err := s.storage.UDLCidUsers(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{"users": users})
}
// handleUDLAggregate handles GET /api/udl/cid/{cid}/aggregate?keys=like,userRating.
func (s *Server) handleUDLAggregate(w http.ResponseWriter, r *http.Request) {
if !s.udlReady(w) {
return
}
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
keysParam := r.URL.Query().Get("keys")
if keysParam == "" {
keysParam = "like,userRating"
}
aggregates := make([]UDLAggregate, 0)
for _, key := range strings.Split(keysParam, ",") {
key = strings.TrimSpace(key)
if key == "" {
continue
}
values, err := s.storage.UDLPublicValues(cid, key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
aggregates = append(aggregates, aggregatePublicValues(key, values))
}
writeJSON(w, http.StatusOK, map[string]interface{}{"aggregates": aggregates})
}
// aggregatePublicValues folds raw JSON scalar values into an UDLAggregate:
// counts booleans as trues and averages numeric values.
func aggregatePublicValues(key string, values []string) UDLAggregate {
agg := UDLAggregate{Key: key, SampleSize: len(values), Count: len(values)}
var sum float64
var numeric int
for _, raw := range values {
var v interface{}
if err := json.Unmarshal([]byte(raw), &v); err != nil {
continue
}
switch t := v.(type) {
case bool:
if t {
agg.Trues++
}
case float64:
sum += t
numeric++
}
}
if numeric > 0 {
avg := math.Round((sum/float64(numeric))*100) / 100
agg.Avg = &avg
}
return agg
}
// udlUserStatsResponse is the shape of GET /api/udl/users/stats.
type udlUserStatsResponse struct {
// Users maps uid -> counts. Accounts holding nothing are absent rather than
// present with zeroes; a caller rendering a list treats "missing" as 0.
Users map[string]storage.UDLUserStats `json:"users"`
}
// handleUDLUserStats handles GET /api/udl/users/stats — how much every account
// holds, in one keyspace walk.
//
// Bulk rather than per-uid because the only caller is the accounts page, which
// needs a number for every row: the counts come from a SCAN (there is no
// per-user cid index — see storage/udl_admin.go), and MATCH filters without
// indexing, so N per-uid calls would walk the whole keyspace N times.
//
// Deliberately not on the critical path of anything. It is an admin read whose
// cost grows with the keyspace, so callers should fetch it after rendering,
// never before.
func (s *Server) handleUDLUserStats(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeErrorSlug(w, http.StatusServiceUnavailable, "storage_unavailable",
"storage not connected", true)
return
}
stats, err := s.storage.UDLAllUserStats()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, udlUserStatsResponse{Users: stats})
}
+112
View File
@@ -0,0 +1,112 @@
package api
import (
"encoding/json"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/metazla/meta-core/internal/claim"
)
func newUDLTestServer(t *testing.T) *Server {
t.Helper()
return &Server{storage: newTestStorage(t)}
}
func udlGet(t *testing.T, s *Server, uid, cid, key string) UDLRecordResponse {
t.Helper()
req := httptest.NewRequest(http.MethodGet, "/api/udl/record?uid="+uid+"&cid="+cid+"&key="+key, nil)
rr := httptest.NewRecorder()
s.handleUDLRecordGet(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("GET record = %d %s", rr.Code, rr.Body)
}
var resp UDLRecordResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
return resp
}
func udlPutBatch(t *testing.T, s *Server, claims ...claim.Claim) UDLPutBatchResponse {
t.Helper()
body, _ := json.Marshal(UDLPutBatchRequest{Records: claims})
req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(string(body)))
rr := httptest.NewRecorder()
s.handleUDLRecordsPut(rr, req)
if rr.Code != http.StatusOK {
t.Fatalf("PUT records = %d %s", rr.Code, rr.Body)
}
var resp UDLPutBatchResponse
if err := json.Unmarshal(rr.Body.Bytes(), &resp); err != nil {
t.Fatal(err)
}
return resp
}
// The facade stores and returns MMUR2 claims: public values in clear,
// private ones as ciphertext, tombstones with their ts.
func TestUDLFacadeRoundTrip(t *testing.T) {
s := newUDLTestServer(t)
me := testSigner(t, 7)
res := udlPutBatch(t, s,
signed(t, me, "self", "profile:name", "Ada Lovelace", 100),
signed(t, me, "bagcsaaa", "like", "true", 100),
signed(t, me, "bagcsaaa", "seek", []byte("ciphertext"), 100),
)
for _, r := range res.Results {
if !r.Accepted {
t.Fatalf("not accepted: %+v", r)
}
}
got := udlGet(t, s, me.UID, "self", "profile:name")
if !got.Exists || got.Claim == nil || got.Claim.Value.Text != "Ada Lovelace" || got.Claim.Verify() != nil {
t.Fatalf("profile = %+v", got)
}
if seek := udlGet(t, s, me.UID, "bagcsaaa", "seek"); seek.Claim == nil || seek.Claim.Value.Kind != claim.KindBytes {
t.Fatalf("seek = %+v", seek)
}
if miss := udlGet(t, s, me.UID, "bagcsaaa", "nope"); miss.Exists || miss.Claim != nil {
t.Fatalf("missing cell = %+v", miss)
}
// Retraction: stored, returned with its ts, no longer "exists".
udlPutBatch(t, s, signed(t, me, "bagcsaaa", "like", nil, 200))
if tomb := udlGet(t, s, me.UID, "bagcsaaa", "like"); tomb.Exists || tomb.Claim == nil || tomb.Claim.TS != 200 {
t.Fatalf("tombstone = %+v", tomb)
}
rr := httptest.NewRecorder()
req := httptest.NewRequest(http.MethodGet, "/api/udl/cid/self/aggregate?keys=profile:name", nil)
req = withVars(req, map[string]string{"cid": "self"})
s.handleUDLAggregate(rr, req)
if !strings.Contains(rr.Body.String(), `"sample_size":1`) {
t.Fatalf("aggregate = %s", rr.Body)
}
}
// A stale claim in a batch is reported in its slot without failing the rest.
func TestUDLRecordsPut_StaleSlotDoesNotFailTheBatch(t *testing.T) {
s := newUDLTestServer(t)
me := testSigner(t, 7)
udlPutBatch(t, s, signed(t, me, "bagcsaaa", "seen:s01e01", "true", 400))
res := udlPutBatch(t, s,
signed(t, me, "bagcsaaa", "seen:s01e01", "false", 300),
signed(t, me, "bagcsaaa", "seen:s01e02", "true", 300),
)
if res.Results[0].Accepted || res.Results[0].Status != "stale" || !res.Results[1].Accepted {
t.Fatalf("results = %+v", res.Results)
}
}
func TestUDLRecordsPut_RejectsEmptyBatch(t *testing.T) {
s := newUDLTestServer(t)
req := httptest.NewRequest(http.MethodPut, "/api/udl/records", strings.NewReader(`{"records":[]}`))
rr := httptest.NewRecorder()
s.handleUDLRecordsPut(rr, req)
if rr.Code != http.StatusBadRequest {
t.Fatalf("empty batch = %d", rr.Code)
}
}
+92
View File
@@ -0,0 +1,92 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"fmt"
"strings"
"unicode/utf8"
)
// DecodeCard recovers the (source, id) pair a card locator (CodeCard, 0x1007)
// wraps. Wire form:
//
// 0x01 ‖ varint(0x1007) ‖ 0x00 (identity mh) ‖ varint(len(digest)) ‖ digest
// digest = varint(len(source)) ‖ source ‖ id (both UTF-8)
//
// Decode stops at the multihash code because ranking needs nothing past it;
// reading *which work* a card addresses needs the digest too, so this is the
// one place in Go that walks the whole thing.
//
// ⚠ Mirror of meta-read `card_cid::decode` and of meta-feeder-sdk
// `compute_card_cid`. The vectors in card_test.go are pinned literals copied
// from meta-read, not round-trips, so a drift on either side fails a test
// instead of silently reading a different work out of the same CID.
//
// Identity multihash only: a 0x1007 CID carrying any other multihash is not
// something anyone mints, and reading its digest as (source, id) would be
// reading hash bytes as text. Bytes past the declared digest length are
// ignored, as meta-read does.
func DecodeCard(cidStr string) (source, id string, err error) {
if len(cidStr) < 2 || cidStr[0] != 'b' {
return "", "", fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
}
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
if err != nil {
return "", "", fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
}
next := func(buf []byte, what string) (uint64, []byte, error) {
v, n := binary.Uvarint(buf)
if n <= 0 {
return 0, nil, fmt.Errorf("cid %q: bad %s varint", cidStr, what)
}
return v, buf[n:], nil
}
version, raw, err := next(raw, "version")
if err != nil {
return "", "", err
}
if version != 1 {
return "", "", fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
}
codec, raw, err := next(raw, "codec")
if err != nil {
return "", "", err
}
if codec != CodeCard {
return "", "", fmt.Errorf("cid %q: codec 0x%x is not a card locator", cidStr, codec)
}
mh, raw, err := next(raw, "multihash-code")
if err != nil {
return "", "", err
}
if mh != 0 {
return "", "", fmt.Errorf("cid %q: card locator with non-identity multihash 0x%x", cidStr, mh)
}
digestLen, raw, err := next(raw, "digest-length")
if err != nil {
return "", "", err
}
if digestLen > uint64(len(raw)) {
return "", "", fmt.Errorf("cid %q: digest truncated", cidStr)
}
digest := raw[:digestLen]
sourceLen, rest, err := next(digest, "source-length")
if err != nil {
return "", "", err
}
if sourceLen > uint64(len(rest)) {
return "", "", fmt.Errorf("cid %q: source truncated", cidStr)
}
source, id = string(rest[:sourceLen]), string(rest[sourceLen:])
if source == "" || id == "" {
return "", "", fmt.Errorf("cid %q: empty source or id", cidStr)
}
if !utf8.ValidString(source) || !utf8.ValidString(id) {
return "", "", fmt.Errorf("cid %q: source or id is not UTF-8", cidStr)
}
return source, id, nil
}
+71
View File
@@ -0,0 +1,71 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"strings"
"testing"
)
// Pinned from meta-read `src/card_cid.rs` / meta-feeder-sdk `compute_card_cid`
// — literals, not round-trips (see DecodeCard).
const (
goldenAnilistCard = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy"
goldenMangadexCard = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
)
// wireCID builds a CIDv1 from raw parts so the negative cases below can vary
// one field at a time.
func wireCID(codec, mh uint64, digest []byte) string {
buf := []byte{0x01}
buf = binary.AppendUvarint(buf, codec)
buf = binary.AppendUvarint(buf, mh)
buf = binary.AppendUvarint(buf, uint64(len(digest)))
buf = append(buf, digest...)
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf))
}
func cardDigest(source, id string) []byte {
d := binary.AppendUvarint(nil, uint64(len(source)))
d = append(d, source...)
return append(d, id...)
}
func TestDecodeCard_GoldenVectors(t *testing.T) {
for _, tc := range []struct {
cid, source, id string
}{
{goldenAnilistCard, "anilist", "manga:105778"},
{goldenMangadexCard, "mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"},
} {
source, id, err := DecodeCard(tc.cid)
if err != nil {
t.Fatalf("DecodeCard(%s): %v", tc.cid, err)
}
if source != tc.source || id != tc.id {
t.Errorf("DecodeCard(%s) = (%q, %q), want (%q, %q)", tc.cid, source, id, tc.source, tc.id)
}
// The test encoder must agree with the pinned literal, or the negative
// cases below are testing a different wire format.
if got := wireCID(CodeCard, 0, cardDigest(tc.source, tc.id)); got != tc.cid {
t.Errorf("wireCID(%q, %q) = %s, want %s", tc.source, tc.id, got, tc.cid)
}
}
}
func TestDecodeCard_RejectsEverythingElse(t *testing.T) {
for name, s := range map[string]string{
"url locator codec": wireCID(CodeURL, 0, cardDigest("anilist", "1")),
"non-identity multihash": wireCID(CodeCard, CodeSha256, cardDigest("anilist", "1")),
"empty id": wireCID(CodeCard, 0, cardDigest("anilist", "")),
"source past digest": wireCID(CodeCard, 0, []byte{0x20, 'a'}),
"not a CID": "title:berserk",
"CIDv0": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
"truncated": goldenAnilistCard[:12],
"empty": "",
} {
if source, id, err := DecodeCard(s); err == nil {
t.Errorf("%s: DecodeCard(%q) = (%q, %q), want error", name, s, source, id)
}
}
}

Some files were not shown because too many files have changed in this diff Show More