Compare commits

...
11 Commits
Author SHA1 Message Date
Worph d268bb83e6 search optimisation
Build and Push Docker Image / build (push) Failing after 13s
2026-07-10 22:56:59 +02:00
Worph 075d6f9bcb cleanup logs 2026-07-10 13:25:16 +02:00
Worph f1cd94b17e bugfix advertise url
Build and Push Docker Image / build (push) Failing after 13s
2026-06-14 18:47:28 +02:00
Pierre 52f5b5cedc multi identity support for metacore 2026-06-06 18:44:52 +02:00
Pierre f8a10091b8 remove in metadata cid_* to put in the cids/ set 2026-05-31 11:41:32 +02:00
Pierre 94ad0c240b bugfix and new internal API 2026-05-25 23:47:30 +02:00
Pierre 415595c1c6 new meta-core api, now redis is not exposed as an "API" 2026-05-24 17:43:17 +02:00
Pierre aa45f4f32e improve editor and add type detection 2026-05-14 23:31:07 +02:00
Pierre 3d65e7e951 doc improvement 2026-05-10 19:47:05 +02:00
Pierre bd833b7be4 improved kv metacore 2026-04-27 10:31:11 +02:00
Pierre 98da535335 improve editor for KV 2026-04-26 15:28:50 +02:00
109 changed files with 13354 additions and 6459 deletions
+2
View File
@@ -23,3 +23,5 @@ go.work
dashboard/node_modules/* dashboard/node_modules/*
dist/* dist/*
dashboard/dist/* dashboard/dist/*
editor/dist/
editor/node_modules/
+18 -9
View File
@@ -7,7 +7,7 @@
# - meta-core Go binary (HTTP API, Mount API, Watcher) # - meta-core Go binary (HTTP API, Mount API, Watcher)
# - nginx (dashboard, WebDAV, API proxy) # - nginx (dashboard, WebDAV, API proxy)
# - Redis (managed by supervisord on leader) # - Redis (managed by supervisord on leader)
# - rclone (remote mount daemon) # - rclone (single mount executor — SMB goes through rclone's :smb: backend)
# - supervisord (process management - runs on leader only) # - supervisord (process management - runs on leader only)
# #
# Architecture: # Architecture:
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts" LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
LABEL org.opencontainers.image.licenses=MIT LABEL org.opencontainers.image.licenses=MIT
# Install runtime dependencies # Pinned rclone release. Alpine's package can lag the latest by several
# minor versions and we depend on VFS cache fixes shipped post-1.65.
ARG RCLONE_VERSION=1.73.5
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
# all remote mounts go through rclone now.
RUN apk add --no-cache \ RUN apk add --no-cache \
# Core services
redis \ redis \
nginx \ nginx \
nginx-mod-http-dav-ext \ nginx-mod-http-dav-ext \
supervisor \ supervisor \
rclone \
fuse3 \ fuse3 \
# Mount utilities
nfs-utils \
cifs-utils \
# Tools
bash \ bash \
curl \ curl \
python3 \ python3 \
ca-certificates \ ca-certificates \
apache2-utils \ apache2-utils \
# Process utilities
procps procps
# Install rclone from the official binary release (pinned).
RUN set -eux; \
apk add --no-cache --virtual .rclone-deps unzip; \
cd /tmp; \
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
unzip -q rclone.zip; \
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
apk del .rclone-deps; \
/usr/local/bin/rclone version | head -2
# Copy Go binary from builder # Copy Go binary from builder
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
RUN chmod +x /usr/local/bin/meta-core RUN chmod +x /usr/local/bin/meta-core
+14
View File
@@ -64,10 +64,24 @@ clean:
rm -f coverage.out coverage.html rm -f coverage.out coverage.html
# Build Docker image # Build Docker image
#
# We tag THREE names:
# - meta-core:$(VERSION) — the canonical versioned tag
# - meta-core:latest — convenience for `docker run meta-core`
# - meta-core:local — the tag every sub-stack docker-compose.yml uses
# (meta-share federation, meta-gateway dev stacks).
# Without this, `make docker` produces an image
# the dev stacks can't see by name, and previously
# -tagged stale `:local` images linger, silently
# breaking downstream API fields (e.g. /urls
# missing webdavUrlInternal → meta-gateway's
# poster auto-store fails). See README.md →
# Quick start → Build for the audit recipe.
docker: docker:
@echo "Building Docker image..." @echo "Building Docker image..."
docker build -t $(DOCKER_IMAGE):$(VERSION) . docker build -t $(DOCKER_IMAGE):$(VERSION) .
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local
# Run locally (for development) # Run locally (for development)
run: build run: build
+277 -423
View File
@@ -1,520 +1,374 @@
# meta-core # meta-core
A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services. Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
## Overview ## Overview
meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul. meta-core runs as its own container (`metacore-app`). Other MetaMesh services
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
is not exposed outside the container — all metadata I/O is mediated by the
HTTP API.
### Key Capabilities ### Key capabilities
| Feature | Description | | Feature | Description |
|---------|-------------| |---------|-------------|
| **Leader Election** | flock-based leader election (Go binary only runs as leader) | | **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash | | **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
| **HTTP API** | Language-agnostic REST interface for metadata and service discovery | | **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
| **Service Discovery** | File-based registry with heartbeats and stale detection | | **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations | | **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
| **File Watching** | Directory scanning, MidHash256 computation, event dispatch | | **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
| **Mount Management** | SMB/rclone configuration, health monitoring | | **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
| **WebDAV Server** | File access with nginx proxy_cache | | **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
| **Event Publishing** | Redis keyspace notifications to meta:events stream | | **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
### Design Characteristics ### Design characteristics
- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes) - **Single-leader** — flock on a shared volume; no external consensus.
- **Automatic failover** - Lock releases on process death, followers detect and re-elect - **HTTP-only externally** — Redis is bound to the container, never published.
- **Thread-safe** - RWMutex protection on all shared state - **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
- **Graceful shutdown** - Signal handling with reverse startup order on-disk Redis layout predates the current build (alpha clean-wipe policy;
- **Static binary** - CGO disabled, single binary deployment see `docs/uuid-rooted-metadata.md`).
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
image with `redis-server`, `rclone`, nginx, and supervisord.
## Architecture ## Architecture
meta-core runs as a **standalone container** that other services connect to via HTTP API:
``` ```
┌───────────────────────────────────────────────────────────────────────┐ ┌──────────────────────────────────────────────────────────────────────────┐
│ meta-core container (standalone) │ │ metacore-app (standalone container) │
│ │ │ │
│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │ │ supervisord │
│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │ │ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │ │ │ writes kv-leader.info, exec supervisord) │
│ │ │ │ │ │ │ │ /health, /services │ │ │ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │ │ ├─ nginx (TLS termination, proxy_cache for /webdav) │
│ │ │ ├─ rclone rcd (mount daemon, RC API) │
│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │ │ ├─ mount-watcher.sh (rclone mount lifecycle) │
└──────────────────────────────────────────────┬────────────────────────┘ │ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
└──────────────────────────────────────────────────────────────────────────┘
│ │
┌──────────────────────────────────────┴──────────────────────────┐ ┌──────────────────────┴──────────────────────┐
│ /meta-core (shared volume) │ │ /meta-core (shared volume) │
│ ├── locks/kv-leader.info (leader URLs) │ │ locks/kv-leader.{lock,info} │
│ ├── db/redis/ (Redis data) │ │ db/redis/ │
│ ├── services/ (hostname-based .json) │ │ services/*.json │
│ └── mounts/ (mount configurations) │ │ mounts/{mounts.json, errors/} │
└─────────────────────────────────────────────────────────────────┘ │ watchers.json │
▲ ▲ ▲ │ cache/ (nginx proxy_cache) │
┌────────────┘ │ └────────────┐ └─────────────────────────────────────────────┘
▲
┌──────────────────────────────┼────────────────────────────────┐
│ │ │ │ │ │
┌───────────────┐ ┌───────────────┐ ┌───────────────┐ ┌─────────────┐ ┌─────────────┐ ┌─────────────┐
│ meta-sort │ │ meta-fuse │ │ meta-stremio │ │ meta-sort │ │ meta-fuse │ │ meta-stremio│
│ (reads info, │ │ (reads info, │ │ (reads info, │ │ │ │ │ │ │
│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │ │ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │
└───────────────┘ └───────────────┘ └───────────────┘ │ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│
└─────────────┘ └─────────────┘ └─────────────┘
``` ```
## Quick Start ## Quick start
### Build ### Build
```bash ```bash
# Build binary make build # build static binary to bin/meta-core
make build make docker # build container image as meta-core:1.0.0 + :latest + :local
make test # run Go unit tests
# Build Docker image
make docker
# Run tests
make test
``` ```
### Run `make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
`:local` tag is the one every sub-stack docker-compose.yml references
(meta-share 10-peer federation, meta-gateway dev stacks); without it the
sub-stacks can't see a freshly-built image by name.
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
> image, but **already-running containers stay on the old image SHA** —
> `docker compose up -d` won't recreate them unless the compose file
> itself changed. After a meta-core source change, force-recreate the
> downstream containers so they pick up the new binary:
>
> ```bash
> make docker
> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
> up -d --force-recreate metagateway-generic-core
> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
> up -d --force-recreate metagateway-torznab-core
> docker compose -f packages/meta-share/dev/docker-compose.yml \
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
> ```
>
> The symptom of a missed recreate is missing API fields on `/urls`
> (e.g. `webdavUrlInternal` absent → meta-gateway's
> `store_poster_for_record` silently fails and torznab posters never
> land). Audit recipe:
>
> ```bash
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
> ```
### Dev container
In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`:
```bash ```bash
# With environment variables cd dev && ./scripts/start.sh
META_CORE_PATH=/meta-core \
FILES_PATH=/files \
SERVICE_NAME=meta-sort \
META_CORE_HTTP_PORT=9000 \
./bin/meta-core
``` ```
### Docker The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend
HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
```bash
docker build -t meta-core .
docker run -v meta-core:/meta-core -v files:/files meta-core
```
## Configuration ## Configuration
| Environment Variable | Default | Description | Environment variables consumed by `internal/config/config.go`:
|---------------------|---------|-------------|
| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db |
| `FILES_PATH` | `/files` | Shared volume for media files |
| `SERVICE_NAME` | `meta-core` | Service identifier |
| `SERVICE_VERSION` | `1.0.0` | Service version |
| `API_PORT` | `8180` | Main service HTTP port |
| `BASE_URL` | - | Stable service URL |
| `REDIS_PORT` | `6379` | Redis port (leader only) |
| `META_CORE_HTTP_PORT` | `9000` | HTTP API port |
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval |
| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold |
| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning |
## API Reference | Variable | Default | Description |
|---|---|---|
| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
| `FILES_PATH` | `/files` | Files volume root. |
| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
| `REDIS_PORT` | `6379` | Local Redis port. |
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind. |
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
### Health & Status Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
retry interval (`ELECTION_RETRY_SECS`, default `5`).
## API reference
All endpoints below are served by `internal/api/server.go`. Examples assume
the in-container Go port `:9000`; from the host use
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
(debug-direct).
### Health, leader, URLs
```bash ```bash
# Health check GET /health # storage + role
curl http://localhost:9000/health GET /status # version, uptime, etc.
# {"status":"ok","redis":true,"timestamp":"..."} GET /leader # current leader info (hostname, PID, timestamps)
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
# Detailed status GET /services # registered services + heartbeats
curl http://localhost:9000/status GET /api/services # alias for /services
# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...} GET /services/{name} # one service
GET /services/cleanup/stats
# Current leader
curl http://localhost:9000/leader
# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...}
``` ```
### Metadata Operations ### Metadata — primary surface
These are the routes other services use. `{hash}` may be a UUID root or
any registered CID alias; CID aliases resolve to the underlying UUID
before reads/writes (`internal/storage/cid_resolution.go`).
```bash ```bash
# List all file hashes GET /meta # list of root IDs
curl http://localhost:9000/meta GET /meta/{hash} # full document (flat keys → nested JSON)
# {"hashIds":["midhash256:abc123",...],"count":42} PUT /meta/{hash} # replace document
PATCH /meta/{hash} # merge into document
# Get metadata for a file DELETE /meta/{hash} # delete root + all CID aliases
curl http://localhost:9000/meta/{hash} GET /meta/{hash}/{key...} # single property (key may contain "/")
# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}} PUT /meta/{hash}/{key...} # set single property
DELETE /meta/{hash}/{key...} # delete single property
# Update metadata POST /meta/{hash}/_add/{key...} # add value to set-valued property
curl -X PUT http://localhost:9000/meta/{hash} \
-H "Content-Type: application/json" \
-d '{"title":"New Title"}'
# Delete metadata
curl -X DELETE http://localhost:9000/meta/{hash}
``` ```
### Data Operations CID-addressed access (public; auth-bypassed in the Caddy perimeter):
```bash ```bash
# Get file path GET /api/meta/{cid} # resolve CID → metadata document
curl http://localhost:9000/data/{hash}/path GET /api/file/{cid}/info # CID resolution metadata
# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true} GET /file/{cid} # serve file bytes (range-aware)
POST /file/cid # compute CID for an uploaded file
# Check if file exists HEAD /data/{hash} # existence + size
curl -I http://localhost:9000/data/{hash} GET /data/{hash}/path # absolute path on /files
# HTTP/1.1 200 OK (or 404)
``` ```
### File Operations (by CID) (There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or
the `/file/{cid}` reverse-lookup path.)
### Editor / KV browser surface
```bash ```bash
# Serve file by CID (looks up poster/backdrop CIDs in metadata) GET /api/metadata/hash-ids
curl http://localhost:9000/file/{cid} --output poster.jpg GET /api/metadata/list
# Returns raw file bytes with appropriate Content-Type header POST /api/metadata/search
POST /api/metadata/batch
POST /api/metadata/clear
GET /api/metadata/{hashId}
PUT /api/metadata/{hashId}
DELETE /api/metadata/{hashId}
GET /api/metadata/{hashId}/property
PUT /api/metadata/{hashId}/property
# Example: fetch a poster image GET /api/kv/info
curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4 GET /api/kv/keys
# Content-Type: image/jpeg GET /api/kv/tree
# [binary image data] GET /api/kv/search
GET /api/kv/find
GET /api/kv/value
PUT /api/kv/value
DELETE /api/kv/value
GET /api/kv/key/{key...}
GET /api/schema
POST /api/schema/rescan
GET /api/snapshot/export
POST /api/snapshot/import
POST /api/snapshot/wipe
``` ```
The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval. ### SSE event streams
### Service Discovery The HTTP-mirror of the Redis Streams. External services never read Redis
directly — they subscribe here with `Last-Event-ID` for resume. See
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
contract (one event per stream entry, opaque IDs, heartbeats, gap
signalling).
```bash ```bash
# List all services GET /api/events/files # file:events stream (watcher events)
curl http://localhost:9000/services GET /api/events/meta # meta:events stream (metadata mutations)
# {"services":[{"name":"meta-sort",...}],"count":3}
# Get specific service
curl http://localhost:9000/services/meta-sort
# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...}
``` ```
### Mount Management ### Mounts
```bash ```bash
# List all mounts GET /api/mounts
curl http://localhost:9000/api/mounts/list POST /api/mounts
# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]} GET /api/mounts/{id}
PUT /api/mounts/{id} # also PATCH
# Add a new mount DELETE /api/mounts/{id}
curl -X POST http://localhost:9000/api/mounts/add \ POST /api/mounts/{id}/mount
-H "Content-Type: application/json" \ POST /api/mounts/{id}/unmount
-d '{"type":"smb","path":"/files/corn/nas","config":{...}}' POST /api/mounts/{id}/safe-unmount
POST /api/mounts/{id}/scan
# Remove a mount GET /api/mounts/rclone/remotes
curl -X DELETE http://localhost:9000/api/mounts/smb-nas
``` ```
### File Watching ### Watchers
```bash ```bash
# Get scan status GET /api/watchers
curl http://localhost:9000/api/scan/status POST /api/watchers
# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847} GET /api/watchers/{id}
PUT /api/watchers/{id}
# Trigger rescan DELETE /api/watchers/{id}
curl -X POST http://localhost:9000/api/scan/trigger POST /api/watchers/{id}/scan
POST /api/watchers/{id}/reset
# Stream file events (SSE) POST /api/watchers/scan-all
curl http://localhost:9000/api/events/stream POST /api/watchers/reset-all
# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."}
``` ```
### KV Browser The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
return a deprecation pointer at the `/api/watchers/*` routes
(`internal/watcher/handlers.go:32-33`).
### Admin / WebDAV
```bash ```bash
# Storage statistics POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
curl http://localhost:9000/api/kv/info GET/PUT/DELETE /webdav/... # mounted on /files
# {"totalKeys":50000,"memoryUsed":"256MB",...}
# List keys with cursor pagination
curl "http://localhost:9000/api/kv/keys?cursor=0&count=100"
# Get specific key value
curl http://localhost:9000/api/kv/key/file:bafkr.../title
``` ```
### Metadata Editor No `/metrics` endpoint is exposed; observability is via container logs and
the `/health` / `/status` JSON.
```bash ## Leader gate
# Get all hash IDs
curl http://localhost:9000/api/metadata/hash-ids
# {"hashIds":["bafkr...","bafks..."]}
# Paginated metadata listing Leader election is a bash flock loop in `docker/leader-election.sh`, not a
curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50" Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
script writes the leader API URL to `kv-leader.info` and `exec`s
supervisord. When the process tree dies, the kernel releases the lock and
any standby instance can take over. There is no follower path inside the
Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
# Search metadata `kv-leader.info` is a single plaintext line containing the API base URL:
curl -X POST http://localhost:9000/api/metadata/search \
-H "Content-Type: application/json" \
-d '{"query":"inception","fields":["title"]}'
# Batch update
curl -X POST http://localhost:9000/api/metadata/batch \
-H "Content-Type: application/json" \
-d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}'
# Clear all metadata
curl -X POST http://localhost:9000/api/metadata/clear
```
### WebDAV Access
```bash
# List directory (returns JSON)
curl http://localhost:9000/webdav/
# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}]
# Get file
curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv
# Upload file
curl -X PUT http://localhost:9000/webdav/plugin/output.txt \
-H "Content-Type: application/octet-stream" \
--data-binary @output.txt
# Create directory
curl -X MKCOL http://localhost:9000/webdav/newdir/
# Delete file/directory
curl -X DELETE http://localhost:9000/webdav/oldfile.txt
```
## Leader Election
meta-core uses POSIX file locking (flock) for distributed leader election:
1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock`
2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info`
3. Losers become **followers**, read leader info, connect to leader's Redis
4. Lock automatically releases when process dies (no stale locks)
### Election Flow
```
Instance 1 Instance 2 Instance 3
│ │ │
├── Try flock ──────────┼───────────────────────┤
│ │ │
[WIN] [LOSE] [LOSE]
LEADER FOLLOWER FOLLOWER
│ │ │
Start Redis Read leader.info Read leader.info
│ │ │
Write leader.info Connect to Redis Connect to Redis
│ │ │
[Health loop 5s] [Health loop 5s] [Health loop 5s]
- Update timestamp - Re-read leader - Re-read leader
- Check Redis alive - Detect changes - Detect changes
```
### Role Callbacks
The election system provides callbacks for role transitions:
```go
election.OnBecomeLeader(func(info LeaderLockInfo) {
// Start Redis, initialize resources
})
election.OnBecomeFollower(func(info LeaderLockInfo) {
// Connect to leader's Redis
})
election.OnLeaderLost(func() {
// Handle leader failure, attempt re-election
})
```
### Lock Info Format
The `kv-leader.info` file contains a plain text API URL:
``` ```
http://10.0.1.50:9000 http://10.0.1.50:9000
``` ```
Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.). Other services read this file, then call `GET /urls` for full discovery.
## Redis Management The dev compose runs a single `metacore-app`; the flock primitive is
retained as a correctness anchor (one process owns Redis and the
writable state under `/meta-core`), not as live failover.
The leader is responsible for spawning and managing the Redis server. ## Metadata storage shape (one-line summary)
### Redis Configuration Roots are opaque UUIDv7 strings; every property is a separate Redis
STRING at `file:<uuid>/<property>`; every known CID for a file is a
reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
The authoritative registry of field semantics and value formats lives in
the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
```bash ## Internal packages
redis-server --port 6379 --bind 0.0.0.0 \
--dir /meta-core/db/redis \
--appendonly yes --appendfilename appendonly.aof \
--dbfilename dump.rdb --save 60 1 \
--loglevel warning
```
### Persistence Strategy
| Method | Configuration | Purpose |
|--------|--------------|---------|
| **AOF** | `appendonly yes` | Write-ahead log for durability |
| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed |
### Failure Handling
- **Redis crash**: Auto-detected via health check, automatically restarted
- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs
- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed
## Metadata Storage
meta-core uses a flat key-value schema in Redis for storing file metadata.
### Storage Format
```
Key: /file/{hashId}/{property}
Value: string
Examples:
/file/midhash256:abc123/title → "Movie Title"
/file/midhash256:abc123/year → "2024"
/file/midhash256:abc123/genres → "action,drama"
/file/midhash256:abc123/filePath → "movies/Movie.mkv"
```
### Storage Client Features
| Feature | Details |
|---------|---------|
| Connection Pool | 10 connections |
| Timeouts | 5s dial, 30s read/write |
| Batch Operations | Pipelined writes for performance |
| Key Scanning | SCAN-based iteration (non-blocking) |
| CID Lookup | Find file paths by poster/backdrop CID |
## Service Discovery
Services register by writing JSON to `/meta-core/services/{name}.json`.
### Registration Format
```json
{
"name": "meta-sort",
"version": "2.0.0",
"api": "http://10.0.1.50:8180",
"status": "running",
"pid": 12345,
"hostname": "meta-sort-dev",
"startedAt": "2024-01-01T00:00:00Z",
"lastHeartbeat": "2024-01-01T00:01:00Z",
"capabilities": ["meta-core"],
"endpoints": {
"health": "http://10.0.1.50:9000/health",
"meta": "http://10.0.1.50:9000/meta"
}
}
```
### Heartbeat & Stale Detection
| Parameter | Default | Purpose |
|-----------|---------|---------|
| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` |
| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this |
Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file.
## Development
```bash
# Format code
make fmt
# Run linter
make lint
# Run tests with coverage
make test-cover
# Clean build artifacts
make clean
```
## Internal Packages
| Package | Purpose | | Package | Purpose |
|---------|---------| |---------|---------|
| `cmd/meta-core` | Main entry point, startup/shutdown orchestration | | `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
| `internal/config` | Environment-based configuration with defaults | | `internal/config` | Env-driven configuration, path helpers. |
| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) | | `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
| `internal/storage` | Redis client wrapper with flat key-value operations | | `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
| `internal/discovery` | Service registration, heartbeat loop, discovery | | `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
| `internal/api` | HTTP server, router, and all endpoint handlers | | `internal/discovery` | Service registry + dead-service cleaner. |
| `internal/events` | Redis keyspace notification publisher to meta:events | | `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch | | `internal/api` | HTTP server + all handlers + SSE event endpoints. |
| `internal/watchers` | Polling-based watcher configuration management | | `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
| `internal/mounts` | SMB/rclone mount configuration and health monitoring | | `internal/snapshot` | Snapshot export / import / wipe. |
| `internal/webdav` | WebDAV protocol handler with caching integration | | `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. |
| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). |
### Startup Sequence ### Startup sequence
``` ```
1. Load configuration from environment 1. Load configuration from environment
2. Create storage client (Redis wrapper) 2. Build LeaderInfoProvider (no election — the bash gate already won)
3. Initialize leader election 3. Connect to local Redis (retry up to 30× / 1s)
4. Register role transition callbacks 4. EnsureSchemaVersion — abort if the existing Redis layout is stale
5. Start election (acquire lock or become follower) 5. Start service discovery + dead-service cleaner
6. Start service discovery (register + heartbeat loop) 6. Construct API server (initialises watcher, mounts, WebDAV)
7. Initialize dead service cleaner 7. Start API server; on first connect, start MetaPublisher + schema Indexer
8. Initialize file watcher (if enabled) and republish existing metadata onto meta:events
9. Initialize mount manager 8. Wait for SIGINT/SIGTERM, shutdown in reverse order
10. Start HTTP API server (WebDAV caching handled by nginx)
11. Wait for SIGINT/SIGTERM
12. Shutdown in reverse order
```
## Integration
### As Sidecar in Docker
Add to existing service's supervisord config:
```ini
[program:meta-core]
command=/usr/local/bin/meta-core
priority=10
autostart=true
autorestart=true
```
### Client Usage (TypeScript)
```typescript
const response = await fetch('http://localhost:9000/meta/' + hashId);
const { metadata } = await response.json();
```
### Client Usage (Python)
```python
import requests
response = requests.get(f'http://localhost:9000/meta/{hash_id}')
metadata = response.json()['metadata']
``` ```
## Dependencies ## Dependencies
| Dependency | Version | Purpose | | Dependency | Purpose |
|------------|---------|---------| |------------|---------|
| `github.com/gorilla/mux` | v1.8.1 | HTTP router | | `github.com/gorilla/mux` | HTTP router |
| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client | | `github.com/redis/go-redis/v9` | Redis client |
| `golang.org/x/net` | v0.20.0 | WebDAV support | | `github.com/google/uuid` | UUIDv7 minting |
| `github.com/google/uuid` | v1.6.0 | UUID generation | | `golang.org/x/net` | WebDAV support |
| Go stdlib | - | context, encoding/json, net, os, sync, syscall | | Go stdlib | context, net/http, os, sync, syscall |
### Runtime Requirements Runtime requirements: Go 1.21+ at build time; the container ships Redis,
rclone, nginx, and supervisord alongside the binary.
- **Go 1.21+** for building
- **Redis** binary available in PATH (leader spawns it)
- **Shared filesystem** accessible by all instances (for flock)
## License ## License
+9
View File
@@ -51,6 +51,15 @@ func main() {
} }
log.Println("[meta-core] Connected to Redis") log.Println("[meta-core] Connected to Redis")
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
// the existing Redis store predates the UUID-rooted schema, so legacy
// data can't silently leak into the new code paths. See
// docs/uuid-rooted-metadata.md.
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
log.Fatalf("[meta-core] %v", err)
}
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
// Create and start service discovery // Create and start service discovery
disc := discovery.NewService(cfg) disc := discovery.NewService(cfg)
if err := disc.Start(); err != nil { if err := disc.Start(); err != nil {
+1 -1
View File
@@ -21,7 +21,7 @@
min-height: 100vh; min-height: 100vh;
} }
</style> </style>
<script type="module" crossorigin src="/assets/index-DJlbGMsa.js"></script> <script type="module" crossorigin src="/assets/index-Cbd1uH5A.js"></script>
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css"> <link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
</head> </head>
<body> <body>
+8
View File
@@ -3,6 +3,7 @@ import Dashboard from './pages/Dashboard';
import Mounts from './pages/Mounts'; import Mounts from './pages/Mounts';
import Watchers from './pages/Watchers'; import Watchers from './pages/Watchers';
import Files from './pages/Files'; import Files from './pages/Files';
import Identity from './pages/Identity';
import ServiceNav from './components/ServiceNav'; import ServiceNav from './components/ServiceNav';
const navStyle: React.CSSProperties = { const navStyle: React.CSSProperties = {
@@ -74,6 +75,12 @@ export default function App() {
> >
Files Files
</NavLink> </NavLink>
<NavLink
to="/identity"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
>
Identity
</NavLink>
<NavLink <NavLink
to="/editor" to="/editor"
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)} style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
@@ -89,6 +96,7 @@ export default function App() {
<Route path="/mounts" element={<Mounts />} /> <Route path="/mounts" element={<Mounts />} />
<Route path="/watchers" element={<Watchers />} /> <Route path="/watchers" element={<Watchers />} />
<Route path="/files" element={<Files />} /> <Route path="/files" element={<Files />} />
<Route path="/identity" element={<Identity />} />
<Route path="/editor" element={<EditorPage />} /> <Route path="/editor" element={<EditorPage />} />
</Routes> </Routes>
</div> </div>
+375
View File
@@ -0,0 +1,375 @@
import { useEffect, useState } from 'react';
// Status returned by GET /api/identity. Private key is never present here —
// it is shown exactly once at generation time (see GeneratedKeyPanel).
interface IdentityStatus {
hasIdentity: boolean;
uid?: string;
curve?: string;
createdAt?: number;
}
interface GeneratedKey {
uid: string;
curve: string;
createdAt: number;
privateKeyHex: string;
}
const cardStyle: React.CSSProperties = {
background: '#16213e',
borderRadius: '8px',
padding: '1.5rem',
marginBottom: '1rem',
};
const buttonStyle: React.CSSProperties = {
padding: '0.5rem 1rem',
background: '#0f3460',
color: '#fff',
border: 'none',
borderRadius: '4px',
cursor: 'pointer',
marginRight: '0.5rem',
};
const dangerButtonStyle: React.CSSProperties = {
...buttonStyle,
background: '#7f1d1d',
};
const inputStyle: React.CSSProperties = {
width: '100%',
padding: '0.5rem',
marginBottom: '0.5rem',
background: '#1a1a2e',
border: '1px solid #0f3460',
borderRadius: '4px',
color: '#fff',
fontFamily: 'monospace',
};
const monoStyle: React.CSSProperties = {
fontFamily: 'monospace',
fontSize: '0.85rem',
wordBreak: 'break-all',
background: '#0a0a18',
border: '1px solid #0f3460',
padding: '0.75rem',
borderRadius: '4px',
};
function formatCreatedAt(unixSec?: number): string {
if (!unixSec) return '';
try {
return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC');
} catch {
return String(unixSec);
}
}
// Read the JSON body of a non-2xx response if present, fall back to status text.
async function errorMessage(res: Response, fallback: string): Promise<string> {
try {
const j = await res.json();
if (j && (j.message || j.error)) return j.message ?? j.error;
} catch {
// not JSON
}
return `${fallback} (HTTP ${res.status})`;
}
function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss: () => void }) {
const [copied, setCopied] = useState(false);
const copy = async () => {
try {
await navigator.clipboard.writeText(data.privateKeyHex);
setCopied(true);
setTimeout(() => setCopied(false), 2000);
} catch {
// Clipboard API blocked (non-HTTPS context, etc.) — fall back to selection.
const sel = window.getSelection();
const range = document.createRange();
const el = document.getElementById('mm-privkey-box');
if (sel && el) {
range.selectNodeContents(el);
sel.removeAllRanges();
sel.addRange(range);
}
}
};
return (
<div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
<div
style={{
background: '#7f1d1d',
color: '#fee2e2',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginBottom: '1rem',
}}
>
<strong>This private key is shown ONCE.</strong> Save it now in a password manager.
It will never be displayed again — there is no recovery path. Losing it means losing access
to every record signed under this identity.
</div>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
<div style={monoStyle}>{data.uid}</div>
<p style={{ color: '#9ca3af', margin: '1rem 0 0.5rem' }}>privateKeyHex (secret):</p>
<div id="mm-privkey-box" style={{ ...monoStyle, background: '#1f0a0a', borderColor: '#7f1d1d' }}>
{data.privateKeyHex}
</div>
<div style={{ marginTop: '1rem' }}>
<button style={buttonStyle} onClick={copy}>
{copied ? 'Copied' : 'Copy private key'}
</button>
<button style={dangerButtonStyle} onClick={onDismiss}>
I have saved it
</button>
</div>
</div>
);
}
function NoIdentityPanel({
onGenerated,
onImported,
}: {
onGenerated: (g: GeneratedKey) => void;
onImported: () => void;
}) {
const [busy, setBusy] = useState(false);
const [importing, setImporting] = useState(false);
const [importHex, setImportHex] = useState('');
const [error, setError] = useState<string | null>(null);
const generate = async () => {
setBusy(true);
setError(null);
try {
const res = await fetch('/api/identity/generate', { method: 'POST' });
if (!res.ok) {
setError(await errorMessage(res, 'Generate failed'));
return;
}
const data = (await res.json()) as GeneratedKey;
onGenerated(data);
} catch (e) {
setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
const importKey = async (e: React.FormEvent) => {
e.preventDefault();
setBusy(true);
setError(null);
try {
const res = await fetch('/api/identity/import', {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ privateKeyHex: importHex.trim() }),
});
if (!res.ok) {
setError(await errorMessage(res, 'Import failed'));
return;
}
setImportHex('');
setImporting(false);
onImported();
} catch (e) {
setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
}
};
return (
<div style={cardStyle}>
<h3 style={{ marginTop: 0 }}>No identity yet</h3>
<p style={{ color: '#9ca3af' }}>
The User Data Layer needs a signing key for your ratings, watch progress, and "liked" flags
to be attributed to you across devices. Nothing happens automatically — generate one below,
or import a key you already have.
</p>
{error && (
<p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
{error}
</p>
)}
<div>
<button style={buttonStyle} disabled={busy} onClick={generate}>
{busy ? 'Working…' : 'Generate new keypair'}
</button>
<button
style={buttonStyle}
disabled={busy}
onClick={() => {
setImporting((x) => !x);
setError(null);
}}
>
{importing ? 'Cancel import' : 'Import existing private key'}
</button>
</div>
{importing && (
<form onSubmit={importKey} style={{ marginTop: '1rem' }}>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>
64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine.
</p>
<input
style={inputStyle}
placeholder="e.g. a3f1…"
value={importHex}
onChange={(e) => setImportHex(e.target.value)}
required
autoFocus
/>
<button type="submit" style={buttonStyle} disabled={busy || !importHex.trim()}>
{busy ? 'Importing…' : 'Import'}
</button>
</form>
)}
</div>
);
}
function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onReplace: () => void }) {
const [confirming, setConfirming] = useState(false);
const [busy, setBusy] = useState(false);
const [error, setError] = useState<string | null>(null);
const remove = async () => {
setBusy(true);
setError(null);
try {
const res = await fetch('/api/identity?confirm=true', { method: 'DELETE' });
if (!res.ok && res.status !== 204) {
setError(await errorMessage(res, 'Delete failed'));
return;
}
onReplace();
} catch (e) {
setError(`Delete failed: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setBusy(false);
setConfirming(false);
}
};
return (
<div style={cardStyle}>
<h3 style={{ marginTop: 0 }}>Active identity</h3>
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid:</p>
<div style={monoStyle}>{status.uid}</div>
<p style={{ color: '#888', marginTop: '1rem' }}>
Curve: {status.curve} · Created: {formatCreatedAt(status.createdAt)}
</p>
<p style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
The private key lives at <code>/meta-core/identity/identity.json</code> on this volume and
is never returned through the API after generation. Other services on this stack (e.g.
meta-watch) sign records by calling <code>POST /api/identity/sign</code>.
</p>
{error && (
<p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
{error}
</p>
)}
{!confirming ? (
<button style={dangerButtonStyle} onClick={() => setConfirming(true)} disabled={busy}>
Replace identity
</button>
) : (
<div
style={{
background: '#1f0a0a',
border: '1px solid #7f1d1d',
padding: '0.75rem 1rem',
borderRadius: '4px',
marginTop: '0.5rem',
}}
>
<p style={{ color: '#fecaca', marginTop: 0 }}>
Replacing the identity deletes the current keypair from disk. Records signed by the
current uid will keep verifying for anyone who has them cached, but this node will no
longer be able to write under that uid. <strong>Make sure you have backed up the key
first.</strong>
</p>
<button style={dangerButtonStyle} onClick={remove} disabled={busy}>
{busy ? 'Deleting…' : 'Yes, delete identity'}
</button>
<button style={buttonStyle} onClick={() => setConfirming(false)} disabled={busy}>
Cancel
</button>
</div>
)}
</div>
);
}
export default function Identity() {
const [status, setStatus] = useState<IdentityStatus | null>(null);
const [generated, setGenerated] = useState<GeneratedKey | null>(null);
const [loading, setLoading] = useState(true);
const [fetchError, setFetchError] = useState<string | null>(null);
const refresh = async () => {
setLoading(true);
setFetchError(null);
try {
const res = await fetch('/api/identity');
if (!res.ok) {
setFetchError(await errorMessage(res, 'Failed to load identity status'));
return;
}
setStatus((await res.json()) as IdentityStatus);
} catch (e) {
setFetchError(`Failed to load identity status: ${e instanceof Error ? e.message : String(e)}`);
} finally {
setLoading(false);
}
};
useEffect(() => {
refresh();
}, []);
return (
<div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
<h1>Identity</h1>
<p style={{ color: '#9ca3af' }}>
Signing key for the User Data Layer (per-user ratings, progress, "liked" flags). One
identity per node. The same key on another device makes that device count as "you".
</p>
{loading && <div style={cardStyle}>Loading…</div>}
{fetchError && (
<div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
<p style={{ color: '#f87171', margin: 0 }}>{fetchError}</p>
<button style={buttonStyle} onClick={refresh}>
Retry
</button>
</div>
)}
{!loading && !fetchError && generated && (
<GeneratedKeyPanel
data={generated}
onDismiss={() => {
setGenerated(null);
refresh();
}}
/>
)}
{!loading && !fetchError && !generated && status && !status.hasIdentity && (
<NoIdentityPanel onGenerated={(g) => setGenerated(g)} onImported={refresh} />
)}
{!loading && !fetchError && !generated && status && status.hasIdentity && (
<HasIdentityPanel status={status} onReplace={refresh} />
)}
</div>
);
}
+220 -42
View File
@@ -1,19 +1,47 @@
import { useEffect, useState } from 'react'; import { useEffect, useState } from 'react';
interface MountIOStats {
source: 'rclone';
bytesRead: number;
bytesWritten?: number;
readBps: number;
writeBps?: number;
readOps?: number;
writeOps?: number;
readIops?: number;
writeIops?: number;
transfersPerSec?: number;
lastSampleAt: number;
intervalMs?: number;
}
interface Mount { interface Mount {
id: string; id: string;
name: string; name: string;
type: 'nfs' | 'smb' | 'rclone'; type: 'smb' | 'rclone';
enabled: boolean; enabled: boolean;
desiredMounted: boolean; desiredMounted: boolean;
mountPath: string; mountPath: string;
mounted: boolean; mounted: boolean;
error?: string; error?: string;
lastChecked: number; lastChecked: number;
// VFS cache configuration applied at mount time. Empty string means "use
// server default" (50G / 24h / 5m).
cacheMaxSize?: string;
cacheMaxAge?: string;
dirCacheTime?: string;
pollingEnabled?: boolean; pollingEnabled?: boolean;
pollingIntervalMs?: number; pollingIntervalMs?: number;
pollingActive?: boolean; pollingActive?: boolean;
lastPolledScan?: number; lastPolledScan?: number;
// Adaptive scan progress (set by the meta-core poller)
scanning?: boolean;
currentScanStartedAt?: number; // ms; only while scanning
lastScanDurationMs?: number;
nextScanAt?: number; // ms; planned start of next adaptive scan
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
// so the same numbers appear on every mount card.
ioStats?: MountIOStats;
} }
interface RcloneRemote { interface RcloneRemote {
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
background: '#7f1d1d', background: '#7f1d1d',
}; };
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
function formatBytes(n: number): string {
if (!isFinite(n) || n < 0) return '—';
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
let i = 0;
let v = n;
while (v >= 1000 && i < units.length - 1) {
v /= 1000;
i++;
}
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
}
function formatRate(bps: number): string {
return `${formatBytes(bps)}/s`;
}
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
// is just noise.
function formatDuration(ms: number): string {
if (ms < 0 || !isFinite(ms)) return '—';
const s = Math.floor(ms / 1000);
if (s < 60) return `${s}s`;
const m = Math.floor(s / 60);
const remS = s % 60;
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
const h = Math.floor(m / 60);
const remM = m % 60;
return remM ? `${h}h ${remM}m` : `${h}h`;
}
// IOStatsLine renders a single line of live throughput info beneath each
// mount card. Source is always rclone after the rclone-only consolidation,
// so we report file-level transfers/s rather than kernel IOPS.
function IOStatsLine({ stats }: { stats: MountIOStats }) {
const opsLabel = 'transfers/s';
const opsValue = stats.transfersPerSec;
const ageMs = Date.now() - stats.lastSampleAt;
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
return (
<p
style={{
color: stale ? '#6b7280' : '#9ca3af',
fontSize: '0.85rem',
margin: '0.25rem 0 0',
}}
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
>
↓ {formatRate(stats.readBps)}
{opsValue !== undefined && opsValue > 0 && (
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
)}
<> · {formatBytes(stats.bytesRead)} read total</>
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
<> · {formatBytes(stats.bytesWritten)} written</>
)}
</p>
);
}
interface ScanProgressProps {
mount: Mount;
now: number;
}
// Progress bar shows scan ETA when running, an idle row with a countdown to
// the next adaptive scan when waiting. We can't show true % complete (we
// don't know the file count up-front), so the running bar is filled by
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
// it pins at 95% to signal "taking longer than last time".
function ScanProgress({ mount, now }: ScanProgressProps) {
if (!mount.pollingEnabled) return null;
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
const lastDuration = mount.lastScanDurationMs ?? 0;
let fill = 0;
let label = '';
if (isScanning) {
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
if (lastDuration > 0) {
fill = Math.min(0.95, elapsed / lastDuration);
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
} else {
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
}
} else {
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
if (lastDuration > 0 && mount.lastPolledScan) {
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
} else if (mount.nextScanAt) {
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
} else {
label = 'Polling enabled — waiting for first scan';
}
}
return (
<div style={{ marginTop: '0.5rem' }}>
<div
style={{
height: '6px',
background: '#0f3460',
borderRadius: '3px',
overflow: 'hidden',
position: 'relative',
}}
>
<div
style={{
width: `${Math.round(fill * 100)}%`,
height: '100%',
background: isScanning ? '#4ade80' : '#374151',
transition: 'width 0.5s linear',
}}
/>
</div>
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
</div>
);
}
export default function Mounts() { export default function Mounts() {
const [mounts, setMounts] = useState<Mount[]>([]); const [mounts, setMounts] = useState<Mount[]>([]);
const [remotes, setRemotes] = useState<RcloneRemote[]>([]); const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
@@ -50,19 +209,23 @@ export default function Mounts() {
const [showForm, setShowForm] = useState(false); const [showForm, setShowForm] = useState(false);
const [formData, setFormData] = useState({ const [formData, setFormData] = useState({
name: '', name: '',
type: 'nfs' as 'nfs' | 'smb' | 'rclone', type: 'smb' as 'smb' | 'rclone',
nfsServer: '',
nfsPath: '',
smbServer: '', smbServer: '',
smbShare: '', smbShare: '',
smbUsername: '', smbUsername: '',
smbPassword: '', smbPassword: '',
rcloneRemote: '', rcloneRemote: '',
rclonePath: '', rclonePath: '',
pollingEnabled: false, cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000, pollingIntervalMs: 30000,
}); });
const [scanning, setScanning] = useState<string | null>(null); const [scanning, setScanning] = useState<string | null>(null);
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
// refetches (which run every 5s).
const [now, setNow] = useState<number>(() => Date.now());
const fetchMounts = async () => { const fetchMounts = async () => {
try { try {
@@ -94,7 +257,11 @@ export default function Mounts() {
fetchMounts(); fetchMounts();
fetchRemotes(); fetchRemotes();
const interval = setInterval(fetchMounts, 5000); const interval = setInterval(fetchMounts, 5000);
return () => clearInterval(interval); const tick = setInterval(() => setNow(Date.now()), 1000);
return () => {
clearInterval(interval);
clearInterval(tick);
};
}, []); }, []);
const handleMount = async (id: string) => { const handleMount = async (id: string) => {
@@ -143,10 +310,7 @@ export default function Mounts() {
type: formData.type, type: formData.type,
}; };
if (formData.type === 'nfs') { if (formData.type === 'smb') {
body.nfsServer = formData.nfsServer;
body.nfsPath = formData.nfsPath;
} else if (formData.type === 'smb') {
body.smbServer = formData.smbServer; body.smbServer = formData.smbServer;
body.smbShare = formData.smbShare; body.smbShare = formData.smbShare;
if (formData.smbUsername) body.smbUsername = formData.smbUsername; if (formData.smbUsername) body.smbUsername = formData.smbUsername;
@@ -156,6 +320,12 @@ export default function Mounts() {
body.rclonePath = formData.rclonePath; body.rclonePath = formData.rclonePath;
} }
// Cache knobs — only forward non-empty values; the server applies defaults
// when fields are omitted.
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
// Add polling configuration // Add polling configuration
body.pollingEnabled = formData.pollingEnabled; body.pollingEnabled = formData.pollingEnabled;
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) { if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
@@ -171,16 +341,17 @@ export default function Mounts() {
setShowForm(false); setShowForm(false);
setFormData({ setFormData({
name: '', name: '',
type: 'nfs', type: 'smb',
nfsServer: '',
nfsPath: '',
smbServer: '', smbServer: '',
smbShare: '', smbShare: '',
smbUsername: '', smbUsername: '',
smbPassword: '', smbPassword: '',
rcloneRemote: '', rcloneRemote: '',
rclonePath: '', rclonePath: '',
pollingEnabled: false, cacheMaxSize: '',
cacheMaxAge: '',
dirCacheTime: '',
pollingEnabled: true,
pollingIntervalMs: 30000, pollingIntervalMs: 30000,
}); });
fetchMounts(); fetchMounts();
@@ -231,32 +402,15 @@ export default function Mounts() {
<select <select
style={inputStyle} style={inputStyle}
value={formData.type} value={formData.type}
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })} onChange={(e) => {
const type = e.target.value as 'smb' | 'rclone';
setFormData({ ...formData, type });
}}
> >
<option value="nfs">NFS</option> <option value="smb">SMB (via rclone)</option>
<option value="smb">SMB/CIFS</option> <option value="rclone">rclone (pre-defined remote)</option>
<option value="rclone">rclone</option>
</select> </select>
{formData.type === 'nfs' && (
<>
<input
style={inputStyle}
placeholder="NFS Server (e.g., 192.168.1.100)"
value={formData.nfsServer}
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
required
/>
<input
style={inputStyle}
placeholder="NFS Path (e.g., /export/media)"
value={formData.nfsPath}
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
required
/>
</>
)}
{formData.type === 'smb' && ( {formData.type === 'smb' && (
<> <>
<input <input
@@ -313,6 +467,30 @@ export default function Mounts() {
</> </>
)} )}
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
VFS cache (leave blank for defaults: 50G / 24h / 5m)
</p>
<input
style={inputStyle}
placeholder="Cache max size (e.g., 50G)"
value={formData.cacheMaxSize}
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Cache max age (e.g., 24h)"
value={formData.cacheMaxAge}
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
/>
<input
style={inputStyle}
placeholder="Dir cache time (e.g., 5m)"
value={formData.dirCacheTime}
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
/>
</div>
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}> <div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}> <label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
<input <input
@@ -362,17 +540,17 @@ export default function Mounts() {
</span> </span>
</p> </p>
{mount.pollingEnabled && ( {mount.pollingEnabled && (
<p style={{ color: '#888', fontSize: '0.9rem' }}> <p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
Polling:{' '} Polling:{' '}
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}> <span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
{mount.pollingActive ? 'Active' : 'Inactive'} {mount.pollingActive ? 'Active' : 'Inactive'}
</span> </span>
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s {' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
{mount.lastPolledScan && ( {' • Adaptive 10× idle'}
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
)}
</p> </p>
)} )}
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
<ScanProgress mount={mount} now={now} />
{mount.error && ( {mount.error && (
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p> <p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
)} )}
+135 -167
View File
@@ -2,11 +2,17 @@
# ============================================================================= # =============================================================================
# Mount Watcher Script # Mount Watcher Script
# #
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts. # Monitors /meta-core/mounts/mounts.json and manages mounts via the local
# Runs as a supervisord program, polling every 5 seconds. # rclone RC daemon. After the rclone-only consolidation, this script no
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
# through rclone's mount/mount endpoint.
# #
# Uses findmnt to check actual mount status (source of truth). # Mount types:
# Writes errors to /meta-core/mounts/errors/{id}.error # - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
# - "rclone" → references a pre-defined remote in rclone.conf
#
# All mounts are read-only by construction (no API surface to override).
# Errors surface in /meta-core/mounts/errors/{id}.error
# ============================================================================= # =============================================================================
set -u set -u
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
FILES_PATH="${FILES_PATH:-/files}" FILES_PATH="${FILES_PATH:-/files}"
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}" POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
# Fall-back VFS cache settings — applied when a mount entry leaves the field
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
# packages/meta-core/internal/mounts/types.go.
DEFAULT_CACHE_MAX_SIZE="50G"
DEFAULT_CACHE_MAX_AGE="24h"
DEFAULT_DIR_CACHE_TIME="5m"
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
# ANSI colors # ANSI colors
RED='\033[0;31m' RED='\033[0;31m'
GREEN='\033[0;32m' GREEN='\033[0;32m'
@@ -54,9 +70,7 @@ init_dirs() {
# Check if path is mounted (uses /proc/mounts for Alpine compatibility) # Check if path is mounted (uses /proc/mounts for Alpine compatibility)
is_mounted() { is_mounted() {
local mount_path="$1" local mount_path="$1"
# Remove trailing slash for consistent matching
local clean_path="${mount_path%/}" local clean_path="${mount_path%/}"
# Check if the path appears as a mount point in /proc/mounts (field 2)
grep -q " ${clean_path} " /proc/mounts 2>/dev/null grep -q " ${clean_path} " /proc/mounts 2>/dev/null
} }
@@ -74,145 +88,102 @@ write_error() {
echo "$error" >> "$ERROR_DIR/$id.error" echo "$error" >> "$ERROR_DIR/$id.error"
} }
# Mount NFS share (always read-only) # Build the rclone remote source string for a mount entry. Echoes the result.
do_mount_nfs() { # For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
local id="$1" # mutate rclone.conf at runtime. For type=rclone, joins the named remote with
local server="$2" # the optional remote path.
local path="$3" build_rclone_fs() {
local mount_path="$4" local type="$1"
local options="${5:-}" local smb_server="$2"
local smb_share="$3"
local smb_username="$4"
local smb_password_obscured="$5"
local smb_domain="$6"
local rclone_remote="$7"
local rclone_path="$8"
mkdir -p "$mount_path" case "$type" in
smb)
# Always mount read-only for safety # On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
local mount_opts="ro" # The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
[ -n "$options" ] && mount_opts="${mount_opts},${options}" # embed without quoting.
local opts="host=${smb_server}"
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}" if [ -n "$smb_username" ]; then
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1) opts="${opts},user=${smb_username}"
local rc=$?
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "NFS mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "NFS mount failed: $output"
fi fi
return $rc if [ -n "$smb_password_obscured" ]; then
opts="${opts},pass=${smb_password_obscured}"
fi
if [ -n "$smb_domain" ]; then
opts="${opts},domain=${smb_domain}"
fi
echo ":smb,${opts}:${smb_share}"
;;
rclone)
echo "${rclone_remote}:${rclone_path}"
;;
*)
echo ""
;;
esac
} }
# Mount SMB/CIFS share (always read-only) # Mount via rclone RC API (always read-only).
do_mount_smb() {
local id="$1"
local server="$2"
local share="$3"
local mount_path="$4"
local username="${5:-}"
local password_obscured="${6:-}"
local domain="${7:-}"
local options="${8:-}"
mkdir -p "$mount_path"
# Always mount read-only for safety
local mount_opts="ro"
# If we have credentials, reveal password and build options
if [ -n "$username" ]; then
mount_opts="${mount_opts},username=${username}"
if [ -n "$password_obscured" ]; then
# Reveal password using rclone (only exists in memory briefly)
local password
password=$(rclone reveal "$password_obscured" 2>/dev/null)
if [ -n "$password" ]; then
mount_opts="${mount_opts},password=${password}"
fi
fi
if [ -n "$domain" ]; then
mount_opts="${mount_opts},domain=${domain}"
fi
fi
# Add any additional options
if [ -n "$options" ]; then
if [ -n "$mount_opts" ]; then
mount_opts="${mount_opts},${options}"
else
mount_opts="$options"
fi
fi
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
# Execute mount directly (not via eval) to handle special chars in password
local output
local rc
if [ -n "$mount_opts" ]; then
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
rc=$?
else
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
rc=$?
fi
# Clear password from any variables (best effort)
password=""
mount_opts=""
if [ $rc -eq 0 ]; then
clear_error "$id"
log_success "SMB mount successful: $mount_path"
else
write_error "$id" "$output"
log_error "SMB mount failed: $output"
fi
return $rc
}
# Mount rclone remote using RC API (always read-only)
do_mount_rclone() { do_mount_rclone() {
local id="$1" local id="$1"
local remote="$2" local fs="$2"
local remote_path="$3" local mount_path="$3"
local mount_path="$4" local cache_max_size="$4"
local options="${5:-}" local cache_max_age="$5"
local dir_cache_time="$6"
mkdir -p "$mount_path" mkdir -p "$mount_path"
# Build the remote source string # Apply defaults for any blank cache fields.
local fs="${remote}:${remote_path}" [ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
# Use rclone RC API to mount (always read-only for safety) log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
# Build JSON body for rclone mount with read-only option # Build JSON body via python3 — embedding the obscured password and
# display fs string into a hand-rolled heredoc is asking for an injection
# bug the next time we add a field with a quote in it.
local json_body local json_body
json_body=$(cat <<EOF json_body=$(python3 -c '
{ import json, sys
"fs": "${fs}", print(json.dumps({
"mountPoint": "${mount_path}", "fs": sys.argv[1],
"mountPoint": sys.argv[2],
"mountOpt": { "mountOpt": {
"AllowOther": true, "AllowOther": True,
"ReadOnly": true "ReadOnly": True,
}, },
"vfsOpt": { "vfsOpt": {
"CacheMode": 2, # CacheMode 3 = "full" — cache both reads and writes on local disk.
"ReadOnly": true # CacheMode 2 ("writes") would silently bypass the cache for reads,
} # which is the opposite of what we want for a read-only mount.
} "CacheMode": 3,
EOF "CacheMaxSize": sys.argv[3],
) "CacheMaxAge": sys.argv[4],
"DirCacheTime": sys.argv[5],
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
# throughput in isolation but, under the 4+ concurrent reads from
# the processing pipeline, multiplies into 16+ simultaneous SMB
# requests and tanks aggregate throughput. Left at default — rely on
# file-level concurrency from the pipeline for parallelism instead.
"ReadOnly": True,
},
}))
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
local output
output=$(curl -s -X POST \ output=$(curl -s -X POST \
-H "Content-Type: application/json" \ -H "Content-Type: application/json" \
-u admin:admin \ -u "$RCLONE_RC_AUTH" \
-d "$json_body" \ -d "$json_body" \
"http://127.0.0.1:5572/mount/mount" 2>&1) "${RCLONE_RC_URL}/mount/mount" 2>&1)
# Wait a moment for mount to establish
sleep 2 sleep 2
if is_mounted "$mount_path"; then if is_mounted "$mount_path"; then
@@ -221,7 +192,6 @@ EOF
return 0 return 0
else else
local error_msg="rclone mount failed" local error_msg="rclone mount failed"
# Try to extract error from response
if echo "$output" | grep -q "error"; then if echo "$output" | grep -q "error"; then
error_msg="$output" error_msg="$output"
fi fi
@@ -231,36 +201,28 @@ EOF
fi fi
} }
# Unmount a path # Unmount a path. Always goes through the rclone RC API now — every mount we
# create lives behind rclone's FUSE driver.
do_unmount() { do_unmount() {
local id="$1" local id="$1"
local mount_path="$2" local mount_path="$2"
local type="$3"
if ! is_mounted "$mount_path"; then if ! is_mounted "$mount_path"; then
log "Already unmounted: $mount_path" log "Already unmounted: $mount_path"
return 0 return 0
fi fi
if [ "$type" = "rclone" ]; then log "Unmounting: $mount_path"
# Use rclone RC API to unmount
log "Unmounting rclone: $mount_path"
curl -s -X POST \ curl -s -X POST \
-H "Content-Type: application/json" \ -H "Content-Type: application/json" \
-u admin:admin \ -u "$RCLONE_RC_AUTH" \
-d "{\"mountPoint\":\"${mount_path}\"}" \ -d "{\"mountPoint\":\"${mount_path}\"}" \
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1 "${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
else
log "Unmounting: $mount_path"
umount "$mount_path" 2>&1
fi
# Wait a moment
sleep 1 sleep 1
# Check if still mounted # Fallback to lazy umount in case the FUSE process is wedged.
if is_mounted "$mount_path"; then if is_mounted "$mount_path"; then
# Force unmount with lazy option
log_warn "Force unmounting (lazy): $mount_path" log_warn "Force unmounting (lazy): $mount_path"
umount -l "$mount_path" 2>&1 umount -l "$mount_path" 2>&1
sleep 1 sleep 1
@@ -277,12 +239,11 @@ do_unmount() {
# Process all mounts from config # Process all mounts from config
process_mounts() { process_mounts() {
# Check if config file exists
if [ ! -f "$CONFIG_FILE" ]; then if [ ! -f "$CONFIG_FILE" ]; then
return return
fi fi
# Parse JSON using python3 # Parse JSON using python3 — pipe-separated fields, NFS dropped.
local mounts local mounts
mounts=$(python3 -c ' mounts=$(python3 -c '
import sys, json import sys, json
@@ -294,15 +255,12 @@ except:
sys.exit(0) sys.exit(0)
for m in data.get("mounts", []): for m in data.get("mounts", []):
# Output pipe-separated values
fields = [ fields = [
m.get("id", ""), m.get("id", ""),
m.get("type", ""), m.get("type", ""),
str(m.get("enabled", True)), str(m.get("enabled", True)),
str(m.get("desiredMounted", False)), str(m.get("desiredMounted", False)),
m.get("mountPath", ""), m.get("mountPath", ""),
m.get("nfsServer", ""),
m.get("nfsPath", ""),
m.get("smbServer", ""), m.get("smbServer", ""),
m.get("smbShare", ""), m.get("smbShare", ""),
m.get("smbUsername", ""), m.get("smbUsername", ""),
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
m.get("smbDomain", ""), m.get("smbDomain", ""),
m.get("rcloneRemote", ""), m.get("rcloneRemote", ""),
m.get("rclonePath", ""), m.get("rclonePath", ""),
m.get("options", "") m.get("cacheMaxSize", ""),
m.get("cacheMaxAge", ""),
m.get("dirCacheTime", ""),
] ]
print("|".join(fields)) print("|".join(fields))
' 2>/dev/null) ' 2>/dev/null)
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
return return
fi fi
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do while IFS='|' read -r id type enabled desired mount_path \
smb_server smb_share smb_username smb_password_obscured smb_domain \
rclone_remote rclone_path \
cache_max_size cache_max_age dir_cache_time; do
# Skip empty lines # Skip empty lines
[ -z "$id" ] && continue [ -z "$id" ] && continue
# Check actual mount status # Skip retired/legacy types — the Go-side migration disables NFS but a
# stale entry could still arrive here on a downgrade/upgrade race.
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
continue
fi
local is_mounted_now=false local is_mounted_now=false
is_mounted "$mount_path" && is_mounted_now=true is_mounted "$mount_path" && is_mounted_now=true
# Handle disabled mounts - unmount if currently mounted # Disabled → unmount if currently up
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
if [ "$is_mounted_now" = true ]; then if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) disabled, unmounting..." log "Mount $id ($mount_path) disabled, unmounting..."
do_unmount "$id" "$mount_path" "$type" do_unmount "$id" "$mount_path"
fi fi
continue continue
fi fi
# Handle desired state
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
# Should be mounted # Should be mounted
if [ "$is_mounted_now" = false ]; then if [ "$is_mounted_now" = false ]; then
local fs
fs=$(build_rclone_fs "$type" \
"$smb_server" "$smb_share" "$smb_username" \
"$smb_password_obscured" "$smb_domain" \
"$rclone_remote" "$rclone_path")
if [ -z "$fs" ]; then
log_error "Could not build rclone fs spec for mount $id (type=$type)"
write_error "$id" "Invalid mount config: missing required fields for type=$type"
continue
fi
log "Mount $id ($mount_path) desired but not mounted, mounting..." log "Mount $id ($mount_path) desired but not mounted, mounting..."
case "$type" in do_mount_rclone "$id" "$fs" "$mount_path" \
nfs) "$cache_max_size" "$cache_max_age" "$dir_cache_time"
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
;;
smb)
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
;;
rclone)
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
;;
*)
log_error "Unknown mount type: $type"
;;
esac
fi fi
else else
# Should be unmounted # Should be unmounted
if [ "$is_mounted_now" = true ]; then if [ "$is_mounted_now" = true ]; then
log "Mount $id ($mount_path) not desired, unmounting..." log "Mount $id ($mount_path) not desired, unmounting..."
do_unmount "$id" "$mount_path" "$type" do_unmount "$id" "$mount_path"
fi fi
fi fi
done <<< "$mounts" done <<< "$mounts"
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
# Main loop # Main loop
main() { main() {
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)" log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
init_dirs init_dirs
+15
View File
@@ -70,6 +70,9 @@ http {
proxy_buffering off; proxy_buffering off;
proxy_cache off; proxy_cache off;
proxy_read_timeout 86400s; proxy_read_timeout 86400s;
# Snapshot import accepts multi-MB ZIP uploads.
client_max_body_size 0;
} }
# ========================================================================= # =========================================================================
@@ -192,6 +195,18 @@ http {
# WebDAV HTML browser (for iframe embedding in dashboard) # WebDAV HTML browser (for iframe embedding in dashboard)
# Returns HTML directory listings instead of JSON # Returns HTML directory listings instead of JSON
# ========================================================================= # =========================================================================
# At the browser root, strip the "../" parent-dir row that autoindex
# otherwise emits — clicking it inside the dashboard iframe would
# navigate to "/" (the dashboard itself), nesting the page inside itself.
location = /webdav-browser/ {
alias /files/;
autoindex on;
charset utf-8;
sub_filter '<a href="../">../</a>' '';
sub_filter_once on;
}
location /webdav-browser/ { location /webdav-browser/ {
alias /files/; alias /files/;
autoindex on; autoindex on;
+2 -2
View File
@@ -71,7 +71,7 @@ startsecs=2
# rclone: Remote storage mount daemon (RC API on port 5572) # rclone: Remote storage mount daemon (RC API on port 5572)
# ============================================================================= # =============================================================================
[program:rclone] [program:rclone]
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
priority=30 priority=30
autostart=true autostart=true
autorestart=true autorestart=true
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
startsecs=3 startsecs=3
# ============================================================================= # =============================================================================
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts # mount-watcher: Monitors mounts.json and routes all mounts through rclone
# ============================================================================= # =============================================================================
[program:mount-watcher] [program:mount-watcher]
command=/app/scripts/mount-watcher.sh command=/app/scripts/mount-watcher.sh
+771
View File
@@ -0,0 +1,771 @@
# API-Mediated Access (Redis Lockdown)
## Status
**Landed** — all four PRs (A/B/C/D) deployed and verified end-to-end on the
dev stack. Redis is unreachable from outside `metacore-app`; every metadata
read, write, and event flows through meta-core's HTTP+SSE surface. Builds
on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
meta-core is the single writer to the metadata keyspace — an assumption
that's now actually enforced rather than aspirational.
See "Implementation outcome" at the bottom for the delta between this doc's
plan and what shipped (notably the write-path CID resolution and the
dual-root migration sweep, which weren't in the original plan but turned
out to be load-bearing).
## Motivation
meta-core was designed as the sidecar that mediates every metadata read and
write — a single point that owns the Redis schema, enforces invariants
(reverse index, canonical_cid reconciliation, schema-version sentinel),
and presents a stable HTTP contract. Other services were meant to talk to
meta-core, not to Redis.
In practice, meta-sort opens its own Redis connection and writes
`file:<hashId>/<property>` keys with raw pipeline SETs
(`packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts:269-296`).
It picked up this shortcut early, before meta-core had a complete write
API, and the shortcut stuck.
This bypass is a real problem in three ways:
1. **Schema invariants leak.** When meta-core landed the UUID-rooted schema
([uuid-rooted-metadata.md](uuid-rooted-metadata.md)), meta-sort kept
writing midhash-rooted entries. The result was *dual roots per file* —
one set of keys at the watcher's UUID, another at meta-sort's midhash.
`file:__index__` doubled in size. The bug was invisible to meta-sort
(its writes succeeded) but broke the reverse-index resolution
downstream services rely on.
2. **Schema evolution is impossible.** Changing the Redis layout requires
coordinating across every service that touches it. The UUID-root
migration would have been a single-PR change if writes already routed
through the API.
3. **Self-hosted distributed deployment widens the threat model.** Peers
are run by different operators. Plugin authors may ship their own
services. We can't expect every binary in the network to honour
undocumented conventions. A buggy or careless service that connects
to Redis and writes the wrong key shape can corrupt the schema for
everybody on that node.
The third point is the one that flipped the design from "soft lock by
convention" to "hard lock by network topology." We're not protecting
against malicious services — that's an out-of-scope threat. We're
protecting against *accidental* corruption from honest mistakes in code
we don't fully control.
## The architecture in one picture
```
meta-core (only Redis writer + reader)
┌──────────────────────────────────────┐
│ HTTP API surface: │
│ POST/PUT/PATCH /api/metadata/* │ ◄── writes
│ GET /api/meta/{cid} │ ◄── reads (by CID)
│ GET /api/metadata/* │ ◄── reads (by id)
│ GET /api/file/{cid} │ ◄── content
│ GET /api/events/files │ ◄── SSE stream
│ GET /api/events/meta │ ◄── SSE stream
│ │ │
│ ▼ │
│ Redis (file:*, cid:*, streams) │
│ ◄── container-local, no exposed │
│ port, no docker-network alias │
└──────────────────────────────────────┘
▲ ▲
│ HTTP │ HTTP SSE
│ │
┌───────────┴────┐ ┌──────┴─────────┐
│ meta-sort │ │ meta-fuse │
│ meta-stremio │ │ meta-share │
│ meta-dup │ │ (any future) │
└────────────────┘ └────────────────┘
```
**The single-writer rule.** meta-core has exclusive write authority over
the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
other service can write — not because of an honoured convention, but
because they can't reach Redis at all.
**Stream reads also mediated.** Events go out through SSE endpoints on
meta-core's HTTP surface. Consumers don't see the underlying Redis
Streams. This is the Kubernetes / Consul pattern (see "Industry context"
below). It costs an extra HTTP hop versus the current direct XREAD; the
benefit is that the storage layer is free to evolve (or be replaced)
without breaking every consumer.
**Redis is private.** No host port mapping, no docker-network alias, no
URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
## Eventing — HTTP Server-Sent Events
SSE is the chosen transport for cross-service event consumption. The
mapping from Redis Streams onto SSE is direct enough that no eventing
feature we currently use is lost.
### Feature mapping (what we keep, how we keep it)
| Redis Streams | SSE equivalent |
|---|---|
| Fan-out to N consumers | meta-core holds N SSE connections, broadcasts events to each |
| Replay from a cursor | `Last-Event-ID` header on reconnect — built into the SSE spec |
| Retention / MAXLEN | Underlying Redis Stream still does the trimming; SSE is a thin shim |
| Backpressure on slow consumers | meta-core's SSE handler blocks on the TCP write; events stay in Redis until trimmed |
| Multiple streams (file:events, meta:events) | Separate endpoints: `/api/events/files`, `/api/events/meta` |
| At-least-once via XACK | Implicit — client persists last-received `id`, sends it on reconnect (see "Cursor semantics and consumer-group migration" below) |
The one Streams feature we don't get is **consumer groups for
horizontally-scaled replicas of the same logical consumer.** We don't use
this today and don't plan to — meta-sort, meta-fuse, etc. each run one
instance. If horizontal scale ever happens, it'll be by sharding hash-IDs
across replicas, not by splitting an event stream. SSE stays adequate.
### Cursor semantics and consumer-group migration
Today three consumers use `XREADGROUP` against named consumer groups, not
bare `XREAD`:
- meta-sort: group `meta-sort-processor`
(`packages/meta-sort/.../events/FileEventConsumer.ts:23,66-74`)
- meta-dup: group on `file:events`
(`packages/meta-dup/.../MetaEventConsumer.ts:102-104`)
- meta-fuse: dual path — `XREADGROUP` on one consumer, plain `XREAD` on
another (`packages/meta-fuse/.../kv/RedisClient.ts:826,907`)
`XREADGROUP` keeps server-side state per group: a Pending Entries List
(PEL) and a `last-delivered-id` cursor. The load-bearing use today is
meta-sort's `processPendingEntries(idleMs=30000)`, which exists to claim
work from a crashed consumer's PEL on restart.
SSE has no equivalent server-side state. After the migration the contract
shifts to: **each consumer persists its own `Last-Event-ID` and resumes
from it on reconnect.** Implications:
- PEL-based crash recovery is gone. A crashed consumer that hadn't
persisted its cursor since entry X resumes from X (its last *durably
persisted* cursor), not from "whatever was in flight." Acceptable for
every current consumer — none run multiple instances per group — but
call out in the PR C migration of meta-sort that the recovery primitive
is changing.
- Each consumer must own its cursor persistence. Cheapest place is the
consumer's own state (a file, a Redis key written under its own
prefix, an in-memory variable if the consumer is OK with at-most-once
on its own crash).
- No server-side "ACK" call. The cursor *is* the ack — when the consumer
successfully writes Last-Event-ID = N to its own state, entries ≤ N
are considered handled. The SSE handler does nothing.
### Endpoint spec
```
GET /api/events/files
Request headers:
Last-Event-ID: <stream-entry-id> (optional; resume from this cursor)
Accept: text/event-stream
Response:
Content-Type: text/event-stream
Cache-Control: no-cache
Connection: keep-alive
X-Accel-Buffering: no (disable nginx response buffering)
```
Same shape for `/api/events/meta`.
### Event format
One SSE event per Redis Stream entry. Format:
```
id: 1747999999999-0
event: add
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1747999999999}
id: 1748000000123-0
event: change
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1748000000123}
```
- `id` — **opaque** cursor. Today it's the Redis Stream entry ID
(`<ms>-<seq>`); consumers MUST treat it as a black-box string they
echo back via `Last-Event-ID`. This contract lets meta-core swap the
backing store later without breaking every consumer's cursor handling.
- `event` — the `type` field from the underlying entry (`add` | `change`
| `delete` | `rename` | `reset` for `file:events`; `set` | `del` |
`expire` for `meta:events`).
- `data` — JSON-encoded rest of the entry. Keys match the existing stream
payload exactly so downstream consumers don't change their parser.
### Backing-store retention and lifecycle
**Contract: the SSE wire is a faithful mirror of the underlying Redis
Streams.** Consumers rely on `Last-Event-ID` resumption working across
*every* kind of meta-core hiccup, including a restart — that's the entire
point of having a cursor protocol. The backing store must therefore
preserve entries across meta-core process restart and bound its growth
explicitly, not implicitly.
Today's behaviour is wrong on both counts:
| Stream | Restart behaviour | Retention |
|---|---|---|
| `meta:events` | **Cleared on every MetaPublisher.Start** (`events/meta_publisher.go:50-58`) | No `MAXLEN` (unbounded) |
| `file:events` | Persists across restart | No `MAXLEN` (unbounded), cleared only on deliberate reset (`watcher/dispatcher.go:82-95`) |
The `meta:events` clear-on-restart is the load-bearing problem. With it
in place, every meta-core restart silently invalidates every consumer's
persisted cursor — the SSE `gap` event would fire on the first reconnect
after every restart, and the consumer would have to re-bootstrap. That
turns "restart meta-core to apply a config change" into "every downstream
service does a full re-sync," which is exactly the regression SSE is
supposed to prevent.
PR A must fix both streams:
1. **Remove the unconditional `Del(MetaEventsStream)` in
`MetaPublisher.Start`.** Entries persist across restart. The republish
path (`RepublishAllMetadata`) stays as a one-shot bootstrap for
missing entries, but it does not pre-truncate the stream.
2. **Add `MAXLEN ~ N` to every `XAdd` on both streams** so growth is
bounded. Approximate trim (`~`) is cheap. Default `N = 100_000` per
stream is fine for the foreseeable workload (a few KB per entry,
bounded total ≈ a few hundred MB) and easy to tune via config.
3. **Keep the deliberate `EmitReset → ClearStream` path on
`file:events`.** That clear is a feature: a reset is the consumer
contract for "throw out everything you knew, re-bootstrap." Consumers
detect it through the `event: reset` payload that's already emitted
as the first entry after the clear (`dispatcher.go:99-`). The SSE
`gap` event handles the race where a consumer's cursor was trimmed
*between* the clear and its reconnect.
After PR A, the cursor contract is **"durable across meta-core restart
and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
when they persist a Last-Event-ID.
### Heartbeats
Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
when no events are flowing. Keeps reverse proxies and load balancers
from killing the connection as idle. The client ignores comment lines
per the SSE spec.
### Reconnect semantics
Per the SSE spec the client (or `EventSource`) auto-reconnects after a
network error, sending `Last-Event-ID`. The server resumes from the next
entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
**Gap handling.** If the requested ID has been trimmed out of the Redis
Stream by retention, the server emits one synthetic event before resuming
from the oldest available entry:
```
event: gap
data: {"requested":"<id>","resumeFrom":"<oldest-id>","reason":"retention"}
```
Consumers that care about gap-free delivery (currently none — the
schema indexer tolerates gaps; ingest tasks rebuild from scratch) can
react. Consumers that don't care ignore it.
### Error handling
- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
- **Redis fails mid-stream**: close the SSE connection. Client reconnects
with `Last-Event-ID`; the cycle resumes when Redis recovers.
- **Client disconnects**: handler goroutine exits cleanly; no per-client
state on meta-core.
### Auth
`/api/events/*` bypasses the hash-lock perimeter, same pattern as
`/api/meta/{cid}` and `/api/file/{cid}`. Event streams are internal
infrastructure and need to be reachable by peer services without going
through Authelia.
**Bypass ≠ public.** "Hash-lock bypass" means the OIDC sidecar doesn't
gate the route; it does *not* automatically mean Caddy proxies it from
the public hostname. The two existing precedents differ on this:
`/api/file/{cid}` is publicly reachable (peer-to-peer content fetch is
its job), `/api/meta/{cid}` likewise. For the SSE endpoints, the choice
is deliberate:
- If kept inside-only (no Caddy upstream for `/api/events/files` and
`/api/events/meta`), only sibling services on the docker network can
subscribe. This matches the rest of the "Redis is private" model and
is the recommended default.
- If exposed publicly, any browser can open an `EventSource` against
the stream without authentication — i.e. anyone can watch every file
event in the library. Only do this if there's a concrete consumer
that needs it and the visibility is acceptable.
PR A's Caddy change (see PR A scope, item 4) makes this an explicit
choice rather than a default behaviour. The recommendation is
inside-only until a concrete external consumer appears.
### Implementation footprint
The SSE handler itself is ~80 lines of Go in meta-core: one handler per
stream, each running an `XREAD … BLOCK 30s` loop and writing
SSE-formatted lines to the response. Go's `net/http` natively supports
streaming responses with `http.Flusher.Flush()`. Standard SSE clients
exist for every language we use: Node `eventsource`, Rust
`eventsource-client` (or `reqwest` + `tokio_util::codec`), Python
`sseclient`, Go `r3labs/sse`.
PR A as a whole is larger than the handler — see the PR A scope below
for the full list (backing-store fixes, Caddy wiring, poll deprecation)
that has to land alongside the handler for the cursor contract to hold.
## Eventing — considered alternatives
Recorded for posterity. None of these are being implemented; if any are
ever revisited it should be a deliberate reopen, not a drift.
- **Long-polling.** Strictly worse than SSE — connection churn on every
event, latency floor at the poll interval, no built-in resumption.
Rejected.
- **Webhooks (meta-core POSTs to subscribers).** Symmetric coupling:
every consumer would need to run an HTTP server reachable from
meta-core. Out-of-order delivery, retry, dead-letter all need
reinventing. Right for cross-organization integrations, wrong for
in-cluster eventing. Rejected.
- **External broker (Kafka / NATS / RabbitMQ).** Industrial-grade
pub/sub. New dependency, new operational surface, overkill for the
scale (single-host dev, ≤20-peer POC). Revisit if MetaMesh ever needs
cross-WAN federation at high throughput. Rejected for now.
- **Two Redises (writes Redis private, events Redis exposed).** Strongest
separation. Doubles operational complexity for no benefit over the
SSE design — once events go through HTTP, the events-Redis isn't
exposed either. Rejected.
- **Redis ACLs (Option B from earlier brainstorm).** Allows direct Redis
access from clients while restricting commands by user. Adds
credential plumbing and ACL config to operate; lets a buggy service
still connect, just with limited verbs. The SSE design closes the
network entirely, which is strictly stronger. Rejected — but the
equivalent escape hatch (open Redis to other services again) stays
available if we ever need it.
## Industry context — where this lands
The pattern we're adopting is the **Kubernetes / etcd** pattern, and
that's the most informative comparison.
Kubernetes is structurally identical to MetaMesh:
- A single source-of-truth store (etcd) holds all cluster state.
- A single service (kube-apiserver) is the sole writer to etcd.
- Many polyglot clients (kubectl, controllers, schedulers, kubelets,
operators) read and react to changes.
- Clients subscribe to *watch streams* for events.
What Kubernetes does, point-for-point:
- **etcd is never reached by clients.** It runs on a separate port,
often a separate network. kube-apiserver is the only thing that holds
etcd credentials. The lock is by network topology, not authentication.
- **Watch is also mediated.** kube-apiserver exposes `/watch` as an HTTP
streaming endpoint (very similar to SSE in shape). Clients don't watch
etcd directly — even though etcd has its own native watch primitive.
- **Storage portability falls out for free.** etcd could be swapped for
another store without any client change. We don't plan to swap Redis,
but the optionality is real and free.
Other systems with the same shape:
- **HashiCorp Consul.** Clients talk to a local agent over HTTP. The
agent talks to the cluster. KV writes and watches both flow through
the agent. Direct raft-store access is impossible.
- **Apache ZooKeeper.** Clients use the ZooKeeper client library which
speaks the ZooKeeper wire protocol; the data store is private.
- **Strapi / Sanity / Contentful** (headless CMS as analog). All
consumers — frontends, backend services, build pipelines — go through
the CMS HTTP API. Direct Postgres or content-store access is impossible
by design.
Where MetaMesh sat *before*: closer to the bad-monolith pattern
(everyone connects directly to the shared Redis). Where it sits *after*
this design: aligned with the canonical microservices pattern, and
specifically with the Kubernetes flavour where streams are mediated too.
### Advantages we're picking up
- **Schema evolution.** One service can change its data layout without
coordinating releases across every other service. The UUID-root
migration is the cautionary tale of what happens without this.
- **Validation choke point.** Every write passes through one code path
that can enforce invariants (reverse index, canonical_cid, type
constraints, schema version).
- **Observability.** Audit logs, request metrics, slow-query detection
all live at one layer.
- **Threat-model fit.** Untrusted-service-in-the-network is now in
scope; network isolation makes the concern moot.
- **Storage portability.** Optionality only, but real.
### Drawbacks (acknowledged)
- **Latency.** One extra hop per write (TCP + HTTP parse + handler).
Local-host overhead is sub-millisecond; mitigations are batched writes
(PATCH the whole metadata blob per plugin completion, not per field)
and Keep-Alive / HTTP/2 connections.
- **SSE-handler plumbing.** ~80 lines per stream endpoint plus per-client
goroutines. Trivial at our scale, real at Kubernetes scale (which is
why they invested in chunked watch with bookmarks, resource versions,
etc.). We can defer the same optimisations forever.
- **API surface size.** Every operation needs a corresponding endpoint.
Most of it already exists; one new SSE endpoint per stream is the
only new surface area introduced here.
## Enforcement — what "lock" actually means
Three layers, in order of effectiveness:
1. **Network isolation (hard).** Redis is on a container-private port.
No host port mapping. No docker-network alias. `kv-leader.info` does
not publish a Redis URL. Any service that wants to talk to Redis
needs container privileges, and at that point the lock isn't the
concern — operating-system isolation is. This is the actual lock.
2. **Documentation (convention).** This file, plus a one-line note in
`CLAUDE.md` and in each consumer service's `CLAUDE.md`: "All metadata
I/O routes through meta-core's HTTP API. Redis is private to
metacore-app." Engineers reading the codebase find the rule on first
contact. This catches the case of a future meta-core PR that
accidentally exposes Redis again.
3. **Code review (convention).** PRs that re-expose Redis (a port
mapping, an alias, a credential leak via leader-info) get rejected
with a pointer to this doc.
No runtime canary or lint is proposed — the network isolation makes
violations physically impossible from outside, and inside meta-core the
small file count makes review sufficient. Layer 1 does the heavy
lifting.
## Migration plan
Four PRs, additive, ordered so the system stays working throughout.
Closing the network access (PR D) is the moment the lock becomes a
lock; everything before it is opt-in migration that consumers can adopt
at their own pace.
**All four PRs have landed.** What follows is the original plan as
written; see "Implementation outcome" at the bottom for the delta
between plan and ship.
### PR A — SSE endpoints in meta-core
Adds `GET /api/events/files` and `GET /api/events/meta`. Implements the
`Last-Event-ID` semantics, gap handling, heartbeats, and the per-stream
XREAD loop. Redis Streams stay reachable directly; consumers can
migrate one at a time.
Scope, in PR-order:
1. Backing-store fixes (see "Backing-store retention and lifecycle"
above) — remove the `meta:events` clear-on-restart, add `MAXLEN ~ N`
to every `XAdd`. These are prerequisites for the SSE cursor contract
to hold; without them PR A is not deliverable.
2. SSE handlers for the two streams.
3. Deprecate `GET /api/events/poll` (`watcher/handlers.go:29`). It
exists today as a long-poll variant over the same `file:events`
stream and is described in code as "always empty in practice"
(`watcher/handlers.go:55-58`). After SSE lands it's redundant — keep
it responding 200 for one release with a `Deprecation` header, then
remove. The `/api/scan/*` deprecation pattern in the same file is
the template.
4. Caddy perimeter wiring (see "Auth" subsection above) — explicit
allow-list for `/api/events/files` and `/api/events/meta` matching
whatever the perimeter policy decides. Either inside-only via the
docker network, or public via Caddy with deliberate authorisation.
Do not let the routes default-leak past the hash-lock.
Roughly 200 lines of Go + ~50 lines of test (covering reconnect from a
cursor, gap emission when the cursor is trimmed, heartbeat cadence,
durability across a meta-core restart).
### PR B — meta-sort writes via HTTP
Swap `RedisClient` for an `HttpKVClient` implementation of the existing
`IKVClient` interface in `packages/meta-sort-core`. All writes
(`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
calls against the **`/meta/{hash}` family** — the canonical write
surface for service-to-service traffic. Per-plugin write coalescing
uses `PATCH /meta/{hash}` (already calls `MergeMetadataFlat`
server-side with the partial-update semantics we need; see
`handlers.go:533-566`).
Reads from Redis stay direct in this PR (no consumer migration yet);
only writes move. Streams stay on direct XREAD.
**Pre-audit (already done):** meta-sort's production `kvClient.*`
surface is `setMetadataFlat` / `deleteMetadataFlat` / `getMetadataFlat`
/ `getMetadata` / `getAllHashIds` / `health` plus the `file:events`
stream consumer. No orphan keyspaces, no generic `kv.set(arbitrary_key)`
callers — the HTTP migration covers the entire write path with no
side bets.
### PR C — Stream consumers migrate to SSE
meta-sort's event consumer (`FileEventConsumer.ts`), meta-fuse's stream
reader, meta-stremio's event hook, meta-dup's monitor, meta-share's
ingest task — each swaps its `XREADGROUP` / `XREAD` loop for an SSE
client. Per-language standard libraries make this small (one HTTP
request, one event-handler closure).
In the same PR, each consumer's `LeaderClient` (TypeScript x2, Python,
plus any new Rust binding for meta-share if added) drops its
*requirement* on the `redisUrl` field — i.e. it stops opening a Redis
connection. It can still tolerate the field being present (PR D removes
it from the publisher side). This ordering matters: PR D's loud-failure
guard cannot ship until every consumer has been demonstrated to boot
without `redisUrl`.
The schema indexer is internal to meta-core and unaffected.
### PR D — Close Redis network access
PR D must land **after** every consumer in PR C has stopped requiring
`redisUrl` (verify by booting each service against a meta-core build
that already omits the field — staging environment or feature-flagged
release). Until that's true, this PR will deadlock the boot sequence.
Then, in this order:
1. Update consumer-side discovery (`LeaderClient` in each service) to
*log loudly* if it sees a `redisUrl` field — guards against
accidental rollback. (Loud log, not hard failure, while the field
is still being written — flip to hard failure once step 4 lands.)
2. Stop publishing `redisUrl` from `/urls` and from `kv-leader.info`.
3. Remove the `meta-core` docker-network alias on the metacore-app
service (other services no longer need DNS for the Redis port).
4. Remove `6380:6379` host port mapping in `dev/docker-compose.yml`.
5. Flip the LeaderClient guard from "warn" to "raise" — any future
consumer that tries to require the field fails loudly at boot.
After PR D, attempting to connect to Redis from anywhere except
metacore-app produces a connection refused. The lock is real.
### Out of scope for this work
- **Redis ACLs / proxy.** Network isolation makes these unnecessary.
Available as an escape hatch if the threat model ever changes.
- **Replacing Redis Streams under the hood.** SSE is the externalised
contract; the backing store is an implementation detail meta-core can
change later.
- **External brokers.** Out of scope absent a concrete driver
(cross-WAN federation at high throughput).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| Writes from non-meta-core services? | Forbidden by network topology | Redis unreachable; not a convention |
| Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
| Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
| Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
| Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
| Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
| **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
| **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
| **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
## Error envelope
Standard shape for all 4xx / 5xx responses from the metadata API:
```json
{
"error": "alias_collision",
"message": "CID midhash256:bafk… already resolves to UUID 01JKR…",
"retryable": false
}
```
Stable `error` slugs the client should switch on:
| Slug | HTTP status | Retryable | Meaning |
|---|---|---|---|
| `alias_collision` | 409 | no | A different UUID already owns the CID the caller is trying to register. Caller bug; escalate. |
| `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
| `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
| `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
| `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
**Implemented.** Every 4xx/5xx response across `internal/api`,
`internal/mounts`, `internal/watcher`, and `internal/watchers` now
emits the typed envelope. The slug-mapping helper is duplicated across
those packages (one ~15-line `slugForStatus` per package) to avoid an
import cycle on the api package; consolidating it into a shared
`internal/httperr` package is a worthwhile follow-up but not load-bearing.
## Remaining open questions
Mostly answered post-implementation:
- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
editor's React UI continues to call it; the surface itself was untouched
by this work. Worth a follow-up to consolidate the editor onto
`/meta/{hash}` and delete the parallel routes.
- **Initial bootstrap flood on `/api/events/files`.** Resolved in
practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
replay every entry on each restart and the StreamingStateBuilder
handles the resulting burst (~2.5k property events on a 50-file
library) without issue. meta-sort persists its cursor so only sees
the flood once per fresh deploy. meta-stremio filters to "interesting
fields" before triggering cache invalidation, so the burst is mostly
no-ops.
- **meta-dup SSE port.** Still pending — meta-dup is in no active
compose file. When revived, copy meta-fuse's pattern wholesale
(SSEEventClient + MetaCoreApiClient). Tracked separately.
## Non-goals
- **Authentication between internal services.** The hash-lock OIDC
perimeter handles external clients; internal service-to-service
trust is by network isolation (docker network for dev, equivalent
isolation in prod deployments). Adding service-to-service auth is a
separate concern.
- **Replacing the public HTTP API surface.** This proposal exercises
what already exists; the only new endpoints are the two SSE streams.
- **Introducing new transport, broker, proxy, or sidecar.** Everything
builds on what's deployed today.
- **Migrating meta-fuse / meta-stremio / meta-dup direct Redis reads
via XSCAN/GET** (separate from event-stream reads). ~~Was a non-goal~~
ended up being required for PR D's network closure to be safe.
meta-fuse and meta-stremio migrated; meta-dup deferred (not deployed).
See PR C / "Implementation outcome" for details.
## Implementation outcome
What actually shipped vs. what this doc planned.
### Delta against the original plan
- **Read-path migration for meta-fuse and meta-stremio is now in scope
and done.** The original "out of scope" carve-out was wrong: closing
the Redis network port doesn't just break stream consumers, it also
breaks every `kvClient.get` / `redis_storage.get_all_videos` call.
Both services grew their own `MetaCoreApiClient` (TS for meta-fuse,
Python for meta-stremio) and now route reads through `/meta/{hash}`
+ `/meta` + `/api/file/{cid}/info`.
- **meta-stremio's HTTP mode auto-recovers from transient meta-core
outages.** `is_connected` re-probes meta-core's `/health` on every
call (cheap; a stateless GET), so a metacore-app restart no longer
latches the addon into a permanent disconnected state. The
background reconnect thread is only needed for legacy Redis mode
where the socket itself is sticky.
- **Hard `redisUrl` guard is gated by `ALLOW_LEGACY_REDIS_URL=1`.** All
three consumer LeaderClients (TS x2 + Python) throw on seeing the
field by default. Setting the env var downgrades to a warn — escape
hatch for a deliberate temporary rollback to a meta-core that still
publishes the field.
- **Write-path CID resolution** turned out to be a hard prerequisite
for the lockdown to work in practice. Documented separately below.
- **PR D ordering matched the doc** (consumer migration first, then
publisher stops emitting, then network closure). The `meta-core`
docker-network alias and the `6380:6379` host port mapping are both
removed; sibling services reach the HTTP/SSE/WebDAV surface via the
canonical `metacore-app` hostname.
- **meta-dup is the one service that didn't migrate.** It's in no
active compose file. The TS port is straightforward when needed: copy
meta-fuse's pattern (SSEEventClient for streams, MetaCoreApiClient
for reads).
### Write-path CID resolution (`storage.ResolveRoot`)
Not in the original plan but load-bearing.
**The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed
every write through `/meta/{hash}`. But `{hash}` is the midhash256 CID
that meta-sort received via file:events, not the watcher's minted UUID.
With no resolution step in the handler, every write went to
`file:<midhash>/*` — a parallel root the watcher's UUID never saw. The
auto-alias hook on `cid_midhash256=<value>` then OVERWROTE the
watcher's `cid:midhash256:<value>` → UUID alias to point at the
midhash itself, leaving the UUID stranded with only the watcher's
sparse fields (filePath, sizeByte, mtimeNano, duplicates) and the
midhash root carrying the rich plugin output. This is the exact
"dual roots per file" failure the doc's Motivation section listed as
the prior bug — it just moved from "convention violation" to "API
contract violation."
**The fix.** Three layers in `internal/storage/cid_resolution.go`:
1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
read/write handler, look up `cid:midhash256:<hash>` in the reverse
index. If found, use the UUID as the storage root. Falls through to
the bare hash for direct-UUID writes / legacy entries.
2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
register `cid:<algo>:<v>` → `<v>`. The dual-root bug's chicken-and-
egg is: meta-sort writes first, registers self-pointing alias,
watcher then can't recover. Refusing the self-write keeps the
watcher's legitimate alias intact.
3. **`MigrateDualRoots` sweep** (`POST /api/admin/migrate-dual-roots`).
One-shot fixer for already-stranded entries: walks every UUID's
`midhash256` field to build the orphan index, merges each stranded
`file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
deletes the stranded root, and re-points the alias.
Verified on the dev stack: 48 stranded entries were reunited; the
editor now shows every file with its full plugin metadata under one
UUID; future writes stay unified.
### Typed error envelope
Done across all four handler packages (`api`, `mounts`, `watcher`,
`watchers`). Each has its own `writeError` returning
`{error: <slug>, message, retryable}`. The slug-mapping helper is
duplicated in each to avoid an import cycle on the `api` package.
### What's NOT done
- **meta-dup SSE + read migration.** Not deployed, no test harness.
When revived, copy meta-fuse's pattern.
- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate
editor-internal API surface is untouched; could be retired in a
follow-up by porting the editor UI to `/meta/{hash}`.
- **Consolidated `internal/httperr` package** for the typed envelope
helpers (currently duplicated 4× to avoid the import cycle). Code
hygiene; functionally a no-op.
### Verification done
- All four service dashboards load cleanly with no console errors.
- meta-sort SSE consumer + HTTP write path verified end-to-end by
delete + inject + reprocess of Sintel.
- meta-fuse SSE consumer rebuilds VFS to 48 files on each boot.
- meta-stremio catalog API now surfaces the full library (47 videos,
1 movie + 5 series + 46 episodes — was 0/0/0 before the Python
read-path migration because LeaderStorage broke when it tried Redis).
- `redis-cli` from any container other than `metacore-app` produces a
connection refused; `cid:midhash256:<H>` aliases all point to UUIDs
(no self-pointing); `file:__index__` has only UUID-style hashIds.
- ALLOW_LEGACY_REDIS_URL=1 escape hatch present in all three consumer
LeaderClients (TS x2 + Python).
### Known test-suite gaps (post-implementation bats run)
Running `docker exec meta-test-runner /app/test/test.sh` after the
lockdown shipped reveals a number of failing suites, almost all of
which are pre-existing test/environment drift unrelated to this work:
| Failure | Root cause | Relation to lockdown |
|---|---|---|
| meta-stremio (~35 tests), stremio-tmdb (6 tests) | Tests call `/manifest.json` directly; dev `.env` sets `STREMIO_HASH_API_SEED=dev-stremio-secret` which puts the addon behind a path-token prefix | Unrelated — pre-existing env mismatch |
| meta-fuse FUSE-mount (3 tests) | Tests use `docker exec meta-fuse` but the container is named `metafuse-app` (Yundera prod-parity rename) | Unrelated — predates this work |
| meta-fuse WebDAV PROPFIND (4 tests) | Tests use Caddy URL with WebDAV basic-auth credentials that no longer match the hash-lock perimeter | Unrelated — predates this work |
| meta-sort "API integration tests" (1 test) | Test runs `docker exec meta-sort pnpm test`; container is `metasort-app` | Unrelated — predates this work |
| meta-core "create and delete mount config" (1 test) | Test creates an NFS mount; production code dropped NFS support ("NFS mounts are no longer supported") | Unrelated — predates this work |
| service-discovery (9 tests) | Cascading from meta-stremio breakage (test expects unauth `/api/services` on stremio) | Unrelated to lockdown directly |
| meta-share / share-1c (multiple) | Tests assert `"phase":"1f"` in `/health`; meta-share has moved past that phase | Unrelated — meta-share evolution |
| flat-keys: "filePath field exists" (1 test) | Test SRANDMEMBERs file:__index__; if it lands on a UUID with no filePath (e.g. a manual test write) it fails | Was hit transiently after my interactive testing left stub entries; resolved by cleanup |
No failure traceable to a real regression from this work was found.
The dev-environment test harness needs an unrelated refresh to catch
up with the prod-parity container renames, the NFS removal, the
STREMIO_HASH_API_SEED default, and meta-share's current phase.
File diff suppressed because it is too large Load Diff
+211 -534
View File
@@ -1,595 +1,272 @@
# Redis Metadata Storage Architecture # Redis Metadata Storage Structure
This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling. How meta-core lays out file metadata in Redis. This document covers the
on-disk shape (UUID-rooted flat string keys, CID reverse index,
per-root sets) and the schema-version gate. It does **not** enumerate
field semantics — that's the job of the repo-root
[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key
registry for everyone who writes to or reads from metadata.
## Overview For the design rationale (why UUIDs instead of content hashes, why a
reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
For the external-access contract (HTTP-only, SSE for events), see
[`api-mediated-access.md`](api-mediated-access.md).
MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables: ## Quick summary
- **Redis keyspace notifications** for field-level change detection - **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates) not content hashes. Examples in this doc use the placeholder
- **meta:events stream** integration for real-time updates across services `01JKR8XW5T4QMV7AHNJ2DEFGHK`.
- Individual field updates without affecting other properties - **Properties are flat Redis STRING keys**, one per leaf:
- Fast VFS lookups using content-based hash IDs `file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
- **CIDs are reverse-index aliases**: `cid:<algorithm>:<value>` → `<uuid>`
(a Redis STRING per known CID).
- **Per-root sets**: `file:<uuid>/cids` (every CID), `file:<uuid>/duplicates`
(alternate paths with the same content), plus `file:__index__` (every UUID).
- **A schema-version sentinel** (`meta-core:schema-version`) refuses to
boot against legacy data.
## Key Structure Implementation lives in `internal/storage/client.go`,
`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
`internal/storage/schema_version.go`.
### Flat Key Storage ## Key structure
Each file property is stored as a separate Redis STRING key: ### Roots
``` ```
Key Pattern: file:{hashId}/{property/path} file:<uuid>/<property/path> → Redis STRING (always)
Key Type: STRING
Value: String value
``` ```
**Example Redis commands:** UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
```bash UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
# Store file metadata (each property is a separate key) time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..." in a given window, which restored some of the debug-ability lost when
SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv" midhash256 stopped being the root.
SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5"
SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm"
# Retrieve single field Properties use `/` as the path separator. Nested objects in source code
GET "file:bafkr4ih5kapbjzq.../fileinfo/duration" are flattened to flat keys on write and reconstructed on read. Examples:
# Retrieve all metadata for a file (scan by prefix)
SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000
```
### Why Flat Keys?
The flat key structure (vs. Redis Hashes) enables:
1. **Keyspace Notifications**: Redis publishes events when individual keys change
2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*`
3. **Selective Updates**: Update one field without touching others
4. **Stream Integration**: The `meta:events` stream captures field-level changes
### File Index
A Redis Set tracks all known file hash IDs:
``` ```
Key: file:__index__ file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv"
Type: Set file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536"
Value: {hashId1, hashId2, hashId3, ...} file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..."
``` ```
This enables fast bulk queries like "get all files" without scanning keys. ### Reverse-index aliases (CIDs)
Every CID known for a file is registered as its own Redis STRING:
```
cid:<algorithm>:<value> → <uuid>
```
Examples:
```
cid:midhash256:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:sha256:abc... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:ipfs:bafy... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
registered CID, independent of which hash algorithm "primary" — there is
no primary. The auto-alias hooks in `SetProperty` and `SetMetadataFlat`
(see `internal/storage/cid_resolution.go`) catch any write of a `cid_*`
or `midhash256` field and register the alias automatically, so plugins
don't need to know about the reverse index.
### Per-root sets
```
file:<uuid>/cids → Redis SET of "<algorithm>:<value>" tokens
file:<uuid>/canonical_cid → STRING — the highest-ranked CID for advertising externally
file:<uuid>/duplicates → Redis SET of alternate paths with same content
```
`canonical_cid` is reconciled by `reconcileCanonicalCIDLocked` whenever
the `cids` set changes; ranking is done by `internal/cid/rank.go`.
### Global index
```
file:__index__ → Redis SET of all UUIDs
```
`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
every file without scanning the keyspace.
```bash ```bash
# Get all hash IDs SMEMBERS file:__index__ # all UUIDs
SMEMBERS "file:__index__" SCARD file:__index__ # total file count
SISMEMBER file:__index__ <uuid>
# Count total files
SCARD "file:__index__"
# Check if a file exists
SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..."
``` ```
### Keyspace Notifications ### Schema-version sentinel
Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration:
```bash
# Required Redis config (set by meta-core)
CONFIG SET notify-keyspace-events "K$"
# K = Keyspace events (published on __keyspace@<db>__:<key>)
# $ = String commands (SET, etc.)
```
Services can subscribe to keyspace events for real-time updates:
```bash
# Subscribe to all file key changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Subscribe to specific field pattern (e.g., TMDB updates)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### meta:events Stream
The `meta:events` stream provides reliable event delivery for file system events from meta-core:
``` ```
Stream Key: meta:events meta-core:schema-version → STRING (currently "2")
Entry Fields:
- type: add | change | delete | rename | reset
- path: File path relative to /files
- midhash256: Content hash (for add/change events)
- oldPath: Previous path (for rename events)
- timestamp: Event timestamp
``` ```
Consumer groups enable reliable processing with acknowledgment: Set at first boot, checked on every subsequent boot
(`internal/storage/schema_version.go`). If the sentinel is missing but
`file:__index__` is non-empty, the gate refuses to boot (legacy data).
If the sentinel is present but doesn't match `SchemaVersion`, the gate
also refuses to boot. Alpha-stage policy: operators wipe and restart;
there is no automated migration.
```bash ## Why flat STRING keys?
# Create consumer group
XGROUP CREATE meta:events mygroup 0 MKSTREAM
# Read new events (blocking) The store does not use Redis Hashes. The reasons are:
XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events >
# Acknowledge processed events 1. **Field-level keyspace notifications.** Redis publishes
XACK meta:events mygroup <entry-id> `__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
``` you'd only get one notification on the hash itself. meta-core's
`MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
notifications and publishes structured events to the `meta:events`
Redis Stream.
2. **Selective updates without read-modify-write.** A plugin writing
`tmdb/poster` doesn't have to read the rest of the document.
3. **Compatibility with the CID alias hooks.** Field writes can trigger
alias-maintenance work in `SetProperty` without the plugin caring.
## Primary Hash ID ## Reads and writes
The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from: The Go API in `internal/storage/client.go`:
1. SHA-256 hash of the middle 1MB of the file | Method | Behaviour |
2. File size in bytes |---|---|
3. IPFS CID v1 encoding (multicodec 0x1000) | `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register any `cid_*` aliases. |
| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
| `AddAlias(uuid, "<algo>:<value>")` | Set `cid:<algo>:<value>` → `<uuid>`, add to `file:<uuid>/cids`, reconcile `canonical_cid`. |
| `ResolveCID("<algo>:<value>")` | `GET cid:<algo>:<value>` → uuid. |
**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...` ## Reconstruction (flat → nested)
This provides: When the HTTP API returns a document, it reconstructs the flattened
- Content-based addressing (same file = same ID) storage back into a nested JSON object:
- Fast computation (only reads 1MB)
- IPFS compatibility for future distributed storage
### Collision Handling
If two different files produce the same midhash (rare), the system falls back to full SHA-256:
```typescript
// Normal case
hashId = midHash256; // "bafkr4ih5kapbjzq..."
// Collision detected (different filePath with same midhash)
hashId = metadata['cid_sha2-256']; // Full hash fallback
```
## Metadata Flattening
Nested objects are flattened into path-based keys for efficient Redis storage.
### Flattening Rules
```typescript
// Input (nested object)
{
title: "Inception",
video: {
codec: "h265",
width: "1920"
},
streams: [
{ language: "eng" },
{ language: "jpn" }
]
}
// Output (flattened for Redis)
title = "Inception"
video/codec = "h265"
video/width = "1920"
streams/0/language = "eng"
streams/1/language = "jpn"
```
**Key rules:**
- Nested objects use `/` as path separator
- Arrays use numeric indices (0, 1, 2...)
- All values stored as strings
- `null`/`undefined` stored as empty strings
- Numbers and booleans converted via `String()`
### Reconstruction
When reading, the flattened structure is reconstructed into nested objects:
```typescript
const metadata = await kvClient.getMetadataFlat(hashId);
// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... }
```
## Plugin Metadata Structure
Each plugin writes metadata using a prefixed key pattern:
``` ```
{plugin-id}/{field} # Simple field flat nested
{plugin-id}/{group}/{field} # Grouped field ───────────────────────────────────── ───────────────────────────
{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays) title = "Inception" title: "Inception"
fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5",
fileinfo/streamdetails/ streamdetails: { video: { 0: {
video/0/codec = "h264" codec: "h264", ... } }, ... } }
streams/0/language = "eng" streams: [ { language: "eng" }, ... ]
``` ```
### Plugin Contribution Flow Rules:
- `/` is the path separator.
- Numeric path segments (`0`, `1`, ...) become array indices on read.
- All values are strings on the wire; type interpretation is the
caller's job (and is documented per-field in `METADATA_KEYS.md`).
- `null`/`undefined` round-trip as empty strings.
## Plugin contribution flow
Plugins don't talk to Redis. They write metadata via meta-core's HTTP
API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
writes under a prefix:
``` ```
File Discovered {plugin-id}/<field> single value
↓ {plugin-id}/<group>/<field> grouped
[FileProcessorPiscina] {plugin-id}/<group>/<n>/<field> indexed (array)
├─ Compute midhash256
├─ Store preliminary metadata
└─ Dispatch plugin tasks
↓
[Plugin Processing]
├─ file-info → fileType, mimeType, sizeByte
├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/...
├─ filename-parser → titles/eng, season, episode, videoType
├─ anime-detector → anime, titles/jpn, titles/jpl
├─ tmdb → tmdbid, plot/eng, genres
└─ full-hash → cid_sha2-256, cid_sha1
↓
[Plugin Callbacks]
└─ Store metadata via flat keys (SET per property)
``` ```
## Core Metadata Types See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full,
authoritative list of fields each plugin writes (content identification,
file stats, video classification, anime detection, subtitle references,
Jellyfin metadata, FFmpeg stream details, etc.) and the value formats
(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc
deliberately does not duplicate that registry — it would drift.
### HashMeta - Content Identification ## Event streams
``` Two Redis Streams are produced inside meta-core:
cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132)
cid_md5 = "bafkr..." # MD5 (multicodec 0xd5)
cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11)
cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12)
cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000)
```
### FileStatMeta - File Properties | Stream | Producer | Contents |
|---|---|---|
| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
``` External consumers do **not** read these streams directly. They subscribe
fileType = "video" # video, audio, image, text via SSE:
mimeType = "video/x-matroska" # MIME type
sizeByte = "4294967296" # File size in bytes
```
### VideoMeta - Classification - `GET /api/events/files` — mirror of `file:events`
- `GET /api/events/meta` — mirror of `meta:events`
``` The SSE wire contract uses `Last-Event-ID` for resume (the opaque
videoType = "movie" # movie, episode, unknown `<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
originalTitle = "Inception" (`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
titles/eng = "Inception" # Localized titles a client's cursor, the handler emits one `event: gap` payload before
titles/jpn = "インセプション" resuming. See `internal/api/sse_events.go` and
season = "1" [`api-mediated-access.md`](api-mediated-access.md) for the full
episode = "5" contract.
movieYear = "2010"
```
### AnimeMeta - Anime Detection Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
inside `MetaPublisher` to drain keyspace notifications, but this is not
``` the external surface — external services use SSE.
anime = "true" # "true" or "false"
titles/jpn = "進撃の巨人" # Japanese title
titles/jpl = "Shingeki no Kyojin" # Romaji
```
### SubtitleMeta - Subtitle References
```
subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID
subtitles/eng/1 = "bafks..."
subtitles/jpn/0 = "bafkt..."
```
### JellyfinMeta - Comprehensive Metadata
```
imdbid = "tt1375666"
tmdbid = "27205"
anidbid = "12345"
plot/eng = "A movie about..."
genre/0 = "Action"
genre/1 = "Sci-Fi"
rating = "8.8"
mpaa = "PG-13"
releasedate = "2010-07-16"
art/poster = "bafkr..." # Reference to poster image CID
art/fanart = "bafks..." # Reference to backdrop image CID
```
## FFmpeg Plugin - Special Case
The FFmpeg plugin extracts detailed stream information with a specific nested structure.
### Stream Details Structure
```
fileinfo/duration = "7200.5"
fileinfo/formatName = "matroska,webm"
fileinfo/streamdetails/video/0/codec = "h264"
fileinfo/streamdetails/video/0/width = "1920"
fileinfo/streamdetails/video/0/height = "1080"
fileinfo/streamdetails/video/0/aspect = "16:9"
fileinfo/streamdetails/video/0/bitrate = "8000000"
fileinfo/streamdetails/audio/0/codec = "aac"
fileinfo/streamdetails/audio/0/language = "eng"
fileinfo/streamdetails/audio/0/channels = "6"
fileinfo/streamdetails/audio/1/codec = "dts"
fileinfo/streamdetails/audio/1/language = "fra"
fileinfo/streamdetails/audio/1/channels = "6"
fileinfo/streamdetails/subtitle/0/language = "eng"
fileinfo/streamdetails/subtitle/0/format = "subrip"
fileinfo/streamdetails/subtitle/1/language = "jpn"
fileinfo/streamdetails/subtitle/1/format = "ass"
fileinfo/streamdetails/embeddedimage/0/type = "cover"
```
### Stream Type Keys
| Stream Type | Key Pattern | Description |
|------------|-------------|-------------|
| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) |
| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks |
| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles |
| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails |
### Common Stream Fields
**Video streams:**
- `codec` - h264, hevc, av1, vp9, etc.
- `width`, `height` - Resolution
- `aspect` - Aspect ratio
- `bitrate` - Video bitrate
- `profile` - Codec profile (main, high, etc.)
- `level` - Codec level
**Audio streams:**
- `codec` - aac, dts, ac3, flac, etc.
- `language` - ISO 639-2 code (eng, jpn, fra)
- `channels` - Channel count (2, 6, 8)
- `bitrate` - Audio bitrate
- `samplerate` - Sample rate
**Subtitle streams:**
- `language` - ISO 639-2 code
- `format` - subrip, ass, pgs, vobsub
## Real-Time Notifications
MetaMesh provides multiple mechanisms for real-time metadata change notifications:
### 1. meta:events Stream (Recommended)
The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups:
```bash
# File system events from meta-core watcher
XREAD STREAMS meta:events 0
# Event types: add, change, delete, rename, reset
```
Benefits:
- **Reliable delivery** via consumer groups and acknowledgment
- **Replay capability** for catching up after restarts
- **Ordered processing** with stream IDs
### 2. Keyspace Notifications
For field-level change detection, subscribe to Redis keyspace notifications:
```bash
# All file metadata changes
PSUBSCRIBE '__keyspace@0__:file:*'
# Specific plugin updates (e.g., TMDB enrichment)
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
```
### Cache Invalidation
The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation:
```
┌─────────────────────────────────────────────────────────────────┐
│ Cache Invalidation Flow │
│ │
│ Redis Keyspace Cache │
│ ┌──────────────┐ Invalidator │
│ │ SET file:*/ │ ──────────► ┌──────────────┐ │
│ │ DEL file:*/ │ subscribe │ Listens to │ │
│ └──────────────┘ │ __keyspace@ │ │
│ │ 0__:file:* │ │
│ └──────┬───────┘ │
│ │ │
│ ▼ │
│ ┌──────────────┐ │
│ │ Remove from │ │
│ │ LRU cache │ │
│ └──────────────┘ │
└─────────────────────────────────────────────────────────────────┘
```
Flow:
1. Cache invalidator subscribes to `__keyspace@0__:file:*`
2. When metadata changes (SET/DEL), invalidator receives notification
3. Invalidator removes the corresponding file from the WebDAV cache
4. Next access fetches fresh copy from source filesystem
This ensures cached files stay synchronized with metadata changes.
### 3. Pub/Sub Channels (Legacy)
Batch notification channels are still supported:
```
meta-sort:file:batch # Batch file updates (every 5 seconds)
meta-sort:scan:reset # Full rescan notification
```
```json
{
"timestamp": 1704067200000,
"changes": [
{ "action": "add", "hashId": "bafkr4ih5..." },
{ "action": "update", "hashId": "bafkr4ih5..." },
{ "action": "remove", "hashId": "bafkr4ih5..." }
]
}
```
## Complete Example
Here's a complete metadata example for a movie file:
```bash
# Query all keys for a file
redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*'
# Example flat keys stored in Redis:
# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..."
# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv"
# file:bafkr4ih5kapbjzq.../fileType = "video"
# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5"
# file:bafkr4ih5kapbjzq.../titles/eng = "Inception"
# ...
# When reconstructed by the API (nested object representation)
{
# Content identification
"cid_midhash256": "bafkr4ih5kapbjzq...",
"cid_sha2-256": "bafks7ij2kbpck...",
"filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv",
# File properties
"fileType": "video",
"mimeType": "video/x-matroska",
"sizeByte": "4294967296",
# Classification
"videoType": "movie",
"originalTitle": "Inception",
"movieYear": "2010",
# Localized titles
"titles": {
"eng": "Inception",
"fra": "Inception",
"jpn": "インセプション"
},
# FFmpeg stream details
"fileinfo": {
"duration": "8878.5",
"formatName": "matroska,webm",
"streamdetails": {
"video": {
"0": {
"codec": "h264",
"width": "1920",
"height": "1080",
"aspect": "16:9",
"bitrate": "8000000"
}
},
"audio": {
"0": { "codec": "dts", "language": "eng", "channels": "6" },
"1": { "codec": "ac3", "language": "fra", "channels": "6" }
},
"subtitle": {
"0": { "language": "eng", "format": "subrip" },
"1": { "language": "fra", "format": "subrip" }
}
}
},
# External metadata (TMDB)
"tmdbid": "27205",
"imdbid": "tt1375666",
"plot": {
"eng": "A thief who steals corporate secrets through dream-sharing technology..."
},
"genre": ["Action", "Science Fiction", "Adventure"],
"rating": "8.4",
"releasedate": "2010-07-16",
# Media assets
"art": {
"poster": "bafkr4poster...",
"fanart": "bafkr4fanart..."
},
# Detection flags
"anime": "false"
}
```
## Key Implementation Files
| Component | File Path |
|-----------|-----------|
| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` |
| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` |
| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` |
| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` |
| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` |
| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` |
| meta-core Storage | `packages/meta-core/internal/storage/client.go` |
| meta-core Events | `packages/meta-core/internal/events/publisher.go` |
| meta-core Cache | `packages/meta-core/internal/cache/` |
## Debugging ## Debugging
```bash ```bash
# List all file hash IDs # Roots
docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__' docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
docker exec metacore-app redis-cli SCARD 'file:__index__'
docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
# Count total files # Full document for one root (flat scan)
docker exec meta-core-dev redis-cli SCARD 'file:__index__' docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
# Get a sample hash ID # One property
docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__' docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
# List all keys for a specific file (flat key scan) # Confirm keys are STRING (not HASH)
docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*' docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
# string
# Get specific field (flat key) # Reverse-index lookups
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration' docker exec metacore-app redis-cli GET 'cid:midhash256:bafkreih...' # → uuid
docker exec metacore-app redis-cli SMEMBERS 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids'
docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid'
# Get filePath for a file # Schema-version sentinel
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath' docker exec metacore-app redis-cli GET 'meta-core:schema-version'
# Verify keys are STRING type (not HASH) # Streams (server-side; external consumers should use SSE)
docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath' docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
# Should return: string docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
# Monitor metadata changes via stream # HTTP equivalents — preferred
docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0 curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>' # CID → document
curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs
# Monitor keyspace notifications (requires notify-keyspace-events enabled) curl -k 'https://metacore-dev.localhost:8083/meta/<uuid>' # document by UUID
docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*' curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE
curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats
# meta-core API debugging
# Cache status
curl http://localhost:9000/api/cache/status
# Clear cache
curl -X POST http://localhost:9000/api/cache/clear
# Watch events stream (SSE)
curl http://localhost:9000/api/events/stream
# KV browser - storage info
curl http://localhost:9000/api/kv/info
# Scan status
curl http://localhost:9000/api/scan/status
``` ```
## meta:events Stream ## Key implementation files
The `meta:events` stream captures metadata change events for real-time notifications: | Component | Path |
|---|---|
```bash | Redis client + flat-key ops | `internal/storage/client.go` |
# View recent events | UUIDv7 minting | `internal/storage/uuid.go` |
docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10 | CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
| Schema-version gate | `internal/storage/schema_version.go` |
# Read events from a consumer group | Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' > | Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
``` | Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
| SSE event mirror | `internal/api/sse_events.go` |
Stream events contain: | Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
- `type`: Event type (add, change, delete, rename, reset)
- `path`: File path
- `midhash256`: Content hash (for add/change)
- `timestamp`: Event timestamp
+322
View File
@@ -0,0 +1,322 @@
# UUID-Rooted Metadata with CID Aliasing
## Status
**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
## Motivation
Today, every metadata entry in Redis is keyed by a content hash, specifically
midhash256:
```
file:midhash256:abc.../filePath
file:midhash256:abc.../title
file:midhash256:abc.../fileinfo/duration
file:midhash256:abc.../tmdb/poster
...
```
Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields
*inside* that entry:
```
file:midhash256:abc.../cid_sha256 = sha256:def...
file:midhash256:abc.../cid_ipfs = bafy...
```
Two problems with this:
1. **midhash256 is privileged by accident.** It IS the key, while every other
CID is a secondary field. There's nothing intrinsically special about
midhash256 — it's just the first hash computed. Hash-algorithm changes, or
new digest types added by plugins, force rewrites of every root. The
"promotion" workarounds people reach for (re-keying entries when a stronger
hash arrives) solve the wrong problem; the privilege itself is the issue.
2. **CID lookup is O(n).** Resolving "given any CID, return that file's
metadata" currently scans every indexed entry (see
`storage/client.go::LookupPathByCID`, which acknowledges this in a
comment). Acceptable for admin tooling, unacceptable for a public API
surface — especially when peers on meta-share may query by any CID variant.
The product requirement is clear: **any CID a client knows about must resolve
to that file's metadata in O(1).** IPFS-compatible CIDs are the externally
meaningful identity for content retrieval; the internal storage layout should
not bias toward any one digest type.
## Goal
A single externally-meaningful contract:
```
GET /api/meta/<cid> → full metadata document for that file
```
Where `<cid>` may be any supported type (midhash256, sha256, IPFS, btih,
sha3, future additions). All CID types are treated symmetrically. No CID type
is "the key."
## Design
### Two-tier key structure
The root key becomes an opaque internal UUID; CIDs are aliases that point to
it.
```
file:<uuid>/<property...> ← canonical storage (UUIDv7, never disclosed externally)
cid:<cid> → <uuid> ← reverse index (one Redis key per known CID)
```
The metadata keyspace under `file:<uuid>/` remains open and flat, exactly as
today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
`file:<uuid>/fileinfo/duration`. Only the root identifier changes.
**Example layout:**
```
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = /watch/Inception.mkv
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size = 4500000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime = 1729872000000000000
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids = SET{midhash256:abc, sha256:def, ipfs:bafy...}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates = SET{/watch/dup1.mkv, /watch/dup2.mkv}
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid = ipfs:bafy...
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = bafy...
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng = ...
cid:midhash256:abc → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:sha256:def → 01JKR8XW5T4QMV7AHNJ2DEFGHK
cid:ipfs:bafy... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
```
### UUID format: UUIDv7
UUIDv7 chosen over UUIDv4 or 64-bit random:
- **128 bits**: collision-free under distributed minting; no central allocator.
- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a
given window. Restores some of the debuggability lost by going opaque.
- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN
locality.
Encoded as a 26-character Crockford Base32 string (ULID-compatible) for
compactness and case-insensitive matching.
### New root fields owned by meta-core
The metadata schema is open — any writer can put any key under
`file:<uuid>/`. This proposal does not change that. It adds three keys whose
sole writer is meta-core itself:
| Key | Type | Purpose |
|-----|------|---------|
| `cids` | Redis SET | All CIDs known for this file. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. |
| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
| `canonical_cid` | STRING | The CID this entry should be presented as externally (meta-share announce, UI display). Selected by the rank function (below). |
Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
`subtitle/<lang>`, …) are unaffected.
Redis SET is a departure from the all-strings convention in the current
storage doc, but `file:__index__` is already a SET — so the precedent exists
for using sets where they're the right structure.
### Canonical CID selection
`canonical_cid` is a label, not a key. Updating it costs one write — no
rename, no race, no broken external references.
Selection rule, highest-wins:
1. IPFS CID (any variant)
2. SHA-256 / SHA-3 family
3. BitTorrent infohash (v2 preferred over v1)
4. midhash256
Ranked by **utility for external retrieval**, not by cryptographic strength.
midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
externally meaningful for content addressing. btih ranks above midhash because
it enables swarm interop even if it's weaker as a digest.
Reconciler runs on the leader, recomputes `canonical_cid` whenever
`AddAlias` registers a new CID for an entry.
## Internal operations
The meta-core storage interface adds five operations. None are exposed
externally — they're internal to meta-core. External callers see only the
CID-based API.
```go
// Mint creates a new root with no aliases. Called when the watcher
// computes a midhash for a previously-unseen file. Returns the new UUID.
Mint(filePath string, size int64, mtime int64) (uuid string, err error)
// AddAlias registers a CID as resolving to this UUID. Updates both the
// cid:<cid> reverse index and the file:<uuid>/cids set atomically.
// Triggers canonical_cid reconciliation.
AddAlias(uuid string, cid string) error
// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
GetByCID(cid string) (uuid string, err error)
// MergeIfAliasExists is called when the watcher mints a fresh UUID for a
// file path but, on hashing, discovers the resulting CID already belongs
// to a different UUID. Merges the new (mostly-empty) UUID into the
// existing one, transferring filePath into the duplicates set, and
// returns the canonical UUID.
MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error)
// DeleteRoot removes the entire entry, including every cid:* alias and
// every file:<uuid>/* key. Used for file deletion events.
DeleteRoot(uuid string) error
```
### Lifecycle: file arrival
```
1. Watcher sees /watch/foo.mkv created.
2. Watcher computes midhash256 → "midhash256:abc...".
3. Watcher calls GetByCID("midhash256:abc...").
→ uuid_X (existing entry) or "" (new file).
4a. If existing (uuid_X):
- Add "/watch/foo.mkv" to file:<uuid_X>/duplicates.
- No new root created.
- meta-dup is notified via the existing keyspace event channel.
4b. If new:
- Mint("/watch/foo.mkv", size, mtime) → uuid_Y.
- AddAlias(uuid_Y, "midhash256:abc...").
- Plugins now process the file, referenced by midhash256:abc...
(UUID is never disclosed to them.)
5. Later, the full-hash plugin computes sha256.
- Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
- meta-core resolves midhash256:abc → uuid_Y via GetByCID.
- meta-core calls AddAlias(uuid_Y, "sha256:def...").
- Reconciler updates canonical_cid if sha256 outranks midhash256.
```
### Lifecycle: file deletion
```
1. Watcher sees /watch/foo.mkv deleted.
2. Resolve which entry owns this path:
a. If /watch/foo.mkv is in some entry's duplicates set:
- Remove from set. Done.
b. If /watch/foo.mkv is the filePath of some entry:
- If duplicates is non-empty: promote one duplicate to filePath.
- If duplicates is empty: DeleteRoot(uuid).
- Iterates the cids set, deletes each cid:<cid> reverse-index key.
- Deletes every file:<uuid>/* key.
- Removes uuid from file:__index__.
```
## Public API
```
GET /api/meta/<cid>
→ 200 { "filePath": "...", "canonical_cid": "...",
"cids": [...], "title": "...", "tmdb": {...}, ... }
→ 404 if no such CID
GET /api/file/<cid> (already exists — implementation now O(1))
→ streams file content
```
The UUID is **not** included in the response. Clients address everything by
CID; meta-core's choice of internal identifier remains a private
implementation detail and may change.
Auth: `/api/meta/<cid>` follows the same pattern as `/api/file/<cid>` and
bypasses the auth perimeter. It's the metadata twin of the content endpoint,
and meta-share calls it from federated search responses.
## Cross-service impact
This is a meta-core schema change. Other services interact only through CIDs,
so the impact is contained but real.
- **meta-sort**: writers must resolve plugin callbacks to a UUID before
writing. The existing `LeaderClient` gains a `getByCID` cache.
- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the
ID format changes.
- **meta-stremio**: queries by CID; needs no logic change beyond its storage
client.
- **meta-dup**: gains a cleaner data source — duplicates are now explicit in
the `duplicates` set, not implicit in colliding writes.
- **meta-share**: announces `canonical_cid` rather than midhash. Federated
search responses include all CIDs from the `cids` set so any peer querying
by any digest gets a hit.
- **Plugins**: no change. They speak CIDs only.
## Migration
One-time sweep, idempotent, runs on first boot after upgrade:
```
For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …):
1. Mint a fresh uuid_Y.
2. AddAlias(uuid_Y, hashID). The existing primary becomes an alias.
3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value).
4. RENAME file:<hashID>/<property> → file:<uuid_Y>/<property> for every
property (Lua script, one entry at a time; non-atomic across entries,
but each entry stays internally consistent).
5. Reconcile canonical_cid.
6. Remove hashID from file:__index__, add uuid_Y.
```
Crash-recoverable: re-running the sweep finds entries with both old and new
roots and resumes by detecting which step completed (presence of
`file:<uuid>/cids` indicates step 3 finished; absence of any
`file:<hashID>/*` keys indicates step 4 finished).
## Decisions recorded
| Question | Decision | Notes |
|---|---|---|
| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. |
| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. |
| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
| `cids` and `duplicates` field type | Redis SET | Departure from "all-strings" convention but consistent with the existing `file:__index__` precedent. |
| Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
## Open questions
- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints
a UUID at discovery (before hashing), then `AddAlias` runs when the midhash
arrives. Requires an explicit "reconcile if alias already exists" step
because two paths with the same content would each get their own UUID until
hashing reveals the collision. Worth doing eventually; not required for
this iteration.
- **meta-share announce strategy.** Announce only `canonical_cid`, or every
entry in `cids`? Maximum discoverability vs. DHT footprint. Decide as part
of the meta-share work, not here.
- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
content-streaming endpoint for external IPFS clients? Adjacent to this
design but not part of it.
- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`)
handle `file:<type>:<hash>/...` with a special split rule for the colon
inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies
— but any external consumer subscribed by hashID pattern needs updating.
Inventory those before flipping the migration.
## Non-goals
- **Not** a content-replication or IPFS-pinning system. Registering an IPFS
CID alias does not pin the file anywhere — it only states "if somebody
hashes this file with IPFS's chunker, this is the CID they'd get."
- **Not** a versioning scheme. Metadata is a snapshot of current state;
historical versions are out of scope.
- **Not** a strong-consistency system. Reverse-index writes happen under the
leader lock; reads from followers may briefly miss a freshly-added alias.
+57 -121
View File
@@ -1,140 +1,76 @@
# @metazla/meta-editor # @meta-core/editor
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface. React + Vite metadata browser/editor that ships inside the meta-core
container. Walks the UUID-rooted Redis keyspace, edits leaf values,
imports/exports snapshots, and consults the live schema indexer.
## Features ## What it talks to
- **Search**: Find files by title, filename, or hash ID The editor is a pure SPA — it has no server of its own. It calls the
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more meta-core HTTP API directly under three prefixes (see `src/api/`):
- **Batch Operations**: Update multiple files simultaneously
- **Real-time**: Direct integration with etcd via REST API
- **User-friendly**: Clean, intuitive interface with form validation
## Development - `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`,
`value`, `key/{key...}`). This is the UUID-exposing surface — keys look
like `file:<uuid>/<property>` and `cid:<algo>:<value>`.
- `/api/schema` — live per-field schema (type hints, value breakdowns),
produced by meta-core's schema indexer that consumes `meta:events`.
- `/api/snapshot/*` — snapshot export / import / wipe.
### Prerequisites In production the editor is served by meta-core's nginx (the Vite build
output is copied into the image at the same path) under
`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to
`/editor/` accordingly.
- Node.js 21.6.2+ ## Project layout
- pnpm 10.19.0+
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
### Running Locally
```bash
# From the root directory
pnpm run start:editor
# Or from the package directory
cd packages/meta-editor
pnpm run dev
```
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
### Building
```bash
# From the root directory
pnpm run build:editor
# Or from the package directory
cd packages/meta-editor
pnpm run build
```
The built files will be in `dist/` directory.
## Docker Deployment
In the Docker development environment, the editor is automatically available at:
``` ```
http://localhost/editor packages/meta-core/editor/
```
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
## API Integration
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
- `POST /api/metadata/search` - Search for files
- `GET /api/metadata/:hashId` - Get file metadata
- `PUT /api/metadata/:hashId` - Update complete metadata
- `PUT /api/metadata/:hashId/property` - Update specific property
- `POST /api/metadata/batch` - Batch update multiple files
See `src/api/metadataApi.ts` for the API client implementation.
## Usage
### Searching for Files
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
2. Enter your search query
3. Click "Search"
4. Results will appear in the left sidebar
### Editing Metadata
1. Select a file from the search results
2. Modify any fields in the editor form
3. Click "Save Changes" to persist updates
4. Use "Reset" to discard changes
### Supported Metadata Fields
**Basic Information:**
- Title, Original Title, Show Title
- Video Type (Movie, TV, Anime, Other)
- Year, Season, Episode
**Jellyfin Metadata:**
- Plot, IMDb ID, TMDb ID, AniDB ID
- Director, Studio, Rating, Runtime
- Genres, Premiered Date, Release Date
**File Information:**
- Filename, File Size, MIME Type
- Hash values (SHA-256, MD5, etc.) - read-only
## Project Structure
```
packages/meta-editor/
├── src/
│ ├── api/ # API client for metadata operations
│ ├── components/ # React components
│ │ ├── SearchBar.tsx # Search interface
│ │ ├── FileList.tsx # Search results list
│ │ └── MetadataEditor.tsx # Main editing form
│ ├── types/ # TypeScript type definitions
│ ├── utils/ # Utility functions
│ ├── App.tsx # Main application component
│ ├── main.tsx # Entry point
│ └── index.css # Global styles
├── index.html ├── index.html
├── vite.config.ts ├── vite.config.ts
├── tsconfig.json ├── tsconfig.json
└── package.json ├── package.json @meta-core/editor
└── src/
├── main.tsx entry
├── api/
│ ├── kvApi.ts /api/kv client
│ ├── schemaApi.ts /api/schema client + cache
│ └── snapshotApi.ts /api/snapshot client
├── kv/ KV browser + leaf/branch editors
├── components/ shared widgets (language combobox, snapshot panel, ...)
└── index.css
``` ```
## Technology Stack ## Local development
- **React 18** - UI framework Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`.
- **TypeScript 5.7** - Type safety
- **Vite 6** - Build tool and dev server
- **Native CSS** - Styling (no CSS framework)
## Contributing ```bash
cd packages/meta-core/editor
pnpm install
pnpm dev # vite dev server on http://localhost:5173
```
When adding new features: The Vite dev server proxies `/api/*` to `http://localhost:3000` (see
`vite.config.ts`). For local development against a running meta-core
container, change that target to wherever meta-core is reachable on your
host — e.g. `http://localhost:18083` for the dev compose's direct-backend
port.
1. Add type definitions to `src/types/index.ts` No root-level `start:editor` or `build:editor` scripts exist; run pnpm
2. Update API client in `src/api/metadataApi.ts` commands from this directory.
3. Create/update components as needed
4. Update this README with new features
## License ## Build
Part of the MetaMesh project. ```bash
pnpm build # tsc -b && vite build → dist/
```
The container build copies `dist/` into the nginx web root so the editor
is served at `/editor/`.
## Tech
- React 18, TypeScript 5
- Vite 5 (`base: '/editor/'`)
- Native CSS — no UI framework
- `iso-639-3` for language code rendering
+13 -2
View File
@@ -1,13 +1,14 @@
{ {
"name": "@meta-sort/editor", "name": "@meta-core/editor",
"version": "0.1.0", "version": "0.1.0",
"lockfileVersion": 3, "lockfileVersion": 3,
"requires": true, "requires": true,
"packages": { "packages": {
"": { "": {
"name": "@meta-sort/editor", "name": "@meta-core/editor",
"version": "0.1.0", "version": "0.1.0",
"dependencies": { "dependencies": {
"iso-639-3": "^3.0.1",
"react": "^18.2.0", "react": "^18.2.0",
"react-dom": "^18.2.0" "react-dom": "^18.2.0"
}, },
@@ -1246,6 +1247,16 @@
"node": ">=6.9.0" "node": ">=6.9.0"
} }
}, },
"node_modules/iso-639-3": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/iso-639-3/-/iso-639-3-3.0.1.tgz",
"integrity": "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==",
"license": "MIT",
"funding": {
"type": "github",
"url": "https://github.com/sponsors/wooorm"
}
},
"node_modules/js-tokens": { "node_modules/js-tokens": {
"version": "4.0.0", "version": "4.0.0",
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz", "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
-185
View File
@@ -1,185 +0,0 @@
.app {
min-height: 100vh;
display: flex;
flex-direction: column;
}
.app-header {
padding: 2rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.app-header h1 {
margin: 0 0 0.5rem 0;
color: #646cff;
}
.app-description {
margin: 0;
color: rgba(255, 255, 255, 0.6);
}
.app-main {
flex: 1;
padding: 2rem;
max-width: 1400px;
width: 100%;
margin: 0 auto;
}
.app-error {
padding: 1rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
border-radius: 8px;
color: #ff6b6b;
margin-bottom: 1rem;
}
.app-loading {
text-align: center;
padding: 3rem 2rem;
color: rgba(255, 255, 255, 0.7);
font-size: 1.1rem;
}
.app-welcome {
text-align: center;
padding: 4rem 2rem;
color: rgba(255, 255, 255, 0.6);
}
.app-welcome h2 {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.8);
}
.app-welcome-features {
margin-top: 1rem;
margin-bottom: 1.5rem;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
}
.app-kv-browser {
margin-top: 1.5rem;
text-align: left;
max-width: 900px;
margin-left: auto;
margin-right: auto;
}
.app-content {
display: grid;
grid-template-columns: 350px 1fr;
gap: 1.5rem;
margin-top: 1rem;
}
.app-sidebar {
min-height: 400px;
}
.app-editor {
min-height: 400px;
}
.app-no-selection {
display: flex;
align-items: center;
justify-content: center;
height: 100%;
min-height: 400px;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
color: rgba(255, 255, 255, 0.5);
text-align: center;
padding: 2rem;
}
.app-footer {
padding: 1.5rem 2rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-size: 0.9rem;
}
.app-footer a {
color: #646cff;
text-decoration: none;
}
.app-footer a:hover {
text-decoration: underline;
}
@media (max-width: 1024px) {
.app-content {
grid-template-columns: 1fr;
}
.app-sidebar {
order: 2;
}
.app-editor {
order: 1;
}
}
@media (max-width: 768px) {
.app-header,
.app-main,
.app-footer {
padding: 1rem;
}
.app-content {
gap: 1rem;
}
}
@media (prefers-color-scheme: light) {
.app-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.app-description {
color: rgba(0, 0, 0, 0.6);
}
.app-error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.app-loading {
color: rgba(0, 0, 0, 0.7);
}
.app-welcome {
color: rgba(0, 0, 0, 0.6);
}
.app-welcome h2 {
color: rgba(0, 0, 0, 0.8);
}
.app-welcome-features {
color: rgba(0, 0, 0, 0.5);
}
.app-no-selection {
background-color: rgba(0, 0, 0, 0.02);
color: rgba(0, 0, 0, 0.5);
}
.app-footer {
border-top-color: rgba(0, 0, 0, 0.1);
color: rgba(0, 0, 0, 0.5);
}
}
-246
View File
@@ -1,246 +0,0 @@
import { useState, useEffect } from 'react';
import { SearchBar } from './components/SearchBar';
import { FileList } from './components/FileList';
import { MetadataEditor } from './components/MetadataEditor';
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
import { KVTreeView } from './components/KVTreeView';
import { MetadataAPI } from './api/metadataApi';
import { SearchResult } from './types';
import './App.css';
function App() {
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
const [bulkEditMode, setBulkEditMode] = useState(false);
const [isSearching, setIsSearching] = useState(false);
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
const [error, setError] = useState<string | null>(null);
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
const getHashFromUrl = (): string | null => {
const path = window.location.pathname;
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
if (fileMatch) {
return fileMatch[1];
}
// Fallback to query param for backwards compatibility
const params = new URLSearchParams(window.location.search);
return params.get('hash');
};
// Update URL when file is selected
const updateUrlForFile = (hashId: string | undefined) => {
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
if (hashId) {
const newPath = `${basePath}/file/${hashId}`;
window.history.pushState({ hashId }, '', newPath);
} else {
window.history.pushState({}, '', basePath || '/editor/');
}
};
// Check for hash parameter in URL on component mount
useEffect(() => {
const loadHashFromUrl = async () => {
const hashParam = getHashFromUrl();
if (hashParam) {
try {
setIsLoadingDirectHash(true);
setError(null);
// Try to get metadata for this hash
const metadata = await MetadataAPI.getMetadata(hashParam);
// Create a search result from the metadata
const searchResult: SearchResult = {
hashId: hashParam,
metadata: metadata,
score: 1.0,
};
// Set the search results and select this file
setSearchResults([searchResult]);
setSelectedHashId(hashParam);
} catch (err: any) {
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsLoadingDirectHash(false);
}
}
};
loadHashFromUrl();
// Handle browser back/forward
const handlePopState = (event: PopStateEvent) => {
const hashId = event.state?.hashId || getHashFromUrl();
if (hashId) {
setSelectedHashId(hashId);
} else {
setSelectedHashId(undefined);
}
};
window.addEventListener('popstate', handlePopState);
return () => window.removeEventListener('popstate', handlePopState);
}, []); // Empty dependency array means this runs once on mount
const handleSearch = async (query: string) => {
try {
setIsSearching(true);
setError(null);
// Unified search - searches across all fields
const response = await MetadataAPI.search({ query });
setSearchResults(response.results);
// Auto-select first result if available
if (response.results.length > 0) {
setSelectedHashId(response.results[0].hashId);
} else {
setSelectedHashId(undefined);
}
} catch (err: any) {
setError(err.message);
setSearchResults([]);
setSelectedHashId(undefined);
} finally {
setIsSearching(false);
}
};
const handleSelectFile = (hashId: string) => {
if (!bulkEditMode) {
setSelectedHashId(hashId);
setSelectedHashIds(new Set());
updateUrlForFile(hashId);
}
};
const handleToggleSelection = (hashId: string) => {
const newSelection = new Set(selectedHashIds);
if (newSelection.has(hashId)) {
newSelection.delete(hashId);
} else {
newSelection.add(hashId);
}
setSelectedHashIds(newSelection);
};
const handleSelectAll = () => {
const allHashIds = new Set(searchResults.map(r => r.hashId));
setSelectedHashIds(allHashIds);
};
const handleClearSelection = () => {
setSelectedHashIds(new Set());
setBulkEditMode(false);
};
const handleBulkEditMode = (enabled: boolean) => {
setBulkEditMode(enabled);
if (enabled) {
setSelectedHashId(undefined);
} else {
setSelectedHashIds(new Set());
}
};
const handleMetadataSaved = () => {
// Optionally refresh the search results to reflect changes
console.log('Metadata saved successfully');
};
const handleBulkMetadataSaved = () => {
console.log('Bulk metadata saved successfully');
handleClearSelection();
};
return (
<div className="app">
<header className="app-header">
<h1>MetaMesh Metadata Editor</h1>
<p className="app-description">
Search and edit metadata for your media files
</p>
</header>
<main className="app-main">
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
{error && <div className="app-error">{error}</div>}
{isLoadingDirectHash && (
<div className="app-loading">Loading file metadata...</div>
)}
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
<div className="app-welcome">
<h2>Welcome to MetaMesh Metadata Editor</h2>
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
<p className="app-welcome-features">
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
</p>
<div className="app-kv-browser">
<KVTreeView />
</div>
</div>
)}
{searchResults.length > 0 && (
<div className="app-content">
<div className="app-sidebar">
<FileList
results={searchResults}
selectedHashId={selectedHashId}
selectedHashIds={selectedHashIds}
bulkEditMode={bulkEditMode}
onSelectFile={handleSelectFile}
onToggleSelection={handleToggleSelection}
onSelectAll={handleSelectAll}
onClearSelection={handleClearSelection}
onBulkEditMode={handleBulkEditMode}
/>
</div>
<div className="app-editor">
{bulkEditMode && selectedHashIds.size > 0 ? (
<BulkMetadataEditor
hashIds={Array.from(selectedHashIds)}
onSaved={handleBulkMetadataSaved}
onCancel={handleClearSelection}
/>
) : selectedHashId ? (
<MetadataEditor
key={selectedHashId}
hashId={selectedHashId}
onSaved={handleMetadataSaved}
/>
) : (
<div className="app-no-selection">
{bulkEditMode
? 'Select files from the list to bulk edit metadata'
: 'Select a file from the list to edit its metadata'}
</div>
)}
</div>
</div>
)}
</main>
<footer className="app-footer">
<p>
MetaMesh Metadata Editor | Part of{' '}
<a href="/" target="_blank">
MetaMesh
</a>
</p>
</footer>
</div>
);
}
export default App;
+136
View File
@@ -0,0 +1,136 @@
// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of
// string-valued keys; this module wraps the S3-style listing, search, and
// per-key get/put/delete operations.
const API_BASE = '/api/kv';
export interface KVTreeResponse {
prefix: string;
delimiter: string;
branches: string[];
leaves: string[];
cursor: string;
hasMore: boolean;
}
export interface KVSearchResponse {
contains: string;
keys: string[];
truncated: boolean;
}
export interface KVFindMatch {
key: string;
value: string;
entryPath: string;
field: string;
}
export interface KVFindResponse {
contains: string;
fields: string[];
matches: KVFindMatch[];
truncated: boolean;
}
export interface KVValueResponse {
key: string;
type: string;
value: string;
exists: boolean;
}
export interface KVInfoResponse {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage: string;
}
async function jsonOrThrow<T>(res: Response): Promise<T> {
if (!res.ok) {
let msg = `HTTP ${res.status}`;
try {
const body = await res.json();
if (body?.message) msg = body.message;
else if (body?.error) msg = body.error;
} catch {}
throw new Error(msg);
}
return res.json();
}
export const KVAPI = {
async info(): Promise<KVInfoResponse> {
return jsonOrThrow(await fetch(`${API_BASE}/info`));
},
async tree(params: {
prefix?: string;
delimiter?: string;
cursor?: string;
max?: number;
}): Promise<KVTreeResponse> {
const q = new URLSearchParams();
if (params.prefix !== undefined) q.set('prefix', params.prefix);
if (params.delimiter !== undefined) q.set('delimiter', params.delimiter);
if (params.cursor) q.set('cursor', params.cursor);
if (params.max) q.set('max', String(params.max));
return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`));
},
// Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap
// is reached. Useful for branches with up to a few thousand children where
// pagination UX would just be a worse spinner.
async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
const seenLeaves = new Set<string>();
const seenBranches = new Set<string>();
let cursor = '';
while (true) {
const page = await this.tree({ prefix, delimiter, cursor, max: 1000 });
for (const k of page.leaves) seenLeaves.add(k);
for (const k of page.branches) seenBranches.add(k);
cursor = page.cursor;
if (!page.hasMore) break;
if (seenLeaves.size + seenBranches.size >= softCap) break;
}
return {
prefix,
delimiter,
branches: [...seenBranches].sort(),
leaves: [...seenLeaves].sort(),
cursor,
hasMore: cursor !== '0' && cursor !== '',
};
},
async search(contains: string, limit = 500): Promise<KVSearchResponse> {
const q = new URLSearchParams({ contains, limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`));
},
async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) });
return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`));
},
async get(key: string): Promise<KVValueResponse> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
},
async put(key: string, value: string): Promise<KVValueResponse> {
const res = await fetch(`${API_BASE}/value`, {
method: 'PUT',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ key, value }),
});
return jsonOrThrow(res);
},
async del(key: string): Promise<{ key: string; deleted: boolean }> {
const q = new URLSearchParams({ key });
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
},
};
-151
View File
@@ -1,151 +0,0 @@
import {
FileMetadata,
SearchResponse,
UpdateResult,
BatchUpdateResponse,
} from '../types';
const API_BASE = '/api/metadata';
export class MetadataAPI {
/**
* Search for files by query, hashId, or property
*/
static async search(params: {
query?: string;
hashId?: string;
property?: string;
propertyValue?: string;
limit?: number;
}): Promise<SearchResponse> {
const response = await fetch(`${API_BASE}/search`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(params),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Search failed');
}
return response.json();
}
/**
* Get metadata for a specific file by hash ID
*/
static async getMetadata(hashId: string): Promise<FileMetadata> {
const response = await fetch(`${API_BASE}/${hashId}`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get metadata');
}
return response.json();
}
/**
* Update complete metadata for a file
*/
static async updateMetadata(
hashId: string,
metadata: FileMetadata
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify(metadata),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update metadata');
}
return response.json();
}
/**
* Update a specific property for a file
*/
static async updateProperty(
hashId: string,
property: string,
value: string
): Promise<UpdateResult> {
const response = await fetch(`${API_BASE}/${hashId}/property`, {
method: 'PUT',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ property, value }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to update property');
}
return response.json();
}
/**
* Batch update multiple files
*/
static async batchUpdate(
updates: Array<{
hashId: string;
metadata?: FileMetadata;
properties?: Record<string, string>;
}>
): Promise<BatchUpdateResponse> {
const response = await fetch(`${API_BASE}/batch`, {
method: 'POST',
headers: {
'Content-Type': 'application/json',
},
body: JSON.stringify({ updates }),
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Batch update failed');
}
return response.json();
}
/**
* Delete metadata for a file
*/
static async deleteMetadata(hashId: string): Promise<void> {
const response = await fetch(`${API_BASE}/${hashId}`, {
method: 'DELETE',
});
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to delete metadata');
}
}
/**
* Get all hash IDs
*/
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
const response = await fetch(`${API_BASE}/hash-ids`);
if (!response.ok) {
const error = await response.json();
throw new Error(error.error || 'Failed to get hash IDs');
}
return response.json();
}
}
+74
View File
@@ -0,0 +1,74 @@
import { useEffect, useState } from 'react';
export type SchemaPrimitive =
| 'string'
| 'int'
| 'float'
| 'bool'
| 'json-object'
| 'json-array'
| 'undefined'
| 'mixed';
export type SchemaHint = 'cid' | 'timestamp';
export type SchemaKeyHint = 'language-code';
export interface SchemaFieldInfo {
type: SchemaPrimitive;
hint?: SchemaHint;
key_hint?: SchemaKeyHint;
breakdown: Record<string, number>;
}
export interface SchemaResponse {
fields: Record<string, SchemaFieldInfo>;
generated_at: string;
source: 'live' | 'rescan';
}
let cached: Promise<SchemaResponse> | null = null;
export async function fetchSchema(): Promise<SchemaResponse> {
if (!cached) {
cached = fetch('/api/schema').then((r) => {
if (!r.ok) {
cached = null;
throw new Error(`schema fetch failed: ${r.status}`);
}
return r.json();
});
}
return cached;
}
/**
* Returns the live schema entry for a field, or undefined if not yet observed.
* Loads /api/schema lazily on first mount and caches the result process-wide.
*/
export function useSchema(): {
schema: SchemaResponse | null;
getFieldHint: (key: string) => SchemaFieldInfo | undefined;
} {
const [schema, setSchema] = useState<SchemaResponse | null>(null);
useEffect(() => {
let alive = true;
fetchSchema()
.then((s) => {
if (alive) setSchema(s);
})
.catch(() => {
// Schema endpoint missing or errored — editor falls back to plugin schemas.
});
return () => {
alive = false;
};
}, []);
const getFieldHint = (key: string): SchemaFieldInfo | undefined => {
if (!schema) return undefined;
return schema.fields[key];
};
return { schema, getFieldHint };
}
+96
View File
@@ -0,0 +1,96 @@
const API_BASE = '/api/snapshot';
export type ImportMode = 'replace' | 'merge';
export type ImportConflict = 'mine' | 'source';
export interface FileImportResult {
hashId: string;
added: number;
updated: number;
unchanged: number;
kept: number;
deleted: number;
error?: string;
}
export interface PluginOutputResult {
written: number;
skipped: number;
}
export interface ImportResult {
mode: ImportMode;
conflict?: ImportConflict;
dryRun: boolean;
totalFiles: number;
filesOk: number;
filesFailed: number;
files?: FileImportResult[];
pluginOutput?: PluginOutputResult;
}
export interface WipeResult {
metadataDeleted: number;
}
export class SnapshotAPI {
/**
* Trigger a download of a metadata snapshot ZIP.
* Browser handles the download via a synthetic <a> click.
*/
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
const parts: string[] = [];
if (includes.pluginOutput) parts.push('plugin-output');
const qs = parts.length ? `?include=${parts.join(',')}` : '';
const a = document.createElement('a');
a.href = `${API_BASE}/export${qs}`;
a.rel = 'noopener';
document.body.appendChild(a);
a.click();
a.remove();
}
/**
* Upload a snapshot ZIP and apply it.
* Mode replace deletes and rewrites each CID's properties.
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
*/
static async runImport(
file: File,
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
): Promise<ImportResult> {
const params = new URLSearchParams();
params.set('mode', opts.mode);
if (opts.mode === 'merge') {
params.set('conflict', opts.conflict ?? 'source');
}
if (opts.dryRun) {
params.set('dry_run', 'true');
}
const fd = new FormData();
fd.append('snapshot', file);
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
method: 'POST',
body: fd,
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Import failed');
}
return response.json();
}
/**
* Wipe Redis metadata. Pass scope=metadata.
*/
static async wipe(scope: 'metadata'): Promise<WipeResult> {
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
method: 'POST',
});
if (!response.ok) {
const err = await response.json().catch(() => ({}));
throw new Error(err.message || err.error || 'Wipe failed');
}
return response.json();
}
}
@@ -1,382 +0,0 @@
.bulk-metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.bulk-metadata-editor-header {
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-editor-header h2 {
margin: 0 0 0.5rem 0;
font-size: 1.5rem;
}
.bulk-info {
margin: 0 0 1rem 0;
color: rgba(255, 255, 255, 0.6);
font-size: 0.95rem;
}
.bulk-metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.bulk-metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.bulk-metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.bulk-metadata-editor-form {
display: flex;
flex-direction: column;
gap: 1.5rem;
}
.bulk-metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.bulk-metadata-section-header {
margin-bottom: 1.5rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.bulk-metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.section-hint {
margin: 0.25rem 0 0 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.section-hint-bulk {
margin: 0.5rem 0 0 0;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
font-weight: 500;
}
/* Plugin-based bulk fields */
.bulk-fields-container {
display: flex;
flex-direction: column;
gap: 1rem;
}
.bulk-field-wrapper {
border: 1px solid rgba(255, 255, 255, 0.05);
border-radius: 4px;
padding: 0.75rem;
background-color: rgba(0, 0, 0, 0.1);
transition: all 0.2s ease;
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.bulk-field-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 0.5rem;
}
.field-key {
font-size: 0.85rem;
font-weight: 600;
color: rgba(255, 255, 255, 0.6);
text-transform: lowercase;
font-family: 'Courier New', monospace;
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.bulk-field-input {
/* Plugin component will render inside this */
}
/* Legacy bulk fields (for compatibility) */
.bulk-metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.bulk-metadata-field:last-child {
margin-bottom: 0;
}
.field-header {
display: flex;
justify-content: space-between;
align-items: center;
}
.field-header label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.clear-field-btn {
width: 20px;
height: 20px;
padding: 0;
border: 1px solid rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
border-radius: 50%;
cursor: pointer;
font-size: 16px;
line-height: 1;
transition: all 0.2s;
display: flex;
align-items: center;
justify-content: center;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
.bulk-metadata-field input,
.bulk-metadata-field textarea,
.bulk-metadata-field select {
width: 100%;
}
.bulk-metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.bulk-metadata-field-row:last-child {
margin-bottom: 0;
}
/* Modified fields summary */
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-fields h3 {
color: #646cff;
}
.modified-fields-list {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.modified-field-item {
display: flex;
gap: 0.75rem;
align-items: center;
padding: 0.75rem;
background-color: rgba(100, 108, 255, 0.05);
border: 1px solid rgba(100, 108, 255, 0.2);
border-radius: 4px;
}
.field-name {
font-weight: 600;
color: #646cff;
min-width: 150px;
font-family: 'Courier New', monospace;
font-size: 0.9rem;
}
.field-value {
flex: 1;
word-break: break-word;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
}
.remove-field-btn {
padding: 0.25rem 0.75rem;
font-size: 0.85rem;
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s ease;
white-space: nowrap;
}
.remove-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
@media (max-width: 768px) {
.bulk-metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.bulk-metadata-editor-actions {
width: 100%;
flex-direction: column;
}
.bulk-metadata-editor-actions button {
width: 100%;
}
.bulk-metadata-field-row {
grid-template-columns: 1fr;
}
.modified-field-item {
flex-direction: column;
align-items: flex-start;
}
.field-name {
min-width: auto;
}
.remove-field-btn {
align-self: flex-end;
}
}
@media (prefers-color-scheme: light) {
.bulk-metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.bulk-info {
color: rgba(0, 0, 0, 0.6);
}
.bulk-metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.section-hint {
color: rgba(0, 0, 0, 0.5);
}
.section-hint-bulk {
color: rgba(0, 0, 0, 0.4);
}
.bulk-field-wrapper {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.bulk-field-wrapper.has-value {
border-color: rgba(100, 108, 255, 0.3);
background-color: rgba(100, 108, 255, 0.05);
}
.field-key {
color: rgba(0, 0, 0, 0.6);
}
.bulk-field-wrapper.has-value .field-key {
color: #646cff;
}
.field-header label {
color: rgba(0, 0, 0, 0.7);
}
.field-value {
color: rgba(0, 0, 0, 0.8);
}
.bulk-metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.bulk-metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.modified-fields {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.3);
}
.modified-field-item {
background-color: rgba(100, 108, 255, 0.05);
border-color: rgba(100, 108, 255, 0.2);
}
.clear-field-btn {
border-color: rgba(255, 0, 0, 0.3);
background-color: rgba(255, 0, 0, 0.1);
color: #cc0000;
}
.clear-field-btn:hover {
background-color: rgba(255, 0, 0, 0.2);
border-color: rgba(255, 0, 0, 0.5);
}
}
@@ -1,280 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './BulkMetadataEditor.css';
interface PluginsResponse {
plugins: PluginSchema[];
}
interface BulkMetadataEditorProps {
hashIds: string[];
onSaved?: () => void;
onCancel?: () => void;
}
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
// Load plugins and metadata on mount
useEffect(() => {
const loadData = async () => {
try {
setIsLoading(true);
setError(null);
// Fetch plugins
const pluginsResponse = await fetch('/api/plugins');
if (pluginsResponse.ok) {
const data: PluginsResponse = await pluginsResponse.json();
// Sort by priority
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
// Load metadata for all selected files
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
const metadata = await Promise.all(metadataPromises);
// Calculate common values
const commons: Record<string, any> = {};
// Get all editable fields from all plugins
const allFields: string[] = [];
plugins.forEach((plugin) => {
if (plugin.metadataSchema) {
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
if (!field.readonly) {
allFields.push(key);
}
});
}
});
allFields.forEach((field) => {
const values = metadata.map((m) => m[field as keyof FileMetadata]);
// Filter out undefined and null values
const definedValues = values.filter((v) => v !== undefined && v !== null);
if (definedValues.length === 0) {
return;
}
// Safely stringify values
const uniqueValues = Array.from(
new Set(
definedValues.map((v) => {
try {
return JSON.stringify(v);
} catch {
return String(v);
}
})
)
);
if (uniqueValues.length === 1) {
try {
commons[field] = JSON.parse(uniqueValues[0]);
} catch {
commons[field] = uniqueValues[0];
}
} else {
commons[field] = '*';
}
});
setCommonValues(commons);
} catch (err: any) {
setError(`Failed to load data: ${err.message}`);
} finally {
setIsLoading(false);
}
};
loadData();
}, [hashIds]);
const handleFieldChange = (field: string, value: any) => {
setEditedFields({
...editedFields,
[field]: value,
});
};
const handleClearField = (field: string) => {
const newFields = { ...editedFields };
delete newFields[field];
setEditedFields(newFields);
};
const handleSave = async () => {
if (Object.keys(editedFields).length === 0) {
setError('No fields have been edited');
return;
}
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
// Build batch update payload
const updates = hashIds.map((hashId) => ({
hashId,
metadata: editedFields,
}));
const result = await MetadataAPI.batchUpdate(updates);
if (result.errors > 0) {
setError(
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
);
console.error('Batch update errors:', result.results.filter((r) => r.error));
} else {
setSuccessMessage(`Successfully updated ${result.success} files!`);
}
if (onSaved) {
onSaved();
}
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const hasChanges = Object.keys(editedFields).length > 0;
// Filter to only editable plugins (those with non-readonly fields)
const editablePlugins = plugins.filter((plugin) => {
if (!plugin.metadataSchema) return false;
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
});
if (isLoading) {
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-loading">
Loading metadata for {hashIds.length} files...
</div>
</div>
);
}
// Build display metadata (edited values or common values)
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
return (
<div className="bulk-metadata-editor">
<div className="bulk-metadata-editor-header">
<h2>Bulk Edit Metadata</h2>
<p className="bulk-info">
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
be updated. Fields showing "*" have different values across files.
</p>
<div className="bulk-metadata-editor-actions">
{onCancel && (
<button onClick={onCancel} disabled={isSaving}>
Cancel
</button>
)}
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
</button>
</div>
</div>
{successMessage && (
<div className="bulk-metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
<div className="bulk-metadata-editor-form">
{editablePlugins.map((plugin) => {
if (!plugin.metadataSchema) return null;
// Get editable fields for this plugin
const editableFields = Object.entries(plugin.metadataSchema).filter(
([, field]) => !field.readonly
);
if (editableFields.length === 0) return null;
return (
<section
key={plugin.id}
className="bulk-metadata-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="bulk-metadata-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="section-hint">{plugin.description}</p>
)}
<p className="section-hint-bulk">
Common values shown. "*" indicates different values. Edit to update all files.
</p>
</div>
<div className="plugin-fields">
{editableFields.map(([key, field]) => (
<DynamicFieldRenderer
key={key}
fieldKey={key}
schema={field}
metadata={displayMetadata as FileMetadata}
onChange={handleFieldChange}
readonly={false}
/>
))}
</div>
</section>
);
})}
{/* Modified Fields Summary */}
{hasChanges && (
<section className="bulk-metadata-section modified-fields">
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
<div className="modified-fields-list">
{Object.entries(editedFields).map(([key, value]) => (
<div key={key} className="modified-field-item">
<span className="field-name">{key}:</span>
<span className="field-value">
{Array.isArray(value)
? value.join(', ')
: typeof value === 'object'
? JSON.stringify(value)
: String(value || '(empty)')}
</span>
<button
className="remove-field-btn"
onClick={() => handleClearField(key)}
title="Remove this field"
>
Remove
</button>
</div>
))}
</div>
</section>
)}
</div>
</div>
);
}
@@ -1,297 +0,0 @@
.dynamic-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 0.5rem;
}
.dynamic-field:last-child {
margin-bottom: 0;
}
.dynamic-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
width: 100%;
padding: 0.5rem 0.75rem;
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-size: 0.9rem;
background: rgba(0, 0, 0, 0.3);
color: #fff;
transition: border-color 0.2s;
}
.dynamic-field input:focus,
.dynamic-field select:focus,
.dynamic-field textarea:focus {
outline: none;
border-color: #646cff;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: rgba(0, 0, 0, 0.2);
color: rgba(255, 255, 255, 0.5);
cursor: not-allowed;
}
.dynamic-field textarea {
resize: vertical;
min-height: 80px;
font-family: inherit;
}
.field-hint {
display: block;
font-size: 0.8rem;
color: rgba(255, 255, 255, 0.4);
}
.checkbox-label {
display: flex;
align-items: center;
gap: 0.5rem;
cursor: pointer;
font-weight: normal;
color: rgba(255, 255, 255, 0.7);
}
.checkbox-label input[type="checkbox"] {
width: 1.1rem;
height: 1.1rem;
cursor: pointer;
accent-color: #646cff;
}
.checkbox-label input:disabled {
cursor: not-allowed;
opacity: 0.5;
}
.readonly-value {
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.7);
font-family: 'Courier New', monospace;
font-size: 0.9rem;
word-break: break-all;
}
.field-link {
color: #646cff;
text-decoration: none;
}
.field-link:hover {
text-decoration: underline;
}
/* CID field styling */
.cid-field .cid-value-container {
display: flex;
align-items: center;
gap: 0.5rem;
}
.cid-field .cid-value {
flex: 1;
padding: 0.5rem 0.75rem;
background: rgba(0, 0, 0, 0.3);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: #a0d8ef;
cursor: pointer;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.cid-field .cid-value:hover {
background: rgba(0, 0, 0, 0.4);
border-color: #646cff;
}
.cid-field .cid-open-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 0.75rem;
background: #646cff;
color: #fff;
border-radius: 4px;
text-decoration: none;
font-size: 0.85rem;
font-weight: 500;
white-space: nowrap;
transition: background 0.2s;
}
.cid-field .cid-open-button:hover {
background: #535bf2;
}
/* JSON field styling */
.json-field .json-viewer {
background: rgba(0, 0, 0, 0.4);
color: #d4d4d4;
padding: 1rem;
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
overflow-x: auto;
max-height: 300px;
overflow-y: auto;
margin: 0;
white-space: pre-wrap;
word-break: break-all;
border: 1px solid rgba(255, 255, 255, 0.1);
}
/* Language string field styling */
.language-string-field .language-entries {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.language-string-field .language-entry {
display: flex;
align-items: center;
gap: 0.5rem;
}
.language-string-field .lang-input {
flex: 1;
min-width: 0;
}
.language-string-field .lang-code {
display: inline-block;
width: 50px;
padding: 0.4rem 0.5rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.8rem;
color: #a0d8ef;
text-align: center;
flex-shrink: 0;
}
.language-string-field .lang-text {
flex: 1;
padding: 0.4rem 0.75rem;
background: rgba(0, 0, 0, 0.2);
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 4px;
color: rgba(255, 255, 255, 0.8);
font-size: 0.9rem;
min-width: 0;
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
}
.language-string-field .lang-remove-btn {
width: 32px;
height: 32px;
flex-shrink: 0;
display: flex;
align-items: center;
justify-content: center;
background: rgba(255, 100, 100, 0.2);
border: 1px solid rgba(255, 100, 100, 0.3);
border-radius: 4px;
color: #ff6b6b;
font-size: 1.2rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-remove-btn:hover {
background: rgba(255, 100, 100, 0.3);
border-color: rgba(255, 100, 100, 0.5);
}
.language-string-field .lang-add-btn {
align-self: flex-start;
padding: 0.4rem 0.75rem;
background: rgba(100, 108, 255, 0.2);
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
color: #646cff;
font-size: 0.85rem;
cursor: pointer;
transition: background 0.2s, border-color 0.2s;
}
.language-string-field .lang-add-btn:hover {
background: rgba(100, 108, 255, 0.3);
border-color: rgba(100, 108, 255, 0.5);
}
/* Light theme */
@media (prefers-color-scheme: light) {
.dynamic-field label {
color: rgba(0, 0, 0, 0.7);
}
.dynamic-field input[type="text"],
.dynamic-field input[type="number"],
.dynamic-field select,
.dynamic-field textarea {
border-color: rgba(0, 0, 0, 0.1);
background: #fff;
color: #333;
}
.dynamic-field input:disabled,
.dynamic-field select:disabled,
.dynamic-field textarea:disabled {
background: #f5f5f5;
color: rgba(0, 0, 0, 0.5);
}
.field-hint {
color: rgba(0, 0, 0, 0.5);
}
.checkbox-label {
color: rgba(0, 0, 0, 0.7);
}
.readonly-value {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
.json-field .json-viewer {
background: #1e1e1e;
border-color: rgba(0, 0, 0, 0.2);
}
.language-string-field .lang-code {
background: rgba(100, 108, 255, 0.1);
color: #4a50c8;
}
.language-string-field .lang-text {
background: #f8f9fa;
border-color: #e9ecef;
color: #495057;
}
}
@@ -1,378 +0,0 @@
import { SchemaField, FileMetadata } from '../types';
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
import './DynamicFieldRenderer.css';
interface DynamicFieldRendererProps {
fieldKey: string;
schema: SchemaField;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
readonly?: boolean;
}
/**
* Renders a single metadata field based on its schema definition
*/
export function DynamicFieldRenderer({
fieldKey,
schema,
metadata,
onChange,
readonly = false,
}: DynamicFieldRendererProps) {
const isReadonly = readonly || schema.readonly;
const fieldType = schema.type || 'string';
// Get value from metadata using path-style key
const getValue = (): any => {
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = fieldKey.split('/');
let value: any = metadata;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
};
// Set value in metadata using path-style key
const setValue = (newValue: any) => {
onChange(fieldKey, newValue);
};
const value = getValue();
// Render link if configured
const renderValueWithLink = (displayValue: string) => {
if (schema.link && displayValue) {
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
return (
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
{displayValue}
</a>
);
}
return displayValue;
};
// Render based on field type
switch (fieldType) {
case 'boolean':
return (
<div className="dynamic-field">
<label className="checkbox-label">
<input
type="checkbox"
checked={Boolean(value)}
onChange={(e) => setValue(e.target.checked)}
disabled={isReadonly}
/>
<span>{schema.label}</span>
</label>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'number':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="number"
value={value ?? ''}
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
disabled={isReadonly}
min={schema.min}
max={schema.max}
placeholder={schema.placeholder}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'select':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<select
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
>
<option value="">Select...</option>
{schema.options?.map((opt) => (
<option key={opt.value} value={opt.value}>
{opt.label}
</option>
))}
</select>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'text':
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<textarea
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
disabled={isReadonly}
placeholder={schema.placeholder}
rows={4}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'array':
// Convert RecordSet to comma-separated string for display
const arrayValue = (() => {
if (!value) return '';
if (typeof value === 'string') return value;
if (typeof value === 'object') {
return Object.values(value).join(', ');
}
return '';
})();
// Convert comma-separated string back to RecordSet
const handleArrayChange = (str: string) => {
const items = str.split(',').map(s => s.trim()).filter(Boolean);
if (items.length === 0) {
setValue(undefined);
return;
}
const recordSet: Record<string, string> = {};
items.forEach((item, idx) => {
recordSet[idx.toString()] = item;
});
setValue(recordSet);
};
return (
<div className="dynamic-field">
<label>{schema.label}</label>
<input
type="text"
value={arrayValue}
onChange={(e) => handleArrayChange(e.target.value)}
disabled={isReadonly}
placeholder={schema.placeholder || 'Comma-separated values'}
/>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'json':
// Aggregate matching keys based on pattern
const jsonData = (() => {
if (!schema.pattern) {
return value;
}
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
const result: Record<string, any> = {};
// Recursively find all matching keys
const findMatching = (obj: any, currentPath: string = '') => {
if (!obj || typeof obj !== 'object') return;
for (const [key, val] of Object.entries(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
// Store in result with relative path
const relativePath = newPath.substring(patternPrefix.length - 1);
if (relativePath) {
setNestedValue(result, relativePath, val);
}
}
if (typeof val === 'object' && val !== null) {
findMatching(val, newPath);
}
}
};
findMatching(metadata);
return Object.keys(result).length > 0 ? result : undefined;
})();
if (!jsonData) {
return null;
}
return (
<div className="dynamic-field json-field">
<label>{schema.label}</label>
<pre className="json-viewer">
{JSON.stringify(jsonData, null, 2)}
</pre>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'cid':
// CID field type - displays CID with "Open in Editor" button
if (!value) {
return null;
}
const cidValue = String(value);
const editorUrl = `/editor/file/${cidValue}`;
return (
<div className="dynamic-field cid-field">
<label>{schema.label}</label>
<div className="cid-value-container">
<code
className="cid-value"
onClick={() => navigator.clipboard.writeText(cidValue)}
title="Click to copy"
>
{cidValue}
</code>
<a
href={editorUrl}
className="cid-open-button"
title="Open in Editor"
target="_blank"
rel="noopener noreferrer"
>
↗ Open
</a>
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'languageString':
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
const langEntries: Array<[string, string]> = value && typeof value === 'object'
? Object.entries(value).map(([k, v]) => [k, String(v)])
: [];
const handleLangCodeChange = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const newValue = { ...value };
const text = newValue[oldLang];
delete newValue[oldLang];
newValue[newLang] = text;
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleLangTextChange = (lang: string, text: string) => {
const newValue = { ...value, [lang]: text };
setValue(newValue);
};
const handleRemoveLang = (lang: string) => {
const newValue = { ...value };
delete newValue[lang];
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
};
const handleAddLang = () => {
// Find a language code not already used, default to 'eng'
const usedLangs = new Set(langEntries.map(([k]) => k));
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
setValue({ ...value, [defaultLang]: '' });
};
if (isReadonly && langEntries.length === 0) {
return null;
}
return (
<div className="dynamic-field language-string-field">
<label>{schema.label}</label>
<div className="language-entries">
{langEntries.map(([lang, text]) => (
<div key={lang} className="language-entry">
{isReadonly ? (
<>
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
<span className="lang-text">{text}</span>
</>
) : (
<>
<LanguageCombobox
value={lang}
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
/>
<input
type="text"
className="lang-input"
value={text}
onChange={(e) => handleLangTextChange(lang, e.target.value)}
placeholder={schema.placeholder || 'Text...'}
/>
<button
type="button"
className="lang-remove-btn"
onClick={() => handleRemoveLang(lang)}
title="Remove"
>
×
</button>
</>
)}
</div>
))}
{!isReadonly && (
<button
type="button"
className="lang-add-btn"
onClick={handleAddLang}
>
+ Add Language
</button>
)}
</div>
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
case 'string':
default:
// Show nothing if no value and readonly
if (isReadonly && !value) {
return null;
}
return (
<div className="dynamic-field">
<label>{schema.label}</label>
{isReadonly ? (
<div className="readonly-value">
{renderValueWithLink(String(value ?? ''))}
</div>
) : (
<input
type="text"
value={value ?? ''}
onChange={(e) => setValue(e.target.value || undefined)}
placeholder={schema.placeholder}
/>
)}
{schema.hint && <small className="field-hint">{schema.hint}</small>}
</div>
);
}
}
// Helper to set nested value in object
function setNestedValue(obj: any, path: string, value: any) {
const parts = path.split('/').filter(Boolean);
let current = obj;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
}
current = current[parts[i]];
}
if (parts.length > 0) {
current[parts[parts.length - 1]] = value;
}
}
@@ -1,63 +0,0 @@
.plugin-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.plugin-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-section-header .plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.plugin-fields {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
/* Responsive grid for fields */
@media (min-width: 768px) {
.plugin-fields {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
gap: 1rem;
}
/* Make text and json fields span full width */
.plugin-fields .dynamic-field:has(textarea),
.plugin-fields .json-field {
grid-column: 1 / -1;
}
}
/* Light theme */
@media (prefers-color-scheme: light) {
.plugin-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-section-header .plugin-description {
color: rgba(0, 0, 0, 0.5);
}
}
@@ -1,126 +0,0 @@
import { PluginSchema, FileMetadata, SchemaField } from '../types';
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
import './DynamicPluginSection.css';
interface DynamicPluginSectionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (key: string, value: any) => void;
}
/**
* Renders a plugin section with all its fields based on schema
*/
export function DynamicPluginSection({
plugin,
metadata,
onChange,
}: DynamicPluginSectionProps) {
const schema = plugin.metadataSchema;
if (!schema || Object.keys(schema).length === 0) {
return null;
}
// Check if plugin has any data in metadata
const hasData = checkPluginHasData(schema, metadata);
if (!hasData) {
return null;
}
// Sort fields: non-readonly first, then by key
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
// Non-readonly fields first
if (a.readonly !== b.readonly) {
return a.readonly ? 1 : -1;
}
// Then alphabetically
return keyA.localeCompare(keyB);
});
return (
<section
className="plugin-section"
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<div className="plugin-section-header">
<h3>{plugin.name}</h3>
{plugin.description && (
<p className="plugin-description">{plugin.description}</p>
)}
</div>
<div className="plugin-fields">
{sortedFields.map(([fieldKey, fieldSchema]) => (
<DynamicFieldRenderer
key={fieldKey}
fieldKey={fieldKey}
schema={fieldSchema}
metadata={metadata}
onChange={onChange}
/>
))}
</div>
</section>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function checkPluginHasData(
schema: Record<string, SchemaField>,
metadata: FileMetadata
): boolean {
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
/**
* Get nested value from object using path-style key
*/
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
/**
* Check if object has any key starting with prefix
*/
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
-174
View File
@@ -1,174 +0,0 @@
.file-list {
border: 1px solid rgba(255, 255, 255, 0.1);
border-radius: 8px;
overflow: hidden;
}
.file-list-header {
padding: 0.75rem 1rem;
background-color: rgba(255, 255, 255, 0.05);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
justify-content: space-between;
align-items: center;
}
.file-count {
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
}
.bulk-edit-toggle {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(100, 108, 255, 0.1);
border: 1px solid rgba(100, 108, 255, 0.3);
color: #646cff;
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.bulk-edit-toggle:hover {
background-color: rgba(100, 108, 255, 0.2);
border-color: rgba(100, 108, 255, 0.5);
}
.bulk-edit-toggle.active {
background-color: rgba(100, 108, 255, 0.3);
border-color: #646cff;
font-weight: 500;
}
.file-list-bulk-actions {
padding: 0.75rem 1rem;
background-color: rgba(100, 108, 255, 0.1);
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
display: flex;
gap: 1rem;
align-items: center;
}
.select-all-btn {
padding: 0.4rem 0.8rem;
font-size: 0.85em;
background-color: rgba(255, 255, 255, 0.1);
border: 1px solid rgba(255, 255, 255, 0.2);
color: rgba(255, 255, 255, 0.9);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.select-all-btn:hover {
background-color: rgba(255, 255, 255, 0.2);
border-color: rgba(255, 255, 255, 0.3);
}
.selection-count {
font-size: 0.9em;
color: #646cff;
font-weight: 500;
}
.file-list-items {
max-height: 500px;
overflow-y: auto;
}
.file-list-item {
padding: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
cursor: pointer;
transition: background-color 0.2s;
display: flex;
gap: 0.75rem;
align-items: flex-start;
}
.file-list-item-checkbox {
padding-top: 0.2rem;
}
.file-list-item-checkbox input[type='checkbox'] {
width: 18px;
height: 18px;
cursor: pointer;
}
.file-list-item-content {
flex: 1;
}
.file-list-item:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.2);
border-left: 3px solid #646cff;
}
.file-list-item-title {
font-weight: 500;
margin-bottom: 0.5rem;
}
.file-list-item-details {
display: flex;
gap: 1rem;
font-size: 0.9em;
color: rgba(255, 255, 255, 0.6);
margin-bottom: 0.25rem;
}
.file-list-item-hash {
font-size: 0.8em;
color: rgba(255, 255, 255, 0.4);
font-family: 'Courier New', monospace;
}
.file-list-empty {
padding: 3rem 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
@media (prefers-color-scheme: light) {
.file-list {
border-color: rgba(0, 0, 0, 0.1);
}
.file-list-header {
background-color: rgba(0, 0, 0, 0.02);
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.file-count {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.file-list-item:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.file-list-item.selected {
background-color: rgba(100, 108, 255, 0.1);
}
.file-list-item-details {
color: rgba(0, 0, 0, 0.6);
}
.file-list-item-hash {
color: rgba(0, 0, 0, 0.4);
}
.file-list-empty {
color: rgba(0, 0, 0, 0.5);
}
}
-123
View File
@@ -1,123 +0,0 @@
import { SearchResult } from '../types';
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
import './FileList.css';
interface FileListProps {
results: SearchResult[];
selectedHashId?: string;
selectedHashIds: Set<string>;
bulkEditMode: boolean;
onSelectFile: (hashId: string) => void;
onToggleSelection: (hashId: string) => void;
onSelectAll: () => void;
onClearSelection: () => void;
onBulkEditMode: (enabled: boolean) => void;
}
export function FileList({
results,
selectedHashId,
selectedHashIds,
bulkEditMode,
onSelectFile,
onToggleSelection,
onSelectAll,
onClearSelection,
onBulkEditMode,
}: FileListProps) {
if (results.length === 0) {
return (
<div className="file-list-empty">
<p>No files found. Try searching for a title, filename, or hash ID.</p>
</div>
);
}
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
// Sort results by file path
const sortedResults = [...results].sort((a, b) => {
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
return pathA.localeCompare(pathB);
});
return (
<div className="file-list">
<div className="file-list-header">
<span className="file-count">{results.length} files found</span>
<button
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
onClick={() => onBulkEditMode(!bulkEditMode)}
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
>
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
</button>
</div>
{bulkEditMode && (
<div className="file-list-bulk-actions">
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
{allSelected ? 'Deselect All' : 'Select All'}
</button>
{selectedHashIds.size > 0 && (
<span className="selection-count">
{selectedHashIds.size} selected
</span>
)}
</div>
)}
<div className="file-list-items">
{sortedResults.map((result) => {
const isSelected = bulkEditMode
? selectedHashIds.has(result.hashId)
: selectedHashId === result.hashId;
return (
<div
key={result.hashId}
className={`file-list-item ${isSelected ? 'selected' : ''}`}
onClick={() => {
if (bulkEditMode) {
onToggleSelection(result.hashId);
} else {
onSelectFile(result.hashId);
}
}}
>
{bulkEditMode && (
<div className="file-list-item-checkbox">
<input
type="checkbox"
checked={selectedHashIds.has(result.hashId)}
onChange={(e) => {
e.stopPropagation();
onToggleSelection(result.hashId);
}}
/>
</div>
)}
<div className="file-list-item-content">
<div className="file-list-item-title">
{getDisplayName(result.metadata)}
</div>
<div className="file-list-item-details">
<span className="file-list-item-type">
{result.metadata.videoType || 'Unknown'}
</span>
<span className="file-list-item-size">
{formatBytes(result.metadata.byteSize)}
</span>
</div>
<div className="file-list-item-hash">
Hash: {truncateHash(result.hashId)}
</div>
</div>
</div>
);
})}
</div>
</div>
);
}
-247
View File
@@ -1,247 +0,0 @@
.kv-tree {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
border: 1px solid rgba(255, 255, 255, 0.1);
overflow: hidden;
}
.kv-tree-header {
display: flex;
align-items: center;
gap: 1rem;
padding: 0.75rem 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
background-color: rgba(0, 0, 0, 0.2);
}
.kv-tree-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.8);
}
.kv-tree-info {
display: flex;
gap: 1rem;
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-refresh {
margin-left: auto;
padding: 0.25rem 0.75rem;
font-size: 0.75rem;
background-color: rgba(100, 108, 255, 0.2);
color: #646cff;
border: 1px solid rgba(100, 108, 255, 0.3);
border-radius: 4px;
cursor: pointer;
transition: all 0.2s;
}
.kv-tree-refresh:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.3);
}
.kv-tree-refresh:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-error {
padding: 0.5rem 1rem;
background-color: rgba(255, 0, 0, 0.1);
color: #ff6b6b;
font-size: 0.8rem;
}
.kv-tree-content {
max-height: 400px;
overflow-y: auto;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
font-size: 0.8rem;
}
.kv-tree-node {
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-node:last-child {
border-bottom: none;
}
.kv-tree-key {
display: flex;
align-items: center;
gap: 0.5rem;
padding: 0.5rem 1rem;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-key:hover {
background-color: rgba(255, 255, 255, 0.05);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-toggle {
width: 1rem;
color: rgba(255, 255, 255, 0.4);
font-size: 0.65rem;
}
.kv-tree-key-name {
flex: 1;
color: #88c0d0;
word-break: break-all;
}
.kv-tree-type {
padding: 0.1rem 0.4rem;
font-size: 0.65rem;
background-color: rgba(255, 255, 255, 0.1);
border-radius: 3px;
color: rgba(255, 255, 255, 0.5);
}
.kv-tree-value {
padding: 0.5rem 1rem 0.5rem 2.5rem;
background-color: rgba(0, 0, 0, 0.2);
border-top: 1px solid rgba(255, 255, 255, 0.05);
}
.kv-tree-fields {
width: 100%;
border-collapse: collapse;
}
.kv-tree-fields tr:hover {
background-color: rgba(255, 255, 255, 0.03);
}
.kv-field-name {
padding: 0.25rem 0.5rem 0.25rem 0;
color: #a3be8c;
white-space: nowrap;
vertical-align: top;
width: 1%;
}
.kv-field-value {
padding: 0.25rem 0;
color: rgba(255, 255, 255, 0.7);
word-break: break-all;
}
.kv-field-value pre {
margin: 0;
font-family: inherit;
white-space: pre-wrap;
font-size: 0.75rem;
}
.kv-tree-load-more {
display: block;
width: 100%;
padding: 0.75rem;
font-size: 0.8rem;
background-color: transparent;
color: #646cff;
border: none;
cursor: pointer;
transition: background-color 0.15s;
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.1);
}
.kv-tree-load-more:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.kv-tree-empty {
padding: 2rem;
text-align: center;
color: rgba(255, 255, 255, 0.4);
}
/* Light mode */
@media (prefers-color-scheme: light) {
.kv-tree {
background-color: rgba(0, 0, 0, 0.02);
border-color: rgba(0, 0, 0, 0.1);
}
.kv-tree-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-header h3 {
color: rgba(0, 0, 0, 0.8);
}
.kv-tree-info {
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-error {
background-color: rgba(255, 0, 0, 0.08);
color: #cc0000;
}
.kv-tree-node {
border-bottom-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-key:hover {
background-color: rgba(0, 0, 0, 0.03);
}
.kv-tree-key.expanded {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-toggle {
color: rgba(0, 0, 0, 0.4);
}
.kv-tree-key-name {
color: #0077aa;
}
.kv-tree-type {
background-color: rgba(0, 0, 0, 0.08);
color: rgba(0, 0, 0, 0.5);
}
.kv-tree-value {
background-color: rgba(0, 0, 0, 0.02);
border-top-color: rgba(0, 0, 0, 0.05);
}
.kv-tree-fields tr:hover {
background-color: rgba(0, 0, 0, 0.02);
}
.kv-field-name {
color: #228822;
}
.kv-field-value {
color: rgba(0, 0, 0, 0.7);
}
.kv-tree-load-more:hover:not(:disabled) {
background-color: rgba(100, 108, 255, 0.08);
}
.kv-tree-empty {
color: rgba(0, 0, 0, 0.4);
}
}
-216
View File
@@ -1,216 +0,0 @@
import { useState, useEffect, useCallback } from 'react';
import './KVTreeView.css';
interface KeyValue {
key: string;
type?: string;
value?: Record<string, any> | string;
expanded?: boolean;
loading?: boolean;
}
interface KVInfo {
prefix: string;
fileCount: number;
keyCount: number;
totalSize: number;
memoryUsage?: string;
}
const API_BASE = '/api/kv';
export function KVTreeView() {
const [keys, setKeys] = useState<KeyValue[]>([]);
const [cursor, setCursor] = useState<string>('0');
const [hasMore, setHasMore] = useState(true);
const [loading, setLoading] = useState(false);
const [error, setError] = useState<string | null>(null);
const [info, setInfo] = useState<KVInfo | null>(null);
// Load KV info on mount
useEffect(() => {
fetch(`${API_BASE}/info`)
.then(res => res.json())
.then(data => setInfo(data))
.catch(err => console.error('Failed to load KV info:', err));
}, []);
// Load initial keys
const loadKeys = useCallback(async (reset = false) => {
if (loading) return;
const currentCursor = reset ? '0' : cursor;
if (!reset && currentCursor === '0' && keys.length > 0) return;
setLoading(true);
setError(null);
try {
const res = await fetch(`${API_BASE}/keys?cursor=${currentCursor}&count=50`);
if (!res.ok) throw new Error('Failed to load keys');
const data = await res.json();
const newKeys: KeyValue[] = data.keys.map((key: string) => ({
key,
expanded: false,
loading: false
}));
if (reset) {
setKeys(newKeys);
} else {
setKeys(prev => [...prev, ...newKeys]);
}
setCursor(data.cursor);
setHasMore(data.hasMore);
} catch (err: any) {
setError(err.message);
} finally {
setLoading(false);
}
}, [cursor, keys.length, loading]);
// Load on mount
useEffect(() => {
loadKeys(true);
}, []);
// Toggle key expansion and load value
const toggleKey = async (index: number) => {
const key = keys[index];
if (key.expanded) {
// Collapse
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: false } : k
));
return;
}
// Expand and load value if not already loaded
if (key.value === undefined) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: true } : k
));
try {
const res = await fetch(`${API_BASE}/key/${encodeURIComponent(key.key)}`);
if (!res.ok) throw new Error('Failed to load value');
const data = await res.json();
setKeys(prev => prev.map((k, i) =>
i === index ? {
...k,
type: data.type,
value: data.value,
expanded: true,
loading: false
} : k
));
} catch (err) {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, loading: false, value: 'Error loading value' } : k
));
}
} else {
setKeys(prev => prev.map((k, i) =>
i === index ? { ...k, expanded: true } : k
));
}
};
const formatValue = (value: any): string => {
if (value === null) return 'null';
if (value === undefined) return 'undefined';
if (typeof value === 'object') {
return JSON.stringify(value, null, 2);
}
return String(value);
};
const formatSize = (bytes: number): string => {
if (bytes < 1024) return `${bytes} B`;
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
};
return (
<div className="kv-tree">
<div className="kv-tree-header">
<h3>KV Database Browser</h3>
{info && (
<div className="kv-tree-info">
<span>Keys: {info.keyCount}</span>
<span>Files: {formatSize(info.totalSize)}</span>
{info.memoryUsage && <span>Memory: {info.memoryUsage}</span>}
</div>
)}
<button
className="kv-tree-refresh"
onClick={() => loadKeys(true)}
disabled={loading}
>
{loading ? 'Loading...' : 'Refresh'}
</button>
</div>
{error && <div className="kv-tree-error">{error}</div>}
<div className="kv-tree-content">
{keys.map((item, index) => (
<div key={item.key} className="kv-tree-node">
<div
className={`kv-tree-key ${item.expanded ? 'expanded' : ''}`}
onClick={() => toggleKey(index)}
>
<span className="kv-tree-toggle">
{item.loading ? '...' : (item.expanded ? '▼' : '▶')}
</span>
<span className="kv-tree-key-name">{item.key}</span>
{item.type && <span className="kv-tree-type">{item.type}</span>}
</div>
{item.expanded && item.value && (
<div className="kv-tree-value">
{typeof item.value === 'object' ? (
<table className="kv-tree-fields">
<tbody>
{Object.entries(item.value).map(([field, val]) => (
<tr key={field}>
<td className="kv-field-name">{field}</td>
<td className="kv-field-value">
<pre>{formatValue(val)}</pre>
</td>
</tr>
))}
</tbody>
</table>
) : (
<pre>{formatValue(item.value)}</pre>
)}
</div>
)}
</div>
))}
{hasMore && (
<button
className="kv-tree-load-more"
onClick={() => loadKeys()}
disabled={loading}
>
{loading ? 'Loading...' : 'Load More'}
</button>
)}
{!loading && keys.length === 0 && (
<div className="kv-tree-empty">No keys found</div>
)}
</div>
</div>
);
}
@@ -8,15 +8,6 @@ for (const lang of iso6393) {
languagesByCode.set(lang.iso6393, lang); languagesByCode.set(lang.iso6393, lang);
} }
// Get display name for a language code
export function getLanguageName(code: string): string {
const lang = languagesByCode.get(code);
if (lang) {
return lang.name;
}
return code.toUpperCase();
}
// Format for display: "English (eng)" or "eng" for unknown // Format for display: "English (eng)" or "eng" for unknown
export function formatLanguageOption(code: string): string { export function formatLanguageOption(code: string): string {
const lang = languagesByCode.get(code); const lang = languagesByCode.get(code);
-438
View File
@@ -1,438 +0,0 @@
.metadata-editor {
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
padding: 1.5rem;
}
.metadata-editor-header {
display: flex;
justify-content: space-between;
align-items: center;
margin-bottom: 1rem;
padding-bottom: 1rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-header h2 {
margin: 0;
font-size: 1.5rem;
}
.metadata-editor-actions {
display: flex;
gap: 0.5rem;
}
.metadata-editor-actions button.primary {
background-color: #646cff;
color: white;
}
.metadata-editor-actions button.primary:hover:not(:disabled) {
background-color: #535bf2;
}
.metadata-editor-actions .download-button {
display: inline-flex;
align-items: center;
padding: 0.5rem 1rem;
background-color: #28a745;
color: white;
border-radius: 4px;
text-decoration: none;
font-size: 0.9rem;
font-weight: 500;
transition: background-color 0.2s;
}
.metadata-editor-actions .download-button:hover {
background-color: #218838;
}
.metadata-editor-message {
padding: 1rem;
border-radius: 4px;
margin-bottom: 1rem;
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border: 1px solid rgba(0, 200, 0, 0.3);
color: #00ff00;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border: 1px solid rgba(255, 0, 0, 0.3);
color: #ff6b6b;
}
.metadata-editor-hashes {
margin-bottom: 1.5rem;
padding: 1rem;
background: rgba(0, 0, 0, 0.3);
border-radius: 8px;
border-left: 4px solid #888;
}
.hashes-header {
margin-bottom: 0.75rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hashes-header h3 {
margin: 0;
font-size: 0.9rem;
color: rgba(255, 255, 255, 0.7);
text-transform: uppercase;
letter-spacing: 0.05em;
}
.hashes-grid {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.hash-value {
display: flex;
flex-direction: column;
gap: 0.15rem;
}
.hash-value-primary {
margin-bottom: 0.5rem;
padding-bottom: 0.5rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.hash-value-primary .hash-label {
color: #4ecdc4;
}
.hash-value-primary .hash-code {
color: #4ecdc4;
font-size: 1rem;
}
.hash-label {
font-size: 0.75rem;
color: rgba(255, 255, 255, 0.4);
text-transform: uppercase;
letter-spacing: 0.03em;
}
.hash-code {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.6);
word-break: break-all;
cursor: pointer;
transition: color 0.2s;
}
.hash-code:hover {
color: #fff;
}
/* Plugin Chips */
.metadata-editor-chips {
display: flex;
flex-wrap: wrap;
gap: 0.5rem;
margin-bottom: 1.5rem;
}
.plugin-chip {
padding: 0.4rem 0.8rem;
border-radius: 16px;
font-size: 0.85rem;
font-weight: 500;
cursor: pointer;
transition: all 0.2s ease;
border: 2px solid var(--chip-color);
color: var(--chip-color);
background: transparent;
}
.plugin-chip:hover {
background-color: var(--chip-color);
color: white;
}
.plugin-chip.active {
color: white;
}
/* Plugin Accordion */
.plugin-accordion {
margin-bottom: 0.5rem;
border-radius: 8px;
overflow: hidden;
background-color: rgba(0, 0, 0, 0.2);
}
.plugin-accordion-header {
width: 100%;
display: flex;
justify-content: space-between;
align-items: center;
padding: 1rem 1.25rem;
border: none;
border-left: 4px solid #646cff;
background-color: rgba(255, 255, 255, 0.05);
color: inherit;
font-size: 1rem;
font-weight: 500;
cursor: pointer;
transition: background-color 0.2s ease;
}
.plugin-accordion-header:hover {
background-color: rgba(255, 255, 255, 0.1);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(255, 255, 255, 0.08);
}
.plugin-accordion-title {
color: inherit;
}
.plugin-accordion-icon {
font-size: 0.75rem;
opacity: 0.6;
transition: transform 0.2s ease;
}
.plugin-accordion-content {
padding: 1rem 1.25rem;
border-top: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-editor-form {
display: flex;
flex-direction: column;
gap: 0.5rem;
}
.metadata-section {
border: 1px solid rgba(255, 255, 255, 0.1);
border-left: 4px solid #646cff;
border-radius: 8px;
padding: 1.5rem;
background-color: rgba(0, 0, 0, 0.2);
}
.metadata-section-header {
margin-bottom: 1rem;
padding-bottom: 0.75rem;
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
}
.metadata-section-header h3 {
margin: 0 0 0.25rem 0;
font-size: 1.1rem;
color: #646cff;
}
.plugin-description {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-field {
display: flex;
flex-direction: column;
gap: 0.5rem;
margin-bottom: 1rem;
}
.metadata-field:last-child {
margin-bottom: 0;
}
.metadata-field label {
font-size: 0.9rem;
font-weight: 500;
color: rgba(255, 255, 255, 0.7);
}
.metadata-field input,
.metadata-field textarea,
.metadata-field select {
width: 100%;
}
.metadata-field-row {
display: grid;
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
gap: 1rem;
margin-bottom: 1rem;
}
.metadata-field-row:last-child {
margin-bottom: 0;
}
.hash-field {
font-family: 'Courier New', monospace;
font-size: 0.9em;
}
.no-hash-info {
margin: 0;
padding: 1rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
padding: 3rem;
text-align: center;
color: rgba(255, 255, 255, 0.5);
}
.metadata-editor-error button {
margin-top: 1rem;
}
@media (max-width: 768px) {
.metadata-editor-header {
flex-direction: column;
align-items: flex-start;
gap: 1rem;
}
.metadata-editor-actions {
width: 100%;
}
.metadata-editor-actions button {
flex: 1;
}
.metadata-field-row {
grid-template-columns: 1fr;
}
}
@media (prefers-color-scheme: light) {
.metadata-editor {
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-editor-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.metadata-section {
border-color: rgba(0, 0, 0, 0.1);
background-color: rgba(0, 0, 0, 0.02);
}
.metadata-section-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.plugin-description {
color: rgba(0, 0, 0, 0.5);
}
.metadata-field label {
color: rgba(0, 0, 0, 0.7);
}
.metadata-editor-message.success {
background-color: rgba(0, 200, 0, 0.1);
border-color: rgba(0, 200, 0, 0.3);
color: #008800;
}
.metadata-editor-message.error {
background-color: rgba(255, 0, 0, 0.1);
border-color: rgba(255, 0, 0, 0.3);
color: #cc0000;
}
.metadata-editor-loading,
.metadata-editor-error,
.metadata-editor-empty {
color: rgba(0, 0, 0, 0.5);
}
.no-hash-info {
color: rgba(0, 0, 0, 0.5);
}
.metadata-editor-hashes {
background: rgba(0, 0, 0, 0.05);
}
.hashes-header {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hashes-header h3 {
color: rgba(0, 0, 0, 0.7);
}
.hash-value-primary {
border-bottom-color: rgba(0, 0, 0, 0.1);
}
.hash-value-primary .hash-label,
.hash-value-primary .hash-code {
color: #0a9e94;
}
.hash-label {
color: rgba(0, 0, 0, 0.4);
}
.hash-code {
color: rgba(0, 0, 0, 0.6);
}
.hash-code:hover {
color: #000;
}
/* Plugin Chips - Light theme */
.plugin-chip {
background: transparent;
}
.plugin-chip:hover,
.plugin-chip.active {
color: white;
}
/* Plugin Accordion - Light theme */
.plugin-accordion {
background-color: rgba(0, 0, 0, 0.02);
}
.plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.03);
}
.plugin-accordion-header:hover {
background-color: rgba(0, 0, 0, 0.06);
}
.plugin-accordion.expanded .plugin-accordion-header {
background-color: rgba(0, 0, 0, 0.05);
}
.plugin-accordion-content {
border-top-color: rgba(0, 0, 0, 0.1);
}
}
-392
View File
@@ -1,392 +0,0 @@
import { useState, useEffect } from 'react';
import { FileMetadata, PluginSchema, PluginsResponse } from '../types';
import { MetadataAPI } from '../api/metadataApi';
import { DynamicPluginSection } from './DynamicPluginSection';
import './MetadataEditor.css';
interface PluginAccordionProps {
plugin: PluginSchema;
metadata: FileMetadata;
onChange: (field: string, value: any) => void;
isExpanded: boolean;
onToggle: () => void;
}
function PluginAccordion({ plugin, metadata, onChange, isExpanded, onToggle }: PluginAccordionProps) {
return (
<div className={`plugin-accordion ${isExpanded ? 'expanded' : ''}`}>
<button
className="plugin-accordion-header"
onClick={onToggle}
style={{ borderLeftColor: plugin.color || '#646cff' }}
>
<span className="plugin-accordion-title">{plugin.name}</span>
<span className="plugin-accordion-icon">{isExpanded ? '▼' : '▶'}</span>
</button>
{isExpanded && (
<div className="plugin-accordion-content">
<DynamicPluginSection
plugin={plugin}
metadata={metadata}
onChange={onChange}
/>
</div>
)}
</div>
);
}
interface MetadataEditorProps {
hashId: string;
onSaved?: () => void;
}
export function MetadataEditor({ hashId, onSaved }: MetadataEditorProps) {
const [metadata, setMetadata] = useState<FileMetadata | null>(null);
const [editedMetadata, setEditedMetadata] = useState<FileMetadata | null>(null);
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
const [isLoading, setIsLoading] = useState(true);
const [isSaving, setIsSaving] = useState(false);
const [error, setError] = useState<string | null>(null);
const [successMessage, setSuccessMessage] = useState<string | null>(null);
const [expandedPlugins, setExpandedPlugins] = useState<Set<string>>(new Set());
// Fetch plugins schema on mount
useEffect(() => {
fetchPlugins();
}, []);
// Fetch metadata when hashId changes
useEffect(() => {
loadMetadata();
}, [hashId]);
const fetchPlugins = async () => {
try {
const response = await fetch('/api/plugins');
if (response.ok) {
const data: PluginsResponse = await response.json();
// Sort by priority (lower = higher priority)
const sorted = [...data.plugins].sort((a, b) =>
(a.priority ?? 100) - (b.priority ?? 100)
);
setPlugins(sorted);
}
} catch (err) {
console.error('Failed to fetch plugin schemas:', err);
}
};
const loadMetadata = async () => {
try {
setIsLoading(true);
setError(null);
const data = await MetadataAPI.getMetadata(hashId);
setMetadata(data);
setEditedMetadata(data);
} catch (err: any) {
setError(err.message);
} finally {
setIsLoading(false);
}
};
const handleFieldChange = (field: string, value: any) => {
if (!editedMetadata) return;
// Handle path-style keys like "stream/duration" or "nfo/title"
const parts = field.split('/');
if (parts.length === 1) {
// Simple key
setEditedMetadata({
...editedMetadata,
[field]: value,
});
} else {
// Nested key - need to update nested structure
const newMetadata = { ...editedMetadata };
let current: any = newMetadata;
for (let i = 0; i < parts.length - 1; i++) {
if (!current[parts[i]]) {
current[parts[i]] = {};
} else {
current[parts[i]] = { ...current[parts[i]] };
}
current = current[parts[i]];
}
current[parts[parts.length - 1]] = value;
setEditedMetadata(newMetadata);
}
};
const handleSave = async () => {
if (!editedMetadata) return;
try {
setIsSaving(true);
setError(null);
setSuccessMessage(null);
await MetadataAPI.updateMetadata(hashId, editedMetadata);
setMetadata(editedMetadata);
setSuccessMessage('Metadata saved successfully!');
if (onSaved) {
onSaved();
}
// Clear success message after 3 seconds
setTimeout(() => setSuccessMessage(null), 3000);
} catch (err: any) {
setError(err.message);
} finally {
setIsSaving(false);
}
};
const handleReset = () => {
setEditedMetadata(metadata);
setError(null);
setSuccessMessage(null);
};
const togglePlugin = (pluginId: string) => {
setExpandedPlugins(prev => {
const next = new Set(prev);
if (next.has(pluginId)) {
next.delete(pluginId);
} else {
next.add(pluginId);
}
return next;
});
};
const scrollToPlugin = (pluginId: string) => {
// Expand the plugin if not already expanded
if (!expandedPlugins.has(pluginId)) {
setExpandedPlugins(prev => new Set(prev).add(pluginId));
}
// Scroll to the plugin after a short delay to allow expansion
setTimeout(() => {
const element = document.getElementById(`plugin-${pluginId}`);
if (element) {
element.scrollIntoView({ behavior: 'smooth', block: 'start' });
}
}, 100);
};
if (isLoading) {
return <div className="metadata-editor-loading">Loading metadata...</div>;
}
if (error && !metadata) {
return (
<div className="metadata-editor-error">
<p>Error loading metadata: {error}</p>
<button onClick={loadMetadata}>Retry</button>
</div>
);
}
if (!metadata || !editedMetadata) {
return <div className="metadata-editor-empty">No metadata available</div>;
}
const hasChanges = JSON.stringify(metadata) !== JSON.stringify(editedMetadata);
// Filter plugins that have data
const pluginsWithData = plugins.filter(plugin => {
if (!plugin.metadataSchema) return false;
return hasPluginData(plugin, editedMetadata);
});
// Get file path for download
const filePath = editedMetadata?.filePath as string | undefined;
const downloadUrl = filePath ? `/api/file/download?path=${encodeURIComponent(filePath)}` : null;
return (
<div className="metadata-editor">
<div className="metadata-editor-header">
<h2>Edit Metadata</h2>
<div className="metadata-editor-actions">
{downloadUrl && (
<a
href={downloadUrl}
className="download-button"
title={`Download: ${filePath}`}
download
>
⬇ Download
</a>
)}
{hasChanges && (
<button onClick={handleReset} disabled={isSaving}>
Reset
</button>
)}
<button
onClick={handleSave}
disabled={isSaving || !hasChanges}
className="primary"
>
{isSaving ? 'Saving...' : 'Save Changes'}
</button>
</div>
</div>
{successMessage && (
<div className="metadata-editor-message success">{successMessage}</div>
)}
{error && <div className="metadata-editor-message error">{error}</div>}
<div className="metadata-editor-hashes">
<div className="hashes-header">
<h3>Content Identifiers</h3>
</div>
<div className="hashes-grid">
<HashValue label="midhash256" value={hashId} primary />
{getHashValues(editedMetadata).map(({ key, label, value }) => (
<HashValue key={key} label={label} value={value} />
))}
</div>
</div>
{pluginsWithData.length > 0 && (
<div className="metadata-editor-chips">
{pluginsWithData.map((plugin) => (
<button
key={plugin.id}
className={`plugin-chip ${expandedPlugins.has(plugin.id) ? 'active' : ''}`}
style={{
'--chip-color': plugin.color || '#646cff',
backgroundColor: expandedPlugins.has(plugin.id) ? plugin.color || '#646cff' : 'transparent'
} as React.CSSProperties}
onClick={() => scrollToPlugin(plugin.id)}
>
{plugin.name}
</button>
))}
</div>
)}
<div className="metadata-editor-form">
{pluginsWithData.length === 0 ? (
<div className="metadata-editor-empty">
No metadata found for this file
</div>
) : (
pluginsWithData.map((plugin) => (
<div key={plugin.id} id={`plugin-${plugin.id}`}>
<PluginAccordion
plugin={plugin}
metadata={editedMetadata}
onChange={handleFieldChange}
isExpanded={expandedPlugins.has(plugin.id)}
onToggle={() => togglePlugin(plugin.id)}
/>
</div>
))
)}
</div>
</div>
);
}
/**
* Check if a plugin has any data in the metadata
*/
function hasPluginData(plugin: PluginSchema, metadata: FileMetadata): boolean {
const schema = plugin.metadataSchema;
if (!schema) return false;
for (const fieldKey of Object.keys(schema)) {
const fieldSchema = schema[fieldKey];
// For JSON type with pattern, check if any matching keys exist
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
if (hasKeyWithPrefix(metadata, patternPrefix)) {
return true;
}
continue;
}
// For regular fields, check if value exists
if (getNestedValue(metadata, fieldKey) !== undefined) {
return true;
}
}
return false;
}
function getNestedValue(obj: any, path: string): any {
const parts = path.split('/');
let value = obj;
for (const part of parts) {
if (value === undefined || value === null) return undefined;
value = value[part];
}
return value;
}
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
if (!obj || typeof obj !== 'object') return false;
for (const key of Object.keys(obj)) {
const newPath = currentPath ? `${currentPath}/${key}` : key;
if (newPath.startsWith(prefix)) {
return true;
}
if (typeof obj[key] === 'object' && obj[key] !== null) {
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
return true;
}
}
}
return false;
}
/**
* Extract all hash values (cid_*) from metadata
*/
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
const hashOrder = [
{ key: 'cid_sha2-256', label: 'SHA-256' },
{ key: 'cid_sha3-256', label: 'SHA3-256' },
{ key: 'cid_sha3-384', label: 'SHA3-384' },
{ key: 'cid_sha1', label: 'SHA-1' },
{ key: 'cid_md5', label: 'MD5' },
{ key: 'cid_crc32', label: 'CRC32' },
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
];
return hashOrder
.filter(({ key }) => (metadata as any)[key])
.map(({ key, label }) => ({
key,
label,
value: (metadata as any)[key],
}));
}
/**
* Display a single hash value with copy functionality
*/
function HashValue({ label, value, primary }: { label: string; value: string; primary?: boolean }) {
const handleCopy = () => {
navigator.clipboard.writeText(value);
};
return (
<div className={`hash-value ${primary ? 'hash-value-primary' : ''}`}>
<span className="hash-label">{label}</span>
<code className="hash-code" onClick={handleCopy} title="Click to copy">
{value}
</code>
</div>
);
}
-52
View File
@@ -1,52 +0,0 @@
.search-bar {
width: 100%;
padding: 1.5rem;
background-color: rgba(255, 255, 255, 0.05);
border-radius: 8px;
margin-bottom: 1rem;
}
.search-controls {
display: flex;
gap: 0.5rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
}
.search-input {
flex: 1;
min-width: 200px;
}
.search-input.unified-search {
font-size: 1rem;
}
.search-bar button {
flex: 0 0 auto;
min-width: 100px;
}
.search-hint {
margin: 0;
font-size: 0.85rem;
color: rgba(255, 255, 255, 0.5);
font-style: italic;
}
@media (max-width: 768px) {
.search-controls {
flex-direction: column;
}
.search-input,
.search-bar button {
width: 100%;
}
}
@media (prefers-color-scheme: light) {
.search-hint {
color: rgba(0, 0, 0, 0.5);
}
}
-40
View File
@@ -1,40 +0,0 @@
import { useState } from 'react';
import './SearchBar.css';
interface SearchBarProps {
onSearch: (query: string) => void;
isLoading: boolean;
}
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
const [query, setQuery] = useState('');
const handleSubmit = (e: React.FormEvent) => {
e.preventDefault();
if (query.trim()) {
onSearch(query.trim());
}
};
return (
<form className="search-bar" onSubmit={handleSubmit}>
<div className="search-controls">
<input
type="text"
value={query}
onChange={(e) => setQuery(e.target.value)}
placeholder="Search by title, filename, filepath, or hash ID..."
className="search-input unified-search"
disabled={isLoading}
/>
<button type="submit" disabled={isLoading || !query.trim()}>
{isLoading ? 'Searching...' : 'Search'}
</button>
</div>
<p className="search-hint">
Search across all fields: title, filename, file path, and hash ID
</p>
</form>
);
}
+209
View File
@@ -0,0 +1,209 @@
.snapshot-panel {
max-width: 920px;
margin: 0 auto;
padding: 1.5rem;
}
.snapshot-panel h2 {
margin: 0 0 0.5rem;
}
.snapshot-intro {
color: var(--text-muted, #888);
margin-bottom: 1.5rem;
}
.snapshot-section {
border: 1px solid var(--border, #e0e0e0);
border-radius: 6px;
padding: 1rem 1.25rem;
margin-bottom: 1.25rem;
}
.snapshot-section.danger {
border-color: #d04a4a;
background: rgba(208, 74, 74, 0.04);
}
.snapshot-section h3 {
margin: 0 0 0.5rem;
}
.snapshot-row {
display: flex;
gap: 1rem;
align-items: flex-start;
margin: 0.75rem 0;
flex-wrap: wrap;
}
.snapshot-file {
display: flex;
flex-direction: column;
gap: 0.25rem;
}
.snapshot-file span {
font-size: 0.85rem;
color: var(--text-muted, #888);
}
.snapshot-fieldset {
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.5rem 0.75rem;
min-width: 280px;
}
.snapshot-fieldset[disabled] {
opacity: 0.5;
}
.snapshot-fieldset legend {
font-weight: 600;
font-size: 0.85rem;
padding: 0 0.25rem;
}
.snapshot-fieldset label {
display: block;
margin: 0.25rem 0;
font-size: 0.9rem;
cursor: pointer;
}
.snapshot-fieldset input[type='radio'] {
margin-right: 0.5rem;
}
.snapshot-btn {
background: var(--bg-button, #f0f0f0);
border: 1px solid var(--border, #c0c0c0);
color: var(--text, #222);
padding: 0.5rem 1rem;
border-radius: 4px;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-btn:hover:not(:disabled) {
background: var(--bg-button-hover, #e0e0e0);
}
.snapshot-btn:disabled {
opacity: 0.5;
cursor: not-allowed;
}
.snapshot-btn.primary {
background: #2a6df4;
border-color: #2a6df4;
color: white;
}
.snapshot-btn.primary:hover:not(:disabled) {
background: #1a5de4;
}
.snapshot-btn.danger {
background: #d04a4a;
border-color: #d04a4a;
color: white;
}
.snapshot-btn.danger:hover:not(:disabled) {
background: #b83838;
}
.snapshot-error {
background: rgba(208, 74, 74, 0.1);
color: #b83838;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-info {
background: rgba(42, 109, 244, 0.08);
color: #1a5de4;
padding: 0.5rem 0.75rem;
border-radius: 4px;
margin-top: 0.75rem;
}
.snapshot-result {
margin-top: 0.75rem;
border: 1px solid var(--border, #e0e0e0);
border-radius: 4px;
padding: 0.75rem;
}
.snapshot-result-summary {
display: flex;
gap: 1rem;
flex-wrap: wrap;
margin-bottom: 0.5rem;
font-size: 0.9rem;
}
.snapshot-result-table {
width: 100%;
border-collapse: collapse;
font-size: 0.9rem;
}
.snapshot-result-table th,
.snapshot-result-table td {
border: 1px solid var(--border, #e0e0e0);
padding: 0.35rem 0.5rem;
text-align: right;
}
.snapshot-result-table th {
background: var(--bg-subtle, #f5f5f5);
font-weight: 600;
}
.snapshot-result-errors {
margin-top: 0.5rem;
font-size: 0.85rem;
}
.snapshot-result-errors summary {
cursor: pointer;
color: #b83838;
}
.snapshot-result-errors ul {
margin: 0.5rem 0 0;
padding-left: 1.25rem;
}
.snapshot-result-errors code {
font-size: 0.8rem;
word-break: break-all;
}
.snapshot-checkbox {
display: flex;
align-items: flex-start;
gap: 0.5rem;
margin: 0.5rem 0 0.75rem;
cursor: pointer;
font-size: 0.95rem;
}
.snapshot-checkbox input {
margin-top: 0.2rem;
}
.snapshot-hint {
color: var(--text-muted, #888);
font-size: 0.9rem;
}
.snapshot-result-aside {
margin-top: 0.5rem;
font-size: 0.85rem;
color: var(--text-muted, #555);
}
+271
View File
@@ -0,0 +1,271 @@
import { useState } from 'react';
import {
SnapshotAPI,
ImportMode,
ImportConflict,
ImportResult,
} from '../api/snapshotApi';
import './SnapshotPanel.css';
export function SnapshotPanel() {
const [exportPluginOutput, setExportPluginOutput] = useState(false);
const [importFile, setImportFile] = useState<File | null>(null);
const [importMode, setImportMode] = useState<ImportMode>('merge');
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
const [importing, setImporting] = useState(false);
const [importResult, setImportResult] = useState<ImportResult | null>(null);
const [importError, setImportError] = useState<string | null>(null);
const [wipeBusy, setWipeBusy] = useState(false);
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
const runImport = async (dryRun: boolean) => {
if (!importFile) {
setImportError('Pick a snapshot ZIP first.');
return;
}
setImporting(true);
setImportError(null);
setImportResult(null);
try {
const result = await SnapshotAPI.runImport(importFile, {
mode: importMode,
conflict: importMode === 'merge' ? importConflict : undefined,
dryRun,
});
setImportResult(result);
} catch (e: any) {
setImportError(e.message || String(e));
} finally {
setImporting(false);
}
};
const runWipe = async () => {
const ok = window.confirm(
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
'edit you made via this editor will be lost. Continue?'
);
if (!ok) return;
setWipeBusy(true);
setWipeMessage(null);
try {
const r = await SnapshotAPI.wipe('metadata');
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
} catch (e: any) {
setWipeMessage(`Failed: ${e.message || e}`);
} finally {
setWipeBusy(false);
}
};
return (
<div className="snapshot-panel">
<h2>Snapshot</h2>
<p className="snapshot-intro">
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
archives — one nested-JSON file per CID, plus a manifest.
</p>
<section className="snapshot-section">
<h3>Export</h3>
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
<label className="snapshot-checkbox">
<input
type="checkbox"
checked={exportPluginOutput}
onChange={(e) => setExportPluginOutput(e.target.checked)}
/>
<span>
Include plugin output
<span className="snapshot-hint">
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
</span>
</span>
</label>
<button
className="snapshot-btn primary"
onClick={() =>
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
}
>
Download snapshot.zip
</button>
</section>
<section className="snapshot-section">
<h3>Import</h3>
<div className="snapshot-row">
<label className="snapshot-file">
<span>Snapshot ZIP</span>
<input
type="file"
accept=".zip,application/zip"
onChange={(e) => {
setImportFile(e.target.files?.[0] ?? null);
setImportResult(null);
setImportError(null);
}}
/>
</label>
</div>
<div className="snapshot-row">
<fieldset className="snapshot-fieldset">
<legend>Mode</legend>
<label>
<input
type="radio"
name="mode"
value="merge"
checked={importMode === 'merge'}
onChange={() => setImportMode('merge')}
/>
Merge — keep existing fields not in the snapshot
</label>
<label>
<input
type="radio"
name="mode"
value="replace"
checked={importMode === 'replace'}
onChange={() => setImportMode('replace')}
/>
Replace — wipe each imported CID and rewrite from snapshot
</label>
</fieldset>
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
<legend>Conflict</legend>
<label>
<input
type="radio"
name="conflict"
value="source"
checked={importConflict === 'source'}
onChange={() => setImportConflict('source')}
/>
Source wins (overwrite my edits)
</label>
<label>
<input
type="radio"
name="conflict"
value="mine"
checked={importConflict === 'mine'}
onChange={() => setImportConflict('mine')}
/>
Mine wins (keep my edits, fill in only missing fields)
</label>
</fieldset>
</div>
<div className="snapshot-row">
<button
className="snapshot-btn"
disabled={!importFile || importing}
onClick={() => runImport(true)}
>
Dry run
</button>
<button
className="snapshot-btn primary"
disabled={!importFile || importing}
onClick={() => runImport(false)}
>
{importing ? 'Importing…' : 'Apply import'}
</button>
</div>
{importError && <div className="snapshot-error">{importError}</div>}
{importResult && <ImportResultView result={importResult} />}
</section>
<section className="snapshot-section danger">
<h3>Wipe metadata</h3>
<p>
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
Files on disk are untouched.
</p>
<button
className="snapshot-btn danger"
disabled={wipeBusy}
onClick={runWipe}
>
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
</button>
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
</section>
</div>
);
}
function ImportResultView({ result }: { result: ImportResult }) {
const totals = (result.files ?? []).reduce(
(acc, f) => {
acc.added += f.added;
acc.updated += f.updated;
acc.unchanged += f.unchanged;
acc.kept += f.kept;
acc.deleted += f.deleted;
return acc;
},
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
);
return (
<div className="snapshot-result">
<div className="snapshot-result-summary">
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
<span>
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
</span>
<span>
mode={result.mode}
{result.conflict ? `, conflict=${result.conflict}` : ''}
</span>
</div>
<table className="snapshot-result-table">
<thead>
<tr>
<th>added</th>
<th>updated</th>
<th>unchanged</th>
<th>kept (mine)</th>
<th>deleted (replace)</th>
</tr>
</thead>
<tbody>
<tr>
<td>{totals.added}</td>
<td>{totals.updated}</td>
<td>{totals.unchanged}</td>
<td>{totals.kept}</td>
<td>{totals.deleted}</td>
</tr>
</tbody>
</table>
{result.pluginOutput && (
<div className="snapshot-result-aside">
plugin-output: {result.pluginOutput.written} written
{result.pluginOutput.skipped > 0
? `, ${result.pluginOutput.skipped} skipped (existed)`
: ''}
</div>
)}
{result.files && result.files.some((f) => f.error) && (
<details className="snapshot-result-errors">
<summary>Per-file errors</summary>
<ul>
{result.files
.filter((f) => f.error)
.map((f) => (
<li key={f.hashId}>
<code>{f.hashId}</code>: {f.error}
</li>
))}
</ul>
</details>
)}
</div>
);
}
+233
View File
@@ -0,0 +1,233 @@
import { useEffect, useMemo, useState } from 'react';
import { KVAPI, KVFindMatch } from '../api/kvApi';
interface Props {
matches: KVFindMatch[];
truncated: boolean;
searching: boolean;
selectedKey: string | null;
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
// FilteredKVTree is the search-active counterpart to KVTree. Instead of
// replacing the tree with a flat results list, it shows only the file
// branches that contain matches, with the matched (field, value) pairs
// rendered inline as sub-lines under each branch.
//
// Branches remain expandable: clicking one fetches the full subtree via
// KVAPI.treeAll just like the main tree, so the user can drill into every
// key of a matching file without clearing the search first.
export function FilteredKVTree({
matches, truncated, searching,
selectedKey, expandedPaths,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: Props) {
// Group matches by their entryPath (file:<hash>/) so each entry is one row
// with potentially multiple hit previews.
const groups = useMemo(() => {
const m = new Map<string, KVFindMatch[]>();
for (const x of matches) {
const arr = m.get(x.entryPath);
if (arr) arr.push(x);
else m.set(x.entryPath, [x]);
}
return m;
}, [matches]);
return (
<div className="kv-filtered">
<div className="kv-search-header">
{searching
? 'searching…'
: `${matches.length} match${matches.length === 1 ? '' : 'es'} in ${groups.size} file${groups.size === 1 ? '' : 's'}${truncated ? ' (truncated)' : ''}`}
</div>
{[...groups.entries()]
.sort(([a], [b]) => a.localeCompare(b))
.map(([entryPath, hits]) => (
<FilteredEntry
key={entryPath}
entryPath={entryPath}
hits={hits}
selectedKey={selectedKey}
expanded={expandedPaths.has(entryPath)}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
))}
</div>
);
}
interface EntryProps {
entryPath: string;
hits: KVFindMatch[];
selectedKey: string | null;
expanded: boolean;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function FilteredEntry({
entryPath, hits, selectedKey, expanded,
onSelectLeaf, onSelectBranch, onToggleBranch,
}: EntryProps) {
// We deliberately do NOT mark the matched-leaf rows as "selected" via the
// tree row's selected style — the inline hit lines are themselves the
// selected indicator (they're highlighted when their key is current).
const isBranchSelected = entryPath === selectedKey;
const HIT_PREVIEW = 4;
return (
<div className="kv-filtered-entry">
<div
className={`kv-tree-row kv-tree-branch${isBranchSelected ? ' selected' : ''}`}
style={{ paddingLeft: 4 }}
onClick={() => {
onToggleBranch(entryPath, !expanded);
onSelectBranch(entryPath);
}}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{entryPath}</span>
<span className="kv-filtered-count" title={`${hits.length} match${hits.length === 1 ? '' : 'es'}`}>
{hits.length}
</span>
</div>
<div className="kv-filtered-hits">
{hits.slice(0, HIT_PREVIEW).map((h) => (
<div
key={h.key}
className={`kv-filtered-hit${h.key === selectedKey ? ' selected' : ''}`}
onClick={(e) => { e.stopPropagation(); onSelectLeaf(h.key); }}
title={h.value}
>
<span className="kv-filtered-hit-field">{h.field}</span>
<span className="kv-filtered-hit-value">{h.value}</span>
</div>
))}
{hits.length > HIT_PREVIEW && (
<div className="kv-filtered-more">
+{hits.length - HIT_PREVIEW} more match{hits.length - HIT_PREVIEW === 1 ? '' : 'es'}
{!expanded && ' · expand to see all keys'}
</div>
)}
</div>
{expanded && (
<LazySubtree
path={entryPath}
depth={1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</div>
);
}
interface SubtreeProps {
path: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
interface SubtreeData {
loading: boolean;
branches: string[];
leaves: string[];
error?: string;
}
// LazySubtree fetches its children once on mount and renders them. Sub-
// branches are themselves LazyBranch components that fetch their own
// children on first expand — recursion all the way down without the global
// expandedPaths state of the main KVTree.
function LazySubtree({ path, depth, selectedKey, onSelectLeaf, onSelectBranch }: SubtreeProps) {
const [data, setData] = useState<SubtreeData>({ loading: true, branches: [], leaves: [] });
useEffect(() => {
let cancelled = false;
KVAPI.treeAll(path, '/', 5000)
.then((r) => { if (!cancelled) setData({ loading: false, branches: r.branches, leaves: r.leaves }); })
.catch((e) => { if (!cancelled) setData({ loading: false, branches: [], leaves: [], error: e.message }); });
return () => { cancelled = true; };
}, [path]);
if (data.loading) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (data.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>{data.error}</div>;
}
return (
<>
{data.branches.map((b) => (
<LazyBranch
key={b}
path={b}
parentPath={path}
depth={depth}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
))}
{data.leaves.map((l) => (
<div
key={l}
className={`kv-tree-row kv-tree-leaf${l === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(l)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{l.slice(path.length)}</span>
</div>
))}
</>
);
}
interface LazyBranchProps {
path: string;
parentPath: string;
depth: number;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
}
function LazyBranch({ path, parentPath, depth, selectedKey, onSelectLeaf, onSelectBranch }: LazyBranchProps) {
const [expanded, setExpanded] = useState(false);
const label = path.slice(parentPath.length);
return (
<>
<div
className={`kv-tree-row kv-tree-branch${path === selectedKey ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => { setExpanded(!expanded); onSelectBranch(path); }}
>
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
<span className="kv-tree-label">{label}</span>
</div>
{expanded && (
<LazySubtree
path={path}
depth={depth + 1}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
/>
)}
</>
);
}
+896
View File
@@ -0,0 +1,896 @@
/* KV Tree Editor — fills the iframe viewport with a 2-pane layout. */
.kv-app {
display: flex;
flex-direction: column;
height: 100vh;
width: 100%;
overflow: hidden;
background: #fafbfc;
color: #1f2330;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
font-size: 13px;
}
/* ── Header ───────────────────────────────────────────────────────────── */
.kv-app-header {
border-bottom: 1px solid #e2e6ee;
padding: 8px 12px;
background: #fff;
flex: 0 0 auto;
}
.kv-app-search {
display: flex;
align-items: center;
gap: 8px;
}
.kv-search-input {
flex: 1;
padding: 7px 11px;
border: 1px solid #d1d6e0;
border-radius: 6px;
font-size: 13px;
background: #fff;
outline: none;
}
.kv-search-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-app-stats {
font-size: 12px;
color: #8089a0;
white-space: nowrap;
}
.kv-app-breadcrumb {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 2px;
font-size: 12px;
}
.kv-crumb-label {
color: #8089a0;
margin-right: 4px;
}
.kv-crumb {
border: none;
background: transparent;
padding: 1px 4px;
border-radius: 3px;
cursor: pointer;
color: #4a5170;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-crumb:hover { background: #eef0fb; color: #2a2f48; }
/* ── Body ─────────────────────────────────────────────────────────────── */
.kv-app-body {
flex: 1 1 auto;
display: flex;
min-height: 0;
}
.kv-app-left {
flex: 0 0 38%;
min-width: 280px;
max-width: 600px;
border-right: 1px solid #e2e6ee;
background: #fff;
overflow: auto;
}
.kv-app-right {
flex: 1 1 auto;
overflow: auto;
padding: 16px 20px;
background: #fafbfc;
}
/* ── Tree ─────────────────────────────────────────────────────────────── */
.kv-tree {
padding: 4px 0;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-tree-list {
list-style: none;
margin: 0;
padding: 0;
}
.kv-tree-row {
display: flex;
align-items: center;
gap: 4px;
padding: 2px 4px;
cursor: pointer;
white-space: nowrap;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-row:hover { background: #f3f5fb; }
.kv-tree-toggle {
display: inline-block;
width: 12px;
color: #8089a0;
font-size: 10px;
}
.kv-tree-leaf-dot {
display: inline-block;
width: 12px;
color: #c5cad6;
}
.kv-tree-label {
overflow: hidden;
text-overflow: ellipsis;
}
.kv-tree-branch .kv-tree-label {
color: #2a3050;
font-weight: 500;
}
.kv-tree-branch .kv-tree-filename {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.85em;
color: #1e3a8a;
white-space: nowrap;
text-overflow: ellipsis;
overflow: hidden;
}
.kv-tree-hash-pill {
margin-left: 0.5rem;
padding: 0.05rem 0.4rem;
background: rgba(100, 108, 255, 0.12);
color: #4a50c8;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 0.7em;
border-radius: 999px;
flex-shrink: 0;
}
.kv-tree-leaf .kv-tree-label {
color: #4a5170;
}
.kv-tree-row.selected {
background: #e7eafd;
color: #2532a3;
}
.kv-tree-row.selected:hover { background: #dde2fc; }
.kv-tree-loading,
.kv-tree-empty,
.kv-tree-error {
font-size: 12px;
color: #8089a0;
padding: 4px 8px;
font-style: italic;
}
.kv-tree-error { color: #c0392b; }
/* ── Right pane (leaf / branch) ───────────────────────────────────────── */
.kv-leaf, .kv-branch {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-leaf-key {
display: flex;
align-items: center;
gap: 6px;
background: #f0f2f8;
border: 1px solid #e2e6ee;
border-radius: 6px;
padding: 8px 10px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
}
.kv-leaf-key-text {
flex: 1;
word-break: break-all;
color: #2a3050;
}
.kv-leaf-meta {
display: flex;
align-items: center;
gap: 8px;
}
.kv-leaf-label {
font-size: 12px;
color: #6e7790;
font-weight: 500;
}
.kv-leaf-type {
border: 1px solid #d1d6e0;
border-radius: 5px;
padding: 4px 8px;
font-size: 12px;
background: #fff;
}
.kv-leaf-hint {
font-size: 12px;
color: #6e7790;
font-style: italic;
}
.kv-leaf-input {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
box-sizing: border-box;
}
.kv-leaf-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-textarea {
width: 100%;
padding: 8px 10px;
border: 1px solid #d1d6e0;
border-radius: 5px;
font-size: 13px;
background: #fff;
outline: none;
resize: vertical;
box-sizing: border-box;
font-family: inherit;
}
.kv-leaf-textarea:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
.kv-leaf-mono {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-url {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-url-open {
font-size: 12px;
color: #6c7afa;
text-decoration: none;
white-space: nowrap;
}
.kv-leaf-actions {
display: flex;
gap: 8px;
}
/* ── CID leaf ─────────────────────────────────────────────────────────── */
.kv-leaf-cid {
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-leaf-cid-controls {
display: flex;
gap: 6px;
align-items: center;
}
.kv-leaf-cid-actions {
display: flex;
gap: 8px;
align-items: center;
flex-wrap: wrap;
}
.kv-leaf-cid-meta {
font-size: 12px;
color: #6b7280;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
}
.kv-leaf-cid-preview {
max-width: 100%;
max-height: 320px;
border-radius: 6px;
border: 1px solid #e2e6ee;
background: #f5f6fa;
object-fit: contain;
align-self: flex-start;
}
.kv-leaf-cid-preview-media {
width: 100%;
max-width: 560px;
}
/* ── File-detail view ─────────────────────────────────────────────────── */
.kv-file {
display: flex;
flex-direction: column;
gap: 14px;
}
.kv-file-header {
padding: 14px 16px;
background: #fff;
border: 1px solid #e2e6ee;
border-radius: 8px;
display: grid;
grid-template-columns: 1fr auto;
grid-template-areas:
"name actions"
"meta actions";
gap: 6px 16px;
align-items: center;
}
.kv-file-name {
grid-area: name;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-weight: 600;
font-size: 14px;
color: #1e3a8a;
word-break: break-all;
}
.kv-file-name-fallback {
color: #9ca3af;
font-style: italic;
}
.kv-file-meta {
grid-area: meta;
display: flex;
gap: 12px;
align-items: baseline;
font-size: 12px;
color: #6b7280;
min-width: 0;
}
.kv-file-hash {
background: rgba(100, 122, 250, 0.1);
color: #4a50c8;
padding: 1px 6px;
border-radius: 999px;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
flex-shrink: 0;
}
.kv-file-path {
overflow: hidden;
text-overflow: ellipsis;
white-space: nowrap;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
}
.kv-file-actions {
grid-area: actions;
display: flex;
gap: 8px;
align-items: center;
}
.kv-file-rows {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row {
display: grid;
grid-template-columns: minmax(140px, 220px) 1fr auto;
gap: 12px;
padding: 8px 12px;
background: #fff;
border: 1px solid #e2e6ee;
border-left: 3px solid #d6deed;
border-radius: 6px;
align-items: start;
}
.kv-file-row.pending-delete {
border-left-color: #dc2626;
background: #fef2f2;
}
.kv-file-row.pending-delete .kv-file-row-label {
text-decoration: line-through;
color: #9ca3af;
}
.kv-file-row-deleted-note {
color: #b91c1c;
font-style: italic;
font-size: 12px;
padding: 6px 0;
}
.kv-file-row-delete {
width: 24px;
height: 24px;
align-self: center;
display: inline-flex;
align-items: center;
justify-content: center;
background: transparent;
border: 1px solid transparent;
border-radius: 4px;
color: #dc2626;
font-size: 16px;
line-height: 1;
cursor: pointer;
opacity: 0.4;
transition: opacity 0.15s, background 0.15s, border-color 0.15s;
}
.kv-file-row:hover .kv-file-row-delete,
.kv-file-row.pending-delete .kv-file-row-delete {
opacity: 1;
}
.kv-file-row-delete:hover {
background: rgba(220, 38, 38, 0.1);
border-color: rgba(220, 38, 38, 0.3);
}
.kv-file-add {
display: flex;
gap: 8px;
align-items: center;
padding: 10px 12px;
background: rgba(100, 122, 250, 0.05);
border: 1px dashed rgba(100, 122, 250, 0.3);
border-radius: 6px;
}
.kv-file-add-prefix {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 11px;
color: #6b7280;
white-space: nowrap;
flex-shrink: 0;
max-width: 220px;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-file-add > input {
flex: 1;
}
.kv-file-add > input:first-of-type {
flex: 0 0 200px;
}
.kv-file-row.dirty {
border-left-color: #fbbf24;
background: #fffbeb;
}
.kv-file-row.pulsed {
animation: kv-row-pulse 1.4s ease-out;
}
@keyframes kv-row-pulse {
0% { background: rgba(100, 122, 250, 0.35); border-left-color: #4a50c8; }
100% { background: transparent; }
}
.kv-file-row-label,
.kv-file-row-label-static {
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
background: none;
border: none;
text-align: left;
cursor: pointer;
padding: 6px 0;
word-break: break-all;
}
.kv-file-row-label-static {
cursor: default;
}
.kv-file-row-label:hover {
color: #4a50c8;
text-decoration: underline;
}
.kv-file-row-value {
display: flex;
min-width: 0;
}
.kv-file-row-value > * {
flex: 1;
min-width: 0;
}
.kv-file-group {
display: flex;
flex-direction: column;
border: 1px solid #e2e6ee;
border-radius: 8px;
background: rgba(100, 122, 250, 0.03);
overflow: hidden;
}
.kv-file-group-header {
display: flex;
align-items: center;
gap: 8px;
padding: 6px 12px;
border: none;
background: rgba(100, 122, 250, 0.1);
cursor: pointer;
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
font-size: 12px;
color: #2a3050;
}
.kv-file-group-header:hover {
background: rgba(100, 122, 250, 0.2);
}
.kv-file-group-chevron {
font-size: 10px;
color: #6b7280;
}
.kv-file-group-name {
font-weight: 600;
color: #1e3a8a;
}
.kv-file-group-count {
margin-left: auto;
padding: 1px 8px;
background: rgba(100, 122, 250, 0.2);
color: #4a50c8;
border-radius: 999px;
font-size: 11px;
}
.kv-file-group-body {
padding: 8px;
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-row-lang .kv-file-lang-entries {
display: flex;
flex-direction: column;
gap: 6px;
}
.kv-file-lang-entry {
display: flex;
gap: 6px;
align-items: center;
}
.kv-file-lang-entry > .kv-leaf-input {
flex: 1;
}
.kv-file-lang-remove {
color: #dc2626;
}
.kv-file-lang-add {
align-self: flex-start;
background: rgba(100, 122, 250, 0.1);
}
/* ── DateTime leaf ────────────────────────────────────────────────────── */
.kv-leaf-datetime {
display: flex;
gap: 8px;
align-items: center;
}
.kv-leaf-datetime-shape {
font-size: 11px;
color: #6b7280;
background: rgba(100, 122, 250, 0.1);
padding: 2px 6px;
border-radius: 999px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
white-space: nowrap;
}
/* ── Branch ───────────────────────────────────────────────────────────── */
.kv-branch-section {
border: 1px solid #e2e6ee;
border-radius: 6px;
background: #fff;
padding: 12px 14px;
display: flex;
flex-direction: column;
gap: 8px;
}
.kv-branch-section h4 {
margin: 0;
font-size: 13px;
color: #2a3050;
}
.kv-branch-row {
display: flex;
align-items: center;
gap: 6px;
}
.kv-branch-prefix {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12px;
color: #6e7790;
background: #f0f2f8;
padding: 6px 8px;
border-radius: 4px;
white-space: nowrap;
max-width: 50%;
overflow: hidden;
text-overflow: ellipsis;
}
.kv-branch-note {
margin: 0;
font-size: 12px;
color: #6e7790;
line-height: 1.4;
}
/* ── Buttons ──────────────────────────────────────────────────────────── */
.kv-btn {
border: 1px solid #d1d6e0;
background: #fff;
color: #2a3050;
padding: 6px 14px;
border-radius: 5px;
font-size: 12.5px;
cursor: pointer;
}
.kv-btn:hover:not(:disabled) { background: #f3f5fb; border-color: #b8c0d0; }
.kv-btn:disabled { opacity: 0.45; cursor: not-allowed; }
.kv-btn-primary {
background: #6c7afa;
border-color: #6c7afa;
color: #fff;
}
.kv-btn-primary:hover:not(:disabled) {
background: #5868e8;
border-color: #5868e8;
}
.kv-btn-danger {
border-color: #e0788a;
color: #c0392b;
}
.kv-btn-danger:hover:not(:disabled) {
background: #fdecef;
border-color: #c0392b;
}
.kv-btn-icon {
border: 1px solid transparent;
background: transparent;
color: #6e7790;
padding: 2px 6px;
border-radius: 4px;
cursor: pointer;
font-size: 13px;
}
.kv-btn-icon:hover { background: #eef0fb; color: #2a3050; }
/* ── Pane status ──────────────────────────────────────────────────────── */
.kv-pane-status {
color: #6e7790;
font-size: 12.5px;
padding: 8px 0;
}
.kv-pane-empty {
text-align: center;
padding: 80px 20px;
font-size: 14px;
}
.kv-pane-hint {
margin-top: 8px;
font-size: 12px;
color: #8089a0;
}
/* ── Field chips (configure search-in fields) ─────────────────────────── */
.kv-chips {
margin-top: 6px;
display: flex;
flex-wrap: wrap;
align-items: center;
gap: 4px;
}
.kv-chips-label {
font-size: 12px;
color: #8089a0;
margin-right: 4px;
}
.kv-chip {
display: inline-flex;
align-items: center;
gap: 4px;
background: #eef0fb;
border: 1px solid #d5dafa;
color: #2a3050;
border-radius: 12px;
padding: 1px 4px 1px 8px;
font-size: 11.5px;
}
.kv-chip code {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11.5px;
}
.kv-chip-x {
border: none;
background: transparent;
color: #6e7790;
cursor: pointer;
padding: 0 4px;
font-size: 14px;
line-height: 1;
}
.kv-chip-x:hover { color: #c0392b; }
.kv-chip-input {
border: 1px dashed #c5cad6;
background: transparent;
border-radius: 12px;
padding: 2px 8px;
font-size: 11.5px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
outline: none;
min-width: 90px;
}
.kv-chip-input:focus { border-style: solid; border-color: #6c7afa; }
/* ── Search header (shared across filter mode) ────────────────────────── */
.kv-search-header {
padding: 6px 10px;
font-size: 11px;
color: #6e7790;
background: #f3f5fb;
border-bottom: 1px solid #e2e6ee;
position: sticky;
top: 0;
z-index: 1;
}
/* ── Filtered tree (search-active state) ──────────────────────────────── */
.kv-filtered {
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 12.5px;
}
.kv-filtered-entry {
border-bottom: 1px solid #f0f2f8;
}
.kv-filtered-count {
margin-left: auto;
background: #eef0fb;
color: #4a5170;
padding: 0 6px;
border-radius: 9px;
font-size: 10.5px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-weight: 600;
}
.kv-filtered-hits {
padding: 2px 4px 4px 24px;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
font-size: 12px;
}
.kv-filtered-hit {
display: flex;
gap: 6px;
align-items: baseline;
padding: 2px 6px;
border-radius: 3px;
cursor: pointer;
line-height: 1.35;
}
.kv-filtered-hit:hover { background: #f3f5fb; }
.kv-filtered-hit.selected {
background: #e7eafd;
}
.kv-filtered-hit-field {
background: #eef0fb;
color: #4a5170;
padding: 0 5px;
border-radius: 3px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 10.5px;
flex: 0 0 auto;
}
.kv-filtered-hit-value {
color: #2a3050;
word-break: break-word;
overflow: hidden;
text-overflow: ellipsis;
display: -webkit-box;
-webkit-line-clamp: 2;
-webkit-box-orient: vertical;
}
.kv-filtered-more {
padding: 2px 6px 0;
font-size: 11px;
font-style: italic;
color: #8089a0;
}
/* ── Toasts ───────────────────────────────────────────────────────────── */
.kv-toast-stack {
position: fixed;
bottom: 16px;
right: 16px;
display: flex;
flex-direction: column-reverse;
gap: 6px;
z-index: 1000;
max-width: 480px;
}
.kv-toast {
background: #2a3050;
color: #fff;
border-radius: 6px;
padding: 9px 12px;
box-shadow: 0 4px 14px rgba(0,0,0,0.15);
font-size: 12.5px;
}
.kv-toast-success { background: #2f9e44; }
.kv-toast-error { background: #c0392b; }
.kv-toast-row {
display: flex;
align-items: flex-start;
gap: 8px;
justify-content: space-between;
}
.kv-toast-text {
flex: 1;
word-break: break-word;
}
.kv-toast-close {
border: none;
background: transparent;
color: rgba(255,255,255,0.8);
cursor: pointer;
font-size: 16px;
line-height: 1;
}
.kv-toast-close:hover { color: #fff; }
.kv-toast-details {
margin: 6px 0 0;
padding-left: 18px;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
font-size: 11px;
color: rgba(255,255,255,0.85);
max-height: 160px;
overflow: auto;
}
.kv-app-gear {
margin-left: 0.25rem;
font-size: 1.05rem;
opacity: 0.6;
}
.kv-app-gear:hover {
opacity: 1;
}
.kv-modal-backdrop {
position: fixed;
inset: 0;
background: rgba(0, 0, 0, 0.45);
z-index: 1000;
display: flex;
align-items: flex-start;
justify-content: center;
padding: 4vh 1rem;
overflow-y: auto;
}
.kv-modal {
position: relative;
background: white;
color: inherit;
border-radius: 8px;
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
width: min(960px, 100%);
max-height: 92vh;
overflow-y: auto;
}
.kv-modal-close {
position: absolute;
top: 0.5rem;
right: 0.75rem;
background: transparent;
border: none;
font-size: 1.6rem;
line-height: 1;
cursor: pointer;
color: inherit;
opacity: 0.6;
z-index: 1;
}
.kv-modal-close:hover {
opacity: 1;
}
+313
View File
@@ -0,0 +1,313 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI, KVFindMatch, KVInfoResponse } from '../api/kvApi';
import { ancestorBranches, KVTree } from './KVTree';
import { FilteredKVTree } from './FilteredKVTree';
import { KVLeafEditor } from './KVLeafEditor';
import { KVBranchEditor } from './KVBranchEditor';
import { KVFileEditor } from './KVFileEditor';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
import { SearchFieldChips } from './SearchFieldChips';
import { ToastStack, makeToast } from './Toast';
import { ToastEntry } from './types';
import { SnapshotPanel } from '../components/SnapshotPanel';
import './KVApp.css';
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
const DEFAULT_SEARCH_FIELDS = ['originalTitle', 'filePath', 'fileName', 'plot/eng'];
function loadSearchFields(): string[] {
try {
const raw = localStorage.getItem(SEARCH_FIELDS_LS_KEY);
if (!raw) return DEFAULT_SEARCH_FIELDS;
const parsed = JSON.parse(raw);
if (Array.isArray(parsed) && parsed.every((x) => typeof x === 'string')) return parsed;
} catch {}
return DEFAULT_SEARCH_FIELDS;
}
// Top-level shell. Two panes inside the iframe:
// left — lazy tree of the keyspace
// right — leaf editor (when a leaf is selected) or branch editor (branch).
//
// A sticky header carries the search bar, the breadcrumb, and basic stats.
export default function KVApp() {
const [snapshotOpen, setSnapshotOpen] = useState(false);
// Selected node. branchPath is set when a branch is selected; selectedKey
// when a leaf is. Mutually exclusive.
const [selectedKey, setSelectedKey] = useState<string | null>(null);
const [branchPath, setBranchPath] = useState<string | null>(null);
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set(['']));
const [searchInput, setSearchInput] = useState('');
const [searchFields, setSearchFields] = useState<string[]>(loadSearchFields);
const [searchResults, setSearchResults] = useState<KVFindMatch[] | null>(null);
const [searchTruncated, setSearchTruncated] = useState(false);
const [searching, setSearching] = useState(false);
const [info, setInfo] = useState<KVInfoResponse | null>(null);
const [reloadToken, setReloadToken] = useState(0);
const [toasts, setToasts] = useState<ToastEntry[]>([]);
const searchTimerRef = useRef<number | null>(null);
// When the right pane is a file-detail view and the user clicks a leaf
// under the same file branch in the tree, we route the click as a
// scroll-anchor inside the file detail instead of a leaf-editor switch.
// The token bumps each click so identical anchors still trigger a scroll.
const [anchor, setAnchor] = useState<{ field: string; token: number } | null>(null);
const isFileBranch = (p: string | null) => !!p && /^file:[^/]+\/$/.test(p);
const handleLeafClick = (key: string) => {
if (isFileBranch(branchPath) && key.startsWith(branchPath!)) {
setAnchor({ field: key.slice(branchPath!.length), token: (anchor?.token ?? 0) + 1 });
return;
}
setSelectedKey(key);
setBranchPath(null);
setAnchor(null);
};
const handleBranchClick = (path: string) => {
setBranchPath(path);
setSelectedKey(null);
setAnchor(null);
};
const pushToast = (t: ToastEntry) => setToasts((cur) => [...cur, t]);
const dismissToast = (id: number) => setToasts((cur) => cur.filter((t) => t.id !== id));
useEffect(() => {
KVAPI.info().then(setInfo).catch(() => {});
}, [reloadToken]);
// Deep-link: /editor/file/<hashId> opens the tree pre-expanded at that hash.
useEffect(() => {
const m = window.location.pathname.match(/\/file\/([a-z0-9]+)$/i);
if (m) {
const hashId = m[1];
const branch = `file:${hashId}/`;
setExpandedPaths((cur) => new Set([...cur, '', 'file:', branch]));
setBranchPath(branch);
}
}, []);
// Persist field list across reloads.
useEffect(() => {
try { localStorage.setItem(SEARCH_FIELDS_LS_KEY, JSON.stringify(searchFields)); } catch {}
}, [searchFields]);
// Debounced live value-search across configured fields. Empty input clears.
useEffect(() => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
if (!searchInput.trim() || searchFields.length === 0) {
setSearchResults(null);
setSearchTruncated(false);
setSearching(false);
return;
}
setSearching(true);
searchTimerRef.current = window.setTimeout(async () => {
try {
const r = await KVAPI.find(searchInput.trim(), searchFields, 200);
setSearchResults(r.matches);
setSearchTruncated(r.truncated);
} catch (e: any) {
pushToast(makeToast('error', `Search failed: ${e.message}`));
} finally {
setSearching(false);
}
}, 250);
return () => {
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
};
}, [searchInput, searchFields]);
// Jump to a key from a search hit: select it, expand all ancestor branches.
const jumpToKey = (key: string) => {
const ancestors = ancestorBranches(key);
setExpandedPaths((cur) => new Set([...cur, '', ...ancestors]));
if (key.endsWith('/') || key.endsWith(':')) {
setBranchPath(key);
setSelectedKey(null);
} else {
setSelectedKey(key);
setBranchPath(null);
}
};
const breadcrumb = useMemo(() => {
const target = selectedKey ?? branchPath ?? '';
if (!target) return null;
const parts: { label: string; path: string }[] = [];
let i = 0;
while (i < target.length) {
let next = -1;
for (const d of [':', '/']) {
const idx = target.indexOf(d, i);
if (idx >= 0 && (next < 0 || idx < next)) next = idx;
}
if (next < 0) {
parts.push({ label: target.slice(i), path: target });
break;
}
parts.push({ label: target.slice(i, next + 1), path: target.slice(0, next + 1) });
i = next + 1;
}
return parts;
}, [selectedKey, branchPath]);
return (
<div className="kv-app">
<header className="kv-app-header">
<div className="kv-app-search">
<input
type="search"
className="kv-search-input"
placeholder="filter by value across configured fields…"
value={searchInput}
onChange={(e) => setSearchInput(e.target.value)}
autoFocus
/>
{searchInput && (
<button className="kv-btn-icon" onClick={() => setSearchInput('')} title="clear">×</button>
)}
<span className="kv-app-stats">
{info ? `${info.keyCount} files · ${info.memoryUsage}` : ''}
</span>
<button
className="kv-btn"
onClick={() => setReloadToken((n) => n + 1)}
title="refresh tree"
>Refresh</button>
<button
className="kv-btn-icon kv-app-gear"
onClick={() => setSnapshotOpen(true)}
title="Snapshot — export / import / wipe"
aria-label="Open snapshot settings"
>
⚙
</button>
</div>
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
{breadcrumb && breadcrumb.length > 0 && (
<div className="kv-app-breadcrumb">
<span className="kv-crumb-label">path:</span>
{breadcrumb.map((b, i) => (
<button
key={i}
className="kv-crumb"
onClick={() => jumpToKey(b.path)}
>{b.label}</button>
))}
</div>
)}
</header>
<div className="kv-app-body">
<aside className="kv-app-left">
{searchResults !== null ? (
<FilteredKVTree
matches={searchResults}
truncated={searchTruncated}
searching={searching}
selectedKey={selectedKey}
expandedPaths={expandedPaths}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
) : (
<KVTree
selectedKey={selectedKey}
expandedPaths={expandedPaths}
reloadToken={reloadToken}
onSelectLeaf={handleLeafClick}
onSelectBranch={handleBranchClick}
onToggleBranch={(p, willOpen) => {
setExpandedPaths((cur) => {
const next = new Set(cur);
if (willOpen) next.add(p); else next.delete(p);
return next;
});
}}
/>
)}
</aside>
<main className="kv-app-right">
{selectedKey ? (
<KVLeafEditor
selectedKey={selectedKey}
reloadToken={reloadToken}
onSaved={() => {
pushToast(makeToast('success', `Saved ${selectedKey}`));
setReloadToken((n) => n + 1);
}}
onDeleted={() => {
pushToast(makeToast('success', `Deleted ${selectedKey}`));
setSelectedKey(null);
setReloadToken((n) => n + 1);
}}
onError={(msg) => pushToast(makeToast('error', msg))}
/>
) : branchPath && FILE_BRANCH_RE.test(branchPath) ? (
<KVFileEditor
key={branchPath}
branchPath={branchPath}
reloadToken={reloadToken}
scrollToField={anchor}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
onSelectLeaf={(k) => { setSelectedKey(k); setBranchPath(null); setAnchor(null); }}
/>
) : branchPath ? (
<KVBranchEditor
branchPath={branchPath}
reloadToken={reloadToken}
onChanged={(info, errors) => {
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
setReloadToken((n) => n + 1);
}}
/>
) : (
<div className="kv-pane-status kv-pane-empty">
Select a key from the tree.
<div className="kv-pane-hint">
Click a branch to add / rename / delete a subtree, or a leaf to edit its value.
</div>
</div>
)}
</main>
</div>
{snapshotOpen && (
<div
className="kv-modal-backdrop"
onClick={() => setSnapshotOpen(false)}
>
<div
className="kv-modal"
onClick={(e) => e.stopPropagation()}
role="dialog"
aria-modal="true"
>
<button
className="kv-modal-close"
onClick={() => setSnapshotOpen(false)}
aria-label="Close"
title="Close"
>×</button>
<SnapshotPanel />
</div>
</div>
)}
<ToastStack toasts={toasts} onDismiss={dismissToast} />
</div>
);
}
+194
View File
@@ -0,0 +1,194 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
interface Props {
// Branch path, ends with the delimiter (":" or "/").
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
}
// Branch operations are non-atomic — each per-key write/delete goes one round
// trip to Redis and may fail individually. Failures collect into a list and
// surface via the parent's toast stack. Per the design, we accept partial
// failure and surface what failed.
export function KVBranchEditor({ branchPath, reloadToken, onChanged }: Props) {
const [count, setCount] = useState<number | null>(null);
const [busy, setBusy] = useState(false);
const [newChild, setNewChild] = useState('');
const [newValue, setNewValue] = useState('');
const [renameTo, setRenameTo] = useState('');
// Use the search endpoint with the branch path as the substring to count
// descendants. Truncated when there are more than the limit; we just show
// an approximate count.
useEffect(() => {
let cancelled = false;
KVAPI.search(branchPath, 5000)
.then((r) => {
if (cancelled) return;
// Filter to true descendants (those starting with branchPath).
const descendants = r.keys.filter((k) => k.startsWith(branchPath));
setCount(descendants.length);
})
.catch(() => { if (!cancelled) setCount(null); });
return () => { cancelled = true; };
}, [branchPath, reloadToken]);
// Walk all descendants and run an op against each, collecting failures.
const forEachDescendant = async (op: (key: string) => Promise<void>): Promise<string[]> => {
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const errors: string[] = [];
for (const k of keys) {
try { await op(k); } catch (e: any) { errors.push(`${k}: ${e.message}`); }
}
return errors;
};
const addChild = async () => {
if (!newChild) return;
const childKey = branchPath + newChild;
setBusy(true);
try {
await KVAPI.put(childKey, newValue);
setNewChild('');
setNewValue('');
onChanged(`Created ${childKey}`);
} catch (e: any) {
onChanged(`Failed to create ${childKey}`, [e.message]);
} finally {
setBusy(false);
}
};
const deleteSubtree = async () => {
if (!confirm(`Delete every key under ${branchPath} (${count ?? '?'} entries)?`)) return;
setBusy(true);
try {
const errors = await forEachDescendant((k) => KVAPI.del(k).then(() => undefined));
if (errors.length === 0) {
onChanged(`Deleted subtree ${branchPath}`);
} else {
onChanged(`Deleted subtree ${branchPath} with ${errors.length} failure(s)`, errors);
}
} finally {
setBusy(false);
}
};
const renameBranch = async () => {
if (!renameTo) return;
const target = renameTo.endsWith('/') || renameTo.endsWith(':') ? renameTo : renameTo + '/';
if (target === branchPath) return;
if (!confirm(`Rename ${branchPath} → ${target}? This copies ${count ?? '?'} keys then deletes the originals.`)) return;
setBusy(true);
try {
// Pass 1: copy
const r = await KVAPI.search(branchPath, 5000);
const keys = r.keys.filter((k) => k.startsWith(branchPath));
const copyErrors: string[] = [];
for (const oldKey of keys) {
try {
const v = await KVAPI.get(oldKey);
if (!v.exists || v.type !== 'string') {
copyErrors.push(`${oldKey}: skipped (type ${v.type || 'missing'})`);
continue;
}
const newKey = target + oldKey.slice(branchPath.length);
await KVAPI.put(newKey, v.value);
} catch (e: any) {
copyErrors.push(`${oldKey}: ${e.message}`);
}
}
// Pass 2: delete originals (only the ones we successfully copied)
const failedOldKeys = new Set(copyErrors.map((e) => e.split(':')[0]));
const deleteErrors: string[] = [];
for (const oldKey of keys) {
if (failedOldKeys.has(oldKey)) continue;
try { await KVAPI.del(oldKey); } catch (e: any) { deleteErrors.push(`${oldKey}: ${e.message}`); }
}
const allErrors = [...copyErrors, ...deleteErrors];
if (allErrors.length === 0) {
onChanged(`Renamed ${branchPath} → ${target}`);
} else {
onChanged(`Renamed ${branchPath} → ${target} with ${allErrors.length} failure(s)`, allErrors);
}
setRenameTo('');
} finally {
setBusy(false);
}
};
return (
<div className="kv-branch">
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{branchPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(branchPath)}
title="copy path"
>⧉</button>
</div>
<div className="kv-pane-status">
{count == null ? 'counting…' : `${count} descendant key${count === 1 ? '' : 's'}`}
</div>
<section className="kv-branch-section">
<h4>Add child</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">{branchPath}</span>
<input
className="kv-leaf-input"
placeholder="child path (may contain /)"
value={newChild}
onChange={(e) => setNewChild(e.target.value)}
disabled={busy}
/>
</div>
<input
className="kv-leaf-input"
placeholder="value"
value={newValue}
onChange={(e) => setNewValue(e.target.value)}
disabled={busy}
/>
<button className="kv-btn kv-btn-primary" disabled={!newChild || busy} onClick={addChild}>
Add
</button>
</section>
<section className="kv-branch-section">
<h4>Rename branch</h4>
<div className="kv-branch-row">
<span className="kv-branch-prefix">→</span>
<input
className="kv-leaf-input"
placeholder="new prefix (e.g. file:newhash/)"
value={renameTo}
onChange={(e) => setRenameTo(e.target.value)}
disabled={busy}
/>
</div>
<p className="kv-branch-note">
Non-atomic: copies each descendant to the new prefix then deletes the original.
Failures are surfaced individually.
</p>
<button className="kv-btn" disabled={!renameTo || busy} onClick={renameBranch}>
Rename
</button>
</section>
<section className="kv-branch-section">
<h4>Delete subtree</h4>
<p className="kv-branch-note">
Deletes every descendant key. Cannot be undone.
</p>
<button className="kv-btn kv-btn-danger" disabled={busy || count === 0} onClick={deleteSubtree}>
Delete {count ?? ''} key{count === 1 ? '' : 's'}
</button>
</section>
</div>
);
}
+675
View File
@@ -0,0 +1,675 @@
import { useEffect, useMemo, useRef, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { SchemaFieldInfo, useSchema } from '../api/schemaApi';
import { LanguageCombobox, formatLanguageOption } from '../components/LanguageCombobox';
import { LeafValueEditor } from './leafRenderers';
import { schemaToFieldType } from './schemaLookup';
import { detectType } from './typeHeuristics';
import { FieldType } from './types';
interface Props {
/** Branch path of a file, e.g. "file:<hash>/". */
branchPath: string;
reloadToken: number;
onChanged: (info: string, errors?: string[]) => void;
/** Navigate to a single leaf for the manual deep editor. */
onSelectLeaf: (key: string) => void;
/**
* When set, scroll to and pulse the row matching this logical field path
* (e.g. "releasedate", "plot", "stream/0"). The `token` lets the same field
* fire a fresh scroll across consecutive clicks.
*/
scrollToField?: { field: string; token: number } | null;
}
const LANG_CODE_RE = /^[a-z]{2,3}$/;
interface LeafRow {
kind: 'leaf';
fieldKey: string;
value: string;
fieldType: FieldType;
info?: SchemaFieldInfo;
}
interface LanguageRow {
kind: 'language-group';
fieldKey: string;
/** Current language → text map. */
languageMap: Record<string, string>;
info?: SchemaFieldInfo;
}
type Row = LeafRow | LanguageRow;
interface RowGroup {
prefix: string;
isCluster: boolean;
rows: Row[];
}
export function KVFileEditor({
branchPath,
reloadToken,
onChanged,
onSelectLeaf,
scrollToField,
}: Props) {
const [metadata, setMetadata] = useState<Record<string, string> | null>(null);
const [edits, setEdits] = useState<Map<string, string | null>>(new Map());
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
const [pulse, setPulse] = useState<string | null>(null);
const [newFieldKey, setNewFieldKey] = useState('');
const [newFieldValue, setNewFieldValue] = useState('');
const [addingField, setAddingField] = useState(false);
// Track whether we've seeded collapsedGroups for the current branch; resets
// when the branch changes so each file gets its defaults reapplied.
const defaultsAppliedRef = useRef<string | null>(null);
const { schema } = useSchema();
// When the parent passes a target field, expand its prefix group (if any)
// and scroll the row into view. We also briefly pulse the row to draw the
// eye — the pulse is reset after the animation duration.
useEffect(() => {
if (!scrollToField) return;
const { field } = scrollToField;
const prefix = field.split('/', 1)[0];
setCollapsedGroups((cur) => {
if (!cur.has(prefix)) return cur;
const next = new Set(cur);
next.delete(prefix);
return next;
});
requestAnimationFrame(() => {
const el = document.querySelector(
`[data-row-key="${cssEscape(field)}"]`,
) as HTMLElement | null;
if (el) {
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
setPulse(field);
window.setTimeout(() => setPulse((p) => (p === field ? null : p)), 1400);
}
});
}, [scrollToField?.field, scrollToField?.token]);
const hashId = branchPath.slice('file:'.length, -1);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.treeAll(branchPath, '/', 5000)
.then(async (resp) => {
// resp.leaves = full keys under branchPath. Recursive descent: also
// walk branch children (they're paths to sub-objects with their own
// leaves, e.g. file:<h>/plot/ → plot/eng).
const all: string[] = [...resp.leaves];
const queue = [...resp.branches];
while (queue.length > 0) {
const next = queue.shift()!;
try {
const sub = await KVAPI.treeAll(next, '/', 5000);
all.push(...sub.leaves);
queue.push(...sub.branches);
} catch {
// Ignore failures on a sub-branch; show what we can.
}
}
if (cancelled) return;
// MGET-equivalent: parallel fetches with a small cap to avoid burst.
const values = await Promise.all(
all.map((k) => KVAPI.get(k).then((r) => [k, r.exists ? r.value : null] as const).catch(() => [k, null] as const)),
);
if (cancelled) return;
const flat: Record<string, string> = {};
for (const [k, v] of values) {
if (v == null) continue;
// Strip the branch prefix so keys are the logical field path
// ("releasedate", "plot/eng", "stream/0").
const fieldKey = k.slice(branchPath.length);
if (fieldKey) flat[fieldKey] = v;
}
setMetadata(flat);
setEdits(new Map());
})
.catch(() => {
if (!cancelled) setMetadata({});
})
.finally(() => {
if (!cancelled) setLoading(false);
});
return () => {
cancelled = true;
};
}, [branchPath, reloadToken]);
const groups = useMemo(
() => groupRows(buildRows(metadata ?? {}, schema)),
[metadata, schema],
);
// Default state: every cluster group (e.g. stream/*, cid_*) starts collapsed.
// Reapplied when the user switches to a different file.
useEffect(() => {
if (groups.length === 0) return;
if (defaultsAppliedRef.current === branchPath) return;
defaultsAppliedRef.current = branchPath;
setCollapsedGroups(new Set(groups.filter((g) => g.isCluster).map((g) => g.prefix)));
}, [groups, branchPath]);
if (loading) {
return <div className="kv-pane-status">loading file…</div>;
}
if (!metadata) {
return <div className="kv-pane-status">no data for {branchPath}</div>;
}
const fileName = metadata['fileName'];
const filePath = metadata['filePath'];
const onLeafChange = (fieldKey: string, value: string) => {
setEdits((prev) => {
const next = new Map(prev);
const original = metadata[fieldKey] ?? null;
if (value === original) next.delete(fieldKey);
else next.set(fieldKey, value);
return next;
});
};
/**
* Toggle a field's pending-delete state. Re-clicking restores; if the field
* is brand-new (no original value) we just drop it from the edits map.
*/
const onLeafDeleteToggle = (fieldKey: string) => {
setEdits((prev) => {
const next = new Map(prev);
if (next.get(fieldKey) === null) {
// Currently pending delete → undo.
next.delete(fieldKey);
} else {
next.set(fieldKey, null);
}
return next;
});
};
const handleAddField = async () => {
const key = newFieldKey.trim();
if (!key) return;
const fullKey = branchPath + key;
setAddingField(true);
try {
await KVAPI.put(fullKey, newFieldValue);
setNewFieldKey('');
setNewFieldValue('');
onChanged(`Added ${fullKey}`);
// Refetch this file so the new field appears in the row list.
const r = await KVAPI.get(fullKey);
if (r.exists) {
setMetadata((cur) => ({ ...(cur ?? {}), [key]: r.value }));
}
} catch (e: any) {
onChanged(`Failed to add ${fullKey}`, [e.message]);
} finally {
setAddingField(false);
}
};
const onLanguageGroupChange = (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => {
setEdits((prev) => {
const next = new Map(prev);
const allLangs = new Set([...Object.keys(originalMap), ...Object.keys(nextMap)]);
for (const lang of allLangs) {
const fieldKey = `${groupKey}/${lang}`;
const orig = originalMap[lang] ?? null;
const cur = nextMap[lang] ?? null;
if (cur === orig) next.delete(fieldKey);
else if (cur == null) next.set(fieldKey, null);
else next.set(fieldKey, cur);
}
return next;
});
};
const handleSave = async () => {
if (edits.size === 0) return;
setSaving(true);
const errors: string[] = [];
for (const [fieldKey, newValue] of edits) {
const fullKey = branchPath + fieldKey;
try {
if (newValue === null) await KVAPI.del(fullKey);
else await KVAPI.put(fullKey, newValue);
} catch (e: any) {
errors.push(`${fieldKey}: ${e.message}`);
}
}
setSaving(false);
if (errors.length === 0) {
onChanged(`Saved ${edits.size} field${edits.size === 1 ? '' : 's'} on ${fileName ?? hashId}`);
} else {
onChanged(
`Saved with ${errors.length} failure(s) on ${fileName ?? hashId}`,
errors,
);
}
};
const handleDiscard = () => setEdits(new Map());
const toggleGroup = (prefix: string) => {
setCollapsedGroups((cur) => {
const next = new Set(cur);
if (next.has(prefix)) next.delete(prefix);
else next.add(prefix);
return next;
});
};
return (
<div className="kv-file">
<div className="kv-file-header">
<div className="kv-file-name">
{fileName ?? <span className="kv-file-name-fallback">(no fileName)</span>}
</div>
<div className="kv-file-meta">
<code className="kv-file-hash" title={hashId}>
{hashId.slice(0, 8)}…{hashId.slice(-6)}
</code>
{filePath && <span className="kv-file-path" title={filePath}>{filePath}</span>}
</div>
<div className="kv-file-actions">
<button
className="kv-btn kv-btn-primary"
disabled={edits.size === 0 || saving}
onClick={handleSave}
>
{saving ? 'saving…' : `Save (${edits.size})`}
</button>
<button
className="kv-btn"
disabled={edits.size === 0 || saving}
onClick={handleDiscard}
>
Discard
</button>
</div>
</div>
<div className="kv-file-rows">
{groups.length === 0 && (
<div className="kv-pane-status">No data yet for this file.</div>
)}
{groups.map((g) => (
<FieldGroupView
key={g.prefix}
group={g}
edits={edits}
collapsed={collapsedGroups.has(g.prefix)}
pulse={pulse}
onToggle={() => toggleGroup(g.prefix)}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageGroupChange}
onJumpToLeaf={(fieldKey) => onSelectLeaf(branchPath + fieldKey)}
/>
))}
</div>
<div className="kv-file-add">
<span className="kv-file-add-prefix">{branchPath}</span>
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
placeholder="field path (may contain /)"
value={newFieldKey}
onChange={(e) => setNewFieldKey(e.target.value)}
disabled={addingField}
/>
<input
type="text"
className="kv-leaf-input"
placeholder="value"
value={newFieldValue}
onChange={(e) => setNewFieldValue(e.target.value)}
onKeyDown={(e) => {
if (e.key === 'Enter' && newFieldKey.trim()) handleAddField();
}}
disabled={addingField}
/>
<button
className="kv-btn kv-btn-primary"
disabled={!newFieldKey.trim() || addingField}
onClick={handleAddField}
>
{addingField ? 'adding…' : '+ Add field'}
</button>
</div>
</div>
);
}
function FieldGroupView({
group,
edits,
collapsed,
pulse,
onToggle,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
group: RowGroup;
edits: Map<string, string | null>;
collapsed: boolean;
pulse: string | null;
onToggle: () => void;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
if (!group.isCluster) {
return (
<>
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</>
);
}
return (
<div className={`kv-file-group ${collapsed ? 'collapsed' : ''}`}>
<button
type="button"
className="kv-file-group-header"
onClick={onToggle}
title={collapsed ? 'expand' : 'collapse'}
>
<span className="kv-file-group-chevron">{collapsed ? '▸' : '▾'}</span>
<span className="kv-file-group-name">{group.prefix}</span>
<span className="kv-file-group-count">{group.rows.length}</span>
</button>
{!collapsed && (
<div className="kv-file-group-body">
{group.rows.map((row) => (
<RowView
key={row.fieldKey}
row={row}
edits={edits}
pulse={pulse}
onLeafChange={onLeafChange}
onLeafDeleteToggle={onLeafDeleteToggle}
onLanguageChange={onLanguageChange}
onJumpToLeaf={onJumpToLeaf}
/>
))}
</div>
)}
</div>
);
}
function RowView({
row,
edits,
pulse,
onLeafChange,
onLeafDeleteToggle,
onLanguageChange,
onJumpToLeaf,
}: {
row: Row;
edits: Map<string, string | null>;
pulse: string | null;
onLeafChange: (fieldKey: string, value: string) => void;
onLeafDeleteToggle: (fieldKey: string) => void;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
onJumpToLeaf: (fieldKey: string) => void;
}) {
const pulsed = pulse === row.fieldKey;
if (row.kind === 'leaf') {
const pendingDelete = edits.get(row.fieldKey) === null;
const draft = edits.has(row.fieldKey) ? (edits.get(row.fieldKey) ?? '') : row.value;
const dirty = edits.has(row.fieldKey);
return (
<div
className={`kv-file-row ${dirty ? 'dirty' : ''} ${pendingDelete ? 'pending-delete' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<button
type="button"
className="kv-file-row-label"
onClick={() => onJumpToLeaf(row.fieldKey)}
title="open in single-leaf editor"
>
{row.fieldKey}
</button>
<div className="kv-file-row-value">
{pendingDelete ? (
<div className="kv-file-row-deleted-note">marked for deletion</div>
) : (
<LeafValueEditor
type={row.fieldType}
value={draft}
onChange={(v) => onLeafChange(row.fieldKey, v)}
/>
)}
</div>
<button
type="button"
className="kv-file-row-delete"
onClick={() => onLeafDeleteToggle(row.fieldKey)}
title={pendingDelete ? 'undo deletion' : 'delete this field'}
aria-label={pendingDelete ? 'undo deletion' : 'delete this field'}
>
{pendingDelete ? '↺' : '×'}
</button>
</div>
);
}
return (
<LanguageGroupRow
row={row}
edits={edits}
pulsed={pulsed}
onLanguageChange={onLanguageChange}
/>
);
}
function LanguageGroupRow({
row,
edits,
pulsed,
onLanguageChange,
}: {
row: LanguageRow;
edits: Map<string, string | null>;
pulsed: boolean;
onLanguageChange: (
groupKey: string,
nextMap: Record<string, string>,
originalMap: Record<string, string>,
) => void;
}) {
// Apply pending edits on top of the original language map.
const effectiveMap: Record<string, string> = { ...row.languageMap };
for (const [k, v] of edits) {
if (!k.startsWith(`${row.fieldKey}/`)) continue;
const lang = k.slice(row.fieldKey.length + 1);
if (v === null) delete effectiveMap[lang];
else effectiveMap[lang] = v;
}
const entries = Object.entries(effectiveMap);
const dirty = [...edits.keys()].some((k) => k.startsWith(`${row.fieldKey}/`));
const updateLang = (lang: string, text: string) => {
onLanguageChange(row.fieldKey, { ...effectiveMap, [lang]: text }, row.languageMap);
};
const renameLang = (oldLang: string, newLang: string) => {
if (oldLang === newLang) return;
const next = { ...effectiveMap };
const text = next[oldLang];
delete next[oldLang];
next[newLang] = text;
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const removeLang = (lang: string) => {
const next = { ...effectiveMap };
delete next[lang];
onLanguageChange(row.fieldKey, next, row.languageMap);
};
const addLang = () => {
const used = new Set(entries.map(([k]) => k));
const defaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa', 'fre'];
const defaultLang = defaults.find((l) => !used.has(l)) || 'eng';
onLanguageChange(row.fieldKey, { ...effectiveMap, [defaultLang]: '' }, row.languageMap);
};
return (
<div
className={`kv-file-row kv-file-row-lang ${dirty ? 'dirty' : ''} ${pulsed ? 'pulsed' : ''}`}
data-row-key={row.fieldKey}
>
<div className="kv-file-row-label-static">{row.fieldKey}</div>
<div className="kv-file-row-value">
<div className="kv-file-lang-entries">
{entries.map(([lang, text]) => (
<div key={lang} className="kv-file-lang-entry">
<LanguageCombobox
value={lang}
onChange={(newLang) => renameLang(lang, newLang)}
/>
<input
type="text"
className="kv-leaf-input"
value={text}
title={formatLanguageOption(lang)}
onChange={(e) => updateLang(lang, e.target.value)}
/>
<button
type="button"
className="kv-btn-icon kv-file-lang-remove"
onClick={() => removeLang(lang)}
title="remove this language"
>
×
</button>
</div>
))}
<button
type="button"
className="kv-btn kv-file-lang-add"
onClick={addLang}
>
+ Add language
</button>
</div>
</div>
</div>
);
}
// ──────────────────────────────────────────────────────────────────────────
// Row model — kept local because it's only used here and is small.
// ──────────────────────────────────────────────────────────────────────────
function buildRows(
flat: Record<string, string>,
schema: ReturnType<typeof useSchema>['schema'],
): Row[] {
const languageGroupKeys = new Set<string>();
if (schema) {
for (const [k, info] of Object.entries(schema.fields)) {
if (info.key_hint === 'language-code') languageGroupKeys.add(k);
}
}
const langMaps: Record<string, Record<string, string>> = {};
const rows: Row[] = [];
for (const [key, value] of Object.entries(flat)) {
const slash = key.lastIndexOf('/');
if (slash > 0) {
const prefix = key.slice(0, slash);
const tail = key.slice(slash + 1);
if (languageGroupKeys.has(prefix) && LANG_CODE_RE.test(tail)) {
if (!langMaps[prefix]) langMaps[prefix] = {};
langMaps[prefix][tail] = value;
continue;
}
}
const info = schema?.fields[key];
rows.push({
kind: 'leaf',
fieldKey: key,
value,
// Schema decides when it can; otherwise heuristic on (path, value).
fieldType: schemaToFieldType(info) ?? detectType(key, value),
info,
});
}
for (const [groupKey, langMap] of Object.entries(langMaps)) {
rows.push({
kind: 'language-group',
fieldKey: groupKey,
languageMap: langMap,
info: schema?.fields[groupKey],
});
}
rows.sort((a, b) => a.fieldKey.localeCompare(b.fieldKey));
return rows;
}
/** Escape characters that would break a CSS attribute selector value. */
function cssEscape(s: string): string {
return s.replace(/["\\]/g, (c) => `\\${c}`);
}
function groupRows(rows: Row[]): RowGroup[] {
const out: RowGroup[] = [];
for (const row of rows) {
const prefix = row.fieldKey.split('/', 1)[0];
const last = out[out.length - 1];
if (last && last.prefix === prefix) {
last.rows.push(row);
} else {
out.push({ prefix, isCluster: false, rows: [row] });
}
}
for (const g of out) g.isCluster = g.rows.length > 1;
return out;
}
+160
View File
@@ -0,0 +1,160 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
import { useSchema } from '../api/schemaApi';
import { ALL_TYPES, describeType, detectType } from './typeHeuristics';
import { lookupSchemaForKey } from './schemaLookup';
import { LeafValueEditor } from './leafRenderers';
import { FieldType } from './types';
interface Props {
// Full Redis key path of the selected leaf.
selectedKey: string;
// Bumped after a delete or external change to force refetch.
reloadToken: number;
onSaved: () => void;
onDeleted: () => void;
onError: (msg: string) => void;
}
export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onError }: Props) {
const [original, setOriginal] = useState<string>('');
const [draft, setDraft] = useState<string>('');
const [exists, setExists] = useState<boolean>(true);
const [redisType, setRedisType] = useState<string>('');
const [loading, setLoading] = useState(true);
const [saving, setSaving] = useState(false);
// Manual override for the inferred type. "auto" defers to schema → heuristic.
const [typeOverride, setTypeOverride] = useState<FieldType>('auto');
const { schema } = useSchema();
const schemaLookup = lookupSchemaForKey(schema, selectedKey);
// Schema is authoritative when it has an opinion; heuristics fill the gap.
const detected = schemaLookup.fieldType ?? detectType(selectedKey, original);
const effectiveType = typeOverride === 'auto' ? detected : typeOverride;
// Reset override when the selected key changes — the previous override
// doesn't apply to a different leaf.
useEffect(() => { setTypeOverride('auto'); }, [selectedKey]);
useEffect(() => {
let cancelled = false;
setLoading(true);
KVAPI.get(selectedKey)
.then((r) => {
if (cancelled) return;
setOriginal(r.value);
setDraft(r.value);
setExists(r.exists);
setRedisType(r.type);
})
.catch((e) => onError(e.message))
.finally(() => { if (!cancelled) setLoading(false); });
return () => { cancelled = true; };
}, [selectedKey, reloadToken, onError]);
const dirty = draft !== original;
const editable = exists ? redisType === 'string' || redisType === '' : true;
const save = async () => {
setSaving(true);
try {
await KVAPI.put(selectedKey, draft);
setOriginal(draft);
setExists(true);
setRedisType('string');
onSaved();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
const del = async () => {
if (!confirm(`Delete ${selectedKey}?`)) return;
setSaving(true);
try {
await KVAPI.del(selectedKey);
onDeleted();
} catch (e: any) {
onError(e.message);
} finally {
setSaving(false);
}
};
if (loading) {
return <div className="kv-pane-status">loading…</div>;
}
if (!editable) {
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-pane-status">
Key has Redis type <code>{redisType}</code> — only string-typed values are editable here.
</div>
</div>
);
}
return (
<div className="kv-leaf">
<KeyHeader keyPath={selectedKey} />
<div className="kv-leaf-meta">
<label className="kv-leaf-label">Type</label>
<select
className="kv-leaf-type"
value={typeOverride}
onChange={(e) => setTypeOverride(e.target.value as FieldType)}
>
{ALL_TYPES.map((t) => (
<option key={t} value={t}>
{t === 'auto' ? `auto (${describeType(detected)})` : describeType(t)}
</option>
))}
</select>
{!exists && <span className="kv-leaf-hint">new key — save to create</span>}
</div>
<LeafValueEditor type={effectiveType} value={draft} onChange={setDraft} />
<div className="kv-leaf-actions">
<button
className="kv-btn kv-btn-primary"
onClick={save}
disabled={!dirty || saving}
>{saving ? 'saving…' : exists ? 'Save' : 'Create'}</button>
<button
className="kv-btn"
onClick={() => setDraft(original)}
disabled={!dirty || saving}
>Reset</button>
{exists && (
<button
className="kv-btn kv-btn-danger"
onClick={del}
disabled={saving}
>Delete</button>
)}
</div>
</div>
);
}
function KeyHeader({ keyPath }: { keyPath: string }) {
return (
<div className="kv-leaf-key">
<code className="kv-leaf-key-text">{keyPath}</code>
<button
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(keyPath)}
title="copy key"
>⧉</button>
</div>
);
}
// Leaf value rendering moved to leafRenderers.tsx (LeafValueEditor)
// so the file-detail view can reuse it per row.
+339
View File
@@ -0,0 +1,339 @@
import { useEffect, useState } from 'react';
import { KVAPI } from '../api/kvApi';
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
const fileNameCache = new Map<string, string | null>();
const fileNamePromises = new Map<string, Promise<void>>();
const fileNameSubscribers = new Set<() => void>();
function notifyFileNameSubscribers() {
for (const cb of fileNameSubscribers) cb();
}
function ensureFileName(branchPath: string): void {
if (fileNameCache.has(branchPath) || fileNamePromises.has(branchPath)) return;
const p = KVAPI.get(branchPath + 'fileName')
.then((r) => {
fileNameCache.set(branchPath, r.exists && r.value ? r.value : null);
})
.catch(() => {
fileNameCache.set(branchPath, null);
})
.finally(() => {
fileNamePromises.delete(branchPath);
notifyFileNameSubscribers();
});
fileNamePromises.set(branchPath, p);
}
function useFileName(branchPath: string | null): string | null {
const [, force] = useState(0);
useEffect(() => {
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return;
ensureFileName(branchPath);
const cb = () => force((n) => n + 1);
fileNameSubscribers.add(cb);
return () => {
fileNameSubscribers.delete(cb);
};
}, [branchPath]);
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return null;
return fileNameCache.get(branchPath) ?? null;
}
function isFileBranch(path: string): boolean {
return FILE_BRANCH_RE.test(path);
}
function shortHashFromFileBranch(path: string): string {
// path = "file:<hash>/" → last 6 chars of hash
const hash = path.slice('file:'.length, -1);
if (hash.length <= 12) return hash;
return `…${hash.slice(-6)}`;
}
// A node in the lazy tree. branch-typed nodes have an associated path that
// ends with the delimiter and lazy-load their children on first expand.
type NodeKind = 'branch' | 'leaf';
interface TreeNode {
kind: NodeKind;
// Display label: the segment of the path under the parent.
label: string;
// Full key path. For branches, ends with the delimiter.
path: string;
}
interface BranchState {
loaded: boolean;
loading: boolean;
error?: string;
children: TreeNode[];
}
// Sub-segment of a path used as the visible label for a branch / leaf.
// Splits on either ":" or "/" so file:abc/title/eng becomes file: > abc > title > eng.
function labelOf(path: string, parentPath: string): string {
const rest = path.slice(parentPath.length);
// The "rest" includes a trailing delimiter for branches; strip it for display.
return rest.replace(/[/:]+$/, '');
}
// Hide internal Redis structures from the tree. The editor is for managing
// file metadata; everything else is meta-core's own bookkeeping and noise
// for the user. Hidden:
// - `meta:` — event streams (file:events, meta:events, etc.)
// - `cid:` — reverse index (cid:<algo>:<value> → uuid), derived
// from each file's cid_* fields; the editor surfaces
// these per-file under their parent root
// - `meta-core:` — internal service state (mounts, watcher config,
// leader info), edited via dedicated UI pages, not
// the KV tree
// - bare top-level leaves under a colon-namespace like file:__index__
// and file:events (Redis sets / streams that aren't user data).
function isHiddenNode(node: TreeNode): boolean {
if (node.kind === 'branch') {
if (node.path === 'meta:' || node.path === 'cid:' || node.path === 'meta-core:') {
return true;
}
}
if (node.kind === 'leaf' && /^[^/]+:[^/]+$/.test(node.path)) return true;
return false;
}
interface Props {
// Selected leaf path (full key) — highlighted.
selectedKey: string | null;
// When set, the tree forces these branches open (used for auto-expand on
// search hit or deep-link).
expandedPaths: Set<string>;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
// Bumping this triggers a full reload of root and any expanded branches.
reloadToken: number;
}
export function KVTree({
selectedKey,
expandedPaths,
onSelectLeaf,
onSelectBranch,
onToggleBranch,
reloadToken,
}: Props) {
// Map of branch path → load state. The empty key "" represents the root.
const [branches, setBranches] = useState<Record<string, BranchState>>({});
// Load (or reload) a branch's children from the API. The root is special-
// cased: we list at prefix="" with delimiter=":" to surface the namespaces.
const loadBranch = async (path: string) => {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: true, children: b[path]?.children ?? [] },
}));
try {
const isRoot = path === '';
const delim = isRoot ? ':' : '/';
const resp = await KVAPI.treeAll(path, delim, 5000);
const branchNodes: TreeNode[] = resp.branches.map((p) => ({
kind: 'branch',
label: labelOf(p, path) + delim,
path: p,
}));
const leafNodes: TreeNode[] = resp.leaves.map((p) => ({
kind: 'leaf',
label: labelOf(p, path),
path: p,
}));
const children = [...branchNodes, ...leafNodes];
setBranches((b) => ({
...b,
[path]: { loaded: true, loading: false, children },
}));
} catch (err: any) {
setBranches((b) => ({
...b,
[path]: { loaded: false, loading: false, children: [], error: err.message },
}));
}
};
// On reloadToken change, refetch root and every currently-expanded branch.
useEffect(() => {
loadBranch('');
for (const p of expandedPaths) {
if (p !== '') loadBranch(p);
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [reloadToken]);
// Whenever expandedPaths gains a path we haven't loaded yet, lazy-load it.
useEffect(() => {
for (const p of expandedPaths) {
if (p === '') continue;
const state = branches[p];
if (!state || (!state.loaded && !state.loading)) {
loadBranch(p);
}
}
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [expandedPaths]);
return (
<div className="kv-tree">
<Branch
path=""
state={branches['']}
depth={0}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={(p, open) => {
onToggleBranch(p, open);
if (open && (!branches[p] || !branches[p].loaded)) loadBranch(p);
}}
/>
</div>
);
}
interface BranchProps {
path: string;
state: BranchState | undefined;
depth: number;
expandedPaths: Set<string>;
branches: Record<string, BranchState>;
selectedKey: string | null;
onSelectLeaf: (key: string) => void;
onSelectBranch: (path: string) => void;
onToggleBranch: (path: string, willOpen: boolean) => void;
}
function Branch({
state, depth, expandedPaths, branches,
selectedKey, onSelectLeaf, onSelectBranch, onToggleBranch,
}: BranchProps) {
if (!state) {
return null;
}
if (state.loading && state.children.length === 0) {
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
}
if (state.error) {
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>error: {state.error}</div>;
}
const visibleChildren = state.children.filter((c) => !isHiddenNode(c));
if (visibleChildren.length === 0) {
return <div className="kv-tree-empty" style={{ paddingLeft: depth * 14 + 18 }}>(empty)</div>;
}
return (
<ul className="kv-tree-list">
{visibleChildren.map((node) => {
if (node.kind === 'branch') {
const open = expandedPaths.has(node.path);
const childState = branches[node.path];
return (
<li key={node.path}>
<BranchRow
node={node}
open={open}
depth={depth}
onToggle={() => {
onToggleBranch(node.path, !open);
onSelectBranch(node.path);
}}
/>
{open && (
<Branch
path={node.path}
state={childState}
depth={depth + 1}
expandedPaths={expandedPaths}
branches={branches}
selectedKey={selectedKey}
onSelectLeaf={onSelectLeaf}
onSelectBranch={onSelectBranch}
onToggleBranch={onToggleBranch}
/>
)}
</li>
);
}
const isSelected = node.path === selectedKey;
return (
<li key={node.path}>
<div
className={`kv-tree-row kv-tree-leaf${isSelected ? ' selected' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={() => onSelectLeaf(node.path)}
>
<span className="kv-tree-leaf-dot">•</span>
<span className="kv-tree-label">{node.label}</span>
</div>
</li>
);
})}
</ul>
);
}
interface BranchRowProps {
node: TreeNode;
open: boolean;
depth: number;
onToggle: () => void;
}
function BranchRow({ node, open, depth, onToggle }: BranchRowProps) {
// Surface the actual filename next to file:<hash>/ branches so the tree
// reads like a file list at that level instead of a wall of hashes.
const fileName = useFileName(isFileBranch(node.path) ? node.path : null);
return (
<div
className={`kv-tree-row kv-tree-branch${open ? ' open' : ''}`}
style={{ paddingLeft: depth * 14 + 4 }}
onClick={onToggle}
title={isFileBranch(node.path) ? node.path : undefined}
>
<span className="kv-tree-toggle">{open ? '▾' : '▸'}</span>
{fileName ? (
<>
<span className="kv-tree-label kv-tree-filename">{fileName}</span>
<span className="kv-tree-hash-pill">{shortHashFromFileBranch(node.path)}</span>
</>
) : (
<span className="kv-tree-label">{node.label}</span>
)}
</div>
);
}
// Helper: given a key path with both ":" and "/" separators, return the list
// of ancestor branch paths that need to be expanded to reveal it.
export function ancestorBranches(key: string): string[] {
const out: string[] = [];
let i = 0;
while (i < key.length) {
const next = nextSep(key, i);
if (next < 0) break;
out.push(key.slice(0, next + 1));
i = next + 1;
}
return out;
}
function nextSep(s: string, from: number): number {
let best = -1;
for (const d of [':', '/']) {
const idx = s.indexOf(d, from);
if (idx >= 0 && (best < 0 || idx < best)) best = idx;
}
return best;
}
// Re-export for convenience in the shell
export type { TreeNode };
+57
View File
@@ -0,0 +1,57 @@
import { useState } from 'react';
interface Props {
fields: string[];
onChange: (fields: string[]) => void;
}
// Pill list of field names that the value-search uses. Each pill is removable;
// the inline input adds a new field on Enter or comma. Persisting to
// localStorage is the parent's responsibility.
export function SearchFieldChips({ fields, onChange }: Props) {
const [input, setInput] = useState('');
const add = (raw: string) => {
const next = raw.trim().replace(/,+$/, '');
if (!next) return;
if (fields.includes(next)) return;
onChange([...fields, next]);
setInput('');
};
const remove = (f: string) => onChange(fields.filter((x) => x !== f));
return (
<div className="kv-chips">
<span className="kv-chips-label">search in:</span>
{fields.map((f) => (
<span key={f} className="kv-chip">
<code>{f}</code>
<button
className="kv-chip-x"
onClick={() => remove(f)}
title={`remove ${f}`}
aria-label={`remove ${f}`}
>×</button>
</span>
))}
<input
className="kv-chip-input"
type="text"
value={input}
placeholder="+ field"
onChange={(e) => {
const v = e.target.value;
if (v.endsWith(',')) add(v);
else setInput(v);
}}
onKeyDown={(e) => {
if (e.key === 'Enter') { e.preventDefault(); add(input); }
if (e.key === 'Backspace' && input === '' && fields.length > 0) {
remove(fields[fields.length - 1]);
}
}}
/>
</div>
);
}
+45
View File
@@ -0,0 +1,45 @@
import { useEffect } from 'react';
import { ToastEntry } from './types';
interface Props {
toasts: ToastEntry[];
onDismiss: (id: number) => void;
}
export function ToastStack({ toasts, onDismiss }: Props) {
return (
<div className="kv-toast-stack">
{toasts.map((t) => (
<Toast key={t.id} entry={t} onDismiss={() => onDismiss(t.id)} />
))}
</div>
);
}
function Toast({ entry, onDismiss }: { entry: ToastEntry; onDismiss: () => void }) {
useEffect(() => {
if (entry.kind === 'error') return;
const timer = setTimeout(onDismiss, 4500);
return () => clearTimeout(timer);
}, [entry.kind, onDismiss]);
return (
<div className={`kv-toast kv-toast-${entry.kind}`}>
<div className="kv-toast-row">
<span className="kv-toast-text">{entry.text}</span>
<button className="kv-toast-close" onClick={onDismiss} aria-label="dismiss">×</button>
</div>
{entry.details && entry.details.length > 0 && (
<ul className="kv-toast-details">
{entry.details.slice(0, 8).map((d, i) => <li key={i}>{d}</li>)}
{entry.details.length > 8 && <li>… {entry.details.length - 8} more</li>}
</ul>
)}
</div>
);
}
let nextId = 1;
export function makeToast(kind: ToastEntry['kind'], text: string, details?: string[]): ToastEntry {
return { id: nextId++, kind, text, details };
}
+295
View File
@@ -0,0 +1,295 @@
import { useEffect, useState } from 'react';
import { LanguageCombobox } from '../components/LanguageCombobox';
import { FieldType } from './types';
interface ValueRendererProps {
value: string;
onChange: (v: string) => void;
}
interface LeafValueEditorProps extends ValueRendererProps {
type: FieldType;
}
/**
* Type-dispatched leaf value editor. Used by both the single-leaf editor
* (KVLeafEditor) and the per-row editor in the file-detail view.
*/
export function LeafValueEditor({ type, value, onChange }: LeafValueEditorProps) {
switch (type) {
case 'bool':
return (
<select
className="kv-leaf-input"
value={value === 'true' ? 'true' : value === 'false' ? 'false' : ''}
onChange={(e) => onChange(e.target.value)}
>
<option value="">(unset)</option>
<option value="true">true</option>
<option value="false">false</option>
</select>
);
case 'number':
return (
<input
type="number"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'datetime':
return <DateTimeLeaf value={value} onChange={onChange} />;
case 'lang':
return <LanguageCombobox value={value} onChange={onChange} />;
case 'url':
return (
<div className="kv-leaf-url">
<input
type="url"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
{value && (
<a className="kv-leaf-url-open" href={value} target="_blank" rel="noreferrer">
open ↗
</a>
)}
</div>
);
case 'json':
return (
<textarea
className="kv-leaf-textarea kv-leaf-mono"
rows={12}
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
);
case 'text':
return (
<textarea
className="kv-leaf-textarea"
rows={8}
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
case 'hash':
return <CidLeaf value={value} onChange={onChange} />;
case 'auto':
case 'string':
default:
return (
<input
type="text"
className="kv-leaf-input"
value={value}
onChange={(e) => onChange(e.target.value)}
/>
);
}
}
interface CidFileInfo {
exists: boolean;
contentType?: string;
size?: number;
filePath?: string;
}
/**
* CID leaf editor. Shows the value as an editable monospace input plus a
* content-typed preview when meta-core can resolve the CID to a file on disk
* (`/api/file/{cid}/info` → image/video/audio rendering, else download-only).
*/
function CidLeaf({ value, onChange }: ValueRendererProps) {
const [info, setInfo] = useState<CidFileInfo | null>(null);
const cid = value.trim();
const fileUrl = cid ? `/file/${cid}` : '';
const downloadName = info?.filePath?.split('/').pop() || cid;
useEffect(() => {
if (!cid) {
setInfo(null);
return;
}
let alive = true;
fetch(`/api/file/${cid}/info`)
.then((r) => (r.ok ? r.json() : { exists: false }))
.then((data: CidFileInfo) => {
if (alive) setInfo(data);
})
.catch(() => {
if (alive) setInfo({ exists: false });
});
return () => {
alive = false;
};
}, [cid]);
const renderPreview = () => {
if (!info?.exists || !info.contentType || !cid) return null;
if (info.contentType.startsWith('image/')) {
return <img src={fileUrl} alt="" className="kv-leaf-cid-preview" loading="lazy" />;
}
if (info.contentType.startsWith('video/')) {
return (
<video
src={fileUrl}
className="kv-leaf-cid-preview kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
if (info.contentType.startsWith('audio/')) {
return (
<audio
src={fileUrl}
className="kv-leaf-cid-preview-media"
controls
preload="metadata"
/>
);
}
return null;
};
return (
<div className="kv-leaf-cid">
{renderPreview()}
<div className="kv-leaf-cid-controls">
<input
type="text"
className="kv-leaf-input kv-leaf-mono"
value={value}
onChange={(e) => onChange(e.target.value)}
spellCheck={false}
/>
<button
type="button"
className="kv-btn-icon"
onClick={() => navigator.clipboard.writeText(value)}
title="copy CID"
>
⧉
</button>
</div>
{info?.exists && cid && (
<div className="kv-leaf-cid-actions">
<a
href={fileUrl}
download={downloadName}
className="kv-btn kv-btn-primary"
title={`Download ${downloadName}${info.size ? ` (${formatBytes(info.size)})` : ''}`}
>
⬇ Download
</a>
<a href={fileUrl} className="kv-btn" target="_blank" rel="noreferrer">
open ↗
</a>
<span className="kv-leaf-cid-meta">
{info.contentType}
{info.size != null ? ` · ${formatBytes(info.size)}` : ''}
</span>
</div>
)}
</div>
);
}
/** Timestamp shapes we round-trip through the picker. */
type TimestampShape = 'iso-date' | 'iso-datetime' | 'unix-s' | 'unix-ms';
function detectShape(raw: string): TimestampShape | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) return 'iso-date';
if (/^\d+$/.test(trimmed)) {
const n = Number.parseInt(trimmed, 10);
if (n >= 1_000_000_000 && n <= 4_102_444_800) return 'unix-s';
if (n >= 1_000_000_000_000 && n <= 4_102_444_800_000) return 'unix-ms';
return null;
}
if (!Number.isNaN(Date.parse(trimmed))) return 'iso-datetime';
return null;
}
function rawToDate(raw: string, shape: TimestampShape): Date | null {
const trimmed = raw.trim();
if (!trimmed) return null;
if (shape === 'unix-s') return new Date(Number.parseInt(trimmed, 10) * 1000);
if (shape === 'unix-ms') return new Date(Number.parseInt(trimmed, 10));
const d = new Date(trimmed);
return Number.isNaN(d.getTime()) ? null : d;
}
function dateToRaw(d: Date, shape: TimestampShape): string {
switch (shape) {
case 'unix-s':
return String(Math.floor(d.getTime() / 1000));
case 'unix-ms':
return String(d.getTime());
case 'iso-date':
return d.toISOString().slice(0, 10);
case 'iso-datetime':
return d.toISOString();
}
}
/**
* Datetime leaf editor. Preserves the original storage shape — ISO date stays
* ISO date, unix seconds stay unix seconds, etc. Unknown shapes fall back to
* iso-datetime.
*/
function DateTimeLeaf({ value, onChange }: ValueRendererProps) {
const shape: TimestampShape = detectShape(value) ?? 'iso-datetime';
const date = rawToDate(value, shape);
const inputType = shape === 'iso-date' ? 'date' : 'datetime-local';
const inputValue = (() => {
if (!date || Number.isNaN(date.getTime())) return '';
if (inputType === 'date') return date.toISOString().slice(0, 10);
// datetime-local wants YYYY-MM-DDTHH:mm in the user's local timezone.
const local = new Date(date.getTime() - date.getTimezoneOffset() * 60000);
return local.toISOString().slice(0, 16);
})();
return (
<div className="kv-leaf-datetime">
<input
type={inputType}
className="kv-leaf-input"
value={inputValue}
onChange={(e) => {
const v = e.target.value;
if (!v) {
onChange('');
return;
}
const parsed = new Date(v);
if (Number.isNaN(parsed.getTime())) return;
onChange(dateToRaw(parsed, shape));
}}
/>
<span className="kv-leaf-datetime-shape" title="Storage format preserved on save">
{shape}
</span>
</div>
);
}
function formatBytes(n: number): string {
if (n < 1024) return `${n} B`;
const units = ['KB', 'MB', 'GB', 'TB'];
let v = n / 1024;
let i = 0;
while (v >= 1024 && i < units.length - 1) {
v /= 1024;
i++;
}
return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[i]}`;
}
+90
View File
@@ -0,0 +1,90 @@
import { SchemaFieldInfo, SchemaResponse } from '../api/schemaApi';
import { FieldType } from './types';
export interface SchemaLookup {
/** The matched schema entry, if any. */
info?: SchemaFieldInfo;
/** Suggested FieldType for the leaf editor, or null when schema can't decide. */
fieldType: FieldType | null;
/**
* True when the leaf is one variant under a language-keyed group
* (e.g. plot/eng). The schema entry returned is the group's, not the leaf's.
*/
isLanguageVariant: boolean;
}
/**
* Resolve the logical schema entry for a full Redis key.
*
* - "file:<hash>/releasedate" → schema.fields["releasedate"]
* - "file:<hash>/plot/eng" → schema.fields["plot"] (language-keyed group)
* - non-file keys → undefined (schema is per-file today)
*/
export function lookupSchemaForKey(
schema: SchemaResponse | null,
key: string,
): SchemaLookup {
if (!schema) {
return { fieldType: null, isLanguageVariant: false };
}
const propertyPath = stripFilePrefix(key);
if (!propertyPath) {
return { fieldType: null, isLanguageVariant: false };
}
const direct = schema.fields[propertyPath];
if (direct) {
return {
info: direct,
fieldType: schemaToFieldType(direct),
isLanguageVariant: false,
};
}
// Language-keyed collapse: tail is a 2/3-letter lang code under a known group.
const lastSlash = propertyPath.lastIndexOf('/');
if (lastSlash > 0) {
const tail = propertyPath.slice(lastSlash + 1);
if (/^[a-z]{2,3}$/.test(tail)) {
const group = schema.fields[propertyPath.slice(0, lastSlash)];
if (group?.key_hint === 'language-code') {
return {
info: group,
fieldType: schemaToFieldType(group),
isLanguageVariant: true,
};
}
}
}
return { fieldType: null, isLanguageVariant: false };
}
export function schemaToFieldType(info: SchemaFieldInfo | undefined): FieldType | null {
if (!info) return null;
if (info.hint === 'cid') return 'hash';
if (info.hint === 'timestamp') return 'datetime';
switch (info.type) {
case 'bool':
return 'bool';
case 'int':
case 'float':
return 'number';
case 'json-object':
case 'json-array':
return 'json';
case 'string':
case 'mixed':
case 'undefined':
default:
return null;
}
}
function stripFilePrefix(key: string): string {
if (!key.startsWith('file:')) return '';
const slash = key.indexOf('/');
if (slash < 0) return '';
return key.slice(slash + 1);
}
+59
View File
@@ -0,0 +1,59 @@
import { FieldType } from './types';
// Path-pattern table. The leftmost segment of the path that matches wins.
// Order matters: more specific patterns should appear first.
const PATH_PATTERNS: Array<[RegExp, FieldType]> = [
[/(^|\/)(language|lang|audioLang|subLang)$/i, 'lang'],
[/(^|\/)(cid_|hash$|sha\d|midhash|md5|crc32)/i, 'hash'],
[/(^|\/)(duration|bitrate|fps|width|height|count|size|sizeByte|increment|episode|season|year|runtime|trackId|index)$/i, 'number'],
[/(^|\/)(enabled|active|disabled|isHidden|hidden|favorite|watched)$/i, 'bool'],
[/(^|\/)(url|uri|href|link|posterUrl|backdropUrl|imageUrl)$/i, 'url'],
[/(^|\/)(.+_at|.+At|.+Date|.+Time|timestamp|createdAt|updatedAt)$/i, 'datetime'],
[/(^|\/)(description|overview|notes?|comment|summary|nfo)$/i, 'text'],
];
function detectFromPath(path: string): FieldType | null {
for (const [re, t] of PATH_PATTERNS) {
if (re.test(path)) return t;
}
return null;
}
function detectFromValue(v: string): FieldType {
if (v === '') return 'string';
if (v === 'true' || v === 'false') return 'bool';
if (/^-?\d+$/.test(v)) return 'number';
if (/^-?\d*\.\d+$/.test(v)) return 'number';
if (/^https?:\/\//i.test(v)) return 'url';
const trimmed = v.trim();
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
try { JSON.parse(trimmed); return 'json'; } catch {}
}
if (v.includes('\n') || v.length > 200) return 'text';
return 'string';
}
// Combine path and value heuristics. Path wins when it matches; otherwise
// fall back to value shape.
export function detectType(path: string, value: string): FieldType {
return detectFromPath(path) ?? detectFromValue(value);
}
export const ALL_TYPES: FieldType[] = [
'auto', 'string', 'text', 'number', 'bool', 'json', 'url', 'lang', 'datetime', 'hash',
];
export function describeType(t: FieldType): string {
switch (t) {
case 'auto': return 'auto';
case 'string': return 'string';
case 'text': return 'text (multiline)';
case 'number': return 'number';
case 'bool': return 'bool';
case 'json': return 'json';
case 'url': return 'url';
case 'lang': return 'language code';
case 'datetime': return 'datetime';
case 'hash': return 'hash / cid';
}
}
+19
View File
@@ -0,0 +1,19 @@
// Inferred type for a leaf value. "auto" defers to the heuristic detector.
export type FieldType =
| 'auto'
| 'string'
| 'number'
| 'bool'
| 'json'
| 'url'
| 'text'
| 'lang'
| 'datetime'
| 'hash';
export interface ToastEntry {
id: number;
kind: 'info' | 'success' | 'error';
text: string;
details?: string[];
}
+2 -2
View File
@@ -1,10 +1,10 @@
import { StrictMode } from 'react'; import { StrictMode } from 'react';
import { createRoot } from 'react-dom/client'; import { createRoot } from 'react-dom/client';
import './index.css'; import './index.css';
import App from './App.tsx'; import KVApp from './kv/KVApp.tsx';
createRoot(document.getElementById('root')!).render( createRoot(document.getElementById('root')!).render(
<StrictMode> <StrictMode>
<App /> <KVApp />
</StrictMode>, </StrictMode>,
); );
-155
View File
@@ -1,155 +0,0 @@
// API Response Types
export interface FileMetadata {
// File info
fileName?: string;
extension?: string;
byteSize?: number;
mimeType?: string;
fileType?: string;
// Filename parsing
title?: string;
titles?: string[];
originaltitle?: string;
showtitle?: string;
season?: number;
episode?: number;
movieYear?: number;
videoType?: string;
// General media metadata
year?: number;
imdbid?: string;
tmdbid?: string;
anidbid?: string;
premiered?: string;
genre?: string[];
releasedate?: string;
plot?: string;
director?: string;
rating?: number;
runtime?: number;
studio?: string;
art?: any;
actor?: any[];
aired?: string;
customrating?: string;
dateadded?: string;
sorttitle?: string;
mpaa?: string;
aspectratio?: string;
collectionnumber?: number;
criticrating?: number;
// Hashes
cid_crc32?: string;
cid_md5?: string;
cid_sha1?: string;
'cid_sha2-256'?: string;
'cid_sha3-256'?: string;
cid_sha3_384?: string;
// Video info
video?: {
codec?: string;
width?: number;
height?: number;
bitrate?: number;
fps?: number;
};
// Audio info
audio?: Array<{
codec?: string;
channels?: number;
language?: string;
bitrate?: number;
}>;
// Subtitles
subtitles?: Array<{
language?: string;
codec?: string;
}>;
// Other
extra?: any;
anime?: any;
fileinfo?: any;
trackers?: string[];
// Allow any additional properties
[key: string]: any;
}
export interface SearchResult {
hashId: string;
metadata: FileMetadata;
score?: number;
}
export interface SearchResponse {
results: SearchResult[];
count: number;
total?: number;
}
export interface UpdateResult {
status: string;
hashId: string;
message?: string;
error?: string;
}
export interface BatchUpdateResponse {
status: string;
total: number;
success: number;
errors: number;
results: Array<{
hashId: string;
status: string;
error?: string;
}>;
}
// Plugin Schema Types
export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
export interface SelectOption {
value: string;
label: string;
}
export interface SchemaField {
label: string;
type?: SchemaFieldType;
readonly?: boolean;
indexed?: boolean;
options?: SelectOption[];
hint?: string;
link?: string;
pattern?: string;
min?: number;
max?: number;
placeholder?: string;
}
export interface PluginSchema {
id: string;
name: string;
version: string;
description?: string;
priority?: number;
color?: string;
dependencies: string[];
active: boolean;
metadataSchema?: Record<string, SchemaField>;
}
export interface PluginsResponse {
plugins: PluginSchema[];
executionOrder: string[];
activeCount: number;
totalCount: number;
}
-53
View File
@@ -1,53 +0,0 @@
/**
* Format bytes to human readable format
*/
export function formatBytes(bytes: number | undefined): string {
if (!bytes) return 'N/A';
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
if (bytes === 0) return '0 B';
const i = Math.floor(Math.log(bytes) / Math.log(1024));
return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i];
}
/**
* Format duration in seconds to HH:MM:SS
*/
export function formatDuration(seconds: number | undefined): string {
if (!seconds) return 'N/A';
const hours = Math.floor(seconds / 3600);
const minutes = Math.floor((seconds % 3600) / 60);
const secs = Math.floor(seconds % 60);
if (hours > 0) {
return `${hours}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
}
return `${minutes}:${secs.toString().padStart(2, '0')}`;
}
/**
* Truncate hash to first and last 6 characters
*/
export function truncateHash(hash: string | undefined): string {
if (!hash) return 'N/A';
if (hash.length <= 16) return hash;
return `${hash.slice(0, 6)}...${hash.slice(-6)}`;
}
/**
* Get display name for a file from metadata
*/
export function getDisplayName(metadata: any): string {
if (metadata.title) {
if (metadata.season && metadata.episode) {
return `${metadata.title} S${String(metadata.season).padStart(2, '0')}E${String(metadata.episode).padStart(2, '0')}`;
}
if (metadata.movieYear) {
return `${metadata.title} (${metadata.movieYear})`;
}
return metadata.title;
}
return metadata.fileName || 'Unknown';
}
+1
View File
@@ -0,0 +1 @@
{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/schemaApi.ts","./src/api/snapshotApi.ts","./src/components/LanguageCombobox.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVFileEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/leafRenderers.tsx","./src/kv/schemaLookup.ts","./src/kv/typeHeuristics.ts","./src/kv/types.ts"],"version":"5.9.3"}
+4 -1
View File
@@ -3,10 +3,13 @@ module github.com/metazla/meta-core
go 1.21 go 1.21
require ( require (
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/gorilla/mux v1.8.1 github.com/gorilla/mux v1.8.1
github.com/mr-tron/base58 v1.3.0
github.com/redis/go-redis/v9 v9.4.0 github.com/redis/go-redis/v9 v9.4.0
golang.org/x/net v0.20.0 golang.org/x/crypto v0.31.0
golang.org/x/net v0.21.0
) )
require ( require (
+10 -2
View File
@@ -4,13 +4,21 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0= github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc= github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk= github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M= github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
+30
View File
@@ -0,0 +1,30 @@
package api
import (
"net/http"
)
// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
// stranded midhash-rooted entries into their UUID roots. See
// storage.MigrateDualRoots for the data-flow detail and
// docs/api-mediated-access.md for the bug background.
//
// Returns {fixed: N} on success. Safe to call repeatedly — entries already
// migrated are skipped on the next pass.
func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
fixed, err := s.storage.MigrateDualRootsWithTimeout()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, map[string]interface{}{
"success": true,
"fixed": fixed,
})
}
+70
View File
@@ -0,0 +1,70 @@
package api
import (
"net/http"
"github.com/gorilla/mux"
)
// handleGetMetaByCID handles GET /api/meta/{cid}.
//
// Resolves any known CID — midhash256, sha256, IPFS, btih, or anything else
// in the reverse index — to the full metadata document for that file.
// Returns:
// - 200 with the document on a hit
// - 404 if the CID isn't registered
// - 503 if storage is down
//
// CIDs in the URL are bare multibase CIDv1 strings ("bafk…") — the midhash,
// a sibling sha2-256, an IPFS CID, a btih, anything registered in the
// reverse index. A CID is self-describing (its algorithm is the multicodec),
// so there is no <algo>: prefix.
//
// This endpoint is the public read API for the CID-resolution layer and
// is auth-bypassed alongside /api/file/{cid} so that meta-share peers and
// IPFS-style clients can use it without going through Authelia.
func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
cid := mux.Vars(r)["cid"]
if cid == "" {
writeError(w, http.StatusBadRequest, "cid is required")
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
uuid, err := s.storage.GetByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if uuid == "" {
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
doc, err := s.storage.GetMetadataDocument(uuid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if doc == nil {
// Reverse index pointed at a UUID with no remaining keys — stale
// alias, probably mid-delete race. Treat as not-found.
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
return
}
// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
// cids / canonical_cid / duplicates), with cid added so the caller can
// correlate when they fan out N lookups. canonical_cid is derived by
// rank from the cids key-set on read — it is not a stored field.
writeJSON(w, http.StatusOK, map[string]interface{}{
"cid": cid,
"metadata": doc.Flat,
"cids": doc.CIDs,
"canonical_cid": doc.Canonical,
"duplicates": doc.Duplicates,
})
}
+138 -5
View File
@@ -63,10 +63,48 @@ type DataPathResponse struct {
Exists bool `json:"exists"` Exists bool `json:"exists"`
} }
// ErrorResponse is the response for errors // ErrorResponse is the typed error envelope returned on every 4xx/5xx.
//
// Shape and slug vocabulary documented in docs/api-mediated-access.md
// ("Error envelope"). Stable slug strings let clients switch on the failure
// category without parsing the message; `retryable` tells callers whether
// a backoff-and-retry has any chance of succeeding.
type ErrorResponse struct { type ErrorResponse struct {
Error string `json:"error"` Error string `json:"error"`
Message string `json:"message,omitempty"` Message string `json:"message,omitempty"`
Retryable bool `json:"retryable"`
}
// Stable error-slug strings. Pinned in the doc; consumers may match on
// these. Do not rename without a deprecation cycle.
const (
ErrAliasCollision = "alias_collision"
ErrUnknownRoot = "unknown_root"
ErrUnknownCID = "unknown_cid"
ErrSchemaViolation = "schema_violation"
ErrStorageUnavailable = "storage_unavailable"
ErrInternal = "internal"
)
// defaultSlugFor maps an HTTP status code to the most likely error slug.
// Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
// instead of "unknown_root" on 404) call writeErrorSlug directly.
func defaultSlugFor(status int) (slug string, retryable bool) {
switch status {
case http.StatusConflict:
return ErrAliasCollision, false
case http.StatusNotFound:
return ErrUnknownRoot, false
case http.StatusBadRequest:
return ErrSchemaViolation, false
case http.StatusServiceUnavailable:
return ErrStorageUnavailable, true
case http.StatusInternalServerError:
return ErrInternal, true
default:
// 4xx → not retryable, 5xx → retryable. Conservative default.
return ErrInternal, status >= 500
}
} }
var startTime = time.Now() var startTime = time.Now()
@@ -169,6 +207,10 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
return return
} }
// Resolve CID → UUID via the reverse index so reads find data wherever
// the watcher minted the root, regardless of which CID the caller knows.
hashID = s.storage.ResolveRoot(hashID)
metadata, err := s.storage.GetMetadataFlat(hashID) metadata, err := s.storage.GetMetadataFlat(hashID)
if err != nil { if err != nil {
writeError(w, http.StatusInternalServerError, err.Error()) writeError(w, http.StatusInternalServerError, err.Error())
@@ -203,6 +245,10 @@ func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
return return
} }
// CID → UUID resolution so writes land on the canonical root, not on
// a parallel midhash-rooted entry the watcher's UUID never sees.
hashID = s.storage.ResolveRoot(hashID)
var metadata map[string]string var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body") writeError(w, http.StatusBadRequest, "invalid JSON body")
@@ -235,6 +281,8 @@ func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
return return
} }
hashID = s.storage.ResolveRoot(hashID)
deleted, err := s.storage.DeleteMetadata(hashID) deleted, err := s.storage.DeleteMetadata(hashID)
if err != nil { if err != nil {
writeError(w, http.StatusInternalServerError, err.Error()) writeError(w, http.StatusInternalServerError, err.Error())
@@ -340,6 +388,63 @@ func (s *Server) handleHeadData(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK) w.WriteHeader(http.StatusOK)
} }
// handleGetFileByCIDInfo handles GET /api/file/{cid}/info.
// Returns lightweight metadata for the file (existence, content type, size,
// relative path) without streaming any bytes. Used by the editor to decide
// whether to render a preview (image/video) or a download-only button.
func (s *Server) handleGetFileByCIDInfo(w http.ResponseWriter, r *http.Request) {
vars := mux.Vars(r)
cid := vars["cid"]
type infoResp struct {
Exists bool `json:"exists"`
ContentType string `json:"contentType,omitempty"`
Size int64 `json:"size,omitempty"`
FilePath string `json:"filePath,omitempty"`
}
w.Header().Set("Content-Type", "application/json")
if cid == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
relPath, err := s.storage.LookupPathByCID(cid)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if relPath == "" {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
return
}
fullPath := filepath.Join(s.config.FilesPath, relPath)
fi, err := os.Stat(fullPath)
if err != nil {
_ = json.NewEncoder(w).Encode(infoResp{Exists: false, FilePath: relPath})
return
}
ext := strings.ToLower(filepath.Ext(fullPath))
contentType, ok := contentTypeByExt[ext]
if !ok {
contentType = "application/octet-stream"
}
_ = json.NewEncoder(w).Encode(infoResp{
Exists: true,
ContentType: contentType,
Size: fi.Size(),
FilePath: relPath,
})
}
// handleGetFileByCID handles GET /file/{cid} // handleGetFileByCID handles GET /file/{cid}
// Serves a file by looking up its CID in poster/backdrop metadata // Serves a file by looking up its CID in poster/backdrop metadata
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) { func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
@@ -364,7 +469,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
} }
if relPath == "" { if relPath == "" {
writeError(w, http.StatusNotFound, "file not found for CID") writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file not found for CID", false)
return return
} }
@@ -374,7 +479,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
// Check if file exists // Check if file exists
fileInfo, err := os.Stat(fullPath) fileInfo, err := os.Stat(fullPath)
if err != nil { if err != nil {
writeError(w, http.StatusNotFound, "file does not exist on disk") writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file does not exist on disk", false)
return return
} }
@@ -465,11 +570,26 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
json.NewEncoder(w).Encode(data) json.NewEncoder(w).Encode(data)
} }
// writeError writes an error response // writeError writes the typed error envelope. The slug + retryable bit
// are inferred from the status (see defaultSlugFor); call writeErrorSlug
// directly when a more specific slug applies.
func writeError(w http.ResponseWriter, status int, message string) { func writeError(w http.ResponseWriter, status int, message string) {
slug, retryable := defaultSlugFor(status)
writeJSON(w, status, ErrorResponse{ writeJSON(w, status, ErrorResponse{
Error: http.StatusText(status), Error: slug,
Message: message, Message: message,
Retryable: retryable,
})
}
// writeErrorSlug writes the typed envelope with an explicit slug and
// retryable bit. Use this for endpoint-specific slugs (alias_collision,
// unknown_cid) where the default mapping from HTTP status would be wrong.
func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, retryable bool) {
writeJSON(w, status, ErrorResponse{
Error: slug,
Message: message,
Retryable: retryable,
}) })
} }
@@ -489,6 +609,11 @@ func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
return return
} }
// PATCH is the hot write path for meta-sort. Resolve so plugin output
// merges into the watcher's UUID root rather than minting a parallel
// midhash-rooted entry.
hashID = s.storage.ResolveRoot(hashID)
var metadata map[string]string var metadata map[string]string
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil { if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body") writeError(w, http.StatusBadRequest, "invalid JSON body")
@@ -530,6 +655,8 @@ func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
return return
} }
hashID = s.storage.ResolveRoot(hashID)
value, err := s.storage.GetProperty(hashID, key) value, err := s.storage.GetProperty(hashID, key)
if err != nil { if err != nil {
writeError(w, http.StatusInternalServerError, err.Error()) writeError(w, http.StatusInternalServerError, err.Error())
@@ -575,6 +702,8 @@ func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
return return
} }
hashID = s.storage.ResolveRoot(hashID)
// Read body // Read body
buf := make([]byte, 1024*1024) // 1MB max buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf) n, err := r.Body.Read(buf)
@@ -629,6 +758,8 @@ func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
return return
} }
hashID = s.storage.ResolveRoot(hashID)
if err := s.storage.DeleteProperty(hashID, key); err != nil { if err := s.storage.DeleteProperty(hashID, key); err != nil {
writeError(w, http.StatusInternalServerError, err.Error()) writeError(w, http.StatusInternalServerError, err.Error())
return return
@@ -664,6 +795,8 @@ func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
return return
} }
hashID = s.storage.ResolveRoot(hashID)
// Read body // Read body
buf := make([]byte, 1024*1024) // 1MB max buf := make([]byte, 1024*1024) // 1MB max
n, err := r.Body.Read(buf) n, err := r.Body.Read(buf)
+277
View File
@@ -0,0 +1,277 @@
package api
import (
"encoding/base64"
"encoding/json"
"net/http"
"strings"
"github.com/metazla/meta-core/internal/identity"
)
// identityStatusResponse is the shape of GET /api/identity. The private key
// is intentionally absent — it is only ever returned once, by
// POST /api/identity/generate, immediately after creation.
type identityStatusResponse struct {
HasIdentity bool `json:"hasIdentity"`
UID string `json:"uid,omitempty"`
Curve string `json:"curve,omitempty"`
CreatedAt int64 `json:"createdAt,omitempty"`
// AccountCount is the total number of accounts in the keystore. Lets
// multi-account clients (meta-watch) tell "single identity" from "pick a
// profile" without a second call. Older single-identity clients ignore it.
AccountCount int `json:"accountCount,omitempty"`
}
// identityAccountInfo is one entry in GET /api/identity/accounts. Never carries
// the private key — that is only ever returned once, by generate.
type identityAccountInfo struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
}
type identityAccountsResponse struct {
Accounts []identityAccountInfo `json:"accounts"`
}
type identityGenerateResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportRequest struct {
PrivateKeyHex string `json:"privateKeyHex"`
}
type identityImportResponse struct {
UID string `json:"uid"`
Curve string `json:"curve"`
CreatedAt int64 `json:"createdAt"`
}
type identityDeleteRequest struct {
Confirm bool `json:"confirm"`
UID string `json:"uid"`
}
type identitySignRequest struct {
BytesB64 string `json:"bytesB64"`
// UID selects which account signs. Optional — when omitted and exactly one
// account exists, that account is used (single-identity back-compat).
UID string `json:"uid,omitempty"`
}
type identitySignResponse struct {
SigB64 string `json:"sigB64"`
}
type identityAEADKeyResponse struct {
AEADKeyB64 string `json:"aeadKeyB64"`
}
// resolveAccount picks the account an operation targets. A non-empty uid must
// match an existing account. An empty uid falls back to the sole account
// (single-identity back-compat); with zero or many accounts and no uid it
// returns a non-zero status for the caller to surface. status == 0 means ok.
func (s *Server) resolveAccount(uid string) (id *identity.Identity, status int, slug, msg string) {
dir := s.config.IdentityAccountsDir()
uid = strings.TrimSpace(uid)
if uid != "" {
got, err := identity.LoadByUID(dir, uid)
if err != nil {
return nil, http.StatusBadRequest, "bad_uid", err.Error()
}
if got == nil {
return nil, http.StatusNotFound, "no_identity", "no identity for the given uid"
}
return got, 0, "", ""
}
list, err := identity.List(dir)
if err != nil {
return nil, http.StatusInternalServerError, "list_failed", err.Error()
}
switch len(list) {
case 0:
return nil, http.StatusNotFound, "no_identity", "no identity is configured"
case 1:
return list[0], 0, "", ""
default:
return nil, http.StatusBadRequest, "uid_required", "multiple accounts exist; specify uid"
}
}
func (s *Server) handleIdentityGet(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if len(list) == 0 {
writeJSON(w, http.StatusOK, identityStatusResponse{HasIdentity: false})
return
}
// Back-compat: report the oldest account as "the" identity. Multi-account
// clients enumerate via GET /api/identity/accounts instead.
def := list[0]
writeJSON(w, http.StatusOK, identityStatusResponse{
HasIdentity: true,
UID: def.UID,
Curve: def.Curve,
CreatedAt: def.CreatedAt,
AccountCount: len(list),
})
}
func (s *Server) handleIdentityAccounts(w http.ResponseWriter, r *http.Request) {
list, err := identity.List(s.config.IdentityAccountsDir())
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
out := make([]identityAccountInfo, 0, len(list))
for _, id := range list {
out = append(out, identityAccountInfo{UID: id.UID, Curve: id.Curve, CreatedAt: id.CreatedAt})
}
writeJSON(w, http.StatusOK, identityAccountsResponse{Accounts: out})
}
func (s *Server) handleIdentityGenerate(w http.ResponseWriter, r *http.Request) {
id, err := identity.Generate()
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if err := identity.SaveAccount(s.config.IdentityAccountsDir(), id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityGenerateResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
PrivateKeyHex: id.PrivateKeyHex,
})
}
func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
var body identityImportRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if strings.TrimSpace(body.PrivateKeyHex) == "" {
writeError(w, http.StatusBadRequest, "privateKeyHex is required")
return
}
id, err := identity.Import(strings.TrimSpace(body.PrivateKeyHex))
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
dir := s.config.IdentityAccountsDir()
if identity.ExistsByUID(dir, id.UID) {
writeErrorSlug(w, http.StatusConflict, "identity_exists",
"an account with this key already exists", false)
return
}
if err := identity.SaveAccount(dir, id); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusCreated, identityImportResponse{
UID: id.UID,
Curve: id.Curve,
CreatedAt: id.CreatedAt,
})
}
func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
var body identityDeleteRequest
// Tolerate empty body — confirm flag in a query param is also acceptable
// for shells where sending a JSON body with DELETE is awkward.
_ = json.NewDecoder(r.Body).Decode(&body)
if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
"identity deletion requires {\"confirm\": true} or ?confirm=true", false)
return
}
dir := s.config.IdentityAccountsDir()
uid := strings.TrimSpace(body.UID)
if uid == "" {
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
}
if uid == "" {
// Back-compat: with no uid, delete the sole account. Ambiguous (and so
// refused) once more than one account exists.
list, err := identity.List(dir)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
switch len(list) {
case 0:
w.WriteHeader(http.StatusNoContent) // idempotent
return
case 1:
uid = list[0].UID
default:
writeErrorSlug(w, http.StatusBadRequest, "uid_required",
"multiple accounts exist; specify uid to delete", false)
return
}
}
if err := identity.DeleteByUID(dir, uid); err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
w.WriteHeader(http.StatusNoContent)
}
func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
var body identitySignRequest
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
uid := body.UID
if strings.TrimSpace(uid) == "" {
uid = r.URL.Query().Get("uid")
}
id, status, slug, msg := s.resolveAccount(uid)
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
raw, err := base64.StdEncoding.DecodeString(body.BytesB64)
if err != nil {
writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
return
}
sig, err := identity.Sign(id, raw)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, identitySignResponse{
SigB64: base64.StdEncoding.EncodeToString(sig),
})
}
func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
if status != 0 {
writeErrorSlug(w, status, slug, msg, false)
return
}
key, err := identity.DeriveAEADKey(id)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, identityAEADKeyResponse{
AEADKeyB64: base64.StdEncoding.EncodeToString(key),
})
}
+286
View File
@@ -1,12 +1,14 @@
package api package api
import ( import (
"encoding/json"
"net/http" "net/http"
"net/url" "net/url"
"strconv" "strconv"
"strings" "strings"
"github.com/gorilla/mux" "github.com/gorilla/mux"
"github.com/metazla/meta-core/internal/storage"
) )
// KVInfoResponse is the response for GET /api/kv/info // KVInfoResponse is the response for GET /api/kv/info
@@ -184,3 +186,287 @@ func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
writeJSON(w, http.StatusOK, response) writeJSON(w, http.StatusOK, response)
} }
// KVTreeResponse is the response for GET /api/kv/tree
type KVTreeResponse struct {
Prefix string `json:"prefix"`
Delimiter string `json:"delimiter"`
Branches []string `json:"branches"`
Leaves []string `json:"leaves"`
Cursor string `json:"cursor"`
HasMore bool `json:"hasMore"`
}
// KVSearchResponse is the response for GET /api/kv/search
type KVSearchResponse struct {
Contains string `json:"contains"`
Keys []string `json:"keys"`
Truncated bool `json:"truncated"`
}
// KVFindResponse is the response for GET /api/kv/find
type KVFindResponse struct {
Contains string `json:"contains"`
Fields []string `json:"fields"`
Matches []storage.FindMatch `json:"matches"`
Truncated bool `json:"truncated"`
}
// KVValueResponse is the response for GET /api/kv/value
type KVValueResponse struct {
Key string `json:"key"`
Type string `json:"type"`
Value string `json:"value"`
Exists bool `json:"exists"`
}
// KVValuePutRequest is the body for PUT /api/kv/value
type KVValuePutRequest struct {
Key string `json:"key"`
Value string `json:"value"`
}
// KVValueDeleteResponse is the response for DELETE /api/kv/value
type KVValueDeleteResponse struct {
Key string `json:"key"`
Deleted bool `json:"deleted"`
}
// handleKVTree handles GET /api/kv/tree?prefix=&delimiter=/&cursor=0&max=500
//
// Lists the immediate children at prefix. When delimiter is set, keys with a
// further delimiter are collapsed into branches. Pass the returned cursor back
// to continue when hasMore is true.
func (s *Server) handleKVTree(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
prefix := q.Get("prefix")
delimiter := q.Get("delimiter")
cursor := q.Get("cursor")
maxResults := 500
if v := q.Get("max"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
maxResults = n
}
}
leaves, branches, nextCursor, err := s.storage.ScanKeys(prefix, delimiter, cursor, maxResults)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVTreeResponse{
Prefix: prefix,
Delimiter: delimiter,
Branches: branches,
Leaves: leaves,
Cursor: nextCursor,
HasMore: nextCursor != "0",
})
}
// handleKVFind handles GET /api/kv/find?contains=&fields=a,b,c&limit=200
//
// Value-search: scans every key whose path ends with /<field> for each
// configured field, then returns those whose value contains the substring
// (case-insensitive). Used as the editor's "filter the view" action.
func (s *Server) handleKVFind(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
fieldsParam := q.Get("fields")
if fieldsParam == "" {
writeError(w, http.StatusBadRequest, "fields is required (comma-separated)")
return
}
fields := strings.Split(fieldsParam, ",")
for i := range fields {
fields[i] = strings.TrimSpace(fields[i])
}
limit := 200
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
matches, truncated, err := s.storage.FindByValue(contains, fields, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if matches == nil {
matches = []storage.FindMatch{}
}
writeJSON(w, http.StatusOK, KVFindResponse{
Contains: contains,
Fields: fields,
Matches: matches,
Truncated: truncated,
})
}
// handleKVSearch handles GET /api/kv/search?contains=&limit=500
func (s *Server) handleKVSearch(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
q := r.URL.Query()
contains := q.Get("contains")
if contains == "" {
writeError(w, http.StatusBadRequest, "contains is required")
return
}
limit := 500
if v := q.Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
limit = n
}
}
keys, truncated, err := s.storage.SearchKeys(contains, limit)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVSearchResponse{
Contains: contains,
Keys: keys,
Truncated: truncated,
})
}
// handleKVValueGet handles GET /api/kv/value?key=...
func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
kind, value, exists, err := s.storage.GetKeyInfo(key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: key,
Type: kind,
Value: value,
Exists: exists,
})
}
// handleKVValuePut handles PUT /api/kv/value with body {key, value}.
// Maintains the file index when the key matches file:<hashId>/...
func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
var req KVValuePutRequest
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
writeError(w, http.StatusBadRequest, "invalid JSON body")
return
}
if req.Key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
if err := s.storage.SetRawValue(req.Key, req.Value); err != nil {
writeError(w, http.StatusConflict, err.Error())
return
}
if hashID := extractFileHashID(req.Key); hashID != "" {
if err := s.storage.IndexAdd(hashID); err != nil {
// Non-fatal: the value was written; index entry is best-effort.
// Surface in the log path of writeJSON via response header.
w.Header().Set("X-KV-Index-Warning", err.Error())
}
}
writeJSON(w, http.StatusOK, KVValueResponse{
Key: req.Key,
Type: "string",
Value: req.Value,
Exists: true,
})
}
// handleKVValueDelete handles DELETE /api/kv/value?key=...
// When the key matches file:<hashId>/..., removes the file index entry if no
// sibling keys remain after the delete.
func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
key := r.URL.Query().Get("key")
if key == "" {
writeError(w, http.StatusBadRequest, "key is required")
return
}
deleted, err := s.storage.DeleteRawKey(key)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
if deleted {
if hashID := extractFileHashID(key); hashID != "" {
if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil {
w.Header().Set("X-KV-Index-Warning", err.Error())
}
}
}
writeJSON(w, http.StatusOK, KVValueDeleteResponse{
Key: key,
Deleted: deleted,
})
}
// extractFileHashID returns the hashID for a key shaped like file:<hashId>/...
// Returns "" when the key does not follow that convention.
func extractFileHashID(key string) string {
if !strings.HasPrefix(key, "file:") {
return ""
}
rest := strings.TrimPrefix(key, "file:")
slash := strings.Index(rest, "/")
if slash <= 0 {
return ""
}
return rest[:slash]
}
+12 -26
View File
@@ -395,7 +395,18 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
return return
} }
// Get all hash IDs // Free-text query: served from the in-memory search index (search_index.go)
// so we match in memory instead of doing a SCAN+MGET Redis round-trip per
// record (that was 40s+ over ~72k records). Full metadata is fetched only
// for the <=limit matches.
if req.Query != "" && req.Property == "" {
s.serveIndexedSearch(w, req)
return
}
// Property filter (rare) or empty query (returns the first `limit`): the
// legacy scan. Empty query only fetches `limit` records; property search is
// low-volume and not on the hot search path.
allHashIDs, err := s.storage.GetAllHashIDs() allHashIDs, err := s.storage.GetAllHashIDs()
if err != nil { if err != nil {
writeError(w, http.StatusInternalServerError, err.Error()) writeError(w, http.StatusInternalServerError, err.Error())
@@ -405,9 +416,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
var results []MetadataSearchResult var results []MetadataSearchResult
total := len(allHashIDs) total := len(allHashIDs)
// Search fields for query
searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
for _, hashID := range allHashIDs { for _, hashID := range allHashIDs {
if len(results) >= req.Limit { if len(results) >= req.Limit {
break break
@@ -428,28 +436,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
} }
} }
// General query search
if req.Query != "" && req.Property == "" {
queryLower := strings.ToLower(req.Query)
// Check if query matches hashId
if strings.Contains(strings.ToLower(hashID), queryLower) {
matches = true
}
// Check search fields
if !matches {
for _, field := range searchFields {
if value, ok := metadata[field]; ok {
if strings.Contains(strings.ToLower(value), queryLower) {
matches = true
break
}
}
}
}
}
// If no query or property filter, return all // If no query or property filter, return all
if req.Query == "" && req.Property == "" { if req.Query == "" && req.Property == "" {
matches = true matches = true
+45
View File
@@ -0,0 +1,45 @@
package api
import (
"encoding/json"
"net/http"
"time"
)
// handleSchemaGet returns the live per-field schema derived from meta:events.
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
snap := s.schemaIndexer.Snapshot()
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(snap)
}
// handleSchemaRescan re-emits a set event for every existing property via the
// meta publisher, waits for the indexer to drain, and returns the refreshed
// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
if s.schemaIndexer == nil {
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
return
}
count, err := s.RepublishMetadata()
if err != nil {
http.Error(w, err.Error(), http.StatusInternalServerError)
return
}
s.schemaIndexer.WaitForDrain(10 * time.Second)
snap := s.schemaIndexer.Snapshot()
snap.Source = "rescan"
w.Header().Set("Content-Type", "application/json")
_ = json.NewEncoder(w).Encode(map[string]any{
"published_events": count,
"schema": snap,
})
}
+110
View File
@@ -0,0 +1,110 @@
package api
import (
"log"
"net/http"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/metazla/meta-core/internal/storage"
)
// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
// from Redis in the background. Records ingest continuously, so a newly stored
// record becomes searchable within one interval; the rebuild is a single
// batched bulk read (storage.GetSearchIndexTuples), off the request path.
const searchIndexRefreshInterval = 60 * time.Second
// searchIndex is an in-memory snapshot of every record's search haystack
// (hashID + lowercased searchable fields), rebuilt periodically from Redis. It
// exists so POST /api/metadata/search matches in memory (string ops over a
// slice) instead of doing a SCAN+MGET Redis round-trip per record — the latter
// took 40s+ over ~72k records and made a live parallel search path unusable.
type searchIndex struct {
mu sync.RWMutex
tuples []storage.SearchTuple
ready atomic.Bool
}
// snapshot returns the current tuples slice and whether the index has built at
// least once. The refresh swaps in a fresh slice under the write lock (never
// mutates in place), so the returned slice is safe to read lock-free.
func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
idx.mu.RLock()
defer idx.mu.RUnlock()
return idx.tuples, idx.ready.Load()
}
// refresh rebuilds the index from Redis once (a batched bulk read).
func (idx *searchIndex) refresh(stor *storage.Client) {
if stor == nil || !stor.IsConnected() {
return
}
tuples, err := stor.GetSearchIndexTuples()
if err != nil {
log.Printf("[API] search index refresh failed: %v", err)
return
}
idx.mu.Lock()
idx.tuples = tuples
idx.mu.Unlock()
idx.ready.Store(true)
}
// runRefreshLoop builds the index immediately, then rebuilds on a ticker for the
// process lifetime.
func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
idx.refresh(stor)
ticker := time.NewTicker(searchIndexRefreshInterval)
defer ticker.Stop()
for range ticker.C {
idx.refresh(stor)
}
}
// serveIndexedSearch answers a free-text (non-property) query from the in-memory
// index: token-AND match against each record's haystack (or a whole-query hashID
// substring, preserving the legacy behaviour), then fetch full metadata for only
// the <=limit matches. No Redis round-trip per scanned record.
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
tuples, ready := s.search.snapshot()
if !ready {
// First call before the background build finished: warm inline with the
// efficient batched read (seconds), never the per-record SCAN fallback.
s.search.refresh(s.storage)
tuples, _ = s.search.snapshot()
}
tokens := strings.Fields(strings.ToLower(req.Query))
queryLower := strings.ToLower(req.Query)
results := make([]MetadataSearchResult, 0, req.Limit)
for _, t := range tuples {
if len(results) >= req.Limit {
break
}
if !matchAllTokens(t.Haystack, tokens) &&
!strings.Contains(strings.ToLower(t.HashID), queryLower) {
continue
}
// Full metadata is cached in the index — return it straight from memory,
// no Redis round-trip per result.
results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
}
writeJSON(w, http.StatusOK, MetadataSearchResponse{
Results: results,
Count: len(results),
Total: len(tuples),
})
}
// matchAllTokens reports whether haystack contains every token (AND semantics).
// An empty token list matches everything.
func matchAllTokens(haystack string, tokens []string) bool {
for _, t := range tokens {
if !strings.Contains(haystack, t) {
return false
}
}
return true
}
+157 -10
View File
@@ -11,8 +11,10 @@ import (
"github.com/metazla/meta-core/internal/config" "github.com/metazla/meta-core/internal/config"
"github.com/metazla/meta-core/internal/discovery" "github.com/metazla/meta-core/internal/discovery"
"github.com/metazla/meta-core/internal/events" "github.com/metazla/meta-core/internal/events"
"github.com/metazla/meta-core/internal/identity"
"github.com/metazla/meta-core/internal/leader" "github.com/metazla/meta-core/internal/leader"
"github.com/metazla/meta-core/internal/mounts" "github.com/metazla/meta-core/internal/mounts"
"github.com/metazla/meta-core/internal/schema"
"github.com/metazla/meta-core/internal/storage" "github.com/metazla/meta-core/internal/storage"
"github.com/metazla/meta-core/internal/watcher" "github.com/metazla/meta-core/internal/watcher"
"github.com/metazla/meta-core/internal/watchers" "github.com/metazla/meta-core/internal/watchers"
@@ -29,6 +31,7 @@ type Server struct {
mountsManager *mounts.Manager mountsManager *mounts.Manager
mountsHandlers *mounts.Handlers mountsHandlers *mounts.Handlers
mountPoller *mounts.Poller mountPoller *mounts.Poller
mountStatsPoller *mounts.StatsPoller
watcherDispatcher *watcher.Dispatcher watcherDispatcher *watcher.Dispatcher
fileWatcher *watcher.Watcher fileWatcher *watcher.Watcher
watcherHandlers *watcher.Handlers watcherHandlers *watcher.Handlers
@@ -36,7 +39,9 @@ type Server struct {
watchersPoller *watchers.Poller watchersPoller *watchers.Poller
watchersHandlers *watchers.Handlers watchersHandlers *watchers.Handlers
metaPublisher *events.MetaPublisher metaPublisher *events.MetaPublisher
schemaIndexer *schema.Indexer
webdavHandler *webdav.Handler webdavHandler *webdav.Handler
search *searchIndex
router *mux.Router router *mux.Router
server *http.Server server *http.Server
} }
@@ -55,6 +60,7 @@ func NewServer(
discovery: disc, discovery: disc,
cleaner: cleaner, cleaner: cleaner,
storage: stor, storage: stor,
search: &searchIndex{},
router: mux.NewRouter(), router: mux.NewRouter(),
} }
@@ -71,6 +77,11 @@ func NewServer(
log.Printf("[API] Warning: failed to initialize file watcher: %v", err) log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
} else { } else {
s.fileWatcher = fileWatcher s.fileWatcher = fileWatcher
// Wire storage so the watcher can persist (path,size,mtime,cid)
// tuples to Redis as it computes midhashes.
if stor != nil {
fileWatcher.SetStorageClient(stor)
}
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher) s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
// Create scan function from watcher for mounts // Create scan function from watcher for mounts
scanFunc = fileWatcher.ScanMountPath scanFunc = fileWatcher.ScanMountPath
@@ -88,6 +99,20 @@ func NewServer(
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher) s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
// Pass republish callback to handlers (handles nil metaPublisher case) // Pass republish callback to handlers (handles nil metaPublisher case)
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata) s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
// Hydrate the in-memory state registry from Redis tuples so
// the first scan tick after boot can skip ComputeMidHash256
// for files whose (size, mtime) match.
configs := watchersManager.List()
roots := make([]watcher.WatcherRoot, 0, len(configs))
for _, c := range configs {
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
}
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
} else {
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
}
} }
} }
} }
@@ -103,11 +128,25 @@ func NewServer(
// Create and configure mount poller // Create and configure mount poller
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc) s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
mountsManager.SetPoller(s.mountPoller) mountsManager.SetPoller(s.mountPoller)
// IO stats poller — queries the local rclone RC API every few seconds
// and exposes live throughput per mount via the same status JSON.
// Stats are daemon-global (rclone limitation); see stats_poller.go.
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
mountsManager.SetStatsPoller(s.mountStatsPoller)
} }
// Initialize WebDAV handler (caching is handled by nginx proxy_cache) // Initialize WebDAV handler (caching is handled by nginx proxy_cache)
s.webdavHandler = webdav.NewHandler(cfg) s.webdavHandler = webdav.NewHandler(cfg)
// One-time migration: fold a pre-multi-account identity.json into the
// per-account keystore so existing single-identity hosts keep their uid.
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
log.Printf("[API] Warning: identity migration failed: %v", err)
} else if migrated {
log.Printf("[API] Migrated legacy identity.json into multi-account keystore")
}
s.setupRoutes() s.setupRoutes()
return s return s
} }
@@ -126,15 +165,40 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST") s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST") s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST") s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
// User Data Layer signing identity (see internal/identity). Auth-gated
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
// Snapshot (export / import / wipe)
s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET") s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT") s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET") s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT") s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE") s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
// Schema inference
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
// KV Browser API routes // KV Browser API routes
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET") s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET") s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
s.router.HandleFunc("/api/kv/tree", s.handleKVTree).Methods("GET")
s.router.HandleFunc("/api/kv/search", s.handleKVSearch).Methods("GET")
s.router.HandleFunc("/api/kv/find", s.handleKVFind).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValueGet).Methods("GET")
s.router.HandleFunc("/api/kv/value", s.handleKVValuePut).Methods("PUT")
s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET") s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
// Metadata operations - base endpoints // Metadata operations - base endpoints
@@ -157,9 +221,27 @@ func (s *Server) setupRoutes() {
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD") s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
// File operations (by CID) // File operations (by CID)
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET") s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST") s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
// Public metadata-by-CID resolver. Reverse-index lookup → full document.
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
// without going through Authelia.
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
// Admin: reunify stranded midhash-rooted entries into their UUID roots
// (the dual-root pattern caused by historical writes that bypassed CID
// resolution).
s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
// SSE event streams. Mediated mirror of the Redis Streams; consumers
// reach them over HTTP instead of touching Redis directly. Auth-bypass
// applies but exposure should be inside-only (do NOT proxy through
// Caddy publicly). See docs/api-mediated-access.md "Auth".
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
// Service discovery // Service discovery
// Both /services and /api/services are supported for consistency with other services // Both /services and /api/services are supported for consistency with other services
s.router.HandleFunc("/services", s.handleListServices).Methods("GET") s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
@@ -170,29 +252,22 @@ func (s *Server) setupRoutes() {
// Mount management routes (if manager initialized) // Mount management routes (if manager initialized)
if s.mountsHandlers != nil { if s.mountsHandlers != nil {
s.mountsHandlers.RegisterRoutes(s.router) s.mountsHandlers.RegisterRoutes(s.router)
log.Println("[API] Mount management routes registered at /api/mounts/*")
} }
log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
log.Println("[API] KV Browser routes registered at /api/kv/*")
// File watcher routes (if watcher initialized) // File watcher routes (if watcher initialized)
if s.watcherHandlers != nil { if s.watcherHandlers != nil {
s.watcherHandlers.RegisterRoutes(s.router) s.watcherHandlers.RegisterRoutes(s.router)
log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
} }
// Watchers management routes (if watchers initialized) // Watchers management routes (if watchers initialized)
if s.watchersHandlers != nil { if s.watchersHandlers != nil {
s.watchersHandlers.RegisterRoutes(s.router) s.watchersHandlers.RegisterRoutes(s.router)
log.Println("[API] Watchers management routes registered at /api/watchers/*")
} }
// WebDAV handler (caching is handled by nginx proxy_cache) // WebDAV handler (caching is handled by nginx proxy_cache)
if s.webdavHandler != nil { if s.webdavHandler != nil {
s.router.PathPrefix("/webdav/").Handler(s.webdavHandler) s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
s.router.PathPrefix("/webdav").Handler(s.webdavHandler) s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
log.Println("[API] WebDAV routes registered at /webdav/*")
} }
// Add middleware // Add middleware
@@ -234,6 +309,20 @@ func (s *Server) Start() error {
} }
} }
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
// mounts), so always-on rather than gated by config.
if s.mountStatsPoller != nil {
s.mountStatsPoller.Start()
}
// Build + periodically refresh the in-memory search index so
// POST /api/metadata/search matches in memory instead of a Redis
// round-trip per record (see search_index.go).
if s.storage != nil {
go s.search.runRefreshLoop(s.storage)
}
// Start meta publisher (if storage is connected) // Start meta publisher (if storage is connected)
// Publishes keyspace notifications to meta:events stream // Publishes keyspace notifications to meta:events stream
if s.storage != nil && s.storage.IsConnected() { if s.storage != nil && s.storage.IsConnected() {
@@ -241,6 +330,15 @@ func (s *Server) Start() error {
if err := s.metaPublisher.Start(); err != nil { if err := s.metaPublisher.Start(); err != nil {
log.Printf("[API] Warning: failed to start meta publisher: %v", err) log.Printf("[API] Warning: failed to start meta publisher: %v", err)
} else { } else {
// Schema indexer consumes the same stream to derive a live
// per-field schema (type/hint/breakdown). Start it before the
// republish flood so it captures the bootstrap events.
s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
if err := s.schemaIndexer.Start(); err != nil {
log.Printf("[API] Warning: failed to start schema indexer: %v", err)
s.schemaIndexer = nil
}
// Republish events for all existing metadata // Republish events for all existing metadata
go func() { go func() {
count, err := s.RepublishMetadata() count, err := s.RepublishMetadata()
@@ -272,6 +370,18 @@ func (s *Server) Stop() error {
} }
} }
// Stop IO stats poller
if s.mountStatsPoller != nil {
s.mountStatsPoller.Stop()
}
// Stop schema indexer before the publisher so the consumer drains cleanly
if s.schemaIndexer != nil {
if err := s.schemaIndexer.Stop(); err != nil {
log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
}
}
// Stop meta publisher // Stop meta publisher
if s.metaPublisher != nil { if s.metaPublisher != nil {
if err := s.metaPublisher.Stop(); err != nil { if err := s.metaPublisher.Stop(); err != nil {
@@ -313,12 +423,49 @@ func (s *Server) RepublishMetadata() (int, error) {
return s.metaPublisher.RepublishAllMetadata(republishFunc) return s.metaPublisher.RepublishAllMetadata(republishFunc)
} }
// loggingMiddleware logs all requests // statusRecorder captures the response status so loggingMiddleware can stay
// silent on success and only report failures. It forwards Flush/Unwrap so SSE
// and other streaming handlers keep working through the wrapper.
type statusRecorder struct {
http.ResponseWriter
status int
written bool
}
func (r *statusRecorder) WriteHeader(code int) {
if !r.written {
r.status = code
r.written = true
}
r.ResponseWriter.WriteHeader(code)
}
func (r *statusRecorder) Write(b []byte) (int, error) {
if !r.written {
r.status = http.StatusOK
r.written = true
}
return r.ResponseWriter.Write(b)
}
func (r *statusRecorder) Flush() {
if f, ok := r.ResponseWriter.(http.Flusher); ok {
f.Flush()
}
}
func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
// loggingMiddleware logs a request only when it fails (status >= 400), keeping
// the nominal case quiet. Successful requests are not logged.
func loggingMiddleware(next http.Handler) http.Handler { func loggingMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
start := time.Now() start := time.Now()
next.ServeHTTP(w, r) rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
log.Printf("[API] %s %s %s", r.Method, r.URL.Path, time.Since(start)) next.ServeHTTP(rec, r)
if rec.status >= 400 {
log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
}
}) })
} }
+167
View File
@@ -0,0 +1,167 @@
package api
import (
"bytes"
"fmt"
"io"
"net/http"
"path/filepath"
"strings"
"time"
"github.com/metazla/meta-core/internal/snapshot"
)
// Bound the upload size so a malicious client can't OOM the server.
// 2 GiB is well above any realistic export.
const maxImportBytes = 2 << 30
// pluginOutputDir is where plugin-produced artifacts live on disk
// (subtitles, posters, etc). Mounted from the host into the meta-core
// container at $FILES_PATH/plugin.
func (s *Server) pluginOutputDir() string {
return filepath.Join(s.config.FilesPath, "plugin")
}
// handleSnapshotExport streams a ZIP snapshot of all metadata.
// GET /api/snapshot/export?include=hash-index,plugin-output
func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// The server has a 30s WriteTimeout (server.go:222). Long exports
// blow past that, leaving a truncated ZIP without a central directory.
// Clear the deadline for this handler so the export can run as long as
// needed. ResponseController is the standard way to do this.
rc := http.NewResponseController(w)
_ = rc.SetWriteDeadline(time.Time{})
include := splitCSVParam(r.URL.Query().Get("include"))
opts := snapshot.ExportOptions{
IncludePluginOutput: include["plugin-output"],
PluginOutputDir: s.pluginOutputDir(),
}
filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
w.Header().Set("Content-Type", "application/zip")
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
// We stream — no Content-Length.
exporter := snapshot.NewExporter(s.storage)
if _, err := exporter.Export(w, opts); err != nil {
// Headers may already be sent; log and bail.
// Client will see a truncated zip, which is the best signal we can give.
fmt.Printf("[Snapshot] export failed: %v\n", err)
return
}
}
// handleSnapshotImport applies an uploaded ZIP snapshot.
// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
// when the snapshot is large. Clear both for this handler.
rc := http.NewResponseController(w)
_ = rc.SetReadDeadline(time.Time{})
_ = rc.SetWriteDeadline(time.Time{})
q := r.URL.Query()
opts := snapshot.ImportOptions{
Mode: snapshot.Mode(q.Get("mode")),
Conflict: snapshot.Conflict(q.Get("conflict")),
DryRun: strings.EqualFold(q.Get("dry_run"), "true"),
PluginOutputDir: s.pluginOutputDir(),
}
if opts.Mode == "" {
opts.Mode = snapshot.ModeMerge
}
if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
opts.Conflict = snapshot.ConflictSource
}
body, err := readImportBody(r)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
importer := snapshot.NewImporter(s.storage)
result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
if err != nil {
writeError(w, http.StatusBadRequest, err.Error())
return
}
writeJSON(w, http.StatusOK, result)
}
// handleSnapshotWipe clears Redis metadata.
// POST /api/snapshot/wipe?scope=metadata
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
if !s.storage.IsConnected() {
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
scopes := splitCSVParam(r.URL.Query().Get("scope"))
if len(scopes) == 0 {
writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
return
}
scope := snapshot.WipeScope{
Metadata: scopes["metadata"],
}
if !scope.Metadata {
writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
return
}
wiper := snapshot.NewWiper(s.storage)
res, err := wiper.Wipe(scope)
if err != nil {
writeError(w, http.StatusInternalServerError, err.Error())
return
}
writeJSON(w, http.StatusOK, res)
}
// readImportBody accepts either a raw application/zip body or a multipart
// upload with a field named "snapshot". Buffers up to maxImportBytes.
func readImportBody(r *http.Request) ([]byte, error) {
r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
ct := r.Header.Get("Content-Type")
if strings.HasPrefix(ct, "multipart/form-data") {
if err := r.ParseMultipartForm(32 << 20); err != nil {
return nil, fmt.Errorf("parse multipart: %w", err)
}
f, _, err := r.FormFile("snapshot")
if err != nil {
return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
}
defer f.Close()
return io.ReadAll(f)
}
return io.ReadAll(r.Body)
}
// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
func splitCSVParam(v string) map[string]bool {
out := map[string]bool{}
for _, part := range strings.Split(v, ",") {
p := strings.TrimSpace(part)
if p != "" {
out[p] = true
}
}
return out
}
+280
View File
@@ -0,0 +1,280 @@
package api
import (
"context"
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"strconv"
"strings"
"time"
"github.com/redis/go-redis/v9"
)
// SSE event stream endpoints for the file:events and meta:events Redis Streams.
//
// Wire contract (see docs/api-mediated-access.md):
// - One SSE event per Redis Stream entry.
// - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
// MUST treat it as a black-box string they echo back via Last-Event-ID.
// - `event:` is the `type` field from the underlying entry.
// - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
// - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
// - On reconnect with Last-Event-ID, resume from the next entry. If the
// cursor has been trimmed by retention, emit one `event: gap` payload
// before resuming from the oldest available entry.
const (
sseHeartbeatInterval = 30 * time.Second
// sseReadBlock controls how long each XREAD blocks before returning.
// Much shorter than sseHeartbeatInterval so the handler stays responsive
// to client disconnects and the heartbeat check runs frequently enough
// that the keep-alive fires within a few seconds of its nominal cadence.
sseReadBlock = 2 * time.Second
// sseReadCount caps how many entries one XREAD returns. Small enough
// to keep per-flush memory low, large enough to drain bursts cheaply.
sseReadCount = 100
)
// handleEventsFiles streams the file:events Redis Stream as SSE.
func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "file:events")
}
// handleEventsMeta streams the meta:events Redis Stream as SSE.
func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
s.streamSSE(w, r, "meta:events")
}
// streamSSE is the shared SSE pump shared by both endpoints. One handler per
// stream — there is no fan-out; each subscriber holds its own XREAD loop.
// That's fine at our scale (≤20 subscribers; one per consumer service).
func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
if !s.storage.IsConnected() {
w.Header().Set("Retry-After", "5")
writeError(w, http.StatusServiceUnavailable, "storage not connected")
return
}
flusher, ok := w.(http.Flusher)
if !ok {
writeError(w, http.StatusInternalServerError, "streaming unsupported")
return
}
// The server-wide WriteTimeout (30s on Server.server in server.go) is
// fine for ordinary handlers but lethal for SSE — it would close every
// connection at exactly 30s, racing the heartbeat. Disable it
// per-request via http.NewResponseController. SSE endpoints never need
// a write deadline; the client disconnect path already cleans up.
rc := http.NewResponseController(w)
if err := rc.SetWriteDeadline(time.Time{}); err != nil {
// Older transports (or nested response writers) may not support
// this. Log and continue — the 30s cap will still apply, but the
// handler is still useful for short-lived subscriptions.
log.Printf("[SSE %s] could not disable write deadline: %v", stream, err)
}
// SSE response headers. X-Accel-Buffering disables nginx response
// buffering; the hash-lock sidecar uses nginx, so this matters.
w.Header().Set("Content-Type", "text/event-stream")
w.Header().Set("Cache-Control", "no-cache")
w.Header().Set("Connection", "keep-alive")
w.Header().Set("X-Accel-Buffering", "no")
w.WriteHeader(http.StatusOK)
flusher.Flush()
client := s.storage.GetRedisClient()
if client == nil {
// Storage was just disconnected — close cleanly.
return
}
ctx := r.Context()
cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
lastWrite := time.Now()
for {
if err := ctx.Err(); err != nil {
return
}
// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
// context's deadline isn't propagated into XREAD's BLOCK arg —
// that's a poll-style timeout on Redis itself — so we keep BLOCK
// short.
readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
result, err := client.XRead(readCtx, &redis.XReadArgs{
Streams: []string{stream, cursor},
Block: sseReadBlock,
Count: sseReadCount,
}).Result()
cancel()
if err != nil {
// redis.Nil = BLOCK timeout with no entries. Anything else
// while the client is still connected is a real failure.
if errors.Is(err, redis.Nil) || isContextTimeout(err) {
// fall through to heartbeat check
} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
return
} else {
log.Printf("[SSE %s] XREAD error: %v", stream, err)
return
}
}
for _, str := range result {
for _, msg := range str.Messages {
writeSSEEntry(w, flusher, msg)
cursor = msg.ID
lastWrite = time.Now()
}
}
// Heartbeat if quiet for >= the interval. Comment line; SSE
// clients ignore it per spec.
if time.Since(lastWrite) >= sseHeartbeatInterval {
if _, err := fmt.Fprint(w, ":keep-alive\n\n"); err != nil {
return
}
flusher.Flush()
lastWrite = time.Now()
}
}
}
// resolveStartCursor decides where to start reading.
// - No Last-Event-ID → start at "$" (only new entries).
// - Last-Event-ID provided AND still in stream → resume from it.
// - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
// and resume from the oldest available entry's predecessor "0", so the
// next XREAD picks up everything still in the stream.
//
// The "still in stream" check is cheap: one XRANGE for the first entry.
func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
if lastID == "" {
return "$"
}
// One-entry XRANGE to find the oldest ID currently in the stream.
xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
defer cancel()
entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
if err != nil {
// Treat any error as "stream not available right now" and resume
// from the cursor — XREAD will either find it or block until new
// entries arrive.
log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
return lastID
}
if len(entries) == 0 {
// Stream is empty. Cursor may be from a previous incarnation; safest
// is to resume from the cursor — XREAD against an empty stream
// returns nothing and blocks until new entries arrive.
return lastID
}
oldest := entries[0].ID
if compareStreamIDs(lastID, oldest) >= 0 {
// Cursor is at or past the oldest — fine to resume.
return lastID
}
// Cursor was trimmed. Emit a gap event so consumers that care can react.
gap := map[string]string{
"requested": lastID,
"resumeFrom": oldest,
"reason": "retention",
}
payload, _ := json.Marshal(gap)
fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
flusher.Flush()
// Return "0" so the next XREAD picks up entries starting from the
// stream's true minimum — including `oldest` itself.
return "0"
}
// writeSSEEntry serialises one Redis Stream entry as an SSE event.
//
// `event:` comes from the entry's `type` field (no prefix munging). `data:`
// is a JSON object containing all OTHER fields. Field values from the Redis
// stream are interface{} (typically strings); we encode whatever the stream
// holds.
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
eventType := "message"
data := make(map[string]interface{}, len(msg.Values))
for k, v := range msg.Values {
if k == "type" {
if s, ok := v.(string); ok {
eventType = s
continue
}
}
data[k] = v
}
payload, err := json.Marshal(data)
if err != nil {
log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
return
}
// Spec: each event ends with a blank line. id/event/data lines are
// terminated with single \n.
fmt.Fprintf(w, "id: %s\n", msg.ID)
fmt.Fprintf(w, "event: %s\n", eventType)
fmt.Fprintf(w, "data: %s\n\n", string(payload))
flusher.Flush()
}
// compareStreamIDs compares two Redis Stream IDs of the form "<ms>-<seq>".
// Returns -1/0/+1. Missing or malformed IDs sort as zero parts.
func compareStreamIDs(a, b string) int {
aMs, aSeq := parseStreamID(a)
bMs, bSeq := parseStreamID(b)
switch {
case aMs < bMs:
return -1
case aMs > bMs:
return 1
case aSeq < bSeq:
return -1
case aSeq > bSeq:
return 1
default:
return 0
}
}
func parseStreamID(id string) (int64, int64) {
parts := strings.SplitN(id, "-", 2)
ms, _ := strconv.ParseInt(parts[0], 10, 64)
var seq int64
if len(parts) == 2 {
seq, _ = strconv.ParseInt(parts[1], 10, 64)
}
return ms, seq
}
// isContextTimeout returns true for the context deadline error wrapped by
// go-redis when XREAD's read context fires. We treat it the same as
// redis.Nil — i.e. an empty poll cycle, not a fatal error.
func isContextTimeout(err error) bool {
if err == nil {
return false
}
if errors.Is(err, context.DeadlineExceeded) {
return true
}
// go-redis sometimes wraps this in its own error type. String match is
// a safety net.
return strings.Contains(err.Error(), "context deadline exceeded")
}
+136
View File
@@ -0,0 +1,136 @@
// Package cid centralizes CID-format helpers: decoding a multibase-base32
// CIDv1 into its multicodec parts, ranking by utility for external
// presentation, and picking which CID is the canonical externally-visible
// identifier of a file.
//
// Sibling CIDs are stored as a bare-CID key-set (`cids/<cid> = "true"`); a
// CID is self-describing — its algorithm is recoverable from the CIDv1
// codec / multihash code, so there is no `<algorithm>:` prefix and no
// per-algorithm field name. Ranking and selection parse the multicodec
// rather than a string prefix.
package cid
import (
"encoding/base32"
"encoding/binary"
"fmt"
"strings"
)
// Multicodec / multihash codes used for ranking. Source of truth for the
// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
// METADATA_KEYS.md §2.
const (
CodecRaw = 0x55 // raw — IPFS content-addressed block
CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block
CodeSha1 = 0x11 // multihash sha1
CodeSha256 = 0x12 // multihash sha2-256
CodeSha3_384 = 0x15 // multihash sha3-384
CodeSha3_256 = 0x16 // multihash sha3-256
CodeMd5 = 0xd5 // multihash md5
CodeCrc32 = 0x0132 // multihash crc32
CodeMidhash256 = 0x1000 // custom — SHA-256(size‖middle 1MB)
CodeBtPiecesRoot = 0xb702 // bittorrent v2 pieces root
CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52)
CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID
CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID
)
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content
// codec and multihash function code. It is the inverse of the encoder in
// internal/watcher/midhash.go. Returns an error for anything that isn't a
// CIDv1 in base32lower (e.g. a CIDv0 "Qm…" or a non-CID string).
func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
if len(cidStr) < 2 || cidStr[0] != 'b' {
return 0, 0, fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
}
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
if err != nil {
return 0, 0, fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
}
version, n := binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad version varint", cidStr)
}
raw = raw[n:]
if version != 1 {
return 0, 0, fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
}
codec, n = binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad codec varint", cidStr)
}
raw = raw[n:]
mhCode, n = binary.Uvarint(raw)
if n <= 0 {
return 0, 0, fmt.Errorf("cid %q: bad multihash-code varint", cidStr)
}
return codec, mhCode, nil
}
// Rank returns a score for a bare CID. Higher = more preferred as the
// public-facing "canonical" identifier for a file. Unknown / undecodable
// CIDs rank 0 so a new digest type can't accidentally outrank an
// established one until it's classified here.
//
// We rank by the **multihash code** (the algorithm), with the custom
// BitTorrent CID codecs checked alongside it. This is deliberate: MetaMesh's
// two CID encoders disagree on the *content codec* — meta-hash/the watcher
// set codec == algorithm code, while the fullhash plugin and gateway wrap
// every digest with the raw codec (0x55) and put the algorithm in the
// multihash code. The multihash code is the algorithm in all of them, so it
// is the only reliable discriminator. (The BT info hash is given a distinct
// multihash code — btih-v2 0x10B7 — so it can't be confused with a plain
// sha2-256, see METADATA_KEYS.md §14.13.)
//
// Ordering rationale (utility-weighted, NOT cryptographic-strength):
// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
// block of the file IS its sha2-256, so they share this tier).
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
// swarm interop, not strength.
// - midhash256 is the floor among real options: internally meaningful,
// externally invisible.
// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
func Rank(cidStr string) int {
codec, mh, err := Decode(cidStr)
if err != nil {
return 0
}
if codec == CodecDagPB {
return 40
}
// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
// multihash code, in either encoder convention.
if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
return 20
}
switch mh {
case CodeSha256, CodeSha3_256, CodeSha3_384:
return 30
case CodeMidhash256:
return 10
default:
// md5, crc32, plain sha1, pieces-root, unknown.
return 0
}
}
// Better picks the higher-ranked of two CIDs. On ties, returns the
// lexicographically smaller one — gives the reconciler a deterministic
// answer so the chosen canonical doesn't churn when peers add the same
// digest under a different ordering.
func Better(a, b string) string {
ra, rb := Rank(a), Rank(b)
if ra > rb {
return a
}
if rb > ra {
return b
}
if a <= b {
return a
}
return b
}
+130
View File
@@ -0,0 +1,130 @@
package cid
import (
"encoding/base32"
"encoding/binary"
"strings"
"testing"
)
// buildCID assembles a multibase-base32 CIDv1 from a content codec, a
// multihash code, and a digest length (digest bytes are zero — only the
// header is under test). Mirrors the encoder in watcher/midhash.go and the
// TS CID.createV1(code, create(code, digest)) path.
func buildCID(codec, mhCode uint64, digestLen int) string {
var b []byte
b = binary.AppendUvarint(b, 1) // CIDv1
b = binary.AppendUvarint(b, codec)
b = binary.AppendUvarint(b, mhCode)
b = binary.AppendUvarint(b, uint64(digestLen))
b = append(b, make([]byte, digestLen)...)
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
return "b" + strings.ToLower(enc)
}
// Real CIDs produced by @metazla/meta-hash (from its main.test.ts). For
// digest CIDs meta-hash sets codec == multihash code == algorithm code.
const (
cidCrc32 = "bagzafmqcathsbnrk"
cidMd5 = "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa"
cidSha1 = "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra"
cidSha256 = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
cidSha3256 = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
cidSha3384 = "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga"
)
func TestDecode_Fixtures(t *testing.T) {
cases := []struct {
name string
cid string
wantMh uint64
}{
{"crc32", cidCrc32, CodeCrc32},
{"md5", cidMd5, CodeMd5},
{"sha1", cidSha1, CodeSha1},
{"sha2-256", cidSha256, CodeSha256},
{"sha3-256", cidSha3256, CodeSha3_256},
{"sha3-384", cidSha3384, CodeSha3_384},
}
for _, tc := range cases {
codec, mh, err := Decode(tc.cid)
if err != nil {
t.Errorf("Decode(%s %q) error: %v", tc.name, tc.cid, err)
continue
}
// meta-hash digest CIDs use codec == multihash code.
if codec != tc.wantMh || mh != tc.wantMh {
t.Errorf("Decode(%s) = codec 0x%x mh 0x%x, want both 0x%x", tc.name, codec, mh, tc.wantMh)
}
}
}
func TestDecode_Rejects(t *testing.T) {
for _, in := range []string{"", "Qmabc", "x", "bzzzz!!"} {
if _, _, err := Decode(in); err == nil {
t.Errorf("Decode(%q) expected error, got nil", in)
}
}
}
func TestRank_Tiers(t *testing.T) {
ipfsDagPB := buildCID(CodecDagPB, CodeSha256, 32) // chunked IPFS file
rawSha256 := buildCID(CodecRaw, CodeSha256, 32) // fullhash-style sha2-256
btihV2meta := buildCID(CodeBtihV2, CodeBtihV2, 32) // meta-hash btih-v2 (codec==mh)
btihV2raw := buildCID(CodecRaw, CodeBtihV2, 32) // fullhash btih-v2 (raw codec)
btV1File := buildCID(CodeBtihV1File, CodeBtihV1File, 20) // gateway per-file btih v1
midhashRaw := buildCID(CodecRaw, CodeMidhash256, 32) // fullhash midhash (raw codec)
cases := []struct {
name string
cid string
want int
}{
{"ipfs-dagpb", ipfsDagPB, 40},
{"sha2-256(fixture, codec==mh)", cidSha256, 30},
{"sha2-256(raw codec)", rawSha256, 30},
{"sha3-256", cidSha3256, 30},
{"sha3-384", cidSha3384, 30},
{"btih-v2(codec==mh)", btihV2meta, 20},
{"btih-v2(raw codec)", btihV2raw, 20},
{"btih-v1-file", btV1File, 20},
{"midhash256(raw codec)", midhashRaw, 10},
{"sha1", cidSha1, 0},
{"md5", cidMd5, 0},
{"crc32", cidCrc32, 0},
{"garbage", "not-a-cid", 0},
}
for _, tc := range cases {
if got := Rank(tc.cid); got != tc.want {
t.Errorf("Rank(%s) = %d, want %d", tc.name, got, tc.want)
}
}
}
func TestRank_StrictOrdering(t *testing.T) {
ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
sha := Rank(cidSha256)
btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
unknown := Rank(cidMd5)
if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
ipfs, sha, btih, mid, unknown)
}
}
func TestBetter(t *testing.T) {
dagpb := buildCID(CodecDagPB, CodeSha256, 32)
// dag-pb (IPFS) beats a sha2-256 digest, regardless of arg order.
if got := Better(dagpb, cidSha256); got != dagpb {
t.Errorf("Better(dagpb, sha256) = %q, want dagpb", got)
}
if got := Better(cidSha256, dagpb); got != dagpb {
t.Errorf("Better(sha256, dagpb) = %q, want dagpb (order shouldn't matter)", got)
}
// Tie → lexicographically smaller wins, deterministic.
a, b := "bafyaaa", "bafyaab"
if Better(a, b) != a || Better(b, a) != a {
t.Errorf("tie-break not deterministic for %q vs %q", a, b)
}
}
+40
View File
@@ -17,6 +17,7 @@ type Config struct {
ServiceVersion string // Service version (default: "1.0.0") ServiceVersion string // Service version (default: "1.0.0")
APIPort int // Service HTTP port (for leader info) APIPort int // Service HTTP port (for leader info)
BaseURL string // Base URL for stable service discovery BaseURL string // Base URL for stable service discovery
PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
// Redis configuration // Redis configuration
RedisPort int // Redis port (default: 6379) RedisPort int // Redis port (default: 6379)
@@ -50,6 +51,7 @@ func Load() *Config {
ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"), ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"),
APIPort: getEnvInt("API_PORT", 8180), APIPort: getEnvInt("API_PORT", 8180),
BaseURL: getEnv("BASE_URL", ""), BaseURL: getEnv("BASE_URL", ""),
PublicURL: getEnv("META_CORE_PUBLIC_URL", ""),
RedisPort: getEnvInt("REDIS_PORT", 6379), RedisPort: getEnvInt("REDIS_PORT", 6379),
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000), HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
HTTPHost: getEnv("META_CORE_HTTP_HOST", "127.0.0.1"), HTTPHost: getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
@@ -104,6 +106,27 @@ func (c *Config) WatchersFilePath() string {
return c.MetaCorePath + "/watchers.json" return c.MetaCorePath + "/watchers.json"
} }
// IdentityDir returns the directory holding the User Data Layer signing
// identity (created by the dashboard's Identity menu — never auto-generated
// at startup).
func (c *Config) IdentityDir() string {
return c.MetaCorePath + "/identity"
}
// IdentityFilePath returns the path to the legacy single-identity JSON. It is
// only read once, by the multi-account migration on startup; new accounts live
// under IdentityAccountsDir.
func (c *Config) IdentityFilePath() string {
return c.IdentityDir() + "/identity.json"
}
// IdentityAccountsDir returns the directory holding one JSON file per account
// in the multi-account keystore ({uid}.json). meta-core is a pure multi-key
// signer — display name / avatar live in the User Data Layer, not here.
func (c *Config) IdentityAccountsDir() string {
return c.IdentityDir() + "/accounts"
}
func getEnv(key, defaultValue string) string { func getEnv(key, defaultValue string) string {
if value := os.Getenv(key); value != "" { if value := os.Getenv(key); value != "" {
return value return value
@@ -132,3 +155,20 @@ func getEnvBool(key string, defaultValue bool) bool {
func (c *Config) DefaultWatchPath() string { func (c *Config) DefaultWatchPath() string {
return c.FilesPath + "/watch" return c.FilesPath + "/watch"
} }
// DefaultWatcherPaths returns every directory that should be watched out of
// the box on a fresh install. The watcher seeds these on first start.
//
// - /files/watch — user-curated media. The main library.
// - /files/plugin — plugin output (TMDB posters/backdrops, extracted
// subtitles, etc.). Indexed so each artefact gets a
// midhash256 alias and CID-based references from
// other files (e.g. `backdrop=<cid>`) resolve through
// the standard reverse index instead of needing a
// path companion field.
func (c *Config) DefaultWatcherPaths() []string {
return []string{
c.FilesPath + "/watch",
c.FilesPath + "/plugin",
}
}
+6 -2
View File
@@ -87,9 +87,13 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
hostname, _ := os.Hostname() hostname, _ := os.Hostname()
ip := getLocalIP() ip := getLocalIP()
// Use BASE_URL for external access, fall back to internal IP // Browser-facing nav URL: prefer META_CORE_PUBLIC_URL (the reachable URL
// when there's no Caddy perimeter in front — e.g. a debug-direct port),
// then BASE_URL (the Caddy perimeter URL), then the internal IP.
var baseUrl string var baseUrl string
if s.config.BaseURL != "" { if s.config.PublicURL != "" {
baseUrl = s.config.PublicURL
} else if s.config.BaseURL != "" {
baseUrl = s.config.BaseURL baseUrl = s.config.BaseURL
} else { } else {
baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort) baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
+19 -12
View File
@@ -14,6 +14,12 @@ import (
const ( const (
// MetaEventsStream is the Redis stream name for metadata events // MetaEventsStream is the Redis stream name for metadata events
MetaEventsStream = "meta:events" MetaEventsStream = "meta:events"
// MetaEventsMaxLen is the approximate cap on stream length. Bounded
// retention is a contract requirement for the SSE wire — see
// docs/api-mediated-access.md "Backing-store retention and lifecycle".
// Cheap (~) trim keeps growth bounded without serialising writes.
MetaEventsMaxLen = 100_000
) )
// MetaPublisher subscribes to Redis keyspace notifications // MetaPublisher subscribes to Redis keyspace notifications
@@ -37,7 +43,12 @@ func NewMetaPublisher(client *redis.Client) *MetaPublisher {
} }
} }
// Start begins subscribing to keyspace notifications // Start begins subscribing to keyspace notifications.
//
// The stream is NOT truncated on startup — the SSE wire contract requires
// Last-Event-ID resumption to survive a meta-core restart, which means
// entries must persist across restart. Growth is bounded instead by
// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
func (p *MetaPublisher) Start() error { func (p *MetaPublisher) Start() error {
p.mu.Lock() p.mu.Lock()
if p.running { if p.running {
@@ -47,16 +58,6 @@ func (p *MetaPublisher) Start() error {
p.running = true p.running = true
p.mu.Unlock() p.mu.Unlock()
// Clear the stream on startup - consumers will re-bootstrap from Redis keys
// This ensures the stream only contains events from the current session
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
if err := p.client.Del(ctx, MetaEventsStream).Err(); err != nil {
log.Printf("[MetaPublisher] Warning: failed to clear stream on startup: %v", err)
} else {
log.Printf("[MetaPublisher] Cleared %s stream for fresh start", MetaEventsStream)
}
p.wg.Add(1) p.wg.Add(1)
go p.subscribeLoop() go p.subscribeLoop()
@@ -133,9 +134,13 @@ func (p *MetaPublisher) publishEvent(channel, operation string) {
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()), "ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
} }
// Add to stream (no MaxLen - stream is cleared on meta-core restart) // Approximate MAXLEN trim keeps the stream bounded across restarts
// (see Start). Cap is intentionally generous; most consumers reconnect
// well within retention.
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{ _, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
Stream: MetaEventsStream, Stream: MetaEventsStream,
MaxLen: MetaEventsMaxLen,
Approx: true,
Values: fields, Values: fields,
}).Result() }).Result()
@@ -154,6 +159,8 @@ func (p *MetaPublisher) publishEventDirect(operation, key string) {
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{ _, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
Stream: MetaEventsStream, Stream: MetaEventsStream,
MaxLen: MetaEventsMaxLen,
Approx: true,
Values: fields, Values: fields,
}).Result() }).Result()
+341
View File
@@ -0,0 +1,341 @@
// Package identity manages the user-controlled signing identity used by the
// User Data Layer (see docs/project-architecture/user-data-layer.md).
//
// The keypair is secp256k1 (D1: wallet forward-compat). The uid is the
// multibase base58btc encoding of the 33-byte compressed pubkey, prefixed
// with "z" per multibase convention.
//
// Storage is plaintext JSON at {META_CORE_PATH}/identity/identity.json,
// matching the trust model of the rest of /meta-core. Signing happens
// in-process; meta-watch calls POST /api/identity/sign rather than holding
// the private key itself.
package identity
import (
"crypto/rand"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
"sort"
"strings"
"time"
"github.com/decred/dcrd/dcrec/secp256k1/v4"
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
"github.com/mr-tron/base58"
"golang.org/x/crypto/hkdf"
)
const (
CurveSecp256k1 = "secp256k1"
// HKDF parameters for deriving the AEAD key from the private key.
// Stable across versions — changing these is a breaking change for
// encrypted records on disk.
aeadKDFSalt = "metamesh-uds-v1"
aeadKDFInfo = "aead-key"
aeadKeyLen = 32
)
// Identity is the persisted form of a user's keypair.
//
// PrivateKeyHex is the 32-byte secp256k1 scalar, lower-case hex, no prefix.
// PublicKeyHex is the 33-byte compressed pubkey, lower-case hex, no prefix.
type Identity struct {
UID string `json:"uid"`
Curve string `json:"curve"`
PublicKeyHex string `json:"publicKeyHex"`
PrivateKeyHex string `json:"privateKeyHex"`
CreatedAt int64 `json:"createdAt"`
}
// Generate creates a fresh secp256k1 identity.
func Generate() (*Identity, error) {
priv, err := secp256k1.GeneratePrivateKey()
if err != nil {
return nil, fmt.Errorf("generate keypair: %w", err)
}
return identityFromPrivKey(priv), nil
}
// Import parses an externally-provided 32-byte secp256k1 scalar and derives
// the matching pubkey. Accepts hex with or without a "0x" prefix.
func Import(privateKeyHex string) (*Identity, error) {
if len(privateKeyHex) >= 2 && (privateKeyHex[:2] == "0x" || privateKeyHex[:2] == "0X") {
privateKeyHex = privateKeyHex[2:]
}
raw, err := hex.DecodeString(privateKeyHex)
if err != nil {
return nil, fmt.Errorf("invalid hex: %w", err)
}
if len(raw) != 32 {
return nil, fmt.Errorf("private key must be 32 bytes, got %d", len(raw))
}
priv := secp256k1.PrivKeyFromBytes(raw)
// Reject the zero scalar (secp256k1 lib does not reject it on its own).
pubBytes := priv.PubKey().SerializeCompressed()
if isAllZero(raw) || isAllZero(pubBytes) {
return nil, errors.New("private key is the curve identity (zero); refusing")
}
return identityFromPrivKey(priv), nil
}
func identityFromPrivKey(priv *secp256k1.PrivateKey) *Identity {
pubBytes := priv.PubKey().SerializeCompressed()
return &Identity{
UID: uidFromCompressedPubkey(pubBytes),
Curve: CurveSecp256k1,
PublicKeyHex: hex.EncodeToString(pubBytes),
PrivateKeyHex: hex.EncodeToString(priv.Serialize()),
CreatedAt: time.Now().UTC().Unix(),
}
}
// uidFromCompressedPubkey encodes the pubkey as a multibase base58btc
// string ("z" prefix). The encoding is deterministic and matches MetaMesh
// CID conventions.
func uidFromCompressedPubkey(pub []byte) string {
return "z" + base58.Encode(pub)
}
// Load reads the identity file. Returns (nil, nil) if it does not exist —
// callers distinguish "no identity" from real errors.
func Load(path string) (*Identity, error) {
data, err := os.ReadFile(path)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("read identity file: %w", err)
}
var id Identity
if err := json.Unmarshal(data, &id); err != nil {
return nil, fmt.Errorf("parse identity file: %w", err)
}
if id.Curve != CurveSecp256k1 {
return nil, fmt.Errorf("unsupported curve on disk: %q", id.Curve)
}
return &id, nil
}
// Save writes the identity atomically (tmp + rename), matching the pattern
// used by discovery/service.go.
func Save(path string, id *Identity) error {
if id == nil {
return errors.New("nil identity")
}
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
return fmt.Errorf("mkdir identity dir: %w", err)
}
data, err := json.MarshalIndent(id, "", " ")
if err != nil {
return fmt.Errorf("marshal identity: %w", err)
}
tmp := path + ".tmp"
if err := os.WriteFile(tmp, data, 0600); err != nil {
return fmt.Errorf("write tmp identity: %w", err)
}
if err := os.Rename(tmp, path); err != nil {
return fmt.Errorf("rename tmp identity: %w", err)
}
return nil
}
// Delete removes the identity file. Idempotent.
func Delete(path string) error {
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
return err
}
return nil
}
// Exists reports whether an identity file is present.
func Exists(path string) bool {
_, err := os.Stat(path)
return err == nil
}
// --- Multi-account keystore -------------------------------------------------
//
// Accounts live as one file per keypair at {dir}/{uid}.json. The uid is a
// multibase base58btc string, so it is filename-safe; validUID still refuses
// anything that could escape the accounts directory.
// validUID accepts the alphanumeric multibase/base58btc shape we mint for uids
// and rejects path separators or traversal sequences.
func validUID(uid string) bool {
if uid == "" || len(uid) > 128 || uid != filepath.Base(uid) {
return false
}
for _, r := range uid {
switch {
case r >= 'a' && r <= 'z':
case r >= 'A' && r <= 'Z':
case r >= '0' && r <= '9':
default:
return false
}
}
return true
}
func accountFilePath(dir, uid string) (string, error) {
if !validUID(uid) {
return "", fmt.Errorf("invalid uid: %q", uid)
}
return filepath.Join(dir, uid+".json"), nil
}
// SaveAccount writes an identity into the multi-account store. It does not
// check for prior existence — callers wanting append-only semantics check
// ExistsByUID first.
func SaveAccount(dir string, id *Identity) error {
if id == nil {
return errors.New("nil identity")
}
path, err := accountFilePath(dir, id.UID)
if err != nil {
return err
}
return Save(path, id)
}
// LoadByUID reads one account. Returns (nil, nil) when absent.
func LoadByUID(dir, uid string) (*Identity, error) {
path, err := accountFilePath(dir, uid)
if err != nil {
return nil, err
}
return Load(path)
}
// ExistsByUID reports whether an account file exists for uid.
func ExistsByUID(dir, uid string) bool {
path, err := accountFilePath(dir, uid)
if err != nil {
return false
}
return Exists(path)
}
// DeleteByUID removes one account. Idempotent.
func DeleteByUID(dir, uid string) error {
path, err := accountFilePath(dir, uid)
if err != nil {
return err
}
return Delete(path)
}
// List returns every account, sorted by CreatedAt ascending (then UID, for a
// stable order among same-second creations). Returns nil when the directory
// does not exist yet.
func List(dir string) ([]*Identity, error) {
entries, err := os.ReadDir(dir)
if err != nil {
if errors.Is(err, os.ErrNotExist) {
return nil, nil
}
return nil, fmt.Errorf("read accounts dir: %w", err)
}
var out []*Identity
for _, e := range entries {
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
continue
}
id, err := Load(filepath.Join(dir, e.Name()))
if err != nil {
return nil, err
}
if id != nil {
out = append(out, id)
}
}
sort.Slice(out, func(i, j int) bool {
if out[i].CreatedAt != out[j].CreatedAt {
return out[i].CreatedAt < out[j].CreatedAt
}
return out[i].UID < out[j].UID
})
return out, nil
}
// MigrateLegacy folds a pre-multi-account identity.json into the accounts
// store as an entry, then removes the legacy file. Idempotent and a no-op when
// the legacy file is absent. Returns true when a migration ran.
func MigrateLegacy(legacyPath, accountsDir string) (bool, error) {
id, err := Load(legacyPath)
if err != nil {
return false, err
}
if id == nil {
return false, nil
}
// Don't clobber an account already present for this uid.
if !ExistsByUID(accountsDir, id.UID) {
if err := SaveAccount(accountsDir, id); err != nil {
return false, err
}
}
if err := Delete(legacyPath); err != nil {
return false, fmt.Errorf("remove legacy identity after migration: %w", err)
}
return true, nil
}
// Sign produces an ECDSA-secp256k1 signature over SHA-256(bytes). The output
// is DER-encoded — the wire format we commit to for User Data Layer records.
func Sign(id *Identity, bytes []byte) ([]byte, error) {
if id == nil {
return nil, errors.New("no identity loaded")
}
raw, err := hex.DecodeString(id.PrivateKeyHex)
if err != nil {
return nil, fmt.Errorf("decode private key: %w", err)
}
if len(raw) != 32 {
return nil, fmt.Errorf("private key length %d", len(raw))
}
priv := secp256k1.PrivKeyFromBytes(raw)
digest := sha256.Sum256(bytes)
sig := ecdsa.Sign(priv, digest[:])
return sig.Serialize(), nil
}
// DeriveAEADKey derives the symmetric key used to encrypt private-tier
// User Data Layer records. The KDF parameters (salt, info, length) are
// fixed for the v1 wire format.
func DeriveAEADKey(id *Identity) ([]byte, error) {
if id == nil {
return nil, errors.New("no identity loaded")
}
raw, err := hex.DecodeString(id.PrivateKeyHex)
if err != nil {
return nil, fmt.Errorf("decode private key: %w", err)
}
r := hkdf.New(sha256.New, raw, []byte(aeadKDFSalt), []byte(aeadKDFInfo))
out := make([]byte, aeadKeyLen)
if _, err := io.ReadFull(r, out); err != nil {
return nil, fmt.Errorf("hkdf: %w", err)
}
return out, nil
}
func isAllZero(b []byte) bool {
for _, x := range b {
if x != 0 {
return false
}
}
return true
}
// Ensure rand.Reader is referenced so dependency-stripped builds don't
// accidentally remove the crypto/rand import that secp256k1.GeneratePrivateKey
// relies on transitively.
var _ = rand.Reader
+308
View File
@@ -0,0 +1,308 @@
package identity
import (
"crypto/sha256"
"encoding/hex"
"os"
"path/filepath"
"strings"
"testing"
"github.com/decred/dcrd/dcrec/secp256k1/v4"
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
)
func TestGenerateUIDIsDeterministicFromPubkey(t *testing.T) {
id, err := Generate()
if err != nil {
t.Fatalf("generate: %v", err)
}
if !strings.HasPrefix(id.UID, "z") {
t.Fatalf("uid missing multibase 'z' prefix: %q", id.UID)
}
if id.Curve != CurveSecp256k1 {
t.Fatalf("curve = %q", id.Curve)
}
if len(id.PrivateKeyHex) != 64 {
t.Fatalf("privkey hex len = %d", len(id.PrivateKeyHex))
}
if len(id.PublicKeyHex) != 66 {
t.Fatalf("compressed pubkey hex len = %d", len(id.PublicKeyHex))
}
// Reimporting yields the same uid.
imported, err := Import(id.PrivateKeyHex)
if err != nil {
t.Fatalf("import: %v", err)
}
if imported.UID != id.UID {
t.Fatalf("uid changed after import roundtrip: %q vs %q", id.UID, imported.UID)
}
if imported.PublicKeyHex != id.PublicKeyHex {
t.Fatalf("pubkey changed after import roundtrip")
}
}
func TestImportRejectsBadInput(t *testing.T) {
if _, err := Import(""); err == nil {
t.Fatal("empty hex should error")
}
if _, err := Import("zz"); err == nil {
t.Fatal("non-hex should error")
}
if _, err := Import("00"); err == nil {
t.Fatal("wrong-length hex should error")
}
if _, err := Import(strings.Repeat("00", 32)); err == nil {
t.Fatal("zero scalar should error")
}
// Hex with 0x prefix should work.
id1, err := Generate()
if err != nil {
t.Fatal(err)
}
id2, err := Import("0x" + id1.PrivateKeyHex)
if err != nil {
t.Fatalf("0x-prefixed hex: %v", err)
}
if id1.UID != id2.UID {
t.Fatal("uid mismatch through 0x prefix")
}
}
func TestSaveLoadRoundtrip(t *testing.T) {
dir := t.TempDir()
path := filepath.Join(dir, "identity.json")
if Exists(path) {
t.Fatal("file should not exist yet")
}
if got, _ := Load(path); got != nil {
t.Fatal("Load on missing file should return nil")
}
id, err := Generate()
if err != nil {
t.Fatal(err)
}
if err := Save(path, id); err != nil {
t.Fatal(err)
}
if !Exists(path) {
t.Fatal("file should exist after Save")
}
loaded, err := Load(path)
if err != nil {
t.Fatal(err)
}
if loaded == nil || loaded.UID != id.UID || loaded.PrivateKeyHex != id.PrivateKeyHex {
t.Fatalf("roundtrip mismatch: %+v vs %+v", id, loaded)
}
// File should be 0600 (private key on disk).
info, err := os.Stat(path)
if err != nil {
t.Fatal(err)
}
if info.Mode().Perm() != 0600 {
t.Errorf("identity file mode = %o, want 0600", info.Mode().Perm())
}
if err := Delete(path); err != nil {
t.Fatal(err)
}
if Exists(path) {
t.Fatal("file should not exist after Delete")
}
// Delete is idempotent.
if err := Delete(path); err != nil {
t.Fatalf("Delete on missing file should be nil: %v", err)
}
}
func TestSignAndVerify(t *testing.T) {
id, err := Generate()
if err != nil {
t.Fatal(err)
}
msg := []byte("hello user data layer")
sig, err := Sign(id, msg)
if err != nil {
t.Fatal(err)
}
if len(sig) == 0 {
t.Fatal("empty sig")
}
// Verify externally with the public key, matching what meta-watch will do.
pubBytes, err := hex.DecodeString(id.PublicKeyHex)
if err != nil {
t.Fatal(err)
}
pub, err := secp256k1.ParsePubKey(pubBytes)
if err != nil {
t.Fatal(err)
}
parsedSig, err := ecdsa.ParseDERSignature(sig)
if err != nil {
t.Fatalf("parse DER sig: %v", err)
}
digest := sha256.Sum256(msg)
if !parsedSig.Verify(digest[:], pub) {
t.Fatal("signature did not verify")
}
// Flipping a byte must break verification.
tampered := append([]byte{}, msg...)
tampered[0] ^= 0x01
digest2 := sha256.Sum256(tampered)
if parsedSig.Verify(digest2[:], pub) {
t.Fatal("signature verified on tampered message")
}
}
func TestDeriveAEADKeyIsDeterministic(t *testing.T) {
id, err := Generate()
if err != nil {
t.Fatal(err)
}
k1, err := DeriveAEADKey(id)
if err != nil {
t.Fatal(err)
}
k2, err := DeriveAEADKey(id)
if err != nil {
t.Fatal(err)
}
if !bytesEqual(k1, k2) {
t.Fatal("AEAD key derivation is non-deterministic")
}
if len(k1) != aeadKeyLen {
t.Fatalf("AEAD key len = %d, want %d", len(k1), aeadKeyLen)
}
// Different identity → different key.
other, _ := Generate()
k3, _ := DeriveAEADKey(other)
if bytesEqual(k1, k3) {
t.Fatal("two distinct identities derived the same AEAD key")
}
}
func TestAccountStoreRoundtrip(t *testing.T) {
dir := filepath.Join(t.TempDir(), "accounts")
// Empty store.
list, err := List(dir)
if err != nil {
t.Fatalf("List on missing dir: %v", err)
}
if len(list) != 0 {
t.Fatalf("expected empty store, got %d", len(list))
}
a, _ := Generate()
b, _ := Generate()
if err := SaveAccount(dir, a); err != nil {
t.Fatal(err)
}
if err := SaveAccount(dir, b); err != nil {
t.Fatal(err)
}
if !ExistsByUID(dir, a.UID) || !ExistsByUID(dir, b.UID) {
t.Fatal("both accounts should exist")
}
list, err = List(dir)
if err != nil {
t.Fatal(err)
}
if len(list) != 2 {
t.Fatalf("expected 2 accounts, got %d", len(list))
}
got, err := LoadByUID(dir, a.UID)
if err != nil || got == nil || got.PrivateKeyHex != a.PrivateKeyHex {
t.Fatalf("LoadByUID roundtrip failed: %+v err=%v", got, err)
}
// Missing uid → (nil, nil).
if got, err := LoadByUID(dir, "zdoesnotexist"); err != nil || got != nil {
t.Fatalf("LoadByUID on missing should be (nil,nil): %+v %v", got, err)
}
// Delete one; the other remains. Delete is idempotent.
if err := DeleteByUID(dir, a.UID); err != nil {
t.Fatal(err)
}
if ExistsByUID(dir, a.UID) {
t.Fatal("account should be gone after delete")
}
if err := DeleteByUID(dir, a.UID); err != nil {
t.Fatalf("delete should be idempotent: %v", err)
}
list, _ = List(dir)
if len(list) != 1 || list[0].UID != b.UID {
t.Fatalf("expected only b remaining, got %+v", list)
}
}
func TestAccountFilePathRejectsTraversal(t *testing.T) {
for _, bad := range []string{"", "../escape", "a/b", "z..", strings.Repeat("z", 200)} {
if _, err := accountFilePath("/tmp/x", bad); err == nil {
t.Fatalf("expected rejection of uid %q", bad)
}
}
// A real uid is accepted.
id, _ := Generate()
if _, err := accountFilePath("/tmp/x", id.UID); err != nil {
t.Fatalf("real uid rejected: %v", err)
}
}
func TestMigrateLegacy(t *testing.T) {
base := t.TempDir()
legacy := filepath.Join(base, "identity.json")
accounts := filepath.Join(base, "accounts")
// No legacy file → no-op.
if migrated, err := MigrateLegacy(legacy, accounts); err != nil || migrated {
t.Fatalf("expected no-op migration, got migrated=%v err=%v", migrated, err)
}
id, _ := Generate()
if err := Save(legacy, id); err != nil {
t.Fatal(err)
}
migrated, err := MigrateLegacy(legacy, accounts)
if err != nil || !migrated {
t.Fatalf("expected migration to run: migrated=%v err=%v", migrated, err)
}
if Exists(legacy) {
t.Fatal("legacy file should be removed after migration")
}
got, err := LoadByUID(accounts, id.UID)
if err != nil || got == nil || got.UID != id.UID {
t.Fatalf("migrated account not found: %+v err=%v", got, err)
}
// Second run is a clean no-op (legacy already gone).
if migrated, err := MigrateLegacy(legacy, accounts); err != nil || migrated {
t.Fatalf("second migration should no-op: migrated=%v err=%v", migrated, err)
}
}
func bytesEqual(a, b []byte) bool {
if len(a) != len(b) {
return false
}
for i := range a {
if a[i] != b[i] {
return false
}
}
return true
}
+5 -1
View File
@@ -75,7 +75,11 @@ func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo {
Hostname: hostname, Hostname: hostname,
BaseUrl: baseUrl, BaseUrl: baseUrl,
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort), ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
RedisUrl: fmt.Sprintf("redis://%s:%d", ip, p.config.RedisPort), // RedisUrl deliberately empty — api-mediated-access PR D removes
// direct Redis exposure. Consumers route metadata I/O through
// meta-core's HTTP API (/meta/{hash}*) and event streams through
// SSE (/api/events/{files,meta}).
RedisUrl: "",
WebdavUrl: webdavUrl, WebdavUrl: webdavUrl,
WebdavUrlInternal: webdavUrlInternal, WebdavUrlInternal: webdavUrlInternal,
Timestamp: time.Now().UnixMilli(), Timestamp: time.Now().UnixMilli(),
+24 -2
View File
@@ -292,13 +292,35 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
json.NewEncoder(w).Encode(data) json.NewEncoder(w).Encode(data)
} }
// writeError writes the typed error envelope (see api/handlers.go for the
// canonical definition + slug vocabulary). Kept local to avoid an import
// cycle with the api package.
func writeError(w http.ResponseWriter, status int, message string) { func writeError(w http.ResponseWriter, status int, message string) {
writeJSON(w, status, map[string]string{ slug, retryable := slugForStatus(status)
"error": http.StatusText(status), writeJSON(w, status, map[string]interface{}{
"error": slug,
"message": message, "message": message,
"retryable": retryable,
}) })
} }
func slugForStatus(status int) (string, bool) {
switch status {
case http.StatusConflict:
return "alias_collision", false
case http.StatusNotFound:
return "unknown_root", false
case http.StatusBadRequest:
return "schema_violation", false
case http.StatusServiceUnavailable:
return "storage_unavailable", true
case http.StatusInternalServerError:
return "internal", true
default:
return "internal", status >= 500
}
}
func parseIntOrDefault(s string, defaultVal int) (int, error) { func parseIntOrDefault(s string, defaultVal int) (int, error) {
var result int var result int
if s == "" { if s == "" {
+93 -12
View File
@@ -24,6 +24,7 @@ type Manager struct {
filesPath string filesPath string
mountsDir string mountsDir string
poller *Poller poller *Poller
statsPoller *StatsPoller
} }
// NewManager creates a new mount manager // NewManager creates a new mount manager
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
return nil, fmt.Errorf("failed to create mount directories: %w", err) return nil, fmt.Errorf("failed to create mount directories: %w", err)
} }
// One-shot migration for legacy NFS entries — disable rather than fail to
// start, so operators can fix their config at their own pace.
if err := m.migrateLegacyMounts(); err != nil {
log.Printf("[Mounts] legacy migration warning: %v", err)
}
return m, nil return m, nil
} }
// migrateLegacyMounts disables any mount entries with type="nfs". The native
// NFS handler was removed when the project consolidated on rclone; an NFS
// entry left enabled would loop the watcher with mount errors.
func (m *Manager) migrateLegacyMounts() error {
mountsFile, err := m.readConfig()
if err != nil {
return err
}
dirty := false
for i := range mountsFile.Mounts {
mnt := &mountsFile.Mounts[i]
if string(mnt.Type) == "nfs" {
if mnt.Enabled || mnt.DesiredMounted {
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
mnt.Enabled = false
mnt.DesiredMounted = false
dirty = true
}
}
}
if dirty {
return m.writeConfig(mountsFile)
}
return nil
}
// SetPoller sets the poller for the manager // SetPoller sets the poller for the manager
func (m *Manager) SetPoller(poller *Poller) { func (m *Manager) SetPoller(poller *Poller) {
m.poller = poller m.poller = poller
} }
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
// set; remains nil-safe when stats are disabled.
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
m.statsPoller = sp
}
// ensureDirs creates required directories // ensureDirs creates required directories
func (m *Manager) ensureDirs() error { func (m *Manager) ensureDirs() error {
dirs := []string{ dirs := []string{
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID) pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
statuses[i].PollingActive = pollingStatus.Active statuses[i].PollingActive = pollingStatus.Active
statuses[i].LastPolledScan = pollingStatus.LastScan statuses[i].LastPolledScan = pollingStatus.LastScan
statuses[i].Scanning = pollingStatus.Scanning
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
statuses[i].NextScanAt = pollingStatus.NextScanAt
}
}
if m.statsPoller != nil {
for i := range statuses {
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
} }
} }
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
pollingStatus := m.poller.GetPollingStatus(mount.ID) pollingStatus := m.poller.GetPollingStatus(mount.ID)
status.PollingActive = pollingStatus.Active status.PollingActive = pollingStatus.Active
status.LastPolledScan = pollingStatus.LastScan status.LastPolledScan = pollingStatus.LastScan
status.Scanning = pollingStatus.Scanning
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
status.NextScanAt = pollingStatus.NextScanAt
}
if m.statsPoller != nil {
status.IOStats = m.statsPoller.GetStats(mount.ID)
} }
return status, nil return status, nil
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
return nil, fmt.Errorf("mount name is required") return nil, fmt.Errorf("mount name is required")
} }
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone { // "nfs" used to be valid; the native handler is gone. Reject loudly so the
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required") // caller knows their integration needs to migrate to a Samba re-export or
// switch to a supported protocol.
if req.Type == MountType("nfs") {
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
}
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
} }
// Validate type-specific fields // Validate type-specific fields
switch req.Type { switch req.Type {
case MountTypeNFS:
if req.NFSServer == "" || req.NFSPath == "" {
return nil, fmt.Errorf("NFS server and path are required")
}
case MountTypeSMB: case MountTypeSMB:
if req.SMBServer == "" || req.SMBShare == "" { if req.SMBServer == "" || req.SMBShare == "" {
return nil, fmt.Errorf("SMB server and share are required") return nil, fmt.Errorf("SMB server and share are required")
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
enabled = *req.Enabled enabled = *req.Enabled
} }
// Default polling settings // Polling defaults on for both supported types — neither SMB-via-rclone nor
pollingEnabled := false // generic rclone delivers kernel inotify, so a periodic re-scan is the only
// way to pick up upstream changes. Explicit PollingEnabled in the request
// wins.
pollingEnabled := true
if req.PollingEnabled != nil { if req.PollingEnabled != nil {
pollingEnabled = *req.PollingEnabled pollingEnabled = *req.PollingEnabled
} }
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
Enabled: enabled, Enabled: enabled,
DesiredMounted: enabled, // Auto-mount if enabled DesiredMounted: enabled, // Auto-mount if enabled
MountPath: mountPath, MountPath: mountPath,
Options: req.Options, CacheMaxSize: req.CacheMaxSize,
CacheMaxAge: req.CacheMaxAge,
DirCacheTime: req.DirCacheTime,
PollingEnabled: pollingEnabled, PollingEnabled: pollingEnabled,
PollingIntervalMs: pollingIntervalMs, PollingIntervalMs: pollingIntervalMs,
} }
// Type-specific fields // Type-specific fields
switch req.Type { switch req.Type {
case MountTypeNFS:
mount.NFSServer = req.NFSServer
mount.NFSPath = req.NFSPath
case MountTypeSMB: case MountTypeSMB:
mount.SMBServer = req.SMBServer mount.SMBServer = req.SMBServer
mount.SMBShare = req.SMBShare mount.SMBShare = req.SMBShare
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
} }
} }
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
// don't validate format here; rclone rejects bad values at mount time.
if val, ok := updates["cacheMaxSize"]; ok {
if s, ok := val.(string); ok {
mount.CacheMaxSize = s
}
}
if val, ok := updates["cacheMaxAge"]; ok {
if s, ok := val.(string); ok {
mount.CacheMaxAge = s
}
}
if val, ok := updates["dirCacheTime"]; ok {
if s, ok := val.(string); ok {
mount.DirCacheTime = s
}
}
if err := m.writeConfig(mountsFile); err != nil { if err := m.writeConfig(mountsFile); err != nil {
return nil, err return nil, err
} }
+174 -29
View File
@@ -9,6 +9,25 @@ import (
// ScanFunc is a function type for scanning a mount path // ScanFunc is a function type for scanning a mount path
type ScanFunc func(path string) (int, error) type ScanFunc func(path string) (int, error)
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
// large remote shares (CORN, etc.) from monopolising the network/CPU.
const IdleFactor = 9
// InitialScanDelay is how long a freshly-started poller waits before its first
// scan. Avoids hammering the system at container startup when many mounts come
// up at once.
const InitialScanDelay = 2 * time.Second
// ReconcileInterval drives the periodic resync loop that keeps poller
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
// — without periodic reconciliation, the poller would never observe the
// transition from `mounted=false` to `mounted=true` and never start scanning.
const ReconcileInterval = 10 * time.Second
// mountPoller manages polling for a single mount // mountPoller manages polling for a single mount
type mountPoller struct { type mountPoller struct {
mountID string mountID string
@@ -20,7 +39,13 @@ type mountPoller struct {
// PollingStatus holds the polling state for a mount // PollingStatus holds the polling state for a mount
type PollingStatus struct { type PollingStatus struct {
Active bool `json:"active"` Active bool `json:"active"`
LastScan int64 `json:"lastScan,omitempty"` LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
LastFileCount int `json:"-"` // last observed file count; used to suppress no-change scan logs
scanned bool // true once at least one scan has completed for this mount
} }
// Poller manages per-mount polling goroutines // Poller manages per-mount polling goroutines
@@ -32,6 +57,7 @@ type Poller struct {
status map[string]*PollingStatus status map[string]*PollingStatus
running bool running bool
reconcileStop chan struct{}
mu sync.RWMutex mu sync.RWMutex
} }
@@ -54,15 +80,35 @@ func (p *Poller) Start() error {
return nil return nil
} }
p.running = true p.running = true
p.reconcileStop = make(chan struct{})
log.Println("[Poller] Starting mount polling service") log.Println("[Poller] Starting mount polling service")
// Initial sync with mount configurations // Initial sync with mount configurations
go p.syncWithMounts() go p.syncWithMounts()
// Periodic reconciliation — picks up mount-watcher.sh transitions
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
go p.reconcileLoop(p.reconcileStop)
return nil return nil
} }
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
ticker := time.NewTicker(ReconcileInterval)
defer ticker.Stop()
for {
select {
case <-stop:
return
case <-ticker.C:
p.syncWithMounts()
}
}
}
// Stop stops all polling goroutines // Stop stops all polling goroutines
func (p *Poller) Stop() error { func (p *Poller) Stop() error {
p.mu.Lock() p.mu.Lock()
@@ -75,6 +121,11 @@ func (p *Poller) Stop() error {
log.Println("[Poller] Stopping mount polling service") log.Println("[Poller] Stopping mount polling service")
if p.reconcileStop != nil {
close(p.reconcileStop)
p.reconcileStop = nil
}
// Stop all pollers // Stop all pollers
for id, poller := range p.pollers { for id, poller := range p.pollers {
p.stopPoller(poller) p.stopPoller(poller)
@@ -127,7 +178,7 @@ func (p *Poller) syncWithMounts() {
existingPoller, exists := p.pollers[mount.ID] existingPoller, exists := p.pollers[mount.ID]
if shouldPoll { if shouldPoll {
// Calculate interval // Calculate interval (used as the floor for adaptive scheduling)
interval := mount.PollingIntervalMs interval := mount.PollingIntervalMs
if interval < MinPollingIntervalMs { if interval < MinPollingIntervalMs {
interval = DefaultPollingIntervalMs interval = DefaultPollingIntervalMs
@@ -135,7 +186,7 @@ func (p *Poller) syncWithMounts() {
if !exists { if !exists {
// Start new poller // Start new poller
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval) log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
p.startPollerForMount(mount.ID, mount.MountPath, interval) p.startPollerForMount(mount.ID, mount.MountPath, interval)
} }
} else if exists { } else if exists {
@@ -156,7 +207,13 @@ func (p *Poller) syncWithMounts() {
} }
} }
// startPollerForMount starts a polling goroutine for a mount // startPollerForMount starts the adaptive polling goroutine for a mount.
//
// The loop runs scans sequentially: scan → measure duration → sleep
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
// cycle on slow shares, while keeping fresh checks on small mounts via the
// configured interval floor.
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) { func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
poller := &mountPoller{ poller := &mountPoller{
mountID: mountID, mountID: mountID,
@@ -170,17 +227,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
} }
p.status[mountID].Active = true p.status[mountID].Active = true
floor := time.Duration(intervalMs) * time.Millisecond
go func() { go func() {
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond) // Initial delay so a freshly-mounted share doesn't get hammered the
defer ticker.Stop() // instant the poller starts.
nextDelay := InitialScanDelay
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
for { for {
select { select {
case <-poller.stopChan: case <-poller.stopChan:
return return
case <-ticker.C: case <-time.After(nextDelay):
p.runPollScan(mountID, mountPath)
} }
// Re-check stop after the wake-up to avoid running a scan during shutdown.
select {
case <-poller.stopChan:
return
default:
}
duration, err := p.executeScan(mountID, mountPath)
if err != nil {
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
}
// Adaptive backoff: idle for IdleFactor × last duration, with the
// configured interval as a floor so tiny mounts still scan often.
nextDelay = duration * IdleFactor
if nextDelay < floor {
nextDelay = floor
}
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
} }
}() }()
} }
@@ -199,54 +279,115 @@ func (p *Poller) stopPoller(poller *mountPoller) {
// Update status // Update status
if status, exists := p.status[poller.mountID]; exists { if status, exists := p.status[poller.mountID]; exists {
status.Active = false status.Active = false
status.NextScanAt = 0
} }
} }
// runPollScan executes a scan for a mount // executeScan runs scanFunc with timing + status tracking. Both the adaptive
func (p *Poller) runPollScan(mountID, mountPath string) { // loop and manual TriggerScan funnel through here so the UI sees a consistent
// "scanning / last duration / last scan" picture regardless of which path
// kicked it off. Returns the scan duration so the caller can compute the next
// adaptive delay.
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
if p.scanFunc == nil { if p.scanFunc == nil {
return return 0, nil
} }
start := time.Now()
startMs := start.UnixMilli()
p.mu.Lock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.Scanning = true
status.CurrentScanStartedAt = startMs
p.mu.Unlock()
fileCount, err := p.scanFunc(mountPath) fileCount, err := p.scanFunc(mountPath)
if err != nil {
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
return
}
// Update status end := time.Now()
duration := end.Sub(start)
// Only log a completed scan when the file count changed (or on the very
// first scan) so a steady mount doesn't emit one line per poll tick.
changed := true
p.mu.Lock() p.mu.Lock()
if status, exists := p.status[mountID]; exists { if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS() status.Scanning = false
status.CurrentScanStartedAt = 0
status.LastScan = end.UnixMilli()
status.LastScanDurationMs = duration.Milliseconds()
changed = !status.scanned || status.LastFileCount != fileCount
status.LastFileCount = fileCount
status.scanned = true
} }
p.mu.Unlock() p.mu.Unlock()
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount) if err == nil && changed {
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
}
return duration, err
} }
// TriggerScan triggers an immediate scan for a mount // setNextScanAt records the projected start time of the next adaptive scan
// (used purely for the UI countdown — the loop's own timer is authoritative).
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
p.mu.Lock()
defer p.mu.Unlock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.NextScanAt = at.UnixMilli()
}
// runPollScan is kept for backwards compatibility with any external caller —
// new code should go through executeScan via the adaptive loop.
func (p *Poller) runPollScan(mountID, mountPath string) {
_, _ = p.executeScan(mountID, mountPath)
}
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
// Runs synchronously; concurrent calls with the adaptive loop are possible
// but rare in practice — both paths update the same status atomically.
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) { func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
if p.scanFunc == nil { if p.scanFunc == nil {
return 0, nil return 0, nil
} }
start := time.Now()
startMs := start.UnixMilli()
p.mu.Lock()
status, exists := p.status[mountID]
if !exists {
status = &PollingStatus{}
p.status[mountID] = status
}
status.Scanning = true
status.CurrentScanStartedAt = startMs
p.mu.Unlock()
fileCount, err := p.scanFunc(mountPath) fileCount, err := p.scanFunc(mountPath)
if err != nil {
return 0, err end := time.Now()
} duration := end.Sub(start)
// Update status
p.mu.Lock() p.mu.Lock()
if status, exists := p.status[mountID]; exists { if status, exists := p.status[mountID]; exists {
status.LastScan = NowMS() status.Scanning = false
} else { status.CurrentScanStartedAt = 0
p.status[mountID] = &PollingStatus{ status.LastScan = end.UnixMilli()
LastScan: NowMS(), status.LastScanDurationMs = duration.Milliseconds()
}
} }
p.mu.Unlock() p.mu.Unlock()
return fileCount, nil return fileCount, err
} }
// GetPollingStatus returns the polling status for a mount // GetPollingStatus returns the polling status for a mount
@@ -258,6 +399,10 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
return &PollingStatus{ return &PollingStatus{
Active: status.Active, Active: status.Active,
LastScan: status.LastScan, LastScan: status.LastScan,
Scanning: status.Scanning,
CurrentScanStartedAt: status.CurrentScanStartedAt,
LastScanDurationMs: status.LastScanDurationMs,
NextScanAt: status.NextScanAt,
} }
} }
return &PollingStatus{} return &PollingStatus{}
+70
View File
@@ -0,0 +1,70 @@
package mounts
import (
"context"
"encoding/json"
"fmt"
"net/http"
"time"
)
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
// time. The poller keeps the previous sample per mount and computes rates
// (Bps, transfers/sec) from the delta to the latest sample.
//
// Post-consolidation: rclone is the only sample source, since all mount types
// execute through the local rclone daemon.
type StatsSample struct {
TakenAt time.Time
Source string // "rclone"
BytesRead int64
BytesWritten int64
ReadOps int64
WriteOps int64
// File-level transfer count from rclone's RC API. Distinct from ReadOps
// (which is unused for rclone — kernel ops counters are not exposed
// through FUSE).
Transfers int64
}
// rcloneCoreStats matches the RC API response from POST /core/stats.
type rcloneCoreStats struct {
Bytes int64 `json:"bytes"`
TotalBytes int64 `json:"totalBytes"`
Transfers int64 `json:"transfers"`
TotalTransfers int64 `json:"totalTransfers"`
}
// SampleRclone queries the local rclone RC API for the cumulative bytes +
// transfers counters. The numbers are global across all active mounts on this
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
// sample for every mounted entry on a tick.
func SampleRclone(ctx context.Context) (*StatsSample, error) {
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
if err != nil {
return nil, fmt.Errorf("build rclone stats request: %w", err)
}
req.SetBasicAuth("admin", "admin")
client := &http.Client{Timeout: 3 * time.Second}
resp, err := client.Do(req)
if err != nil {
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
}
defer resp.Body.Close()
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
}
var stats rcloneCoreStats
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
return nil, fmt.Errorf("decode rclone stats: %w", err)
}
return &StatsSample{
Source: "rclone",
TakenAt: time.Now(),
BytesRead: stats.Bytes,
Transfers: stats.Transfers,
}, nil
}
+185
View File
@@ -0,0 +1,185 @@
package mounts
import (
"context"
"log"
"sync"
"time"
)
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
// spot — short enough that throughput numbers feel live, long enough that
// /proc reads aren't a load source themselves.
const StatsInterval = 5 * time.Second
// StatsPoller samples cumulative counters from the local rclone RC API and
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
//
// Post-consolidation note: every mount type executes through rclone, so
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
// rclone is the only counter source. core/stats is daemon-global across all
// active mounts; the same sample is applied to each mount's status. If
// per-mount byte counters become important, the path forward is rclone stats
// groups (out of scope for this consolidation).
type StatsPoller struct {
manager *Manager
mu sync.RWMutex
lastSamples map[string]*StatsSample // mountID -> previous raw sample
latestStats map[string]*MountIOStats
stopChan chan struct{}
running bool
pollInterval time.Duration
}
// NewStatsPoller constructs a StatsPoller backed by the given manager.
func NewStatsPoller(manager *Manager) *StatsPoller {
return &StatsPoller{
manager: manager,
lastSamples: make(map[string]*StatsSample),
latestStats: make(map[string]*MountIOStats),
pollInterval: StatsInterval,
}
}
// Start launches the sampling loop. Idempotent.
func (p *StatsPoller) Start() {
p.mu.Lock()
if p.running {
p.mu.Unlock()
return
}
p.running = true
p.stopChan = make(chan struct{})
stop := p.stopChan
p.mu.Unlock()
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
go p.loop(stop)
}
// Stop terminates the sampling loop. Idempotent.
func (p *StatsPoller) Stop() {
p.mu.Lock()
defer p.mu.Unlock()
if !p.running {
return
}
p.running = false
if p.stopChan != nil {
close(p.stopChan)
p.stopChan = nil
}
}
// GetStats returns the latest computed IO stats for a mount, or nil if no
// stats have been observed yet.
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
p.mu.RLock()
defer p.mu.RUnlock()
s, ok := p.latestStats[mountID]
if !ok {
return nil
}
// Return a copy so the caller can't mutate our cache.
cp := *s
return &cp
}
func (p *StatsPoller) loop(stop <-chan struct{}) {
// Take the first sample immediately so rates start populating on the
// second tick instead of waiting two intervals.
p.tick()
t := time.NewTicker(p.pollInterval)
defer t.Stop()
for {
select {
case <-stop:
return
case <-t.C:
p.tick()
}
}
}
func (p *StatsPoller) tick() {
mounts, err := p.manager.listMountsWithoutPollingStatus()
if err != nil {
log.Printf("[StatsPoller] listMounts failed: %v", err)
return
}
// Sample rclone once per tick — its RC API gives global stats across all
// active rclone mounts on this host. Both supported mount types (smb,
// rclone) execute through the same rclone daemon, so the same sample is
// applied to every mounted entry.
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
defer cancel()
sample, err := SampleRclone(ctx)
if err != nil {
log.Printf("[StatsPoller] rclone sample failed: %v", err)
return
}
if sample == nil {
return
}
for _, m := range mounts {
if !m.Mounted {
continue
}
p.mu.Lock()
prev := p.lastSamples[m.ID]
p.lastSamples[m.ID] = sample
stats := computeStats(prev, sample)
p.latestStats[m.ID] = stats
p.mu.Unlock()
}
}
// computeStats turns a (previous, current) raw-sample pair into the rate-
// based MountIOStats the API exposes. With no previous sample the rates are
// zero — operators will see cumulative counters but Bps/IOPS only on the
// second poll.
func computeStats(prev, cur *StatsSample) *MountIOStats {
out := &MountIOStats{
Source: cur.Source,
BytesRead: cur.BytesRead,
BytesWritten: cur.BytesWritten,
ReadOps: cur.ReadOps,
WriteOps: cur.WriteOps,
LastSampleAt: cur.TakenAt.UnixMilli(),
}
if prev == nil {
return out
}
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
if dt <= 0 {
return out
}
out.IntervalMs = int64(dt * 1000)
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
// negative deltas as "no movement" rather than letting the rate go
// negative or implausibly huge.
if d := cur.BytesRead - prev.BytesRead; d > 0 {
out.ReadBps = float64(d) / dt
}
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
out.WriteBps = float64(d) / dt
}
if d := cur.ReadOps - prev.ReadOps; d > 0 {
out.ReadIops = float64(d) / dt
}
if d := cur.WriteOps - prev.WriteOps; d > 0 {
out.WriteIops = float64(d) / dt
}
if cur.Source == "rclone" {
if d := cur.Transfers - prev.Transfers; d > 0 {
out.TransfersPerSec = float64(d) / dt
}
}
return out
}
+64 -14
View File
@@ -2,11 +2,19 @@ package mounts
import "time" import "time"
// MountType represents the type of mount // MountType represents the type of mount.
//
// All types execute through the local rclone daemon. "smb" is a UX-level alias
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
// the generic escape hatch for any pre-defined remote in rclone.conf.
//
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
// regardless.
type MountType string type MountType string
const ( const (
MountTypeNFS MountType = "nfs"
MountTypeSMB MountType = "smb" MountTypeSMB MountType = "smb"
MountTypeRclone MountType = "rclone" MountTypeRclone MountType = "rclone"
) )
@@ -17,6 +25,15 @@ const (
MinPollingIntervalMs = 5000 // 5 seconds MinPollingIntervalMs = 5000 // 5 seconds
) )
// VFS cache defaults — applied per-mount when the caller leaves the field
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
// Duration) untouched.
const (
DefaultCacheMaxSize = "50G"
DefaultCacheMaxAge = "24h"
DefaultDirCacheTime = "5m"
)
// MountConfig represents a mount configuration // MountConfig represents a mount configuration
type MountConfig struct { type MountConfig struct {
// Common fields // Common fields
@@ -26,23 +43,26 @@ type MountConfig struct {
Enabled bool `json:"enabled"` Enabled bool `json:"enabled"`
DesiredMounted bool `json:"desiredMounted"` DesiredMounted bool `json:"desiredMounted"`
MountPath string `json:"mountPath"` MountPath string `json:"mountPath"`
Options string `json:"options,omitempty"`
// NFS-specific fields // SMB-specific fields (also valid for type=smb routed through rclone-smb)
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
// SMB-specific fields
SMBServer string `json:"smbServer,omitempty"` SMBServer string `json:"smbServer,omitempty"`
SMBShare string `json:"smbShare,omitempty"` SMBShare string `json:"smbShare,omitempty"`
SMBUsername string `json:"smbUsername,omitempty"` SMBUsername string `json:"smbUsername,omitempty"`
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"` SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
SMBDomain string `json:"smbDomain,omitempty"` SMBDomain string `json:"smbDomain,omitempty"`
// rclone-specific fields // rclone-specific fields (escape hatch — references a pre-defined remote)
RcloneRemote string `json:"rcloneRemote,omitempty"` RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"` RclonePath string `json:"rclonePath,omitempty"`
// VFS cache configuration — applied to the rclone mount call. All three
// fields are optional; blanks fall back to Default* constants above.
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
// DirCacheTime use rclone's Duration ("24h", "30m").
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
DirCacheTime string `json:"dirCacheTime,omitempty"`
// Polling configuration // Polling configuration
PollingEnabled bool `json:"pollingEnabled"` PollingEnabled bool `json:"pollingEnabled"`
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000 PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
@@ -56,6 +76,36 @@ type MountStatus struct {
LastChecked int64 `json:"lastChecked"` LastChecked int64 `json:"lastChecked"`
PollingActive bool `json:"pollingActive,omitempty"` PollingActive bool `json:"pollingActive,omitempty"`
LastPolledScan int64 `json:"lastPolledScan,omitempty"` LastPolledScan int64 `json:"lastPolledScan,omitempty"`
// Progress tracking — surface the poller's adaptive scan state to the UI.
Scanning bool `json:"scanning,omitempty"`
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
// /proc/self/mountstats, or rclone's RC API depending on mount type.
IOStats *MountIOStats `json:"ioStats,omitempty"`
}
// MountIOStats captures throughput + operation rate for a single mount.
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
// the last two samples divided by the interval.
type MountIOStats struct {
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
BytesRead int64 `json:"bytesRead"`
BytesWritten int64 `json:"bytesWritten,omitempty"`
ReadBps float64 `json:"readBps"`
WriteBps float64 `json:"writeBps,omitempty"`
ReadOps int64 `json:"readOps,omitempty"`
WriteOps int64 `json:"writeOps,omitempty"`
// ReadIops and WriteIops are op counters per second — number of read/write
// requests issued by the kernel against the share. Distinct from
// TransfersPerSec which is a file-level metric only relevant to rclone.
ReadIops float64 `json:"readIops,omitempty"`
WriteIops float64 `json:"writeIops,omitempty"`
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
LastSampleAt int64 `json:"lastSampleAt"` // ms
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
} }
// MountsFile represents the mounts.json file structure // MountsFile represents the mounts.json file structure
@@ -75,11 +125,6 @@ type CreateMountRequest struct {
Name string `json:"name"` Name string `json:"name"`
Type MountType `json:"type"` Type MountType `json:"type"`
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
Options string `json:"options,omitempty"`
// NFS
NFSServer string `json:"nfsServer,omitempty"`
NFSPath string `json:"nfsPath,omitempty"`
// SMB // SMB
SMBServer string `json:"smbServer,omitempty"` SMBServer string `json:"smbServer,omitempty"`
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
RcloneRemote string `json:"rcloneRemote,omitempty"` RcloneRemote string `json:"rcloneRemote,omitempty"`
RclonePath string `json:"rclonePath,omitempty"` RclonePath string `json:"rclonePath,omitempty"`
// VFS cache (optional — falls back to Default* constants when unset)
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
DirCacheTime string `json:"dirCacheTime,omitempty"`
// Polling // Polling
PollingEnabled *bool `json:"pollingEnabled,omitempty"` PollingEnabled *bool `json:"pollingEnabled,omitempty"`
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"` PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
+103
View File
@@ -0,0 +1,103 @@
package schema
import (
"encoding/json"
"regexp"
"strconv"
"strings"
"time"
)
// cidRegex matches the CID shapes produced by @metazla/meta-hash:
// - CIDv1 multibase-base32: every algorithm (crc32, md5, sha1, sha2-256,
// sha3-256, sha3-384, midhash256, btih-v2, …) is serialized via
// CID.createV1(code, digest).toString(), which always yields a lowercase
// base32 string starting with "ba" (the "b" is the multibase tag, the
// "a" is the CIDv1 version byte under base32-encoding-of-0x01).
// - CIDv0 base58btc: "Qm" + 44 chars, kept for compatibility with other
// services that may still emit v0.
//
// Minimum CIDv1 length 16 chars matches the shortest real output we observed
// (crc32: "bafk3eaqerjhiwhy").
var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
// ClassifyValue returns the (primitive, hint) for a Redis value.
// exists=false (key missing) and JSON null literal both map to PrimUndefined.
// Empty string is a valid string.
func ClassifyValue(value string, exists bool) (Primitive, Hint) {
if !exists {
return PrimUndefined, ""
}
trimmed := strings.TrimSpace(value)
if trimmed == "null" {
return PrimUndefined, ""
}
if trimmed == "true" || trimmed == "false" {
return PrimBool, ""
}
if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
if isTimestampInt(trimmed) {
return PrimInt, HintTimestamp
}
return PrimInt, ""
}
if _, err := strconv.ParseFloat(trimmed, 64); err == nil {
return PrimFloat, ""
}
if len(trimmed) > 0 {
switch trimmed[0] {
case '{':
var v map[string]any
if json.Unmarshal([]byte(trimmed), &v) == nil {
return PrimJSONObj, ""
}
case '[':
var v []any
if json.Unmarshal([]byte(trimmed), &v) == nil {
return PrimJSONArr, ""
}
}
}
if cidRegex.MatchString(value) {
return PrimString, HintCID
}
if isTimestampString(value) {
return PrimString, HintTimestamp
}
return PrimString, ""
}
// isTimestampInt accepts unix-seconds in [2001-09, 2100] or unix-millis in the
// same range. Anything else is treated as a plain int.
func isTimestampInt(s string) bool {
n, err := strconv.ParseInt(s, 10, 64)
if err != nil || n <= 0 {
return false
}
if n >= 1_000_000_000 && n <= 4_102_444_800 {
return true
}
if n >= 1_000_000_000_000 && n <= 4_102_444_800_000 {
return true
}
return false
}
func isTimestampString(s string) bool {
if _, err := time.Parse(time.RFC3339, s); err == nil {
return true
}
if _, err := time.Parse("2006-01-02", s); err == nil {
return true
}
return false
}
// breakdownLabel returns the most specific label for a classified value.
// Hints take precedence over primitives.
func breakdownLabel(prim Primitive, hint Hint) string {
if hint != "" {
return string(hint)
}
return string(prim)
}
+355
View File
@@ -0,0 +1,355 @@
package schema
import (
"context"
"log"
"regexp"
"strings"
"sync"
"sync/atomic"
"time"
"github.com/metazla/meta-core/internal/events"
"github.com/redis/go-redis/v9"
)
var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
// fieldState is the per-field accumulator. perKey lets us undo a previous
// contribution when the same Redis key is re-SET or DELed.
type fieldState struct {
counts map[string]int
primitives map[Primitive]int
hintCounts map[Hint]int
typedCount int
undefinedCount int
languageKeyed bool
perKey map[string]labelPair
}
type labelPair struct {
prim Primitive
hint Hint
}
func newFieldState() *fieldState {
return &fieldState{
counts: make(map[string]int),
primitives: make(map[Primitive]int),
hintCounts: make(map[Hint]int),
perKey: make(map[string]labelPair),
}
}
// Indexer maintains a live field-schema by consuming meta:events.
type Indexer struct {
client *redis.Client
keyPrefix string
mu sync.RWMutex
fields map[string]*fieldState
processed uint64 // total events processed since start
ctx context.Context
cancel context.CancelFunc
wg sync.WaitGroup
runMu sync.Mutex
running bool
}
// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
// so the indexer can build absolute Redis keys for GET calls.
func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
ctx, cancel := context.WithCancel(context.Background())
return &Indexer{
client: client,
keyPrefix: keyPrefix,
fields: make(map[string]*fieldState),
ctx: ctx,
cancel: cancel,
}
}
// Start begins consuming the meta:events stream from the earliest available ID.
func (i *Indexer) Start() error {
i.runMu.Lock()
if i.running {
i.runMu.Unlock()
return nil
}
i.running = true
i.runMu.Unlock()
i.wg.Add(1)
go i.consumeLoop()
log.Println("[SchemaIndexer] Started - consuming meta:events stream")
return nil
}
// Stop terminates the consumer goroutine.
func (i *Indexer) Stop() error {
i.runMu.Lock()
if !i.running {
i.runMu.Unlock()
return nil
}
i.running = false
i.runMu.Unlock()
i.cancel()
i.wg.Wait()
log.Println("[SchemaIndexer] Stopped")
return nil
}
// ProcessedCount returns the total number of stream events processed.
// Used by WaitForDrain to detect when a rescan flood has been absorbed.
func (i *Indexer) ProcessedCount() uint64 {
return atomic.LoadUint64(&i.processed)
}
// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
// the timeout elapses. Suitable for use right after RepublishMetadata.
func (i *Indexer) WaitForDrain(timeout time.Duration) {
deadline := time.Now().Add(timeout)
last := i.ProcessedCount()
stableSince := time.Now()
for time.Now().Before(deadline) {
time.Sleep(100 * time.Millisecond)
now := i.ProcessedCount()
if now != last {
last = now
stableSince = time.Now()
continue
}
if time.Since(stableSince) >= 300*time.Millisecond {
return
}
}
}
func (i *Indexer) consumeLoop() {
defer i.wg.Done()
lastID := "0"
for {
select {
case <-i.ctx.Done():
return
default:
}
res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
Streams: []string{events.MetaEventsStream, lastID},
Block: 2 * time.Second,
Count: 500,
}).Result()
if err != nil {
if err == redis.Nil || i.ctx.Err() != nil {
continue
}
log.Printf("[SchemaIndexer] XRead error: %v", err)
time.Sleep(1 * time.Second)
continue
}
for _, stream := range res {
for _, msg := range stream.Messages {
lastID = msg.ID
i.processEvent(msg.Values)
atomic.AddUint64(&i.processed, 1)
}
}
}
}
func (i *Indexer) processEvent(values map[string]any) {
op, _ := values["type"].(string)
key, _ := values["key"].(string)
if key == "" {
return
}
fieldPath, keyHint, ok := parseFieldPath(key)
if !ok {
return
}
switch op {
case "set":
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
cancel()
exists := err != redis.Nil
if err != nil && err != redis.Nil {
// Treat transient errors as "value disappeared" — nothing to record.
return
}
prim, hint := ClassifyValue(val, exists)
i.upsert(fieldPath, key, keyHint, prim, hint)
case "del", "expire":
i.remove(fieldPath, key)
}
}
// parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
// path used as the map key. rootID is opaque to the indexer — currently a
// UUIDv7, historically a midhash256:abc token; both shapes parse identically
// because the split is on the first '/' after "file:".
//
// When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as
// a language code: the field name collapses to the prefix and key_hint is set.
func parseFieldPath(key string) (string, KeyHint, bool) {
if !strings.HasPrefix(key, "file:") {
return "", "", false
}
rest := strings.TrimPrefix(key, "file:")
slash := strings.Index(rest, "/")
if slash < 0 {
return "", "", false
}
property := rest[slash+1:]
if property == "" {
return "", "", false
}
if last := strings.LastIndex(property, "/"); last >= 0 {
tail := property[last+1:]
if langCodeRegex.MatchString(tail) {
return property[:last], KeyHintLanguageCode, true
}
}
return property, "", true
}
func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
i.mu.Lock()
defer i.mu.Unlock()
fs, ok := i.fields[fieldPath]
if !ok {
fs = newFieldState()
i.fields[fieldPath] = fs
}
if keyHint != "" {
fs.languageKeyed = true
}
if old, ok := fs.perKey[redisKey]; ok {
decContribution(fs, old)
}
fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
incContribution(fs, labelPair{prim: prim, hint: hint})
}
func (i *Indexer) remove(fieldPath, redisKey string) {
i.mu.Lock()
defer i.mu.Unlock()
fs, ok := i.fields[fieldPath]
if !ok {
return
}
old, ok := fs.perKey[redisKey]
if !ok {
return
}
delete(fs.perKey, redisKey)
decContribution(fs, old)
if len(fs.perKey) == 0 {
delete(i.fields, fieldPath)
}
}
func incContribution(fs *fieldState, lp labelPair) {
label := breakdownLabel(lp.prim, lp.hint)
fs.counts[label]++
if lp.prim == PrimUndefined {
fs.undefinedCount++
return
}
fs.typedCount++
fs.primitives[lp.prim]++
if lp.hint != "" {
fs.hintCounts[lp.hint]++
}
}
func decContribution(fs *fieldState, lp labelPair) {
label := breakdownLabel(lp.prim, lp.hint)
fs.counts[label]--
if fs.counts[label] <= 0 {
delete(fs.counts, label)
}
if lp.prim == PrimUndefined {
fs.undefinedCount--
return
}
fs.typedCount--
fs.primitives[lp.prim]--
if fs.primitives[lp.prim] <= 0 {
delete(fs.primitives, lp.prim)
}
if lp.hint != "" {
fs.hintCounts[lp.hint]--
if fs.hintCounts[lp.hint] <= 0 {
delete(fs.hintCounts, lp.hint)
}
}
}
// Snapshot returns the current schema. Safe to call concurrently with consumption.
func (i *Indexer) Snapshot() *SchemaResponse {
i.mu.RLock()
defer i.mu.RUnlock()
out := &SchemaResponse{
Fields: make(map[string]*FieldSchema, len(i.fields)),
GeneratedAt: time.Now().UTC(),
Source: "live",
}
for fp, fs := range i.fields {
out.Fields[fp] = computeFieldSchema(fs)
}
return out
}
func computeFieldSchema(fs *fieldState) *FieldSchema {
out := &FieldSchema{
Breakdown: make(map[string]int, len(fs.counts)),
}
for k, v := range fs.counts {
out.Breakdown[k] = v
}
nPrim := len(fs.primitives)
hasUndef := fs.undefinedCount > 0
switch {
case nPrim == 0 && hasUndef:
out.Type = PrimUndefined
case nPrim == 1 && !hasUndef:
for p := range fs.primitives {
out.Type = p
}
default:
out.Type = PrimMixed
}
// Strict hint promotion: exactly one hint observed, and it covers every
// typed (non-undefined) value. Hints are never set on mixed fields.
if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 {
for h, c := range fs.hintCounts {
if c == fs.typedCount {
out.Hint = h
}
}
}
if fs.languageKeyed {
out.KeyHint = KeyHintLanguageCode
}
return out
}
+48
View File
@@ -0,0 +1,48 @@
package schema
import "time"
// Primitive is the coarse value type used by the schema indexer.
// PrimMixed appears only as an aggregate label on a field, never per value.
type Primitive string
const (
PrimString Primitive = "string"
PrimInt Primitive = "int"
PrimFloat Primitive = "float"
PrimBool Primitive = "bool"
PrimJSONObj Primitive = "json-object"
PrimJSONArr Primitive = "json-array"
PrimUndefined Primitive = "undefined"
PrimMixed Primitive = "mixed"
)
// Hint is a semantic refinement on top of a primitive (e.g. cid, timestamp).
// A hint is promoted onto a field only when every non-undefined value matches it.
type Hint string
const (
HintCID Hint = "cid"
HintTimestamp Hint = "timestamp"
)
// KeyHint describes the shape of the subkey dimension when a field is keyed
// by a structured discriminator (e.g. title/eng, title/fre → language-code).
type KeyHint string
const KeyHintLanguageCode KeyHint = "language-code"
// FieldSchema is the per-field entry returned by GET /api/schema.
type FieldSchema struct {
Type Primitive `json:"type"`
Hint Hint `json:"hint,omitempty"`
KeyHint KeyHint `json:"key_hint,omitempty"`
Breakdown map[string]int `json:"breakdown"`
}
// SchemaResponse is the top-level payload for GET /api/schema.
type SchemaResponse struct {
Fields map[string]*FieldSchema `json:"fields"`
GeneratedAt time.Time `json:"generated_at"`
Source string `json:"source"`
}
+143
View File
@@ -0,0 +1,143 @@
package snapshot
import (
"archive/zip"
"errors"
"fmt"
"io"
"log"
"os"
"path/filepath"
"strings"
)
// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
// Trailing slash because ZIP entry names are forward-slash relative paths.
const PluginOutputArchivePrefix = "cache/plugin-output/"
// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
// file under it into zw at PluginOutputArchivePrefix + relPath.
//
// Returns the number of files added. A missing srcDir is not an error: it
// just means no plugin has produced output yet, so the section stays empty.
func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
info, err := os.Stat(srcDir)
if errors.Is(err, os.ErrNotExist) {
return 0, nil
}
if err != nil {
return 0, fmt.Errorf("stat %s: %w", srcDir, err)
}
if !info.IsDir() {
return 0, fmt.Errorf("%s is not a directory", srcDir)
}
count := 0
err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
if walkErr != nil {
return walkErr
}
if d.IsDir() {
return nil
}
rel, err := filepath.Rel(srcDir, path)
if err != nil {
return err
}
entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
w, err := zw.Create(entry)
if err != nil {
return err
}
f, err := os.Open(path)
if err != nil {
return err
}
defer f.Close()
if _, err := io.Copy(w, f); err != nil {
return err
}
count++
return nil
})
if err != nil {
return count, fmt.Errorf("walk %s: %w", srcDir, err)
}
return count, nil
}
// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
// the ZIP and writes it under dstDir. Existing files are overwritten when
// overwrite=true; otherwise they're left alone (treats existing files as the
// "mine wins" choice — same toggle the metadata import uses).
//
// Path traversal is rejected: any entry whose relative path resolves outside
// dstDir is skipped with a warning.
func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
if dstDir == "" {
return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
}
if err := os.MkdirAll(dstDir, 0o755); err != nil {
return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
}
cleanRoot, err := filepath.Abs(dstDir)
if err != nil {
return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
}
for _, f := range zr.File {
if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
continue
}
if strings.HasSuffix(f.Name, "/") {
continue
}
rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
// Reject anything that escapes the dst root.
if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
skipped++
continue
}
if !overwrite {
if _, err := os.Stat(dst); err == nil {
skipped++
continue
}
}
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
}
if err := writeZipFile(f, dst); err != nil {
return written, skipped, err
}
written++
}
return written, skipped, nil
}
func writeZipFile(f *zip.File, dst string) error {
rc, err := f.Open()
if err != nil {
return fmt.Errorf("open zip entry %s: %w", f.Name, err)
}
defer rc.Close()
out, err := os.Create(dst)
if err != nil {
return fmt.Errorf("create %s: %w", dst, err)
}
defer out.Close()
if _, err := io.Copy(out, rc); err != nil {
return fmt.Errorf("write %s: %w", dst, err)
}
return nil
}
+132
View File
@@ -0,0 +1,132 @@
package snapshot
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"log"
"os"
"time"
"github.com/metazla/meta-core/internal/storage"
)
// ExportOptions controls what an export includes.
type ExportOptions struct {
IncludePluginOutput bool
// PluginOutputDir is the absolute path on disk that holds plugin-produced
// artifacts (typically $FILES_PATH/plugin). Required when
// IncludePluginOutput is true.
PluginOutputDir string
// SourceHost is recorded in the manifest so imports can attribute
// where the snapshot came from (defaults to OS hostname).
SourceHost string
}
// Exporter writes snapshot ZIPs from a live Redis store.
type Exporter struct {
storage *storage.Client
}
func NewExporter(stor *storage.Client) *Exporter {
return &Exporter{storage: stor}
}
// Export streams a ZIP archive to w. The archive layout is:
//
// manifest.json
// index.json
// metadata/{cid}.json (one per known CID)
//
// Optional sections (hash-index, plugin-output) are gated by ExportOptions
// but are not yet implemented — they live on filesystems meta-core does not
// own, and need cross-service collection. The flag is plumbed for forward
// compatibility but does nothing today.
func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
if e.storage == nil || !e.storage.IsConnected() {
return Manifest{}, fmt.Errorf("storage not connected")
}
hashIDs, err := e.storage.GetAllHashIDs()
if err != nil {
return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
}
zw := zip.NewWriter(w)
defer zw.Close()
// 1. manifest.json — written last after we know FileCount, but we need
// the entry order to put it first in the archive for readers. Solve
// by buffering in memory: it's tiny.
host := opts.SourceHost
if host == "" {
if h, err := os.Hostname(); err == nil {
host = h
}
}
manifest := Manifest{
SchemaVersion: ManifestSchemaVersion,
GeneratedAt: time.Now().UTC(),
SourceHost: host,
FileCount: len(hashIDs),
Includes: Includes{
Metadata: true,
PluginOutput: opts.IncludePluginOutput,
},
}
if err := writeJSON(zw, ManifestFile, manifest); err != nil {
return Manifest{}, fmt.Errorf("write manifest: %w", err)
}
// 2. index.json — flat list of CIDs. Redundant with metadata/ but
// cheap and useful for readers that only need the keyset.
if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
return Manifest{}, fmt.Errorf("write index: %w", err)
}
// 3. metadata/{cid}.json — one nested document per CID.
for _, hashID := range hashIDs {
flat, err := e.storage.GetMetadataFlat(hashID)
if err != nil {
log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
continue
}
if len(flat) == 0 {
continue
}
nested := Reconstruct(flat)
entry := MetadataDir + hashID + ".json"
if err := writeJSON(zw, entry, nested); err != nil {
return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
}
}
// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
// streams each file into the ZIP. Heavy (~120 MB on a populated dev
// stack), so it's gated by the ExportOptions toggle.
if opts.IncludePluginOutput {
if opts.PluginOutputDir == "" {
return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
}
n, err := addPluginOutput(zw, opts.PluginOutputDir)
if err != nil {
return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
}
log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
}
log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
len(hashIDs), host, opts.IncludePluginOutput)
return manifest, nil
}
func writeJSON(zw *zip.Writer, name string, body interface{}) error {
w, err := zw.Create(name)
if err != nil {
return err
}
enc := json.NewEncoder(w)
enc.SetIndent("", " ")
return enc.Encode(body)
}
+195
View File
@@ -0,0 +1,195 @@
package snapshot
import (
"fmt"
"regexp"
"sort"
"strconv"
"strings"
)
// Flat keys use "/" as the separator. Property paths inside Redis are
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
const flatSep = "/"
// numericKey matches a path segment that is a non-negative decimal integer.
// Used to detect array slots (".../video/0/codec").
var numericKey = regexp.MustCompile(`^\d+$`)
// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
// Keys are property paths (no leading slash, no file:{cid}/ prefix).
//
// Sibling keys whose path segments are all sequential integers starting at 0
// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
// are coerced from string to int/float/bool/null when they parse cleanly so
// the exported JSON is human-readable; on import we round-trip back to strings.
func Reconstruct(flat map[string]string) interface{} {
root := map[string]interface{}{}
for path, value := range flat {
parts := strings.Split(path, flatSep)
setPath(root, parts, parseValue(value))
}
return convertNumericMapsToArrays(root)
}
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
// flat property map whose values are all strings (Redis-ready).
//
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
// values are stored as "" to preserve the field's presence — this matches
// what flattenMetadata in the TS implementation does.
func Flatten(value interface{}) map[string]string {
out := map[string]string{}
flattenInto(value, "", out)
return out
}
func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
if len(parts) == 0 {
return
}
cur := root
for i, part := range parts {
if i == len(parts)-1 {
cur[part] = leaf
return
}
next, ok := cur[part].(map[string]interface{})
if !ok {
next = map[string]interface{}{}
cur[part] = next
}
cur = next
}
}
func convertNumericMapsToArrays(v interface{}) interface{} {
switch t := v.(type) {
case map[string]interface{}:
// Recurse first so child arrays are detected before parent.
for k, child := range t {
t[k] = convertNumericMapsToArrays(child)
}
if !looksLikeArray(t) {
return t
}
// Sort numerically and emit as []interface{}.
idx := make([]int, 0, len(t))
for k := range t {
n, _ := strconv.Atoi(k)
idx = append(idx, n)
}
sort.Ints(idx)
arr := make([]interface{}, len(idx))
for i, n := range idx {
arr[i] = t[strconv.Itoa(n)]
}
return arr
case []interface{}:
for i := range t {
t[i] = convertNumericMapsToArrays(t[i])
}
return t
default:
return v
}
}
// looksLikeArray returns true when every key in m is a non-negative integer
// and the keys form a contiguous range starting at 0. Empty maps return false
// — we can't tell whether the original was {} or [], so prefer {}.
func looksLikeArray(m map[string]interface{}) bool {
if len(m) == 0 {
return false
}
seen := make(map[int]bool, len(m))
for k := range m {
if !numericKey.MatchString(k) {
return false
}
n, err := strconv.Atoi(k)
if err != nil || n < 0 {
return false
}
seen[n] = true
}
for i := 0; i < len(m); i++ {
if !seen[i] {
return false
}
}
return true
}
func flattenInto(v interface{}, path string, out map[string]string) {
switch t := v.(type) {
case nil:
out[path] = ""
case map[string]interface{}:
if len(t) == 0 && path != "" {
// Empty objects don't have a Redis representation; skip.
return
}
for k, child := range t {
next := k
if path != "" {
next = path + flatSep + k
}
flattenInto(child, next, out)
}
case []interface{}:
for i, child := range t {
next := strconv.Itoa(i)
if path != "" {
next = path + flatSep + next
}
flattenInto(child, next, out)
}
case string:
out[path] = t
case bool:
if t {
out[path] = "true"
} else {
out[path] = "false"
}
case float64:
// JSON numbers always decode to float64; emit ints without decimals.
if t == float64(int64(t)) {
out[path] = strconv.FormatInt(int64(t), 10)
} else {
out[path] = strconv.FormatFloat(t, 'f', -1, 64)
}
default:
out[path] = fmt.Sprintf("%v", t)
}
}
// parseValue mirrors the TS reconstruct logic: convert obvious primitives
// back to their typed form for nicer exported JSON, leave anything ambiguous
// as a string so we don't lose data (e.g. "5.1" channel layouts).
func parseValue(s string) interface{} {
if s == "" {
return nil
}
switch s {
case "true":
return true
case "false":
return false
case "null":
return nil
}
// Only treat as number when the string is a clean integer or decimal,
// no leading zeros / trailing junk. Matches the TS regex.
if isCleanNumber(s) {
if n, err := strconv.ParseFloat(s, 64); err == nil {
return n
}
}
return s
}
var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }
+262
View File
@@ -0,0 +1,262 @@
package snapshot
import (
"archive/zip"
"encoding/json"
"fmt"
"io"
"log"
"strings"
"github.com/metazla/meta-core/internal/storage"
)
// Mode controls how an import reconciles existing CIDs.
type Mode string
const (
// ModeReplace deletes every existing property of an imported CID and
// writes the snapshot's values verbatim. Untouched CIDs are left alone.
ModeReplace Mode = "replace"
// ModeMerge keeps unmentioned existing properties and only touches the
// fields present in the snapshot. Conflict resolution within merge is
// controlled by Conflict.
ModeMerge Mode = "merge"
)
// Conflict picks a winner for fields present on both sides during merge.
type Conflict string
const (
// ConflictMine keeps the existing (target) value when both sides have one.
ConflictMine Conflict = "mine"
// ConflictSource overwrites the existing value with the snapshot's value.
ConflictSource Conflict = "source"
)
// ImportOptions controls how Import applies a snapshot.
type ImportOptions struct {
Mode Mode
Conflict Conflict
DryRun bool
// PluginOutputDir is the on-disk destination for cache/plugin-output/*
// entries. When empty, plugin-output is silently skipped on import.
PluginOutputDir string
}
// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
type FileResult struct {
HashID string `json:"hashId"`
Added int `json:"added"` // new property keys written
Updated int `json:"updated"` // existing property keys whose value changed
Unchanged int `json:"unchanged"` // existing property keys whose value matched
Kept int `json:"kept"` // existing property keys preserved by ConflictMine
Deleted int `json:"deleted"` // existing property keys removed (replace mode)
Error string `json:"error,omitempty"`
}
// PluginOutputResult summarizes what happened to cache/plugin-output entries.
type PluginOutputResult struct {
Written int `json:"written"`
Skipped int `json:"skipped"`
}
// ImportResult is the response body for the import endpoint.
type ImportResult struct {
Mode Mode `json:"mode"`
Conflict Conflict `json:"conflict,omitempty"`
DryRun bool `json:"dryRun"`
TotalFiles int `json:"totalFiles"`
FilesOK int `json:"filesOk"`
FilesFailed int `json:"filesFailed"`
Files []FileResult `json:"files,omitempty"`
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
}
// Importer applies snapshot ZIPs to a live Redis store.
type Importer struct {
storage *storage.Client
}
func NewImporter(stor *storage.Client) *Importer {
return &Importer{storage: stor}
}
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
// *bytes.Reader after buffering the upload) and applies it to Redis.
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
if i.storage == nil || !i.storage.IsConnected() {
return nil, fmt.Errorf("storage not connected")
}
if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
}
if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
}
zr, err := zip.NewReader(r, size)
if err != nil {
return nil, fmt.Errorf("open zip: %w", err)
}
var manifest Manifest
if err := readJSON(zr, ManifestFile, &manifest); err != nil {
return nil, fmt.Errorf("read manifest: %w", err)
}
if manifest.SchemaVersion > ManifestSchemaVersion {
return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
manifest.SchemaVersion, ManifestSchemaVersion)
}
if manifest.SchemaVersion < ManifestSchemaVersion {
return nil, fmt.Errorf("snapshot schemaVersion %d is no longer supported (current: %d); v1 used content-hash root keys which are incompatible with the UUID-rooted schema",
manifest.SchemaVersion, ManifestSchemaVersion)
}
result := &ImportResult{
Mode: opts.Mode,
Conflict: opts.Conflict,
DryRun: opts.DryRun,
}
for _, f := range zr.File {
if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
continue
}
hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
if hashID == "" {
continue
}
fr := i.applyOne(f, hashID, opts)
result.TotalFiles++
if fr.Error != "" {
result.FilesFailed++
} else {
result.FilesOK++
}
result.Files = append(result.Files, fr)
}
// Plugin-output extraction. Only applied on real imports (not dry-run).
// If the snapshot doesn't contain a cache/plugin-output/ section, this is
// a no-op. If the destination is unset, we skip silently — the manifest's
// pluginOutput flag tells the caller it was on the source side either way.
if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
// Existing files are kept under ConflictMine / replace-doesn't-apply,
// overwritten under ConflictSource. Replace mode behaves like
// "overwrite" for cache files (it's the destructive choice for
// metadata; mirror that intent here).
overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
if err != nil {
return result, fmt.Errorf("extract plugin-output: %w", err)
}
result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
}
log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
return result, nil
}
func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
fr := FileResult{HashID: hashID}
rc, err := f.Open()
if err != nil {
fr.Error = fmt.Sprintf("open entry: %v", err)
return fr
}
defer rc.Close()
var nested interface{}
if err := json.NewDecoder(rc).Decode(&nested); err != nil {
fr.Error = fmt.Sprintf("decode json: %v", err)
return fr
}
incoming := Flatten(nested)
existing, err := i.storage.GetMetadataFlat(hashID)
if err != nil {
fr.Error = fmt.Sprintf("read existing: %v", err)
return fr
}
if existing == nil {
existing = map[string]string{}
}
switch opts.Mode {
case ModeReplace:
// Diff for reporting.
toWrite := map[string]string{}
for k, v := range incoming {
toWrite[k] = v
if cur, ok := existing[k]; ok {
if cur == v {
fr.Unchanged++
} else {
fr.Updated++
}
} else {
fr.Added++
}
}
for k := range existing {
if _, ok := incoming[k]; !ok {
fr.Deleted++
}
}
if !opts.DryRun {
if _, err := i.storage.DeleteMetadata(hashID); err != nil {
fr.Error = fmt.Sprintf("delete existing: %v", err)
return fr
}
if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
fr.Error = fmt.Sprintf("write: %v", err)
return fr
}
}
case ModeMerge:
toWrite := map[string]string{}
for k, v := range incoming {
cur, present := existing[k]
switch {
case !present:
toWrite[k] = v
fr.Added++
case cur == v:
fr.Unchanged++
case opts.Conflict == ConflictSource:
toWrite[k] = v
fr.Updated++
case opts.Conflict == ConflictMine:
fr.Kept++
}
}
if !opts.DryRun && len(toWrite) > 0 {
if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
fr.Error = fmt.Sprintf("merge: %v", err)
return fr
}
}
}
return fr
}
func readJSON(zr *zip.Reader, name string, out interface{}) error {
for _, f := range zr.File {
if f.Name != name {
continue
}
rc, err := f.Open()
if err != nil {
return err
}
defer rc.Close()
return json.NewDecoder(rc).Decode(out)
}
return fmt.Errorf("entry %q not found", name)
}
+53
View File
@@ -0,0 +1,53 @@
// Package snapshot implements export/import/wipe of MetaMesh metadata.
//
// The on-disk format is a ZIP archive containing one nested-JSON file per
// indexed root. Schema is defined by ManifestSchemaVersion; readers must
// reject archives whose schemaVersion exceeds what they know.
//
// v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries
// under metadata/<root>.json carry the flat string properties; the cids
// and duplicates Redis SETs are NOT serialized yet — they are rebuilt
// transparently on import because SetMetadataFlat / MergeMetadataFlat
// auto-register reverse-index aliases for any cid_*/midhash256 field they
// see (see internal/storage/cid_resolution.go).
//
// v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc".
// v1 archives can no longer be imported — they would resurrect the
// privileged-hashID schema this layer was built to retire.
package snapshot
import "time"
// ManifestSchemaVersion is the on-disk format version. Bump when the layout
// of metadata/{root}.json or the manifest changes in a non-additive way.
const ManifestSchemaVersion = 2
// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
const MetadataDir = "metadata/"
// ManifestFile is the path inside the ZIP that holds the manifest JSON.
const ManifestFile = "manifest.json"
// IndexFile is the path inside the ZIP that holds the file index (CID list).
const IndexFile = "index.json"
// Manifest is the metadata header stored at the root of every snapshot ZIP.
type Manifest struct {
SchemaVersion int `json:"schemaVersion"`
GeneratedAt time.Time `json:"generatedAt"`
SourceHost string `json:"sourceHost,omitempty"`
MetaCoreVer string `json:"metaCoreVersion,omitempty"`
FileCount int `json:"fileCount"`
Includes Includes `json:"includes"`
}
// Includes records which optional sections are present in a snapshot.
type Includes struct {
Metadata bool `json:"metadata"`
PluginOutput bool `json:"pluginOutput,omitempty"`
}
// IndexFileBody is the structure of index.json inside the ZIP.
type IndexFileBody struct {
HashIDs []string `json:"hashIds"`
}
+42
View File
@@ -0,0 +1,42 @@
package snapshot
import (
"fmt"
"github.com/metazla/meta-core/internal/storage"
)
// WipeScope picks which classes of data to clear.
type WipeScope struct {
Metadata bool // file:* keys + file:__index__
}
// WipeResult reports what was cleared.
type WipeResult struct {
MetadataDeleted int64 `json:"metadataDeleted"`
}
// Wiper clears metadata from Redis.
type Wiper struct {
storage *storage.Client
}
func NewWiper(stor *storage.Client) *Wiper {
return &Wiper{storage: stor}
}
// Wipe applies the requested scope. Returns the count of files removed.
func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
if w.storage == nil || !w.storage.IsConnected() {
return nil, fmt.Errorf("storage not connected")
}
res := &WipeResult{}
if scope.Metadata {
n, err := w.storage.ClearAllMetadata()
if err != nil {
return nil, fmt.Errorf("clear metadata: %w", err)
}
res.MetadataDeleted = n
}
return res, nil
}
+426
View File
@@ -0,0 +1,426 @@
package storage
import (
"context"
"fmt"
"strconv"
"strings"
"time"
"github.com/metazla/meta-core/internal/cid"
"github.com/redis/go-redis/v9"
)
// This file implements the CID-resolution layer that lives on top of the
// flat metadata storage in client.go. The short version:
//
// - Roots are opaque UUIDv7s — file:<uuid>/<property> for every property.
// The UUID is an internal implementation detail; the mental model is
// "N CIDs → one record of keys→metadata".
// - Every known CID for a file is registered as a reverse-index entry:
// cid:<bareCid> → <uuid> (Redis STRING)
// and as a member of the per-root bare-CID key-set:
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
// multicodec (see internal/cid), so there is no per-algorithm field name
// and no <algo>: token prefix. The canonical externally-visible CID is
// *derived by rank* from the key-set on read (GetMetadataDocument), never
// stored.
//
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
// any write of a cids/<cid> field and register the reverse-index entry, so
// writers that just emit the key-set members don't need to know about the
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
// already held by the caller, avoiding nested RLock (which can deadlock
// against a waiting writer).
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
// with a bare CID, it forms the full key: "cid:bafk…".
const CIDIndexPrefix = "cid:"
// CIDsKeyPrefix is the field-name prefix for the bare-CID key-set. Sibling
// CIDs live as flat keys file:<uuid>/cids/<cid> = "true"; the set IS the
// keyset, the value is the sentinel. Read by GetMetadataDocument (hoisted
// into the typed CIDs array) and by DeleteRoot/DeleteMetadata (to find the
// reverse-index entries that need cleanup).
const CIDsKeyPrefix = "cids/"
// DuplicatesField is the per-root SET key holding additional file paths
// whose content hashes to the same CIDs as this root. The watcher writes
// here when it sees a new path for content that's already indexed; meta-dup
// reads from it.
const DuplicatesField = "duplicates"
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
// cidStr is a bare multibase CIDv1 ("bafk…").
func (c *Client) buildCIDIndexKey(cidStr string) string {
return c.buildKey(CIDIndexPrefix + cidStr)
}
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
// in the file index. Returns the new UUID. The caller is expected to follow
// up with AddAlias for at least one CID — a root without any alias is
// reachable only via the index, which works but defeats the point.
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
uuid, err := NewUUID()
if err != nil {
return "", fmt.Errorf("uuid: %w", err)
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
pipe := c.client.Pipeline()
pipe.Set(ctx, prefix+"filePath", filePath, 0)
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
if _, err := pipe.Exec(ctx); err != nil {
return "", fmt.Errorf("mint pipeline: %w", err)
}
return uuid, nil
}
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
// Idempotent: registering the same CID twice is a no-op.
func (c *Client) AddAlias(uuid, cidStr string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.addAliasLocked(ctx, uuid, cidStr)
}
// addAliasLocked is the lock-free body of AddAlias. The caller must already
// hold c.mu (read or write). Split out so the alias-maintenance hooks
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
// the work without re-entering the lock — nested RLock can deadlock against
// a waiting writer.
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
if uuid == "" || cidStr == "" {
return fmt.Errorf("uuid and cid are required")
}
keysetKey := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + cidStr
// Self-pointing alias guard. If the caller is trying to register
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
// dual-root failure mode in disguise — it means somebody wrote
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
// write; leave any existing alias alone. Still record the key-set
// membership so the field-level write succeeds.
if uuid == cidStr {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
}
// Guard against the dual-root pattern: if cid:<cid> already points at a
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
// just record the key-set membership so it's still discoverable via the
// root → cids enumeration.
indexKey := c.buildCIDIndexKey(cidStr)
existing, err := c.client.Get(ctx, indexKey).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("read existing alias: %w", err)
}
if existing != "" && existing != uuid {
return c.client.Set(ctx, keysetKey, "true", 0).Err()
}
pipe := c.client.Pipeline()
pipe.Set(ctx, indexKey, uuid, 0)
pipe.Set(ctx, keysetKey, "true", 0)
if _, err := pipe.Exec(ctx); err != nil {
return fmt.Errorf("add alias pipeline: %w", err)
}
return nil
}
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
// lookup, an SSE consumer) to the actual storage root key.
//
// Callers send a bare CID (e.g. the midhash "bafk…" the watcher publishes on
// file:events, or any sibling CID). Roots are kept opaque and every CID is
// aliased into the reverse index:
//
// cid:bafk… → 01KSBHM*
//
// Without resolution, every /meta/<cid> write creates a parallel cid-rooted
// entry that the watcher's UUID never sees.
//
// Behaviour:
// - empty hash → empty (caller's problem)
// - hash matches a reverse-index entry → return UUID
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
//
// Errors are converted to "no resolution" — if Redis is down or the lookup
// fails we let the caller proceed with the bare hash and surface the Redis
// failure on the actual operation.
func (c *Client) ResolveRoot(hash string) string {
if hash == "" {
return hash
}
uuid, err := c.GetByCID(hash)
if err != nil || uuid == "" {
return hash
}
return uuid
}
// GetByCID resolves a bare CID to the UUID of its root. Returns ("", nil)
// when the CID is unknown; the empty string is the "not found" signal rather
// than a typed error so callers can branch without errors.As.
func (c *Client) GetByCID(cidStr string) (string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", fmt.Errorf("not connected")
}
if cidStr == "" {
return "", nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
if err == redis.Nil {
return "", nil
}
if err != nil {
return "", fmt.Errorf("get cid index: %w", err)
}
return uuid, nil
}
// AddDuplicatePath records that path is another physical location whose
// content matches uuid. Returns true if path was newly added (not already
// in the set). The watcher calls this when it sees a file whose midhash
// matches an existing root but at a different filePath.
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
if uuid == "" || path == "" {
return false, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
if err != nil {
return false, fmt.Errorf("sadd duplicates: %w", err)
}
return added > 0, nil
}
// cidsForRootLocked scans the bare-CID key-set for a root and returns its
// members (bare CIDs, with the cids/ prefix stripped). Caller must hold c.mu.
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
prefix := c.buildKeyPrefix(uuid) + CIDsKeyPrefix
var out []string
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, fmt.Errorf("scan cids: %w", err)
}
for _, k := range keys {
out = append(out, strings.TrimPrefix(k, prefix))
}
cursor = next
if cursor == 0 {
break
}
}
return out, nil
}
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
// (including the cids/<cid> key-set), every reverse-index entry for those
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
// is a successful no-op.
func (c *Client) DeleteRoot(uuid string) error {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
if uuid == "" {
return nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(uuid)
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
// the list of reverse-index entries to clean up.
cids, err := c.cidsForRootLocked(ctx, uuid)
if err != nil {
return err
}
// 2. Delete every reverse-index entry in one shot.
if len(cids) > 0 {
keys := make([]string, 0, len(cids))
for _, cidStr := range cids {
keys = append(keys, c.buildCIDIndexKey(cidStr))
}
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del cid index: %w", err)
}
}
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return fmt.Errorf("scan root keys: %w", err)
}
if len(keys) > 0 {
if err := c.client.Del(ctx, keys...).Err(); err != nil {
return fmt.Errorf("del root keys: %w", err)
}
}
cursor = next
if cursor == 0 {
break
}
}
// 4. Remove from the file index set.
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
return fmt.Errorf("srem index: %w", err)
}
return nil
}
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
// root. If it's a key-set member — field name starts with "cids/" — it
// registers the implied CID in the reverse index. Writers emit cids/<cid>
// members without knowing about the reverse index; this hook keeps it in
// sync transparently.
//
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
// (logged, not returned): an out-of-sync reverse index is recoverable via a
// sweep, while a failed property write is not.
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
cidStr := cidFromKeysetField(field)
if cidStr == "" {
return nil
}
return c.addAliasLocked(ctx, uuid, cidStr)
}
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
// over every entry in a metadata map. Used by SetMetadataFlat /
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
// hold c.mu.
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
for k := range metadata {
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
}
}
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
// or "" if the field isn't a key-set member.
func cidFromKeysetField(field string) string {
if !strings.HasPrefix(field, CIDsKeyPrefix) {
return ""
}
return strings.TrimPrefix(field, CIDsKeyPrefix)
}
// MetadataDocument is the complete state of one file as the public API
// returns it: every flat string field (the cids/<cid> members included),
// plus the CIDs hoisted into a typed array, the canonical CID derived by
// rank, and the duplicates set. Distinct from GetMetadataFlat (which only
// returns the raw string keys).
type MetadataDocument struct {
Flat map[string]string `json:"metadata"`
CIDs []string `json:"cids"`
Canonical string `json:"canonical_cid,omitempty"`
Duplicates []string `json:"duplicates"`
}
// GetMetadataDocument returns the full document for a root: every string
// property under file:<uuid>/* (cids/<cid> members included in Flat), the
// CIDs hoisted into a typed array, the canonical CID computed by rank, and
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
// never existed).
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
if uuid == "" {
return nil, nil
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
flat, err := c.getMetadataFlatInternal(ctx, uuid)
if err != nil {
return nil, err
}
prefix := c.buildKeyPrefix(uuid)
// Hoist the cids/<cid> key-set members into a typed array and derive the
// canonical CID by rank. The members remain in Flat too — that's how
// flat-map consumers (meta-share ingest) see them.
cids := make([]string, 0)
canonical := ""
for k := range flat {
if !strings.HasPrefix(k, CIDsKeyPrefix) {
continue
}
cidStr := strings.TrimPrefix(k, CIDsKeyPrefix)
cids = append(cids, cidStr)
if canonical == "" {
canonical = cidStr
} else {
canonical = cid.Better(canonical, cidStr)
}
}
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
if err != nil && err != redis.Nil {
return nil, fmt.Errorf("smembers duplicates: %w", err)
}
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
return nil, nil
}
return &MetadataDocument{
Flat: flat,
CIDs: cids,
Canonical: canonical,
Duplicates: dups,
}, nil
}
+717 -43
View File
@@ -4,6 +4,8 @@ import (
"context" "context"
"fmt" "fmt"
"log" "log"
"sort"
"strconv"
"strings" "strings"
"sync" "sync"
"time" "time"
@@ -190,6 +192,10 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
return fmt.Errorf("sadd index failed: %w", err) return fmt.Errorf("sadd index failed: %w", err)
} }
// Auto-register reverse-index aliases for any cid_* or midhash256 fields
// in the payload. See cid_resolution.go for the rules.
c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
return nil return nil
} }
@@ -261,11 +267,17 @@ func (c *Client) SetProperty(hashID, property, value string) error {
return fmt.Errorf("sadd index failed: %w", err) return fmt.Errorf("sadd index failed: %w", err)
} }
// Auto-register reverse-index alias if this is a CID field.
_ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value)
return nil return nil
} }
// DeleteMetadata deletes all metadata for a file // DeleteMetadata deletes all metadata for a file
// Uses SCAN + DEL for flat keys: file:{hashId}/* // Uses SCAN + DEL for flat keys: file:{hashId}/*
// Also unmaps every reverse-index entry registered for this root, so a
// subsequent GET /api/meta/<old-cid> correctly returns 404 instead of
// pointing at a UUID with no keys.
func (c *Client) DeleteMetadata(hashID string) (int64, error) { func (c *Client) DeleteMetadata(hashID string) (int64, error) {
c.mu.RLock() c.mu.RLock()
defer c.mu.RUnlock() defer c.mu.RUnlock()
@@ -279,6 +291,22 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) {
prefix := c.buildKeyPrefix(hashID) prefix := c.buildKeyPrefix(hashID)
// Read the cids key-set first — once file:<uuid>/* is gone we've lost the
// list of reverse-index keys that need cleanup.
cids, err := c.cidsForRootLocked(ctx, hashID)
if err != nil {
return 0, err
}
if len(cids) > 0 {
idxKeys := make([]string, 0, len(cids))
for _, cidStr := range cids {
idxKeys = append(idxKeys, c.buildCIDIndexKey(cidStr))
}
if err := c.client.Del(ctx, idxKeys...).Err(); err != nil {
return 0, fmt.Errorf("del cid index: %w", err)
}
}
// Scan and delete all keys with this prefix // Scan and delete all keys with this prefix
var deletedCount int64 var deletedCount int64
var cursor uint64 var cursor uint64
@@ -319,53 +347,73 @@ func (c *Client) CountFiles() (int, error) {
return len(hashIDs), nil return len(hashIDs), nil
} }
// LookupPathByCID searches all file metadata for a matching CID // LookupPathByCID resolves a bare CID to its file path on disk via the
// Returns the file path if found in poster/posterPath or backdrop/backdropPath // reverse index. O(1) lookup followed by a single GET of file:<uuid>/filePath.
func (c *Client) LookupPathByCID(cid string) (string, error) { //
c.mu.RLock() // Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS
defer c.mu.RUnlock() // CID, a btih, etc. (they all alias to the same root). The bare CID is what
// the editor passes to /api/file/{cid}/info and what plugins write (e.g.
if c.client == nil { // `backdrop=<bare-cid>`).
return "", fmt.Errorf("not connected") //
} // Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is
// resolved through this same path: the plugin output directory is a
if cid == "" { // watcher root (see config.DefaultWatcherPaths), so every plugin artefact
return "", nil // gets a midhash alias and resolves through the reverse index like any
} // other file. There is no path-companion sidecar field.
func (c *Client) LookupPathByCID(cidStr string) (string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second) uuid, err := c.GetByCID(cidStr)
defer cancel()
// Get all hash IDs first
hashIDs, err := c.getAllHashIDsInternal(ctx)
if err != nil { if err != nil {
return "", err return "", err
} }
if uuid == "" {
// For each hash, check poster and backdrop CIDs using flat keys
for _, hashID := range hashIDs {
prefix := c.buildKeyPrefix(hashID)
// Check poster
posterCID, err := c.client.Get(ctx, prefix+"poster").Result()
if err == nil && posterCID == cid {
posterPath, err := c.client.Get(ctx, prefix+"posterPath").Result()
if err == nil && posterPath != "" {
return posterPath, nil
}
}
// Check backdrop
backdropCID, err := c.client.Get(ctx, prefix+"backdrop").Result()
if err == nil && backdropCID == cid {
backdropPath, err := c.client.Get(ctx, prefix+"backdropPath").Result()
if err == nil && backdropPath != "" {
return backdropPath, nil
}
}
}
return "", nil return "", nil
}
fp, err := c.GetProperty(uuid, "filePath")
if err != nil {
return "", err
}
return normalizeFilesRelativePath(fp), nil
}
// normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
// safely filepath.Join(FilesPath, …) without double-prefixing. Different
// plugins store paths inconsistently — poster/backdrop use FilesPath-relative
// ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…").
func normalizeFilesRelativePath(p string) string {
return strings.TrimPrefix(p, "/files")
}
// getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock.
// Callers must already hold c.mu.
func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) {
prefix := c.buildKeyPrefix(hashID)
result := make(map[string]string)
var cursor uint64
for {
keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
if err != nil {
return nil, fmt.Errorf("scan failed: %w", err)
}
if len(keys) > 0 {
values, err := c.client.MGet(ctx, keys...).Result()
if err != nil {
return nil, fmt.Errorf("mget failed: %w", err)
}
for i, key := range keys {
if values[i] == nil {
continue
}
field := strings.TrimPrefix(key, prefix)
result[field] = values[i].(string)
}
}
cursor = nextCursor
if cursor == 0 {
break
}
}
return result, nil
} }
// getAllHashIDsInternal is an internal version that doesn't acquire locks // getAllHashIDsInternal is an internal version that doesn't acquire locks
@@ -413,6 +461,10 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
return len(metadata), fmt.Errorf("sadd index failed: %w", err) return len(metadata), fmt.Errorf("sadd index failed: %w", err)
} }
// Auto-register reverse-index aliases for any cid_* or midhash256 fields
// in the payload. See cid_resolution.go for the rules.
c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
return len(metadata), nil return len(metadata), nil
} }
@@ -535,6 +587,211 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
return true, c.client.Set(ctx, key, newValue, 0).Err() return true, c.client.Set(ctx, key, newValue, 0).Err()
} }
// FileTuple holds the (uuid, path, size, mtime, midhash) record for one
// file. Used by the watcher hydrator at startup to avoid re-mid-hashing
// files whose tuple matches what's already in Redis.
//
// HashID is the root identifier (UUIDv7 in the current model). Hydration
// uses HashID + (FilePath, Size, MtimeNano) to seed the in-memory state so
// the next scan can skip re-mid-hashing files whose size+mtime are unchanged.
type FileTuple struct {
HashID string
FilePath string
Size int64
MtimeNano int64
}
// GetTuplesForAllFiles returns one FileTuple per indexed root, fetched in
// batched MGETs (no SCAN per root). Skips files missing any of filePath,
// sizeByte, or mtimeNano. Sub-second on local Redis for ~10K files.
//
// The midhash is intentionally NOT fetched here: it now lives only as a
// cids/<cid> key-set member (no named field), and the skip-rehash decision
// is purely size+mtime — so reading it would cost a SCAN per file for no
// benefit.
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
if err != nil {
return nil, fmt.Errorf("smembers index: %w", err)
}
if len(hashIDs) == 0 {
return nil, nil
}
// Build the key list: 4 keys per root, in a fixed order so we can decode
// the MGET response by position. fields[i] for hashIDs[j] sits at
// keys[j*4+i].
const fieldsPerFile = 3
fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"}
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
for _, h := range hashIDs {
prefix := c.buildKeyPrefix(h)
for _, f := range fields {
keys = append(keys, prefix+f)
}
}
// Chunked MGET — Redis caps args per command; 1000 is a safe batch.
const batch = 1000
values := make([]interface{}, 0, len(keys))
for start := 0; start < len(keys); start += batch {
end := start + batch
if end > len(keys) {
end = len(keys)
}
chunk, err := c.client.MGet(ctx, keys[start:end]...).Result()
if err != nil {
return nil, fmt.Errorf("mget chunk [%d:%d]: %w", start, end, err)
}
values = append(values, chunk...)
}
out := make([]FileTuple, 0, len(hashIDs))
for i, h := range hashIDs {
base := i * fieldsPerFile
filePath, _ := values[base].(string)
sizeStr, _ := values[base+1].(string)
mtimeStr, _ := values[base+2].(string)
if filePath == "" || sizeStr == "" || mtimeStr == "" {
continue
}
size, err := strconv.ParseInt(sizeStr, 10, 64)
if err != nil {
continue
}
mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
if err != nil {
continue
}
out = append(out, FileTuple{
HashID: h,
FilePath: filePath,
Size: size,
MtimeNano: mtime,
})
}
return out, nil
}
// SearchTuple is one record's search state: its hashID, a lowercased haystack of
// its searchable text fields (for token/substring match), and the full flat
// metadata map (so a search hit is returned straight from memory — no per-result
// Redis fetch).
type SearchTuple struct {
HashID string
Haystack string
Fields map[string]string
}
// searchIndexFields are the metadata fields folded into a record's search
// haystack — must stay in sync with the fields the search handler documents.
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
// GetSearchIndexTuples reads EVERY record's full metadata in a single keyspace
// SCAN pass over the flat `file:{hashID}/{field}` keys plus batched MGETs,
// grouped by record — one bulk pass instead of a SCAN+MGET per record (which is
// O(records) Redis round-trips, ~40s over 72k field keys). It is the bulk source
// the API folds into its in-memory search index, so per-request search does zero
// Redis round-trips (match AND result metadata both come from memory).
func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
defer cancel()
filePrefix := c.buildKey("file:") // key layout: {filePrefix}{hashID}/{field}
indexKey := c.buildIndexKey() // {filePrefix}__index__ (a SET, skip it)
pattern := filePrefix + "*"
records := make(map[string]map[string]string)
keyBatch := make([]string, 0, 1000)
flush := func() error {
if len(keyBatch) == 0 {
return nil
}
values, err := c.client.MGet(ctx, keyBatch...).Result()
if err != nil {
return fmt.Errorf("mget: %w", err)
}
for i, k := range keyBatch {
v, ok := values[i].(string)
if !ok {
continue
}
// {filePrefix}{hashID}/{field}; hashID has no '/', field may.
rest := strings.TrimPrefix(k, filePrefix)
slash := strings.IndexByte(rest, '/')
if slash < 0 {
continue
}
hashID, field := rest[:slash], rest[slash+1:]
m := records[hashID]
if m == nil {
m = make(map[string]string)
records[hashID] = m
}
m[field] = v
}
keyBatch = keyBatch[:0]
return nil
}
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
if err != nil {
return nil, fmt.Errorf("scan: %w", err)
}
for _, k := range keys {
if k == indexKey {
continue
}
keyBatch = append(keyBatch, k)
if len(keyBatch) >= 1000 {
if err := flush(); err != nil {
return nil, err
}
}
}
cursor = next
if cursor == 0 {
break
}
}
if err := flush(); err != nil {
return nil, err
}
out := make([]SearchTuple, 0, len(records))
for hashID, fields := range records {
var sb strings.Builder
for _, f := range searchIndexFields {
if v, ok := fields[f]; ok && v != "" {
sb.WriteString(strings.ToLower(v))
sb.WriteByte('\n')
}
}
out = append(out, SearchTuple{HashID: hashID, Haystack: sb.String(), Fields: fields})
}
return out, nil
}
// GetMemoryInfo returns Redis memory usage information // GetMemoryInfo returns Redis memory usage information
func (c *Client) GetMemoryInfo() (string, error) { func (c *Client) GetMemoryInfo() (string, error) {
c.mu.RLock() c.mu.RLock()
@@ -620,6 +877,378 @@ func (c *Client) ClearAllMetadata() (int64, error) {
return deletedCount, nil return deletedCount, nil
} }
// ScanKeys lists the immediate children at the given prefix, S3-style.
//
// Walks the keyspace under prefix using Redis SCAN. When delimiter is set,
// keys with a delimiter occurrence after the prefix collapse into a branch
// path (the prefix up to and including that first delimiter); the rest are
// returned as leaves. When delimiter is empty all matching keys are leaves.
//
// Continues SCAN-ing until either the keyspace is exhausted (nextCursor == "0")
// or leaves+branches reaches maxResults — whichever comes first. The caller
// passes nextCursor back to continue. Results are deduped and sorted.
func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ([]string, []string, string, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, nil, "", fmt.Errorf("not connected")
}
if maxResults <= 0 {
maxResults = 500
}
var cursor uint64
if cursorStr != "" {
if v, err := strconv.ParseUint(cursorStr, 10, 64); err == nil {
cursor = v
}
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
fullPrefix := c.buildKey(prefix)
pattern := escapeGlob(fullPrefix) + "*"
leafSet := make(map[string]struct{})
branchSet := make(map[string]struct{})
for {
// SCAN COUNT is a hint; ask for a larger batch when grouping by
// delimiter since many keys collapse into one branch.
scanCount := int64(maxResults * 4)
if scanCount > 5000 {
scanCount = 5000
}
keys, next, err := c.client.Scan(ctx, cursor, pattern, scanCount).Result()
if err != nil {
return nil, nil, "", fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
userKey := strings.TrimPrefix(k, c.prefix)
suffix := strings.TrimPrefix(userKey, prefix)
if delimiter != "" {
if idx := strings.Index(suffix, delimiter); idx >= 0 {
branchSet[prefix+suffix[:idx+len(delimiter)]] = struct{}{}
continue
}
}
leafSet[userKey] = struct{}{}
}
cursor = next
if cursor == 0 {
break
}
if len(leafSet)+len(branchSet) >= maxResults {
break
}
}
leaves := make([]string, 0, len(leafSet))
for k := range leafSet {
leaves = append(leaves, k)
}
branches := make([]string, 0, len(branchSet))
for k := range branchSet {
branches = append(branches, k)
}
sort.Strings(leaves)
sort.Strings(branches)
return leaves, branches, strconv.FormatUint(cursor, 10), nil
}
// FindMatch is a single value-search hit.
type FindMatch struct {
Key string `json:"key"`
Value string `json:"value"`
EntryPath string `json:"entryPath"`
Field string `json:"field"`
}
// FindByValue scans every key whose path ends with /<field> for each field
// in fields, MGETs them, and returns the keys whose value contains the
// substring (case-insensitive). Returns truncated when limit was hit before
// the scan exhausted.
//
// EntryPath is the parent prefix up to (and including) the slash before
// field — i.e. the file branch path. Useful for revealing the matched entry
// in the tree.
func (c *Client) FindByValue(contains string, fields []string, limit int) ([]FindMatch, bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, false, fmt.Errorf("not connected")
}
if contains == "" || len(fields) == 0 {
return nil, false, nil
}
if limit <= 0 {
limit = 200
}
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
defer cancel()
needle := strings.ToLower(contains)
var matches []FindMatch
truncated := false
for _, field := range fields {
field = strings.TrimSpace(field)
if field == "" {
continue
}
if len(matches) >= limit {
truncated = true
break
}
// Walk all keys ending with /field via SCAN MATCH "*/<field>".
pattern := "*/" + escapeGlob(field)
if c.prefix != "" {
pattern = escapeGlob(c.prefix) + pattern
}
var keysToFetch []string
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
if err != nil {
return nil, false, fmt.Errorf("scan failed: %w", err)
}
keysToFetch = append(keysToFetch, keys...)
cursor = next
if cursor == 0 {
break
}
if len(keysToFetch) > 20000 {
break
}
}
// MGET in chunks (Redis limits the number of args per command).
for chunkStart := 0; chunkStart < len(keysToFetch); chunkStart += 500 {
chunkEnd := chunkStart + 500
if chunkEnd > len(keysToFetch) {
chunkEnd = len(keysToFetch)
}
chunk := keysToFetch[chunkStart:chunkEnd]
values, err := c.client.MGet(ctx, chunk...).Result()
if err != nil {
return nil, false, fmt.Errorf("mget failed: %w", err)
}
for i, v := range values {
if v == nil {
continue
}
sv, _ := v.(string)
if !strings.Contains(strings.ToLower(sv), needle) {
continue
}
userKey := strings.TrimPrefix(chunk[i], c.prefix)
entryPath := strings.TrimSuffix(userKey, field)
matches = append(matches, FindMatch{
Key: userKey,
Value: sv,
EntryPath: entryPath,
Field: field,
})
if len(matches) >= limit {
truncated = true
break
}
}
if truncated {
break
}
}
}
sort.Slice(matches, func(i, j int) bool { return matches[i].Key < matches[j].Key })
return matches, truncated, nil
}
// SearchKeys returns up to limit keys whose full path contains the substring.
// truncated is true when the search stopped at limit before exhausting Redis.
func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, false, fmt.Errorf("not connected")
}
if contains == "" {
return nil, false, nil
}
if limit <= 0 {
limit = 500
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
pattern := escapeGlob(c.prefix) + "*" + escapeGlob(contains) + "*"
results := make([]string, 0, limit)
truncated := false
var cursor uint64
for {
keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
if err != nil {
return nil, false, fmt.Errorf("scan failed: %w", err)
}
for _, k := range keys {
if len(results) >= limit {
truncated = true
break
}
results = append(results, strings.TrimPrefix(k, c.prefix))
}
cursor = next
if truncated || cursor == 0 {
break
}
}
sort.Strings(results)
return results, truncated, nil
}
// GetKeyInfo returns the Redis type ("string", "set", "hash", ...) and, for
// string-typed keys, the value. exists is false when the key is absent.
func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, err error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return "", "", false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
full := c.buildKey(key)
t, err := c.client.Type(ctx, full).Result()
if err != nil {
return "", "", false, fmt.Errorf("type failed: %w", err)
}
if t == "none" {
return "", "", false, nil
}
if t != "string" {
return t, "", true, nil
}
v, err := c.client.Get(ctx, full).Result()
if err == redis.Nil {
return "", "", false, nil
}
if err != nil {
return "", "", false, fmt.Errorf("get failed: %w", err)
}
return "string", v, true, nil
}
// SetRawValue writes a string value at the exact key. Refuses to overwrite a
// key whose existing Redis type is not "string".
func (c *Client) SetRawValue(key, value string) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
full := c.buildKey(key)
t, err := c.client.Type(ctx, full).Result()
if err != nil {
return fmt.Errorf("type failed: %w", err)
}
if t != "none" && t != "string" {
return fmt.Errorf("key has incompatible type %q (only string is editable)", t)
}
return c.client.Set(ctx, full, value, 0).Err()
}
// DeleteRawKey deletes the exact key. Returns true if the key existed.
func (c *Client) DeleteRawKey(key string) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
n, err := c.client.Del(ctx, c.buildKey(key)).Result()
if err != nil {
return false, err
}
return n > 0, nil
}
// IndexAdd adds a hash ID to the file index set. Idempotent.
func (c *Client) IndexAdd(hashID string) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
return c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err()
}
// IndexRemoveIfEmpty removes hashID from the index set when no
// file:<hashID>/* keys remain. Returns true if the index entry was removed.
func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return false, fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
prefix := c.buildKeyPrefix(hashID)
keys, _, err := c.client.Scan(ctx, 0, prefix+"*", 1).Result()
if err != nil {
return false, fmt.Errorf("scan failed: %w", err)
}
if len(keys) > 0 {
return false, nil
}
removed, err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Result()
if err != nil {
return false, fmt.Errorf("srem failed: %w", err)
}
return removed > 0, nil
}
// escapeGlob escapes Redis SCAN MATCH glob metacharacters so a literal string
// can be embedded inside a pattern. Redis uses *, ?, [, \ as metacharacters.
func escapeGlob(s string) string {
var b strings.Builder
b.Grow(len(s))
for _, r := range s {
switch r {
case '*', '?', '[', ']', '\\':
b.WriteByte('\\')
}
b.WriteRune(r)
}
return b.String()
}
// GetRedisClient returns the underlying redis client for advanced operations // GetRedisClient returns the underlying redis client for advanced operations
func (c *Client) GetRedisClient() *redis.Client { func (c *Client) GetRedisClient() *redis.Client {
c.mu.RLock() c.mu.RLock()
@@ -685,6 +1314,51 @@ func (c *Client) ClearStream(stream string) error {
return nil return nil
} }
// StreamEntry represents one entry from a Redis stream.
type StreamEntry struct {
ID string
Values map[string]interface{}
}
// XRevRange reads stream entries from newest to oldest.
// start defaults to "+" (newest) and stop defaults to "-" (oldest) when empty.
// If count > 0, at most that many entries are returned.
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
c.mu.RLock()
defer c.mu.RUnlock()
if c.client == nil {
return nil, fmt.Errorf("not connected")
}
if start == "" {
start = "+"
}
if stop == "" {
stop = "-"
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
var msgs []redis.XMessage
var err error
if count > 0 {
msgs, err = c.client.XRevRangeN(ctx, stream, start, stop, count).Result()
} else {
msgs, err = c.client.XRevRange(ctx, stream, start, stop).Result()
}
if err != nil {
return nil, fmt.Errorf("xrevrange failed: %w", err)
}
entries := make([]StreamEntry, 0, len(msgs))
for _, m := range msgs {
entries = append(entries, StreamEntry{ID: m.ID, Values: m.Values})
}
return entries, nil
}
// GetStreamLength returns the length of a Redis stream // GetStreamLength returns the length of a Redis stream
func (c *Client) GetStreamLength(stream string) (int64, error) { func (c *Client) GetStreamLength(stream string) (int64, error) {
c.mu.RLock() c.mu.RLock()
+41
View File
@@ -0,0 +1,41 @@
package storage
import (
"context"
"fmt"
)
// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
// legacy storage model: /meta/{hash} write handlers that didn't resolve
// CID → UUID first created a parallel midhash-rooted entry alongside the
// watcher's UUID root.
//
// That pattern is now structurally impossible:
// - writes resolve via ResolveRoot before landing (see handlers), and
// - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
// UUID roots, with the reverse index cid:<cid> → uuid as the single
// resolution path.
//
// The legacy on-disk shape this migration targeted (midhash-rooted entries,
// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
// cut-over wipes Redis, so there is nothing to migrate. The function is kept
// as a no-op so the admin endpoint and its wrappers continue to compile and
// respond.
func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
if c.client == nil {
return 0, fmt.Errorf("not connected")
}
return 0, nil
}
// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
// handler and the startup hook.
func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
return c.MigrateDualRoots(context.Background())
}
// MigrateDualRootsReport returns a human-readable summary. With the dual-root
// pattern retired (see MigrateDualRoots), this always reports nothing to do.
func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
}
+83
View File
@@ -0,0 +1,83 @@
package storage
import (
"context"
"fmt"
"strconv"
"time"
"github.com/redis/go-redis/v9"
)
// SchemaVersion is the Redis-layout version this build of meta-core expects.
// Distinct from the snapshot manifest version (which is on-disk format);
// this number describes the in-memory Redis schema — primarily, "are roots
// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
//
// Bump only when a Redis-layout change can't be detected from the data
// itself or healed by the auto-alias hooks. Each bump is a flag day in
// alpha: operators wipe and restart.
const SchemaVersion = 2
// SchemaVersionKey is the sentinel under which we record SchemaVersion in
// Redis. Checked on every meta-core boot.
const SchemaVersionKey = "meta-core:schema-version"
// EnsureSchemaVersion verifies the Redis store matches the expected schema
// version. Called once at startup, before any other reads/writes.
//
// Behavior:
// - sentinel present and matches expected → OK
// - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
// - sentinel missing AND file:__index__ non-empty → legacy data, refuse
// - sentinel present but mismatched → refuse, point operator at clean:all
//
// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
// The error message names the wipe remediation because in alpha there
// is no automated migration — operators bring up a clean store.
func (c *Client) EnsureSchemaVersion(expected int) error {
c.mu.Lock()
defer c.mu.Unlock()
if c.client == nil {
return fmt.Errorf("not connected")
}
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
defer cancel()
key := c.buildKey(SchemaVersionKey)
have, err := c.client.Get(ctx, key).Result()
if err != nil && err != redis.Nil {
return fmt.Errorf("get schema version: %w", err)
}
if err == redis.Nil {
// Sentinel absent. Fresh store iff there's no file:__index__ membership.
n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
if sErr != nil && sErr != redis.Nil {
return fmt.Errorf("scard file:__index__: %w", sErr)
}
if n > 0 {
return fmt.Errorf(
"redis has %d indexed roots but no %q sentinel — this is alpha "+
"data from before the UUID-rooted schema landed. Run "+
"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
"to wipe and restart, or downgrade meta-core to a compatible build",
n, SchemaVersionKey)
}
if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
return fmt.Errorf("set schema version: %w", err)
}
return nil
}
if have == strconv.Itoa(expected) {
return nil
}
return fmt.Errorf(
"redis schema version is %q, this build of meta-core expects %d. "+
"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
"build that matches the existing data",
have, expected)
}
+62
View File
@@ -0,0 +1,62 @@
package storage
import (
"github.com/google/uuid"
)
// NewUUID returns a new UUIDv7 encoded as 26-char Crockford Base32 (ULID
// layout). Used as the root key for file metadata entries.
//
// Why this format over canonical 8-4-4-4-12 hex:
// - 26 chars vs 36 — shorter Redis keys, less prefix space waste
// - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
// given time window, restoring some of the debug-ability lost vs the
// previous midhash256:abc rooting
// - Case-insensitive — no ambiguity when copying from logs
// - Crockford alphabet skips I/L/O/U, so eyeballed inspection of a key
// can't be confused for an ambient digit/letter typo
func NewUUID() (string, error) {
u, err := uuid.NewV7()
if err != nil {
return "", err
}
return encodeCrockford(u[:]), nil
}
const crockfordAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
// encodeCrockford encodes a 16-byte value as 26 Crockford Base32 chars.
// 128 bits packs into 130 bits (26 × 5); the top 2 bits of the first char
// carry the high bits of byte 0, the remaining 3 are implicit zero. This
// matches the ULID encoding so the output is interchangeable with ULIDs in
// any tool that accepts them.
func encodeCrockford(b []byte) string {
out := make([]byte, 26)
out[0] = crockfordAlphabet[(b[0]&0xE0)>>5]
out[1] = crockfordAlphabet[b[0]&0x1F]
out[2] = crockfordAlphabet[(b[1]&0xF8)>>3]
out[3] = crockfordAlphabet[((b[1]&0x07)<<2)|((b[2]&0xC0)>>6)]
out[4] = crockfordAlphabet[(b[2]&0x3E)>>1]
out[5] = crockfordAlphabet[((b[2]&0x01)<<4)|((b[3]&0xF0)>>4)]
out[6] = crockfordAlphabet[((b[3]&0x0F)<<1)|((b[4]&0x80)>>7)]
out[7] = crockfordAlphabet[(b[4]&0x7C)>>2]
out[8] = crockfordAlphabet[((b[4]&0x03)<<3)|((b[5]&0xE0)>>5)]
out[9] = crockfordAlphabet[b[5]&0x1F]
out[10] = crockfordAlphabet[(b[6]&0xF8)>>3]
out[11] = crockfordAlphabet[((b[6]&0x07)<<2)|((b[7]&0xC0)>>6)]
out[12] = crockfordAlphabet[(b[7]&0x3E)>>1]
out[13] = crockfordAlphabet[((b[7]&0x01)<<4)|((b[8]&0xF0)>>4)]
out[14] = crockfordAlphabet[((b[8]&0x0F)<<1)|((b[9]&0x80)>>7)]
out[15] = crockfordAlphabet[(b[9]&0x7C)>>2]
out[16] = crockfordAlphabet[((b[9]&0x03)<<3)|((b[10]&0xE0)>>5)]
out[17] = crockfordAlphabet[b[10]&0x1F]
out[18] = crockfordAlphabet[(b[11]&0xF8)>>3]
out[19] = crockfordAlphabet[((b[11]&0x07)<<2)|((b[12]&0xC0)>>6)]
out[20] = crockfordAlphabet[(b[12]&0x3E)>>1]
out[21] = crockfordAlphabet[((b[12]&0x01)<<4)|((b[13]&0xF0)>>4)]
out[22] = crockfordAlphabet[((b[13]&0x0F)<<1)|((b[14]&0x80)>>7)]
out[23] = crockfordAlphabet[(b[14]&0x7C)>>2]
out[24] = crockfordAlphabet[((b[14]&0x03)<<3)|((b[15]&0xE0)>>5)]
out[25] = crockfordAlphabet[b[15]&0x1F]
return string(out)
}
+74
View File
@@ -0,0 +1,74 @@
package storage
import (
"strings"
"testing"
"time"
)
func TestNewUUID_Shape(t *testing.T) {
id, err := NewUUID()
if err != nil {
t.Fatalf("NewUUID: %v", err)
}
if len(id) != 26 {
t.Errorf("expected 26 chars, got %d (%q)", len(id), id)
}
const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
for i, c := range id {
if !strings.ContainsRune(alphabet, c) {
t.Errorf("char %d (%q) not in Crockford alphabet (uuid=%q)", i, c, id)
}
}
}
func TestNewUUID_Unique(t *testing.T) {
const n = 1000
seen := make(map[string]struct{}, n)
for i := 0; i < n; i++ {
id, err := NewUUID()
if err != nil {
t.Fatalf("NewUUID at i=%d: %v", i, err)
}
if _, dup := seen[id]; dup {
t.Fatalf("collision after %d mints: %q", i+1, id)
}
seen[id] = struct{}{}
}
}
func TestNewUUID_TimeSortable(t *testing.T) {
// UUIDv7 has a millisecond timestamp prefix; consecutive mints in the
// same millisecond may not be strictly ordered (random tail), but
// mints across milliseconds are. Just check that an early ID
// lexicographically sorts before a later one when there's a real
// time gap. Doesn't test the no-gap case — the v7 spec only
// guarantees ordering across millisecond boundaries.
a, _ := NewUUID()
time.Sleep(2 * time.Millisecond)
b, _ := NewUUID()
if a >= b {
t.Errorf("expected %q < %q (UUIDv7 should be roughly time-sortable)", a, b)
}
}
func TestCIDFromKeysetField(t *testing.T) {
cases := []struct {
field string
want string
}{
{"cids/bafkmidhash", "bafkmidhash"},
{"cids/baejbeisha256", "baejbeisha256"},
{"cids/bafyipfs", "bafyipfs"},
{"title", ""}, // non-CID field
{"tmdb/title", ""}, // non-CID nested
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
{"midhash256", ""}, // legacy named field — no longer recognized
{"cids", ""}, // prefix without a member
}
for _, tc := range cases {
if got := cidFromKeysetField(tc.field); got != tc.want {
t.Errorf("cidFromKeysetField(%q) = %q, want %q", tc.field, got, tc.want)
}
}
}

Some files were not shown because too many files have changed in this diff Show More