Compare commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
f1cd94b17e | ||
|
|
52f5b5cedc | ||
|
|
f8a10091b8 | ||
|
|
94ad0c240b | ||
|
|
415595c1c6 | ||
|
|
aa45f4f32e | ||
|
|
3d65e7e951 | ||
|
|
bd833b7be4 | ||
|
|
98da535335 |
@@ -23,3 +23,5 @@ go.work
|
|||||||
dashboard/node_modules/*
|
dashboard/node_modules/*
|
||||||
dist/*
|
dist/*
|
||||||
dashboard/dist/*
|
dashboard/dist/*
|
||||||
|
editor/dist/
|
||||||
|
editor/node_modules/
|
||||||
|
|||||||
+18
-9
@@ -7,7 +7,7 @@
|
|||||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||||
# - nginx (dashboard, WebDAV, API proxy)
|
# - nginx (dashboard, WebDAV, API proxy)
|
||||||
# - Redis (managed by supervisord on leader)
|
# - Redis (managed by supervisord on leader)
|
||||||
# - rclone (remote mount daemon)
|
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
|
||||||
# - supervisord (process management - runs on leader only)
|
# - supervisord (process management - runs on leader only)
|
||||||
#
|
#
|
||||||
# Architecture:
|
# Architecture:
|
||||||
@@ -67,27 +67,36 @@ LABEL org.opencontainers.image.source=https://github.com/worph/meta-core
|
|||||||
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
|
LABEL org.opencontainers.image.description="MetaMesh meta-core - centralized data layer with leader election, Redis, WebDAV, mounts"
|
||||||
LABEL org.opencontainers.image.licenses=MIT
|
LABEL org.opencontainers.image.licenses=MIT
|
||||||
|
|
||||||
# Install runtime dependencies
|
# Pinned rclone release. Alpine's package can lag the latest by several
|
||||||
|
# minor versions and we depend on VFS cache fixes shipped post-1.65.
|
||||||
|
ARG RCLONE_VERSION=1.73.5
|
||||||
|
|
||||||
|
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
|
||||||
|
# all remote mounts go through rclone now.
|
||||||
RUN apk add --no-cache \
|
RUN apk add --no-cache \
|
||||||
# Core services
|
|
||||||
redis \
|
redis \
|
||||||
nginx \
|
nginx \
|
||||||
nginx-mod-http-dav-ext \
|
nginx-mod-http-dav-ext \
|
||||||
supervisor \
|
supervisor \
|
||||||
rclone \
|
|
||||||
fuse3 \
|
fuse3 \
|
||||||
# Mount utilities
|
|
||||||
nfs-utils \
|
|
||||||
cifs-utils \
|
|
||||||
# Tools
|
|
||||||
bash \
|
bash \
|
||||||
curl \
|
curl \
|
||||||
python3 \
|
python3 \
|
||||||
ca-certificates \
|
ca-certificates \
|
||||||
apache2-utils \
|
apache2-utils \
|
||||||
# Process utilities
|
|
||||||
procps
|
procps
|
||||||
|
|
||||||
|
# Install rclone from the official binary release (pinned).
|
||||||
|
RUN set -eux; \
|
||||||
|
apk add --no-cache --virtual .rclone-deps unzip; \
|
||||||
|
cd /tmp; \
|
||||||
|
curl -fsSL "https://downloads.rclone.org/v${RCLONE_VERSION}/rclone-v${RCLONE_VERSION}-linux-amd64.zip" -o rclone.zip; \
|
||||||
|
unzip -q rclone.zip; \
|
||||||
|
install -m 0755 "rclone-v${RCLONE_VERSION}-linux-amd64/rclone" /usr/local/bin/rclone; \
|
||||||
|
rm -rf rclone.zip "rclone-v${RCLONE_VERSION}-linux-amd64"; \
|
||||||
|
apk del .rclone-deps; \
|
||||||
|
/usr/local/bin/rclone version | head -2
|
||||||
|
|
||||||
# Copy Go binary from builder
|
# Copy Go binary from builder
|
||||||
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
|
COPY --from=go-builder /build/meta-core /usr/local/bin/meta-core
|
||||||
RUN chmod +x /usr/local/bin/meta-core
|
RUN chmod +x /usr/local/bin/meta-core
|
||||||
|
|||||||
@@ -64,10 +64,24 @@ clean:
|
|||||||
rm -f coverage.out coverage.html
|
rm -f coverage.out coverage.html
|
||||||
|
|
||||||
# Build Docker image
|
# Build Docker image
|
||||||
|
#
|
||||||
|
# We tag THREE names:
|
||||||
|
# - meta-core:$(VERSION) — the canonical versioned tag
|
||||||
|
# - meta-core:latest — convenience for `docker run meta-core`
|
||||||
|
# - meta-core:local — the tag every sub-stack docker-compose.yml uses
|
||||||
|
# (meta-share federation, meta-gateway dev stacks).
|
||||||
|
# Without this, `make docker` produces an image
|
||||||
|
# the dev stacks can't see by name, and previously
|
||||||
|
# -tagged stale `:local` images linger, silently
|
||||||
|
# breaking downstream API fields (e.g. /urls
|
||||||
|
# missing webdavUrlInternal → meta-gateway's
|
||||||
|
# poster auto-store fails). See README.md →
|
||||||
|
# Quick start → Build for the audit recipe.
|
||||||
docker:
|
docker:
|
||||||
@echo "Building Docker image..."
|
@echo "Building Docker image..."
|
||||||
docker build -t $(DOCKER_IMAGE):$(VERSION) .
|
docker build -t $(DOCKER_IMAGE):$(VERSION) .
|
||||||
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
|
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest
|
||||||
|
docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local
|
||||||
|
|
||||||
# Run locally (for development)
|
# Run locally (for development)
|
||||||
run: build
|
run: build
|
||||||
|
|||||||
@@ -1,520 +1,374 @@
|
|||||||
# meta-core
|
# meta-core
|
||||||
|
|
||||||
A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services.
|
Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
|
||||||
|
metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
|
||||||
|
|
||||||
## Overview
|
## Overview
|
||||||
|
|
||||||
meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul.
|
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
|
||||||
|
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
|
||||||
|
reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
|
||||||
|
is not exposed outside the container — all metadata I/O is mediated by the
|
||||||
|
HTTP API.
|
||||||
|
|
||||||
### Key Capabilities
|
### Key capabilities
|
||||||
|
|
||||||
| Feature | Description |
|
| Feature | Description |
|
||||||
|---------|-------------|
|
|---------|-------------|
|
||||||
| **Leader Election** | flock-based leader election (Go binary only runs as leader) |
|
| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
|
||||||
| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash |
|
| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
|
||||||
| **HTTP API** | Language-agnostic REST interface for metadata and service discovery |
|
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
|
||||||
| **Service Discovery** | File-based registry with heartbeats and stale detection |
|
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
|
||||||
| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations |
|
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
|
||||||
| **File Watching** | Directory scanning, MidHash256 computation, event dispatch |
|
| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
|
||||||
| **Mount Management** | SMB/rclone configuration, health monitoring |
|
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
|
||||||
| **WebDAV Server** | File access with nginx proxy_cache |
|
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
|
||||||
| **Event Publishing** | Redis keyspace notifications to meta:events stream |
|
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
|
||||||
|
|
||||||
### Design Characteristics
|
### Design characteristics
|
||||||
|
|
||||||
- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes)
|
- **Single-leader** — flock on a shared volume; no external consensus.
|
||||||
- **Automatic failover** - Lock releases on process death, followers detect and re-elect
|
- **HTTP-only externally** — Redis is bound to the container, never published.
|
||||||
- **Thread-safe** - RWMutex protection on all shared state
|
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
|
||||||
- **Graceful shutdown** - Signal handling with reverse startup order
|
on-disk Redis layout predates the current build (alpha clean-wipe policy;
|
||||||
- **Static binary** - CGO disabled, single binary deployment
|
see `docs/uuid-rooted-metadata.md`).
|
||||||
|
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
|
||||||
|
image with `redis-server`, `rclone`, nginx, and supervisord.
|
||||||
|
|
||||||
## Architecture
|
## Architecture
|
||||||
|
|
||||||
meta-core runs as a **standalone container** that other services connect to via HTTP API:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
┌───────────────────────────────────────────────────────────────────────┐
|
┌──────────────────────────────────────────────────────────────────────────┐
|
||||||
│ meta-core container (standalone) │
|
│ metacore-app (standalone container) │
|
||||||
│ │
|
│ │
|
||||||
│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │
|
│ supervisord │
|
||||||
│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │
|
│ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
|
||||||
│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │
|
│ │ writes kv-leader.info, exec supervisord) │
|
||||||
│ │ │ │ │ │ │ │ /health, /services │ │
|
│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
|
||||||
│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │
|
│ ├─ nginx (TLS termination, proxy_cache for /webdav) │
|
||||||
│ │
|
│ ├─ rclone rcd (mount daemon, RC API) │
|
||||||
│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │
|
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
|
||||||
└──────────────────────────────────────────────┬────────────────────────┘
|
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
|
||||||
|
└──────────────────────────────────────────────────────────────────────────┘
|
||||||
│
|
│
|
||||||
┌──────────────────────────────────────┴──────────────────────────┐
|
┌──────────────────────┴──────────────────────┐
|
||||||
│ /meta-core (shared volume) │
|
│ /meta-core (shared volume) │
|
||||||
│ ├── locks/kv-leader.info (leader URLs) │
|
│ locks/kv-leader.{lock,info} │
|
||||||
│ ├── db/redis/ (Redis data) │
|
│ db/redis/ │
|
||||||
│ ├── services/ (hostname-based .json) │
|
│ services/*.json │
|
||||||
│ └── mounts/ (mount configurations) │
|
│ mounts/{mounts.json, errors/} │
|
||||||
└─────────────────────────────────────────────────────────────────┘
|
│ watchers.json │
|
||||||
▲ ▲ ▲
|
│ cache/ (nginx proxy_cache) │
|
||||||
┌────────────┘ │ └────────────┐
|
└─────────────────────────────────────────────┘
|
||||||
|
▲
|
||||||
|
┌──────────────────────────────┼────────────────────────────────┐
|
||||||
│ │ │
|
│ │ │
|
||||||
┌───────────────┐ ┌───────────────┐ ┌───────────────┐
|
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||||
│ meta-sort │ │ meta-fuse │ │ meta-stremio│
|
│ meta-sort │ │ meta-fuse │ │ meta-stremio│
|
||||||
│ (reads info, │ │ (reads info, │ │ (reads info, │
|
│ │ │ │ │ │
|
||||||
│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │
|
│ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │
|
||||||
└───────────────┘ └───────────────┘ └───────────────┘
|
│ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│
|
||||||
|
└─────────────┘ └─────────────┘ └─────────────┘
|
||||||
```
|
```
|
||||||
|
|
||||||
## Quick Start
|
## Quick start
|
||||||
|
|
||||||
### Build
|
### Build
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Build binary
|
make build # build static binary to bin/meta-core
|
||||||
make build
|
make docker # build container image as meta-core:1.0.0 + :latest + :local
|
||||||
|
make test # run Go unit tests
|
||||||
# Build Docker image
|
|
||||||
make docker
|
|
||||||
|
|
||||||
# Run tests
|
|
||||||
make test
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Run
|
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
|
||||||
|
`:local` tag is the one every sub-stack docker-compose.yml references
|
||||||
|
(meta-share 10-peer federation, meta-gateway dev stacks); without it the
|
||||||
|
sub-stacks can't see a freshly-built image by name.
|
||||||
|
|
||||||
|
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
|
||||||
|
> image, but **already-running containers stay on the old image SHA** —
|
||||||
|
> `docker compose up -d` won't recreate them unless the compose file
|
||||||
|
> itself changed. After a meta-core source change, force-recreate the
|
||||||
|
> downstream containers so they pick up the new binary:
|
||||||
|
>
|
||||||
|
> ```bash
|
||||||
|
> make docker
|
||||||
|
> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
|
||||||
|
> up -d --force-recreate metagateway-generic-core
|
||||||
|
> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
|
||||||
|
> up -d --force-recreate metagateway-torznab-core
|
||||||
|
> docker compose -f packages/meta-share/dev/docker-compose.yml \
|
||||||
|
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
|
||||||
|
> ```
|
||||||
|
>
|
||||||
|
> The symptom of a missed recreate is missing API fields on `/urls`
|
||||||
|
> (e.g. `webdavUrlInternal` absent → meta-gateway's
|
||||||
|
> `store_poster_for_record` silently fails and torznab posters never
|
||||||
|
> land). Audit recipe:
|
||||||
|
>
|
||||||
|
> ```bash
|
||||||
|
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
|
||||||
|
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
|
||||||
|
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
|
||||||
|
> ```
|
||||||
|
|
||||||
|
### Dev container
|
||||||
|
|
||||||
|
In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# With environment variables
|
cd dev && ./scripts/start.sh
|
||||||
META_CORE_PATH=/meta-core \
|
|
||||||
FILES_PATH=/files \
|
|
||||||
SERVICE_NAME=meta-sort \
|
|
||||||
META_CORE_HTTP_PORT=9000 \
|
|
||||||
./bin/meta-core
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Docker
|
The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend
|
||||||
|
HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
|
||||||
```bash
|
|
||||||
docker build -t meta-core .
|
|
||||||
docker run -v meta-core:/meta-core -v files:/files meta-core
|
|
||||||
```
|
|
||||||
|
|
||||||
## Configuration
|
## Configuration
|
||||||
|
|
||||||
| Environment Variable | Default | Description |
|
Environment variables consumed by `internal/config/config.go`:
|
||||||
|---------------------|---------|-------------|
|
|
||||||
| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db |
|
|
||||||
| `FILES_PATH` | `/files` | Shared volume for media files |
|
|
||||||
| `SERVICE_NAME` | `meta-core` | Service identifier |
|
|
||||||
| `SERVICE_VERSION` | `1.0.0` | Service version |
|
|
||||||
| `API_PORT` | `8180` | Main service HTTP port |
|
|
||||||
| `BASE_URL` | - | Stable service URL |
|
|
||||||
| `REDIS_PORT` | `6379` | Redis port (leader only) |
|
|
||||||
| `META_CORE_HTTP_PORT` | `9000` | HTTP API port |
|
|
||||||
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address |
|
|
||||||
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval |
|
|
||||||
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval |
|
|
||||||
| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold |
|
|
||||||
| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning |
|
|
||||||
|
|
||||||
## API Reference
|
| Variable | Default | Description |
|
||||||
|
|---|---|---|
|
||||||
|
| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
|
||||||
|
| `FILES_PATH` | `/files` | Files volume root. |
|
||||||
|
| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
|
||||||
|
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
|
||||||
|
| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
|
||||||
|
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
|
||||||
|
| `REDIS_PORT` | `6379` | Local Redis port. |
|
||||||
|
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
|
||||||
|
| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind. |
|
||||||
|
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
|
||||||
|
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
|
||||||
|
| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
|
||||||
|
| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
|
||||||
|
| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
|
||||||
|
| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
|
||||||
|
| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
|
||||||
|
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
|
||||||
|
|
||||||
### Health & Status
|
Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
|
||||||
|
no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
|
||||||
|
identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
|
||||||
|
retry interval (`ELECTION_RETRY_SECS`, default `5`).
|
||||||
|
|
||||||
|
## API reference
|
||||||
|
|
||||||
|
All endpoints below are served by `internal/api/server.go`. Examples assume
|
||||||
|
the in-container Go port `:9000`; from the host use
|
||||||
|
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
|
||||||
|
(debug-direct).
|
||||||
|
|
||||||
|
### Health, leader, URLs
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Health check
|
GET /health # storage + role
|
||||||
curl http://localhost:9000/health
|
GET /status # version, uptime, etc.
|
||||||
# {"status":"ok","redis":true,"timestamp":"..."}
|
GET /leader # current leader info (hostname, PID, timestamps)
|
||||||
|
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
|
||||||
# Detailed status
|
GET /services # registered services + heartbeats
|
||||||
curl http://localhost:9000/status
|
GET /api/services # alias for /services
|
||||||
# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...}
|
GET /services/{name} # one service
|
||||||
|
GET /services/cleanup/stats
|
||||||
# Current leader
|
|
||||||
curl http://localhost:9000/leader
|
|
||||||
# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Metadata Operations
|
### Metadata — primary surface
|
||||||
|
|
||||||
|
These are the routes other services use. `{hash}` may be a UUID root or
|
||||||
|
any registered CID alias; CID aliases resolve to the underlying UUID
|
||||||
|
before reads/writes (`internal/storage/cid_resolution.go`).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# List all file hashes
|
GET /meta # list of root IDs
|
||||||
curl http://localhost:9000/meta
|
GET /meta/{hash} # full document (flat keys → nested JSON)
|
||||||
# {"hashIds":["midhash256:abc123",...],"count":42}
|
PUT /meta/{hash} # replace document
|
||||||
|
PATCH /meta/{hash} # merge into document
|
||||||
# Get metadata for a file
|
DELETE /meta/{hash} # delete root + all CID aliases
|
||||||
curl http://localhost:9000/meta/{hash}
|
GET /meta/{hash}/{key...} # single property (key may contain "/")
|
||||||
# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}}
|
PUT /meta/{hash}/{key...} # set single property
|
||||||
|
DELETE /meta/{hash}/{key...} # delete single property
|
||||||
# Update metadata
|
POST /meta/{hash}/_add/{key...} # add value to set-valued property
|
||||||
curl -X PUT http://localhost:9000/meta/{hash} \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"title":"New Title"}'
|
|
||||||
|
|
||||||
# Delete metadata
|
|
||||||
curl -X DELETE http://localhost:9000/meta/{hash}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Data Operations
|
CID-addressed access (public; auth-bypassed in the Caddy perimeter):
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Get file path
|
GET /api/meta/{cid} # resolve CID → metadata document
|
||||||
curl http://localhost:9000/data/{hash}/path
|
GET /api/file/{cid}/info # CID resolution metadata
|
||||||
# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true}
|
GET /file/{cid} # serve file bytes (range-aware)
|
||||||
|
POST /file/cid # compute CID for an uploaded file
|
||||||
# Check if file exists
|
HEAD /data/{hash} # existence + size
|
||||||
curl -I http://localhost:9000/data/{hash}
|
GET /data/{hash}/path # absolute path on /files
|
||||||
# HTTP/1.1 200 OK (or 404)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### File Operations (by CID)
|
(There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or
|
||||||
|
the `/file/{cid}` reverse-lookup path.)
|
||||||
|
|
||||||
|
### Editor / KV browser surface
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Serve file by CID (looks up poster/backdrop CIDs in metadata)
|
GET /api/metadata/hash-ids
|
||||||
curl http://localhost:9000/file/{cid} --output poster.jpg
|
GET /api/metadata/list
|
||||||
# Returns raw file bytes with appropriate Content-Type header
|
POST /api/metadata/search
|
||||||
|
POST /api/metadata/batch
|
||||||
|
POST /api/metadata/clear
|
||||||
|
GET /api/metadata/{hashId}
|
||||||
|
PUT /api/metadata/{hashId}
|
||||||
|
DELETE /api/metadata/{hashId}
|
||||||
|
GET /api/metadata/{hashId}/property
|
||||||
|
PUT /api/metadata/{hashId}/property
|
||||||
|
|
||||||
# Example: fetch a poster image
|
GET /api/kv/info
|
||||||
curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4
|
GET /api/kv/keys
|
||||||
# Content-Type: image/jpeg
|
GET /api/kv/tree
|
||||||
# [binary image data]
|
GET /api/kv/search
|
||||||
|
GET /api/kv/find
|
||||||
|
GET /api/kv/value
|
||||||
|
PUT /api/kv/value
|
||||||
|
DELETE /api/kv/value
|
||||||
|
GET /api/kv/key/{key...}
|
||||||
|
|
||||||
|
GET /api/schema
|
||||||
|
POST /api/schema/rescan
|
||||||
|
|
||||||
|
GET /api/snapshot/export
|
||||||
|
POST /api/snapshot/import
|
||||||
|
POST /api/snapshot/wipe
|
||||||
```
|
```
|
||||||
|
|
||||||
The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval.
|
### SSE event streams
|
||||||
|
|
||||||
### Service Discovery
|
The HTTP-mirror of the Redis Streams. External services never read Redis
|
||||||
|
directly — they subscribe here with `Last-Event-ID` for resume. See
|
||||||
|
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
|
||||||
|
contract (one event per stream entry, opaque IDs, heartbeats, gap
|
||||||
|
signalling).
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# List all services
|
GET /api/events/files # file:events stream (watcher events)
|
||||||
curl http://localhost:9000/services
|
GET /api/events/meta # meta:events stream (metadata mutations)
|
||||||
# {"services":[{"name":"meta-sort",...}],"count":3}
|
|
||||||
|
|
||||||
# Get specific service
|
|
||||||
curl http://localhost:9000/services/meta-sort
|
|
||||||
# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Mount Management
|
### Mounts
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# List all mounts
|
GET /api/mounts
|
||||||
curl http://localhost:9000/api/mounts/list
|
POST /api/mounts
|
||||||
# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]}
|
GET /api/mounts/{id}
|
||||||
|
PUT /api/mounts/{id} # also PATCH
|
||||||
# Add a new mount
|
DELETE /api/mounts/{id}
|
||||||
curl -X POST http://localhost:9000/api/mounts/add \
|
POST /api/mounts/{id}/mount
|
||||||
-H "Content-Type: application/json" \
|
POST /api/mounts/{id}/unmount
|
||||||
-d '{"type":"smb","path":"/files/corn/nas","config":{...}}'
|
POST /api/mounts/{id}/safe-unmount
|
||||||
|
POST /api/mounts/{id}/scan
|
||||||
# Remove a mount
|
GET /api/mounts/rclone/remotes
|
||||||
curl -X DELETE http://localhost:9000/api/mounts/smb-nas
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### File Watching
|
### Watchers
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Get scan status
|
GET /api/watchers
|
||||||
curl http://localhost:9000/api/scan/status
|
POST /api/watchers
|
||||||
# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847}
|
GET /api/watchers/{id}
|
||||||
|
PUT /api/watchers/{id}
|
||||||
# Trigger rescan
|
DELETE /api/watchers/{id}
|
||||||
curl -X POST http://localhost:9000/api/scan/trigger
|
POST /api/watchers/{id}/scan
|
||||||
|
POST /api/watchers/{id}/reset
|
||||||
# Stream file events (SSE)
|
POST /api/watchers/scan-all
|
||||||
curl http://localhost:9000/api/events/stream
|
POST /api/watchers/reset-all
|
||||||
# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."}
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### KV Browser
|
The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
|
||||||
|
return a deprecation pointer at the `/api/watchers/*` routes
|
||||||
|
(`internal/watcher/handlers.go:32-33`).
|
||||||
|
|
||||||
|
### Admin / WebDAV
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Storage statistics
|
POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
|
||||||
curl http://localhost:9000/api/kv/info
|
GET/PUT/DELETE /webdav/... # mounted on /files
|
||||||
# {"totalKeys":50000,"memoryUsed":"256MB",...}
|
|
||||||
|
|
||||||
# List keys with cursor pagination
|
|
||||||
curl "http://localhost:9000/api/kv/keys?cursor=0&count=100"
|
|
||||||
|
|
||||||
# Get specific key value
|
|
||||||
curl http://localhost:9000/api/kv/key/file:bafkr.../title
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Metadata Editor
|
No `/metrics` endpoint is exposed; observability is via container logs and
|
||||||
|
the `/health` / `/status` JSON.
|
||||||
|
|
||||||
```bash
|
## Leader gate
|
||||||
# Get all hash IDs
|
|
||||||
curl http://localhost:9000/api/metadata/hash-ids
|
|
||||||
# {"hashIds":["bafkr...","bafks..."]}
|
|
||||||
|
|
||||||
# Paginated metadata listing
|
Leader election is a bash flock loop in `docker/leader-election.sh`, not a
|
||||||
curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50"
|
Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
|
||||||
|
script writes the leader API URL to `kv-leader.info` and `exec`s
|
||||||
|
supervisord. When the process tree dies, the kernel releases the lock and
|
||||||
|
any standby instance can take over. There is no follower path inside the
|
||||||
|
Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
|
||||||
|
|
||||||
# Search metadata
|
`kv-leader.info` is a single plaintext line containing the API base URL:
|
||||||
curl -X POST http://localhost:9000/api/metadata/search \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"query":"inception","fields":["title"]}'
|
|
||||||
|
|
||||||
# Batch update
|
|
||||||
curl -X POST http://localhost:9000/api/metadata/batch \
|
|
||||||
-H "Content-Type: application/json" \
|
|
||||||
-d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}'
|
|
||||||
|
|
||||||
# Clear all metadata
|
|
||||||
curl -X POST http://localhost:9000/api/metadata/clear
|
|
||||||
```
|
|
||||||
|
|
||||||
### WebDAV Access
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# List directory (returns JSON)
|
|
||||||
curl http://localhost:9000/webdav/
|
|
||||||
# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}]
|
|
||||||
|
|
||||||
# Get file
|
|
||||||
curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv
|
|
||||||
|
|
||||||
# Upload file
|
|
||||||
curl -X PUT http://localhost:9000/webdav/plugin/output.txt \
|
|
||||||
-H "Content-Type: application/octet-stream" \
|
|
||||||
--data-binary @output.txt
|
|
||||||
|
|
||||||
# Create directory
|
|
||||||
curl -X MKCOL http://localhost:9000/webdav/newdir/
|
|
||||||
|
|
||||||
# Delete file/directory
|
|
||||||
curl -X DELETE http://localhost:9000/webdav/oldfile.txt
|
|
||||||
```
|
|
||||||
|
|
||||||
## Leader Election
|
|
||||||
|
|
||||||
meta-core uses POSIX file locking (flock) for distributed leader election:
|
|
||||||
|
|
||||||
1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock`
|
|
||||||
2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info`
|
|
||||||
3. Losers become **followers**, read leader info, connect to leader's Redis
|
|
||||||
4. Lock automatically releases when process dies (no stale locks)
|
|
||||||
|
|
||||||
### Election Flow
|
|
||||||
|
|
||||||
```
|
|
||||||
Instance 1 Instance 2 Instance 3
|
|
||||||
│ │ │
|
|
||||||
├── Try flock ──────────┼───────────────────────┤
|
|
||||||
│ │ │
|
|
||||||
[WIN] [LOSE] [LOSE]
|
|
||||||
LEADER FOLLOWER FOLLOWER
|
|
||||||
│ │ │
|
|
||||||
Start Redis Read leader.info Read leader.info
|
|
||||||
│ │ │
|
|
||||||
Write leader.info Connect to Redis Connect to Redis
|
|
||||||
│ │ │
|
|
||||||
[Health loop 5s] [Health loop 5s] [Health loop 5s]
|
|
||||||
- Update timestamp - Re-read leader - Re-read leader
|
|
||||||
- Check Redis alive - Detect changes - Detect changes
|
|
||||||
```
|
|
||||||
|
|
||||||
### Role Callbacks
|
|
||||||
|
|
||||||
The election system provides callbacks for role transitions:
|
|
||||||
|
|
||||||
```go
|
|
||||||
election.OnBecomeLeader(func(info LeaderLockInfo) {
|
|
||||||
// Start Redis, initialize resources
|
|
||||||
})
|
|
||||||
election.OnBecomeFollower(func(info LeaderLockInfo) {
|
|
||||||
// Connect to leader's Redis
|
|
||||||
})
|
|
||||||
election.OnLeaderLost(func() {
|
|
||||||
// Handle leader failure, attempt re-election
|
|
||||||
})
|
|
||||||
```
|
|
||||||
|
|
||||||
### Lock Info Format
|
|
||||||
|
|
||||||
The `kv-leader.info` file contains a plain text API URL:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
http://10.0.1.50:9000
|
http://10.0.1.50:9000
|
||||||
```
|
```
|
||||||
|
|
||||||
Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.).
|
Other services read this file, then call `GET /urls` for full discovery.
|
||||||
|
|
||||||
## Redis Management
|
The dev compose runs a single `metacore-app`; the flock primitive is
|
||||||
|
retained as a correctness anchor (one process owns Redis and the
|
||||||
|
writable state under `/meta-core`), not as live failover.
|
||||||
|
|
||||||
The leader is responsible for spawning and managing the Redis server.
|
## Metadata storage shape (one-line summary)
|
||||||
|
|
||||||
### Redis Configuration
|
Roots are opaque UUIDv7 strings; every property is a separate Redis
|
||||||
|
STRING at `file:<uuid>/<property>`; every known CID for a file is a
|
||||||
|
reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
|
||||||
|
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
|
||||||
|
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
|
||||||
|
The authoritative registry of field semantics and value formats lives in
|
||||||
|
the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||||
|
|
||||||
```bash
|
## Internal packages
|
||||||
redis-server --port 6379 --bind 0.0.0.0 \
|
|
||||||
--dir /meta-core/db/redis \
|
|
||||||
--appendonly yes --appendfilename appendonly.aof \
|
|
||||||
--dbfilename dump.rdb --save 60 1 \
|
|
||||||
--loglevel warning
|
|
||||||
```
|
|
||||||
|
|
||||||
### Persistence Strategy
|
|
||||||
|
|
||||||
| Method | Configuration | Purpose |
|
|
||||||
|--------|--------------|---------|
|
|
||||||
| **AOF** | `appendonly yes` | Write-ahead log for durability |
|
|
||||||
| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed |
|
|
||||||
|
|
||||||
### Failure Handling
|
|
||||||
|
|
||||||
- **Redis crash**: Auto-detected via health check, automatically restarted
|
|
||||||
- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs
|
|
||||||
- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed
|
|
||||||
|
|
||||||
## Metadata Storage
|
|
||||||
|
|
||||||
meta-core uses a flat key-value schema in Redis for storing file metadata.
|
|
||||||
|
|
||||||
### Storage Format
|
|
||||||
|
|
||||||
```
|
|
||||||
Key: /file/{hashId}/{property}
|
|
||||||
Value: string
|
|
||||||
|
|
||||||
Examples:
|
|
||||||
/file/midhash256:abc123/title → "Movie Title"
|
|
||||||
/file/midhash256:abc123/year → "2024"
|
|
||||||
/file/midhash256:abc123/genres → "action,drama"
|
|
||||||
/file/midhash256:abc123/filePath → "movies/Movie.mkv"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Storage Client Features
|
|
||||||
|
|
||||||
| Feature | Details |
|
|
||||||
|---------|---------|
|
|
||||||
| Connection Pool | 10 connections |
|
|
||||||
| Timeouts | 5s dial, 30s read/write |
|
|
||||||
| Batch Operations | Pipelined writes for performance |
|
|
||||||
| Key Scanning | SCAN-based iteration (non-blocking) |
|
|
||||||
| CID Lookup | Find file paths by poster/backdrop CID |
|
|
||||||
|
|
||||||
## Service Discovery
|
|
||||||
|
|
||||||
Services register by writing JSON to `/meta-core/services/{name}.json`.
|
|
||||||
|
|
||||||
### Registration Format
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"name": "meta-sort",
|
|
||||||
"version": "2.0.0",
|
|
||||||
"api": "http://10.0.1.50:8180",
|
|
||||||
"status": "running",
|
|
||||||
"pid": 12345,
|
|
||||||
"hostname": "meta-sort-dev",
|
|
||||||
"startedAt": "2024-01-01T00:00:00Z",
|
|
||||||
"lastHeartbeat": "2024-01-01T00:01:00Z",
|
|
||||||
"capabilities": ["meta-core"],
|
|
||||||
"endpoints": {
|
|
||||||
"health": "http://10.0.1.50:9000/health",
|
|
||||||
"meta": "http://10.0.1.50:9000/meta"
|
|
||||||
}
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
### Heartbeat & Stale Detection
|
|
||||||
|
|
||||||
| Parameter | Default | Purpose |
|
|
||||||
|-----------|---------|---------|
|
|
||||||
| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` |
|
|
||||||
| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this |
|
|
||||||
|
|
||||||
Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file.
|
|
||||||
|
|
||||||
## Development
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Format code
|
|
||||||
make fmt
|
|
||||||
|
|
||||||
# Run linter
|
|
||||||
make lint
|
|
||||||
|
|
||||||
# Run tests with coverage
|
|
||||||
make test-cover
|
|
||||||
|
|
||||||
# Clean build artifacts
|
|
||||||
make clean
|
|
||||||
```
|
|
||||||
|
|
||||||
## Internal Packages
|
|
||||||
|
|
||||||
| Package | Purpose |
|
| Package | Purpose |
|
||||||
|---------|---------|
|
|---------|---------|
|
||||||
| `cmd/meta-core` | Main entry point, startup/shutdown orchestration |
|
| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
|
||||||
| `internal/config` | Environment-based configuration with defaults |
|
| `internal/config` | Env-driven configuration, path helpers. |
|
||||||
| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) |
|
| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
|
||||||
| `internal/storage` | Redis client wrapper with flat key-value operations |
|
| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
|
||||||
| `internal/discovery` | Service registration, heartbeat loop, discovery |
|
| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
|
||||||
| `internal/api` | HTTP server, router, and all endpoint handlers |
|
| `internal/discovery` | Service registry + dead-service cleaner. |
|
||||||
| `internal/events` | Redis keyspace notification publisher to meta:events |
|
| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
|
||||||
| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch |
|
| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
|
||||||
| `internal/watchers` | Polling-based watcher configuration management |
|
| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
|
||||||
| `internal/mounts` | SMB/rclone mount configuration and health monitoring |
|
| `internal/snapshot` | Snapshot export / import / wipe. |
|
||||||
| `internal/webdav` | WebDAV protocol handler with caching integration |
|
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
|
||||||
|
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
|
||||||
|
| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. |
|
||||||
|
| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). |
|
||||||
|
|
||||||
### Startup Sequence
|
### Startup sequence
|
||||||
|
|
||||||
```
|
```
|
||||||
1. Load configuration from environment
|
1. Load configuration from environment
|
||||||
2. Create storage client (Redis wrapper)
|
2. Build LeaderInfoProvider (no election — the bash gate already won)
|
||||||
3. Initialize leader election
|
3. Connect to local Redis (retry up to 30× / 1s)
|
||||||
4. Register role transition callbacks
|
4. EnsureSchemaVersion — abort if the existing Redis layout is stale
|
||||||
5. Start election (acquire lock or become follower)
|
5. Start service discovery + dead-service cleaner
|
||||||
6. Start service discovery (register + heartbeat loop)
|
6. Construct API server (initialises watcher, mounts, WebDAV)
|
||||||
7. Initialize dead service cleaner
|
7. Start API server; on first connect, start MetaPublisher + schema Indexer
|
||||||
8. Initialize file watcher (if enabled)
|
and republish existing metadata onto meta:events
|
||||||
9. Initialize mount manager
|
8. Wait for SIGINT/SIGTERM, shutdown in reverse order
|
||||||
10. Start HTTP API server (WebDAV caching handled by nginx)
|
|
||||||
11. Wait for SIGINT/SIGTERM
|
|
||||||
12. Shutdown in reverse order
|
|
||||||
```
|
|
||||||
|
|
||||||
## Integration
|
|
||||||
|
|
||||||
### As Sidecar in Docker
|
|
||||||
|
|
||||||
Add to existing service's supervisord config:
|
|
||||||
|
|
||||||
```ini
|
|
||||||
[program:meta-core]
|
|
||||||
command=/usr/local/bin/meta-core
|
|
||||||
priority=10
|
|
||||||
autostart=true
|
|
||||||
autorestart=true
|
|
||||||
```
|
|
||||||
|
|
||||||
### Client Usage (TypeScript)
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const response = await fetch('http://localhost:9000/meta/' + hashId);
|
|
||||||
const { metadata } = await response.json();
|
|
||||||
```
|
|
||||||
|
|
||||||
### Client Usage (Python)
|
|
||||||
|
|
||||||
```python
|
|
||||||
import requests
|
|
||||||
response = requests.get(f'http://localhost:9000/meta/{hash_id}')
|
|
||||||
metadata = response.json()['metadata']
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Dependencies
|
## Dependencies
|
||||||
|
|
||||||
| Dependency | Version | Purpose |
|
| Dependency | Purpose |
|
||||||
|------------|---------|---------|
|
|------------|---------|
|
||||||
| `github.com/gorilla/mux` | v1.8.1 | HTTP router |
|
| `github.com/gorilla/mux` | HTTP router |
|
||||||
| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client |
|
| `github.com/redis/go-redis/v9` | Redis client |
|
||||||
| `golang.org/x/net` | v0.20.0 | WebDAV support |
|
| `github.com/google/uuid` | UUIDv7 minting |
|
||||||
| `github.com/google/uuid` | v1.6.0 | UUID generation |
|
| `golang.org/x/net` | WebDAV support |
|
||||||
| Go stdlib | - | context, encoding/json, net, os, sync, syscall |
|
| Go stdlib | context, net/http, os, sync, syscall |
|
||||||
|
|
||||||
### Runtime Requirements
|
Runtime requirements: Go 1.21+ at build time; the container ships Redis,
|
||||||
|
rclone, nginx, and supervisord alongside the binary.
|
||||||
- **Go 1.21+** for building
|
|
||||||
- **Redis** binary available in PATH (leader spawns it)
|
|
||||||
- **Shared filesystem** accessible by all instances (for flock)
|
|
||||||
|
|
||||||
## License
|
## License
|
||||||
|
|
||||||
|
|||||||
@@ -51,6 +51,15 @@ func main() {
|
|||||||
}
|
}
|
||||||
log.Println("[meta-core] Connected to Redis")
|
log.Println("[meta-core] Connected to Redis")
|
||||||
|
|
||||||
|
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
|
||||||
|
// the existing Redis store predates the UUID-rooted schema, so legacy
|
||||||
|
// data can't silently leak into the new code paths. See
|
||||||
|
// docs/uuid-rooted-metadata.md.
|
||||||
|
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
|
||||||
|
log.Fatalf("[meta-core] %v", err)
|
||||||
|
}
|
||||||
|
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
|
||||||
|
|
||||||
// Create and start service discovery
|
// Create and start service discovery
|
||||||
disc := discovery.NewService(cfg)
|
disc := discovery.NewService(cfg)
|
||||||
if err := disc.Start(); err != nil {
|
if err := disc.Start(); err != nil {
|
||||||
|
|||||||
Vendored
+1
-1
@@ -21,7 +21,7 @@
|
|||||||
min-height: 100vh;
|
min-height: 100vh;
|
||||||
}
|
}
|
||||||
</style>
|
</style>
|
||||||
<script type="module" crossorigin src="/assets/index-DJlbGMsa.js"></script>
|
<script type="module" crossorigin src="/assets/index-Cbd1uH5A.js"></script>
|
||||||
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
|
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
|
||||||
</head>
|
</head>
|
||||||
<body>
|
<body>
|
||||||
|
|||||||
@@ -3,6 +3,7 @@ import Dashboard from './pages/Dashboard';
|
|||||||
import Mounts from './pages/Mounts';
|
import Mounts from './pages/Mounts';
|
||||||
import Watchers from './pages/Watchers';
|
import Watchers from './pages/Watchers';
|
||||||
import Files from './pages/Files';
|
import Files from './pages/Files';
|
||||||
|
import Identity from './pages/Identity';
|
||||||
import ServiceNav from './components/ServiceNav';
|
import ServiceNav from './components/ServiceNav';
|
||||||
|
|
||||||
const navStyle: React.CSSProperties = {
|
const navStyle: React.CSSProperties = {
|
||||||
@@ -74,6 +75,12 @@ export default function App() {
|
|||||||
>
|
>
|
||||||
Files
|
Files
|
||||||
</NavLink>
|
</NavLink>
|
||||||
|
<NavLink
|
||||||
|
to="/identity"
|
||||||
|
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
|
||||||
|
>
|
||||||
|
Identity
|
||||||
|
</NavLink>
|
||||||
<NavLink
|
<NavLink
|
||||||
to="/editor"
|
to="/editor"
|
||||||
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
|
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
|
||||||
@@ -89,6 +96,7 @@ export default function App() {
|
|||||||
<Route path="/mounts" element={<Mounts />} />
|
<Route path="/mounts" element={<Mounts />} />
|
||||||
<Route path="/watchers" element={<Watchers />} />
|
<Route path="/watchers" element={<Watchers />} />
|
||||||
<Route path="/files" element={<Files />} />
|
<Route path="/files" element={<Files />} />
|
||||||
|
<Route path="/identity" element={<Identity />} />
|
||||||
<Route path="/editor" element={<EditorPage />} />
|
<Route path="/editor" element={<EditorPage />} />
|
||||||
</Routes>
|
</Routes>
|
||||||
</div>
|
</div>
|
||||||
|
|||||||
@@ -0,0 +1,375 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
// Status returned by GET /api/identity. Private key is never present here —
|
||||||
|
// it is shown exactly once at generation time (see GeneratedKeyPanel).
|
||||||
|
interface IdentityStatus {
|
||||||
|
hasIdentity: boolean;
|
||||||
|
uid?: string;
|
||||||
|
curve?: string;
|
||||||
|
createdAt?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface GeneratedKey {
|
||||||
|
uid: string;
|
||||||
|
curve: string;
|
||||||
|
createdAt: number;
|
||||||
|
privateKeyHex: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
const cardStyle: React.CSSProperties = {
|
||||||
|
background: '#16213e',
|
||||||
|
borderRadius: '8px',
|
||||||
|
padding: '1.5rem',
|
||||||
|
marginBottom: '1rem',
|
||||||
|
};
|
||||||
|
|
||||||
|
const buttonStyle: React.CSSProperties = {
|
||||||
|
padding: '0.5rem 1rem',
|
||||||
|
background: '#0f3460',
|
||||||
|
color: '#fff',
|
||||||
|
border: 'none',
|
||||||
|
borderRadius: '4px',
|
||||||
|
cursor: 'pointer',
|
||||||
|
marginRight: '0.5rem',
|
||||||
|
};
|
||||||
|
|
||||||
|
const dangerButtonStyle: React.CSSProperties = {
|
||||||
|
...buttonStyle,
|
||||||
|
background: '#7f1d1d',
|
||||||
|
};
|
||||||
|
|
||||||
|
const inputStyle: React.CSSProperties = {
|
||||||
|
width: '100%',
|
||||||
|
padding: '0.5rem',
|
||||||
|
marginBottom: '0.5rem',
|
||||||
|
background: '#1a1a2e',
|
||||||
|
border: '1px solid #0f3460',
|
||||||
|
borderRadius: '4px',
|
||||||
|
color: '#fff',
|
||||||
|
fontFamily: 'monospace',
|
||||||
|
};
|
||||||
|
|
||||||
|
const monoStyle: React.CSSProperties = {
|
||||||
|
fontFamily: 'monospace',
|
||||||
|
fontSize: '0.85rem',
|
||||||
|
wordBreak: 'break-all',
|
||||||
|
background: '#0a0a18',
|
||||||
|
border: '1px solid #0f3460',
|
||||||
|
padding: '0.75rem',
|
||||||
|
borderRadius: '4px',
|
||||||
|
};
|
||||||
|
|
||||||
|
function formatCreatedAt(unixSec?: number): string {
|
||||||
|
if (!unixSec) return '';
|
||||||
|
try {
|
||||||
|
return new Date(unixSec * 1000).toISOString().replace('T', ' ').replace(/\..+$/, ' UTC');
|
||||||
|
} catch {
|
||||||
|
return String(unixSec);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Read the JSON body of a non-2xx response if present, fall back to status text.
|
||||||
|
async function errorMessage(res: Response, fallback: string): Promise<string> {
|
||||||
|
try {
|
||||||
|
const j = await res.json();
|
||||||
|
if (j && (j.message || j.error)) return j.message ?? j.error;
|
||||||
|
} catch {
|
||||||
|
// not JSON
|
||||||
|
}
|
||||||
|
return `${fallback} (HTTP ${res.status})`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function GeneratedKeyPanel({ data, onDismiss }: { data: GeneratedKey; onDismiss: () => void }) {
|
||||||
|
const [copied, setCopied] = useState(false);
|
||||||
|
const copy = async () => {
|
||||||
|
try {
|
||||||
|
await navigator.clipboard.writeText(data.privateKeyHex);
|
||||||
|
setCopied(true);
|
||||||
|
setTimeout(() => setCopied(false), 2000);
|
||||||
|
} catch {
|
||||||
|
// Clipboard API blocked (non-HTTPS context, etc.) — fall back to selection.
|
||||||
|
const sel = window.getSelection();
|
||||||
|
const range = document.createRange();
|
||||||
|
const el = document.getElementById('mm-privkey-box');
|
||||||
|
if (sel && el) {
|
||||||
|
range.selectNodeContents(el);
|
||||||
|
sel.removeAllRanges();
|
||||||
|
sel.addRange(range);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
return (
|
||||||
|
<div style={{ ...cardStyle, border: '2px solid #dc2626' }}>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
background: '#7f1d1d',
|
||||||
|
color: '#fee2e2',
|
||||||
|
padding: '0.75rem 1rem',
|
||||||
|
borderRadius: '4px',
|
||||||
|
marginBottom: '1rem',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<strong>This private key is shown ONCE.</strong> Save it now in a password manager.
|
||||||
|
It will never be displayed again — there is no recovery path. Losing it means losing access
|
||||||
|
to every record signed under this identity.
|
||||||
|
</div>
|
||||||
|
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid (public):</p>
|
||||||
|
<div style={monoStyle}>{data.uid}</div>
|
||||||
|
<p style={{ color: '#9ca3af', margin: '1rem 0 0.5rem' }}>privateKeyHex (secret):</p>
|
||||||
|
<div id="mm-privkey-box" style={{ ...monoStyle, background: '#1f0a0a', borderColor: '#7f1d1d' }}>
|
||||||
|
{data.privateKeyHex}
|
||||||
|
</div>
|
||||||
|
<div style={{ marginTop: '1rem' }}>
|
||||||
|
<button style={buttonStyle} onClick={copy}>
|
||||||
|
{copied ? 'Copied' : 'Copy private key'}
|
||||||
|
</button>
|
||||||
|
<button style={dangerButtonStyle} onClick={onDismiss}>
|
||||||
|
I have saved it
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function NoIdentityPanel({
|
||||||
|
onGenerated,
|
||||||
|
onImported,
|
||||||
|
}: {
|
||||||
|
onGenerated: (g: GeneratedKey) => void;
|
||||||
|
onImported: () => void;
|
||||||
|
}) {
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [importing, setImporting] = useState(false);
|
||||||
|
const [importHex, setImportHex] = useState('');
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const generate = async () => {
|
||||||
|
setBusy(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/identity/generate', { method: 'POST' });
|
||||||
|
if (!res.ok) {
|
||||||
|
setError(await errorMessage(res, 'Generate failed'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const data = (await res.json()) as GeneratedKey;
|
||||||
|
onGenerated(data);
|
||||||
|
} catch (e) {
|
||||||
|
setError(`Generate failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const importKey = async (e: React.FormEvent) => {
|
||||||
|
e.preventDefault();
|
||||||
|
setBusy(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/identity/import', {
|
||||||
|
method: 'POST',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ privateKeyHex: importHex.trim() }),
|
||||||
|
});
|
||||||
|
if (!res.ok) {
|
||||||
|
setError(await errorMessage(res, 'Import failed'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setImportHex('');
|
||||||
|
setImporting(false);
|
||||||
|
onImported();
|
||||||
|
} catch (e) {
|
||||||
|
setError(`Import failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={cardStyle}>
|
||||||
|
<h3 style={{ marginTop: 0 }}>No identity yet</h3>
|
||||||
|
<p style={{ color: '#9ca3af' }}>
|
||||||
|
The User Data Layer needs a signing key for your ratings, watch progress, and "liked" flags
|
||||||
|
to be attributed to you across devices. Nothing happens automatically — generate one below,
|
||||||
|
or import a key you already have.
|
||||||
|
</p>
|
||||||
|
{error && (
|
||||||
|
<p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
|
||||||
|
{error}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
<div>
|
||||||
|
<button style={buttonStyle} disabled={busy} onClick={generate}>
|
||||||
|
{busy ? 'Working…' : 'Generate new keypair'}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
style={buttonStyle}
|
||||||
|
disabled={busy}
|
||||||
|
onClick={() => {
|
||||||
|
setImporting((x) => !x);
|
||||||
|
setError(null);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
{importing ? 'Cancel import' : 'Import existing private key'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{importing && (
|
||||||
|
<form onSubmit={importKey} style={{ marginTop: '1rem' }}>
|
||||||
|
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>
|
||||||
|
64 hex chars (32-byte secp256k1 scalar). A "0x" prefix is fine.
|
||||||
|
</p>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="e.g. a3f1…"
|
||||||
|
value={importHex}
|
||||||
|
onChange={(e) => setImportHex(e.target.value)}
|
||||||
|
required
|
||||||
|
autoFocus
|
||||||
|
/>
|
||||||
|
<button type="submit" style={buttonStyle} disabled={busy || !importHex.trim()}>
|
||||||
|
{busy ? 'Importing…' : 'Import'}
|
||||||
|
</button>
|
||||||
|
</form>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function HasIdentityPanel({ status, onReplace }: { status: IdentityStatus; onReplace: () => void }) {
|
||||||
|
const [confirming, setConfirming] = useState(false);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [error, setError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const remove = async () => {
|
||||||
|
setBusy(true);
|
||||||
|
setError(null);
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/identity?confirm=true', { method: 'DELETE' });
|
||||||
|
if (!res.ok && res.status !== 204) {
|
||||||
|
setError(await errorMessage(res, 'Delete failed'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
onReplace();
|
||||||
|
} catch (e) {
|
||||||
|
setError(`Delete failed: ${e instanceof Error ? e.message : String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
setConfirming(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={cardStyle}>
|
||||||
|
<h3 style={{ marginTop: 0 }}>Active identity</h3>
|
||||||
|
<p style={{ color: '#9ca3af', margin: '0 0 0.5rem' }}>uid:</p>
|
||||||
|
<div style={monoStyle}>{status.uid}</div>
|
||||||
|
<p style={{ color: '#888', marginTop: '1rem' }}>
|
||||||
|
Curve: {status.curve} · Created: {formatCreatedAt(status.createdAt)}
|
||||||
|
</p>
|
||||||
|
<p style={{ color: '#9ca3af', fontSize: '0.85rem' }}>
|
||||||
|
The private key lives at <code>/meta-core/identity/identity.json</code> on this volume and
|
||||||
|
is never returned through the API after generation. Other services on this stack (e.g.
|
||||||
|
meta-watch) sign records by calling <code>POST /api/identity/sign</code>.
|
||||||
|
</p>
|
||||||
|
{error && (
|
||||||
|
<p style={{ color: '#f87171', background: '#1f0a0a', padding: '0.5rem 0.75rem', borderRadius: '4px' }}>
|
||||||
|
{error}
|
||||||
|
</p>
|
||||||
|
)}
|
||||||
|
{!confirming ? (
|
||||||
|
<button style={dangerButtonStyle} onClick={() => setConfirming(true)} disabled={busy}>
|
||||||
|
Replace identity
|
||||||
|
</button>
|
||||||
|
) : (
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
background: '#1f0a0a',
|
||||||
|
border: '1px solid #7f1d1d',
|
||||||
|
padding: '0.75rem 1rem',
|
||||||
|
borderRadius: '4px',
|
||||||
|
marginTop: '0.5rem',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<p style={{ color: '#fecaca', marginTop: 0 }}>
|
||||||
|
Replacing the identity deletes the current keypair from disk. Records signed by the
|
||||||
|
current uid will keep verifying for anyone who has them cached, but this node will no
|
||||||
|
longer be able to write under that uid. <strong>Make sure you have backed up the key
|
||||||
|
first.</strong>
|
||||||
|
</p>
|
||||||
|
<button style={dangerButtonStyle} onClick={remove} disabled={busy}>
|
||||||
|
{busy ? 'Deleting…' : 'Yes, delete identity'}
|
||||||
|
</button>
|
||||||
|
<button style={buttonStyle} onClick={() => setConfirming(false)} disabled={busy}>
|
||||||
|
Cancel
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
export default function Identity() {
|
||||||
|
const [status, setStatus] = useState<IdentityStatus | null>(null);
|
||||||
|
const [generated, setGenerated] = useState<GeneratedKey | null>(null);
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [fetchError, setFetchError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const refresh = async () => {
|
||||||
|
setLoading(true);
|
||||||
|
setFetchError(null);
|
||||||
|
try {
|
||||||
|
const res = await fetch('/api/identity');
|
||||||
|
if (!res.ok) {
|
||||||
|
setFetchError(await errorMessage(res, 'Failed to load identity status'));
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setStatus((await res.json()) as IdentityStatus);
|
||||||
|
} catch (e) {
|
||||||
|
setFetchError(`Failed to load identity status: ${e instanceof Error ? e.message : String(e)}`);
|
||||||
|
} finally {
|
||||||
|
setLoading(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
refresh();
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ padding: '2rem', maxWidth: '900px', margin: '0 auto' }}>
|
||||||
|
<h1>Identity</h1>
|
||||||
|
<p style={{ color: '#9ca3af' }}>
|
||||||
|
Signing key for the User Data Layer (per-user ratings, progress, "liked" flags). One
|
||||||
|
identity per node. The same key on another device makes that device count as "you".
|
||||||
|
</p>
|
||||||
|
|
||||||
|
{loading && <div style={cardStyle}>Loading…</div>}
|
||||||
|
{fetchError && (
|
||||||
|
<div style={{ ...cardStyle, border: '1px solid #7f1d1d' }}>
|
||||||
|
<p style={{ color: '#f87171', margin: 0 }}>{fetchError}</p>
|
||||||
|
<button style={buttonStyle} onClick={refresh}>
|
||||||
|
Retry
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!loading && !fetchError && generated && (
|
||||||
|
<GeneratedKeyPanel
|
||||||
|
data={generated}
|
||||||
|
onDismiss={() => {
|
||||||
|
setGenerated(null);
|
||||||
|
refresh();
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!loading && !fetchError && !generated && status && !status.hasIdentity && (
|
||||||
|
<NoIdentityPanel onGenerated={(g) => setGenerated(g)} onImported={refresh} />
|
||||||
|
)}
|
||||||
|
|
||||||
|
{!loading && !fetchError && !generated && status && status.hasIdentity && (
|
||||||
|
<HasIdentityPanel status={status} onReplace={refresh} />
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
+220
-42
@@ -1,19 +1,47 @@
|
|||||||
import { useEffect, useState } from 'react';
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
interface MountIOStats {
|
||||||
|
source: 'rclone';
|
||||||
|
bytesRead: number;
|
||||||
|
bytesWritten?: number;
|
||||||
|
readBps: number;
|
||||||
|
writeBps?: number;
|
||||||
|
readOps?: number;
|
||||||
|
writeOps?: number;
|
||||||
|
readIops?: number;
|
||||||
|
writeIops?: number;
|
||||||
|
transfersPerSec?: number;
|
||||||
|
lastSampleAt: number;
|
||||||
|
intervalMs?: number;
|
||||||
|
}
|
||||||
|
|
||||||
interface Mount {
|
interface Mount {
|
||||||
id: string;
|
id: string;
|
||||||
name: string;
|
name: string;
|
||||||
type: 'nfs' | 'smb' | 'rclone';
|
type: 'smb' | 'rclone';
|
||||||
enabled: boolean;
|
enabled: boolean;
|
||||||
desiredMounted: boolean;
|
desiredMounted: boolean;
|
||||||
mountPath: string;
|
mountPath: string;
|
||||||
mounted: boolean;
|
mounted: boolean;
|
||||||
error?: string;
|
error?: string;
|
||||||
lastChecked: number;
|
lastChecked: number;
|
||||||
|
// VFS cache configuration applied at mount time. Empty string means "use
|
||||||
|
// server default" (50G / 24h / 5m).
|
||||||
|
cacheMaxSize?: string;
|
||||||
|
cacheMaxAge?: string;
|
||||||
|
dirCacheTime?: string;
|
||||||
pollingEnabled?: boolean;
|
pollingEnabled?: boolean;
|
||||||
pollingIntervalMs?: number;
|
pollingIntervalMs?: number;
|
||||||
pollingActive?: boolean;
|
pollingActive?: boolean;
|
||||||
lastPolledScan?: number;
|
lastPolledScan?: number;
|
||||||
|
// Adaptive scan progress (set by the meta-core poller)
|
||||||
|
scanning?: boolean;
|
||||||
|
currentScanStartedAt?: number; // ms; only while scanning
|
||||||
|
lastScanDurationMs?: number;
|
||||||
|
nextScanAt?: number; // ms; planned start of next adaptive scan
|
||||||
|
// Live IO — populated by StatsPoller every ~5s. Daemon-global from rclone,
|
||||||
|
// so the same numbers appear on every mount card.
|
||||||
|
ioStats?: MountIOStats;
|
||||||
}
|
}
|
||||||
|
|
||||||
interface RcloneRemote {
|
interface RcloneRemote {
|
||||||
@@ -43,6 +71,137 @@ const dangerButtonStyle: React.CSSProperties = {
|
|||||||
background: '#7f1d1d',
|
background: '#7f1d1d',
|
||||||
};
|
};
|
||||||
|
|
||||||
|
// Format byte counts/rates with K/M/G/T prefix (decimal — matches what users
|
||||||
|
// expect for network speeds; binary prefixes feel pedantic on a "MB/s"
|
||||||
|
// readout). Returns e.g. "12.4 MB", "850 KB", "2.3 GB".
|
||||||
|
function formatBytes(n: number): string {
|
||||||
|
if (!isFinite(n) || n < 0) return '—';
|
||||||
|
const units = ['B', 'KB', 'MB', 'GB', 'TB'];
|
||||||
|
let i = 0;
|
||||||
|
let v = n;
|
||||||
|
while (v >= 1000 && i < units.length - 1) {
|
||||||
|
v /= 1000;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return v >= 100 || i === 0 ? `${v.toFixed(0)} ${units[i]}` : `${v.toFixed(1)} ${units[i]}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatRate(bps: number): string {
|
||||||
|
return `${formatBytes(bps)}/s`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Render a humanised duration: 5s, 1m 23s, 2h 14m. Sub-second collapses to 0s
|
||||||
|
// rather than "0ms" — at the second-tick cadence the UI uses, finer precision
|
||||||
|
// is just noise.
|
||||||
|
function formatDuration(ms: number): string {
|
||||||
|
if (ms < 0 || !isFinite(ms)) return '—';
|
||||||
|
const s = Math.floor(ms / 1000);
|
||||||
|
if (s < 60) return `${s}s`;
|
||||||
|
const m = Math.floor(s / 60);
|
||||||
|
const remS = s % 60;
|
||||||
|
if (m < 60) return remS ? `${m}m ${remS}s` : `${m}m`;
|
||||||
|
const h = Math.floor(m / 60);
|
||||||
|
const remM = m % 60;
|
||||||
|
return remM ? `${h}h ${remM}m` : `${h}h`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// IOStatsLine renders a single line of live throughput info beneath each
|
||||||
|
// mount card. Source is always rclone after the rclone-only consolidation,
|
||||||
|
// so we report file-level transfers/s rather than kernel IOPS.
|
||||||
|
function IOStatsLine({ stats }: { stats: MountIOStats }) {
|
||||||
|
const opsLabel = 'transfers/s';
|
||||||
|
const opsValue = stats.transfersPerSec;
|
||||||
|
|
||||||
|
const ageMs = Date.now() - stats.lastSampleAt;
|
||||||
|
const stale = ageMs > 30000; // >30s old means sampler hasn't ticked recently
|
||||||
|
|
||||||
|
return (
|
||||||
|
<p
|
||||||
|
style={{
|
||||||
|
color: stale ? '#6b7280' : '#9ca3af',
|
||||||
|
fontSize: '0.85rem',
|
||||||
|
margin: '0.25rem 0 0',
|
||||||
|
}}
|
||||||
|
title={`source: ${stats.source} · sample age: ${Math.round(ageMs / 1000)}s`}
|
||||||
|
>
|
||||||
|
↓ {formatRate(stats.readBps)}
|
||||||
|
{opsValue !== undefined && opsValue > 0 && (
|
||||||
|
<> · {opsValue.toFixed(opsValue >= 10 ? 0 : 1)} {opsLabel}</>
|
||||||
|
)}
|
||||||
|
<> · {formatBytes(stats.bytesRead)} read total</>
|
||||||
|
{stats.bytesWritten !== undefined && stats.bytesWritten > 0 && (
|
||||||
|
<> · {formatBytes(stats.bytesWritten)} written</>
|
||||||
|
)}
|
||||||
|
</p>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface ScanProgressProps {
|
||||||
|
mount: Mount;
|
||||||
|
now: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Progress bar shows scan ETA when running, an idle row with a countdown to
|
||||||
|
// the next adaptive scan when waiting. We can't show true % complete (we
|
||||||
|
// don't know the file count up-front), so the running bar is filled by
|
||||||
|
// elapsed/lastDuration capped at 95% — once we exceed the previous duration,
|
||||||
|
// it pins at 95% to signal "taking longer than last time".
|
||||||
|
function ScanProgress({ mount, now }: ScanProgressProps) {
|
||||||
|
if (!mount.pollingEnabled) return null;
|
||||||
|
|
||||||
|
const isScanning = !!mount.scanning && !!mount.currentScanStartedAt;
|
||||||
|
const lastDuration = mount.lastScanDurationMs ?? 0;
|
||||||
|
|
||||||
|
let fill = 0;
|
||||||
|
let label = '';
|
||||||
|
|
||||||
|
if (isScanning) {
|
||||||
|
const elapsed = Math.max(0, now - (mount.currentScanStartedAt ?? now));
|
||||||
|
if (lastDuration > 0) {
|
||||||
|
fill = Math.min(0.95, elapsed / lastDuration);
|
||||||
|
label = `Scanning… ${formatDuration(elapsed)} elapsed (last took ${formatDuration(lastDuration)})`;
|
||||||
|
} else {
|
||||||
|
fill = 0.4; // first scan ever — no ETA available, show indeterminate-ish
|
||||||
|
label = `Scanning… ${formatDuration(elapsed)} elapsed (first scan, no ETA yet)`;
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
const sinceLast = mount.lastPolledScan ? now - mount.lastPolledScan : 0;
|
||||||
|
const untilNext = mount.nextScanAt ? Math.max(0, mount.nextScanAt - now) : 0;
|
||||||
|
if (lastDuration > 0 && mount.lastPolledScan) {
|
||||||
|
label = `Last scan ${formatDuration(sinceLast)} ago (took ${formatDuration(lastDuration)})`;
|
||||||
|
if (mount.nextScanAt) label += ` • next in ${formatDuration(untilNext)}`;
|
||||||
|
} else if (mount.nextScanAt) {
|
||||||
|
label = `Waiting — next scan in ${formatDuration(untilNext)}`;
|
||||||
|
} else {
|
||||||
|
label = 'Polling enabled — waiting for first scan';
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div style={{ marginTop: '0.5rem' }}>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
height: '6px',
|
||||||
|
background: '#0f3460',
|
||||||
|
borderRadius: '3px',
|
||||||
|
overflow: 'hidden',
|
||||||
|
position: 'relative',
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
style={{
|
||||||
|
width: `${Math.round(fill * 100)}%`,
|
||||||
|
height: '100%',
|
||||||
|
background: isScanning ? '#4ade80' : '#374151',
|
||||||
|
transition: 'width 0.5s linear',
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<p style={{ color: '#9ca3af', fontSize: '0.8rem', margin: '0.25rem 0 0' }}>{label}</p>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
export default function Mounts() {
|
export default function Mounts() {
|
||||||
const [mounts, setMounts] = useState<Mount[]>([]);
|
const [mounts, setMounts] = useState<Mount[]>([]);
|
||||||
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
|
const [remotes, setRemotes] = useState<RcloneRemote[]>([]);
|
||||||
@@ -50,19 +209,23 @@ export default function Mounts() {
|
|||||||
const [showForm, setShowForm] = useState(false);
|
const [showForm, setShowForm] = useState(false);
|
||||||
const [formData, setFormData] = useState({
|
const [formData, setFormData] = useState({
|
||||||
name: '',
|
name: '',
|
||||||
type: 'nfs' as 'nfs' | 'smb' | 'rclone',
|
type: 'smb' as 'smb' | 'rclone',
|
||||||
nfsServer: '',
|
|
||||||
nfsPath: '',
|
|
||||||
smbServer: '',
|
smbServer: '',
|
||||||
smbShare: '',
|
smbShare: '',
|
||||||
smbUsername: '',
|
smbUsername: '',
|
||||||
smbPassword: '',
|
smbPassword: '',
|
||||||
rcloneRemote: '',
|
rcloneRemote: '',
|
||||||
rclonePath: '',
|
rclonePath: '',
|
||||||
pollingEnabled: false,
|
cacheMaxSize: '',
|
||||||
|
cacheMaxAge: '',
|
||||||
|
dirCacheTime: '',
|
||||||
|
pollingEnabled: true,
|
||||||
pollingIntervalMs: 30000,
|
pollingIntervalMs: 30000,
|
||||||
});
|
});
|
||||||
const [scanning, setScanning] = useState<string | null>(null);
|
const [scanning, setScanning] = useState<string | null>(null);
|
||||||
|
// 1Hz ticker so the progress bar's elapsed/countdown updates between API
|
||||||
|
// refetches (which run every 5s).
|
||||||
|
const [now, setNow] = useState<number>(() => Date.now());
|
||||||
|
|
||||||
const fetchMounts = async () => {
|
const fetchMounts = async () => {
|
||||||
try {
|
try {
|
||||||
@@ -94,7 +257,11 @@ export default function Mounts() {
|
|||||||
fetchMounts();
|
fetchMounts();
|
||||||
fetchRemotes();
|
fetchRemotes();
|
||||||
const interval = setInterval(fetchMounts, 5000);
|
const interval = setInterval(fetchMounts, 5000);
|
||||||
return () => clearInterval(interval);
|
const tick = setInterval(() => setNow(Date.now()), 1000);
|
||||||
|
return () => {
|
||||||
|
clearInterval(interval);
|
||||||
|
clearInterval(tick);
|
||||||
|
};
|
||||||
}, []);
|
}, []);
|
||||||
|
|
||||||
const handleMount = async (id: string) => {
|
const handleMount = async (id: string) => {
|
||||||
@@ -143,10 +310,7 @@ export default function Mounts() {
|
|||||||
type: formData.type,
|
type: formData.type,
|
||||||
};
|
};
|
||||||
|
|
||||||
if (formData.type === 'nfs') {
|
if (formData.type === 'smb') {
|
||||||
body.nfsServer = formData.nfsServer;
|
|
||||||
body.nfsPath = formData.nfsPath;
|
|
||||||
} else if (formData.type === 'smb') {
|
|
||||||
body.smbServer = formData.smbServer;
|
body.smbServer = formData.smbServer;
|
||||||
body.smbShare = formData.smbShare;
|
body.smbShare = formData.smbShare;
|
||||||
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
|
if (formData.smbUsername) body.smbUsername = formData.smbUsername;
|
||||||
@@ -156,6 +320,12 @@ export default function Mounts() {
|
|||||||
body.rclonePath = formData.rclonePath;
|
body.rclonePath = formData.rclonePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Cache knobs — only forward non-empty values; the server applies defaults
|
||||||
|
// when fields are omitted.
|
||||||
|
if (formData.cacheMaxSize) body.cacheMaxSize = formData.cacheMaxSize;
|
||||||
|
if (formData.cacheMaxAge) body.cacheMaxAge = formData.cacheMaxAge;
|
||||||
|
if (formData.dirCacheTime) body.dirCacheTime = formData.dirCacheTime;
|
||||||
|
|
||||||
// Add polling configuration
|
// Add polling configuration
|
||||||
body.pollingEnabled = formData.pollingEnabled;
|
body.pollingEnabled = formData.pollingEnabled;
|
||||||
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
|
if (formData.pollingEnabled && formData.pollingIntervalMs >= 5000) {
|
||||||
@@ -171,16 +341,17 @@ export default function Mounts() {
|
|||||||
setShowForm(false);
|
setShowForm(false);
|
||||||
setFormData({
|
setFormData({
|
||||||
name: '',
|
name: '',
|
||||||
type: 'nfs',
|
type: 'smb',
|
||||||
nfsServer: '',
|
|
||||||
nfsPath: '',
|
|
||||||
smbServer: '',
|
smbServer: '',
|
||||||
smbShare: '',
|
smbShare: '',
|
||||||
smbUsername: '',
|
smbUsername: '',
|
||||||
smbPassword: '',
|
smbPassword: '',
|
||||||
rcloneRemote: '',
|
rcloneRemote: '',
|
||||||
rclonePath: '',
|
rclonePath: '',
|
||||||
pollingEnabled: false,
|
cacheMaxSize: '',
|
||||||
|
cacheMaxAge: '',
|
||||||
|
dirCacheTime: '',
|
||||||
|
pollingEnabled: true,
|
||||||
pollingIntervalMs: 30000,
|
pollingIntervalMs: 30000,
|
||||||
});
|
});
|
||||||
fetchMounts();
|
fetchMounts();
|
||||||
@@ -231,32 +402,15 @@ export default function Mounts() {
|
|||||||
<select
|
<select
|
||||||
style={inputStyle}
|
style={inputStyle}
|
||||||
value={formData.type}
|
value={formData.type}
|
||||||
onChange={(e) => setFormData({ ...formData, type: e.target.value as 'nfs' | 'smb' | 'rclone' })}
|
onChange={(e) => {
|
||||||
|
const type = e.target.value as 'smb' | 'rclone';
|
||||||
|
setFormData({ ...formData, type });
|
||||||
|
}}
|
||||||
>
|
>
|
||||||
<option value="nfs">NFS</option>
|
<option value="smb">SMB (via rclone)</option>
|
||||||
<option value="smb">SMB/CIFS</option>
|
<option value="rclone">rclone (pre-defined remote)</option>
|
||||||
<option value="rclone">rclone</option>
|
|
||||||
</select>
|
</select>
|
||||||
|
|
||||||
{formData.type === 'nfs' && (
|
|
||||||
<>
|
|
||||||
<input
|
|
||||||
style={inputStyle}
|
|
||||||
placeholder="NFS Server (e.g., 192.168.1.100)"
|
|
||||||
value={formData.nfsServer}
|
|
||||||
onChange={(e) => setFormData({ ...formData, nfsServer: e.target.value })}
|
|
||||||
required
|
|
||||||
/>
|
|
||||||
<input
|
|
||||||
style={inputStyle}
|
|
||||||
placeholder="NFS Path (e.g., /export/media)"
|
|
||||||
value={formData.nfsPath}
|
|
||||||
onChange={(e) => setFormData({ ...formData, nfsPath: e.target.value })}
|
|
||||||
required
|
|
||||||
/>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{formData.type === 'smb' && (
|
{formData.type === 'smb' && (
|
||||||
<>
|
<>
|
||||||
<input
|
<input
|
||||||
@@ -313,6 +467,30 @@ export default function Mounts() {
|
|||||||
</>
|
</>
|
||||||
)}
|
)}
|
||||||
|
|
||||||
|
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||||
|
<p style={{ color: '#9ca3af', fontSize: '0.85rem', marginTop: 0, marginBottom: '0.5rem' }}>
|
||||||
|
VFS cache (leave blank for defaults: 50G / 24h / 5m)
|
||||||
|
</p>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Cache max size (e.g., 50G)"
|
||||||
|
value={formData.cacheMaxSize}
|
||||||
|
onChange={(e) => setFormData({ ...formData, cacheMaxSize: e.target.value })}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Cache max age (e.g., 24h)"
|
||||||
|
value={formData.cacheMaxAge}
|
||||||
|
onChange={(e) => setFormData({ ...formData, cacheMaxAge: e.target.value })}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
style={inputStyle}
|
||||||
|
placeholder="Dir cache time (e.g., 5m)"
|
||||||
|
value={formData.dirCacheTime}
|
||||||
|
onChange={(e) => setFormData({ ...formData, dirCacheTime: e.target.value })}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
|
||||||
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
<div style={{ marginTop: '1rem', borderTop: '1px solid #0f3460', paddingTop: '1rem' }}>
|
||||||
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
|
<label style={{ display: 'flex', alignItems: 'center', marginBottom: '0.5rem', cursor: 'pointer' }}>
|
||||||
<input
|
<input
|
||||||
@@ -362,17 +540,17 @@ export default function Mounts() {
|
|||||||
</span>
|
</span>
|
||||||
</p>
|
</p>
|
||||||
{mount.pollingEnabled && (
|
{mount.pollingEnabled && (
|
||||||
<p style={{ color: '#888', fontSize: '0.9rem' }}>
|
<p style={{ color: '#888', fontSize: '0.9rem', margin: '0.25rem 0 0' }}>
|
||||||
Polling:{' '}
|
Polling:{' '}
|
||||||
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
|
<span style={{ color: mount.pollingActive ? '#4ade80' : '#f59e0b' }}>
|
||||||
{mount.pollingActive ? 'Active' : 'Inactive'}
|
{mount.pollingActive ? 'Active' : 'Inactive'}
|
||||||
</span>
|
</span>
|
||||||
{' | Interval: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
{' • Floor: '}{(mount.pollingIntervalMs || 30000) / 1000}s
|
||||||
{mount.lastPolledScan && (
|
{' • Adaptive 10× idle'}
|
||||||
<> | Last scan: {new Date(mount.lastPolledScan).toLocaleTimeString()}</>
|
|
||||||
)}
|
|
||||||
</p>
|
</p>
|
||||||
)}
|
)}
|
||||||
|
{mount.ioStats && <IOStatsLine stats={mount.ioStats} />}
|
||||||
|
<ScanProgress mount={mount} now={now} />
|
||||||
{mount.error && (
|
{mount.error && (
|
||||||
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
|
<p style={{ color: '#f87171', marginTop: '0.5rem' }}>Error: {mount.error}</p>
|
||||||
)}
|
)}
|
||||||
|
|||||||
+135
-167
@@ -2,11 +2,17 @@
|
|||||||
# =============================================================================
|
# =============================================================================
|
||||||
# Mount Watcher Script
|
# Mount Watcher Script
|
||||||
#
|
#
|
||||||
# Monitors /meta-core/mounts/mounts.json and manages NFS, SMB, and rclone mounts.
|
# Monitors /meta-core/mounts/mounts.json and manages mounts via the local
|
||||||
# Runs as a supervisord program, polling every 5 seconds.
|
# rclone RC daemon. After the rclone-only consolidation, this script no
|
||||||
|
# longer shells out to mount.cifs / mount.nfs — every mount type is executed
|
||||||
|
# through rclone's mount/mount endpoint.
|
||||||
#
|
#
|
||||||
# Uses findmnt to check actual mount status (source of truth).
|
# Mount types:
|
||||||
# Writes errors to /meta-core/mounts/errors/{id}.error
|
# - "smb" → on-the-fly :smb: remote synthesised from SMB-* fields
|
||||||
|
# - "rclone" → references a pre-defined remote in rclone.conf
|
||||||
|
#
|
||||||
|
# All mounts are read-only by construction (no API surface to override).
|
||||||
|
# Errors surface in /meta-core/mounts/errors/{id}.error
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
|
|
||||||
set -u
|
set -u
|
||||||
@@ -17,6 +23,16 @@ ERROR_DIR="$CONFIG_DIR/errors"
|
|||||||
FILES_PATH="${FILES_PATH:-/files}"
|
FILES_PATH="${FILES_PATH:-/files}"
|
||||||
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
POLL_INTERVAL="${MOUNT_POLL_INTERVAL:-5}"
|
||||||
|
|
||||||
|
# Fall-back VFS cache settings — applied when a mount entry leaves the field
|
||||||
|
# blank. Must stay in sync with DefaultCacheMaxSize/MaxAge/DirCacheTime in
|
||||||
|
# packages/meta-core/internal/mounts/types.go.
|
||||||
|
DEFAULT_CACHE_MAX_SIZE="50G"
|
||||||
|
DEFAULT_CACHE_MAX_AGE="24h"
|
||||||
|
DEFAULT_DIR_CACHE_TIME="5m"
|
||||||
|
|
||||||
|
RCLONE_RC_URL="${RCLONE_RC_URL:-http://127.0.0.1:5572}"
|
||||||
|
RCLONE_RC_AUTH="${RCLONE_RC_AUTH:-admin:admin}"
|
||||||
|
|
||||||
# ANSI colors
|
# ANSI colors
|
||||||
RED='\033[0;31m'
|
RED='\033[0;31m'
|
||||||
GREEN='\033[0;32m'
|
GREEN='\033[0;32m'
|
||||||
@@ -54,9 +70,7 @@ init_dirs() {
|
|||||||
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
|
# Check if path is mounted (uses /proc/mounts for Alpine compatibility)
|
||||||
is_mounted() {
|
is_mounted() {
|
||||||
local mount_path="$1"
|
local mount_path="$1"
|
||||||
# Remove trailing slash for consistent matching
|
|
||||||
local clean_path="${mount_path%/}"
|
local clean_path="${mount_path%/}"
|
||||||
# Check if the path appears as a mount point in /proc/mounts (field 2)
|
|
||||||
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
|
grep -q " ${clean_path} " /proc/mounts 2>/dev/null
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -74,145 +88,102 @@ write_error() {
|
|||||||
echo "$error" >> "$ERROR_DIR/$id.error"
|
echo "$error" >> "$ERROR_DIR/$id.error"
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mount NFS share (always read-only)
|
# Build the rclone remote source string for a mount entry. Echoes the result.
|
||||||
do_mount_nfs() {
|
# For type=smb, synthesises an on-the-fly :smb: remote so we don't have to
|
||||||
local id="$1"
|
# mutate rclone.conf at runtime. For type=rclone, joins the named remote with
|
||||||
local server="$2"
|
# the optional remote path.
|
||||||
local path="$3"
|
build_rclone_fs() {
|
||||||
local mount_path="$4"
|
local type="$1"
|
||||||
local options="${5:-}"
|
local smb_server="$2"
|
||||||
|
local smb_share="$3"
|
||||||
|
local smb_username="$4"
|
||||||
|
local smb_password_obscured="$5"
|
||||||
|
local smb_domain="$6"
|
||||||
|
local rclone_remote="$7"
|
||||||
|
local rclone_path="$8"
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
case "$type" in
|
||||||
|
smb)
|
||||||
# Always mount read-only for safety
|
# On-the-fly remote: :<backend>,k1=v1,k2=v2:<path>
|
||||||
local mount_opts="ro"
|
# The obscured password is base64-ish (A-Za-z0-9+/_-=) so safe to
|
||||||
[ -n "$options" ] && mount_opts="${mount_opts},${options}"
|
# embed without quoting.
|
||||||
|
local opts="host=${smb_server}"
|
||||||
log "Mounting NFS (read-only): ${server}:${path} -> ${mount_path}"
|
if [ -n "$smb_username" ]; then
|
||||||
output=$(mount -t nfs -o "${mount_opts}" "${server}:${path}" "${mount_path}" 2>&1)
|
opts="${opts},user=${smb_username}"
|
||||||
local rc=$?
|
|
||||||
|
|
||||||
if [ $rc -eq 0 ]; then
|
|
||||||
clear_error "$id"
|
|
||||||
log_success "NFS mount successful: $mount_path"
|
|
||||||
else
|
|
||||||
write_error "$id" "$output"
|
|
||||||
log_error "NFS mount failed: $output"
|
|
||||||
fi
|
fi
|
||||||
return $rc
|
if [ -n "$smb_password_obscured" ]; then
|
||||||
|
opts="${opts},pass=${smb_password_obscured}"
|
||||||
|
fi
|
||||||
|
if [ -n "$smb_domain" ]; then
|
||||||
|
opts="${opts},domain=${smb_domain}"
|
||||||
|
fi
|
||||||
|
echo ":smb,${opts}:${smb_share}"
|
||||||
|
;;
|
||||||
|
rclone)
|
||||||
|
echo "${rclone_remote}:${rclone_path}"
|
||||||
|
;;
|
||||||
|
*)
|
||||||
|
echo ""
|
||||||
|
;;
|
||||||
|
esac
|
||||||
}
|
}
|
||||||
|
|
||||||
# Mount SMB/CIFS share (always read-only)
|
# Mount via rclone RC API (always read-only).
|
||||||
do_mount_smb() {
|
|
||||||
local id="$1"
|
|
||||||
local server="$2"
|
|
||||||
local share="$3"
|
|
||||||
local mount_path="$4"
|
|
||||||
local username="${5:-}"
|
|
||||||
local password_obscured="${6:-}"
|
|
||||||
local domain="${7:-}"
|
|
||||||
local options="${8:-}"
|
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
|
||||||
|
|
||||||
# Always mount read-only for safety
|
|
||||||
local mount_opts="ro"
|
|
||||||
|
|
||||||
# If we have credentials, reveal password and build options
|
|
||||||
if [ -n "$username" ]; then
|
|
||||||
mount_opts="${mount_opts},username=${username}"
|
|
||||||
|
|
||||||
if [ -n "$password_obscured" ]; then
|
|
||||||
# Reveal password using rclone (only exists in memory briefly)
|
|
||||||
local password
|
|
||||||
password=$(rclone reveal "$password_obscured" 2>/dev/null)
|
|
||||||
if [ -n "$password" ]; then
|
|
||||||
mount_opts="${mount_opts},password=${password}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
if [ -n "$domain" ]; then
|
|
||||||
mount_opts="${mount_opts},domain=${domain}"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Add any additional options
|
|
||||||
if [ -n "$options" ]; then
|
|
||||||
if [ -n "$mount_opts" ]; then
|
|
||||||
mount_opts="${mount_opts},${options}"
|
|
||||||
else
|
|
||||||
mount_opts="$options"
|
|
||||||
fi
|
|
||||||
fi
|
|
||||||
|
|
||||||
log "Mounting SMB (read-only): //${server}/${share} -> ${mount_path}"
|
|
||||||
|
|
||||||
# Execute mount directly (not via eval) to handle special chars in password
|
|
||||||
local output
|
|
||||||
local rc
|
|
||||||
if [ -n "$mount_opts" ]; then
|
|
||||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" -o "${mount_opts}" 2>&1)
|
|
||||||
rc=$?
|
|
||||||
else
|
|
||||||
output=$(mount -t cifs "//${server}/${share}" "${mount_path}" 2>&1)
|
|
||||||
rc=$?
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Clear password from any variables (best effort)
|
|
||||||
password=""
|
|
||||||
mount_opts=""
|
|
||||||
|
|
||||||
if [ $rc -eq 0 ]; then
|
|
||||||
clear_error "$id"
|
|
||||||
log_success "SMB mount successful: $mount_path"
|
|
||||||
else
|
|
||||||
write_error "$id" "$output"
|
|
||||||
log_error "SMB mount failed: $output"
|
|
||||||
fi
|
|
||||||
return $rc
|
|
||||||
}
|
|
||||||
|
|
||||||
# Mount rclone remote using RC API (always read-only)
|
|
||||||
do_mount_rclone() {
|
do_mount_rclone() {
|
||||||
local id="$1"
|
local id="$1"
|
||||||
local remote="$2"
|
local fs="$2"
|
||||||
local remote_path="$3"
|
local mount_path="$3"
|
||||||
local mount_path="$4"
|
local cache_max_size="$4"
|
||||||
local options="${5:-}"
|
local cache_max_age="$5"
|
||||||
|
local dir_cache_time="$6"
|
||||||
|
|
||||||
mkdir -p "$mount_path"
|
mkdir -p "$mount_path"
|
||||||
|
|
||||||
# Build the remote source string
|
# Apply defaults for any blank cache fields.
|
||||||
local fs="${remote}:${remote_path}"
|
[ -z "$cache_max_size" ] && cache_max_size="$DEFAULT_CACHE_MAX_SIZE"
|
||||||
|
[ -z "$cache_max_age" ] && cache_max_age="$DEFAULT_CACHE_MAX_AGE"
|
||||||
|
[ -z "$dir_cache_time" ] && dir_cache_time="$DEFAULT_DIR_CACHE_TIME"
|
||||||
|
|
||||||
# Use rclone RC API to mount (always read-only for safety)
|
log "Mounting (rclone, read-only): ${fs} -> ${mount_path} (cache ${cache_max_size}/${cache_max_age}, dir-cache ${dir_cache_time})"
|
||||||
log "Mounting rclone (read-only): ${fs} -> ${mount_path}"
|
|
||||||
|
|
||||||
# Build JSON body for rclone mount with read-only option
|
# Build JSON body via python3 — embedding the obscured password and
|
||||||
|
# display fs string into a hand-rolled heredoc is asking for an injection
|
||||||
|
# bug the next time we add a field with a quote in it.
|
||||||
local json_body
|
local json_body
|
||||||
json_body=$(cat <<EOF
|
json_body=$(python3 -c '
|
||||||
{
|
import json, sys
|
||||||
"fs": "${fs}",
|
print(json.dumps({
|
||||||
"mountPoint": "${mount_path}",
|
"fs": sys.argv[1],
|
||||||
|
"mountPoint": sys.argv[2],
|
||||||
"mountOpt": {
|
"mountOpt": {
|
||||||
"AllowOther": true,
|
"AllowOther": True,
|
||||||
"ReadOnly": true
|
"ReadOnly": True,
|
||||||
},
|
},
|
||||||
"vfsOpt": {
|
"vfsOpt": {
|
||||||
"CacheMode": 2,
|
# CacheMode 3 = "full" — cache both reads and writes on local disk.
|
||||||
"ReadOnly": true
|
# CacheMode 2 ("writes") would silently bypass the cache for reads,
|
||||||
}
|
# which is the opposite of what we want for a read-only mount.
|
||||||
}
|
"CacheMode": 3,
|
||||||
EOF
|
"CacheMaxSize": sys.argv[3],
|
||||||
)
|
"CacheMaxAge": sys.argv[4],
|
||||||
|
"DirCacheTime": sys.argv[5],
|
||||||
|
# Note: ChunkStreams (per-file parallel TCP) helps single-stream
|
||||||
|
# throughput in isolation but, under the 4+ concurrent reads from
|
||||||
|
# the processing pipeline, multiplies into 16+ simultaneous SMB
|
||||||
|
# requests and tanks aggregate throughput. Left at default — rely on
|
||||||
|
# file-level concurrency from the pipeline for parallelism instead.
|
||||||
|
"ReadOnly": True,
|
||||||
|
},
|
||||||
|
}))
|
||||||
|
' "$fs" "$mount_path" "$cache_max_size" "$cache_max_age" "$dir_cache_time")
|
||||||
|
|
||||||
|
local output
|
||||||
output=$(curl -s -X POST \
|
output=$(curl -s -X POST \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-u admin:admin \
|
-u "$RCLONE_RC_AUTH" \
|
||||||
-d "$json_body" \
|
-d "$json_body" \
|
||||||
"http://127.0.0.1:5572/mount/mount" 2>&1)
|
"${RCLONE_RC_URL}/mount/mount" 2>&1)
|
||||||
|
|
||||||
# Wait a moment for mount to establish
|
|
||||||
sleep 2
|
sleep 2
|
||||||
|
|
||||||
if is_mounted "$mount_path"; then
|
if is_mounted "$mount_path"; then
|
||||||
@@ -221,7 +192,6 @@ EOF
|
|||||||
return 0
|
return 0
|
||||||
else
|
else
|
||||||
local error_msg="rclone mount failed"
|
local error_msg="rclone mount failed"
|
||||||
# Try to extract error from response
|
|
||||||
if echo "$output" | grep -q "error"; then
|
if echo "$output" | grep -q "error"; then
|
||||||
error_msg="$output"
|
error_msg="$output"
|
||||||
fi
|
fi
|
||||||
@@ -231,36 +201,28 @@ EOF
|
|||||||
fi
|
fi
|
||||||
}
|
}
|
||||||
|
|
||||||
# Unmount a path
|
# Unmount a path. Always goes through the rclone RC API now — every mount we
|
||||||
|
# create lives behind rclone's FUSE driver.
|
||||||
do_unmount() {
|
do_unmount() {
|
||||||
local id="$1"
|
local id="$1"
|
||||||
local mount_path="$2"
|
local mount_path="$2"
|
||||||
local type="$3"
|
|
||||||
|
|
||||||
if ! is_mounted "$mount_path"; then
|
if ! is_mounted "$mount_path"; then
|
||||||
log "Already unmounted: $mount_path"
|
log "Already unmounted: $mount_path"
|
||||||
return 0
|
return 0
|
||||||
fi
|
fi
|
||||||
|
|
||||||
if [ "$type" = "rclone" ]; then
|
log "Unmounting: $mount_path"
|
||||||
# Use rclone RC API to unmount
|
|
||||||
log "Unmounting rclone: $mount_path"
|
|
||||||
curl -s -X POST \
|
curl -s -X POST \
|
||||||
-H "Content-Type: application/json" \
|
-H "Content-Type: application/json" \
|
||||||
-u admin:admin \
|
-u "$RCLONE_RC_AUTH" \
|
||||||
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
-d "{\"mountPoint\":\"${mount_path}\"}" \
|
||||||
"http://127.0.0.1:5572/mount/unmount" > /dev/null 2>&1
|
"${RCLONE_RC_URL}/mount/unmount" > /dev/null 2>&1
|
||||||
else
|
|
||||||
log "Unmounting: $mount_path"
|
|
||||||
umount "$mount_path" 2>&1
|
|
||||||
fi
|
|
||||||
|
|
||||||
# Wait a moment
|
|
||||||
sleep 1
|
sleep 1
|
||||||
|
|
||||||
# Check if still mounted
|
# Fallback to lazy umount in case the FUSE process is wedged.
|
||||||
if is_mounted "$mount_path"; then
|
if is_mounted "$mount_path"; then
|
||||||
# Force unmount with lazy option
|
|
||||||
log_warn "Force unmounting (lazy): $mount_path"
|
log_warn "Force unmounting (lazy): $mount_path"
|
||||||
umount -l "$mount_path" 2>&1
|
umount -l "$mount_path" 2>&1
|
||||||
sleep 1
|
sleep 1
|
||||||
@@ -277,12 +239,11 @@ do_unmount() {
|
|||||||
|
|
||||||
# Process all mounts from config
|
# Process all mounts from config
|
||||||
process_mounts() {
|
process_mounts() {
|
||||||
# Check if config file exists
|
|
||||||
if [ ! -f "$CONFIG_FILE" ]; then
|
if [ ! -f "$CONFIG_FILE" ]; then
|
||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Parse JSON using python3
|
# Parse JSON using python3 — pipe-separated fields, NFS dropped.
|
||||||
local mounts
|
local mounts
|
||||||
mounts=$(python3 -c '
|
mounts=$(python3 -c '
|
||||||
import sys, json
|
import sys, json
|
||||||
@@ -294,15 +255,12 @@ except:
|
|||||||
sys.exit(0)
|
sys.exit(0)
|
||||||
|
|
||||||
for m in data.get("mounts", []):
|
for m in data.get("mounts", []):
|
||||||
# Output pipe-separated values
|
|
||||||
fields = [
|
fields = [
|
||||||
m.get("id", ""),
|
m.get("id", ""),
|
||||||
m.get("type", ""),
|
m.get("type", ""),
|
||||||
str(m.get("enabled", True)),
|
str(m.get("enabled", True)),
|
||||||
str(m.get("desiredMounted", False)),
|
str(m.get("desiredMounted", False)),
|
||||||
m.get("mountPath", ""),
|
m.get("mountPath", ""),
|
||||||
m.get("nfsServer", ""),
|
|
||||||
m.get("nfsPath", ""),
|
|
||||||
m.get("smbServer", ""),
|
m.get("smbServer", ""),
|
||||||
m.get("smbShare", ""),
|
m.get("smbShare", ""),
|
||||||
m.get("smbUsername", ""),
|
m.get("smbUsername", ""),
|
||||||
@@ -310,7 +268,9 @@ for m in data.get("mounts", []):
|
|||||||
m.get("smbDomain", ""),
|
m.get("smbDomain", ""),
|
||||||
m.get("rcloneRemote", ""),
|
m.get("rcloneRemote", ""),
|
||||||
m.get("rclonePath", ""),
|
m.get("rclonePath", ""),
|
||||||
m.get("options", "")
|
m.get("cacheMaxSize", ""),
|
||||||
|
m.get("cacheMaxAge", ""),
|
||||||
|
m.get("dirCacheTime", ""),
|
||||||
]
|
]
|
||||||
print("|".join(fields))
|
print("|".join(fields))
|
||||||
' 2>/dev/null)
|
' 2>/dev/null)
|
||||||
@@ -319,48 +279,56 @@ for m in data.get("mounts", []):
|
|||||||
return
|
return
|
||||||
fi
|
fi
|
||||||
|
|
||||||
while IFS='|' read -r id type enabled desired mount_path nfs_server nfs_path smb_server smb_share smb_username smb_password_obscured smb_domain rclone_remote rclone_path options; do
|
while IFS='|' read -r id type enabled desired mount_path \
|
||||||
|
smb_server smb_share smb_username smb_password_obscured smb_domain \
|
||||||
|
rclone_remote rclone_path \
|
||||||
|
cache_max_size cache_max_age dir_cache_time; do
|
||||||
# Skip empty lines
|
# Skip empty lines
|
||||||
[ -z "$id" ] && continue
|
[ -z "$id" ] && continue
|
||||||
|
|
||||||
# Check actual mount status
|
# Skip retired/legacy types — the Go-side migration disables NFS but a
|
||||||
|
# stale entry could still arrive here on a downgrade/upgrade race.
|
||||||
|
if [ "$type" != "smb" ] && [ "$type" != "rclone" ]; then
|
||||||
|
log_warn "Skipping unsupported mount type '$type' (id=$id) — only smb and rclone are supported"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
local is_mounted_now=false
|
local is_mounted_now=false
|
||||||
is_mounted "$mount_path" && is_mounted_now=true
|
is_mounted "$mount_path" && is_mounted_now=true
|
||||||
|
|
||||||
# Handle disabled mounts - unmount if currently mounted
|
# Disabled → unmount if currently up
|
||||||
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
if [ "$enabled" != "True" ] && [ "$enabled" != "true" ]; then
|
||||||
if [ "$is_mounted_now" = true ]; then
|
if [ "$is_mounted_now" = true ]; then
|
||||||
log "Mount $id ($mount_path) disabled, unmounting..."
|
log "Mount $id ($mount_path) disabled, unmounting..."
|
||||||
do_unmount "$id" "$mount_path" "$type"
|
do_unmount "$id" "$mount_path"
|
||||||
fi
|
fi
|
||||||
continue
|
continue
|
||||||
fi
|
fi
|
||||||
|
|
||||||
# Handle desired state
|
|
||||||
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
if [ "$desired" = "True" ] || [ "$desired" = "true" ]; then
|
||||||
# Should be mounted
|
# Should be mounted
|
||||||
if [ "$is_mounted_now" = false ]; then
|
if [ "$is_mounted_now" = false ]; then
|
||||||
|
local fs
|
||||||
|
fs=$(build_rclone_fs "$type" \
|
||||||
|
"$smb_server" "$smb_share" "$smb_username" \
|
||||||
|
"$smb_password_obscured" "$smb_domain" \
|
||||||
|
"$rclone_remote" "$rclone_path")
|
||||||
|
|
||||||
|
if [ -z "$fs" ]; then
|
||||||
|
log_error "Could not build rclone fs spec for mount $id (type=$type)"
|
||||||
|
write_error "$id" "Invalid mount config: missing required fields for type=$type"
|
||||||
|
continue
|
||||||
|
fi
|
||||||
|
|
||||||
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
log "Mount $id ($mount_path) desired but not mounted, mounting..."
|
||||||
case "$type" in
|
do_mount_rclone "$id" "$fs" "$mount_path" \
|
||||||
nfs)
|
"$cache_max_size" "$cache_max_age" "$dir_cache_time"
|
||||||
do_mount_nfs "$id" "$nfs_server" "$nfs_path" "$mount_path" "$options"
|
|
||||||
;;
|
|
||||||
smb)
|
|
||||||
do_mount_smb "$id" "$smb_server" "$smb_share" "$mount_path" "$smb_username" "$smb_password_obscured" "$smb_domain" "$options"
|
|
||||||
;;
|
|
||||||
rclone)
|
|
||||||
do_mount_rclone "$id" "$rclone_remote" "$rclone_path" "$mount_path" "$options"
|
|
||||||
;;
|
|
||||||
*)
|
|
||||||
log_error "Unknown mount type: $type"
|
|
||||||
;;
|
|
||||||
esac
|
|
||||||
fi
|
fi
|
||||||
else
|
else
|
||||||
# Should be unmounted
|
# Should be unmounted
|
||||||
if [ "$is_mounted_now" = true ]; then
|
if [ "$is_mounted_now" = true ]; then
|
||||||
log "Mount $id ($mount_path) not desired, unmounting..."
|
log "Mount $id ($mount_path) not desired, unmounting..."
|
||||||
do_unmount "$id" "$mount_path" "$type"
|
do_unmount "$id" "$mount_path"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
done <<< "$mounts"
|
done <<< "$mounts"
|
||||||
@@ -368,7 +336,7 @@ for m in data.get("mounts", []):
|
|||||||
|
|
||||||
# Main loop
|
# Main loop
|
||||||
main() {
|
main() {
|
||||||
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s)"
|
log "Starting mount watcher (poll interval: ${POLL_INTERVAL}s, rclone-only)"
|
||||||
|
|
||||||
init_dirs
|
init_dirs
|
||||||
|
|
||||||
|
|||||||
@@ -70,6 +70,9 @@ http {
|
|||||||
proxy_buffering off;
|
proxy_buffering off;
|
||||||
proxy_cache off;
|
proxy_cache off;
|
||||||
proxy_read_timeout 86400s;
|
proxy_read_timeout 86400s;
|
||||||
|
|
||||||
|
# Snapshot import accepts multi-MB ZIP uploads.
|
||||||
|
client_max_body_size 0;
|
||||||
}
|
}
|
||||||
|
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
@@ -192,6 +195,18 @@ http {
|
|||||||
# WebDAV HTML browser (for iframe embedding in dashboard)
|
# WebDAV HTML browser (for iframe embedding in dashboard)
|
||||||
# Returns HTML directory listings instead of JSON
|
# Returns HTML directory listings instead of JSON
|
||||||
# =========================================================================
|
# =========================================================================
|
||||||
|
|
||||||
|
# At the browser root, strip the "../" parent-dir row that autoindex
|
||||||
|
# otherwise emits — clicking it inside the dashboard iframe would
|
||||||
|
# navigate to "/" (the dashboard itself), nesting the page inside itself.
|
||||||
|
location = /webdav-browser/ {
|
||||||
|
alias /files/;
|
||||||
|
autoindex on;
|
||||||
|
charset utf-8;
|
||||||
|
sub_filter '<a href="../">../</a>' '';
|
||||||
|
sub_filter_once on;
|
||||||
|
}
|
||||||
|
|
||||||
location /webdav-browser/ {
|
location /webdav-browser/ {
|
||||||
alias /files/;
|
alias /files/;
|
||||||
autoindex on;
|
autoindex on;
|
||||||
|
|||||||
@@ -71,7 +71,7 @@ startsecs=2
|
|||||||
# rclone: Remote storage mount daemon (RC API on port 5572)
|
# rclone: Remote storage mount daemon (RC API on port 5572)
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
[program:rclone]
|
[program:rclone]
|
||||||
command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
command=/usr/local/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin
|
||||||
priority=30
|
priority=30
|
||||||
autostart=true
|
autostart=true
|
||||||
autorestart=true
|
autorestart=true
|
||||||
@@ -80,7 +80,7 @@ stderr_logfile=/var/log/supervisor/rclone_error.log
|
|||||||
startsecs=3
|
startsecs=3
|
||||||
|
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
# mount-watcher: Monitors mounts.json and manages NFS/SMB/rclone mounts
|
# mount-watcher: Monitors mounts.json and routes all mounts through rclone
|
||||||
# =============================================================================
|
# =============================================================================
|
||||||
[program:mount-watcher]
|
[program:mount-watcher]
|
||||||
command=/app/scripts/mount-watcher.sh
|
command=/app/scripts/mount-watcher.sh
|
||||||
|
|||||||
@@ -0,0 +1,771 @@
|
|||||||
|
# API-Mediated Access (Redis Lockdown)
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
**Landed** — all four PRs (A/B/C/D) deployed and verified end-to-end on the
|
||||||
|
dev stack. Redis is unreachable from outside `metacore-app`; every metadata
|
||||||
|
read, write, and event flows through meta-core's HTTP+SSE surface. Builds
|
||||||
|
on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
|
||||||
|
meta-core is the single writer to the metadata keyspace — an assumption
|
||||||
|
that's now actually enforced rather than aspirational.
|
||||||
|
|
||||||
|
See "Implementation outcome" at the bottom for the delta between this doc's
|
||||||
|
plan and what shipped (notably the write-path CID resolution and the
|
||||||
|
dual-root migration sweep, which weren't in the original plan but turned
|
||||||
|
out to be load-bearing).
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
meta-core was designed as the sidecar that mediates every metadata read and
|
||||||
|
write — a single point that owns the Redis schema, enforces invariants
|
||||||
|
(reverse index, canonical_cid reconciliation, schema-version sentinel),
|
||||||
|
and presents a stable HTTP contract. Other services were meant to talk to
|
||||||
|
meta-core, not to Redis.
|
||||||
|
|
||||||
|
In practice, meta-sort opens its own Redis connection and writes
|
||||||
|
`file:<hashId>/<property>` keys with raw pipeline SETs
|
||||||
|
(`packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts:269-296`).
|
||||||
|
It picked up this shortcut early, before meta-core had a complete write
|
||||||
|
API, and the shortcut stuck.
|
||||||
|
|
||||||
|
This bypass is a real problem in three ways:
|
||||||
|
|
||||||
|
1. **Schema invariants leak.** When meta-core landed the UUID-rooted schema
|
||||||
|
([uuid-rooted-metadata.md](uuid-rooted-metadata.md)), meta-sort kept
|
||||||
|
writing midhash-rooted entries. The result was *dual roots per file* —
|
||||||
|
one set of keys at the watcher's UUID, another at meta-sort's midhash.
|
||||||
|
`file:__index__` doubled in size. The bug was invisible to meta-sort
|
||||||
|
(its writes succeeded) but broke the reverse-index resolution
|
||||||
|
downstream services rely on.
|
||||||
|
|
||||||
|
2. **Schema evolution is impossible.** Changing the Redis layout requires
|
||||||
|
coordinating across every service that touches it. The UUID-root
|
||||||
|
migration would have been a single-PR change if writes already routed
|
||||||
|
through the API.
|
||||||
|
|
||||||
|
3. **Self-hosted distributed deployment widens the threat model.** Peers
|
||||||
|
are run by different operators. Plugin authors may ship their own
|
||||||
|
services. We can't expect every binary in the network to honour
|
||||||
|
undocumented conventions. A buggy or careless service that connects
|
||||||
|
to Redis and writes the wrong key shape can corrupt the schema for
|
||||||
|
everybody on that node.
|
||||||
|
|
||||||
|
The third point is the one that flipped the design from "soft lock by
|
||||||
|
convention" to "hard lock by network topology." We're not protecting
|
||||||
|
against malicious services — that's an out-of-scope threat. We're
|
||||||
|
protecting against *accidental* corruption from honest mistakes in code
|
||||||
|
we don't fully control.
|
||||||
|
|
||||||
|
## The architecture in one picture
|
||||||
|
|
||||||
|
```
|
||||||
|
meta-core (only Redis writer + reader)
|
||||||
|
┌──────────────────────────────────────┐
|
||||||
|
│ HTTP API surface: │
|
||||||
|
│ POST/PUT/PATCH /api/metadata/* │ ◄── writes
|
||||||
|
│ GET /api/meta/{cid} │ ◄── reads (by CID)
|
||||||
|
│ GET /api/metadata/* │ ◄── reads (by id)
|
||||||
|
│ GET /api/file/{cid} │ ◄── content
|
||||||
|
│ GET /api/events/files │ ◄── SSE stream
|
||||||
|
│ GET /api/events/meta │ ◄── SSE stream
|
||||||
|
│ │ │
|
||||||
|
│ ▼ │
|
||||||
|
│ Redis (file:*, cid:*, streams) │
|
||||||
|
│ ◄── container-local, no exposed │
|
||||||
|
│ port, no docker-network alias │
|
||||||
|
└──────────────────────────────────────┘
|
||||||
|
▲ ▲
|
||||||
|
│ HTTP │ HTTP SSE
|
||||||
|
│ │
|
||||||
|
┌───────────┴────┐ ┌──────┴─────────┐
|
||||||
|
│ meta-sort │ │ meta-fuse │
|
||||||
|
│ meta-stremio │ │ meta-share │
|
||||||
|
│ meta-dup │ │ (any future) │
|
||||||
|
└────────────────┘ └────────────────┘
|
||||||
|
```
|
||||||
|
|
||||||
|
**The single-writer rule.** meta-core has exclusive write authority over
|
||||||
|
the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
|
||||||
|
other service can write — not because of an honoured convention, but
|
||||||
|
because they can't reach Redis at all.
|
||||||
|
|
||||||
|
**Stream reads also mediated.** Events go out through SSE endpoints on
|
||||||
|
meta-core's HTTP surface. Consumers don't see the underlying Redis
|
||||||
|
Streams. This is the Kubernetes / Consul pattern (see "Industry context"
|
||||||
|
below). It costs an extra HTTP hop versus the current direct XREAD; the
|
||||||
|
benefit is that the storage layer is free to evolve (or be replaced)
|
||||||
|
without breaking every consumer.
|
||||||
|
|
||||||
|
**Redis is private.** No host port mapping, no docker-network alias, no
|
||||||
|
URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
|
||||||
|
|
||||||
|
## Eventing — HTTP Server-Sent Events
|
||||||
|
|
||||||
|
SSE is the chosen transport for cross-service event consumption. The
|
||||||
|
mapping from Redis Streams onto SSE is direct enough that no eventing
|
||||||
|
feature we currently use is lost.
|
||||||
|
|
||||||
|
### Feature mapping (what we keep, how we keep it)
|
||||||
|
|
||||||
|
| Redis Streams | SSE equivalent |
|
||||||
|
|---|---|
|
||||||
|
| Fan-out to N consumers | meta-core holds N SSE connections, broadcasts events to each |
|
||||||
|
| Replay from a cursor | `Last-Event-ID` header on reconnect — built into the SSE spec |
|
||||||
|
| Retention / MAXLEN | Underlying Redis Stream still does the trimming; SSE is a thin shim |
|
||||||
|
| Backpressure on slow consumers | meta-core's SSE handler blocks on the TCP write; events stay in Redis until trimmed |
|
||||||
|
| Multiple streams (file:events, meta:events) | Separate endpoints: `/api/events/files`, `/api/events/meta` |
|
||||||
|
| At-least-once via XACK | Implicit — client persists last-received `id`, sends it on reconnect (see "Cursor semantics and consumer-group migration" below) |
|
||||||
|
|
||||||
|
The one Streams feature we don't get is **consumer groups for
|
||||||
|
horizontally-scaled replicas of the same logical consumer.** We don't use
|
||||||
|
this today and don't plan to — meta-sort, meta-fuse, etc. each run one
|
||||||
|
instance. If horizontal scale ever happens, it'll be by sharding hash-IDs
|
||||||
|
across replicas, not by splitting an event stream. SSE stays adequate.
|
||||||
|
|
||||||
|
### Cursor semantics and consumer-group migration
|
||||||
|
|
||||||
|
Today three consumers use `XREADGROUP` against named consumer groups, not
|
||||||
|
bare `XREAD`:
|
||||||
|
|
||||||
|
- meta-sort: group `meta-sort-processor`
|
||||||
|
(`packages/meta-sort/.../events/FileEventConsumer.ts:23,66-74`)
|
||||||
|
- meta-dup: group on `file:events`
|
||||||
|
(`packages/meta-dup/.../MetaEventConsumer.ts:102-104`)
|
||||||
|
- meta-fuse: dual path — `XREADGROUP` on one consumer, plain `XREAD` on
|
||||||
|
another (`packages/meta-fuse/.../kv/RedisClient.ts:826,907`)
|
||||||
|
|
||||||
|
`XREADGROUP` keeps server-side state per group: a Pending Entries List
|
||||||
|
(PEL) and a `last-delivered-id` cursor. The load-bearing use today is
|
||||||
|
meta-sort's `processPendingEntries(idleMs=30000)`, which exists to claim
|
||||||
|
work from a crashed consumer's PEL on restart.
|
||||||
|
|
||||||
|
SSE has no equivalent server-side state. After the migration the contract
|
||||||
|
shifts to: **each consumer persists its own `Last-Event-ID` and resumes
|
||||||
|
from it on reconnect.** Implications:
|
||||||
|
|
||||||
|
- PEL-based crash recovery is gone. A crashed consumer that hadn't
|
||||||
|
persisted its cursor since entry X resumes from X (its last *durably
|
||||||
|
persisted* cursor), not from "whatever was in flight." Acceptable for
|
||||||
|
every current consumer — none run multiple instances per group — but
|
||||||
|
call out in the PR C migration of meta-sort that the recovery primitive
|
||||||
|
is changing.
|
||||||
|
- Each consumer must own its cursor persistence. Cheapest place is the
|
||||||
|
consumer's own state (a file, a Redis key written under its own
|
||||||
|
prefix, an in-memory variable if the consumer is OK with at-most-once
|
||||||
|
on its own crash).
|
||||||
|
- No server-side "ACK" call. The cursor *is* the ack — when the consumer
|
||||||
|
successfully writes Last-Event-ID = N to its own state, entries ≤ N
|
||||||
|
are considered handled. The SSE handler does nothing.
|
||||||
|
|
||||||
|
### Endpoint spec
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/events/files
|
||||||
|
Request headers:
|
||||||
|
Last-Event-ID: <stream-entry-id> (optional; resume from this cursor)
|
||||||
|
Accept: text/event-stream
|
||||||
|
Response:
|
||||||
|
Content-Type: text/event-stream
|
||||||
|
Cache-Control: no-cache
|
||||||
|
Connection: keep-alive
|
||||||
|
X-Accel-Buffering: no (disable nginx response buffering)
|
||||||
|
```
|
||||||
|
|
||||||
|
Same shape for `/api/events/meta`.
|
||||||
|
|
||||||
|
### Event format
|
||||||
|
|
||||||
|
One SSE event per Redis Stream entry. Format:
|
||||||
|
|
||||||
|
```
|
||||||
|
id: 1747999999999-0
|
||||||
|
event: add
|
||||||
|
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1747999999999}
|
||||||
|
|
||||||
|
id: 1748000000123-0
|
||||||
|
event: change
|
||||||
|
data: {"path":"/watch/foo.mkv","size":4500000000,"midhash256":"bafk…","timestamp":1748000000123}
|
||||||
|
```
|
||||||
|
|
||||||
|
- `id` — **opaque** cursor. Today it's the Redis Stream entry ID
|
||||||
|
(`<ms>-<seq>`); consumers MUST treat it as a black-box string they
|
||||||
|
echo back via `Last-Event-ID`. This contract lets meta-core swap the
|
||||||
|
backing store later without breaking every consumer's cursor handling.
|
||||||
|
- `event` — the `type` field from the underlying entry (`add` | `change`
|
||||||
|
| `delete` | `rename` | `reset` for `file:events`; `set` | `del` |
|
||||||
|
`expire` for `meta:events`).
|
||||||
|
- `data` — JSON-encoded rest of the entry. Keys match the existing stream
|
||||||
|
payload exactly so downstream consumers don't change their parser.
|
||||||
|
|
||||||
|
### Backing-store retention and lifecycle
|
||||||
|
|
||||||
|
**Contract: the SSE wire is a faithful mirror of the underlying Redis
|
||||||
|
Streams.** Consumers rely on `Last-Event-ID` resumption working across
|
||||||
|
*every* kind of meta-core hiccup, including a restart — that's the entire
|
||||||
|
point of having a cursor protocol. The backing store must therefore
|
||||||
|
preserve entries across meta-core process restart and bound its growth
|
||||||
|
explicitly, not implicitly.
|
||||||
|
|
||||||
|
Today's behaviour is wrong on both counts:
|
||||||
|
|
||||||
|
| Stream | Restart behaviour | Retention |
|
||||||
|
|---|---|---|
|
||||||
|
| `meta:events` | **Cleared on every MetaPublisher.Start** (`events/meta_publisher.go:50-58`) | No `MAXLEN` (unbounded) |
|
||||||
|
| `file:events` | Persists across restart | No `MAXLEN` (unbounded), cleared only on deliberate reset (`watcher/dispatcher.go:82-95`) |
|
||||||
|
|
||||||
|
The `meta:events` clear-on-restart is the load-bearing problem. With it
|
||||||
|
in place, every meta-core restart silently invalidates every consumer's
|
||||||
|
persisted cursor — the SSE `gap` event would fire on the first reconnect
|
||||||
|
after every restart, and the consumer would have to re-bootstrap. That
|
||||||
|
turns "restart meta-core to apply a config change" into "every downstream
|
||||||
|
service does a full re-sync," which is exactly the regression SSE is
|
||||||
|
supposed to prevent.
|
||||||
|
|
||||||
|
PR A must fix both streams:
|
||||||
|
|
||||||
|
1. **Remove the unconditional `Del(MetaEventsStream)` in
|
||||||
|
`MetaPublisher.Start`.** Entries persist across restart. The republish
|
||||||
|
path (`RepublishAllMetadata`) stays as a one-shot bootstrap for
|
||||||
|
missing entries, but it does not pre-truncate the stream.
|
||||||
|
2. **Add `MAXLEN ~ N` to every `XAdd` on both streams** so growth is
|
||||||
|
bounded. Approximate trim (`~`) is cheap. Default `N = 100_000` per
|
||||||
|
stream is fine for the foreseeable workload (a few KB per entry,
|
||||||
|
bounded total ≈ a few hundred MB) and easy to tune via config.
|
||||||
|
3. **Keep the deliberate `EmitReset → ClearStream` path on
|
||||||
|
`file:events`.** That clear is a feature: a reset is the consumer
|
||||||
|
contract for "throw out everything you knew, re-bootstrap." Consumers
|
||||||
|
detect it through the `event: reset` payload that's already emitted
|
||||||
|
as the first entry after the clear (`dispatcher.go:99-`). The SSE
|
||||||
|
`gap` event handles the race where a consumer's cursor was trimmed
|
||||||
|
*between* the clear and its reconnect.
|
||||||
|
|
||||||
|
After PR A, the cursor contract is **"durable across meta-core restart
|
||||||
|
and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
|
||||||
|
when they persist a Last-Event-ID.
|
||||||
|
|
||||||
|
### Heartbeats
|
||||||
|
|
||||||
|
Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
|
||||||
|
when no events are flowing. Keeps reverse proxies and load balancers
|
||||||
|
from killing the connection as idle. The client ignores comment lines
|
||||||
|
per the SSE spec.
|
||||||
|
|
||||||
|
### Reconnect semantics
|
||||||
|
|
||||||
|
Per the SSE spec the client (or `EventSource`) auto-reconnects after a
|
||||||
|
network error, sending `Last-Event-ID`. The server resumes from the next
|
||||||
|
entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
|
||||||
|
|
||||||
|
**Gap handling.** If the requested ID has been trimmed out of the Redis
|
||||||
|
Stream by retention, the server emits one synthetic event before resuming
|
||||||
|
from the oldest available entry:
|
||||||
|
|
||||||
|
```
|
||||||
|
event: gap
|
||||||
|
data: {"requested":"<id>","resumeFrom":"<oldest-id>","reason":"retention"}
|
||||||
|
```
|
||||||
|
|
||||||
|
Consumers that care about gap-free delivery (currently none — the
|
||||||
|
schema indexer tolerates gaps; ingest tasks rebuild from scratch) can
|
||||||
|
react. Consumers that don't care ignore it.
|
||||||
|
|
||||||
|
### Error handling
|
||||||
|
|
||||||
|
- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
|
||||||
|
- **Redis fails mid-stream**: close the SSE connection. Client reconnects
|
||||||
|
with `Last-Event-ID`; the cycle resumes when Redis recovers.
|
||||||
|
- **Client disconnects**: handler goroutine exits cleanly; no per-client
|
||||||
|
state on meta-core.
|
||||||
|
|
||||||
|
### Auth
|
||||||
|
|
||||||
|
`/api/events/*` bypasses the hash-lock perimeter, same pattern as
|
||||||
|
`/api/meta/{cid}` and `/api/file/{cid}`. Event streams are internal
|
||||||
|
infrastructure and need to be reachable by peer services without going
|
||||||
|
through Authelia.
|
||||||
|
|
||||||
|
**Bypass ≠ public.** "Hash-lock bypass" means the OIDC sidecar doesn't
|
||||||
|
gate the route; it does *not* automatically mean Caddy proxies it from
|
||||||
|
the public hostname. The two existing precedents differ on this:
|
||||||
|
`/api/file/{cid}` is publicly reachable (peer-to-peer content fetch is
|
||||||
|
its job), `/api/meta/{cid}` likewise. For the SSE endpoints, the choice
|
||||||
|
is deliberate:
|
||||||
|
|
||||||
|
- If kept inside-only (no Caddy upstream for `/api/events/files` and
|
||||||
|
`/api/events/meta`), only sibling services on the docker network can
|
||||||
|
subscribe. This matches the rest of the "Redis is private" model and
|
||||||
|
is the recommended default.
|
||||||
|
- If exposed publicly, any browser can open an `EventSource` against
|
||||||
|
the stream without authentication — i.e. anyone can watch every file
|
||||||
|
event in the library. Only do this if there's a concrete consumer
|
||||||
|
that needs it and the visibility is acceptable.
|
||||||
|
|
||||||
|
PR A's Caddy change (see PR A scope, item 4) makes this an explicit
|
||||||
|
choice rather than a default behaviour. The recommendation is
|
||||||
|
inside-only until a concrete external consumer appears.
|
||||||
|
|
||||||
|
### Implementation footprint
|
||||||
|
|
||||||
|
The SSE handler itself is ~80 lines of Go in meta-core: one handler per
|
||||||
|
stream, each running an `XREAD … BLOCK 30s` loop and writing
|
||||||
|
SSE-formatted lines to the response. Go's `net/http` natively supports
|
||||||
|
streaming responses with `http.Flusher.Flush()`. Standard SSE clients
|
||||||
|
exist for every language we use: Node `eventsource`, Rust
|
||||||
|
`eventsource-client` (or `reqwest` + `tokio_util::codec`), Python
|
||||||
|
`sseclient`, Go `r3labs/sse`.
|
||||||
|
|
||||||
|
PR A as a whole is larger than the handler — see the PR A scope below
|
||||||
|
for the full list (backing-store fixes, Caddy wiring, poll deprecation)
|
||||||
|
that has to land alongside the handler for the cursor contract to hold.
|
||||||
|
|
||||||
|
## Eventing — considered alternatives
|
||||||
|
|
||||||
|
Recorded for posterity. None of these are being implemented; if any are
|
||||||
|
ever revisited it should be a deliberate reopen, not a drift.
|
||||||
|
|
||||||
|
- **Long-polling.** Strictly worse than SSE — connection churn on every
|
||||||
|
event, latency floor at the poll interval, no built-in resumption.
|
||||||
|
Rejected.
|
||||||
|
- **Webhooks (meta-core POSTs to subscribers).** Symmetric coupling:
|
||||||
|
every consumer would need to run an HTTP server reachable from
|
||||||
|
meta-core. Out-of-order delivery, retry, dead-letter all need
|
||||||
|
reinventing. Right for cross-organization integrations, wrong for
|
||||||
|
in-cluster eventing. Rejected.
|
||||||
|
- **External broker (Kafka / NATS / RabbitMQ).** Industrial-grade
|
||||||
|
pub/sub. New dependency, new operational surface, overkill for the
|
||||||
|
scale (single-host dev, ≤20-peer POC). Revisit if MetaMesh ever needs
|
||||||
|
cross-WAN federation at high throughput. Rejected for now.
|
||||||
|
- **Two Redises (writes Redis private, events Redis exposed).** Strongest
|
||||||
|
separation. Doubles operational complexity for no benefit over the
|
||||||
|
SSE design — once events go through HTTP, the events-Redis isn't
|
||||||
|
exposed either. Rejected.
|
||||||
|
- **Redis ACLs (Option B from earlier brainstorm).** Allows direct Redis
|
||||||
|
access from clients while restricting commands by user. Adds
|
||||||
|
credential plumbing and ACL config to operate; lets a buggy service
|
||||||
|
still connect, just with limited verbs. The SSE design closes the
|
||||||
|
network entirely, which is strictly stronger. Rejected — but the
|
||||||
|
equivalent escape hatch (open Redis to other services again) stays
|
||||||
|
available if we ever need it.
|
||||||
|
|
||||||
|
## Industry context — where this lands
|
||||||
|
|
||||||
|
The pattern we're adopting is the **Kubernetes / etcd** pattern, and
|
||||||
|
that's the most informative comparison.
|
||||||
|
|
||||||
|
Kubernetes is structurally identical to MetaMesh:
|
||||||
|
- A single source-of-truth store (etcd) holds all cluster state.
|
||||||
|
- A single service (kube-apiserver) is the sole writer to etcd.
|
||||||
|
- Many polyglot clients (kubectl, controllers, schedulers, kubelets,
|
||||||
|
operators) read and react to changes.
|
||||||
|
- Clients subscribe to *watch streams* for events.
|
||||||
|
|
||||||
|
What Kubernetes does, point-for-point:
|
||||||
|
- **etcd is never reached by clients.** It runs on a separate port,
|
||||||
|
often a separate network. kube-apiserver is the only thing that holds
|
||||||
|
etcd credentials. The lock is by network topology, not authentication.
|
||||||
|
- **Watch is also mediated.** kube-apiserver exposes `/watch` as an HTTP
|
||||||
|
streaming endpoint (very similar to SSE in shape). Clients don't watch
|
||||||
|
etcd directly — even though etcd has its own native watch primitive.
|
||||||
|
- **Storage portability falls out for free.** etcd could be swapped for
|
||||||
|
another store without any client change. We don't plan to swap Redis,
|
||||||
|
but the optionality is real and free.
|
||||||
|
|
||||||
|
Other systems with the same shape:
|
||||||
|
- **HashiCorp Consul.** Clients talk to a local agent over HTTP. The
|
||||||
|
agent talks to the cluster. KV writes and watches both flow through
|
||||||
|
the agent. Direct raft-store access is impossible.
|
||||||
|
- **Apache ZooKeeper.** Clients use the ZooKeeper client library which
|
||||||
|
speaks the ZooKeeper wire protocol; the data store is private.
|
||||||
|
- **Strapi / Sanity / Contentful** (headless CMS as analog). All
|
||||||
|
consumers — frontends, backend services, build pipelines — go through
|
||||||
|
the CMS HTTP API. Direct Postgres or content-store access is impossible
|
||||||
|
by design.
|
||||||
|
|
||||||
|
Where MetaMesh sat *before*: closer to the bad-monolith pattern
|
||||||
|
(everyone connects directly to the shared Redis). Where it sits *after*
|
||||||
|
this design: aligned with the canonical microservices pattern, and
|
||||||
|
specifically with the Kubernetes flavour where streams are mediated too.
|
||||||
|
|
||||||
|
### Advantages we're picking up
|
||||||
|
|
||||||
|
- **Schema evolution.** One service can change its data layout without
|
||||||
|
coordinating releases across every other service. The UUID-root
|
||||||
|
migration is the cautionary tale of what happens without this.
|
||||||
|
- **Validation choke point.** Every write passes through one code path
|
||||||
|
that can enforce invariants (reverse index, canonical_cid, type
|
||||||
|
constraints, schema version).
|
||||||
|
- **Observability.** Audit logs, request metrics, slow-query detection
|
||||||
|
all live at one layer.
|
||||||
|
- **Threat-model fit.** Untrusted-service-in-the-network is now in
|
||||||
|
scope; network isolation makes the concern moot.
|
||||||
|
- **Storage portability.** Optionality only, but real.
|
||||||
|
|
||||||
|
### Drawbacks (acknowledged)
|
||||||
|
|
||||||
|
- **Latency.** One extra hop per write (TCP + HTTP parse + handler).
|
||||||
|
Local-host overhead is sub-millisecond; mitigations are batched writes
|
||||||
|
(PATCH the whole metadata blob per plugin completion, not per field)
|
||||||
|
and Keep-Alive / HTTP/2 connections.
|
||||||
|
- **SSE-handler plumbing.** ~80 lines per stream endpoint plus per-client
|
||||||
|
goroutines. Trivial at our scale, real at Kubernetes scale (which is
|
||||||
|
why they invested in chunked watch with bookmarks, resource versions,
|
||||||
|
etc.). We can defer the same optimisations forever.
|
||||||
|
- **API surface size.** Every operation needs a corresponding endpoint.
|
||||||
|
Most of it already exists; one new SSE endpoint per stream is the
|
||||||
|
only new surface area introduced here.
|
||||||
|
|
||||||
|
## Enforcement — what "lock" actually means
|
||||||
|
|
||||||
|
Three layers, in order of effectiveness:
|
||||||
|
|
||||||
|
1. **Network isolation (hard).** Redis is on a container-private port.
|
||||||
|
No host port mapping. No docker-network alias. `kv-leader.info` does
|
||||||
|
not publish a Redis URL. Any service that wants to talk to Redis
|
||||||
|
needs container privileges, and at that point the lock isn't the
|
||||||
|
concern — operating-system isolation is. This is the actual lock.
|
||||||
|
|
||||||
|
2. **Documentation (convention).** This file, plus a one-line note in
|
||||||
|
`CLAUDE.md` and in each consumer service's `CLAUDE.md`: "All metadata
|
||||||
|
I/O routes through meta-core's HTTP API. Redis is private to
|
||||||
|
metacore-app." Engineers reading the codebase find the rule on first
|
||||||
|
contact. This catches the case of a future meta-core PR that
|
||||||
|
accidentally exposes Redis again.
|
||||||
|
|
||||||
|
3. **Code review (convention).** PRs that re-expose Redis (a port
|
||||||
|
mapping, an alias, a credential leak via leader-info) get rejected
|
||||||
|
with a pointer to this doc.
|
||||||
|
|
||||||
|
No runtime canary or lint is proposed — the network isolation makes
|
||||||
|
violations physically impossible from outside, and inside meta-core the
|
||||||
|
small file count makes review sufficient. Layer 1 does the heavy
|
||||||
|
lifting.
|
||||||
|
|
||||||
|
## Migration plan
|
||||||
|
|
||||||
|
Four PRs, additive, ordered so the system stays working throughout.
|
||||||
|
Closing the network access (PR D) is the moment the lock becomes a
|
||||||
|
lock; everything before it is opt-in migration that consumers can adopt
|
||||||
|
at their own pace.
|
||||||
|
|
||||||
|
**All four PRs have landed.** What follows is the original plan as
|
||||||
|
written; see "Implementation outcome" at the bottom for the delta
|
||||||
|
between plan and ship.
|
||||||
|
|
||||||
|
### PR A — SSE endpoints in meta-core
|
||||||
|
|
||||||
|
Adds `GET /api/events/files` and `GET /api/events/meta`. Implements the
|
||||||
|
`Last-Event-ID` semantics, gap handling, heartbeats, and the per-stream
|
||||||
|
XREAD loop. Redis Streams stay reachable directly; consumers can
|
||||||
|
migrate one at a time.
|
||||||
|
|
||||||
|
Scope, in PR-order:
|
||||||
|
|
||||||
|
1. Backing-store fixes (see "Backing-store retention and lifecycle"
|
||||||
|
above) — remove the `meta:events` clear-on-restart, add `MAXLEN ~ N`
|
||||||
|
to every `XAdd`. These are prerequisites for the SSE cursor contract
|
||||||
|
to hold; without them PR A is not deliverable.
|
||||||
|
2. SSE handlers for the two streams.
|
||||||
|
3. Deprecate `GET /api/events/poll` (`watcher/handlers.go:29`). It
|
||||||
|
exists today as a long-poll variant over the same `file:events`
|
||||||
|
stream and is described in code as "always empty in practice"
|
||||||
|
(`watcher/handlers.go:55-58`). After SSE lands it's redundant — keep
|
||||||
|
it responding 200 for one release with a `Deprecation` header, then
|
||||||
|
remove. The `/api/scan/*` deprecation pattern in the same file is
|
||||||
|
the template.
|
||||||
|
4. Caddy perimeter wiring (see "Auth" subsection above) — explicit
|
||||||
|
allow-list for `/api/events/files` and `/api/events/meta` matching
|
||||||
|
whatever the perimeter policy decides. Either inside-only via the
|
||||||
|
docker network, or public via Caddy with deliberate authorisation.
|
||||||
|
Do not let the routes default-leak past the hash-lock.
|
||||||
|
|
||||||
|
Roughly 200 lines of Go + ~50 lines of test (covering reconnect from a
|
||||||
|
cursor, gap emission when the cursor is trimmed, heartbeat cadence,
|
||||||
|
durability across a meta-core restart).
|
||||||
|
|
||||||
|
### PR B — meta-sort writes via HTTP
|
||||||
|
|
||||||
|
Swap `RedisClient` for an `HttpKVClient` implementation of the existing
|
||||||
|
`IKVClient` interface in `packages/meta-sort-core`. All writes
|
||||||
|
(`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
|
||||||
|
calls against the **`/meta/{hash}` family** — the canonical write
|
||||||
|
surface for service-to-service traffic. Per-plugin write coalescing
|
||||||
|
uses `PATCH /meta/{hash}` (already calls `MergeMetadataFlat`
|
||||||
|
server-side with the partial-update semantics we need; see
|
||||||
|
`handlers.go:533-566`).
|
||||||
|
|
||||||
|
Reads from Redis stay direct in this PR (no consumer migration yet);
|
||||||
|
only writes move. Streams stay on direct XREAD.
|
||||||
|
|
||||||
|
**Pre-audit (already done):** meta-sort's production `kvClient.*`
|
||||||
|
surface is `setMetadataFlat` / `deleteMetadataFlat` / `getMetadataFlat`
|
||||||
|
/ `getMetadata` / `getAllHashIds` / `health` plus the `file:events`
|
||||||
|
stream consumer. No orphan keyspaces, no generic `kv.set(arbitrary_key)`
|
||||||
|
callers — the HTTP migration covers the entire write path with no
|
||||||
|
side bets.
|
||||||
|
|
||||||
|
### PR C — Stream consumers migrate to SSE
|
||||||
|
|
||||||
|
meta-sort's event consumer (`FileEventConsumer.ts`), meta-fuse's stream
|
||||||
|
reader, meta-stremio's event hook, meta-dup's monitor, meta-share's
|
||||||
|
ingest task — each swaps its `XREADGROUP` / `XREAD` loop for an SSE
|
||||||
|
client. Per-language standard libraries make this small (one HTTP
|
||||||
|
request, one event-handler closure).
|
||||||
|
|
||||||
|
In the same PR, each consumer's `LeaderClient` (TypeScript x2, Python,
|
||||||
|
plus any new Rust binding for meta-share if added) drops its
|
||||||
|
*requirement* on the `redisUrl` field — i.e. it stops opening a Redis
|
||||||
|
connection. It can still tolerate the field being present (PR D removes
|
||||||
|
it from the publisher side). This ordering matters: PR D's loud-failure
|
||||||
|
guard cannot ship until every consumer has been demonstrated to boot
|
||||||
|
without `redisUrl`.
|
||||||
|
|
||||||
|
The schema indexer is internal to meta-core and unaffected.
|
||||||
|
|
||||||
|
### PR D — Close Redis network access
|
||||||
|
|
||||||
|
PR D must land **after** every consumer in PR C has stopped requiring
|
||||||
|
`redisUrl` (verify by booting each service against a meta-core build
|
||||||
|
that already omits the field — staging environment or feature-flagged
|
||||||
|
release). Until that's true, this PR will deadlock the boot sequence.
|
||||||
|
|
||||||
|
Then, in this order:
|
||||||
|
|
||||||
|
1. Update consumer-side discovery (`LeaderClient` in each service) to
|
||||||
|
*log loudly* if it sees a `redisUrl` field — guards against
|
||||||
|
accidental rollback. (Loud log, not hard failure, while the field
|
||||||
|
is still being written — flip to hard failure once step 4 lands.)
|
||||||
|
2. Stop publishing `redisUrl` from `/urls` and from `kv-leader.info`.
|
||||||
|
3. Remove the `meta-core` docker-network alias on the metacore-app
|
||||||
|
service (other services no longer need DNS for the Redis port).
|
||||||
|
4. Remove `6380:6379` host port mapping in `dev/docker-compose.yml`.
|
||||||
|
5. Flip the LeaderClient guard from "warn" to "raise" — any future
|
||||||
|
consumer that tries to require the field fails loudly at boot.
|
||||||
|
|
||||||
|
After PR D, attempting to connect to Redis from anywhere except
|
||||||
|
metacore-app produces a connection refused. The lock is real.
|
||||||
|
|
||||||
|
### Out of scope for this work
|
||||||
|
|
||||||
|
- **Redis ACLs / proxy.** Network isolation makes these unnecessary.
|
||||||
|
Available as an escape hatch if the threat model ever changes.
|
||||||
|
- **Replacing Redis Streams under the hood.** SSE is the externalised
|
||||||
|
contract; the backing store is an implementation detail meta-core can
|
||||||
|
change later.
|
||||||
|
- **External brokers.** Out of scope absent a concrete driver
|
||||||
|
(cross-WAN federation at high throughput).
|
||||||
|
|
||||||
|
## Decisions recorded
|
||||||
|
|
||||||
|
| Question | Decision | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| Writes from non-meta-core services? | Forbidden by network topology | Redis unreachable; not a convention |
|
||||||
|
| Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
|
||||||
|
| Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
|
||||||
|
| Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
|
||||||
|
| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
|
||||||
|
| Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
|
||||||
|
| Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
|
||||||
|
| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
|
||||||
|
| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
|
||||||
|
| **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
|
||||||
|
| **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
|
||||||
|
| **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
|
||||||
|
|
||||||
|
## Error envelope
|
||||||
|
|
||||||
|
Standard shape for all 4xx / 5xx responses from the metadata API:
|
||||||
|
|
||||||
|
```json
|
||||||
|
{
|
||||||
|
"error": "alias_collision",
|
||||||
|
"message": "CID midhash256:bafk… already resolves to UUID 01JKR…",
|
||||||
|
"retryable": false
|
||||||
|
}
|
||||||
|
```
|
||||||
|
|
||||||
|
Stable `error` slugs the client should switch on:
|
||||||
|
|
||||||
|
| Slug | HTTP status | Retryable | Meaning |
|
||||||
|
|---|---|---|---|
|
||||||
|
| `alias_collision` | 409 | no | A different UUID already owns the CID the caller is trying to register. Caller bug; escalate. |
|
||||||
|
| `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
|
||||||
|
| `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
|
||||||
|
| `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
|
||||||
|
| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
|
||||||
|
| `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
|
||||||
|
|
||||||
|
**Implemented.** Every 4xx/5xx response across `internal/api`,
|
||||||
|
`internal/mounts`, `internal/watcher`, and `internal/watchers` now
|
||||||
|
emits the typed envelope. The slug-mapping helper is duplicated across
|
||||||
|
those packages (one ~15-line `slugForStatus` per package) to avoid an
|
||||||
|
import cycle on the api package; consolidating it into a shared
|
||||||
|
`internal/httperr` package is a worthwhile follow-up but not load-bearing.
|
||||||
|
|
||||||
|
## Remaining open questions
|
||||||
|
|
||||||
|
Mostly answered post-implementation:
|
||||||
|
|
||||||
|
- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
|
||||||
|
editor's React UI continues to call it; the surface itself was untouched
|
||||||
|
by this work. Worth a follow-up to consolidate the editor onto
|
||||||
|
`/meta/{hash}` and delete the parallel routes.
|
||||||
|
|
||||||
|
- **Initial bootstrap flood on `/api/events/files`.** Resolved in
|
||||||
|
practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
|
||||||
|
replay every entry on each restart and the StreamingStateBuilder
|
||||||
|
handles the resulting burst (~2.5k property events on a 50-file
|
||||||
|
library) without issue. meta-sort persists its cursor so only sees
|
||||||
|
the flood once per fresh deploy. meta-stremio filters to "interesting
|
||||||
|
fields" before triggering cache invalidation, so the burst is mostly
|
||||||
|
no-ops.
|
||||||
|
|
||||||
|
- **meta-dup SSE port.** Still pending — meta-dup is in no active
|
||||||
|
compose file. When revived, copy meta-fuse's pattern wholesale
|
||||||
|
(SSEEventClient + MetaCoreApiClient). Tracked separately.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **Authentication between internal services.** The hash-lock OIDC
|
||||||
|
perimeter handles external clients; internal service-to-service
|
||||||
|
trust is by network isolation (docker network for dev, equivalent
|
||||||
|
isolation in prod deployments). Adding service-to-service auth is a
|
||||||
|
separate concern.
|
||||||
|
- **Replacing the public HTTP API surface.** This proposal exercises
|
||||||
|
what already exists; the only new endpoints are the two SSE streams.
|
||||||
|
- **Introducing new transport, broker, proxy, or sidecar.** Everything
|
||||||
|
builds on what's deployed today.
|
||||||
|
- **Migrating meta-fuse / meta-stremio / meta-dup direct Redis reads
|
||||||
|
via XSCAN/GET** (separate from event-stream reads). ~~Was a non-goal~~
|
||||||
|
ended up being required for PR D's network closure to be safe.
|
||||||
|
meta-fuse and meta-stremio migrated; meta-dup deferred (not deployed).
|
||||||
|
See PR C / "Implementation outcome" for details.
|
||||||
|
|
||||||
|
## Implementation outcome
|
||||||
|
|
||||||
|
What actually shipped vs. what this doc planned.
|
||||||
|
|
||||||
|
### Delta against the original plan
|
||||||
|
|
||||||
|
- **Read-path migration for meta-fuse and meta-stremio is now in scope
|
||||||
|
and done.** The original "out of scope" carve-out was wrong: closing
|
||||||
|
the Redis network port doesn't just break stream consumers, it also
|
||||||
|
breaks every `kvClient.get` / `redis_storage.get_all_videos` call.
|
||||||
|
Both services grew their own `MetaCoreApiClient` (TS for meta-fuse,
|
||||||
|
Python for meta-stremio) and now route reads through `/meta/{hash}`
|
||||||
|
+ `/meta` + `/api/file/{cid}/info`.
|
||||||
|
- **meta-stremio's HTTP mode auto-recovers from transient meta-core
|
||||||
|
outages.** `is_connected` re-probes meta-core's `/health` on every
|
||||||
|
call (cheap; a stateless GET), so a metacore-app restart no longer
|
||||||
|
latches the addon into a permanent disconnected state. The
|
||||||
|
background reconnect thread is only needed for legacy Redis mode
|
||||||
|
where the socket itself is sticky.
|
||||||
|
- **Hard `redisUrl` guard is gated by `ALLOW_LEGACY_REDIS_URL=1`.** All
|
||||||
|
three consumer LeaderClients (TS x2 + Python) throw on seeing the
|
||||||
|
field by default. Setting the env var downgrades to a warn — escape
|
||||||
|
hatch for a deliberate temporary rollback to a meta-core that still
|
||||||
|
publishes the field.
|
||||||
|
- **Write-path CID resolution** turned out to be a hard prerequisite
|
||||||
|
for the lockdown to work in practice. Documented separately below.
|
||||||
|
- **PR D ordering matched the doc** (consumer migration first, then
|
||||||
|
publisher stops emitting, then network closure). The `meta-core`
|
||||||
|
docker-network alias and the `6380:6379` host port mapping are both
|
||||||
|
removed; sibling services reach the HTTP/SSE/WebDAV surface via the
|
||||||
|
canonical `metacore-app` hostname.
|
||||||
|
- **meta-dup is the one service that didn't migrate.** It's in no
|
||||||
|
active compose file. The TS port is straightforward when needed: copy
|
||||||
|
meta-fuse's pattern (SSEEventClient for streams, MetaCoreApiClient
|
||||||
|
for reads).
|
||||||
|
|
||||||
|
### Write-path CID resolution (`storage.ResolveRoot`)
|
||||||
|
|
||||||
|
Not in the original plan but load-bearing.
|
||||||
|
|
||||||
|
**The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed
|
||||||
|
every write through `/meta/{hash}`. But `{hash}` is the midhash256 CID
|
||||||
|
that meta-sort received via file:events, not the watcher's minted UUID.
|
||||||
|
With no resolution step in the handler, every write went to
|
||||||
|
`file:<midhash>/*` — a parallel root the watcher's UUID never saw. The
|
||||||
|
auto-alias hook on `cid_midhash256=<value>` then OVERWROTE the
|
||||||
|
watcher's `cid:midhash256:<value>` → UUID alias to point at the
|
||||||
|
midhash itself, leaving the UUID stranded with only the watcher's
|
||||||
|
sparse fields (filePath, sizeByte, mtimeNano, duplicates) and the
|
||||||
|
midhash root carrying the rich plugin output. This is the exact
|
||||||
|
"dual roots per file" failure the doc's Motivation section listed as
|
||||||
|
the prior bug — it just moved from "convention violation" to "API
|
||||||
|
contract violation."
|
||||||
|
|
||||||
|
**The fix.** Three layers in `internal/storage/cid_resolution.go`:
|
||||||
|
|
||||||
|
1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}`
|
||||||
|
read/write handler, look up `cid:midhash256:<hash>` in the reverse
|
||||||
|
index. If found, use the UUID as the storage root. Falls through to
|
||||||
|
the bare hash for direct-UUID writes / legacy entries.
|
||||||
|
2. **Self-pointing alias guard in `addAliasLocked`** — refuse to
|
||||||
|
register `cid:<algo>:<v>` → `<v>`. The dual-root bug's chicken-and-
|
||||||
|
egg is: meta-sort writes first, registers self-pointing alias,
|
||||||
|
watcher then can't recover. Refusing the self-write keeps the
|
||||||
|
watcher's legitimate alias intact.
|
||||||
|
3. **`MigrateDualRoots` sweep** (`POST /api/admin/migrate-dual-roots`).
|
||||||
|
One-shot fixer for already-stranded entries: walks every UUID's
|
||||||
|
`midhash256` field to build the orphan index, merges each stranded
|
||||||
|
`file:<midhash>/*` into the matching UUID via `MergeMetadataFlat`,
|
||||||
|
deletes the stranded root, and re-points the alias.
|
||||||
|
|
||||||
|
Verified on the dev stack: 48 stranded entries were reunited; the
|
||||||
|
editor now shows every file with its full plugin metadata under one
|
||||||
|
UUID; future writes stay unified.
|
||||||
|
|
||||||
|
### Typed error envelope
|
||||||
|
|
||||||
|
Done across all four handler packages (`api`, `mounts`, `watcher`,
|
||||||
|
`watchers`). Each has its own `writeError` returning
|
||||||
|
`{error: <slug>, message, retryable}`. The slug-mapping helper is
|
||||||
|
duplicated in each to avoid an import cycle on the `api` package.
|
||||||
|
|
||||||
|
### What's NOT done
|
||||||
|
|
||||||
|
- **meta-dup SSE + read migration.** Not deployed, no test harness.
|
||||||
|
When revived, copy meta-fuse's pattern.
|
||||||
|
- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate
|
||||||
|
editor-internal API surface is untouched; could be retired in a
|
||||||
|
follow-up by porting the editor UI to `/meta/{hash}`.
|
||||||
|
- **Consolidated `internal/httperr` package** for the typed envelope
|
||||||
|
helpers (currently duplicated 4× to avoid the import cycle). Code
|
||||||
|
hygiene; functionally a no-op.
|
||||||
|
|
||||||
|
### Verification done
|
||||||
|
|
||||||
|
- All four service dashboards load cleanly with no console errors.
|
||||||
|
- meta-sort SSE consumer + HTTP write path verified end-to-end by
|
||||||
|
delete + inject + reprocess of Sintel.
|
||||||
|
- meta-fuse SSE consumer rebuilds VFS to 48 files on each boot.
|
||||||
|
- meta-stremio catalog API now surfaces the full library (47 videos,
|
||||||
|
1 movie + 5 series + 46 episodes — was 0/0/0 before the Python
|
||||||
|
read-path migration because LeaderStorage broke when it tried Redis).
|
||||||
|
- `redis-cli` from any container other than `metacore-app` produces a
|
||||||
|
connection refused; `cid:midhash256:<H>` aliases all point to UUIDs
|
||||||
|
(no self-pointing); `file:__index__` has only UUID-style hashIds.
|
||||||
|
- ALLOW_LEGACY_REDIS_URL=1 escape hatch present in all three consumer
|
||||||
|
LeaderClients (TS x2 + Python).
|
||||||
|
|
||||||
|
### Known test-suite gaps (post-implementation bats run)
|
||||||
|
|
||||||
|
Running `docker exec meta-test-runner /app/test/test.sh` after the
|
||||||
|
lockdown shipped reveals a number of failing suites, almost all of
|
||||||
|
which are pre-existing test/environment drift unrelated to this work:
|
||||||
|
|
||||||
|
| Failure | Root cause | Relation to lockdown |
|
||||||
|
|---|---|---|
|
||||||
|
| meta-stremio (~35 tests), stremio-tmdb (6 tests) | Tests call `/manifest.json` directly; dev `.env` sets `STREMIO_HASH_API_SEED=dev-stremio-secret` which puts the addon behind a path-token prefix | Unrelated — pre-existing env mismatch |
|
||||||
|
| meta-fuse FUSE-mount (3 tests) | Tests use `docker exec meta-fuse` but the container is named `metafuse-app` (Yundera prod-parity rename) | Unrelated — predates this work |
|
||||||
|
| meta-fuse WebDAV PROPFIND (4 tests) | Tests use Caddy URL with WebDAV basic-auth credentials that no longer match the hash-lock perimeter | Unrelated — predates this work |
|
||||||
|
| meta-sort "API integration tests" (1 test) | Test runs `docker exec meta-sort pnpm test`; container is `metasort-app` | Unrelated — predates this work |
|
||||||
|
| meta-core "create and delete mount config" (1 test) | Test creates an NFS mount; production code dropped NFS support ("NFS mounts are no longer supported") | Unrelated — predates this work |
|
||||||
|
| service-discovery (9 tests) | Cascading from meta-stremio breakage (test expects unauth `/api/services` on stremio) | Unrelated to lockdown directly |
|
||||||
|
| meta-share / share-1c (multiple) | Tests assert `"phase":"1f"` in `/health`; meta-share has moved past that phase | Unrelated — meta-share evolution |
|
||||||
|
| flat-keys: "filePath field exists" (1 test) | Test SRANDMEMBERs file:__index__; if it lands on a UUID with no filePath (e.g. a manual test write) it fails | Was hit transiently after my interactive testing left stub entries; resolved by cleanup |
|
||||||
|
|
||||||
|
No failure traceable to a real regression from this work was found.
|
||||||
|
The dev-environment test harness needs an unrelated refresh to catch
|
||||||
|
up with the prod-parity container renames, the NFS removal, the
|
||||||
|
STREMIO_HASH_API_SEED default, and meta-share's current phase.
|
||||||
+388
-891
File diff suppressed because it is too large
Load Diff
+211
-534
@@ -1,595 +1,272 @@
|
|||||||
# Redis Metadata Storage Architecture
|
# Redis Metadata Storage Structure
|
||||||
|
|
||||||
This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling.
|
How meta-core lays out file metadata in Redis. This document covers the
|
||||||
|
on-disk shape (UUID-rooted flat string keys, CID reverse index,
|
||||||
|
per-root sets) and the schema-version gate. It does **not** enumerate
|
||||||
|
field semantics — that's the job of the repo-root
|
||||||
|
[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key
|
||||||
|
registry for everyone who writes to or reads from metadata.
|
||||||
|
|
||||||
## Overview
|
For the design rationale (why UUIDs instead of content hashes, why a
|
||||||
|
reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md).
|
||||||
|
For the external-access contract (HTTP-only, SSE for events), see
|
||||||
|
[`api-mediated-access.md`](api-mediated-access.md).
|
||||||
|
|
||||||
MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables:
|
## Quick summary
|
||||||
|
|
||||||
- **Redis keyspace notifications** for field-level change detection
|
- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout),
|
||||||
- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates)
|
not content hashes. Examples in this doc use the placeholder
|
||||||
- **meta:events stream** integration for real-time updates across services
|
`01JKR8XW5T4QMV7AHNJ2DEFGHK`.
|
||||||
- Individual field updates without affecting other properties
|
- **Properties are flat Redis STRING keys**, one per leaf:
|
||||||
- Fast VFS lookups using content-based hash IDs
|
`file:<uuid>/<property/path>` → string value. *Not* Redis Hashes.
|
||||||
|
- **CIDs are reverse-index aliases**: `cid:<algorithm>:<value>` → `<uuid>`
|
||||||
|
(a Redis STRING per known CID).
|
||||||
|
- **Per-root sets**: `file:<uuid>/cids` (every CID), `file:<uuid>/duplicates`
|
||||||
|
(alternate paths with the same content), plus `file:__index__` (every UUID).
|
||||||
|
- **A schema-version sentinel** (`meta-core:schema-version`) refuses to
|
||||||
|
boot against legacy data.
|
||||||
|
|
||||||
## Key Structure
|
Implementation lives in `internal/storage/client.go`,
|
||||||
|
`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and
|
||||||
|
`internal/storage/schema_version.go`.
|
||||||
|
|
||||||
### Flat Key Storage
|
## Key structure
|
||||||
|
|
||||||
Each file property is stored as a separate Redis STRING key:
|
### Roots
|
||||||
|
|
||||||
```
|
```
|
||||||
Key Pattern: file:{hashId}/{property/path}
|
file:<uuid>/<property/path> → Redis STRING (always)
|
||||||
Key Type: STRING
|
|
||||||
Value: String value
|
|
||||||
```
|
```
|
||||||
|
|
||||||
**Example Redis commands:**
|
UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`):
|
||||||
```bash
|
UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The
|
||||||
# Store file metadata (each property is a separate key)
|
time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted
|
||||||
SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..."
|
in a given window, which restored some of the debug-ability lost when
|
||||||
SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv"
|
midhash256 stopped being the root.
|
||||||
SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5"
|
|
||||||
SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm"
|
|
||||||
|
|
||||||
# Retrieve single field
|
Properties use `/` as the path separator. Nested objects in source code
|
||||||
GET "file:bafkr4ih5kapbjzq.../fileinfo/duration"
|
are flattened to flat keys on write and reconstructed on read. Examples:
|
||||||
|
|
||||||
# Retrieve all metadata for a file (scan by prefix)
|
|
||||||
SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000
|
|
||||||
```
|
|
||||||
|
|
||||||
### Why Flat Keys?
|
|
||||||
|
|
||||||
The flat key structure (vs. Redis Hashes) enables:
|
|
||||||
|
|
||||||
1. **Keyspace Notifications**: Redis publishes events when individual keys change
|
|
||||||
2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*`
|
|
||||||
3. **Selective Updates**: Update one field without touching others
|
|
||||||
4. **Stream Integration**: The `meta:events` stream captures field-level changes
|
|
||||||
|
|
||||||
### File Index
|
|
||||||
|
|
||||||
A Redis Set tracks all known file hash IDs:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
Key: file:__index__
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv"
|
||||||
Type: Set
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536"
|
||||||
Value: {hashId1, hashId2, hashId3, ...}
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000"
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5"
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264"
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..."
|
||||||
```
|
```
|
||||||
|
|
||||||
This enables fast bulk queries like "get all files" without scanning keys.
|
### Reverse-index aliases (CIDs)
|
||||||
|
|
||||||
|
Every CID known for a file is registered as its own Redis STRING:
|
||||||
|
|
||||||
|
```
|
||||||
|
cid:<algorithm>:<value> → <uuid>
|
||||||
|
```
|
||||||
|
|
||||||
|
Examples:
|
||||||
|
|
||||||
|
```
|
||||||
|
cid:midhash256:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
cid:sha256:abc... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
cid:ipfs:bafy... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
```
|
||||||
|
|
||||||
|
The reverse index makes `GET /api/meta/<cid>` an O(1) lookup for any
|
||||||
|
registered CID, independent of which hash algorithm "primary" — there is
|
||||||
|
no primary. The auto-alias hooks in `SetProperty` and `SetMetadataFlat`
|
||||||
|
(see `internal/storage/cid_resolution.go`) catch any write of a `cid_*`
|
||||||
|
or `midhash256` field and register the alias automatically, so plugins
|
||||||
|
don't need to know about the reverse index.
|
||||||
|
|
||||||
|
### Per-root sets
|
||||||
|
|
||||||
|
```
|
||||||
|
file:<uuid>/cids → Redis SET of "<algorithm>:<value>" tokens
|
||||||
|
file:<uuid>/canonical_cid → STRING — the highest-ranked CID for advertising externally
|
||||||
|
file:<uuid>/duplicates → Redis SET of alternate paths with same content
|
||||||
|
```
|
||||||
|
|
||||||
|
`canonical_cid` is reconciled by `reconcileCanonicalCIDLocked` whenever
|
||||||
|
the `cids` set changes; ranking is done by `internal/cid/rank.go`.
|
||||||
|
|
||||||
|
### Global index
|
||||||
|
|
||||||
|
```
|
||||||
|
file:__index__ → Redis SET of all UUIDs
|
||||||
|
```
|
||||||
|
|
||||||
|
`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate
|
||||||
|
every file without scanning the keyspace.
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# Get all hash IDs
|
SMEMBERS file:__index__ # all UUIDs
|
||||||
SMEMBERS "file:__index__"
|
SCARD file:__index__ # total file count
|
||||||
|
SISMEMBER file:__index__ <uuid>
|
||||||
# Count total files
|
|
||||||
SCARD "file:__index__"
|
|
||||||
|
|
||||||
# Check if a file exists
|
|
||||||
SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..."
|
|
||||||
```
|
```
|
||||||
|
|
||||||
### Keyspace Notifications
|
### Schema-version sentinel
|
||||||
|
|
||||||
Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Required Redis config (set by meta-core)
|
|
||||||
CONFIG SET notify-keyspace-events "K$"
|
|
||||||
|
|
||||||
# K = Keyspace events (published on __keyspace@<db>__:<key>)
|
|
||||||
# $ = String commands (SET, etc.)
|
|
||||||
```
|
|
||||||
|
|
||||||
Services can subscribe to keyspace events for real-time updates:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Subscribe to all file key changes
|
|
||||||
PSUBSCRIBE '__keyspace@0__:file:*'
|
|
||||||
|
|
||||||
# Subscribe to specific field pattern (e.g., TMDB updates)
|
|
||||||
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
|
|
||||||
```
|
|
||||||
|
|
||||||
### meta:events Stream
|
|
||||||
|
|
||||||
The `meta:events` stream provides reliable event delivery for file system events from meta-core:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
Stream Key: meta:events
|
meta-core:schema-version → STRING (currently "2")
|
||||||
Entry Fields:
|
|
||||||
- type: add | change | delete | rename | reset
|
|
||||||
- path: File path relative to /files
|
|
||||||
- midhash256: Content hash (for add/change events)
|
|
||||||
- oldPath: Previous path (for rename events)
|
|
||||||
- timestamp: Event timestamp
|
|
||||||
```
|
```
|
||||||
|
|
||||||
Consumer groups enable reliable processing with acknowledgment:
|
Set at first boot, checked on every subsequent boot
|
||||||
|
(`internal/storage/schema_version.go`). If the sentinel is missing but
|
||||||
|
`file:__index__` is non-empty, the gate refuses to boot (legacy data).
|
||||||
|
If the sentinel is present but doesn't match `SchemaVersion`, the gate
|
||||||
|
also refuses to boot. Alpha-stage policy: operators wipe and restart;
|
||||||
|
there is no automated migration.
|
||||||
|
|
||||||
```bash
|
## Why flat STRING keys?
|
||||||
# Create consumer group
|
|
||||||
XGROUP CREATE meta:events mygroup 0 MKSTREAM
|
|
||||||
|
|
||||||
# Read new events (blocking)
|
The store does not use Redis Hashes. The reasons are:
|
||||||
XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events >
|
|
||||||
|
|
||||||
# Acknowledge processed events
|
1. **Field-level keyspace notifications.** Redis publishes
|
||||||
XACK meta:events mygroup <entry-id>
|
`__keyspace@0__:<key>` on every SET/DEL of a STRING key. With Hashes
|
||||||
```
|
you'd only get one notification on the hash itself. meta-core's
|
||||||
|
`MetaPublisher` (`internal/events/meta_publisher.go`) consumes these
|
||||||
|
notifications and publishes structured events to the `meta:events`
|
||||||
|
Redis Stream.
|
||||||
|
2. **Selective updates without read-modify-write.** A plugin writing
|
||||||
|
`tmdb/poster` doesn't have to read the rest of the document.
|
||||||
|
3. **Compatibility with the CID alias hooks.** Field writes can trigger
|
||||||
|
alias-maintenance work in `SetProperty` without the plugin caring.
|
||||||
|
|
||||||
## Primary Hash ID
|
## Reads and writes
|
||||||
|
|
||||||
The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from:
|
The Go API in `internal/storage/client.go`:
|
||||||
|
|
||||||
1. SHA-256 hash of the middle 1MB of the file
|
| Method | Behaviour |
|
||||||
2. File size in bytes
|
|---|---|
|
||||||
3. IPFS CID v1 encoding (multicodec 0x1000)
|
| `GetMetadataFlat(uuid)` | `SCAN` for `file:<uuid>/*`, `MGET` the values, return `map[string]string`. |
|
||||||
|
| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register any `cid_*` aliases. |
|
||||||
|
| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. |
|
||||||
|
| `GetProperty(uuid, field)` | `GET file:<uuid>/<field>`. |
|
||||||
|
| `SetProperty(uuid, field, value)` | `SET file:<uuid>/<field>`; auto-register alias if the field is a CID. |
|
||||||
|
| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:<uuid>/*`, unmap every reverse-index entry. |
|
||||||
|
| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. |
|
||||||
|
| `AddAlias(uuid, "<algo>:<value>")` | Set `cid:<algo>:<value>` → `<uuid>`, add to `file:<uuid>/cids`, reconcile `canonical_cid`. |
|
||||||
|
| `ResolveCID("<algo>:<value>")` | `GET cid:<algo>:<value>` → uuid. |
|
||||||
|
|
||||||
**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...`
|
## Reconstruction (flat → nested)
|
||||||
|
|
||||||
This provides:
|
When the HTTP API returns a document, it reconstructs the flattened
|
||||||
- Content-based addressing (same file = same ID)
|
storage back into a nested JSON object:
|
||||||
- Fast computation (only reads 1MB)
|
|
||||||
- IPFS compatibility for future distributed storage
|
|
||||||
|
|
||||||
### Collision Handling
|
|
||||||
|
|
||||||
If two different files produce the same midhash (rare), the system falls back to full SHA-256:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Normal case
|
|
||||||
hashId = midHash256; // "bafkr4ih5kapbjzq..."
|
|
||||||
|
|
||||||
// Collision detected (different filePath with same midhash)
|
|
||||||
hashId = metadata['cid_sha2-256']; // Full hash fallback
|
|
||||||
```
|
|
||||||
|
|
||||||
## Metadata Flattening
|
|
||||||
|
|
||||||
Nested objects are flattened into path-based keys for efficient Redis storage.
|
|
||||||
|
|
||||||
### Flattening Rules
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
// Input (nested object)
|
|
||||||
{
|
|
||||||
title: "Inception",
|
|
||||||
video: {
|
|
||||||
codec: "h265",
|
|
||||||
width: "1920"
|
|
||||||
},
|
|
||||||
streams: [
|
|
||||||
{ language: "eng" },
|
|
||||||
{ language: "jpn" }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
|
|
||||||
// Output (flattened for Redis)
|
|
||||||
title = "Inception"
|
|
||||||
video/codec = "h265"
|
|
||||||
video/width = "1920"
|
|
||||||
streams/0/language = "eng"
|
|
||||||
streams/1/language = "jpn"
|
|
||||||
```
|
|
||||||
|
|
||||||
**Key rules:**
|
|
||||||
- Nested objects use `/` as path separator
|
|
||||||
- Arrays use numeric indices (0, 1, 2...)
|
|
||||||
- All values stored as strings
|
|
||||||
- `null`/`undefined` stored as empty strings
|
|
||||||
- Numbers and booleans converted via `String()`
|
|
||||||
|
|
||||||
### Reconstruction
|
|
||||||
|
|
||||||
When reading, the flattened structure is reconstructed into nested objects:
|
|
||||||
|
|
||||||
```typescript
|
|
||||||
const metadata = await kvClient.getMetadataFlat(hashId);
|
|
||||||
// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... }
|
|
||||||
```
|
|
||||||
|
|
||||||
## Plugin Metadata Structure
|
|
||||||
|
|
||||||
Each plugin writes metadata using a prefixed key pattern:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
{plugin-id}/{field} # Simple field
|
flat nested
|
||||||
{plugin-id}/{group}/{field} # Grouped field
|
───────────────────────────────────── ───────────────────────────
|
||||||
{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays)
|
title = "Inception" title: "Inception"
|
||||||
|
fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5",
|
||||||
|
fileinfo/streamdetails/ streamdetails: { video: { 0: {
|
||||||
|
video/0/codec = "h264" codec: "h264", ... } }, ... } }
|
||||||
|
streams/0/language = "eng" streams: [ { language: "eng" }, ... ]
|
||||||
```
|
```
|
||||||
|
|
||||||
### Plugin Contribution Flow
|
Rules:
|
||||||
|
|
||||||
|
- `/` is the path separator.
|
||||||
|
- Numeric path segments (`0`, `1`, ...) become array indices on read.
|
||||||
|
- All values are strings on the wire; type interpretation is the
|
||||||
|
caller's job (and is documented per-field in `METADATA_KEYS.md`).
|
||||||
|
- `null`/`undefined` round-trip as empty strings.
|
||||||
|
|
||||||
|
## Plugin contribution flow
|
||||||
|
|
||||||
|
Plugins don't talk to Redis. They write metadata via meta-core's HTTP
|
||||||
|
API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin
|
||||||
|
writes under a prefix:
|
||||||
|
|
||||||
```
|
```
|
||||||
File Discovered
|
{plugin-id}/<field> single value
|
||||||
↓
|
{plugin-id}/<group>/<field> grouped
|
||||||
[FileProcessorPiscina]
|
{plugin-id}/<group>/<n>/<field> indexed (array)
|
||||||
├─ Compute midhash256
|
|
||||||
├─ Store preliminary metadata
|
|
||||||
└─ Dispatch plugin tasks
|
|
||||||
↓
|
|
||||||
[Plugin Processing]
|
|
||||||
├─ file-info → fileType, mimeType, sizeByte
|
|
||||||
├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/...
|
|
||||||
├─ filename-parser → titles/eng, season, episode, videoType
|
|
||||||
├─ anime-detector → anime, titles/jpn, titles/jpl
|
|
||||||
├─ tmdb → tmdbid, plot/eng, genres
|
|
||||||
└─ full-hash → cid_sha2-256, cid_sha1
|
|
||||||
↓
|
|
||||||
[Plugin Callbacks]
|
|
||||||
└─ Store metadata via flat keys (SET per property)
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## Core Metadata Types
|
See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full,
|
||||||
|
authoritative list of fields each plugin writes (content identification,
|
||||||
|
file stats, video classification, anime detection, subtitle references,
|
||||||
|
Jellyfin metadata, FFmpeg stream details, etc.) and the value formats
|
||||||
|
(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc
|
||||||
|
deliberately does not duplicate that registry — it would drift.
|
||||||
|
|
||||||
### HashMeta - Content Identification
|
## Event streams
|
||||||
|
|
||||||
```
|
Two Redis Streams are produced inside meta-core:
|
||||||
cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132)
|
|
||||||
cid_md5 = "bafkr..." # MD5 (multicodec 0xd5)
|
|
||||||
cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11)
|
|
||||||
cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12)
|
|
||||||
cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000)
|
|
||||||
```
|
|
||||||
|
|
||||||
### FileStatMeta - File Properties
|
| Stream | Producer | Contents |
|
||||||
|
|---|---|---|
|
||||||
|
| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. |
|
||||||
|
| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. |
|
||||||
|
|
||||||
```
|
External consumers do **not** read these streams directly. They subscribe
|
||||||
fileType = "video" # video, audio, image, text
|
via SSE:
|
||||||
mimeType = "video/x-matroska" # MIME type
|
|
||||||
sizeByte = "4294967296" # File size in bytes
|
|
||||||
```
|
|
||||||
|
|
||||||
### VideoMeta - Classification
|
- `GET /api/events/files` — mirror of `file:events`
|
||||||
|
- `GET /api/events/meta` — mirror of `meta:events`
|
||||||
|
|
||||||
```
|
The SSE wire contract uses `Last-Event-ID` for resume (the opaque
|
||||||
videoType = "movie" # movie, episode, unknown
|
`<ms>-<seq>` Redis Stream entry ID). Heartbeats are SSE comments
|
||||||
originalTitle = "Inception"
|
(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past
|
||||||
titles/eng = "Inception" # Localized titles
|
a client's cursor, the handler emits one `event: gap` payload before
|
||||||
titles/jpn = "インセプション"
|
resuming. See `internal/api/sse_events.go` and
|
||||||
season = "1"
|
[`api-mediated-access.md`](api-mediated-access.md) for the full
|
||||||
episode = "5"
|
contract.
|
||||||
movieYear = "2010"
|
|
||||||
```
|
|
||||||
|
|
||||||
### AnimeMeta - Anime Detection
|
Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK)
|
||||||
|
inside `MetaPublisher` to drain keyspace notifications, but this is not
|
||||||
```
|
the external surface — external services use SSE.
|
||||||
anime = "true" # "true" or "false"
|
|
||||||
titles/jpn = "進撃の巨人" # Japanese title
|
|
||||||
titles/jpl = "Shingeki no Kyojin" # Romaji
|
|
||||||
```
|
|
||||||
|
|
||||||
### SubtitleMeta - Subtitle References
|
|
||||||
|
|
||||||
```
|
|
||||||
subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID
|
|
||||||
subtitles/eng/1 = "bafks..."
|
|
||||||
subtitles/jpn/0 = "bafkt..."
|
|
||||||
```
|
|
||||||
|
|
||||||
### JellyfinMeta - Comprehensive Metadata
|
|
||||||
|
|
||||||
```
|
|
||||||
imdbid = "tt1375666"
|
|
||||||
tmdbid = "27205"
|
|
||||||
anidbid = "12345"
|
|
||||||
plot/eng = "A movie about..."
|
|
||||||
genre/0 = "Action"
|
|
||||||
genre/1 = "Sci-Fi"
|
|
||||||
rating = "8.8"
|
|
||||||
mpaa = "PG-13"
|
|
||||||
releasedate = "2010-07-16"
|
|
||||||
art/poster = "bafkr..." # Reference to poster image CID
|
|
||||||
art/fanart = "bafks..." # Reference to backdrop image CID
|
|
||||||
```
|
|
||||||
|
|
||||||
## FFmpeg Plugin - Special Case
|
|
||||||
|
|
||||||
The FFmpeg plugin extracts detailed stream information with a specific nested structure.
|
|
||||||
|
|
||||||
### Stream Details Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
fileinfo/duration = "7200.5"
|
|
||||||
fileinfo/formatName = "matroska,webm"
|
|
||||||
fileinfo/streamdetails/video/0/codec = "h264"
|
|
||||||
fileinfo/streamdetails/video/0/width = "1920"
|
|
||||||
fileinfo/streamdetails/video/0/height = "1080"
|
|
||||||
fileinfo/streamdetails/video/0/aspect = "16:9"
|
|
||||||
fileinfo/streamdetails/video/0/bitrate = "8000000"
|
|
||||||
fileinfo/streamdetails/audio/0/codec = "aac"
|
|
||||||
fileinfo/streamdetails/audio/0/language = "eng"
|
|
||||||
fileinfo/streamdetails/audio/0/channels = "6"
|
|
||||||
fileinfo/streamdetails/audio/1/codec = "dts"
|
|
||||||
fileinfo/streamdetails/audio/1/language = "fra"
|
|
||||||
fileinfo/streamdetails/audio/1/channels = "6"
|
|
||||||
fileinfo/streamdetails/subtitle/0/language = "eng"
|
|
||||||
fileinfo/streamdetails/subtitle/0/format = "subrip"
|
|
||||||
fileinfo/streamdetails/subtitle/1/language = "jpn"
|
|
||||||
fileinfo/streamdetails/subtitle/1/format = "ass"
|
|
||||||
fileinfo/streamdetails/embeddedimage/0/type = "cover"
|
|
||||||
```
|
|
||||||
|
|
||||||
### Stream Type Keys
|
|
||||||
|
|
||||||
| Stream Type | Key Pattern | Description |
|
|
||||||
|------------|-------------|-------------|
|
|
||||||
| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) |
|
|
||||||
| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks |
|
|
||||||
| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles |
|
|
||||||
| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails |
|
|
||||||
|
|
||||||
### Common Stream Fields
|
|
||||||
|
|
||||||
**Video streams:**
|
|
||||||
- `codec` - h264, hevc, av1, vp9, etc.
|
|
||||||
- `width`, `height` - Resolution
|
|
||||||
- `aspect` - Aspect ratio
|
|
||||||
- `bitrate` - Video bitrate
|
|
||||||
- `profile` - Codec profile (main, high, etc.)
|
|
||||||
- `level` - Codec level
|
|
||||||
|
|
||||||
**Audio streams:**
|
|
||||||
- `codec` - aac, dts, ac3, flac, etc.
|
|
||||||
- `language` - ISO 639-2 code (eng, jpn, fra)
|
|
||||||
- `channels` - Channel count (2, 6, 8)
|
|
||||||
- `bitrate` - Audio bitrate
|
|
||||||
- `samplerate` - Sample rate
|
|
||||||
|
|
||||||
**Subtitle streams:**
|
|
||||||
- `language` - ISO 639-2 code
|
|
||||||
- `format` - subrip, ass, pgs, vobsub
|
|
||||||
|
|
||||||
## Real-Time Notifications
|
|
||||||
|
|
||||||
MetaMesh provides multiple mechanisms for real-time metadata change notifications:
|
|
||||||
|
|
||||||
### 1. meta:events Stream (Recommended)
|
|
||||||
|
|
||||||
The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# File system events from meta-core watcher
|
|
||||||
XREAD STREAMS meta:events 0
|
|
||||||
|
|
||||||
# Event types: add, change, delete, rename, reset
|
|
||||||
```
|
|
||||||
|
|
||||||
Benefits:
|
|
||||||
- **Reliable delivery** via consumer groups and acknowledgment
|
|
||||||
- **Replay capability** for catching up after restarts
|
|
||||||
- **Ordered processing** with stream IDs
|
|
||||||
|
|
||||||
### 2. Keyspace Notifications
|
|
||||||
|
|
||||||
For field-level change detection, subscribe to Redis keyspace notifications:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# All file metadata changes
|
|
||||||
PSUBSCRIBE '__keyspace@0__:file:*'
|
|
||||||
|
|
||||||
# Specific plugin updates (e.g., TMDB enrichment)
|
|
||||||
PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*'
|
|
||||||
```
|
|
||||||
|
|
||||||
### Cache Invalidation
|
|
||||||
|
|
||||||
The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation:
|
|
||||||
|
|
||||||
```
|
|
||||||
┌─────────────────────────────────────────────────────────────────┐
|
|
||||||
│ Cache Invalidation Flow │
|
|
||||||
│ │
|
|
||||||
│ Redis Keyspace Cache │
|
|
||||||
│ ┌──────────────┐ Invalidator │
|
|
||||||
│ │ SET file:*/ │ ──────────► ┌──────────────┐ │
|
|
||||||
│ │ DEL file:*/ │ subscribe │ Listens to │ │
|
|
||||||
│ └──────────────┘ │ __keyspace@ │ │
|
|
||||||
│ │ 0__:file:* │ │
|
|
||||||
│ └──────┬───────┘ │
|
|
||||||
│ │ │
|
|
||||||
│ ▼ │
|
|
||||||
│ ┌──────────────┐ │
|
|
||||||
│ │ Remove from │ │
|
|
||||||
│ │ LRU cache │ │
|
|
||||||
│ └──────────────┘ │
|
|
||||||
└─────────────────────────────────────────────────────────────────┘
|
|
||||||
```
|
|
||||||
|
|
||||||
Flow:
|
|
||||||
1. Cache invalidator subscribes to `__keyspace@0__:file:*`
|
|
||||||
2. When metadata changes (SET/DEL), invalidator receives notification
|
|
||||||
3. Invalidator removes the corresponding file from the WebDAV cache
|
|
||||||
4. Next access fetches fresh copy from source filesystem
|
|
||||||
|
|
||||||
This ensures cached files stay synchronized with metadata changes.
|
|
||||||
|
|
||||||
### 3. Pub/Sub Channels (Legacy)
|
|
||||||
|
|
||||||
Batch notification channels are still supported:
|
|
||||||
|
|
||||||
```
|
|
||||||
meta-sort:file:batch # Batch file updates (every 5 seconds)
|
|
||||||
meta-sort:scan:reset # Full rescan notification
|
|
||||||
```
|
|
||||||
|
|
||||||
```json
|
|
||||||
{
|
|
||||||
"timestamp": 1704067200000,
|
|
||||||
"changes": [
|
|
||||||
{ "action": "add", "hashId": "bafkr4ih5..." },
|
|
||||||
{ "action": "update", "hashId": "bafkr4ih5..." },
|
|
||||||
{ "action": "remove", "hashId": "bafkr4ih5..." }
|
|
||||||
]
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Complete Example
|
|
||||||
|
|
||||||
Here's a complete metadata example for a movie file:
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# Query all keys for a file
|
|
||||||
redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*'
|
|
||||||
|
|
||||||
# Example flat keys stored in Redis:
|
|
||||||
# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..."
|
|
||||||
# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv"
|
|
||||||
# file:bafkr4ih5kapbjzq.../fileType = "video"
|
|
||||||
# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5"
|
|
||||||
# file:bafkr4ih5kapbjzq.../titles/eng = "Inception"
|
|
||||||
# ...
|
|
||||||
|
|
||||||
# When reconstructed by the API (nested object representation)
|
|
||||||
{
|
|
||||||
# Content identification
|
|
||||||
"cid_midhash256": "bafkr4ih5kapbjzq...",
|
|
||||||
"cid_sha2-256": "bafks7ij2kbpck...",
|
|
||||||
"filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv",
|
|
||||||
|
|
||||||
# File properties
|
|
||||||
"fileType": "video",
|
|
||||||
"mimeType": "video/x-matroska",
|
|
||||||
"sizeByte": "4294967296",
|
|
||||||
|
|
||||||
# Classification
|
|
||||||
"videoType": "movie",
|
|
||||||
"originalTitle": "Inception",
|
|
||||||
"movieYear": "2010",
|
|
||||||
|
|
||||||
# Localized titles
|
|
||||||
"titles": {
|
|
||||||
"eng": "Inception",
|
|
||||||
"fra": "Inception",
|
|
||||||
"jpn": "インセプション"
|
|
||||||
},
|
|
||||||
|
|
||||||
# FFmpeg stream details
|
|
||||||
"fileinfo": {
|
|
||||||
"duration": "8878.5",
|
|
||||||
"formatName": "matroska,webm",
|
|
||||||
"streamdetails": {
|
|
||||||
"video": {
|
|
||||||
"0": {
|
|
||||||
"codec": "h264",
|
|
||||||
"width": "1920",
|
|
||||||
"height": "1080",
|
|
||||||
"aspect": "16:9",
|
|
||||||
"bitrate": "8000000"
|
|
||||||
}
|
|
||||||
},
|
|
||||||
"audio": {
|
|
||||||
"0": { "codec": "dts", "language": "eng", "channels": "6" },
|
|
||||||
"1": { "codec": "ac3", "language": "fra", "channels": "6" }
|
|
||||||
},
|
|
||||||
"subtitle": {
|
|
||||||
"0": { "language": "eng", "format": "subrip" },
|
|
||||||
"1": { "language": "fra", "format": "subrip" }
|
|
||||||
}
|
|
||||||
}
|
|
||||||
},
|
|
||||||
|
|
||||||
# External metadata (TMDB)
|
|
||||||
"tmdbid": "27205",
|
|
||||||
"imdbid": "tt1375666",
|
|
||||||
"plot": {
|
|
||||||
"eng": "A thief who steals corporate secrets through dream-sharing technology..."
|
|
||||||
},
|
|
||||||
"genre": ["Action", "Science Fiction", "Adventure"],
|
|
||||||
"rating": "8.4",
|
|
||||||
"releasedate": "2010-07-16",
|
|
||||||
|
|
||||||
# Media assets
|
|
||||||
"art": {
|
|
||||||
"poster": "bafkr4poster...",
|
|
||||||
"fanart": "bafkr4fanart..."
|
|
||||||
},
|
|
||||||
|
|
||||||
# Detection flags
|
|
||||||
"anime": "false"
|
|
||||||
}
|
|
||||||
```
|
|
||||||
|
|
||||||
## Key Implementation Files
|
|
||||||
|
|
||||||
| Component | File Path |
|
|
||||||
|-----------|-----------|
|
|
||||||
| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` |
|
|
||||||
| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` |
|
|
||||||
| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` |
|
|
||||||
| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` |
|
|
||||||
| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` |
|
|
||||||
| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` |
|
|
||||||
| meta-core Storage | `packages/meta-core/internal/storage/client.go` |
|
|
||||||
| meta-core Events | `packages/meta-core/internal/events/publisher.go` |
|
|
||||||
| meta-core Cache | `packages/meta-core/internal/cache/` |
|
|
||||||
|
|
||||||
## Debugging
|
## Debugging
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
# List all file hash IDs
|
# Roots
|
||||||
docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__'
|
docker exec metacore-app redis-cli SMEMBERS 'file:__index__'
|
||||||
|
docker exec metacore-app redis-cli SCARD 'file:__index__'
|
||||||
|
docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__'
|
||||||
|
|
||||||
# Count total files
|
# Full document for one root (flat scan)
|
||||||
docker exec meta-core-dev redis-cli SCARD 'file:__index__'
|
docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*'
|
||||||
|
|
||||||
# Get a sample hash ID
|
# One property
|
||||||
docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__'
|
docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration'
|
||||||
|
|
||||||
# List all keys for a specific file (flat key scan)
|
# Confirm keys are STRING (not HASH)
|
||||||
docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*'
|
docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath'
|
||||||
|
# string
|
||||||
|
|
||||||
# Get specific field (flat key)
|
# Reverse-index lookups
|
||||||
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration'
|
docker exec metacore-app redis-cli GET 'cid:midhash256:bafkreih...' # → uuid
|
||||||
|
docker exec metacore-app redis-cli SMEMBERS 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids'
|
||||||
|
docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid'
|
||||||
|
|
||||||
# Get filePath for a file
|
# Schema-version sentinel
|
||||||
docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath'
|
docker exec metacore-app redis-cli GET 'meta-core:schema-version'
|
||||||
|
|
||||||
# Verify keys are STRING type (not HASH)
|
# Streams (server-side; external consumers should use SSE)
|
||||||
docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath'
|
docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10
|
||||||
# Should return: string
|
docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10
|
||||||
|
|
||||||
# Monitor metadata changes via stream
|
# HTTP equivalents — preferred
|
||||||
docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0
|
curl -k 'https://metacore-dev.localhost:8083/api/meta/<cid>' # CID → document
|
||||||
|
curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs
|
||||||
# Monitor keyspace notifications (requires notify-keyspace-events enabled)
|
curl -k 'https://metacore-dev.localhost:8083/meta/<uuid>' # document by UUID
|
||||||
docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*'
|
curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE
|
||||||
|
curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats
|
||||||
# meta-core API debugging
|
|
||||||
# Cache status
|
|
||||||
curl http://localhost:9000/api/cache/status
|
|
||||||
|
|
||||||
# Clear cache
|
|
||||||
curl -X POST http://localhost:9000/api/cache/clear
|
|
||||||
|
|
||||||
# Watch events stream (SSE)
|
|
||||||
curl http://localhost:9000/api/events/stream
|
|
||||||
|
|
||||||
# KV browser - storage info
|
|
||||||
curl http://localhost:9000/api/kv/info
|
|
||||||
|
|
||||||
# Scan status
|
|
||||||
curl http://localhost:9000/api/scan/status
|
|
||||||
```
|
```
|
||||||
|
|
||||||
## meta:events Stream
|
## Key implementation files
|
||||||
|
|
||||||
The `meta:events` stream captures metadata change events for real-time notifications:
|
| Component | Path |
|
||||||
|
|---|---|
|
||||||
```bash
|
| Redis client + flat-key ops | `internal/storage/client.go` |
|
||||||
# View recent events
|
| UUIDv7 minting | `internal/storage/uuid.go` |
|
||||||
docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10
|
| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` |
|
||||||
|
| Schema-version gate | `internal/storage/schema_version.go` |
|
||||||
# Read events from a consumer group
|
| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` |
|
||||||
docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' >
|
| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` |
|
||||||
```
|
| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` |
|
||||||
|
| SSE event mirror | `internal/api/sse_events.go` |
|
||||||
Stream events contain:
|
| Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) |
|
||||||
- `type`: Event type (add, change, delete, rename, reset)
|
|
||||||
- `path`: File path
|
|
||||||
- `midhash256`: Content hash (for add/change)
|
|
||||||
- `timestamp`: Event timestamp
|
|
||||||
|
|||||||
@@ -0,0 +1,322 @@
|
|||||||
|
# UUID-Rooted Metadata with CID Aliasing
|
||||||
|
|
||||||
|
## Status
|
||||||
|
|
||||||
|
**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale.
|
||||||
|
|
||||||
|
## Motivation
|
||||||
|
|
||||||
|
Today, every metadata entry in Redis is keyed by a content hash, specifically
|
||||||
|
midhash256:
|
||||||
|
|
||||||
|
```
|
||||||
|
file:midhash256:abc.../filePath
|
||||||
|
file:midhash256:abc.../title
|
||||||
|
file:midhash256:abc.../fileinfo/duration
|
||||||
|
file:midhash256:abc.../tmdb/poster
|
||||||
|
...
|
||||||
|
```
|
||||||
|
|
||||||
|
Other digests (sha256, IPFS CIDs, BitTorrent infohashes) live as fields
|
||||||
|
*inside* that entry:
|
||||||
|
|
||||||
|
```
|
||||||
|
file:midhash256:abc.../cid_sha256 = sha256:def...
|
||||||
|
file:midhash256:abc.../cid_ipfs = bafy...
|
||||||
|
```
|
||||||
|
|
||||||
|
Two problems with this:
|
||||||
|
|
||||||
|
1. **midhash256 is privileged by accident.** It IS the key, while every other
|
||||||
|
CID is a secondary field. There's nothing intrinsically special about
|
||||||
|
midhash256 — it's just the first hash computed. Hash-algorithm changes, or
|
||||||
|
new digest types added by plugins, force rewrites of every root. The
|
||||||
|
"promotion" workarounds people reach for (re-keying entries when a stronger
|
||||||
|
hash arrives) solve the wrong problem; the privilege itself is the issue.
|
||||||
|
|
||||||
|
2. **CID lookup is O(n).** Resolving "given any CID, return that file's
|
||||||
|
metadata" currently scans every indexed entry (see
|
||||||
|
`storage/client.go::LookupPathByCID`, which acknowledges this in a
|
||||||
|
comment). Acceptable for admin tooling, unacceptable for a public API
|
||||||
|
surface — especially when peers on meta-share may query by any CID variant.
|
||||||
|
|
||||||
|
The product requirement is clear: **any CID a client knows about must resolve
|
||||||
|
to that file's metadata in O(1).** IPFS-compatible CIDs are the externally
|
||||||
|
meaningful identity for content retrieval; the internal storage layout should
|
||||||
|
not bias toward any one digest type.
|
||||||
|
|
||||||
|
## Goal
|
||||||
|
|
||||||
|
A single externally-meaningful contract:
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/meta/<cid> → full metadata document for that file
|
||||||
|
```
|
||||||
|
|
||||||
|
Where `<cid>` may be any supported type (midhash256, sha256, IPFS, btih,
|
||||||
|
sha3, future additions). All CID types are treated symmetrically. No CID type
|
||||||
|
is "the key."
|
||||||
|
|
||||||
|
## Design
|
||||||
|
|
||||||
|
### Two-tier key structure
|
||||||
|
|
||||||
|
The root key becomes an opaque internal UUID; CIDs are aliases that point to
|
||||||
|
it.
|
||||||
|
|
||||||
|
```
|
||||||
|
file:<uuid>/<property...> ← canonical storage (UUIDv7, never disclosed externally)
|
||||||
|
cid:<cid> → <uuid> ← reverse index (one Redis key per known CID)
|
||||||
|
```
|
||||||
|
|
||||||
|
The metadata keyspace under `file:<uuid>/` remains open and flat, exactly as
|
||||||
|
today — anything that wrote `file:<midhash>/fileinfo/duration` now writes
|
||||||
|
`file:<uuid>/fileinfo/duration`. Only the root identifier changes.
|
||||||
|
|
||||||
|
**Example layout:**
|
||||||
|
|
||||||
|
```
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = /watch/Inception.mkv
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/size = 4500000000
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtime = 1729872000000000000
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids = SET{midhash256:abc, sha256:def, ipfs:bafy...}
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/duplicates = SET{/watch/dup1.mkv, /watch/dup2.mkv}
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid = ipfs:bafy...
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception"
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = 7200.5
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = bafy...
|
||||||
|
file:01JKR8XW5T4QMV7AHNJ2DEFGHK/subtitle/eng = ...
|
||||||
|
|
||||||
|
cid:midhash256:abc → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
cid:sha256:def → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
cid:ipfs:bafy... → 01JKR8XW5T4QMV7AHNJ2DEFGHK
|
||||||
|
```
|
||||||
|
|
||||||
|
### UUID format: UUIDv7
|
||||||
|
|
||||||
|
UUIDv7 chosen over UUIDv4 or 64-bit random:
|
||||||
|
|
||||||
|
- **128 bits**: collision-free under distributed minting; no central allocator.
|
||||||
|
- **Time-sortable prefix**: `KEYS file:01JKR8*` retrieves entries minted in a
|
||||||
|
given window. Restores some of the debuggability lost by going opaque.
|
||||||
|
- **Monotonic-ish**: adjacent UUIDs cluster on disk; modestly better SCAN
|
||||||
|
locality.
|
||||||
|
|
||||||
|
Encoded as a 26-character Crockford Base32 string (ULID-compatible) for
|
||||||
|
compactness and case-insensitive matching.
|
||||||
|
|
||||||
|
### New root fields owned by meta-core
|
||||||
|
|
||||||
|
The metadata schema is open — any writer can put any key under
|
||||||
|
`file:<uuid>/`. This proposal does not change that. It adds three keys whose
|
||||||
|
sole writer is meta-core itself:
|
||||||
|
|
||||||
|
| Key | Type | Purpose |
|
||||||
|
|-----|------|---------|
|
||||||
|
| `cids` | Redis SET | All CIDs known for this file. Mirror of the `cid:*` reverse index. Source of truth for delete-time GC. |
|
||||||
|
| `duplicates` | Redis SET | Additional file paths whose content matches this entry. One UUID = one set of metadata; multiple physical files may share that content. Consumed by meta-dup. |
|
||||||
|
| `canonical_cid` | STRING | The CID this entry should be presented as externally (meta-share announce, UI display). Selected by the rank function (below). |
|
||||||
|
|
||||||
|
Existing meta-core-owned fields (`filePath`, `size`, `mtime`, `midhash256`)
|
||||||
|
keep their names and writers. Plugin-written fields (`tmdb/*`, `fileinfo/*`,
|
||||||
|
`subtitle/<lang>`, …) are unaffected.
|
||||||
|
|
||||||
|
Redis SET is a departure from the all-strings convention in the current
|
||||||
|
storage doc, but `file:__index__` is already a SET — so the precedent exists
|
||||||
|
for using sets where they're the right structure.
|
||||||
|
|
||||||
|
### Canonical CID selection
|
||||||
|
|
||||||
|
`canonical_cid` is a label, not a key. Updating it costs one write — no
|
||||||
|
rename, no race, no broken external references.
|
||||||
|
|
||||||
|
Selection rule, highest-wins:
|
||||||
|
|
||||||
|
1. IPFS CID (any variant)
|
||||||
|
2. SHA-256 / SHA-3 family
|
||||||
|
3. BitTorrent infohash (v2 preferred over v1)
|
||||||
|
4. midhash256
|
||||||
|
|
||||||
|
Ranked by **utility for external retrieval**, not by cryptographic strength.
|
||||||
|
midhash256 and sha256 are both 256-bit, but only sha256/IPFS CIDs are
|
||||||
|
externally meaningful for content addressing. btih ranks above midhash because
|
||||||
|
it enables swarm interop even if it's weaker as a digest.
|
||||||
|
|
||||||
|
Reconciler runs on the leader, recomputes `canonical_cid` whenever
|
||||||
|
`AddAlias` registers a new CID for an entry.
|
||||||
|
|
||||||
|
## Internal operations
|
||||||
|
|
||||||
|
The meta-core storage interface adds five operations. None are exposed
|
||||||
|
externally — they're internal to meta-core. External callers see only the
|
||||||
|
CID-based API.
|
||||||
|
|
||||||
|
```go
|
||||||
|
// Mint creates a new root with no aliases. Called when the watcher
|
||||||
|
// computes a midhash for a previously-unseen file. Returns the new UUID.
|
||||||
|
Mint(filePath string, size int64, mtime int64) (uuid string, err error)
|
||||||
|
|
||||||
|
// AddAlias registers a CID as resolving to this UUID. Updates both the
|
||||||
|
// cid:<cid> reverse index and the file:<uuid>/cids set atomically.
|
||||||
|
// Triggers canonical_cid reconciliation.
|
||||||
|
AddAlias(uuid string, cid string) error
|
||||||
|
|
||||||
|
// GetByCID resolves any known CID to its UUID. Returns ("", nil) if unknown.
|
||||||
|
GetByCID(cid string) (uuid string, err error)
|
||||||
|
|
||||||
|
// MergeIfAliasExists is called when the watcher mints a fresh UUID for a
|
||||||
|
// file path but, on hashing, discovers the resulting CID already belongs
|
||||||
|
// to a different UUID. Merges the new (mostly-empty) UUID into the
|
||||||
|
// existing one, transferring filePath into the duplicates set, and
|
||||||
|
// returns the canonical UUID.
|
||||||
|
MergeIfAliasExists(newUUID, cid string) (canonicalUUID string, merged bool, err error)
|
||||||
|
|
||||||
|
// DeleteRoot removes the entire entry, including every cid:* alias and
|
||||||
|
// every file:<uuid>/* key. Used for file deletion events.
|
||||||
|
DeleteRoot(uuid string) error
|
||||||
|
```
|
||||||
|
|
||||||
|
### Lifecycle: file arrival
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Watcher sees /watch/foo.mkv created.
|
||||||
|
2. Watcher computes midhash256 → "midhash256:abc...".
|
||||||
|
3. Watcher calls GetByCID("midhash256:abc...").
|
||||||
|
→ uuid_X (existing entry) or "" (new file).
|
||||||
|
|
||||||
|
4a. If existing (uuid_X):
|
||||||
|
- Add "/watch/foo.mkv" to file:<uuid_X>/duplicates.
|
||||||
|
- No new root created.
|
||||||
|
- meta-dup is notified via the existing keyspace event channel.
|
||||||
|
|
||||||
|
4b. If new:
|
||||||
|
- Mint("/watch/foo.mkv", size, mtime) → uuid_Y.
|
||||||
|
- AddAlias(uuid_Y, "midhash256:abc...").
|
||||||
|
- Plugins now process the file, referenced by midhash256:abc...
|
||||||
|
(UUID is never disclosed to them.)
|
||||||
|
|
||||||
|
5. Later, the full-hash plugin computes sha256.
|
||||||
|
- Plugin reports back: "for CID midhash256:abc..., sha256 is sha256:def...".
|
||||||
|
- meta-core resolves midhash256:abc → uuid_Y via GetByCID.
|
||||||
|
- meta-core calls AddAlias(uuid_Y, "sha256:def...").
|
||||||
|
- Reconciler updates canonical_cid if sha256 outranks midhash256.
|
||||||
|
```
|
||||||
|
|
||||||
|
### Lifecycle: file deletion
|
||||||
|
|
||||||
|
```
|
||||||
|
1. Watcher sees /watch/foo.mkv deleted.
|
||||||
|
2. Resolve which entry owns this path:
|
||||||
|
a. If /watch/foo.mkv is in some entry's duplicates set:
|
||||||
|
- Remove from set. Done.
|
||||||
|
b. If /watch/foo.mkv is the filePath of some entry:
|
||||||
|
- If duplicates is non-empty: promote one duplicate to filePath.
|
||||||
|
- If duplicates is empty: DeleteRoot(uuid).
|
||||||
|
- Iterates the cids set, deletes each cid:<cid> reverse-index key.
|
||||||
|
- Deletes every file:<uuid>/* key.
|
||||||
|
- Removes uuid from file:__index__.
|
||||||
|
```
|
||||||
|
|
||||||
|
## Public API
|
||||||
|
|
||||||
|
```
|
||||||
|
GET /api/meta/<cid>
|
||||||
|
→ 200 { "filePath": "...", "canonical_cid": "...",
|
||||||
|
"cids": [...], "title": "...", "tmdb": {...}, ... }
|
||||||
|
→ 404 if no such CID
|
||||||
|
|
||||||
|
GET /api/file/<cid> (already exists — implementation now O(1))
|
||||||
|
→ streams file content
|
||||||
|
```
|
||||||
|
|
||||||
|
The UUID is **not** included in the response. Clients address everything by
|
||||||
|
CID; meta-core's choice of internal identifier remains a private
|
||||||
|
implementation detail and may change.
|
||||||
|
|
||||||
|
Auth: `/api/meta/<cid>` follows the same pattern as `/api/file/<cid>` and
|
||||||
|
bypasses the auth perimeter. It's the metadata twin of the content endpoint,
|
||||||
|
and meta-share calls it from federated search responses.
|
||||||
|
|
||||||
|
## Cross-service impact
|
||||||
|
|
||||||
|
This is a meta-core schema change. Other services interact only through CIDs,
|
||||||
|
so the impact is contained but real.
|
||||||
|
|
||||||
|
- **meta-sort**: writers must resolve plugin callbacks to a UUID before
|
||||||
|
writing. The existing `LeaderClient` gains a `getByCID` cache.
|
||||||
|
- **meta-fuse**: VFS rebuild already iterates `file:__index__`; only the
|
||||||
|
ID format changes.
|
||||||
|
- **meta-stremio**: queries by CID; needs no logic change beyond its storage
|
||||||
|
client.
|
||||||
|
- **meta-dup**: gains a cleaner data source — duplicates are now explicit in
|
||||||
|
the `duplicates` set, not implicit in colliding writes.
|
||||||
|
- **meta-share**: announces `canonical_cid` rather than midhash. Federated
|
||||||
|
search responses include all CIDs from the `cids` set so any peer querying
|
||||||
|
by any digest gets a hit.
|
||||||
|
- **Plugins**: no change. They speak CIDs only.
|
||||||
|
|
||||||
|
## Migration
|
||||||
|
|
||||||
|
One-time sweep, idempotent, runs on first boot after upgrade:
|
||||||
|
|
||||||
|
```
|
||||||
|
For each hashID in file:__index__ (current shape: midhash256:abc, sha256:def, …):
|
||||||
|
1. Mint a fresh uuid_Y.
|
||||||
|
2. AddAlias(uuid_Y, hashID). The existing primary becomes an alias.
|
||||||
|
3. For each cid_* field on the old entry, AddAlias(uuid_Y, that value).
|
||||||
|
4. RENAME file:<hashID>/<property> → file:<uuid_Y>/<property> for every
|
||||||
|
property (Lua script, one entry at a time; non-atomic across entries,
|
||||||
|
but each entry stays internally consistent).
|
||||||
|
5. Reconcile canonical_cid.
|
||||||
|
6. Remove hashID from file:__index__, add uuid_Y.
|
||||||
|
```
|
||||||
|
|
||||||
|
Crash-recoverable: re-running the sweep finds entries with both old and new
|
||||||
|
roots and resumes by detecting which step completed (presence of
|
||||||
|
`file:<uuid>/cids` indicates step 3 finished; absence of any
|
||||||
|
`file:<hashID>/*` keys indicates step 4 finished).
|
||||||
|
|
||||||
|
## Decisions recorded
|
||||||
|
|
||||||
|
| Question | Decision | Notes |
|
||||||
|
|---|---|---|
|
||||||
|
| When is the UUID minted? | At first-hash | Pragmatic for the initial migration. Long-term target: at discovery, fully decoupled from hashing. Tracked as an open question below. |
|
||||||
|
| UUID format | UUIDv7, 26-char Crockford Base32 | Time-sortable, 128-bit, distributed-safe. |
|
||||||
|
| UUID visible externally? | No | Pure meta-core internal. Clients address by CID only. |
|
||||||
|
| Merge semantics | One UUID per content; additional paths into `duplicates` | One file (logical) = one metadata set. Multiple physical paths supported via `duplicates`. |
|
||||||
|
| Canonical CID rank | IPFS > sha-family > btih > midhash | Utility-weighted, not strength-weighted. |
|
||||||
|
| `cids` and `duplicates` field type | Redis SET | Departure from "all-strings" convention but consistent with the existing `file:__index__` precedent. |
|
||||||
|
| Reverse-index format | Flat `cid:<cid>` STRING keys | Simple, scan-friendly, no hot-key contention. |
|
||||||
|
|
||||||
|
## Open questions
|
||||||
|
|
||||||
|
- **Discovery-time UUID minting.** The cleaner long-term model: watcher mints
|
||||||
|
a UUID at discovery (before hashing), then `AddAlias` runs when the midhash
|
||||||
|
arrives. Requires an explicit "reconcile if alias already exists" step
|
||||||
|
because two paths with the same content would each get their own UUID until
|
||||||
|
hashing reveals the collision. Worth doing eventually; not required for
|
||||||
|
this iteration.
|
||||||
|
|
||||||
|
- **meta-share announce strategy.** Announce only `canonical_cid`, or every
|
||||||
|
entry in `cids`? Maximum discoverability vs. DHT footprint. Decide as part
|
||||||
|
of the meta-share work, not here.
|
||||||
|
|
||||||
|
- **IPFS gateway endpoint.** Should meta-core expose `/ipfs/<cid>` as a
|
||||||
|
content-streaming endpoint for external IPFS clients? Adjacent to this
|
||||||
|
design but not part of it.
|
||||||
|
|
||||||
|
- **Keyspace-notification consumers.** Today's parsers (`schema/indexer.go`)
|
||||||
|
handle `file:<type>:<hash>/...` with a special split rule for the colon
|
||||||
|
inside `midhash256:abc`. With UUIDs (no colons), `parseFieldPath` simplifies
|
||||||
|
— but any external consumer subscribed by hashID pattern needs updating.
|
||||||
|
Inventory those before flipping the migration.
|
||||||
|
|
||||||
|
## Non-goals
|
||||||
|
|
||||||
|
- **Not** a content-replication or IPFS-pinning system. Registering an IPFS
|
||||||
|
CID alias does not pin the file anywhere — it only states "if somebody
|
||||||
|
hashes this file with IPFS's chunker, this is the CID they'd get."
|
||||||
|
- **Not** a versioning scheme. Metadata is a snapshot of current state;
|
||||||
|
historical versions are out of scope.
|
||||||
|
- **Not** a strong-consistency system. Reverse-index writes happen under the
|
||||||
|
leader lock; reads from followers may briefly miss a freshly-added alias.
|
||||||
+57
-121
@@ -1,140 +1,76 @@
|
|||||||
# @metazla/meta-editor
|
# @meta-core/editor
|
||||||
|
|
||||||
User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface.
|
React + Vite metadata browser/editor that ships inside the meta-core
|
||||||
|
container. Walks the UUID-rooted Redis keyspace, edits leaf values,
|
||||||
|
imports/exports snapshots, and consults the live schema indexer.
|
||||||
|
|
||||||
## Features
|
## What it talks to
|
||||||
|
|
||||||
- **Search**: Find files by title, filename, or hash ID
|
The editor is a pure SPA — it has no server of its own. It calls the
|
||||||
- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more
|
meta-core HTTP API directly under three prefixes (see `src/api/`):
|
||||||
- **Batch Operations**: Update multiple files simultaneously
|
|
||||||
- **Real-time**: Direct integration with etcd via REST API
|
|
||||||
- **User-friendly**: Clean, intuitive interface with form validation
|
|
||||||
|
|
||||||
## Development
|
- `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`,
|
||||||
|
`value`, `key/{key...}`). This is the UUID-exposing surface — keys look
|
||||||
|
like `file:<uuid>/<property>` and `cid:<algo>:<value>`.
|
||||||
|
- `/api/schema` — live per-field schema (type hints, value breakdowns),
|
||||||
|
produced by meta-core's schema indexer that consumes `meta:events`.
|
||||||
|
- `/api/snapshot/*` — snapshot export / import / wipe.
|
||||||
|
|
||||||
### Prerequisites
|
In production the editor is served by meta-core's nginx (the Vite build
|
||||||
|
output is copied into the image at the same path) under
|
||||||
|
`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to
|
||||||
|
`/editor/` accordingly.
|
||||||
|
|
||||||
- Node.js 21.6.2+
|
## Project layout
|
||||||
- pnpm 10.19.0+
|
|
||||||
- Meta-mesh API server running (provides the `/api/metadata/*` endpoints)
|
|
||||||
|
|
||||||
### Running Locally
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# From the root directory
|
|
||||||
pnpm run start:editor
|
|
||||||
|
|
||||||
# Or from the package directory
|
|
||||||
cd packages/meta-editor
|
|
||||||
pnpm run dev
|
|
||||||
```
|
|
||||||
|
|
||||||
The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`.
|
|
||||||
|
|
||||||
### Building
|
|
||||||
|
|
||||||
```bash
|
|
||||||
# From the root directory
|
|
||||||
pnpm run build:editor
|
|
||||||
|
|
||||||
# Or from the package directory
|
|
||||||
cd packages/meta-editor
|
|
||||||
pnpm run build
|
|
||||||
```
|
|
||||||
|
|
||||||
The built files will be in `dist/` directory.
|
|
||||||
|
|
||||||
## Docker Deployment
|
|
||||||
|
|
||||||
In the Docker development environment, the editor is automatically available at:
|
|
||||||
|
|
||||||
```
|
```
|
||||||
http://localhost/editor
|
packages/meta-core/editor/
|
||||||
```
|
|
||||||
|
|
||||||
The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`.
|
|
||||||
|
|
||||||
## API Integration
|
|
||||||
|
|
||||||
The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API:
|
|
||||||
|
|
||||||
- `POST /api/metadata/search` - Search for files
|
|
||||||
- `GET /api/metadata/:hashId` - Get file metadata
|
|
||||||
- `PUT /api/metadata/:hashId` - Update complete metadata
|
|
||||||
- `PUT /api/metadata/:hashId/property` - Update specific property
|
|
||||||
- `POST /api/metadata/batch` - Batch update multiple files
|
|
||||||
|
|
||||||
See `src/api/metadataApi.ts` for the API client implementation.
|
|
||||||
|
|
||||||
## Usage
|
|
||||||
|
|
||||||
### Searching for Files
|
|
||||||
|
|
||||||
1. Select search type: "Search by Title/Filename" or "Search by Hash ID"
|
|
||||||
2. Enter your search query
|
|
||||||
3. Click "Search"
|
|
||||||
4. Results will appear in the left sidebar
|
|
||||||
|
|
||||||
### Editing Metadata
|
|
||||||
|
|
||||||
1. Select a file from the search results
|
|
||||||
2. Modify any fields in the editor form
|
|
||||||
3. Click "Save Changes" to persist updates
|
|
||||||
4. Use "Reset" to discard changes
|
|
||||||
|
|
||||||
### Supported Metadata Fields
|
|
||||||
|
|
||||||
**Basic Information:**
|
|
||||||
- Title, Original Title, Show Title
|
|
||||||
- Video Type (Movie, TV, Anime, Other)
|
|
||||||
- Year, Season, Episode
|
|
||||||
|
|
||||||
**Jellyfin Metadata:**
|
|
||||||
- Plot, IMDb ID, TMDb ID, AniDB ID
|
|
||||||
- Director, Studio, Rating, Runtime
|
|
||||||
- Genres, Premiered Date, Release Date
|
|
||||||
|
|
||||||
**File Information:**
|
|
||||||
- Filename, File Size, MIME Type
|
|
||||||
- Hash values (SHA-256, MD5, etc.) - read-only
|
|
||||||
|
|
||||||
## Project Structure
|
|
||||||
|
|
||||||
```
|
|
||||||
packages/meta-editor/
|
|
||||||
├── src/
|
|
||||||
│ ├── api/ # API client for metadata operations
|
|
||||||
│ ├── components/ # React components
|
|
||||||
│ │ ├── SearchBar.tsx # Search interface
|
|
||||||
│ │ ├── FileList.tsx # Search results list
|
|
||||||
│ │ └── MetadataEditor.tsx # Main editing form
|
|
||||||
│ ├── types/ # TypeScript type definitions
|
|
||||||
│ ├── utils/ # Utility functions
|
|
||||||
│ ├── App.tsx # Main application component
|
|
||||||
│ ├── main.tsx # Entry point
|
|
||||||
│ └── index.css # Global styles
|
|
||||||
├── index.html
|
├── index.html
|
||||||
├── vite.config.ts
|
├── vite.config.ts
|
||||||
├── tsconfig.json
|
├── tsconfig.json
|
||||||
└── package.json
|
├── package.json @meta-core/editor
|
||||||
|
└── src/
|
||||||
|
├── main.tsx entry
|
||||||
|
├── api/
|
||||||
|
│ ├── kvApi.ts /api/kv client
|
||||||
|
│ ├── schemaApi.ts /api/schema client + cache
|
||||||
|
│ └── snapshotApi.ts /api/snapshot client
|
||||||
|
├── kv/ KV browser + leaf/branch editors
|
||||||
|
├── components/ shared widgets (language combobox, snapshot panel, ...)
|
||||||
|
└── index.css
|
||||||
```
|
```
|
||||||
|
|
||||||
## Technology Stack
|
## Local development
|
||||||
|
|
||||||
- **React 18** - UI framework
|
Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`.
|
||||||
- **TypeScript 5.7** - Type safety
|
|
||||||
- **Vite 6** - Build tool and dev server
|
|
||||||
- **Native CSS** - Styling (no CSS framework)
|
|
||||||
|
|
||||||
## Contributing
|
```bash
|
||||||
|
cd packages/meta-core/editor
|
||||||
|
pnpm install
|
||||||
|
pnpm dev # vite dev server on http://localhost:5173
|
||||||
|
```
|
||||||
|
|
||||||
When adding new features:
|
The Vite dev server proxies `/api/*` to `http://localhost:3000` (see
|
||||||
|
`vite.config.ts`). For local development against a running meta-core
|
||||||
|
container, change that target to wherever meta-core is reachable on your
|
||||||
|
host — e.g. `http://localhost:18083` for the dev compose's direct-backend
|
||||||
|
port.
|
||||||
|
|
||||||
1. Add type definitions to `src/types/index.ts`
|
No root-level `start:editor` or `build:editor` scripts exist; run pnpm
|
||||||
2. Update API client in `src/api/metadataApi.ts`
|
commands from this directory.
|
||||||
3. Create/update components as needed
|
|
||||||
4. Update this README with new features
|
|
||||||
|
|
||||||
## License
|
## Build
|
||||||
|
|
||||||
Part of the MetaMesh project.
|
```bash
|
||||||
|
pnpm build # tsc -b && vite build → dist/
|
||||||
|
```
|
||||||
|
|
||||||
|
The container build copies `dist/` into the nginx web root so the editor
|
||||||
|
is served at `/editor/`.
|
||||||
|
|
||||||
|
## Tech
|
||||||
|
|
||||||
|
- React 18, TypeScript 5
|
||||||
|
- Vite 5 (`base: '/editor/'`)
|
||||||
|
- Native CSS — no UI framework
|
||||||
|
- `iso-639-3` for language code rendering
|
||||||
|
|||||||
Generated
+13
-2
@@ -1,13 +1,14 @@
|
|||||||
{
|
{
|
||||||
"name": "@meta-sort/editor",
|
"name": "@meta-core/editor",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"lockfileVersion": 3,
|
"lockfileVersion": 3,
|
||||||
"requires": true,
|
"requires": true,
|
||||||
"packages": {
|
"packages": {
|
||||||
"": {
|
"": {
|
||||||
"name": "@meta-sort/editor",
|
"name": "@meta-core/editor",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"iso-639-3": "^3.0.1",
|
||||||
"react": "^18.2.0",
|
"react": "^18.2.0",
|
||||||
"react-dom": "^18.2.0"
|
"react-dom": "^18.2.0"
|
||||||
},
|
},
|
||||||
@@ -1246,6 +1247,16 @@
|
|||||||
"node": ">=6.9.0"
|
"node": ">=6.9.0"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
|
"node_modules/iso-639-3": {
|
||||||
|
"version": "3.0.1",
|
||||||
|
"resolved": "https://registry.npmjs.org/iso-639-3/-/iso-639-3-3.0.1.tgz",
|
||||||
|
"integrity": "sha512-SdljCYXOexv/JmbQ0tvigHN43yECoscVpe2y2hlEqy/CStXQlroPhZLj7zKLRiGqLJfw8k7B973UAMDoQczVgQ==",
|
||||||
|
"license": "MIT",
|
||||||
|
"funding": {
|
||||||
|
"type": "github",
|
||||||
|
"url": "https://github.com/sponsors/wooorm"
|
||||||
|
}
|
||||||
|
},
|
||||||
"node_modules/js-tokens": {
|
"node_modules/js-tokens": {
|
||||||
"version": "4.0.0",
|
"version": "4.0.0",
|
||||||
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
"resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",
|
||||||
|
|||||||
@@ -1,185 +0,0 @@
|
|||||||
.app {
|
|
||||||
min-height: 100vh;
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-header {
|
|
||||||
padding: 2rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-header h1 {
|
|
||||||
margin: 0 0 0.5rem 0;
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-description {
|
|
||||||
margin: 0;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-main {
|
|
||||||
flex: 1;
|
|
||||||
padding: 2rem;
|
|
||||||
max-width: 1400px;
|
|
||||||
width: 100%;
|
|
||||||
margin: 0 auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-error {
|
|
||||||
padding: 1rem;
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border: 1px solid rgba(255, 0, 0, 0.3);
|
|
||||||
border-radius: 8px;
|
|
||||||
color: #ff6b6b;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-loading {
|
|
||||||
text-align: center;
|
|
||||||
padding: 3rem 2rem;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
font-size: 1.1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome {
|
|
||||||
text-align: center;
|
|
||||||
padding: 4rem 2rem;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome h2 {
|
|
||||||
margin: 0 0 1rem 0;
|
|
||||||
color: rgba(255, 255, 255, 0.8);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome-features {
|
|
||||||
margin-top: 1rem;
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-kv-browser {
|
|
||||||
margin-top: 1.5rem;
|
|
||||||
text-align: left;
|
|
||||||
max-width: 900px;
|
|
||||||
margin-left: auto;
|
|
||||||
margin-right: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-content {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: 350px 1fr;
|
|
||||||
gap: 1.5rem;
|
|
||||||
margin-top: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-sidebar {
|
|
||||||
min-height: 400px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-editor {
|
|
||||||
min-height: 400px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-no-selection {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
height: 100%;
|
|
||||||
min-height: 400px;
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 8px;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
text-align: center;
|
|
||||||
padding: 2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-footer {
|
|
||||||
padding: 1.5rem 2rem;
|
|
||||||
border-top: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
text-align: center;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-size: 0.9rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-footer a {
|
|
||||||
color: #646cff;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-footer a:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 1024px) {
|
|
||||||
.app-content {
|
|
||||||
grid-template-columns: 1fr;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-sidebar {
|
|
||||||
order: 2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-editor {
|
|
||||||
order: 1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 768px) {
|
|
||||||
.app-header,
|
|
||||||
.app-main,
|
|
||||||
.app-footer {
|
|
||||||
padding: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-content {
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.app-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-description {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border-color: rgba(255, 0, 0, 0.3);
|
|
||||||
color: #cc0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-loading {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome h2 {
|
|
||||||
color: rgba(0, 0, 0, 0.8);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-welcome-features {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-no-selection {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.app-footer {
|
|
||||||
border-top-color: rgba(0, 0, 0, 0.1);
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,246 +0,0 @@
|
|||||||
import { useState, useEffect } from 'react';
|
|
||||||
import { SearchBar } from './components/SearchBar';
|
|
||||||
import { FileList } from './components/FileList';
|
|
||||||
import { MetadataEditor } from './components/MetadataEditor';
|
|
||||||
import { BulkMetadataEditor } from './components/BulkMetadataEditor';
|
|
||||||
import { KVTreeView } from './components/KVTreeView';
|
|
||||||
import { MetadataAPI } from './api/metadataApi';
|
|
||||||
import { SearchResult } from './types';
|
|
||||||
import './App.css';
|
|
||||||
|
|
||||||
function App() {
|
|
||||||
const [searchResults, setSearchResults] = useState<SearchResult[]>([]);
|
|
||||||
const [selectedHashId, setSelectedHashId] = useState<string | undefined>();
|
|
||||||
const [selectedHashIds, setSelectedHashIds] = useState<Set<string>>(new Set());
|
|
||||||
const [bulkEditMode, setBulkEditMode] = useState(false);
|
|
||||||
const [isSearching, setIsSearching] = useState(false);
|
|
||||||
const [isLoadingDirectHash, setIsLoadingDirectHash] = useState(false);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
|
|
||||||
// Extract hash from URL path (supports /editor/file/hash or /editor/?hash=...)
|
|
||||||
const getHashFromUrl = (): string | null => {
|
|
||||||
const path = window.location.pathname;
|
|
||||||
const fileMatch = path.match(/\/file\/([a-z0-9]+)$/i);
|
|
||||||
if (fileMatch) {
|
|
||||||
return fileMatch[1];
|
|
||||||
}
|
|
||||||
// Fallback to query param for backwards compatibility
|
|
||||||
const params = new URLSearchParams(window.location.search);
|
|
||||||
return params.get('hash');
|
|
||||||
};
|
|
||||||
|
|
||||||
// Update URL when file is selected
|
|
||||||
const updateUrlForFile = (hashId: string | undefined) => {
|
|
||||||
const basePath = window.location.pathname.replace(/\/file\/[a-z0-9]+$/i, '').replace(/\/$/, '');
|
|
||||||
if (hashId) {
|
|
||||||
const newPath = `${basePath}/file/${hashId}`;
|
|
||||||
window.history.pushState({ hashId }, '', newPath);
|
|
||||||
} else {
|
|
||||||
window.history.pushState({}, '', basePath || '/editor/');
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
// Check for hash parameter in URL on component mount
|
|
||||||
useEffect(() => {
|
|
||||||
const loadHashFromUrl = async () => {
|
|
||||||
const hashParam = getHashFromUrl();
|
|
||||||
|
|
||||||
if (hashParam) {
|
|
||||||
try {
|
|
||||||
setIsLoadingDirectHash(true);
|
|
||||||
setError(null);
|
|
||||||
|
|
||||||
// Try to get metadata for this hash
|
|
||||||
const metadata = await MetadataAPI.getMetadata(hashParam);
|
|
||||||
|
|
||||||
// Create a search result from the metadata
|
|
||||||
const searchResult: SearchResult = {
|
|
||||||
hashId: hashParam,
|
|
||||||
metadata: metadata,
|
|
||||||
score: 1.0,
|
|
||||||
};
|
|
||||||
|
|
||||||
// Set the search results and select this file
|
|
||||||
setSearchResults([searchResult]);
|
|
||||||
setSelectedHashId(hashParam);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(`Failed to load file with hash ${hashParam}: ${err.message}`);
|
|
||||||
setSearchResults([]);
|
|
||||||
setSelectedHashId(undefined);
|
|
||||||
} finally {
|
|
||||||
setIsLoadingDirectHash(false);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
loadHashFromUrl();
|
|
||||||
|
|
||||||
// Handle browser back/forward
|
|
||||||
const handlePopState = (event: PopStateEvent) => {
|
|
||||||
const hashId = event.state?.hashId || getHashFromUrl();
|
|
||||||
if (hashId) {
|
|
||||||
setSelectedHashId(hashId);
|
|
||||||
} else {
|
|
||||||
setSelectedHashId(undefined);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
window.addEventListener('popstate', handlePopState);
|
|
||||||
return () => window.removeEventListener('popstate', handlePopState);
|
|
||||||
}, []); // Empty dependency array means this runs once on mount
|
|
||||||
|
|
||||||
const handleSearch = async (query: string) => {
|
|
||||||
try {
|
|
||||||
setIsSearching(true);
|
|
||||||
setError(null);
|
|
||||||
|
|
||||||
// Unified search - searches across all fields
|
|
||||||
const response = await MetadataAPI.search({ query });
|
|
||||||
|
|
||||||
setSearchResults(response.results);
|
|
||||||
|
|
||||||
// Auto-select first result if available
|
|
||||||
if (response.results.length > 0) {
|
|
||||||
setSelectedHashId(response.results[0].hashId);
|
|
||||||
} else {
|
|
||||||
setSelectedHashId(undefined);
|
|
||||||
}
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
setSearchResults([]);
|
|
||||||
setSelectedHashId(undefined);
|
|
||||||
} finally {
|
|
||||||
setIsSearching(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleSelectFile = (hashId: string) => {
|
|
||||||
if (!bulkEditMode) {
|
|
||||||
setSelectedHashId(hashId);
|
|
||||||
setSelectedHashIds(new Set());
|
|
||||||
updateUrlForFile(hashId);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleToggleSelection = (hashId: string) => {
|
|
||||||
const newSelection = new Set(selectedHashIds);
|
|
||||||
if (newSelection.has(hashId)) {
|
|
||||||
newSelection.delete(hashId);
|
|
||||||
} else {
|
|
||||||
newSelection.add(hashId);
|
|
||||||
}
|
|
||||||
setSelectedHashIds(newSelection);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleSelectAll = () => {
|
|
||||||
const allHashIds = new Set(searchResults.map(r => r.hashId));
|
|
||||||
setSelectedHashIds(allHashIds);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleClearSelection = () => {
|
|
||||||
setSelectedHashIds(new Set());
|
|
||||||
setBulkEditMode(false);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleBulkEditMode = (enabled: boolean) => {
|
|
||||||
setBulkEditMode(enabled);
|
|
||||||
if (enabled) {
|
|
||||||
setSelectedHashId(undefined);
|
|
||||||
} else {
|
|
||||||
setSelectedHashIds(new Set());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleMetadataSaved = () => {
|
|
||||||
// Optionally refresh the search results to reflect changes
|
|
||||||
console.log('Metadata saved successfully');
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleBulkMetadataSaved = () => {
|
|
||||||
console.log('Bulk metadata saved successfully');
|
|
||||||
handleClearSelection();
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="app">
|
|
||||||
<header className="app-header">
|
|
||||||
<h1>MetaMesh Metadata Editor</h1>
|
|
||||||
<p className="app-description">
|
|
||||||
Search and edit metadata for your media files
|
|
||||||
</p>
|
|
||||||
</header>
|
|
||||||
|
|
||||||
<main className="app-main">
|
|
||||||
<SearchBar onSearch={handleSearch} isLoading={isSearching} />
|
|
||||||
|
|
||||||
{error && <div className="app-error">{error}</div>}
|
|
||||||
|
|
||||||
{isLoadingDirectHash && (
|
|
||||||
<div className="app-loading">Loading file metadata...</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!isSearching && !isLoadingDirectHash && searchResults.length === 0 && !error && (
|
|
||||||
<div className="app-welcome">
|
|
||||||
<h2>Welcome to MetaMesh Metadata Editor</h2>
|
|
||||||
<p>Search for files by title, filename, file path, or hash ID to get started.</p>
|
|
||||||
<p className="app-welcome-features">
|
|
||||||
Unified search across all fields | Single file editing | Bulk editing | Direct access via URL
|
|
||||||
</p>
|
|
||||||
<div className="app-kv-browser">
|
|
||||||
<KVTreeView />
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{searchResults.length > 0 && (
|
|
||||||
<div className="app-content">
|
|
||||||
<div className="app-sidebar">
|
|
||||||
<FileList
|
|
||||||
results={searchResults}
|
|
||||||
selectedHashId={selectedHashId}
|
|
||||||
selectedHashIds={selectedHashIds}
|
|
||||||
bulkEditMode={bulkEditMode}
|
|
||||||
onSelectFile={handleSelectFile}
|
|
||||||
onToggleSelection={handleToggleSelection}
|
|
||||||
onSelectAll={handleSelectAll}
|
|
||||||
onClearSelection={handleClearSelection}
|
|
||||||
onBulkEditMode={handleBulkEditMode}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
<div className="app-editor">
|
|
||||||
{bulkEditMode && selectedHashIds.size > 0 ? (
|
|
||||||
<BulkMetadataEditor
|
|
||||||
hashIds={Array.from(selectedHashIds)}
|
|
||||||
onSaved={handleBulkMetadataSaved}
|
|
||||||
onCancel={handleClearSelection}
|
|
||||||
/>
|
|
||||||
) : selectedHashId ? (
|
|
||||||
<MetadataEditor
|
|
||||||
key={selectedHashId}
|
|
||||||
hashId={selectedHashId}
|
|
||||||
onSaved={handleMetadataSaved}
|
|
||||||
/>
|
|
||||||
) : (
|
|
||||||
<div className="app-no-selection">
|
|
||||||
{bulkEditMode
|
|
||||||
? 'Select files from the list to bulk edit metadata'
|
|
||||||
: 'Select a file from the list to edit its metadata'}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</main>
|
|
||||||
|
|
||||||
<footer className="app-footer">
|
|
||||||
<p>
|
|
||||||
MetaMesh Metadata Editor | Part of{' '}
|
|
||||||
<a href="/" target="_blank">
|
|
||||||
MetaMesh
|
|
||||||
</a>
|
|
||||||
</p>
|
|
||||||
</footer>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
export default App;
|
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
// Client for the meta-core /api/kv endpoints. The keyspace is a slash-tree of
|
||||||
|
// string-valued keys; this module wraps the S3-style listing, search, and
|
||||||
|
// per-key get/put/delete operations.
|
||||||
|
|
||||||
|
const API_BASE = '/api/kv';
|
||||||
|
|
||||||
|
export interface KVTreeResponse {
|
||||||
|
prefix: string;
|
||||||
|
delimiter: string;
|
||||||
|
branches: string[];
|
||||||
|
leaves: string[];
|
||||||
|
cursor: string;
|
||||||
|
hasMore: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KVSearchResponse {
|
||||||
|
contains: string;
|
||||||
|
keys: string[];
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KVFindMatch {
|
||||||
|
key: string;
|
||||||
|
value: string;
|
||||||
|
entryPath: string;
|
||||||
|
field: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KVFindResponse {
|
||||||
|
contains: string;
|
||||||
|
fields: string[];
|
||||||
|
matches: KVFindMatch[];
|
||||||
|
truncated: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KVValueResponse {
|
||||||
|
key: string;
|
||||||
|
type: string;
|
||||||
|
value: string;
|
||||||
|
exists: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface KVInfoResponse {
|
||||||
|
prefix: string;
|
||||||
|
fileCount: number;
|
||||||
|
keyCount: number;
|
||||||
|
totalSize: number;
|
||||||
|
memoryUsage: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function jsonOrThrow<T>(res: Response): Promise<T> {
|
||||||
|
if (!res.ok) {
|
||||||
|
let msg = `HTTP ${res.status}`;
|
||||||
|
try {
|
||||||
|
const body = await res.json();
|
||||||
|
if (body?.message) msg = body.message;
|
||||||
|
else if (body?.error) msg = body.error;
|
||||||
|
} catch {}
|
||||||
|
throw new Error(msg);
|
||||||
|
}
|
||||||
|
return res.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
export const KVAPI = {
|
||||||
|
async info(): Promise<KVInfoResponse> {
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/info`));
|
||||||
|
},
|
||||||
|
|
||||||
|
async tree(params: {
|
||||||
|
prefix?: string;
|
||||||
|
delimiter?: string;
|
||||||
|
cursor?: string;
|
||||||
|
max?: number;
|
||||||
|
}): Promise<KVTreeResponse> {
|
||||||
|
const q = new URLSearchParams();
|
||||||
|
if (params.prefix !== undefined) q.set('prefix', params.prefix);
|
||||||
|
if (params.delimiter !== undefined) q.set('delimiter', params.delimiter);
|
||||||
|
if (params.cursor) q.set('cursor', params.cursor);
|
||||||
|
if (params.max) q.set('max', String(params.max));
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/tree?${q}`));
|
||||||
|
},
|
||||||
|
|
||||||
|
// Walk multiple SCAN cycles until the keyspace is exhausted or a soft cap
|
||||||
|
// is reached. Useful for branches with up to a few thousand children where
|
||||||
|
// pagination UX would just be a worse spinner.
|
||||||
|
async treeAll(prefix: string, delimiter: string, softCap = 5000): Promise<KVTreeResponse> {
|
||||||
|
const seenLeaves = new Set<string>();
|
||||||
|
const seenBranches = new Set<string>();
|
||||||
|
let cursor = '';
|
||||||
|
while (true) {
|
||||||
|
const page = await this.tree({ prefix, delimiter, cursor, max: 1000 });
|
||||||
|
for (const k of page.leaves) seenLeaves.add(k);
|
||||||
|
for (const k of page.branches) seenBranches.add(k);
|
||||||
|
cursor = page.cursor;
|
||||||
|
if (!page.hasMore) break;
|
||||||
|
if (seenLeaves.size + seenBranches.size >= softCap) break;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
prefix,
|
||||||
|
delimiter,
|
||||||
|
branches: [...seenBranches].sort(),
|
||||||
|
leaves: [...seenLeaves].sort(),
|
||||||
|
cursor,
|
||||||
|
hasMore: cursor !== '0' && cursor !== '',
|
||||||
|
};
|
||||||
|
},
|
||||||
|
|
||||||
|
async search(contains: string, limit = 500): Promise<KVSearchResponse> {
|
||||||
|
const q = new URLSearchParams({ contains, limit: String(limit) });
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/search?${q}`));
|
||||||
|
},
|
||||||
|
|
||||||
|
async find(contains: string, fields: string[], limit = 200): Promise<KVFindResponse> {
|
||||||
|
const q = new URLSearchParams({ contains, fields: fields.join(','), limit: String(limit) });
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/find?${q}`));
|
||||||
|
},
|
||||||
|
|
||||||
|
async get(key: string): Promise<KVValueResponse> {
|
||||||
|
const q = new URLSearchParams({ key });
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`));
|
||||||
|
},
|
||||||
|
|
||||||
|
async put(key: string, value: string): Promise<KVValueResponse> {
|
||||||
|
const res = await fetch(`${API_BASE}/value`, {
|
||||||
|
method: 'PUT',
|
||||||
|
headers: { 'Content-Type': 'application/json' },
|
||||||
|
body: JSON.stringify({ key, value }),
|
||||||
|
});
|
||||||
|
return jsonOrThrow(res);
|
||||||
|
},
|
||||||
|
|
||||||
|
async del(key: string): Promise<{ key: string; deleted: boolean }> {
|
||||||
|
const q = new URLSearchParams({ key });
|
||||||
|
return jsonOrThrow(await fetch(`${API_BASE}/value?${q}`, { method: 'DELETE' }));
|
||||||
|
},
|
||||||
|
};
|
||||||
@@ -1,151 +0,0 @@
|
|||||||
import {
|
|
||||||
FileMetadata,
|
|
||||||
SearchResponse,
|
|
||||||
UpdateResult,
|
|
||||||
BatchUpdateResponse,
|
|
||||||
} from '../types';
|
|
||||||
|
|
||||||
const API_BASE = '/api/metadata';
|
|
||||||
|
|
||||||
export class MetadataAPI {
|
|
||||||
/**
|
|
||||||
* Search for files by query, hashId, or property
|
|
||||||
*/
|
|
||||||
static async search(params: {
|
|
||||||
query?: string;
|
|
||||||
hashId?: string;
|
|
||||||
property?: string;
|
|
||||||
propertyValue?: string;
|
|
||||||
limit?: number;
|
|
||||||
}): Promise<SearchResponse> {
|
|
||||||
const response = await fetch(`${API_BASE}/search`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(params),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Search failed');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get metadata for a specific file by hash ID
|
|
||||||
*/
|
|
||||||
static async getMetadata(hashId: string): Promise<FileMetadata> {
|
|
||||||
const response = await fetch(`${API_BASE}/${hashId}`);
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Failed to get metadata');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update complete metadata for a file
|
|
||||||
*/
|
|
||||||
static async updateMetadata(
|
|
||||||
hashId: string,
|
|
||||||
metadata: FileMetadata
|
|
||||||
): Promise<UpdateResult> {
|
|
||||||
const response = await fetch(`${API_BASE}/${hashId}`, {
|
|
||||||
method: 'PUT',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify(metadata),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Failed to update metadata');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Update a specific property for a file
|
|
||||||
*/
|
|
||||||
static async updateProperty(
|
|
||||||
hashId: string,
|
|
||||||
property: string,
|
|
||||||
value: string
|
|
||||||
): Promise<UpdateResult> {
|
|
||||||
const response = await fetch(`${API_BASE}/${hashId}/property`, {
|
|
||||||
method: 'PUT',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ property, value }),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Failed to update property');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Batch update multiple files
|
|
||||||
*/
|
|
||||||
static async batchUpdate(
|
|
||||||
updates: Array<{
|
|
||||||
hashId: string;
|
|
||||||
metadata?: FileMetadata;
|
|
||||||
properties?: Record<string, string>;
|
|
||||||
}>
|
|
||||||
): Promise<BatchUpdateResponse> {
|
|
||||||
const response = await fetch(`${API_BASE}/batch`, {
|
|
||||||
method: 'POST',
|
|
||||||
headers: {
|
|
||||||
'Content-Type': 'application/json',
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ updates }),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Batch update failed');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Delete metadata for a file
|
|
||||||
*/
|
|
||||||
static async deleteMetadata(hashId: string): Promise<void> {
|
|
||||||
const response = await fetch(`${API_BASE}/${hashId}`, {
|
|
||||||
method: 'DELETE',
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Failed to delete metadata');
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get all hash IDs
|
|
||||||
*/
|
|
||||||
static async getAllHashIds(): Promise<{ hashIds: string[]; count: number }> {
|
|
||||||
const response = await fetch(`${API_BASE}/hash-ids`);
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const error = await response.json();
|
|
||||||
throw new Error(error.error || 'Failed to get hash IDs');
|
|
||||||
}
|
|
||||||
|
|
||||||
return response.json();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
|
||||||
|
export type SchemaPrimitive =
|
||||||
|
| 'string'
|
||||||
|
| 'int'
|
||||||
|
| 'float'
|
||||||
|
| 'bool'
|
||||||
|
| 'json-object'
|
||||||
|
| 'json-array'
|
||||||
|
| 'undefined'
|
||||||
|
| 'mixed';
|
||||||
|
|
||||||
|
export type SchemaHint = 'cid' | 'timestamp';
|
||||||
|
export type SchemaKeyHint = 'language-code';
|
||||||
|
|
||||||
|
export interface SchemaFieldInfo {
|
||||||
|
type: SchemaPrimitive;
|
||||||
|
hint?: SchemaHint;
|
||||||
|
key_hint?: SchemaKeyHint;
|
||||||
|
breakdown: Record<string, number>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface SchemaResponse {
|
||||||
|
fields: Record<string, SchemaFieldInfo>;
|
||||||
|
generated_at: string;
|
||||||
|
source: 'live' | 'rescan';
|
||||||
|
}
|
||||||
|
|
||||||
|
let cached: Promise<SchemaResponse> | null = null;
|
||||||
|
|
||||||
|
export async function fetchSchema(): Promise<SchemaResponse> {
|
||||||
|
if (!cached) {
|
||||||
|
cached = fetch('/api/schema').then((r) => {
|
||||||
|
if (!r.ok) {
|
||||||
|
cached = null;
|
||||||
|
throw new Error(`schema fetch failed: ${r.status}`);
|
||||||
|
}
|
||||||
|
return r.json();
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return cached;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Returns the live schema entry for a field, or undefined if not yet observed.
|
||||||
|
* Loads /api/schema lazily on first mount and caches the result process-wide.
|
||||||
|
*/
|
||||||
|
export function useSchema(): {
|
||||||
|
schema: SchemaResponse | null;
|
||||||
|
getFieldHint: (key: string) => SchemaFieldInfo | undefined;
|
||||||
|
} {
|
||||||
|
const [schema, setSchema] = useState<SchemaResponse | null>(null);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let alive = true;
|
||||||
|
fetchSchema()
|
||||||
|
.then((s) => {
|
||||||
|
if (alive) setSchema(s);
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
// Schema endpoint missing or errored — editor falls back to plugin schemas.
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
alive = false;
|
||||||
|
};
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
const getFieldHint = (key: string): SchemaFieldInfo | undefined => {
|
||||||
|
if (!schema) return undefined;
|
||||||
|
return schema.fields[key];
|
||||||
|
};
|
||||||
|
|
||||||
|
return { schema, getFieldHint };
|
||||||
|
}
|
||||||
@@ -0,0 +1,96 @@
|
|||||||
|
const API_BASE = '/api/snapshot';
|
||||||
|
|
||||||
|
export type ImportMode = 'replace' | 'merge';
|
||||||
|
export type ImportConflict = 'mine' | 'source';
|
||||||
|
|
||||||
|
export interface FileImportResult {
|
||||||
|
hashId: string;
|
||||||
|
added: number;
|
||||||
|
updated: number;
|
||||||
|
unchanged: number;
|
||||||
|
kept: number;
|
||||||
|
deleted: number;
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PluginOutputResult {
|
||||||
|
written: number;
|
||||||
|
skipped: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ImportResult {
|
||||||
|
mode: ImportMode;
|
||||||
|
conflict?: ImportConflict;
|
||||||
|
dryRun: boolean;
|
||||||
|
totalFiles: number;
|
||||||
|
filesOk: number;
|
||||||
|
filesFailed: number;
|
||||||
|
files?: FileImportResult[];
|
||||||
|
pluginOutput?: PluginOutputResult;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WipeResult {
|
||||||
|
metadataDeleted: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export class SnapshotAPI {
|
||||||
|
/**
|
||||||
|
* Trigger a download of a metadata snapshot ZIP.
|
||||||
|
* Browser handles the download via a synthetic <a> click.
|
||||||
|
*/
|
||||||
|
static downloadExport(includes: { pluginOutput?: boolean } = {}): void {
|
||||||
|
const parts: string[] = [];
|
||||||
|
if (includes.pluginOutput) parts.push('plugin-output');
|
||||||
|
const qs = parts.length ? `?include=${parts.join(',')}` : '';
|
||||||
|
const a = document.createElement('a');
|
||||||
|
a.href = `${API_BASE}/export${qs}`;
|
||||||
|
a.rel = 'noopener';
|
||||||
|
document.body.appendChild(a);
|
||||||
|
a.click();
|
||||||
|
a.remove();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Upload a snapshot ZIP and apply it.
|
||||||
|
* Mode replace deletes and rewrites each CID's properties.
|
||||||
|
* Mode merge preserves untouched fields; conflict picks who wins on overlap.
|
||||||
|
*/
|
||||||
|
static async runImport(
|
||||||
|
file: File,
|
||||||
|
opts: { mode: ImportMode; conflict?: ImportConflict; dryRun?: boolean }
|
||||||
|
): Promise<ImportResult> {
|
||||||
|
const params = new URLSearchParams();
|
||||||
|
params.set('mode', opts.mode);
|
||||||
|
if (opts.mode === 'merge') {
|
||||||
|
params.set('conflict', opts.conflict ?? 'source');
|
||||||
|
}
|
||||||
|
if (opts.dryRun) {
|
||||||
|
params.set('dry_run', 'true');
|
||||||
|
}
|
||||||
|
const fd = new FormData();
|
||||||
|
fd.append('snapshot', file);
|
||||||
|
const response = await fetch(`${API_BASE}/import?${params.toString()}`, {
|
||||||
|
method: 'POST',
|
||||||
|
body: fd,
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const err = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(err.message || err.error || 'Import failed');
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Wipe Redis metadata. Pass scope=metadata.
|
||||||
|
*/
|
||||||
|
static async wipe(scope: 'metadata'): Promise<WipeResult> {
|
||||||
|
const response = await fetch(`${API_BASE}/wipe?scope=${scope}`, {
|
||||||
|
method: 'POST',
|
||||||
|
});
|
||||||
|
if (!response.ok) {
|
||||||
|
const err = await response.json().catch(() => ({}));
|
||||||
|
throw new Error(err.message || err.error || 'Wipe failed');
|
||||||
|
}
|
||||||
|
return response.json();
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -1,382 +0,0 @@
|
|||||||
.bulk-metadata-editor {
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-header {
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding-bottom: 1rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-header h2 {
|
|
||||||
margin: 0 0 0.5rem 0;
|
|
||||||
font-size: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-info {
|
|
||||||
margin: 0 0 1rem 0;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
font-size: 0.95rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-actions {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-actions button.primary {
|
|
||||||
background-color: #646cff;
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-actions button.primary:hover:not(:disabled) {
|
|
||||||
background-color: #535bf2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-message {
|
|
||||||
padding: 1rem;
|
|
||||||
border-radius: 4px;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-message.success {
|
|
||||||
background-color: rgba(0, 200, 0, 0.1);
|
|
||||||
border: 1px solid rgba(0, 200, 0, 0.3);
|
|
||||||
color: #00ff00;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-message.error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border: 1px solid rgba(255, 0, 0, 0.3);
|
|
||||||
color: #ff6b6b;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-form {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-section {
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-left: 4px solid #646cff;
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1.5rem;
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-section-header {
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
padding-bottom: 0.75rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-section-header h3 {
|
|
||||||
margin: 0 0 0.25rem 0;
|
|
||||||
font-size: 1.1rem;
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.section-hint {
|
|
||||||
margin: 0.25rem 0 0 0;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
.section-hint-bulk {
|
|
||||||
margin: 0.5rem 0 0 0;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Plugin-based bulk fields */
|
|
||||||
.bulk-fields-container {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper {
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 4px;
|
|
||||||
padding: 0.75rem;
|
|
||||||
background-color: rgba(0, 0, 0, 0.1);
|
|
||||||
transition: all 0.2s ease;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper.has-value {
|
|
||||||
border-color: rgba(100, 108, 255, 0.3);
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-header {
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-key {
|
|
||||||
font-size: 0.85rem;
|
|
||||||
font-weight: 600;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
text-transform: lowercase;
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper.has-value .field-key {
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-input {
|
|
||||||
/* Plugin component will render inside this */
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Legacy bulk fields (for compatibility) */
|
|
||||||
.bulk-metadata-field {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-field:last-child {
|
|
||||||
margin-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-header {
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-header label {
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.clear-field-btn {
|
|
||||||
width: 20px;
|
|
||||||
height: 20px;
|
|
||||||
padding: 0;
|
|
||||||
border: 1px solid rgba(255, 0, 0, 0.3);
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
color: #ff6b6b;
|
|
||||||
border-radius: 50%;
|
|
||||||
cursor: pointer;
|
|
||||||
font-size: 16px;
|
|
||||||
line-height: 1;
|
|
||||||
transition: all 0.2s;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.clear-field-btn:hover {
|
|
||||||
background-color: rgba(255, 0, 0, 0.2);
|
|
||||||
border-color: rgba(255, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-field input,
|
|
||||||
.bulk-metadata-field textarea,
|
|
||||||
.bulk-metadata-field select {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-field-row {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
||||||
gap: 1rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-field-row:last-child {
|
|
||||||
margin-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Modified fields summary */
|
|
||||||
.modified-fields {
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
border-color: rgba(100, 108, 255, 0.3);
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-fields h3 {
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-fields-list {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-field-item {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.75rem;
|
|
||||||
align-items: center;
|
|
||||||
padding: 0.75rem;
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
border: 1px solid rgba(100, 108, 255, 0.2);
|
|
||||||
border-radius: 4px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-name {
|
|
||||||
font-weight: 600;
|
|
||||||
color: #646cff;
|
|
||||||
min-width: 150px;
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-value {
|
|
||||||
flex: 1;
|
|
||||||
word-break: break-word;
|
|
||||||
color: rgba(255, 255, 255, 0.8);
|
|
||||||
font-size: 0.9rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.remove-field-btn {
|
|
||||||
padding: 0.25rem 0.75rem;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border: 1px solid rgba(255, 0, 0, 0.3);
|
|
||||||
color: #ff6b6b;
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s ease;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.remove-field-btn:hover {
|
|
||||||
background-color: rgba(255, 0, 0, 0.2);
|
|
||||||
border-color: rgba(255, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 768px) {
|
|
||||||
.bulk-metadata-editor-header {
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: flex-start;
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-actions {
|
|
||||||
width: 100%;
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-actions button {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-field-row {
|
|
||||||
grid-template-columns: 1fr;
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-field-item {
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: flex-start;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-name {
|
|
||||||
min-width: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.remove-field-btn {
|
|
||||||
align-self: flex-end;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.bulk-metadata-editor {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-info {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-section {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-section-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.section-hint {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.section-hint-bulk {
|
|
||||||
color: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper.has-value {
|
|
||||||
border-color: rgba(100, 108, 255, 0.3);
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-key {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-field-wrapper.has-value .field-key {
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-header label {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-value {
|
|
||||||
color: rgba(0, 0, 0, 0.8);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-message.success {
|
|
||||||
background-color: rgba(0, 200, 0, 0.1);
|
|
||||||
border-color: rgba(0, 200, 0, 0.3);
|
|
||||||
color: #008800;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-metadata-editor-message.error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border-color: rgba(255, 0, 0, 0.3);
|
|
||||||
color: #cc0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-fields {
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
border-color: rgba(100, 108, 255, 0.3);
|
|
||||||
}
|
|
||||||
|
|
||||||
.modified-field-item {
|
|
||||||
background-color: rgba(100, 108, 255, 0.05);
|
|
||||||
border-color: rgba(100, 108, 255, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.clear-field-btn {
|
|
||||||
border-color: rgba(255, 0, 0, 0.3);
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
color: #cc0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
.clear-field-btn:hover {
|
|
||||||
background-color: rgba(255, 0, 0, 0.2);
|
|
||||||
border-color: rgba(255, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,280 +0,0 @@
|
|||||||
import { useState, useEffect } from 'react';
|
|
||||||
import { FileMetadata, PluginSchema } from '../types';
|
|
||||||
import { MetadataAPI } from '../api/metadataApi';
|
|
||||||
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
|
|
||||||
import './BulkMetadataEditor.css';
|
|
||||||
|
|
||||||
interface PluginsResponse {
|
|
||||||
plugins: PluginSchema[];
|
|
||||||
}
|
|
||||||
|
|
||||||
interface BulkMetadataEditorProps {
|
|
||||||
hashIds: string[];
|
|
||||||
onSaved?: () => void;
|
|
||||||
onCancel?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function BulkMetadataEditor({ hashIds, onSaved, onCancel }: BulkMetadataEditorProps) {
|
|
||||||
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
|
|
||||||
const [commonValues, setCommonValues] = useState<Record<string, any>>({});
|
|
||||||
const [editedFields, setEditedFields] = useState<Partial<FileMetadata>>({});
|
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
|
||||||
const [isSaving, setIsSaving] = useState(false);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
const [successMessage, setSuccessMessage] = useState<string | null>(null);
|
|
||||||
|
|
||||||
// Load plugins and metadata on mount
|
|
||||||
useEffect(() => {
|
|
||||||
const loadData = async () => {
|
|
||||||
try {
|
|
||||||
setIsLoading(true);
|
|
||||||
setError(null);
|
|
||||||
|
|
||||||
// Fetch plugins
|
|
||||||
const pluginsResponse = await fetch('/api/plugins');
|
|
||||||
if (pluginsResponse.ok) {
|
|
||||||
const data: PluginsResponse = await pluginsResponse.json();
|
|
||||||
// Sort by priority
|
|
||||||
const sorted = [...data.plugins].sort((a, b) =>
|
|
||||||
(a.priority ?? 100) - (b.priority ?? 100)
|
|
||||||
);
|
|
||||||
setPlugins(sorted);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Load metadata for all selected files
|
|
||||||
const metadataPromises = hashIds.map((hashId) => MetadataAPI.getMetadata(hashId));
|
|
||||||
const metadata = await Promise.all(metadataPromises);
|
|
||||||
|
|
||||||
// Calculate common values
|
|
||||||
const commons: Record<string, any> = {};
|
|
||||||
|
|
||||||
// Get all editable fields from all plugins
|
|
||||||
const allFields: string[] = [];
|
|
||||||
plugins.forEach((plugin) => {
|
|
||||||
if (plugin.metadataSchema) {
|
|
||||||
Object.entries(plugin.metadataSchema).forEach(([key, field]) => {
|
|
||||||
if (!field.readonly) {
|
|
||||||
allFields.push(key);
|
|
||||||
}
|
|
||||||
});
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
allFields.forEach((field) => {
|
|
||||||
const values = metadata.map((m) => m[field as keyof FileMetadata]);
|
|
||||||
|
|
||||||
// Filter out undefined and null values
|
|
||||||
const definedValues = values.filter((v) => v !== undefined && v !== null);
|
|
||||||
|
|
||||||
if (definedValues.length === 0) {
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Safely stringify values
|
|
||||||
const uniqueValues = Array.from(
|
|
||||||
new Set(
|
|
||||||
definedValues.map((v) => {
|
|
||||||
try {
|
|
||||||
return JSON.stringify(v);
|
|
||||||
} catch {
|
|
||||||
return String(v);
|
|
||||||
}
|
|
||||||
})
|
|
||||||
)
|
|
||||||
);
|
|
||||||
|
|
||||||
if (uniqueValues.length === 1) {
|
|
||||||
try {
|
|
||||||
commons[field] = JSON.parse(uniqueValues[0]);
|
|
||||||
} catch {
|
|
||||||
commons[field] = uniqueValues[0];
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
commons[field] = '*';
|
|
||||||
}
|
|
||||||
});
|
|
||||||
|
|
||||||
setCommonValues(commons);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(`Failed to load data: ${err.message}`);
|
|
||||||
} finally {
|
|
||||||
setIsLoading(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
loadData();
|
|
||||||
}, [hashIds]);
|
|
||||||
|
|
||||||
const handleFieldChange = (field: string, value: any) => {
|
|
||||||
setEditedFields({
|
|
||||||
...editedFields,
|
|
||||||
[field]: value,
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleClearField = (field: string) => {
|
|
||||||
const newFields = { ...editedFields };
|
|
||||||
delete newFields[field];
|
|
||||||
setEditedFields(newFields);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleSave = async () => {
|
|
||||||
if (Object.keys(editedFields).length === 0) {
|
|
||||||
setError('No fields have been edited');
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
|
||||||
setIsSaving(true);
|
|
||||||
setError(null);
|
|
||||||
setSuccessMessage(null);
|
|
||||||
|
|
||||||
// Build batch update payload
|
|
||||||
const updates = hashIds.map((hashId) => ({
|
|
||||||
hashId,
|
|
||||||
metadata: editedFields,
|
|
||||||
}));
|
|
||||||
|
|
||||||
const result = await MetadataAPI.batchUpdate(updates);
|
|
||||||
|
|
||||||
if (result.errors > 0) {
|
|
||||||
setError(
|
|
||||||
`Updated ${result.success} files successfully, but ${result.errors} failed. Check console for details.`
|
|
||||||
);
|
|
||||||
console.error('Batch update errors:', result.results.filter((r) => r.error));
|
|
||||||
} else {
|
|
||||||
setSuccessMessage(`Successfully updated ${result.success} files!`);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (onSaved) {
|
|
||||||
onSaved();
|
|
||||||
}
|
|
||||||
|
|
||||||
setTimeout(() => setSuccessMessage(null), 3000);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
} finally {
|
|
||||||
setIsSaving(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const hasChanges = Object.keys(editedFields).length > 0;
|
|
||||||
|
|
||||||
// Filter to only editable plugins (those with non-readonly fields)
|
|
||||||
const editablePlugins = plugins.filter((plugin) => {
|
|
||||||
if (!plugin.metadataSchema) return false;
|
|
||||||
return Object.values(plugin.metadataSchema).some((field) => !field.readonly);
|
|
||||||
});
|
|
||||||
|
|
||||||
if (isLoading) {
|
|
||||||
return (
|
|
||||||
<div className="bulk-metadata-editor">
|
|
||||||
<div className="bulk-metadata-editor-loading">
|
|
||||||
Loading metadata for {hashIds.length} files...
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
// Build display metadata (edited values or common values)
|
|
||||||
const displayMetadata: Record<string, any> = { ...commonValues, ...editedFields };
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="bulk-metadata-editor">
|
|
||||||
<div className="bulk-metadata-editor-header">
|
|
||||||
<h2>Bulk Edit Metadata</h2>
|
|
||||||
<p className="bulk-info">
|
|
||||||
Editing {hashIds.length} file{hashIds.length !== 1 ? 's' : ''} - Only modified fields will
|
|
||||||
be updated. Fields showing "*" have different values across files.
|
|
||||||
</p>
|
|
||||||
<div className="bulk-metadata-editor-actions">
|
|
||||||
{onCancel && (
|
|
||||||
<button onClick={onCancel} disabled={isSaving}>
|
|
||||||
Cancel
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
<button onClick={handleSave} disabled={isSaving || !hasChanges} className="primary">
|
|
||||||
{isSaving ? 'Saving...' : `Update ${hashIds.length} File${hashIds.length !== 1 ? 's' : ''}`}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{successMessage && (
|
|
||||||
<div className="bulk-metadata-editor-message success">{successMessage}</div>
|
|
||||||
)}
|
|
||||||
{error && <div className="bulk-metadata-editor-message error">{error}</div>}
|
|
||||||
|
|
||||||
<div className="bulk-metadata-editor-form">
|
|
||||||
{editablePlugins.map((plugin) => {
|
|
||||||
if (!plugin.metadataSchema) return null;
|
|
||||||
|
|
||||||
// Get editable fields for this plugin
|
|
||||||
const editableFields = Object.entries(plugin.metadataSchema).filter(
|
|
||||||
([, field]) => !field.readonly
|
|
||||||
);
|
|
||||||
|
|
||||||
if (editableFields.length === 0) return null;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<section
|
|
||||||
key={plugin.id}
|
|
||||||
className="bulk-metadata-section"
|
|
||||||
style={{ borderLeftColor: plugin.color || '#646cff' }}
|
|
||||||
>
|
|
||||||
<div className="bulk-metadata-section-header">
|
|
||||||
<h3>{plugin.name}</h3>
|
|
||||||
{plugin.description && (
|
|
||||||
<p className="section-hint">{plugin.description}</p>
|
|
||||||
)}
|
|
||||||
<p className="section-hint-bulk">
|
|
||||||
Common values shown. "*" indicates different values. Edit to update all files.
|
|
||||||
</p>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="plugin-fields">
|
|
||||||
{editableFields.map(([key, field]) => (
|
|
||||||
<DynamicFieldRenderer
|
|
||||||
key={key}
|
|
||||||
fieldKey={key}
|
|
||||||
schema={field}
|
|
||||||
metadata={displayMetadata as FileMetadata}
|
|
||||||
onChange={handleFieldChange}
|
|
||||||
readonly={false}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
|
|
||||||
{/* Modified Fields Summary */}
|
|
||||||
{hasChanges && (
|
|
||||||
<section className="bulk-metadata-section modified-fields">
|
|
||||||
<h3>Fields to be Updated ({Object.keys(editedFields).length})</h3>
|
|
||||||
<div className="modified-fields-list">
|
|
||||||
{Object.entries(editedFields).map(([key, value]) => (
|
|
||||||
<div key={key} className="modified-field-item">
|
|
||||||
<span className="field-name">{key}:</span>
|
|
||||||
<span className="field-value">
|
|
||||||
{Array.isArray(value)
|
|
||||||
? value.join(', ')
|
|
||||||
: typeof value === 'object'
|
|
||||||
? JSON.stringify(value)
|
|
||||||
: String(value || '(empty)')}
|
|
||||||
</span>
|
|
||||||
<button
|
|
||||||
className="remove-field-btn"
|
|
||||||
onClick={() => handleClearField(key)}
|
|
||||||
title="Remove this field"
|
|
||||||
>
|
|
||||||
Remove
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,297 +0,0 @@
|
|||||||
.dynamic-field {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field:last-child {
|
|
||||||
margin-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field label {
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field input[type="text"],
|
|
||||||
.dynamic-field input[type="number"],
|
|
||||||
.dynamic-field select,
|
|
||||||
.dynamic-field textarea {
|
|
||||||
width: 100%;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 4px;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
color: #fff;
|
|
||||||
transition: border-color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field input:focus,
|
|
||||||
.dynamic-field select:focus,
|
|
||||||
.dynamic-field textarea:focus {
|
|
||||||
outline: none;
|
|
||||||
border-color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field input:disabled,
|
|
||||||
.dynamic-field select:disabled,
|
|
||||||
.dynamic-field textarea:disabled {
|
|
||||||
background: rgba(0, 0, 0, 0.2);
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
cursor: not-allowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field textarea {
|
|
||||||
resize: vertical;
|
|
||||||
min-height: 80px;
|
|
||||||
font-family: inherit;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-hint {
|
|
||||||
display: block;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.checkbox-label {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
cursor: pointer;
|
|
||||||
font-weight: normal;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.checkbox-label input[type="checkbox"] {
|
|
||||||
width: 1.1rem;
|
|
||||||
height: 1.1rem;
|
|
||||||
cursor: pointer;
|
|
||||||
accent-color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.checkbox-label input:disabled {
|
|
||||||
cursor: not-allowed;
|
|
||||||
opacity: 0.5;
|
|
||||||
}
|
|
||||||
|
|
||||||
.readonly-value {
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
background: rgba(0, 0, 0, 0.2);
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 4px;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
word-break: break-all;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-link {
|
|
||||||
color: #646cff;
|
|
||||||
text-decoration: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-link:hover {
|
|
||||||
text-decoration: underline;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* CID field styling */
|
|
||||||
.cid-field .cid-value-container {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.cid-field .cid-value {
|
|
||||||
flex: 1;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 4px;
|
|
||||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: #a0d8ef;
|
|
||||||
cursor: pointer;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.cid-field .cid-value:hover {
|
|
||||||
background: rgba(0, 0, 0, 0.4);
|
|
||||||
border-color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.cid-field .cid-open-button {
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
padding: 0.5rem 0.75rem;
|
|
||||||
background: #646cff;
|
|
||||||
color: #fff;
|
|
||||||
border-radius: 4px;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
font-weight: 500;
|
|
||||||
white-space: nowrap;
|
|
||||||
transition: background 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.cid-field .cid-open-button:hover {
|
|
||||||
background: #535bf2;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* JSON field styling */
|
|
||||||
.json-field .json-viewer {
|
|
||||||
background: rgba(0, 0, 0, 0.4);
|
|
||||||
color: #d4d4d4;
|
|
||||||
padding: 1rem;
|
|
||||||
border-radius: 4px;
|
|
||||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
overflow-x: auto;
|
|
||||||
max-height: 300px;
|
|
||||||
overflow-y: auto;
|
|
||||||
margin: 0;
|
|
||||||
white-space: pre-wrap;
|
|
||||||
word-break: break-all;
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Language string field styling */
|
|
||||||
.language-string-field .language-entries {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .language-entry {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-input {
|
|
||||||
flex: 1;
|
|
||||||
min-width: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-code {
|
|
||||||
display: inline-block;
|
|
||||||
width: 50px;
|
|
||||||
padding: 0.4rem 0.5rem;
|
|
||||||
background: rgba(100, 108, 255, 0.2);
|
|
||||||
border: 1px solid rgba(100, 108, 255, 0.3);
|
|
||||||
border-radius: 4px;
|
|
||||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
color: #a0d8ef;
|
|
||||||
text-align: center;
|
|
||||||
flex-shrink: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-text {
|
|
||||||
flex: 1;
|
|
||||||
padding: 0.4rem 0.75rem;
|
|
||||||
background: rgba(0, 0, 0, 0.2);
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 4px;
|
|
||||||
color: rgba(255, 255, 255, 0.8);
|
|
||||||
font-size: 0.9rem;
|
|
||||||
min-width: 0;
|
|
||||||
overflow: hidden;
|
|
||||||
text-overflow: ellipsis;
|
|
||||||
white-space: nowrap;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-remove-btn {
|
|
||||||
width: 32px;
|
|
||||||
height: 32px;
|
|
||||||
flex-shrink: 0;
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
justify-content: center;
|
|
||||||
background: rgba(255, 100, 100, 0.2);
|
|
||||||
border: 1px solid rgba(255, 100, 100, 0.3);
|
|
||||||
border-radius: 4px;
|
|
||||||
color: #ff6b6b;
|
|
||||||
font-size: 1.2rem;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background 0.2s, border-color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-remove-btn:hover {
|
|
||||||
background: rgba(255, 100, 100, 0.3);
|
|
||||||
border-color: rgba(255, 100, 100, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-add-btn {
|
|
||||||
align-self: flex-start;
|
|
||||||
padding: 0.4rem 0.75rem;
|
|
||||||
background: rgba(100, 108, 255, 0.2);
|
|
||||||
border: 1px solid rgba(100, 108, 255, 0.3);
|
|
||||||
border-radius: 4px;
|
|
||||||
color: #646cff;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background 0.2s, border-color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-add-btn:hover {
|
|
||||||
background: rgba(100, 108, 255, 0.3);
|
|
||||||
border-color: rgba(100, 108, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Light theme */
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.dynamic-field label {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field input[type="text"],
|
|
||||||
.dynamic-field input[type="number"],
|
|
||||||
.dynamic-field select,
|
|
||||||
.dynamic-field textarea {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background: #fff;
|
|
||||||
color: #333;
|
|
||||||
}
|
|
||||||
|
|
||||||
.dynamic-field input:disabled,
|
|
||||||
.dynamic-field select:disabled,
|
|
||||||
.dynamic-field textarea:disabled {
|
|
||||||
background: #f5f5f5;
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.field-hint {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.checkbox-label {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.readonly-value {
|
|
||||||
background: #f8f9fa;
|
|
||||||
border-color: #e9ecef;
|
|
||||||
color: #495057;
|
|
||||||
}
|
|
||||||
|
|
||||||
.json-field .json-viewer {
|
|
||||||
background: #1e1e1e;
|
|
||||||
border-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-code {
|
|
||||||
background: rgba(100, 108, 255, 0.1);
|
|
||||||
color: #4a50c8;
|
|
||||||
}
|
|
||||||
|
|
||||||
.language-string-field .lang-text {
|
|
||||||
background: #f8f9fa;
|
|
||||||
border-color: #e9ecef;
|
|
||||||
color: #495057;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,378 +0,0 @@
|
|||||||
import { SchemaField, FileMetadata } from '../types';
|
|
||||||
import { LanguageCombobox, formatLanguageOption } from './LanguageCombobox';
|
|
||||||
import './DynamicFieldRenderer.css';
|
|
||||||
|
|
||||||
interface DynamicFieldRendererProps {
|
|
||||||
fieldKey: string;
|
|
||||||
schema: SchemaField;
|
|
||||||
metadata: FileMetadata;
|
|
||||||
onChange: (key: string, value: any) => void;
|
|
||||||
readonly?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Renders a single metadata field based on its schema definition
|
|
||||||
*/
|
|
||||||
export function DynamicFieldRenderer({
|
|
||||||
fieldKey,
|
|
||||||
schema,
|
|
||||||
metadata,
|
|
||||||
onChange,
|
|
||||||
readonly = false,
|
|
||||||
}: DynamicFieldRendererProps) {
|
|
||||||
const isReadonly = readonly || schema.readonly;
|
|
||||||
const fieldType = schema.type || 'string';
|
|
||||||
|
|
||||||
// Get value from metadata using path-style key
|
|
||||||
const getValue = (): any => {
|
|
||||||
// Handle path-style keys like "stream/duration" or "nfo/title"
|
|
||||||
const parts = fieldKey.split('/');
|
|
||||||
let value: any = metadata;
|
|
||||||
for (const part of parts) {
|
|
||||||
if (value === undefined || value === null) return undefined;
|
|
||||||
value = value[part];
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Set value in metadata using path-style key
|
|
||||||
const setValue = (newValue: any) => {
|
|
||||||
onChange(fieldKey, newValue);
|
|
||||||
};
|
|
||||||
|
|
||||||
const value = getValue();
|
|
||||||
|
|
||||||
// Render link if configured
|
|
||||||
const renderValueWithLink = (displayValue: string) => {
|
|
||||||
if (schema.link && displayValue) {
|
|
||||||
const url = schema.link.replace('{value}', encodeURIComponent(displayValue));
|
|
||||||
return (
|
|
||||||
<a href={url} target="_blank" rel="noopener noreferrer" className="field-link">
|
|
||||||
{displayValue}
|
|
||||||
</a>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
return displayValue;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Render based on field type
|
|
||||||
switch (fieldType) {
|
|
||||||
case 'boolean':
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label className="checkbox-label">
|
|
||||||
<input
|
|
||||||
type="checkbox"
|
|
||||||
checked={Boolean(value)}
|
|
||||||
onChange={(e) => setValue(e.target.checked)}
|
|
||||||
disabled={isReadonly}
|
|
||||||
/>
|
|
||||||
<span>{schema.label}</span>
|
|
||||||
</label>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'number':
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<input
|
|
||||||
type="number"
|
|
||||||
value={value ?? ''}
|
|
||||||
onChange={(e) => setValue(e.target.value ? Number(e.target.value) : undefined)}
|
|
||||||
disabled={isReadonly}
|
|
||||||
min={schema.min}
|
|
||||||
max={schema.max}
|
|
||||||
placeholder={schema.placeholder}
|
|
||||||
/>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'select':
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<select
|
|
||||||
value={value ?? ''}
|
|
||||||
onChange={(e) => setValue(e.target.value || undefined)}
|
|
||||||
disabled={isReadonly}
|
|
||||||
>
|
|
||||||
<option value="">Select...</option>
|
|
||||||
{schema.options?.map((opt) => (
|
|
||||||
<option key={opt.value} value={opt.value}>
|
|
||||||
{opt.label}
|
|
||||||
</option>
|
|
||||||
))}
|
|
||||||
</select>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'text':
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<textarea
|
|
||||||
value={value ?? ''}
|
|
||||||
onChange={(e) => setValue(e.target.value || undefined)}
|
|
||||||
disabled={isReadonly}
|
|
||||||
placeholder={schema.placeholder}
|
|
||||||
rows={4}
|
|
||||||
/>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'array':
|
|
||||||
// Convert RecordSet to comma-separated string for display
|
|
||||||
const arrayValue = (() => {
|
|
||||||
if (!value) return '';
|
|
||||||
if (typeof value === 'string') return value;
|
|
||||||
if (typeof value === 'object') {
|
|
||||||
return Object.values(value).join(', ');
|
|
||||||
}
|
|
||||||
return '';
|
|
||||||
})();
|
|
||||||
|
|
||||||
// Convert comma-separated string back to RecordSet
|
|
||||||
const handleArrayChange = (str: string) => {
|
|
||||||
const items = str.split(',').map(s => s.trim()).filter(Boolean);
|
|
||||||
if (items.length === 0) {
|
|
||||||
setValue(undefined);
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
const recordSet: Record<string, string> = {};
|
|
||||||
items.forEach((item, idx) => {
|
|
||||||
recordSet[idx.toString()] = item;
|
|
||||||
});
|
|
||||||
setValue(recordSet);
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
value={arrayValue}
|
|
||||||
onChange={(e) => handleArrayChange(e.target.value)}
|
|
||||||
disabled={isReadonly}
|
|
||||||
placeholder={schema.placeholder || 'Comma-separated values'}
|
|
||||||
/>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'json':
|
|
||||||
// Aggregate matching keys based on pattern
|
|
||||||
const jsonData = (() => {
|
|
||||||
if (!schema.pattern) {
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Pattern like "stream/video/{n}/*" - aggregate all matching keys
|
|
||||||
const patternPrefix = schema.pattern.replace(/\{n\}.*$/, '');
|
|
||||||
const result: Record<string, any> = {};
|
|
||||||
|
|
||||||
// Recursively find all matching keys
|
|
||||||
const findMatching = (obj: any, currentPath: string = '') => {
|
|
||||||
if (!obj || typeof obj !== 'object') return;
|
|
||||||
|
|
||||||
for (const [key, val] of Object.entries(obj)) {
|
|
||||||
const newPath = currentPath ? `${currentPath}/${key}` : key;
|
|
||||||
if (newPath.startsWith(patternPrefix.replace(/\/$/, ''))) {
|
|
||||||
// Store in result with relative path
|
|
||||||
const relativePath = newPath.substring(patternPrefix.length - 1);
|
|
||||||
if (relativePath) {
|
|
||||||
setNestedValue(result, relativePath, val);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
if (typeof val === 'object' && val !== null) {
|
|
||||||
findMatching(val, newPath);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
findMatching(metadata);
|
|
||||||
return Object.keys(result).length > 0 ? result : undefined;
|
|
||||||
})();
|
|
||||||
|
|
||||||
if (!jsonData) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field json-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<pre className="json-viewer">
|
|
||||||
{JSON.stringify(jsonData, null, 2)}
|
|
||||||
</pre>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'cid':
|
|
||||||
// CID field type - displays CID with "Open in Editor" button
|
|
||||||
if (!value) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
const cidValue = String(value);
|
|
||||||
const editorUrl = `/editor/file/${cidValue}`;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field cid-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<div className="cid-value-container">
|
|
||||||
<code
|
|
||||||
className="cid-value"
|
|
||||||
onClick={() => navigator.clipboard.writeText(cidValue)}
|
|
||||||
title="Click to copy"
|
|
||||||
>
|
|
||||||
{cidValue}
|
|
||||||
</code>
|
|
||||||
<a
|
|
||||||
href={editorUrl}
|
|
||||||
className="cid-open-button"
|
|
||||||
title="Open in Editor"
|
|
||||||
target="_blank"
|
|
||||||
rel="noopener noreferrer"
|
|
||||||
>
|
|
||||||
↗ Open
|
|
||||||
</a>
|
|
||||||
</div>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'languageString':
|
|
||||||
// Language-keyed string object: { "eng": "English Title", "jpn": "日本語タイトル" }
|
|
||||||
const langEntries: Array<[string, string]> = value && typeof value === 'object'
|
|
||||||
? Object.entries(value).map(([k, v]) => [k, String(v)])
|
|
||||||
: [];
|
|
||||||
|
|
||||||
const handleLangCodeChange = (oldLang: string, newLang: string) => {
|
|
||||||
if (oldLang === newLang) return;
|
|
||||||
const newValue = { ...value };
|
|
||||||
const text = newValue[oldLang];
|
|
||||||
delete newValue[oldLang];
|
|
||||||
newValue[newLang] = text;
|
|
||||||
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleLangTextChange = (lang: string, text: string) => {
|
|
||||||
const newValue = { ...value, [lang]: text };
|
|
||||||
setValue(newValue);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleRemoveLang = (lang: string) => {
|
|
||||||
const newValue = { ...value };
|
|
||||||
delete newValue[lang];
|
|
||||||
setValue(Object.keys(newValue).length > 0 ? newValue : undefined);
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleAddLang = () => {
|
|
||||||
// Find a language code not already used, default to 'eng'
|
|
||||||
const usedLangs = new Set(langEntries.map(([k]) => k));
|
|
||||||
const commonDefaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa'];
|
|
||||||
const defaultLang = commonDefaults.find(l => !usedLangs.has(l)) || 'eng';
|
|
||||||
setValue({ ...value, [defaultLang]: '' });
|
|
||||||
};
|
|
||||||
|
|
||||||
if (isReadonly && langEntries.length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field language-string-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
<div className="language-entries">
|
|
||||||
{langEntries.map(([lang, text]) => (
|
|
||||||
<div key={lang} className="language-entry">
|
|
||||||
{isReadonly ? (
|
|
||||||
<>
|
|
||||||
<span className="lang-code" title={formatLanguageOption(lang)}>{lang}</span>
|
|
||||||
<span className="lang-text">{text}</span>
|
|
||||||
</>
|
|
||||||
) : (
|
|
||||||
<>
|
|
||||||
<LanguageCombobox
|
|
||||||
value={lang}
|
|
||||||
onChange={(newLang) => handleLangCodeChange(lang, newLang)}
|
|
||||||
/>
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
className="lang-input"
|
|
||||||
value={text}
|
|
||||||
onChange={(e) => handleLangTextChange(lang, e.target.value)}
|
|
||||||
placeholder={schema.placeholder || 'Text...'}
|
|
||||||
/>
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
className="lang-remove-btn"
|
|
||||||
onClick={() => handleRemoveLang(lang)}
|
|
||||||
title="Remove"
|
|
||||||
>
|
|
||||||
×
|
|
||||||
</button>
|
|
||||||
</>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
{!isReadonly && (
|
|
||||||
<button
|
|
||||||
type="button"
|
|
||||||
className="lang-add-btn"
|
|
||||||
onClick={handleAddLang}
|
|
||||||
>
|
|
||||||
+ Add Language
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
|
|
||||||
case 'string':
|
|
||||||
default:
|
|
||||||
// Show nothing if no value and readonly
|
|
||||||
if (isReadonly && !value) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="dynamic-field">
|
|
||||||
<label>{schema.label}</label>
|
|
||||||
{isReadonly ? (
|
|
||||||
<div className="readonly-value">
|
|
||||||
{renderValueWithLink(String(value ?? ''))}
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
value={value ?? ''}
|
|
||||||
onChange={(e) => setValue(e.target.value || undefined)}
|
|
||||||
placeholder={schema.placeholder}
|
|
||||||
/>
|
|
||||||
)}
|
|
||||||
{schema.hint && <small className="field-hint">{schema.hint}</small>}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Helper to set nested value in object
|
|
||||||
function setNestedValue(obj: any, path: string, value: any) {
|
|
||||||
const parts = path.split('/').filter(Boolean);
|
|
||||||
let current = obj;
|
|
||||||
for (let i = 0; i < parts.length - 1; i++) {
|
|
||||||
if (!current[parts[i]]) {
|
|
||||||
current[parts[i]] = {};
|
|
||||||
}
|
|
||||||
current = current[parts[i]];
|
|
||||||
}
|
|
||||||
if (parts.length > 0) {
|
|
||||||
current[parts[parts.length - 1]] = value;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,63 +0,0 @@
|
|||||||
.plugin-section {
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-left: 4px solid #646cff;
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1.5rem;
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-section-header {
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding-bottom: 0.75rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-section-header h3 {
|
|
||||||
margin: 0 0 0.25rem 0;
|
|
||||||
font-size: 1.1rem;
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-section-header .plugin-description {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-fields {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Responsive grid for fields */
|
|
||||||
@media (min-width: 768px) {
|
|
||||||
.plugin-fields {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(250px, 1fr));
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Make text and json fields span full width */
|
|
||||||
.plugin-fields .dynamic-field:has(textarea),
|
|
||||||
.plugin-fields .json-field {
|
|
||||||
grid-column: 1 / -1;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Light theme */
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.plugin-section {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-section-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-section-header .plugin-description {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,126 +0,0 @@
|
|||||||
import { PluginSchema, FileMetadata, SchemaField } from '../types';
|
|
||||||
import { DynamicFieldRenderer } from './DynamicFieldRenderer';
|
|
||||||
import './DynamicPluginSection.css';
|
|
||||||
|
|
||||||
interface DynamicPluginSectionProps {
|
|
||||||
plugin: PluginSchema;
|
|
||||||
metadata: FileMetadata;
|
|
||||||
onChange: (key: string, value: any) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Renders a plugin section with all its fields based on schema
|
|
||||||
*/
|
|
||||||
export function DynamicPluginSection({
|
|
||||||
plugin,
|
|
||||||
metadata,
|
|
||||||
onChange,
|
|
||||||
}: DynamicPluginSectionProps) {
|
|
||||||
const schema = plugin.metadataSchema;
|
|
||||||
|
|
||||||
if (!schema || Object.keys(schema).length === 0) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check if plugin has any data in metadata
|
|
||||||
const hasData = checkPluginHasData(schema, metadata);
|
|
||||||
if (!hasData) {
|
|
||||||
return null;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Sort fields: non-readonly first, then by key
|
|
||||||
const sortedFields = Object.entries(schema).sort(([keyA, a], [keyB, b]) => {
|
|
||||||
// Non-readonly fields first
|
|
||||||
if (a.readonly !== b.readonly) {
|
|
||||||
return a.readonly ? 1 : -1;
|
|
||||||
}
|
|
||||||
// Then alphabetically
|
|
||||||
return keyA.localeCompare(keyB);
|
|
||||||
});
|
|
||||||
|
|
||||||
return (
|
|
||||||
<section
|
|
||||||
className="plugin-section"
|
|
||||||
style={{ borderLeftColor: plugin.color || '#646cff' }}
|
|
||||||
>
|
|
||||||
<div className="plugin-section-header">
|
|
||||||
<h3>{plugin.name}</h3>
|
|
||||||
{plugin.description && (
|
|
||||||
<p className="plugin-description">{plugin.description}</p>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
<div className="plugin-fields">
|
|
||||||
{sortedFields.map(([fieldKey, fieldSchema]) => (
|
|
||||||
<DynamicFieldRenderer
|
|
||||||
key={fieldKey}
|
|
||||||
fieldKey={fieldKey}
|
|
||||||
schema={fieldSchema}
|
|
||||||
metadata={metadata}
|
|
||||||
onChange={onChange}
|
|
||||||
/>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</section>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a plugin has any data in the metadata
|
|
||||||
*/
|
|
||||||
function checkPluginHasData(
|
|
||||||
schema: Record<string, SchemaField>,
|
|
||||||
metadata: FileMetadata
|
|
||||||
): boolean {
|
|
||||||
for (const fieldKey of Object.keys(schema)) {
|
|
||||||
const fieldSchema = schema[fieldKey];
|
|
||||||
|
|
||||||
// For JSON type with pattern, check if any matching keys exist
|
|
||||||
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
|
|
||||||
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
|
|
||||||
if (hasKeyWithPrefix(metadata, patternPrefix)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// For regular fields, check if value exists
|
|
||||||
if (getNestedValue(metadata, fieldKey) !== undefined) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get nested value from object using path-style key
|
|
||||||
*/
|
|
||||||
function getNestedValue(obj: any, path: string): any {
|
|
||||||
const parts = path.split('/');
|
|
||||||
let value = obj;
|
|
||||||
for (const part of parts) {
|
|
||||||
if (value === undefined || value === null) return undefined;
|
|
||||||
value = value[part];
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if object has any key starting with prefix
|
|
||||||
*/
|
|
||||||
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
|
|
||||||
if (!obj || typeof obj !== 'object') return false;
|
|
||||||
|
|
||||||
for (const key of Object.keys(obj)) {
|
|
||||||
const newPath = currentPath ? `${currentPath}/${key}` : key;
|
|
||||||
if (newPath.startsWith(prefix)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (typeof obj[key] === 'object' && obj[key] !== null) {
|
|
||||||
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
@@ -1,174 +0,0 @@
|
|||||||
.file-list {
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 8px;
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-header {
|
|
||||||
padding: 0.75rem 1rem;
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-count {
|
|
||||||
font-size: 0.9em;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-edit-toggle {
|
|
||||||
padding: 0.4rem 0.8rem;
|
|
||||||
font-size: 0.85em;
|
|
||||||
background-color: rgba(100, 108, 255, 0.1);
|
|
||||||
border: 1px solid rgba(100, 108, 255, 0.3);
|
|
||||||
color: #646cff;
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-edit-toggle:hover {
|
|
||||||
background-color: rgba(100, 108, 255, 0.2);
|
|
||||||
border-color: rgba(100, 108, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.bulk-edit-toggle.active {
|
|
||||||
background-color: rgba(100, 108, 255, 0.3);
|
|
||||||
border-color: #646cff;
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-bulk-actions {
|
|
||||||
padding: 0.75rem 1rem;
|
|
||||||
background-color: rgba(100, 108, 255, 0.1);
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
display: flex;
|
|
||||||
gap: 1rem;
|
|
||||||
align-items: center;
|
|
||||||
}
|
|
||||||
|
|
||||||
.select-all-btn {
|
|
||||||
padding: 0.4rem 0.8rem;
|
|
||||||
font-size: 0.85em;
|
|
||||||
background-color: rgba(255, 255, 255, 0.1);
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.2);
|
|
||||||
color: rgba(255, 255, 255, 0.9);
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.select-all-btn:hover {
|
|
||||||
background-color: rgba(255, 255, 255, 0.2);
|
|
||||||
border-color: rgba(255, 255, 255, 0.3);
|
|
||||||
}
|
|
||||||
|
|
||||||
.selection-count {
|
|
||||||
font-size: 0.9em;
|
|
||||||
color: #646cff;
|
|
||||||
font-weight: 500;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-items {
|
|
||||||
max-height: 500px;
|
|
||||||
overflow-y: auto;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item {
|
|
||||||
padding: 1rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color 0.2s;
|
|
||||||
display: flex;
|
|
||||||
gap: 0.75rem;
|
|
||||||
align-items: flex-start;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-checkbox {
|
|
||||||
padding-top: 0.2rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-checkbox input[type='checkbox'] {
|
|
||||||
width: 18px;
|
|
||||||
height: 18px;
|
|
||||||
cursor: pointer;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-content {
|
|
||||||
flex: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item:hover {
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item.selected {
|
|
||||||
background-color: rgba(100, 108, 255, 0.2);
|
|
||||||
border-left: 3px solid #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-title {
|
|
||||||
font-weight: 500;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-details {
|
|
||||||
display: flex;
|
|
||||||
gap: 1rem;
|
|
||||||
font-size: 0.9em;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
margin-bottom: 0.25rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-hash {
|
|
||||||
font-size: 0.8em;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-empty {
|
|
||||||
padding: 3rem 1rem;
|
|
||||||
text-align: center;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.file-list {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-header {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-count {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item:hover {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item.selected {
|
|
||||||
background-color: rgba(100, 108, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-details {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-item-hash {
|
|
||||||
color: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.file-list-empty {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,123 +0,0 @@
|
|||||||
import { SearchResult } from '../types';
|
|
||||||
import { getDisplayName, truncateHash, formatBytes } from '../utils/formatting';
|
|
||||||
import './FileList.css';
|
|
||||||
|
|
||||||
interface FileListProps {
|
|
||||||
results: SearchResult[];
|
|
||||||
selectedHashId?: string;
|
|
||||||
selectedHashIds: Set<string>;
|
|
||||||
bulkEditMode: boolean;
|
|
||||||
onSelectFile: (hashId: string) => void;
|
|
||||||
onToggleSelection: (hashId: string) => void;
|
|
||||||
onSelectAll: () => void;
|
|
||||||
onClearSelection: () => void;
|
|
||||||
onBulkEditMode: (enabled: boolean) => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function FileList({
|
|
||||||
results,
|
|
||||||
selectedHashId,
|
|
||||||
selectedHashIds,
|
|
||||||
bulkEditMode,
|
|
||||||
onSelectFile,
|
|
||||||
onToggleSelection,
|
|
||||||
onSelectAll,
|
|
||||||
onClearSelection,
|
|
||||||
onBulkEditMode,
|
|
||||||
}: FileListProps) {
|
|
||||||
if (results.length === 0) {
|
|
||||||
return (
|
|
||||||
<div className="file-list-empty">
|
|
||||||
<p>No files found. Try searching for a title, filename, or hash ID.</p>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const allSelected = results.length > 0 && results.every(r => selectedHashIds.has(r.hashId));
|
|
||||||
|
|
||||||
// Sort results by file path
|
|
||||||
const sortedResults = [...results].sort((a, b) => {
|
|
||||||
const pathA = a.metadata.filePath || a.metadata.privateFilePath || '';
|
|
||||||
const pathB = b.metadata.filePath || b.metadata.privateFilePath || '';
|
|
||||||
return pathA.localeCompare(pathB);
|
|
||||||
});
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="file-list">
|
|
||||||
<div className="file-list-header">
|
|
||||||
<span className="file-count">{results.length} files found</span>
|
|
||||||
<button
|
|
||||||
className={`bulk-edit-toggle ${bulkEditMode ? 'active' : ''}`}
|
|
||||||
onClick={() => onBulkEditMode(!bulkEditMode)}
|
|
||||||
title={bulkEditMode ? 'Exit bulk edit mode' : 'Enter bulk edit mode'}
|
|
||||||
>
|
|
||||||
{bulkEditMode ? '✓ Bulk Edit' : 'Bulk Edit'}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{bulkEditMode && (
|
|
||||||
<div className="file-list-bulk-actions">
|
|
||||||
<button onClick={allSelected ? onClearSelection : onSelectAll} className="select-all-btn">
|
|
||||||
{allSelected ? 'Deselect All' : 'Select All'}
|
|
||||||
</button>
|
|
||||||
{selectedHashIds.size > 0 && (
|
|
||||||
<span className="selection-count">
|
|
||||||
{selectedHashIds.size} selected
|
|
||||||
</span>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
<div className="file-list-items">
|
|
||||||
{sortedResults.map((result) => {
|
|
||||||
const isSelected = bulkEditMode
|
|
||||||
? selectedHashIds.has(result.hashId)
|
|
||||||
: selectedHashId === result.hashId;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div
|
|
||||||
key={result.hashId}
|
|
||||||
className={`file-list-item ${isSelected ? 'selected' : ''}`}
|
|
||||||
onClick={() => {
|
|
||||||
if (bulkEditMode) {
|
|
||||||
onToggleSelection(result.hashId);
|
|
||||||
} else {
|
|
||||||
onSelectFile(result.hashId);
|
|
||||||
}
|
|
||||||
}}
|
|
||||||
>
|
|
||||||
{bulkEditMode && (
|
|
||||||
<div className="file-list-item-checkbox">
|
|
||||||
<input
|
|
||||||
type="checkbox"
|
|
||||||
checked={selectedHashIds.has(result.hashId)}
|
|
||||||
onChange={(e) => {
|
|
||||||
e.stopPropagation();
|
|
||||||
onToggleSelection(result.hashId);
|
|
||||||
}}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
<div className="file-list-item-content">
|
|
||||||
<div className="file-list-item-title">
|
|
||||||
{getDisplayName(result.metadata)}
|
|
||||||
</div>
|
|
||||||
<div className="file-list-item-details">
|
|
||||||
<span className="file-list-item-type">
|
|
||||||
{result.metadata.videoType || 'Unknown'}
|
|
||||||
</span>
|
|
||||||
<span className="file-list-item-size">
|
|
||||||
{formatBytes(result.metadata.byteSize)}
|
|
||||||
</span>
|
|
||||||
</div>
|
|
||||||
<div className="file-list-item-hash">
|
|
||||||
Hash: {truncateHash(result.hashId)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
})}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,247 +0,0 @@
|
|||||||
.kv-tree {
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 8px;
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
overflow: hidden;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-header {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 1rem;
|
|
||||||
padding: 0.75rem 1rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-header h3 {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: rgba(255, 255, 255, 0.8);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-info {
|
|
||||||
display: flex;
|
|
||||||
gap: 1rem;
|
|
||||||
font-size: 0.75rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-refresh {
|
|
||||||
margin-left: auto;
|
|
||||||
padding: 0.25rem 0.75rem;
|
|
||||||
font-size: 0.75rem;
|
|
||||||
background-color: rgba(100, 108, 255, 0.2);
|
|
||||||
color: #646cff;
|
|
||||||
border: 1px solid rgba(100, 108, 255, 0.3);
|
|
||||||
border-radius: 4px;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-refresh:hover:not(:disabled) {
|
|
||||||
background-color: rgba(100, 108, 255, 0.3);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-refresh:disabled {
|
|
||||||
opacity: 0.5;
|
|
||||||
cursor: not-allowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-error {
|
|
||||||
padding: 0.5rem 1rem;
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
color: #ff6b6b;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-content {
|
|
||||||
max-height: 400px;
|
|
||||||
overflow-y: auto;
|
|
||||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', monospace;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-node {
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-node:last-child {
|
|
||||||
border-bottom: none;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key {
|
|
||||||
display: flex;
|
|
||||||
align-items: center;
|
|
||||||
gap: 0.5rem;
|
|
||||||
padding: 0.5rem 1rem;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key:hover {
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key.expanded {
|
|
||||||
background-color: rgba(100, 108, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-toggle {
|
|
||||||
width: 1rem;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
font-size: 0.65rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key-name {
|
|
||||||
flex: 1;
|
|
||||||
color: #88c0d0;
|
|
||||||
word-break: break-all;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-type {
|
|
||||||
padding: 0.1rem 0.4rem;
|
|
||||||
font-size: 0.65rem;
|
|
||||||
background-color: rgba(255, 255, 255, 0.1);
|
|
||||||
border-radius: 3px;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-value {
|
|
||||||
padding: 0.5rem 1rem 0.5rem 2.5rem;
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
border-top: 1px solid rgba(255, 255, 255, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-fields {
|
|
||||||
width: 100%;
|
|
||||||
border-collapse: collapse;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-fields tr:hover {
|
|
||||||
background-color: rgba(255, 255, 255, 0.03);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-field-name {
|
|
||||||
padding: 0.25rem 0.5rem 0.25rem 0;
|
|
||||||
color: #a3be8c;
|
|
||||||
white-space: nowrap;
|
|
||||||
vertical-align: top;
|
|
||||||
width: 1%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-field-value {
|
|
||||||
padding: 0.25rem 0;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
word-break: break-all;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-field-value pre {
|
|
||||||
margin: 0;
|
|
||||||
font-family: inherit;
|
|
||||||
white-space: pre-wrap;
|
|
||||||
font-size: 0.75rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-load-more {
|
|
||||||
display: block;
|
|
||||||
width: 100%;
|
|
||||||
padding: 0.75rem;
|
|
||||||
font-size: 0.8rem;
|
|
||||||
background-color: transparent;
|
|
||||||
color: #646cff;
|
|
||||||
border: none;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color 0.15s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-load-more:hover:not(:disabled) {
|
|
||||||
background-color: rgba(100, 108, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-load-more:disabled {
|
|
||||||
opacity: 0.5;
|
|
||||||
cursor: not-allowed;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-empty {
|
|
||||||
padding: 2rem;
|
|
||||||
text-align: center;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Light mode */
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.kv-tree {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.03);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-header h3 {
|
|
||||||
color: rgba(0, 0, 0, 0.8);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-info {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.08);
|
|
||||||
color: #cc0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-node {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key:hover {
|
|
||||||
background-color: rgba(0, 0, 0, 0.03);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key.expanded {
|
|
||||||
background-color: rgba(100, 108, 255, 0.08);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-toggle {
|
|
||||||
color: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-key-name {
|
|
||||||
color: #0077aa;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-type {
|
|
||||||
background-color: rgba(0, 0, 0, 0.08);
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-value {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
border-top-color: rgba(0, 0, 0, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-fields tr:hover {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-field-name {
|
|
||||||
color: #228822;
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-field-value {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-load-more:hover:not(:disabled) {
|
|
||||||
background-color: rgba(100, 108, 255, 0.08);
|
|
||||||
}
|
|
||||||
|
|
||||||
.kv-tree-empty {
|
|
||||||
color: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,216 +0,0 @@
|
|||||||
import { useState, useEffect, useCallback } from 'react';
|
|
||||||
import './KVTreeView.css';
|
|
||||||
|
|
||||||
interface KeyValue {
|
|
||||||
key: string;
|
|
||||||
type?: string;
|
|
||||||
value?: Record<string, any> | string;
|
|
||||||
expanded?: boolean;
|
|
||||||
loading?: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
interface KVInfo {
|
|
||||||
prefix: string;
|
|
||||||
fileCount: number;
|
|
||||||
keyCount: number;
|
|
||||||
totalSize: number;
|
|
||||||
memoryUsage?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
const API_BASE = '/api/kv';
|
|
||||||
|
|
||||||
export function KVTreeView() {
|
|
||||||
const [keys, setKeys] = useState<KeyValue[]>([]);
|
|
||||||
const [cursor, setCursor] = useState<string>('0');
|
|
||||||
const [hasMore, setHasMore] = useState(true);
|
|
||||||
const [loading, setLoading] = useState(false);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
const [info, setInfo] = useState<KVInfo | null>(null);
|
|
||||||
|
|
||||||
// Load KV info on mount
|
|
||||||
useEffect(() => {
|
|
||||||
fetch(`${API_BASE}/info`)
|
|
||||||
.then(res => res.json())
|
|
||||||
.then(data => setInfo(data))
|
|
||||||
.catch(err => console.error('Failed to load KV info:', err));
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Load initial keys
|
|
||||||
const loadKeys = useCallback(async (reset = false) => {
|
|
||||||
if (loading) return;
|
|
||||||
|
|
||||||
const currentCursor = reset ? '0' : cursor;
|
|
||||||
if (!reset && currentCursor === '0' && keys.length > 0) return;
|
|
||||||
|
|
||||||
setLoading(true);
|
|
||||||
setError(null);
|
|
||||||
|
|
||||||
try {
|
|
||||||
const res = await fetch(`${API_BASE}/keys?cursor=${currentCursor}&count=50`);
|
|
||||||
if (!res.ok) throw new Error('Failed to load keys');
|
|
||||||
|
|
||||||
const data = await res.json();
|
|
||||||
|
|
||||||
const newKeys: KeyValue[] = data.keys.map((key: string) => ({
|
|
||||||
key,
|
|
||||||
expanded: false,
|
|
||||||
loading: false
|
|
||||||
}));
|
|
||||||
|
|
||||||
if (reset) {
|
|
||||||
setKeys(newKeys);
|
|
||||||
} else {
|
|
||||||
setKeys(prev => [...prev, ...newKeys]);
|
|
||||||
}
|
|
||||||
|
|
||||||
setCursor(data.cursor);
|
|
||||||
setHasMore(data.hasMore);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
} finally {
|
|
||||||
setLoading(false);
|
|
||||||
}
|
|
||||||
}, [cursor, keys.length, loading]);
|
|
||||||
|
|
||||||
// Load on mount
|
|
||||||
useEffect(() => {
|
|
||||||
loadKeys(true);
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Toggle key expansion and load value
|
|
||||||
const toggleKey = async (index: number) => {
|
|
||||||
const key = keys[index];
|
|
||||||
|
|
||||||
if (key.expanded) {
|
|
||||||
// Collapse
|
|
||||||
setKeys(prev => prev.map((k, i) =>
|
|
||||||
i === index ? { ...k, expanded: false } : k
|
|
||||||
));
|
|
||||||
return;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Expand and load value if not already loaded
|
|
||||||
if (key.value === undefined) {
|
|
||||||
setKeys(prev => prev.map((k, i) =>
|
|
||||||
i === index ? { ...k, loading: true } : k
|
|
||||||
));
|
|
||||||
|
|
||||||
try {
|
|
||||||
const res = await fetch(`${API_BASE}/key/${encodeURIComponent(key.key)}`);
|
|
||||||
if (!res.ok) throw new Error('Failed to load value');
|
|
||||||
|
|
||||||
const data = await res.json();
|
|
||||||
|
|
||||||
setKeys(prev => prev.map((k, i) =>
|
|
||||||
i === index ? {
|
|
||||||
...k,
|
|
||||||
type: data.type,
|
|
||||||
value: data.value,
|
|
||||||
expanded: true,
|
|
||||||
loading: false
|
|
||||||
} : k
|
|
||||||
));
|
|
||||||
} catch (err) {
|
|
||||||
setKeys(prev => prev.map((k, i) =>
|
|
||||||
i === index ? { ...k, loading: false, value: 'Error loading value' } : k
|
|
||||||
));
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
setKeys(prev => prev.map((k, i) =>
|
|
||||||
i === index ? { ...k, expanded: true } : k
|
|
||||||
));
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const formatValue = (value: any): string => {
|
|
||||||
if (value === null) return 'null';
|
|
||||||
if (value === undefined) return 'undefined';
|
|
||||||
if (typeof value === 'object') {
|
|
||||||
return JSON.stringify(value, null, 2);
|
|
||||||
}
|
|
||||||
return String(value);
|
|
||||||
};
|
|
||||||
|
|
||||||
const formatSize = (bytes: number): string => {
|
|
||||||
if (bytes < 1024) return `${bytes} B`;
|
|
||||||
if (bytes < 1024 * 1024) return `${(bytes / 1024).toFixed(1)} KB`;
|
|
||||||
if (bytes < 1024 * 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`;
|
|
||||||
return `${(bytes / (1024 * 1024 * 1024)).toFixed(2)} GB`;
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="kv-tree">
|
|
||||||
<div className="kv-tree-header">
|
|
||||||
<h3>KV Database Browser</h3>
|
|
||||||
{info && (
|
|
||||||
<div className="kv-tree-info">
|
|
||||||
<span>Keys: {info.keyCount}</span>
|
|
||||||
<span>Files: {formatSize(info.totalSize)}</span>
|
|
||||||
{info.memoryUsage && <span>Memory: {info.memoryUsage}</span>}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
<button
|
|
||||||
className="kv-tree-refresh"
|
|
||||||
onClick={() => loadKeys(true)}
|
|
||||||
disabled={loading}
|
|
||||||
>
|
|
||||||
{loading ? 'Loading...' : 'Refresh'}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{error && <div className="kv-tree-error">{error}</div>}
|
|
||||||
|
|
||||||
<div className="kv-tree-content">
|
|
||||||
{keys.map((item, index) => (
|
|
||||||
<div key={item.key} className="kv-tree-node">
|
|
||||||
<div
|
|
||||||
className={`kv-tree-key ${item.expanded ? 'expanded' : ''}`}
|
|
||||||
onClick={() => toggleKey(index)}
|
|
||||||
>
|
|
||||||
<span className="kv-tree-toggle">
|
|
||||||
{item.loading ? '...' : (item.expanded ? '▼' : '▶')}
|
|
||||||
</span>
|
|
||||||
<span className="kv-tree-key-name">{item.key}</span>
|
|
||||||
{item.type && <span className="kv-tree-type">{item.type}</span>}
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{item.expanded && item.value && (
|
|
||||||
<div className="kv-tree-value">
|
|
||||||
{typeof item.value === 'object' ? (
|
|
||||||
<table className="kv-tree-fields">
|
|
||||||
<tbody>
|
|
||||||
{Object.entries(item.value).map(([field, val]) => (
|
|
||||||
<tr key={field}>
|
|
||||||
<td className="kv-field-name">{field}</td>
|
|
||||||
<td className="kv-field-value">
|
|
||||||
<pre>{formatValue(val)}</pre>
|
|
||||||
</td>
|
|
||||||
</tr>
|
|
||||||
))}
|
|
||||||
</tbody>
|
|
||||||
</table>
|
|
||||||
) : (
|
|
||||||
<pre>{formatValue(item.value)}</pre>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
))}
|
|
||||||
|
|
||||||
{hasMore && (
|
|
||||||
<button
|
|
||||||
className="kv-tree-load-more"
|
|
||||||
onClick={() => loadKeys()}
|
|
||||||
disabled={loading}
|
|
||||||
>
|
|
||||||
{loading ? 'Loading...' : 'Load More'}
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
|
|
||||||
{!loading && keys.length === 0 && (
|
|
||||||
<div className="kv-tree-empty">No keys found</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -8,15 +8,6 @@ for (const lang of iso6393) {
|
|||||||
languagesByCode.set(lang.iso6393, lang);
|
languagesByCode.set(lang.iso6393, lang);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Get display name for a language code
|
|
||||||
export function getLanguageName(code: string): string {
|
|
||||||
const lang = languagesByCode.get(code);
|
|
||||||
if (lang) {
|
|
||||||
return lang.name;
|
|
||||||
}
|
|
||||||
return code.toUpperCase();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Format for display: "English (eng)" or "eng" for unknown
|
// Format for display: "English (eng)" or "eng" for unknown
|
||||||
export function formatLanguageOption(code: string): string {
|
export function formatLanguageOption(code: string): string {
|
||||||
const lang = languagesByCode.get(code);
|
const lang = languagesByCode.get(code);
|
||||||
|
|||||||
@@ -1,438 +0,0 @@
|
|||||||
.metadata-editor {
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-header {
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding-bottom: 1rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-header h2 {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions button.primary {
|
|
||||||
background-color: #646cff;
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions button.primary:hover:not(:disabled) {
|
|
||||||
background-color: #535bf2;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions .download-button {
|
|
||||||
display: inline-flex;
|
|
||||||
align-items: center;
|
|
||||||
padding: 0.5rem 1rem;
|
|
||||||
background-color: #28a745;
|
|
||||||
color: white;
|
|
||||||
border-radius: 4px;
|
|
||||||
text-decoration: none;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-weight: 500;
|
|
||||||
transition: background-color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions .download-button:hover {
|
|
||||||
background-color: #218838;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-message {
|
|
||||||
padding: 1rem;
|
|
||||||
border-radius: 4px;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-message.success {
|
|
||||||
background-color: rgba(0, 200, 0, 0.1);
|
|
||||||
border: 1px solid rgba(0, 200, 0, 0.3);
|
|
||||||
color: #00ff00;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-message.error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border: 1px solid rgba(255, 0, 0, 0.3);
|
|
||||||
color: #ff6b6b;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-hashes {
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
padding: 1rem;
|
|
||||||
background: rgba(0, 0, 0, 0.3);
|
|
||||||
border-radius: 8px;
|
|
||||||
border-left: 4px solid #888;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hashes-header {
|
|
||||||
margin-bottom: 0.75rem;
|
|
||||||
padding-bottom: 0.5rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hashes-header h3 {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.9rem;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.05em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hashes-grid {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.15rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value-primary {
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
padding-bottom: 0.5rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value-primary .hash-label {
|
|
||||||
color: #4ecdc4;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value-primary .hash-code {
|
|
||||||
color: #4ecdc4;
|
|
||||||
font-size: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-label {
|
|
||||||
font-size: 0.75rem;
|
|
||||||
color: rgba(255, 255, 255, 0.4);
|
|
||||||
text-transform: uppercase;
|
|
||||||
letter-spacing: 0.03em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-code {
|
|
||||||
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.6);
|
|
||||||
word-break: break-all;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: color 0.2s;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-code:hover {
|
|
||||||
color: #fff;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Plugin Chips */
|
|
||||||
.metadata-editor-chips {
|
|
||||||
display: flex;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
gap: 0.5rem;
|
|
||||||
margin-bottom: 1.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-chip {
|
|
||||||
padding: 0.4rem 0.8rem;
|
|
||||||
border-radius: 16px;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
font-weight: 500;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: all 0.2s ease;
|
|
||||||
border: 2px solid var(--chip-color);
|
|
||||||
color: var(--chip-color);
|
|
||||||
background: transparent;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-chip:hover {
|
|
||||||
background-color: var(--chip-color);
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-chip.active {
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Plugin Accordion */
|
|
||||||
.plugin-accordion {
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
border-radius: 8px;
|
|
||||||
overflow: hidden;
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-header {
|
|
||||||
width: 100%;
|
|
||||||
display: flex;
|
|
||||||
justify-content: space-between;
|
|
||||||
align-items: center;
|
|
||||||
padding: 1rem 1.25rem;
|
|
||||||
border: none;
|
|
||||||
border-left: 4px solid #646cff;
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
color: inherit;
|
|
||||||
font-size: 1rem;
|
|
||||||
font-weight: 500;
|
|
||||||
cursor: pointer;
|
|
||||||
transition: background-color 0.2s ease;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-header:hover {
|
|
||||||
background-color: rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion.expanded .plugin-accordion-header {
|
|
||||||
background-color: rgba(255, 255, 255, 0.08);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-title {
|
|
||||||
color: inherit;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-icon {
|
|
||||||
font-size: 0.75rem;
|
|
||||||
opacity: 0.6;
|
|
||||||
transition: transform 0.2s ease;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-content {
|
|
||||||
padding: 1rem 1.25rem;
|
|
||||||
border-top: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-form {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-section {
|
|
||||||
border: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
border-left: 4px solid #646cff;
|
|
||||||
border-radius: 8px;
|
|
||||||
padding: 1.5rem;
|
|
||||||
background-color: rgba(0, 0, 0, 0.2);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-section-header {
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
padding-bottom: 0.75rem;
|
|
||||||
border-bottom: 1px solid rgba(255, 255, 255, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-section-header h3 {
|
|
||||||
margin: 0 0 0.25rem 0;
|
|
||||||
font-size: 1.1rem;
|
|
||||||
color: #646cff;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-description {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field {
|
|
||||||
display: flex;
|
|
||||||
flex-direction: column;
|
|
||||||
gap: 0.5rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field:last-child {
|
|
||||||
margin-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field label {
|
|
||||||
font-size: 0.9rem;
|
|
||||||
font-weight: 500;
|
|
||||||
color: rgba(255, 255, 255, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field input,
|
|
||||||
.metadata-field textarea,
|
|
||||||
.metadata-field select {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field-row {
|
|
||||||
display: grid;
|
|
||||||
grid-template-columns: repeat(auto-fit, minmax(200px, 1fr));
|
|
||||||
gap: 1rem;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field-row:last-child {
|
|
||||||
margin-bottom: 0;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-field {
|
|
||||||
font-family: 'Courier New', monospace;
|
|
||||||
font-size: 0.9em;
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-hash-info {
|
|
||||||
margin: 0;
|
|
||||||
padding: 1rem;
|
|
||||||
text-align: center;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-loading,
|
|
||||||
.metadata-editor-error,
|
|
||||||
.metadata-editor-empty {
|
|
||||||
padding: 3rem;
|
|
||||||
text-align: center;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-error button {
|
|
||||||
margin-top: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 768px) {
|
|
||||||
.metadata-editor-header {
|
|
||||||
flex-direction: column;
|
|
||||||
align-items: flex-start;
|
|
||||||
gap: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-actions button {
|
|
||||||
flex: 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field-row {
|
|
||||||
grid-template-columns: 1fr;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.metadata-editor {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-section {
|
|
||||||
border-color: rgba(0, 0, 0, 0.1);
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-section-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-description {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-field label {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-message.success {
|
|
||||||
background-color: rgba(0, 200, 0, 0.1);
|
|
||||||
border-color: rgba(0, 200, 0, 0.3);
|
|
||||||
color: #008800;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-message.error {
|
|
||||||
background-color: rgba(255, 0, 0, 0.1);
|
|
||||||
border-color: rgba(255, 0, 0, 0.3);
|
|
||||||
color: #cc0000;
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-loading,
|
|
||||||
.metadata-editor-error,
|
|
||||||
.metadata-editor-empty {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.no-hash-info {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
|
|
||||||
.metadata-editor-hashes {
|
|
||||||
background: rgba(0, 0, 0, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hashes-header {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hashes-header h3 {
|
|
||||||
color: rgba(0, 0, 0, 0.7);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value-primary {
|
|
||||||
border-bottom-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-value-primary .hash-label,
|
|
||||||
.hash-value-primary .hash-code {
|
|
||||||
color: #0a9e94;
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-label {
|
|
||||||
color: rgba(0, 0, 0, 0.4);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-code {
|
|
||||||
color: rgba(0, 0, 0, 0.6);
|
|
||||||
}
|
|
||||||
|
|
||||||
.hash-code:hover {
|
|
||||||
color: #000;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Plugin Chips - Light theme */
|
|
||||||
.plugin-chip {
|
|
||||||
background: transparent;
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-chip:hover,
|
|
||||||
.plugin-chip.active {
|
|
||||||
color: white;
|
|
||||||
}
|
|
||||||
|
|
||||||
/* Plugin Accordion - Light theme */
|
|
||||||
.plugin-accordion {
|
|
||||||
background-color: rgba(0, 0, 0, 0.02);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-header {
|
|
||||||
background-color: rgba(0, 0, 0, 0.03);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-header:hover {
|
|
||||||
background-color: rgba(0, 0, 0, 0.06);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion.expanded .plugin-accordion-header {
|
|
||||||
background-color: rgba(0, 0, 0, 0.05);
|
|
||||||
}
|
|
||||||
|
|
||||||
.plugin-accordion-content {
|
|
||||||
border-top-color: rgba(0, 0, 0, 0.1);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,392 +0,0 @@
|
|||||||
import { useState, useEffect } from 'react';
|
|
||||||
import { FileMetadata, PluginSchema, PluginsResponse } from '../types';
|
|
||||||
import { MetadataAPI } from '../api/metadataApi';
|
|
||||||
import { DynamicPluginSection } from './DynamicPluginSection';
|
|
||||||
import './MetadataEditor.css';
|
|
||||||
|
|
||||||
interface PluginAccordionProps {
|
|
||||||
plugin: PluginSchema;
|
|
||||||
metadata: FileMetadata;
|
|
||||||
onChange: (field: string, value: any) => void;
|
|
||||||
isExpanded: boolean;
|
|
||||||
onToggle: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
function PluginAccordion({ plugin, metadata, onChange, isExpanded, onToggle }: PluginAccordionProps) {
|
|
||||||
return (
|
|
||||||
<div className={`plugin-accordion ${isExpanded ? 'expanded' : ''}`}>
|
|
||||||
<button
|
|
||||||
className="plugin-accordion-header"
|
|
||||||
onClick={onToggle}
|
|
||||||
style={{ borderLeftColor: plugin.color || '#646cff' }}
|
|
||||||
>
|
|
||||||
<span className="plugin-accordion-title">{plugin.name}</span>
|
|
||||||
<span className="plugin-accordion-icon">{isExpanded ? '▼' : '▶'}</span>
|
|
||||||
</button>
|
|
||||||
{isExpanded && (
|
|
||||||
<div className="plugin-accordion-content">
|
|
||||||
<DynamicPluginSection
|
|
||||||
plugin={plugin}
|
|
||||||
metadata={metadata}
|
|
||||||
onChange={onChange}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
interface MetadataEditorProps {
|
|
||||||
hashId: string;
|
|
||||||
onSaved?: () => void;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function MetadataEditor({ hashId, onSaved }: MetadataEditorProps) {
|
|
||||||
const [metadata, setMetadata] = useState<FileMetadata | null>(null);
|
|
||||||
const [editedMetadata, setEditedMetadata] = useState<FileMetadata | null>(null);
|
|
||||||
const [plugins, setPlugins] = useState<PluginSchema[]>([]);
|
|
||||||
const [isLoading, setIsLoading] = useState(true);
|
|
||||||
const [isSaving, setIsSaving] = useState(false);
|
|
||||||
const [error, setError] = useState<string | null>(null);
|
|
||||||
const [successMessage, setSuccessMessage] = useState<string | null>(null);
|
|
||||||
const [expandedPlugins, setExpandedPlugins] = useState<Set<string>>(new Set());
|
|
||||||
|
|
||||||
// Fetch plugins schema on mount
|
|
||||||
useEffect(() => {
|
|
||||||
fetchPlugins();
|
|
||||||
}, []);
|
|
||||||
|
|
||||||
// Fetch metadata when hashId changes
|
|
||||||
useEffect(() => {
|
|
||||||
loadMetadata();
|
|
||||||
}, [hashId]);
|
|
||||||
|
|
||||||
const fetchPlugins = async () => {
|
|
||||||
try {
|
|
||||||
const response = await fetch('/api/plugins');
|
|
||||||
if (response.ok) {
|
|
||||||
const data: PluginsResponse = await response.json();
|
|
||||||
// Sort by priority (lower = higher priority)
|
|
||||||
const sorted = [...data.plugins].sort((a, b) =>
|
|
||||||
(a.priority ?? 100) - (b.priority ?? 100)
|
|
||||||
);
|
|
||||||
setPlugins(sorted);
|
|
||||||
}
|
|
||||||
} catch (err) {
|
|
||||||
console.error('Failed to fetch plugin schemas:', err);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const loadMetadata = async () => {
|
|
||||||
try {
|
|
||||||
setIsLoading(true);
|
|
||||||
setError(null);
|
|
||||||
const data = await MetadataAPI.getMetadata(hashId);
|
|
||||||
setMetadata(data);
|
|
||||||
setEditedMetadata(data);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
} finally {
|
|
||||||
setIsLoading(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleFieldChange = (field: string, value: any) => {
|
|
||||||
if (!editedMetadata) return;
|
|
||||||
|
|
||||||
// Handle path-style keys like "stream/duration" or "nfo/title"
|
|
||||||
const parts = field.split('/');
|
|
||||||
|
|
||||||
if (parts.length === 1) {
|
|
||||||
// Simple key
|
|
||||||
setEditedMetadata({
|
|
||||||
...editedMetadata,
|
|
||||||
[field]: value,
|
|
||||||
});
|
|
||||||
} else {
|
|
||||||
// Nested key - need to update nested structure
|
|
||||||
const newMetadata = { ...editedMetadata };
|
|
||||||
let current: any = newMetadata;
|
|
||||||
|
|
||||||
for (let i = 0; i < parts.length - 1; i++) {
|
|
||||||
if (!current[parts[i]]) {
|
|
||||||
current[parts[i]] = {};
|
|
||||||
} else {
|
|
||||||
current[parts[i]] = { ...current[parts[i]] };
|
|
||||||
}
|
|
||||||
current = current[parts[i]];
|
|
||||||
}
|
|
||||||
|
|
||||||
current[parts[parts.length - 1]] = value;
|
|
||||||
setEditedMetadata(newMetadata);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleSave = async () => {
|
|
||||||
if (!editedMetadata) return;
|
|
||||||
|
|
||||||
try {
|
|
||||||
setIsSaving(true);
|
|
||||||
setError(null);
|
|
||||||
setSuccessMessage(null);
|
|
||||||
|
|
||||||
await MetadataAPI.updateMetadata(hashId, editedMetadata);
|
|
||||||
setMetadata(editedMetadata);
|
|
||||||
setSuccessMessage('Metadata saved successfully!');
|
|
||||||
|
|
||||||
if (onSaved) {
|
|
||||||
onSaved();
|
|
||||||
}
|
|
||||||
|
|
||||||
// Clear success message after 3 seconds
|
|
||||||
setTimeout(() => setSuccessMessage(null), 3000);
|
|
||||||
} catch (err: any) {
|
|
||||||
setError(err.message);
|
|
||||||
} finally {
|
|
||||||
setIsSaving(false);
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
const handleReset = () => {
|
|
||||||
setEditedMetadata(metadata);
|
|
||||||
setError(null);
|
|
||||||
setSuccessMessage(null);
|
|
||||||
};
|
|
||||||
|
|
||||||
const togglePlugin = (pluginId: string) => {
|
|
||||||
setExpandedPlugins(prev => {
|
|
||||||
const next = new Set(prev);
|
|
||||||
if (next.has(pluginId)) {
|
|
||||||
next.delete(pluginId);
|
|
||||||
} else {
|
|
||||||
next.add(pluginId);
|
|
||||||
}
|
|
||||||
return next;
|
|
||||||
});
|
|
||||||
};
|
|
||||||
|
|
||||||
const scrollToPlugin = (pluginId: string) => {
|
|
||||||
// Expand the plugin if not already expanded
|
|
||||||
if (!expandedPlugins.has(pluginId)) {
|
|
||||||
setExpandedPlugins(prev => new Set(prev).add(pluginId));
|
|
||||||
}
|
|
||||||
// Scroll to the plugin after a short delay to allow expansion
|
|
||||||
setTimeout(() => {
|
|
||||||
const element = document.getElementById(`plugin-${pluginId}`);
|
|
||||||
if (element) {
|
|
||||||
element.scrollIntoView({ behavior: 'smooth', block: 'start' });
|
|
||||||
}
|
|
||||||
}, 100);
|
|
||||||
};
|
|
||||||
|
|
||||||
if (isLoading) {
|
|
||||||
return <div className="metadata-editor-loading">Loading metadata...</div>;
|
|
||||||
}
|
|
||||||
|
|
||||||
if (error && !metadata) {
|
|
||||||
return (
|
|
||||||
<div className="metadata-editor-error">
|
|
||||||
<p>Error loading metadata: {error}</p>
|
|
||||||
<button onClick={loadMetadata}>Retry</button>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
if (!metadata || !editedMetadata) {
|
|
||||||
return <div className="metadata-editor-empty">No metadata available</div>;
|
|
||||||
}
|
|
||||||
|
|
||||||
const hasChanges = JSON.stringify(metadata) !== JSON.stringify(editedMetadata);
|
|
||||||
|
|
||||||
// Filter plugins that have data
|
|
||||||
const pluginsWithData = plugins.filter(plugin => {
|
|
||||||
if (!plugin.metadataSchema) return false;
|
|
||||||
return hasPluginData(plugin, editedMetadata);
|
|
||||||
});
|
|
||||||
|
|
||||||
// Get file path for download
|
|
||||||
const filePath = editedMetadata?.filePath as string | undefined;
|
|
||||||
const downloadUrl = filePath ? `/api/file/download?path=${encodeURIComponent(filePath)}` : null;
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className="metadata-editor">
|
|
||||||
<div className="metadata-editor-header">
|
|
||||||
<h2>Edit Metadata</h2>
|
|
||||||
<div className="metadata-editor-actions">
|
|
||||||
{downloadUrl && (
|
|
||||||
<a
|
|
||||||
href={downloadUrl}
|
|
||||||
className="download-button"
|
|
||||||
title={`Download: ${filePath}`}
|
|
||||||
download
|
|
||||||
>
|
|
||||||
⬇ Download
|
|
||||||
</a>
|
|
||||||
)}
|
|
||||||
{hasChanges && (
|
|
||||||
<button onClick={handleReset} disabled={isSaving}>
|
|
||||||
Reset
|
|
||||||
</button>
|
|
||||||
)}
|
|
||||||
<button
|
|
||||||
onClick={handleSave}
|
|
||||||
disabled={isSaving || !hasChanges}
|
|
||||||
className="primary"
|
|
||||||
>
|
|
||||||
{isSaving ? 'Saving...' : 'Save Changes'}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{successMessage && (
|
|
||||||
<div className="metadata-editor-message success">{successMessage}</div>
|
|
||||||
)}
|
|
||||||
{error && <div className="metadata-editor-message error">{error}</div>}
|
|
||||||
|
|
||||||
<div className="metadata-editor-hashes">
|
|
||||||
<div className="hashes-header">
|
|
||||||
<h3>Content Identifiers</h3>
|
|
||||||
</div>
|
|
||||||
<div className="hashes-grid">
|
|
||||||
<HashValue label="midhash256" value={hashId} primary />
|
|
||||||
{getHashValues(editedMetadata).map(({ key, label, value }) => (
|
|
||||||
<HashValue key={key} label={label} value={value} />
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
|
|
||||||
{pluginsWithData.length > 0 && (
|
|
||||||
<div className="metadata-editor-chips">
|
|
||||||
{pluginsWithData.map((plugin) => (
|
|
||||||
<button
|
|
||||||
key={plugin.id}
|
|
||||||
className={`plugin-chip ${expandedPlugins.has(plugin.id) ? 'active' : ''}`}
|
|
||||||
style={{
|
|
||||||
'--chip-color': plugin.color || '#646cff',
|
|
||||||
backgroundColor: expandedPlugins.has(plugin.id) ? plugin.color || '#646cff' : 'transparent'
|
|
||||||
} as React.CSSProperties}
|
|
||||||
onClick={() => scrollToPlugin(plugin.id)}
|
|
||||||
>
|
|
||||||
{plugin.name}
|
|
||||||
</button>
|
|
||||||
))}
|
|
||||||
</div>
|
|
||||||
)}
|
|
||||||
|
|
||||||
<div className="metadata-editor-form">
|
|
||||||
{pluginsWithData.length === 0 ? (
|
|
||||||
<div className="metadata-editor-empty">
|
|
||||||
No metadata found for this file
|
|
||||||
</div>
|
|
||||||
) : (
|
|
||||||
pluginsWithData.map((plugin) => (
|
|
||||||
<div key={plugin.id} id={`plugin-${plugin.id}`}>
|
|
||||||
<PluginAccordion
|
|
||||||
plugin={plugin}
|
|
||||||
metadata={editedMetadata}
|
|
||||||
onChange={handleFieldChange}
|
|
||||||
isExpanded={expandedPlugins.has(plugin.id)}
|
|
||||||
onToggle={() => togglePlugin(plugin.id)}
|
|
||||||
/>
|
|
||||||
</div>
|
|
||||||
))
|
|
||||||
)}
|
|
||||||
</div>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Check if a plugin has any data in the metadata
|
|
||||||
*/
|
|
||||||
function hasPluginData(plugin: PluginSchema, metadata: FileMetadata): boolean {
|
|
||||||
const schema = plugin.metadataSchema;
|
|
||||||
if (!schema) return false;
|
|
||||||
|
|
||||||
for (const fieldKey of Object.keys(schema)) {
|
|
||||||
const fieldSchema = schema[fieldKey];
|
|
||||||
|
|
||||||
// For JSON type with pattern, check if any matching keys exist
|
|
||||||
if (fieldSchema.type === 'json' && fieldSchema.pattern) {
|
|
||||||
const patternPrefix = fieldSchema.pattern.replace(/\{n\}.*$/, '').replace(/\/$/, '');
|
|
||||||
if (hasKeyWithPrefix(metadata, patternPrefix)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
continue;
|
|
||||||
}
|
|
||||||
|
|
||||||
// For regular fields, check if value exists
|
|
||||||
if (getNestedValue(metadata, fieldKey) !== undefined) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
function getNestedValue(obj: any, path: string): any {
|
|
||||||
const parts = path.split('/');
|
|
||||||
let value = obj;
|
|
||||||
for (const part of parts) {
|
|
||||||
if (value === undefined || value === null) return undefined;
|
|
||||||
value = value[part];
|
|
||||||
}
|
|
||||||
return value;
|
|
||||||
}
|
|
||||||
|
|
||||||
function hasKeyWithPrefix(obj: any, prefix: string, currentPath: string = ''): boolean {
|
|
||||||
if (!obj || typeof obj !== 'object') return false;
|
|
||||||
|
|
||||||
for (const key of Object.keys(obj)) {
|
|
||||||
const newPath = currentPath ? `${currentPath}/${key}` : key;
|
|
||||||
if (newPath.startsWith(prefix)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
if (typeof obj[key] === 'object' && obj[key] !== null) {
|
|
||||||
if (hasKeyWithPrefix(obj[key], prefix, newPath)) {
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Extract all hash values (cid_*) from metadata
|
|
||||||
*/
|
|
||||||
function getHashValues(metadata: FileMetadata): { key: string; label: string; value: string }[] {
|
|
||||||
const hashOrder = [
|
|
||||||
{ key: 'cid_sha2-256', label: 'SHA-256' },
|
|
||||||
{ key: 'cid_sha3-256', label: 'SHA3-256' },
|
|
||||||
{ key: 'cid_sha3-384', label: 'SHA3-384' },
|
|
||||||
{ key: 'cid_sha1', label: 'SHA-1' },
|
|
||||||
{ key: 'cid_md5', label: 'MD5' },
|
|
||||||
{ key: 'cid_crc32', label: 'CRC32' },
|
|
||||||
{ key: 'cid_btih_v2', label: 'BitTorrent v2' },
|
|
||||||
];
|
|
||||||
|
|
||||||
return hashOrder
|
|
||||||
.filter(({ key }) => (metadata as any)[key])
|
|
||||||
.map(({ key, label }) => ({
|
|
||||||
key,
|
|
||||||
label,
|
|
||||||
value: (metadata as any)[key],
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Display a single hash value with copy functionality
|
|
||||||
*/
|
|
||||||
function HashValue({ label, value, primary }: { label: string; value: string; primary?: boolean }) {
|
|
||||||
const handleCopy = () => {
|
|
||||||
navigator.clipboard.writeText(value);
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<div className={`hash-value ${primary ? 'hash-value-primary' : ''}`}>
|
|
||||||
<span className="hash-label">{label}</span>
|
|
||||||
<code className="hash-code" onClick={handleCopy} title="Click to copy">
|
|
||||||
{value}
|
|
||||||
</code>
|
|
||||||
</div>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -1,52 +0,0 @@
|
|||||||
.search-bar {
|
|
||||||
width: 100%;
|
|
||||||
padding: 1.5rem;
|
|
||||||
background-color: rgba(255, 255, 255, 0.05);
|
|
||||||
border-radius: 8px;
|
|
||||||
margin-bottom: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-controls {
|
|
||||||
display: flex;
|
|
||||||
gap: 0.5rem;
|
|
||||||
flex-wrap: wrap;
|
|
||||||
margin-bottom: 0.5rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-input {
|
|
||||||
flex: 1;
|
|
||||||
min-width: 200px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-input.unified-search {
|
|
||||||
font-size: 1rem;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-bar button {
|
|
||||||
flex: 0 0 auto;
|
|
||||||
min-width: 100px;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-hint {
|
|
||||||
margin: 0;
|
|
||||||
font-size: 0.85rem;
|
|
||||||
color: rgba(255, 255, 255, 0.5);
|
|
||||||
font-style: italic;
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (max-width: 768px) {
|
|
||||||
.search-controls {
|
|
||||||
flex-direction: column;
|
|
||||||
}
|
|
||||||
|
|
||||||
.search-input,
|
|
||||||
.search-bar button {
|
|
||||||
width: 100%;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
@media (prefers-color-scheme: light) {
|
|
||||||
.search-hint {
|
|
||||||
color: rgba(0, 0, 0, 0.5);
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,40 +0,0 @@
|
|||||||
import { useState } from 'react';
|
|
||||||
import './SearchBar.css';
|
|
||||||
|
|
||||||
interface SearchBarProps {
|
|
||||||
onSearch: (query: string) => void;
|
|
||||||
isLoading: boolean;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function SearchBar({ onSearch, isLoading }: SearchBarProps) {
|
|
||||||
const [query, setQuery] = useState('');
|
|
||||||
|
|
||||||
const handleSubmit = (e: React.FormEvent) => {
|
|
||||||
e.preventDefault();
|
|
||||||
if (query.trim()) {
|
|
||||||
onSearch(query.trim());
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
return (
|
|
||||||
<form className="search-bar" onSubmit={handleSubmit}>
|
|
||||||
<div className="search-controls">
|
|
||||||
<input
|
|
||||||
type="text"
|
|
||||||
value={query}
|
|
||||||
onChange={(e) => setQuery(e.target.value)}
|
|
||||||
placeholder="Search by title, filename, filepath, or hash ID..."
|
|
||||||
className="search-input unified-search"
|
|
||||||
disabled={isLoading}
|
|
||||||
/>
|
|
||||||
|
|
||||||
<button type="submit" disabled={isLoading || !query.trim()}>
|
|
||||||
{isLoading ? 'Searching...' : 'Search'}
|
|
||||||
</button>
|
|
||||||
</div>
|
|
||||||
<p className="search-hint">
|
|
||||||
Search across all fields: title, filename, file path, and hash ID
|
|
||||||
</p>
|
|
||||||
</form>
|
|
||||||
);
|
|
||||||
}
|
|
||||||
@@ -0,0 +1,209 @@
|
|||||||
|
.snapshot-panel {
|
||||||
|
max-width: 920px;
|
||||||
|
margin: 0 auto;
|
||||||
|
padding: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-panel h2 {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-intro {
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
margin-bottom: 1.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 1rem 1.25rem;
|
||||||
|
margin-bottom: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section.danger {
|
||||||
|
border-color: #d04a4a;
|
||||||
|
background: rgba(208, 74, 74, 0.04);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-section h3 {
|
||||||
|
margin: 0 0 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-row {
|
||||||
|
display: flex;
|
||||||
|
gap: 1rem;
|
||||||
|
align-items: flex-start;
|
||||||
|
margin: 0.75rem 0;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-file {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-file span {
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
min-width: 280px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset[disabled] {
|
||||||
|
opacity: 0.5;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset legend {
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
padding: 0 0.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset label {
|
||||||
|
display: block;
|
||||||
|
margin: 0.25rem 0;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-fieldset input[type='radio'] {
|
||||||
|
margin-right: 0.5rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn {
|
||||||
|
background: var(--bg-button, #f0f0f0);
|
||||||
|
border: 1px solid var(--border, #c0c0c0);
|
||||||
|
color: var(--text, #222);
|
||||||
|
padding: 0.5rem 1rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn:hover:not(:disabled) {
|
||||||
|
background: var(--bg-button-hover, #e0e0e0);
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn:disabled {
|
||||||
|
opacity: 0.5;
|
||||||
|
cursor: not-allowed;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.primary {
|
||||||
|
background: #2a6df4;
|
||||||
|
border-color: #2a6df4;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.primary:hover:not(:disabled) {
|
||||||
|
background: #1a5de4;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.danger {
|
||||||
|
background: #d04a4a;
|
||||||
|
border-color: #d04a4a;
|
||||||
|
color: white;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-btn.danger:hover:not(:disabled) {
|
||||||
|
background: #b83838;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-error {
|
||||||
|
background: rgba(208, 74, 74, 0.1);
|
||||||
|
color: #b83838;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-info {
|
||||||
|
background: rgba(42, 109, 244, 0.08);
|
||||||
|
color: #1a5de4;
|
||||||
|
padding: 0.5rem 0.75rem;
|
||||||
|
border-radius: 4px;
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result {
|
||||||
|
margin-top: 0.75rem;
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
border-radius: 4px;
|
||||||
|
padding: 0.75rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-summary {
|
||||||
|
display: flex;
|
||||||
|
gap: 1rem;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
margin-bottom: 0.5rem;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table {
|
||||||
|
width: 100%;
|
||||||
|
border-collapse: collapse;
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table th,
|
||||||
|
.snapshot-result-table td {
|
||||||
|
border: 1px solid var(--border, #e0e0e0);
|
||||||
|
padding: 0.35rem 0.5rem;
|
||||||
|
text-align: right;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-table th {
|
||||||
|
background: var(--bg-subtle, #f5f5f5);
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors {
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors summary {
|
||||||
|
cursor: pointer;
|
||||||
|
color: #b83838;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors ul {
|
||||||
|
margin: 0.5rem 0 0;
|
||||||
|
padding-left: 1.25rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-errors code {
|
||||||
|
font-size: 0.8rem;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-checkbox {
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
gap: 0.5rem;
|
||||||
|
margin: 0.5rem 0 0.75rem;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 0.95rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-checkbox input {
|
||||||
|
margin-top: 0.2rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-hint {
|
||||||
|
color: var(--text-muted, #888);
|
||||||
|
font-size: 0.9rem;
|
||||||
|
}
|
||||||
|
|
||||||
|
.snapshot-result-aside {
|
||||||
|
margin-top: 0.5rem;
|
||||||
|
font-size: 0.85rem;
|
||||||
|
color: var(--text-muted, #555);
|
||||||
|
}
|
||||||
@@ -0,0 +1,271 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
import {
|
||||||
|
SnapshotAPI,
|
||||||
|
ImportMode,
|
||||||
|
ImportConflict,
|
||||||
|
ImportResult,
|
||||||
|
} from '../api/snapshotApi';
|
||||||
|
import './SnapshotPanel.css';
|
||||||
|
|
||||||
|
export function SnapshotPanel() {
|
||||||
|
const [exportPluginOutput, setExportPluginOutput] = useState(false);
|
||||||
|
const [importFile, setImportFile] = useState<File | null>(null);
|
||||||
|
const [importMode, setImportMode] = useState<ImportMode>('merge');
|
||||||
|
const [importConflict, setImportConflict] = useState<ImportConflict>('source');
|
||||||
|
const [importing, setImporting] = useState(false);
|
||||||
|
const [importResult, setImportResult] = useState<ImportResult | null>(null);
|
||||||
|
const [importError, setImportError] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const [wipeBusy, setWipeBusy] = useState(false);
|
||||||
|
const [wipeMessage, setWipeMessage] = useState<string | null>(null);
|
||||||
|
|
||||||
|
const runImport = async (dryRun: boolean) => {
|
||||||
|
if (!importFile) {
|
||||||
|
setImportError('Pick a snapshot ZIP first.');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setImporting(true);
|
||||||
|
setImportError(null);
|
||||||
|
setImportResult(null);
|
||||||
|
try {
|
||||||
|
const result = await SnapshotAPI.runImport(importFile, {
|
||||||
|
mode: importMode,
|
||||||
|
conflict: importMode === 'merge' ? importConflict : undefined,
|
||||||
|
dryRun,
|
||||||
|
});
|
||||||
|
setImportResult(result);
|
||||||
|
} catch (e: any) {
|
||||||
|
setImportError(e.message || String(e));
|
||||||
|
} finally {
|
||||||
|
setImporting(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const runWipe = async () => {
|
||||||
|
const ok = window.confirm(
|
||||||
|
'This deletes ALL metadata in Redis. Files on disk are not touched, but every manual ' +
|
||||||
|
'edit you made via this editor will be lost. Continue?'
|
||||||
|
);
|
||||||
|
if (!ok) return;
|
||||||
|
setWipeBusy(true);
|
||||||
|
setWipeMessage(null);
|
||||||
|
try {
|
||||||
|
const r = await SnapshotAPI.wipe('metadata');
|
||||||
|
setWipeMessage(`Deleted ${r.metadataDeleted} file metadata entries.`);
|
||||||
|
} catch (e: any) {
|
||||||
|
setWipeMessage(`Failed: ${e.message || e}`);
|
||||||
|
} finally {
|
||||||
|
setWipeBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="snapshot-panel">
|
||||||
|
<h2>Snapshot</h2>
|
||||||
|
<p className="snapshot-intro">
|
||||||
|
Export, import, or wipe the metadata KV store. Snapshots are portable ZIP
|
||||||
|
archives — one nested-JSON file per CID, plus a manifest.
|
||||||
|
</p>
|
||||||
|
|
||||||
|
<section className="snapshot-section">
|
||||||
|
<h3>Export</h3>
|
||||||
|
<p>Downloads every CID's metadata as a ZIP. Optional cache files can be added below.</p>
|
||||||
|
<label className="snapshot-checkbox">
|
||||||
|
<input
|
||||||
|
type="checkbox"
|
||||||
|
checked={exportPluginOutput}
|
||||||
|
onChange={(e) => setExportPluginOutput(e.target.checked)}
|
||||||
|
/>
|
||||||
|
<span>
|
||||||
|
Include plugin output
|
||||||
|
<span className="snapshot-hint">
|
||||||
|
{' '}— TMDB posters/backdrops, extracted subtitles. Adds ~120 MB on a populated stack.
|
||||||
|
</span>
|
||||||
|
</span>
|
||||||
|
</label>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn primary"
|
||||||
|
onClick={() =>
|
||||||
|
SnapshotAPI.downloadExport({ pluginOutput: exportPluginOutput })
|
||||||
|
}
|
||||||
|
>
|
||||||
|
Download snapshot.zip
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="snapshot-section">
|
||||||
|
<h3>Import</h3>
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<label className="snapshot-file">
|
||||||
|
<span>Snapshot ZIP</span>
|
||||||
|
<input
|
||||||
|
type="file"
|
||||||
|
accept=".zip,application/zip"
|
||||||
|
onChange={(e) => {
|
||||||
|
setImportFile(e.target.files?.[0] ?? null);
|
||||||
|
setImportResult(null);
|
||||||
|
setImportError(null);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</label>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<fieldset className="snapshot-fieldset">
|
||||||
|
<legend>Mode</legend>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="mode"
|
||||||
|
value="merge"
|
||||||
|
checked={importMode === 'merge'}
|
||||||
|
onChange={() => setImportMode('merge')}
|
||||||
|
/>
|
||||||
|
Merge — keep existing fields not in the snapshot
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="mode"
|
||||||
|
value="replace"
|
||||||
|
checked={importMode === 'replace'}
|
||||||
|
onChange={() => setImportMode('replace')}
|
||||||
|
/>
|
||||||
|
Replace — wipe each imported CID and rewrite from snapshot
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
|
||||||
|
<fieldset className="snapshot-fieldset" disabled={importMode !== 'merge'}>
|
||||||
|
<legend>Conflict</legend>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="conflict"
|
||||||
|
value="source"
|
||||||
|
checked={importConflict === 'source'}
|
||||||
|
onChange={() => setImportConflict('source')}
|
||||||
|
/>
|
||||||
|
Source wins (overwrite my edits)
|
||||||
|
</label>
|
||||||
|
<label>
|
||||||
|
<input
|
||||||
|
type="radio"
|
||||||
|
name="conflict"
|
||||||
|
value="mine"
|
||||||
|
checked={importConflict === 'mine'}
|
||||||
|
onChange={() => setImportConflict('mine')}
|
||||||
|
/>
|
||||||
|
Mine wins (keep my edits, fill in only missing fields)
|
||||||
|
</label>
|
||||||
|
</fieldset>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="snapshot-row">
|
||||||
|
<button
|
||||||
|
className="snapshot-btn"
|
||||||
|
disabled={!importFile || importing}
|
||||||
|
onClick={() => runImport(true)}
|
||||||
|
>
|
||||||
|
Dry run
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn primary"
|
||||||
|
disabled={!importFile || importing}
|
||||||
|
onClick={() => runImport(false)}
|
||||||
|
>
|
||||||
|
{importing ? 'Importing…' : 'Apply import'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{importError && <div className="snapshot-error">{importError}</div>}
|
||||||
|
{importResult && <ImportResultView result={importResult} />}
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="snapshot-section danger">
|
||||||
|
<h3>Wipe metadata</h3>
|
||||||
|
<p>
|
||||||
|
Deletes every <code>file:*</code> key plus <code>file:__index__</code>.
|
||||||
|
Files on disk are untouched.
|
||||||
|
</p>
|
||||||
|
<button
|
||||||
|
className="snapshot-btn danger"
|
||||||
|
disabled={wipeBusy}
|
||||||
|
onClick={runWipe}
|
||||||
|
>
|
||||||
|
{wipeBusy ? 'Wiping…' : 'Wipe all metadata'}
|
||||||
|
</button>
|
||||||
|
{wipeMessage && <div className="snapshot-info">{wipeMessage}</div>}
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function ImportResultView({ result }: { result: ImportResult }) {
|
||||||
|
const totals = (result.files ?? []).reduce(
|
||||||
|
(acc, f) => {
|
||||||
|
acc.added += f.added;
|
||||||
|
acc.updated += f.updated;
|
||||||
|
acc.unchanged += f.unchanged;
|
||||||
|
acc.kept += f.kept;
|
||||||
|
acc.deleted += f.deleted;
|
||||||
|
return acc;
|
||||||
|
},
|
||||||
|
{ added: 0, updated: 0, unchanged: 0, kept: 0, deleted: 0 }
|
||||||
|
);
|
||||||
|
return (
|
||||||
|
<div className="snapshot-result">
|
||||||
|
<div className="snapshot-result-summary">
|
||||||
|
<strong>{result.dryRun ? 'Dry run preview' : 'Import applied'}</strong>
|
||||||
|
<span>
|
||||||
|
{result.filesOk} ok / {result.filesFailed} failed of {result.totalFiles} files
|
||||||
|
</span>
|
||||||
|
<span>
|
||||||
|
mode={result.mode}
|
||||||
|
{result.conflict ? `, conflict=${result.conflict}` : ''}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
<table className="snapshot-result-table">
|
||||||
|
<thead>
|
||||||
|
<tr>
|
||||||
|
<th>added</th>
|
||||||
|
<th>updated</th>
|
||||||
|
<th>unchanged</th>
|
||||||
|
<th>kept (mine)</th>
|
||||||
|
<th>deleted (replace)</th>
|
||||||
|
</tr>
|
||||||
|
</thead>
|
||||||
|
<tbody>
|
||||||
|
<tr>
|
||||||
|
<td>{totals.added}</td>
|
||||||
|
<td>{totals.updated}</td>
|
||||||
|
<td>{totals.unchanged}</td>
|
||||||
|
<td>{totals.kept}</td>
|
||||||
|
<td>{totals.deleted}</td>
|
||||||
|
</tr>
|
||||||
|
</tbody>
|
||||||
|
</table>
|
||||||
|
{result.pluginOutput && (
|
||||||
|
<div className="snapshot-result-aside">
|
||||||
|
plugin-output: {result.pluginOutput.written} written
|
||||||
|
{result.pluginOutput.skipped > 0
|
||||||
|
? `, ${result.pluginOutput.skipped} skipped (existed)`
|
||||||
|
: ''}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
{result.files && result.files.some((f) => f.error) && (
|
||||||
|
<details className="snapshot-result-errors">
|
||||||
|
<summary>Per-file errors</summary>
|
||||||
|
<ul>
|
||||||
|
{result.files
|
||||||
|
.filter((f) => f.error)
|
||||||
|
.map((f) => (
|
||||||
|
<li key={f.hashId}>
|
||||||
|
<code>{f.hashId}</code>: {f.error}
|
||||||
|
</li>
|
||||||
|
))}
|
||||||
|
</ul>
|
||||||
|
</details>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,233 @@
|
|||||||
|
import { useEffect, useMemo, useState } from 'react';
|
||||||
|
import { KVAPI, KVFindMatch } from '../api/kvApi';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
matches: KVFindMatch[];
|
||||||
|
truncated: boolean;
|
||||||
|
searching: boolean;
|
||||||
|
selectedKey: string | null;
|
||||||
|
expandedPaths: Set<string>;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
onToggleBranch: (path: string, willOpen: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// FilteredKVTree is the search-active counterpart to KVTree. Instead of
|
||||||
|
// replacing the tree with a flat results list, it shows only the file
|
||||||
|
// branches that contain matches, with the matched (field, value) pairs
|
||||||
|
// rendered inline as sub-lines under each branch.
|
||||||
|
//
|
||||||
|
// Branches remain expandable: clicking one fetches the full subtree via
|
||||||
|
// KVAPI.treeAll just like the main tree, so the user can drill into every
|
||||||
|
// key of a matching file without clearing the search first.
|
||||||
|
export function FilteredKVTree({
|
||||||
|
matches, truncated, searching,
|
||||||
|
selectedKey, expandedPaths,
|
||||||
|
onSelectLeaf, onSelectBranch, onToggleBranch,
|
||||||
|
}: Props) {
|
||||||
|
// Group matches by their entryPath (file:<hash>/) so each entry is one row
|
||||||
|
// with potentially multiple hit previews.
|
||||||
|
const groups = useMemo(() => {
|
||||||
|
const m = new Map<string, KVFindMatch[]>();
|
||||||
|
for (const x of matches) {
|
||||||
|
const arr = m.get(x.entryPath);
|
||||||
|
if (arr) arr.push(x);
|
||||||
|
else m.set(x.entryPath, [x]);
|
||||||
|
}
|
||||||
|
return m;
|
||||||
|
}, [matches]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-filtered">
|
||||||
|
<div className="kv-search-header">
|
||||||
|
{searching
|
||||||
|
? 'searching…'
|
||||||
|
: `${matches.length} match${matches.length === 1 ? '' : 'es'} in ${groups.size} file${groups.size === 1 ? '' : 's'}${truncated ? ' (truncated)' : ''}`}
|
||||||
|
</div>
|
||||||
|
{[...groups.entries()]
|
||||||
|
.sort(([a], [b]) => a.localeCompare(b))
|
||||||
|
.map(([entryPath, hits]) => (
|
||||||
|
<FilteredEntry
|
||||||
|
key={entryPath}
|
||||||
|
entryPath={entryPath}
|
||||||
|
hits={hits}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
expanded={expandedPaths.has(entryPath)}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
onToggleBranch={onToggleBranch}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface EntryProps {
|
||||||
|
entryPath: string;
|
||||||
|
hits: KVFindMatch[];
|
||||||
|
selectedKey: string | null;
|
||||||
|
expanded: boolean;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
onToggleBranch: (path: string, willOpen: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function FilteredEntry({
|
||||||
|
entryPath, hits, selectedKey, expanded,
|
||||||
|
onSelectLeaf, onSelectBranch, onToggleBranch,
|
||||||
|
}: EntryProps) {
|
||||||
|
// We deliberately do NOT mark the matched-leaf rows as "selected" via the
|
||||||
|
// tree row's selected style — the inline hit lines are themselves the
|
||||||
|
// selected indicator (they're highlighted when their key is current).
|
||||||
|
const isBranchSelected = entryPath === selectedKey;
|
||||||
|
const HIT_PREVIEW = 4;
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-filtered-entry">
|
||||||
|
<div
|
||||||
|
className={`kv-tree-row kv-tree-branch${isBranchSelected ? ' selected' : ''}`}
|
||||||
|
style={{ paddingLeft: 4 }}
|
||||||
|
onClick={() => {
|
||||||
|
onToggleBranch(entryPath, !expanded);
|
||||||
|
onSelectBranch(entryPath);
|
||||||
|
}}
|
||||||
|
>
|
||||||
|
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
|
||||||
|
<span className="kv-tree-label">{entryPath}</span>
|
||||||
|
<span className="kv-filtered-count" title={`${hits.length} match${hits.length === 1 ? '' : 'es'}`}>
|
||||||
|
{hits.length}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="kv-filtered-hits">
|
||||||
|
{hits.slice(0, HIT_PREVIEW).map((h) => (
|
||||||
|
<div
|
||||||
|
key={h.key}
|
||||||
|
className={`kv-filtered-hit${h.key === selectedKey ? ' selected' : ''}`}
|
||||||
|
onClick={(e) => { e.stopPropagation(); onSelectLeaf(h.key); }}
|
||||||
|
title={h.value}
|
||||||
|
>
|
||||||
|
<span className="kv-filtered-hit-field">{h.field}</span>
|
||||||
|
<span className="kv-filtered-hit-value">{h.value}</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
{hits.length > HIT_PREVIEW && (
|
||||||
|
<div className="kv-filtered-more">
|
||||||
|
+{hits.length - HIT_PREVIEW} more match{hits.length - HIT_PREVIEW === 1 ? '' : 'es'}
|
||||||
|
{!expanded && ' · expand to see all keys'}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{expanded && (
|
||||||
|
<LazySubtree
|
||||||
|
path={entryPath}
|
||||||
|
depth={1}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SubtreeProps {
|
||||||
|
path: string;
|
||||||
|
depth: number;
|
||||||
|
selectedKey: string | null;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface SubtreeData {
|
||||||
|
loading: boolean;
|
||||||
|
branches: string[];
|
||||||
|
leaves: string[];
|
||||||
|
error?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
// LazySubtree fetches its children once on mount and renders them. Sub-
|
||||||
|
// branches are themselves LazyBranch components that fetch their own
|
||||||
|
// children on first expand — recursion all the way down without the global
|
||||||
|
// expandedPaths state of the main KVTree.
|
||||||
|
function LazySubtree({ path, depth, selectedKey, onSelectLeaf, onSelectBranch }: SubtreeProps) {
|
||||||
|
const [data, setData] = useState<SubtreeData>({ loading: true, branches: [], leaves: [] });
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
KVAPI.treeAll(path, '/', 5000)
|
||||||
|
.then((r) => { if (!cancelled) setData({ loading: false, branches: r.branches, leaves: r.leaves }); })
|
||||||
|
.catch((e) => { if (!cancelled) setData({ loading: false, branches: [], leaves: [], error: e.message }); });
|
||||||
|
return () => { cancelled = true; };
|
||||||
|
}, [path]);
|
||||||
|
|
||||||
|
if (data.loading) {
|
||||||
|
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
|
||||||
|
}
|
||||||
|
if (data.error) {
|
||||||
|
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>{data.error}</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{data.branches.map((b) => (
|
||||||
|
<LazyBranch
|
||||||
|
key={b}
|
||||||
|
path={b}
|
||||||
|
parentPath={path}
|
||||||
|
depth={depth}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
{data.leaves.map((l) => (
|
||||||
|
<div
|
||||||
|
key={l}
|
||||||
|
className={`kv-tree-row kv-tree-leaf${l === selectedKey ? ' selected' : ''}`}
|
||||||
|
style={{ paddingLeft: depth * 14 + 4 }}
|
||||||
|
onClick={() => onSelectLeaf(l)}
|
||||||
|
>
|
||||||
|
<span className="kv-tree-leaf-dot">•</span>
|
||||||
|
<span className="kv-tree-label">{l.slice(path.length)}</span>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LazyBranchProps {
|
||||||
|
path: string;
|
||||||
|
parentPath: string;
|
||||||
|
depth: number;
|
||||||
|
selectedKey: string | null;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function LazyBranch({ path, parentPath, depth, selectedKey, onSelectLeaf, onSelectBranch }: LazyBranchProps) {
|
||||||
|
const [expanded, setExpanded] = useState(false);
|
||||||
|
const label = path.slice(parentPath.length);
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
<div
|
||||||
|
className={`kv-tree-row kv-tree-branch${path === selectedKey ? ' selected' : ''}`}
|
||||||
|
style={{ paddingLeft: depth * 14 + 4 }}
|
||||||
|
onClick={() => { setExpanded(!expanded); onSelectBranch(path); }}
|
||||||
|
>
|
||||||
|
<span className="kv-tree-toggle">{expanded ? '▾' : '▸'}</span>
|
||||||
|
<span className="kv-tree-label">{label}</span>
|
||||||
|
</div>
|
||||||
|
{expanded && (
|
||||||
|
<LazySubtree
|
||||||
|
path={path}
|
||||||
|
depth={depth + 1}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,896 @@
|
|||||||
|
/* KV Tree Editor — fills the iframe viewport with a 2-pane layout. */
|
||||||
|
|
||||||
|
.kv-app {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
height: 100vh;
|
||||||
|
width: 100%;
|
||||||
|
overflow: hidden;
|
||||||
|
background: #fafbfc;
|
||||||
|
color: #1f2330;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, 'Helvetica Neue', Arial, sans-serif;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Header ───────────────────────────────────────────────────────────── */
|
||||||
|
.kv-app-header {
|
||||||
|
border-bottom: 1px solid #e2e6ee;
|
||||||
|
padding: 8px 12px;
|
||||||
|
background: #fff;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-search {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-search-input {
|
||||||
|
flex: 1;
|
||||||
|
padding: 7px 11px;
|
||||||
|
border: 1px solid #d1d6e0;
|
||||||
|
border-radius: 6px;
|
||||||
|
font-size: 13px;
|
||||||
|
background: #fff;
|
||||||
|
outline: none;
|
||||||
|
}
|
||||||
|
.kv-search-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
|
||||||
|
|
||||||
|
.kv-app-stats {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #8089a0;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-breadcrumb {
|
||||||
|
margin-top: 6px;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-items: center;
|
||||||
|
gap: 2px;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
.kv-crumb-label {
|
||||||
|
color: #8089a0;
|
||||||
|
margin-right: 4px;
|
||||||
|
}
|
||||||
|
.kv-crumb {
|
||||||
|
border: none;
|
||||||
|
background: transparent;
|
||||||
|
padding: 1px 4px;
|
||||||
|
border-radius: 3px;
|
||||||
|
cursor: pointer;
|
||||||
|
color: #4a5170;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
}
|
||||||
|
.kv-crumb:hover { background: #eef0fb; color: #2a2f48; }
|
||||||
|
|
||||||
|
/* ── Body ─────────────────────────────────────────────────────────────── */
|
||||||
|
.kv-app-body {
|
||||||
|
flex: 1 1 auto;
|
||||||
|
display: flex;
|
||||||
|
min-height: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-left {
|
||||||
|
flex: 0 0 38%;
|
||||||
|
min-width: 280px;
|
||||||
|
max-width: 600px;
|
||||||
|
border-right: 1px solid #e2e6ee;
|
||||||
|
background: #fff;
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-right {
|
||||||
|
flex: 1 1 auto;
|
||||||
|
overflow: auto;
|
||||||
|
padding: 16px 20px;
|
||||||
|
background: #fafbfc;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Tree ─────────────────────────────────────────────────────────────── */
|
||||||
|
.kv-tree {
|
||||||
|
padding: 4px 0;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 12.5px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-list {
|
||||||
|
list-style: none;
|
||||||
|
margin: 0;
|
||||||
|
padding: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
padding: 2px 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
white-space: nowrap;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
.kv-tree-row:hover { background: #f3f5fb; }
|
||||||
|
|
||||||
|
.kv-tree-toggle {
|
||||||
|
display: inline-block;
|
||||||
|
width: 12px;
|
||||||
|
color: #8089a0;
|
||||||
|
font-size: 10px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-leaf-dot {
|
||||||
|
display: inline-block;
|
||||||
|
width: 12px;
|
||||||
|
color: #c5cad6;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-label {
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-branch .kv-tree-label {
|
||||||
|
color: #2a3050;
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-branch .kv-tree-filename {
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 0.85em;
|
||||||
|
color: #1e3a8a;
|
||||||
|
white-space: nowrap;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-hash-pill {
|
||||||
|
margin-left: 0.5rem;
|
||||||
|
padding: 0.05rem 0.4rem;
|
||||||
|
background: rgba(100, 108, 255, 0.12);
|
||||||
|
color: #4a50c8;
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 0.7em;
|
||||||
|
border-radius: 999px;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-leaf .kv-tree-label {
|
||||||
|
color: #4a5170;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-tree-row.selected {
|
||||||
|
background: #e7eafd;
|
||||||
|
color: #2532a3;
|
||||||
|
}
|
||||||
|
.kv-tree-row.selected:hover { background: #dde2fc; }
|
||||||
|
|
||||||
|
.kv-tree-loading,
|
||||||
|
.kv-tree-empty,
|
||||||
|
.kv-tree-error {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #8089a0;
|
||||||
|
padding: 4px 8px;
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
.kv-tree-error { color: #c0392b; }
|
||||||
|
|
||||||
|
/* ── Right pane (leaf / branch) ───────────────────────────────────────── */
|
||||||
|
.kv-leaf, .kv-branch {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 14px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-leaf-key {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
background: #f0f2f8;
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 8px 10px;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
.kv-leaf-key-text {
|
||||||
|
flex: 1;
|
||||||
|
word-break: break-all;
|
||||||
|
color: #2a3050;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-leaf-meta {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.kv-leaf-label {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6e7790;
|
||||||
|
font-weight: 500;
|
||||||
|
}
|
||||||
|
.kv-leaf-type {
|
||||||
|
border: 1px solid #d1d6e0;
|
||||||
|
border-radius: 5px;
|
||||||
|
padding: 4px 8px;
|
||||||
|
font-size: 12px;
|
||||||
|
background: #fff;
|
||||||
|
}
|
||||||
|
.kv-leaf-hint {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6e7790;
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-leaf-input {
|
||||||
|
width: 100%;
|
||||||
|
padding: 8px 10px;
|
||||||
|
border: 1px solid #d1d6e0;
|
||||||
|
border-radius: 5px;
|
||||||
|
font-size: 13px;
|
||||||
|
background: #fff;
|
||||||
|
outline: none;
|
||||||
|
box-sizing: border-box;
|
||||||
|
}
|
||||||
|
.kv-leaf-input:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
|
||||||
|
|
||||||
|
.kv-leaf-textarea {
|
||||||
|
width: 100%;
|
||||||
|
padding: 8px 10px;
|
||||||
|
border: 1px solid #d1d6e0;
|
||||||
|
border-radius: 5px;
|
||||||
|
font-size: 13px;
|
||||||
|
background: #fff;
|
||||||
|
outline: none;
|
||||||
|
resize: vertical;
|
||||||
|
box-sizing: border-box;
|
||||||
|
font-family: inherit;
|
||||||
|
}
|
||||||
|
.kv-leaf-textarea:focus { border-color: #6c7afa; box-shadow: 0 0 0 2px rgba(108, 122, 250, 0.2); }
|
||||||
|
|
||||||
|
.kv-leaf-mono {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-leaf-url {
|
||||||
|
display: flex;
|
||||||
|
gap: 6px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.kv-leaf-url-open {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6c7afa;
|
||||||
|
text-decoration: none;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-leaf-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── CID leaf ─────────────────────────────────────────────────────────── */
|
||||||
|
.kv-leaf-cid {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.kv-leaf-cid-controls {
|
||||||
|
display: flex;
|
||||||
|
gap: 6px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.kv-leaf-cid-actions {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
}
|
||||||
|
.kv-leaf-cid-meta {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6b7280;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
}
|
||||||
|
.kv-leaf-cid-preview {
|
||||||
|
max-width: 100%;
|
||||||
|
max-height: 320px;
|
||||||
|
border-radius: 6px;
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
background: #f5f6fa;
|
||||||
|
object-fit: contain;
|
||||||
|
align-self: flex-start;
|
||||||
|
}
|
||||||
|
.kv-leaf-cid-preview-media {
|
||||||
|
width: 100%;
|
||||||
|
max-width: 560px;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── File-detail view ─────────────────────────────────────────────────── */
|
||||||
|
.kv-file {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 14px;
|
||||||
|
}
|
||||||
|
.kv-file-header {
|
||||||
|
padding: 14px 16px;
|
||||||
|
background: #fff;
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
border-radius: 8px;
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: 1fr auto;
|
||||||
|
grid-template-areas:
|
||||||
|
"name actions"
|
||||||
|
"meta actions";
|
||||||
|
gap: 6px 16px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.kv-file-name {
|
||||||
|
grid-area: name;
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-weight: 600;
|
||||||
|
font-size: 14px;
|
||||||
|
color: #1e3a8a;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.kv-file-name-fallback {
|
||||||
|
color: #9ca3af;
|
||||||
|
font-style: italic;
|
||||||
|
}
|
||||||
|
.kv-file-meta {
|
||||||
|
grid-area: meta;
|
||||||
|
display: flex;
|
||||||
|
gap: 12px;
|
||||||
|
align-items: baseline;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6b7280;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.kv-file-hash {
|
||||||
|
background: rgba(100, 122, 250, 0.1);
|
||||||
|
color: #4a50c8;
|
||||||
|
padding: 1px 6px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
flex-shrink: 0;
|
||||||
|
}
|
||||||
|
.kv-file-path {
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
white-space: nowrap;
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
.kv-file-actions {
|
||||||
|
grid-area: actions;
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-file-rows {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
.kv-file-row {
|
||||||
|
display: grid;
|
||||||
|
grid-template-columns: minmax(140px, 220px) 1fr auto;
|
||||||
|
gap: 12px;
|
||||||
|
padding: 8px 12px;
|
||||||
|
background: #fff;
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
border-left: 3px solid #d6deed;
|
||||||
|
border-radius: 6px;
|
||||||
|
align-items: start;
|
||||||
|
}
|
||||||
|
.kv-file-row.pending-delete {
|
||||||
|
border-left-color: #dc2626;
|
||||||
|
background: #fef2f2;
|
||||||
|
}
|
||||||
|
.kv-file-row.pending-delete .kv-file-row-label {
|
||||||
|
text-decoration: line-through;
|
||||||
|
color: #9ca3af;
|
||||||
|
}
|
||||||
|
.kv-file-row-deleted-note {
|
||||||
|
color: #b91c1c;
|
||||||
|
font-style: italic;
|
||||||
|
font-size: 12px;
|
||||||
|
padding: 6px 0;
|
||||||
|
}
|
||||||
|
.kv-file-row-delete {
|
||||||
|
width: 24px;
|
||||||
|
height: 24px;
|
||||||
|
align-self: center;
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
justify-content: center;
|
||||||
|
background: transparent;
|
||||||
|
border: 1px solid transparent;
|
||||||
|
border-radius: 4px;
|
||||||
|
color: #dc2626;
|
||||||
|
font-size: 16px;
|
||||||
|
line-height: 1;
|
||||||
|
cursor: pointer;
|
||||||
|
opacity: 0.4;
|
||||||
|
transition: opacity 0.15s, background 0.15s, border-color 0.15s;
|
||||||
|
}
|
||||||
|
.kv-file-row:hover .kv-file-row-delete,
|
||||||
|
.kv-file-row.pending-delete .kv-file-row-delete {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
.kv-file-row-delete:hover {
|
||||||
|
background: rgba(220, 38, 38, 0.1);
|
||||||
|
border-color: rgba(220, 38, 38, 0.3);
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-file-add {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
padding: 10px 12px;
|
||||||
|
background: rgba(100, 122, 250, 0.05);
|
||||||
|
border: 1px dashed rgba(100, 122, 250, 0.3);
|
||||||
|
border-radius: 6px;
|
||||||
|
}
|
||||||
|
.kv-file-add-prefix {
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 11px;
|
||||||
|
color: #6b7280;
|
||||||
|
white-space: nowrap;
|
||||||
|
flex-shrink: 0;
|
||||||
|
max-width: 220px;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
.kv-file-add > input {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.kv-file-add > input:first-of-type {
|
||||||
|
flex: 0 0 200px;
|
||||||
|
}
|
||||||
|
.kv-file-row.dirty {
|
||||||
|
border-left-color: #fbbf24;
|
||||||
|
background: #fffbeb;
|
||||||
|
}
|
||||||
|
.kv-file-row.pulsed {
|
||||||
|
animation: kv-row-pulse 1.4s ease-out;
|
||||||
|
}
|
||||||
|
@keyframes kv-row-pulse {
|
||||||
|
0% { background: rgba(100, 122, 250, 0.35); border-left-color: #4a50c8; }
|
||||||
|
100% { background: transparent; }
|
||||||
|
}
|
||||||
|
.kv-file-row-label,
|
||||||
|
.kv-file-row-label-static {
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #2a3050;
|
||||||
|
background: none;
|
||||||
|
border: none;
|
||||||
|
text-align: left;
|
||||||
|
cursor: pointer;
|
||||||
|
padding: 6px 0;
|
||||||
|
word-break: break-all;
|
||||||
|
}
|
||||||
|
.kv-file-row-label-static {
|
||||||
|
cursor: default;
|
||||||
|
}
|
||||||
|
.kv-file-row-label:hover {
|
||||||
|
color: #4a50c8;
|
||||||
|
text-decoration: underline;
|
||||||
|
}
|
||||||
|
.kv-file-row-value {
|
||||||
|
display: flex;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
.kv-file-row-value > * {
|
||||||
|
flex: 1;
|
||||||
|
min-width: 0;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-file-group {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
border-radius: 8px;
|
||||||
|
background: rgba(100, 122, 250, 0.03);
|
||||||
|
overflow: hidden;
|
||||||
|
}
|
||||||
|
.kv-file-group-header {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 8px;
|
||||||
|
padding: 6px 12px;
|
||||||
|
border: none;
|
||||||
|
background: rgba(100, 122, 250, 0.1);
|
||||||
|
cursor: pointer;
|
||||||
|
font-family: 'Monaco', 'Menlo', 'Ubuntu Mono', 'Courier New', monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #2a3050;
|
||||||
|
}
|
||||||
|
.kv-file-group-header:hover {
|
||||||
|
background: rgba(100, 122, 250, 0.2);
|
||||||
|
}
|
||||||
|
.kv-file-group-chevron {
|
||||||
|
font-size: 10px;
|
||||||
|
color: #6b7280;
|
||||||
|
}
|
||||||
|
.kv-file-group-name {
|
||||||
|
font-weight: 600;
|
||||||
|
color: #1e3a8a;
|
||||||
|
}
|
||||||
|
.kv-file-group-count {
|
||||||
|
margin-left: auto;
|
||||||
|
padding: 1px 8px;
|
||||||
|
background: rgba(100, 122, 250, 0.2);
|
||||||
|
color: #4a50c8;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-size: 11px;
|
||||||
|
}
|
||||||
|
.kv-file-group-body {
|
||||||
|
padding: 8px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-file-row-lang .kv-file-lang-entries {
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
.kv-file-lang-entry {
|
||||||
|
display: flex;
|
||||||
|
gap: 6px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.kv-file-lang-entry > .kv-leaf-input {
|
||||||
|
flex: 1;
|
||||||
|
}
|
||||||
|
.kv-file-lang-remove {
|
||||||
|
color: #dc2626;
|
||||||
|
}
|
||||||
|
.kv-file-lang-add {
|
||||||
|
align-self: flex-start;
|
||||||
|
background: rgba(100, 122, 250, 0.1);
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── DateTime leaf ────────────────────────────────────────────────────── */
|
||||||
|
.kv-leaf-datetime {
|
||||||
|
display: flex;
|
||||||
|
gap: 8px;
|
||||||
|
align-items: center;
|
||||||
|
}
|
||||||
|
.kv-leaf-datetime-shape {
|
||||||
|
font-size: 11px;
|
||||||
|
color: #6b7280;
|
||||||
|
background: rgba(100, 122, 250, 0.1);
|
||||||
|
padding: 2px 6px;
|
||||||
|
border-radius: 999px;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
white-space: nowrap;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Branch ───────────────────────────────────────────────────────────── */
|
||||||
|
.kv-branch-section {
|
||||||
|
border: 1px solid #e2e6ee;
|
||||||
|
border-radius: 6px;
|
||||||
|
background: #fff;
|
||||||
|
padding: 12px 14px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column;
|
||||||
|
gap: 8px;
|
||||||
|
}
|
||||||
|
.kv-branch-section h4 {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 13px;
|
||||||
|
color: #2a3050;
|
||||||
|
}
|
||||||
|
.kv-branch-row {
|
||||||
|
display: flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 6px;
|
||||||
|
}
|
||||||
|
.kv-branch-prefix {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6e7790;
|
||||||
|
background: #f0f2f8;
|
||||||
|
padding: 6px 8px;
|
||||||
|
border-radius: 4px;
|
||||||
|
white-space: nowrap;
|
||||||
|
max-width: 50%;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
}
|
||||||
|
.kv-branch-note {
|
||||||
|
margin: 0;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #6e7790;
|
||||||
|
line-height: 1.4;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Buttons ──────────────────────────────────────────────────────────── */
|
||||||
|
.kv-btn {
|
||||||
|
border: 1px solid #d1d6e0;
|
||||||
|
background: #fff;
|
||||||
|
color: #2a3050;
|
||||||
|
padding: 6px 14px;
|
||||||
|
border-radius: 5px;
|
||||||
|
font-size: 12.5px;
|
||||||
|
cursor: pointer;
|
||||||
|
}
|
||||||
|
.kv-btn:hover:not(:disabled) { background: #f3f5fb; border-color: #b8c0d0; }
|
||||||
|
.kv-btn:disabled { opacity: 0.45; cursor: not-allowed; }
|
||||||
|
|
||||||
|
.kv-btn-primary {
|
||||||
|
background: #6c7afa;
|
||||||
|
border-color: #6c7afa;
|
||||||
|
color: #fff;
|
||||||
|
}
|
||||||
|
.kv-btn-primary:hover:not(:disabled) {
|
||||||
|
background: #5868e8;
|
||||||
|
border-color: #5868e8;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-btn-danger {
|
||||||
|
border-color: #e0788a;
|
||||||
|
color: #c0392b;
|
||||||
|
}
|
||||||
|
.kv-btn-danger:hover:not(:disabled) {
|
||||||
|
background: #fdecef;
|
||||||
|
border-color: #c0392b;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-btn-icon {
|
||||||
|
border: 1px solid transparent;
|
||||||
|
background: transparent;
|
||||||
|
color: #6e7790;
|
||||||
|
padding: 2px 6px;
|
||||||
|
border-radius: 4px;
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 13px;
|
||||||
|
}
|
||||||
|
.kv-btn-icon:hover { background: #eef0fb; color: #2a3050; }
|
||||||
|
|
||||||
|
/* ── Pane status ──────────────────────────────────────────────────────── */
|
||||||
|
.kv-pane-status {
|
||||||
|
color: #6e7790;
|
||||||
|
font-size: 12.5px;
|
||||||
|
padding: 8px 0;
|
||||||
|
}
|
||||||
|
.kv-pane-empty {
|
||||||
|
text-align: center;
|
||||||
|
padding: 80px 20px;
|
||||||
|
font-size: 14px;
|
||||||
|
}
|
||||||
|
.kv-pane-hint {
|
||||||
|
margin-top: 8px;
|
||||||
|
font-size: 12px;
|
||||||
|
color: #8089a0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Field chips (configure search-in fields) ─────────────────────────── */
|
||||||
|
.kv-chips {
|
||||||
|
margin-top: 6px;
|
||||||
|
display: flex;
|
||||||
|
flex-wrap: wrap;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
}
|
||||||
|
.kv-chips-label {
|
||||||
|
font-size: 12px;
|
||||||
|
color: #8089a0;
|
||||||
|
margin-right: 4px;
|
||||||
|
}
|
||||||
|
.kv-chip {
|
||||||
|
display: inline-flex;
|
||||||
|
align-items: center;
|
||||||
|
gap: 4px;
|
||||||
|
background: #eef0fb;
|
||||||
|
border: 1px solid #d5dafa;
|
||||||
|
color: #2a3050;
|
||||||
|
border-radius: 12px;
|
||||||
|
padding: 1px 4px 1px 8px;
|
||||||
|
font-size: 11.5px;
|
||||||
|
}
|
||||||
|
.kv-chip code {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 11.5px;
|
||||||
|
}
|
||||||
|
.kv-chip-x {
|
||||||
|
border: none;
|
||||||
|
background: transparent;
|
||||||
|
color: #6e7790;
|
||||||
|
cursor: pointer;
|
||||||
|
padding: 0 4px;
|
||||||
|
font-size: 14px;
|
||||||
|
line-height: 1;
|
||||||
|
}
|
||||||
|
.kv-chip-x:hover { color: #c0392b; }
|
||||||
|
.kv-chip-input {
|
||||||
|
border: 1px dashed #c5cad6;
|
||||||
|
background: transparent;
|
||||||
|
border-radius: 12px;
|
||||||
|
padding: 2px 8px;
|
||||||
|
font-size: 11.5px;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
outline: none;
|
||||||
|
min-width: 90px;
|
||||||
|
}
|
||||||
|
.kv-chip-input:focus { border-style: solid; border-color: #6c7afa; }
|
||||||
|
|
||||||
|
/* ── Search header (shared across filter mode) ────────────────────────── */
|
||||||
|
.kv-search-header {
|
||||||
|
padding: 6px 10px;
|
||||||
|
font-size: 11px;
|
||||||
|
color: #6e7790;
|
||||||
|
background: #f3f5fb;
|
||||||
|
border-bottom: 1px solid #e2e6ee;
|
||||||
|
position: sticky;
|
||||||
|
top: 0;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Filtered tree (search-active state) ──────────────────────────────── */
|
||||||
|
.kv-filtered {
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 12.5px;
|
||||||
|
}
|
||||||
|
.kv-filtered-entry {
|
||||||
|
border-bottom: 1px solid #f0f2f8;
|
||||||
|
}
|
||||||
|
.kv-filtered-count {
|
||||||
|
margin-left: auto;
|
||||||
|
background: #eef0fb;
|
||||||
|
color: #4a5170;
|
||||||
|
padding: 0 6px;
|
||||||
|
border-radius: 9px;
|
||||||
|
font-size: 10.5px;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
|
font-weight: 600;
|
||||||
|
}
|
||||||
|
.kv-filtered-hits {
|
||||||
|
padding: 2px 4px 4px 24px;
|
||||||
|
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
|
||||||
|
font-size: 12px;
|
||||||
|
}
|
||||||
|
.kv-filtered-hit {
|
||||||
|
display: flex;
|
||||||
|
gap: 6px;
|
||||||
|
align-items: baseline;
|
||||||
|
padding: 2px 6px;
|
||||||
|
border-radius: 3px;
|
||||||
|
cursor: pointer;
|
||||||
|
line-height: 1.35;
|
||||||
|
}
|
||||||
|
.kv-filtered-hit:hover { background: #f3f5fb; }
|
||||||
|
.kv-filtered-hit.selected {
|
||||||
|
background: #e7eafd;
|
||||||
|
}
|
||||||
|
.kv-filtered-hit-field {
|
||||||
|
background: #eef0fb;
|
||||||
|
color: #4a5170;
|
||||||
|
padding: 0 5px;
|
||||||
|
border-radius: 3px;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 10.5px;
|
||||||
|
flex: 0 0 auto;
|
||||||
|
}
|
||||||
|
.kv-filtered-hit-value {
|
||||||
|
color: #2a3050;
|
||||||
|
word-break: break-word;
|
||||||
|
overflow: hidden;
|
||||||
|
text-overflow: ellipsis;
|
||||||
|
display: -webkit-box;
|
||||||
|
-webkit-line-clamp: 2;
|
||||||
|
-webkit-box-orient: vertical;
|
||||||
|
}
|
||||||
|
.kv-filtered-more {
|
||||||
|
padding: 2px 6px 0;
|
||||||
|
font-size: 11px;
|
||||||
|
font-style: italic;
|
||||||
|
color: #8089a0;
|
||||||
|
}
|
||||||
|
|
||||||
|
/* ── Toasts ───────────────────────────────────────────────────────────── */
|
||||||
|
.kv-toast-stack {
|
||||||
|
position: fixed;
|
||||||
|
bottom: 16px;
|
||||||
|
right: 16px;
|
||||||
|
display: flex;
|
||||||
|
flex-direction: column-reverse;
|
||||||
|
gap: 6px;
|
||||||
|
z-index: 1000;
|
||||||
|
max-width: 480px;
|
||||||
|
}
|
||||||
|
.kv-toast {
|
||||||
|
background: #2a3050;
|
||||||
|
color: #fff;
|
||||||
|
border-radius: 6px;
|
||||||
|
padding: 9px 12px;
|
||||||
|
box-shadow: 0 4px 14px rgba(0,0,0,0.15);
|
||||||
|
font-size: 12.5px;
|
||||||
|
}
|
||||||
|
.kv-toast-success { background: #2f9e44; }
|
||||||
|
.kv-toast-error { background: #c0392b; }
|
||||||
|
.kv-toast-row {
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
gap: 8px;
|
||||||
|
justify-content: space-between;
|
||||||
|
}
|
||||||
|
.kv-toast-text {
|
||||||
|
flex: 1;
|
||||||
|
word-break: break-word;
|
||||||
|
}
|
||||||
|
.kv-toast-close {
|
||||||
|
border: none;
|
||||||
|
background: transparent;
|
||||||
|
color: rgba(255,255,255,0.8);
|
||||||
|
cursor: pointer;
|
||||||
|
font-size: 16px;
|
||||||
|
line-height: 1;
|
||||||
|
}
|
||||||
|
.kv-toast-close:hover { color: #fff; }
|
||||||
|
.kv-toast-details {
|
||||||
|
margin: 6px 0 0;
|
||||||
|
padding-left: 18px;
|
||||||
|
font-family: ui-monospace, SFMono-Regular, Menlo, monospace;
|
||||||
|
font-size: 11px;
|
||||||
|
color: rgba(255,255,255,0.85);
|
||||||
|
max-height: 160px;
|
||||||
|
overflow: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-gear {
|
||||||
|
margin-left: 0.25rem;
|
||||||
|
font-size: 1.05rem;
|
||||||
|
opacity: 0.6;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-app-gear:hover {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-backdrop {
|
||||||
|
position: fixed;
|
||||||
|
inset: 0;
|
||||||
|
background: rgba(0, 0, 0, 0.45);
|
||||||
|
z-index: 1000;
|
||||||
|
display: flex;
|
||||||
|
align-items: flex-start;
|
||||||
|
justify-content: center;
|
||||||
|
padding: 4vh 1rem;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal {
|
||||||
|
position: relative;
|
||||||
|
background: white;
|
||||||
|
color: inherit;
|
||||||
|
border-radius: 8px;
|
||||||
|
box-shadow: 0 10px 40px rgba(0, 0, 0, 0.3);
|
||||||
|
width: min(960px, 100%);
|
||||||
|
max-height: 92vh;
|
||||||
|
overflow-y: auto;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-close {
|
||||||
|
position: absolute;
|
||||||
|
top: 0.5rem;
|
||||||
|
right: 0.75rem;
|
||||||
|
background: transparent;
|
||||||
|
border: none;
|
||||||
|
font-size: 1.6rem;
|
||||||
|
line-height: 1;
|
||||||
|
cursor: pointer;
|
||||||
|
color: inherit;
|
||||||
|
opacity: 0.6;
|
||||||
|
z-index: 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
.kv-modal-close:hover {
|
||||||
|
opacity: 1;
|
||||||
|
}
|
||||||
@@ -0,0 +1,313 @@
|
|||||||
|
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||||
|
import { KVAPI, KVFindMatch, KVInfoResponse } from '../api/kvApi';
|
||||||
|
import { ancestorBranches, KVTree } from './KVTree';
|
||||||
|
import { FilteredKVTree } from './FilteredKVTree';
|
||||||
|
import { KVLeafEditor } from './KVLeafEditor';
|
||||||
|
import { KVBranchEditor } from './KVBranchEditor';
|
||||||
|
import { KVFileEditor } from './KVFileEditor';
|
||||||
|
|
||||||
|
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
|
||||||
|
import { SearchFieldChips } from './SearchFieldChips';
|
||||||
|
import { ToastStack, makeToast } from './Toast';
|
||||||
|
import { ToastEntry } from './types';
|
||||||
|
import { SnapshotPanel } from '../components/SnapshotPanel';
|
||||||
|
import './KVApp.css';
|
||||||
|
|
||||||
|
const SEARCH_FIELDS_LS_KEY = 'kv-editor.searchFields.v1';
|
||||||
|
const DEFAULT_SEARCH_FIELDS = ['originalTitle', 'filePath', 'fileName', 'plot/eng'];
|
||||||
|
|
||||||
|
function loadSearchFields(): string[] {
|
||||||
|
try {
|
||||||
|
const raw = localStorage.getItem(SEARCH_FIELDS_LS_KEY);
|
||||||
|
if (!raw) return DEFAULT_SEARCH_FIELDS;
|
||||||
|
const parsed = JSON.parse(raw);
|
||||||
|
if (Array.isArray(parsed) && parsed.every((x) => typeof x === 'string')) return parsed;
|
||||||
|
} catch {}
|
||||||
|
return DEFAULT_SEARCH_FIELDS;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Top-level shell. Two panes inside the iframe:
|
||||||
|
// left — lazy tree of the keyspace
|
||||||
|
// right — leaf editor (when a leaf is selected) or branch editor (branch).
|
||||||
|
//
|
||||||
|
// A sticky header carries the search bar, the breadcrumb, and basic stats.
|
||||||
|
export default function KVApp() {
|
||||||
|
const [snapshotOpen, setSnapshotOpen] = useState(false);
|
||||||
|
// Selected node. branchPath is set when a branch is selected; selectedKey
|
||||||
|
// when a leaf is. Mutually exclusive.
|
||||||
|
const [selectedKey, setSelectedKey] = useState<string | null>(null);
|
||||||
|
const [branchPath, setBranchPath] = useState<string | null>(null);
|
||||||
|
const [expandedPaths, setExpandedPaths] = useState<Set<string>>(new Set(['']));
|
||||||
|
const [searchInput, setSearchInput] = useState('');
|
||||||
|
const [searchFields, setSearchFields] = useState<string[]>(loadSearchFields);
|
||||||
|
const [searchResults, setSearchResults] = useState<KVFindMatch[] | null>(null);
|
||||||
|
const [searchTruncated, setSearchTruncated] = useState(false);
|
||||||
|
const [searching, setSearching] = useState(false);
|
||||||
|
const [info, setInfo] = useState<KVInfoResponse | null>(null);
|
||||||
|
const [reloadToken, setReloadToken] = useState(0);
|
||||||
|
const [toasts, setToasts] = useState<ToastEntry[]>([]);
|
||||||
|
const searchTimerRef = useRef<number | null>(null);
|
||||||
|
// When the right pane is a file-detail view and the user clicks a leaf
|
||||||
|
// under the same file branch in the tree, we route the click as a
|
||||||
|
// scroll-anchor inside the file detail instead of a leaf-editor switch.
|
||||||
|
// The token bumps each click so identical anchors still trigger a scroll.
|
||||||
|
const [anchor, setAnchor] = useState<{ field: string; token: number } | null>(null);
|
||||||
|
const isFileBranch = (p: string | null) => !!p && /^file:[^/]+\/$/.test(p);
|
||||||
|
|
||||||
|
const handleLeafClick = (key: string) => {
|
||||||
|
if (isFileBranch(branchPath) && key.startsWith(branchPath!)) {
|
||||||
|
setAnchor({ field: key.slice(branchPath!.length), token: (anchor?.token ?? 0) + 1 });
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSelectedKey(key);
|
||||||
|
setBranchPath(null);
|
||||||
|
setAnchor(null);
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleBranchClick = (path: string) => {
|
||||||
|
setBranchPath(path);
|
||||||
|
setSelectedKey(null);
|
||||||
|
setAnchor(null);
|
||||||
|
};
|
||||||
|
|
||||||
|
const pushToast = (t: ToastEntry) => setToasts((cur) => [...cur, t]);
|
||||||
|
const dismissToast = (id: number) => setToasts((cur) => cur.filter((t) => t.id !== id));
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
KVAPI.info().then(setInfo).catch(() => {});
|
||||||
|
}, [reloadToken]);
|
||||||
|
|
||||||
|
// Deep-link: /editor/file/<hashId> opens the tree pre-expanded at that hash.
|
||||||
|
useEffect(() => {
|
||||||
|
const m = window.location.pathname.match(/\/file\/([a-z0-9]+)$/i);
|
||||||
|
if (m) {
|
||||||
|
const hashId = m[1];
|
||||||
|
const branch = `file:${hashId}/`;
|
||||||
|
setExpandedPaths((cur) => new Set([...cur, '', 'file:', branch]));
|
||||||
|
setBranchPath(branch);
|
||||||
|
}
|
||||||
|
}, []);
|
||||||
|
|
||||||
|
// Persist field list across reloads.
|
||||||
|
useEffect(() => {
|
||||||
|
try { localStorage.setItem(SEARCH_FIELDS_LS_KEY, JSON.stringify(searchFields)); } catch {}
|
||||||
|
}, [searchFields]);
|
||||||
|
|
||||||
|
// Debounced live value-search across configured fields. Empty input clears.
|
||||||
|
useEffect(() => {
|
||||||
|
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
|
||||||
|
if (!searchInput.trim() || searchFields.length === 0) {
|
||||||
|
setSearchResults(null);
|
||||||
|
setSearchTruncated(false);
|
||||||
|
setSearching(false);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
setSearching(true);
|
||||||
|
searchTimerRef.current = window.setTimeout(async () => {
|
||||||
|
try {
|
||||||
|
const r = await KVAPI.find(searchInput.trim(), searchFields, 200);
|
||||||
|
setSearchResults(r.matches);
|
||||||
|
setSearchTruncated(r.truncated);
|
||||||
|
} catch (e: any) {
|
||||||
|
pushToast(makeToast('error', `Search failed: ${e.message}`));
|
||||||
|
} finally {
|
||||||
|
setSearching(false);
|
||||||
|
}
|
||||||
|
}, 250);
|
||||||
|
return () => {
|
||||||
|
if (searchTimerRef.current) window.clearTimeout(searchTimerRef.current);
|
||||||
|
};
|
||||||
|
}, [searchInput, searchFields]);
|
||||||
|
|
||||||
|
// Jump to a key from a search hit: select it, expand all ancestor branches.
|
||||||
|
const jumpToKey = (key: string) => {
|
||||||
|
const ancestors = ancestorBranches(key);
|
||||||
|
setExpandedPaths((cur) => new Set([...cur, '', ...ancestors]));
|
||||||
|
if (key.endsWith('/') || key.endsWith(':')) {
|
||||||
|
setBranchPath(key);
|
||||||
|
setSelectedKey(null);
|
||||||
|
} else {
|
||||||
|
setSelectedKey(key);
|
||||||
|
setBranchPath(null);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const breadcrumb = useMemo(() => {
|
||||||
|
const target = selectedKey ?? branchPath ?? '';
|
||||||
|
if (!target) return null;
|
||||||
|
const parts: { label: string; path: string }[] = [];
|
||||||
|
let i = 0;
|
||||||
|
while (i < target.length) {
|
||||||
|
let next = -1;
|
||||||
|
for (const d of [':', '/']) {
|
||||||
|
const idx = target.indexOf(d, i);
|
||||||
|
if (idx >= 0 && (next < 0 || idx < next)) next = idx;
|
||||||
|
}
|
||||||
|
if (next < 0) {
|
||||||
|
parts.push({ label: target.slice(i), path: target });
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
parts.push({ label: target.slice(i, next + 1), path: target.slice(0, next + 1) });
|
||||||
|
i = next + 1;
|
||||||
|
}
|
||||||
|
return parts;
|
||||||
|
}, [selectedKey, branchPath]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-app">
|
||||||
|
<header className="kv-app-header">
|
||||||
|
<div className="kv-app-search">
|
||||||
|
<input
|
||||||
|
type="search"
|
||||||
|
className="kv-search-input"
|
||||||
|
placeholder="filter by value across configured fields…"
|
||||||
|
value={searchInput}
|
||||||
|
onChange={(e) => setSearchInput(e.target.value)}
|
||||||
|
autoFocus
|
||||||
|
/>
|
||||||
|
{searchInput && (
|
||||||
|
<button className="kv-btn-icon" onClick={() => setSearchInput('')} title="clear">×</button>
|
||||||
|
)}
|
||||||
|
<span className="kv-app-stats">
|
||||||
|
{info ? `${info.keyCount} files · ${info.memoryUsage}` : ''}
|
||||||
|
</span>
|
||||||
|
<button
|
||||||
|
className="kv-btn"
|
||||||
|
onClick={() => setReloadToken((n) => n + 1)}
|
||||||
|
title="refresh tree"
|
||||||
|
>Refresh</button>
|
||||||
|
<button
|
||||||
|
className="kv-btn-icon kv-app-gear"
|
||||||
|
onClick={() => setSnapshotOpen(true)}
|
||||||
|
title="Snapshot — export / import / wipe"
|
||||||
|
aria-label="Open snapshot settings"
|
||||||
|
>
|
||||||
|
⚙
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
<SearchFieldChips fields={searchFields} onChange={setSearchFields} />
|
||||||
|
{breadcrumb && breadcrumb.length > 0 && (
|
||||||
|
<div className="kv-app-breadcrumb">
|
||||||
|
<span className="kv-crumb-label">path:</span>
|
||||||
|
{breadcrumb.map((b, i) => (
|
||||||
|
<button
|
||||||
|
key={i}
|
||||||
|
className="kv-crumb"
|
||||||
|
onClick={() => jumpToKey(b.path)}
|
||||||
|
>{b.label}</button>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</header>
|
||||||
|
|
||||||
|
<div className="kv-app-body">
|
||||||
|
<aside className="kv-app-left">
|
||||||
|
{searchResults !== null ? (
|
||||||
|
<FilteredKVTree
|
||||||
|
matches={searchResults}
|
||||||
|
truncated={searchTruncated}
|
||||||
|
searching={searching}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
expandedPaths={expandedPaths}
|
||||||
|
onSelectLeaf={handleLeafClick}
|
||||||
|
onSelectBranch={handleBranchClick}
|
||||||
|
onToggleBranch={(p, willOpen) => {
|
||||||
|
setExpandedPaths((cur) => {
|
||||||
|
const next = new Set(cur);
|
||||||
|
if (willOpen) next.add(p); else next.delete(p);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<KVTree
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
expandedPaths={expandedPaths}
|
||||||
|
reloadToken={reloadToken}
|
||||||
|
onSelectLeaf={handleLeafClick}
|
||||||
|
onSelectBranch={handleBranchClick}
|
||||||
|
onToggleBranch={(p, willOpen) => {
|
||||||
|
setExpandedPaths((cur) => {
|
||||||
|
const next = new Set(cur);
|
||||||
|
if (willOpen) next.add(p); else next.delete(p);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</aside>
|
||||||
|
<main className="kv-app-right">
|
||||||
|
{selectedKey ? (
|
||||||
|
<KVLeafEditor
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
reloadToken={reloadToken}
|
||||||
|
onSaved={() => {
|
||||||
|
pushToast(makeToast('success', `Saved ${selectedKey}`));
|
||||||
|
setReloadToken((n) => n + 1);
|
||||||
|
}}
|
||||||
|
onDeleted={() => {
|
||||||
|
pushToast(makeToast('success', `Deleted ${selectedKey}`));
|
||||||
|
setSelectedKey(null);
|
||||||
|
setReloadToken((n) => n + 1);
|
||||||
|
}}
|
||||||
|
onError={(msg) => pushToast(makeToast('error', msg))}
|
||||||
|
/>
|
||||||
|
) : branchPath && FILE_BRANCH_RE.test(branchPath) ? (
|
||||||
|
<KVFileEditor
|
||||||
|
key={branchPath}
|
||||||
|
branchPath={branchPath}
|
||||||
|
reloadToken={reloadToken}
|
||||||
|
scrollToField={anchor}
|
||||||
|
onChanged={(info, errors) => {
|
||||||
|
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
|
||||||
|
setReloadToken((n) => n + 1);
|
||||||
|
}}
|
||||||
|
onSelectLeaf={(k) => { setSelectedKey(k); setBranchPath(null); setAnchor(null); }}
|
||||||
|
/>
|
||||||
|
) : branchPath ? (
|
||||||
|
<KVBranchEditor
|
||||||
|
branchPath={branchPath}
|
||||||
|
reloadToken={reloadToken}
|
||||||
|
onChanged={(info, errors) => {
|
||||||
|
pushToast(makeToast(errors && errors.length > 0 ? 'error' : 'success', info, errors));
|
||||||
|
setReloadToken((n) => n + 1);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
) : (
|
||||||
|
<div className="kv-pane-status kv-pane-empty">
|
||||||
|
Select a key from the tree.
|
||||||
|
<div className="kv-pane-hint">
|
||||||
|
Click a branch to add / rename / delete a subtree, or a leaf to edit its value.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</main>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
{snapshotOpen && (
|
||||||
|
<div
|
||||||
|
className="kv-modal-backdrop"
|
||||||
|
onClick={() => setSnapshotOpen(false)}
|
||||||
|
>
|
||||||
|
<div
|
||||||
|
className="kv-modal"
|
||||||
|
onClick={(e) => e.stopPropagation()}
|
||||||
|
role="dialog"
|
||||||
|
aria-modal="true"
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
className="kv-modal-close"
|
||||||
|
onClick={() => setSnapshotOpen(false)}
|
||||||
|
aria-label="Close"
|
||||||
|
title="Close"
|
||||||
|
>×</button>
|
||||||
|
<SnapshotPanel />
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
|
||||||
|
<ToastStack toasts={toasts} onDismiss={dismissToast} />
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
@@ -0,0 +1,194 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { KVAPI } from '../api/kvApi';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
// Branch path, ends with the delimiter (":" or "/").
|
||||||
|
branchPath: string;
|
||||||
|
reloadToken: number;
|
||||||
|
onChanged: (info: string, errors?: string[]) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Branch operations are non-atomic — each per-key write/delete goes one round
|
||||||
|
// trip to Redis and may fail individually. Failures collect into a list and
|
||||||
|
// surface via the parent's toast stack. Per the design, we accept partial
|
||||||
|
// failure and surface what failed.
|
||||||
|
export function KVBranchEditor({ branchPath, reloadToken, onChanged }: Props) {
|
||||||
|
const [count, setCount] = useState<number | null>(null);
|
||||||
|
const [busy, setBusy] = useState(false);
|
||||||
|
const [newChild, setNewChild] = useState('');
|
||||||
|
const [newValue, setNewValue] = useState('');
|
||||||
|
const [renameTo, setRenameTo] = useState('');
|
||||||
|
|
||||||
|
// Use the search endpoint with the branch path as the substring to count
|
||||||
|
// descendants. Truncated when there are more than the limit; we just show
|
||||||
|
// an approximate count.
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
KVAPI.search(branchPath, 5000)
|
||||||
|
.then((r) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
// Filter to true descendants (those starting with branchPath).
|
||||||
|
const descendants = r.keys.filter((k) => k.startsWith(branchPath));
|
||||||
|
setCount(descendants.length);
|
||||||
|
})
|
||||||
|
.catch(() => { if (!cancelled) setCount(null); });
|
||||||
|
return () => { cancelled = true; };
|
||||||
|
}, [branchPath, reloadToken]);
|
||||||
|
|
||||||
|
// Walk all descendants and run an op against each, collecting failures.
|
||||||
|
const forEachDescendant = async (op: (key: string) => Promise<void>): Promise<string[]> => {
|
||||||
|
const r = await KVAPI.search(branchPath, 5000);
|
||||||
|
const keys = r.keys.filter((k) => k.startsWith(branchPath));
|
||||||
|
const errors: string[] = [];
|
||||||
|
for (const k of keys) {
|
||||||
|
try { await op(k); } catch (e: any) { errors.push(`${k}: ${e.message}`); }
|
||||||
|
}
|
||||||
|
return errors;
|
||||||
|
};
|
||||||
|
|
||||||
|
const addChild = async () => {
|
||||||
|
if (!newChild) return;
|
||||||
|
const childKey = branchPath + newChild;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
await KVAPI.put(childKey, newValue);
|
||||||
|
setNewChild('');
|
||||||
|
setNewValue('');
|
||||||
|
onChanged(`Created ${childKey}`);
|
||||||
|
} catch (e: any) {
|
||||||
|
onChanged(`Failed to create ${childKey}`, [e.message]);
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const deleteSubtree = async () => {
|
||||||
|
if (!confirm(`Delete every key under ${branchPath} (${count ?? '?'} entries)?`)) return;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
const errors = await forEachDescendant((k) => KVAPI.del(k).then(() => undefined));
|
||||||
|
if (errors.length === 0) {
|
||||||
|
onChanged(`Deleted subtree ${branchPath}`);
|
||||||
|
} else {
|
||||||
|
onChanged(`Deleted subtree ${branchPath} with ${errors.length} failure(s)`, errors);
|
||||||
|
}
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const renameBranch = async () => {
|
||||||
|
if (!renameTo) return;
|
||||||
|
const target = renameTo.endsWith('/') || renameTo.endsWith(':') ? renameTo : renameTo + '/';
|
||||||
|
if (target === branchPath) return;
|
||||||
|
if (!confirm(`Rename ${branchPath} → ${target}? This copies ${count ?? '?'} keys then deletes the originals.`)) return;
|
||||||
|
setBusy(true);
|
||||||
|
try {
|
||||||
|
// Pass 1: copy
|
||||||
|
const r = await KVAPI.search(branchPath, 5000);
|
||||||
|
const keys = r.keys.filter((k) => k.startsWith(branchPath));
|
||||||
|
const copyErrors: string[] = [];
|
||||||
|
for (const oldKey of keys) {
|
||||||
|
try {
|
||||||
|
const v = await KVAPI.get(oldKey);
|
||||||
|
if (!v.exists || v.type !== 'string') {
|
||||||
|
copyErrors.push(`${oldKey}: skipped (type ${v.type || 'missing'})`);
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const newKey = target + oldKey.slice(branchPath.length);
|
||||||
|
await KVAPI.put(newKey, v.value);
|
||||||
|
} catch (e: any) {
|
||||||
|
copyErrors.push(`${oldKey}: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// Pass 2: delete originals (only the ones we successfully copied)
|
||||||
|
const failedOldKeys = new Set(copyErrors.map((e) => e.split(':')[0]));
|
||||||
|
const deleteErrors: string[] = [];
|
||||||
|
for (const oldKey of keys) {
|
||||||
|
if (failedOldKeys.has(oldKey)) continue;
|
||||||
|
try { await KVAPI.del(oldKey); } catch (e: any) { deleteErrors.push(`${oldKey}: ${e.message}`); }
|
||||||
|
}
|
||||||
|
const allErrors = [...copyErrors, ...deleteErrors];
|
||||||
|
if (allErrors.length === 0) {
|
||||||
|
onChanged(`Renamed ${branchPath} → ${target}`);
|
||||||
|
} else {
|
||||||
|
onChanged(`Renamed ${branchPath} → ${target} with ${allErrors.length} failure(s)`, allErrors);
|
||||||
|
}
|
||||||
|
setRenameTo('');
|
||||||
|
} finally {
|
||||||
|
setBusy(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-branch">
|
||||||
|
<div className="kv-leaf-key">
|
||||||
|
<code className="kv-leaf-key-text">{branchPath}</code>
|
||||||
|
<button
|
||||||
|
className="kv-btn-icon"
|
||||||
|
onClick={() => navigator.clipboard.writeText(branchPath)}
|
||||||
|
title="copy path"
|
||||||
|
>⧉</button>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="kv-pane-status">
|
||||||
|
{count == null ? 'counting…' : `${count} descendant key${count === 1 ? '' : 's'}`}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<section className="kv-branch-section">
|
||||||
|
<h4>Add child</h4>
|
||||||
|
<div className="kv-branch-row">
|
||||||
|
<span className="kv-branch-prefix">{branchPath}</span>
|
||||||
|
<input
|
||||||
|
className="kv-leaf-input"
|
||||||
|
placeholder="child path (may contain /)"
|
||||||
|
value={newChild}
|
||||||
|
onChange={(e) => setNewChild(e.target.value)}
|
||||||
|
disabled={busy}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<input
|
||||||
|
className="kv-leaf-input"
|
||||||
|
placeholder="value"
|
||||||
|
value={newValue}
|
||||||
|
onChange={(e) => setNewValue(e.target.value)}
|
||||||
|
disabled={busy}
|
||||||
|
/>
|
||||||
|
<button className="kv-btn kv-btn-primary" disabled={!newChild || busy} onClick={addChild}>
|
||||||
|
Add
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="kv-branch-section">
|
||||||
|
<h4>Rename branch</h4>
|
||||||
|
<div className="kv-branch-row">
|
||||||
|
<span className="kv-branch-prefix">→</span>
|
||||||
|
<input
|
||||||
|
className="kv-leaf-input"
|
||||||
|
placeholder="new prefix (e.g. file:newhash/)"
|
||||||
|
value={renameTo}
|
||||||
|
onChange={(e) => setRenameTo(e.target.value)}
|
||||||
|
disabled={busy}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
<p className="kv-branch-note">
|
||||||
|
Non-atomic: copies each descendant to the new prefix then deletes the original.
|
||||||
|
Failures are surfaced individually.
|
||||||
|
</p>
|
||||||
|
<button className="kv-btn" disabled={!renameTo || busy} onClick={renameBranch}>
|
||||||
|
Rename
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
|
||||||
|
<section className="kv-branch-section">
|
||||||
|
<h4>Delete subtree</h4>
|
||||||
|
<p className="kv-branch-note">
|
||||||
|
Deletes every descendant key. Cannot be undone.
|
||||||
|
</p>
|
||||||
|
<button className="kv-btn kv-btn-danger" disabled={busy || count === 0} onClick={deleteSubtree}>
|
||||||
|
Delete {count ?? ''} key{count === 1 ? '' : 's'}
|
||||||
|
</button>
|
||||||
|
</section>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,675 @@
|
|||||||
|
import { useEffect, useMemo, useRef, useState } from 'react';
|
||||||
|
import { KVAPI } from '../api/kvApi';
|
||||||
|
import { SchemaFieldInfo, useSchema } from '../api/schemaApi';
|
||||||
|
import { LanguageCombobox, formatLanguageOption } from '../components/LanguageCombobox';
|
||||||
|
import { LeafValueEditor } from './leafRenderers';
|
||||||
|
import { schemaToFieldType } from './schemaLookup';
|
||||||
|
import { detectType } from './typeHeuristics';
|
||||||
|
import { FieldType } from './types';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
/** Branch path of a file, e.g. "file:<hash>/". */
|
||||||
|
branchPath: string;
|
||||||
|
reloadToken: number;
|
||||||
|
onChanged: (info: string, errors?: string[]) => void;
|
||||||
|
/** Navigate to a single leaf for the manual deep editor. */
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
/**
|
||||||
|
* When set, scroll to and pulse the row matching this logical field path
|
||||||
|
* (e.g. "releasedate", "plot", "stream/0"). The `token` lets the same field
|
||||||
|
* fire a fresh scroll across consecutive clicks.
|
||||||
|
*/
|
||||||
|
scrollToField?: { field: string; token: number } | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
const LANG_CODE_RE = /^[a-z]{2,3}$/;
|
||||||
|
|
||||||
|
interface LeafRow {
|
||||||
|
kind: 'leaf';
|
||||||
|
fieldKey: string;
|
||||||
|
value: string;
|
||||||
|
fieldType: FieldType;
|
||||||
|
info?: SchemaFieldInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LanguageRow {
|
||||||
|
kind: 'language-group';
|
||||||
|
fieldKey: string;
|
||||||
|
/** Current language → text map. */
|
||||||
|
languageMap: Record<string, string>;
|
||||||
|
info?: SchemaFieldInfo;
|
||||||
|
}
|
||||||
|
|
||||||
|
type Row = LeafRow | LanguageRow;
|
||||||
|
|
||||||
|
interface RowGroup {
|
||||||
|
prefix: string;
|
||||||
|
isCluster: boolean;
|
||||||
|
rows: Row[];
|
||||||
|
}
|
||||||
|
|
||||||
|
export function KVFileEditor({
|
||||||
|
branchPath,
|
||||||
|
reloadToken,
|
||||||
|
onChanged,
|
||||||
|
onSelectLeaf,
|
||||||
|
scrollToField,
|
||||||
|
}: Props) {
|
||||||
|
const [metadata, setMetadata] = useState<Record<string, string> | null>(null);
|
||||||
|
const [edits, setEdits] = useState<Map<string, string | null>>(new Map());
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
const [collapsedGroups, setCollapsedGroups] = useState<Set<string>>(new Set());
|
||||||
|
const [pulse, setPulse] = useState<string | null>(null);
|
||||||
|
const [newFieldKey, setNewFieldKey] = useState('');
|
||||||
|
const [newFieldValue, setNewFieldValue] = useState('');
|
||||||
|
const [addingField, setAddingField] = useState(false);
|
||||||
|
// Track whether we've seeded collapsedGroups for the current branch; resets
|
||||||
|
// when the branch changes so each file gets its defaults reapplied.
|
||||||
|
const defaultsAppliedRef = useRef<string | null>(null);
|
||||||
|
|
||||||
|
const { schema } = useSchema();
|
||||||
|
|
||||||
|
// When the parent passes a target field, expand its prefix group (if any)
|
||||||
|
// and scroll the row into view. We also briefly pulse the row to draw the
|
||||||
|
// eye — the pulse is reset after the animation duration.
|
||||||
|
useEffect(() => {
|
||||||
|
if (!scrollToField) return;
|
||||||
|
const { field } = scrollToField;
|
||||||
|
const prefix = field.split('/', 1)[0];
|
||||||
|
setCollapsedGroups((cur) => {
|
||||||
|
if (!cur.has(prefix)) return cur;
|
||||||
|
const next = new Set(cur);
|
||||||
|
next.delete(prefix);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
requestAnimationFrame(() => {
|
||||||
|
const el = document.querySelector(
|
||||||
|
`[data-row-key="${cssEscape(field)}"]`,
|
||||||
|
) as HTMLElement | null;
|
||||||
|
if (el) {
|
||||||
|
el.scrollIntoView({ behavior: 'smooth', block: 'center' });
|
||||||
|
setPulse(field);
|
||||||
|
window.setTimeout(() => setPulse((p) => (p === field ? null : p)), 1400);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
}, [scrollToField?.field, scrollToField?.token]);
|
||||||
|
|
||||||
|
const hashId = branchPath.slice('file:'.length, -1);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
KVAPI.treeAll(branchPath, '/', 5000)
|
||||||
|
.then(async (resp) => {
|
||||||
|
// resp.leaves = full keys under branchPath. Recursive descent: also
|
||||||
|
// walk branch children (they're paths to sub-objects with their own
|
||||||
|
// leaves, e.g. file:<h>/plot/ → plot/eng).
|
||||||
|
const all: string[] = [...resp.leaves];
|
||||||
|
const queue = [...resp.branches];
|
||||||
|
while (queue.length > 0) {
|
||||||
|
const next = queue.shift()!;
|
||||||
|
try {
|
||||||
|
const sub = await KVAPI.treeAll(next, '/', 5000);
|
||||||
|
all.push(...sub.leaves);
|
||||||
|
queue.push(...sub.branches);
|
||||||
|
} catch {
|
||||||
|
// Ignore failures on a sub-branch; show what we can.
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (cancelled) return;
|
||||||
|
// MGET-equivalent: parallel fetches with a small cap to avoid burst.
|
||||||
|
const values = await Promise.all(
|
||||||
|
all.map((k) => KVAPI.get(k).then((r) => [k, r.exists ? r.value : null] as const).catch(() => [k, null] as const)),
|
||||||
|
);
|
||||||
|
if (cancelled) return;
|
||||||
|
const flat: Record<string, string> = {};
|
||||||
|
for (const [k, v] of values) {
|
||||||
|
if (v == null) continue;
|
||||||
|
// Strip the branch prefix so keys are the logical field path
|
||||||
|
// ("releasedate", "plot/eng", "stream/0").
|
||||||
|
const fieldKey = k.slice(branchPath.length);
|
||||||
|
if (fieldKey) flat[fieldKey] = v;
|
||||||
|
}
|
||||||
|
setMetadata(flat);
|
||||||
|
setEdits(new Map());
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
if (!cancelled) setMetadata({});
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
if (!cancelled) setLoading(false);
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
cancelled = true;
|
||||||
|
};
|
||||||
|
}, [branchPath, reloadToken]);
|
||||||
|
|
||||||
|
const groups = useMemo(
|
||||||
|
() => groupRows(buildRows(metadata ?? {}, schema)),
|
||||||
|
[metadata, schema],
|
||||||
|
);
|
||||||
|
|
||||||
|
// Default state: every cluster group (e.g. stream/*, cid_*) starts collapsed.
|
||||||
|
// Reapplied when the user switches to a different file.
|
||||||
|
useEffect(() => {
|
||||||
|
if (groups.length === 0) return;
|
||||||
|
if (defaultsAppliedRef.current === branchPath) return;
|
||||||
|
defaultsAppliedRef.current = branchPath;
|
||||||
|
setCollapsedGroups(new Set(groups.filter((g) => g.isCluster).map((g) => g.prefix)));
|
||||||
|
}, [groups, branchPath]);
|
||||||
|
|
||||||
|
if (loading) {
|
||||||
|
return <div className="kv-pane-status">loading file…</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!metadata) {
|
||||||
|
return <div className="kv-pane-status">no data for {branchPath}</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
const fileName = metadata['fileName'];
|
||||||
|
const filePath = metadata['filePath'];
|
||||||
|
|
||||||
|
const onLeafChange = (fieldKey: string, value: string) => {
|
||||||
|
setEdits((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
const original = metadata[fieldKey] ?? null;
|
||||||
|
if (value === original) next.delete(fieldKey);
|
||||||
|
else next.set(fieldKey, value);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Toggle a field's pending-delete state. Re-clicking restores; if the field
|
||||||
|
* is brand-new (no original value) we just drop it from the edits map.
|
||||||
|
*/
|
||||||
|
const onLeafDeleteToggle = (fieldKey: string) => {
|
||||||
|
setEdits((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
if (next.get(fieldKey) === null) {
|
||||||
|
// Currently pending delete → undo.
|
||||||
|
next.delete(fieldKey);
|
||||||
|
} else {
|
||||||
|
next.set(fieldKey, null);
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleAddField = async () => {
|
||||||
|
const key = newFieldKey.trim();
|
||||||
|
if (!key) return;
|
||||||
|
const fullKey = branchPath + key;
|
||||||
|
setAddingField(true);
|
||||||
|
try {
|
||||||
|
await KVAPI.put(fullKey, newFieldValue);
|
||||||
|
setNewFieldKey('');
|
||||||
|
setNewFieldValue('');
|
||||||
|
onChanged(`Added ${fullKey}`);
|
||||||
|
// Refetch this file so the new field appears in the row list.
|
||||||
|
const r = await KVAPI.get(fullKey);
|
||||||
|
if (r.exists) {
|
||||||
|
setMetadata((cur) => ({ ...(cur ?? {}), [key]: r.value }));
|
||||||
|
}
|
||||||
|
} catch (e: any) {
|
||||||
|
onChanged(`Failed to add ${fullKey}`, [e.message]);
|
||||||
|
} finally {
|
||||||
|
setAddingField(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const onLanguageGroupChange = (
|
||||||
|
groupKey: string,
|
||||||
|
nextMap: Record<string, string>,
|
||||||
|
originalMap: Record<string, string>,
|
||||||
|
) => {
|
||||||
|
setEdits((prev) => {
|
||||||
|
const next = new Map(prev);
|
||||||
|
const allLangs = new Set([...Object.keys(originalMap), ...Object.keys(nextMap)]);
|
||||||
|
for (const lang of allLangs) {
|
||||||
|
const fieldKey = `${groupKey}/${lang}`;
|
||||||
|
const orig = originalMap[lang] ?? null;
|
||||||
|
const cur = nextMap[lang] ?? null;
|
||||||
|
if (cur === orig) next.delete(fieldKey);
|
||||||
|
else if (cur == null) next.set(fieldKey, null);
|
||||||
|
else next.set(fieldKey, cur);
|
||||||
|
}
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleSave = async () => {
|
||||||
|
if (edits.size === 0) return;
|
||||||
|
setSaving(true);
|
||||||
|
const errors: string[] = [];
|
||||||
|
for (const [fieldKey, newValue] of edits) {
|
||||||
|
const fullKey = branchPath + fieldKey;
|
||||||
|
try {
|
||||||
|
if (newValue === null) await KVAPI.del(fullKey);
|
||||||
|
else await KVAPI.put(fullKey, newValue);
|
||||||
|
} catch (e: any) {
|
||||||
|
errors.push(`${fieldKey}: ${e.message}`);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
setSaving(false);
|
||||||
|
if (errors.length === 0) {
|
||||||
|
onChanged(`Saved ${edits.size} field${edits.size === 1 ? '' : 's'} on ${fileName ?? hashId}`);
|
||||||
|
} else {
|
||||||
|
onChanged(
|
||||||
|
`Saved with ${errors.length} failure(s) on ${fileName ?? hashId}`,
|
||||||
|
errors,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const handleDiscard = () => setEdits(new Map());
|
||||||
|
|
||||||
|
const toggleGroup = (prefix: string) => {
|
||||||
|
setCollapsedGroups((cur) => {
|
||||||
|
const next = new Set(cur);
|
||||||
|
if (next.has(prefix)) next.delete(prefix);
|
||||||
|
else next.add(prefix);
|
||||||
|
return next;
|
||||||
|
});
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-file">
|
||||||
|
<div className="kv-file-header">
|
||||||
|
<div className="kv-file-name">
|
||||||
|
{fileName ?? <span className="kv-file-name-fallback">(no fileName)</span>}
|
||||||
|
</div>
|
||||||
|
<div className="kv-file-meta">
|
||||||
|
<code className="kv-file-hash" title={hashId}>
|
||||||
|
{hashId.slice(0, 8)}…{hashId.slice(-6)}
|
||||||
|
</code>
|
||||||
|
{filePath && <span className="kv-file-path" title={filePath}>{filePath}</span>}
|
||||||
|
</div>
|
||||||
|
<div className="kv-file-actions">
|
||||||
|
<button
|
||||||
|
className="kv-btn kv-btn-primary"
|
||||||
|
disabled={edits.size === 0 || saving}
|
||||||
|
onClick={handleSave}
|
||||||
|
>
|
||||||
|
{saving ? 'saving…' : `Save (${edits.size})`}
|
||||||
|
</button>
|
||||||
|
<button
|
||||||
|
className="kv-btn"
|
||||||
|
disabled={edits.size === 0 || saving}
|
||||||
|
onClick={handleDiscard}
|
||||||
|
>
|
||||||
|
Discard
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="kv-file-rows">
|
||||||
|
{groups.length === 0 && (
|
||||||
|
<div className="kv-pane-status">No data yet for this file.</div>
|
||||||
|
)}
|
||||||
|
{groups.map((g) => (
|
||||||
|
<FieldGroupView
|
||||||
|
key={g.prefix}
|
||||||
|
group={g}
|
||||||
|
edits={edits}
|
||||||
|
collapsed={collapsedGroups.has(g.prefix)}
|
||||||
|
pulse={pulse}
|
||||||
|
onToggle={() => toggleGroup(g.prefix)}
|
||||||
|
onLeafChange={onLeafChange}
|
||||||
|
onLeafDeleteToggle={onLeafDeleteToggle}
|
||||||
|
onLanguageChange={onLanguageGroupChange}
|
||||||
|
onJumpToLeaf={(fieldKey) => onSelectLeaf(branchPath + fieldKey)}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<div className="kv-file-add">
|
||||||
|
<span className="kv-file-add-prefix">{branchPath}</span>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="kv-leaf-input kv-leaf-mono"
|
||||||
|
placeholder="field path (may contain /)"
|
||||||
|
value={newFieldKey}
|
||||||
|
onChange={(e) => setNewFieldKey(e.target.value)}
|
||||||
|
disabled={addingField}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="kv-leaf-input"
|
||||||
|
placeholder="value"
|
||||||
|
value={newFieldValue}
|
||||||
|
onChange={(e) => setNewFieldValue(e.target.value)}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === 'Enter' && newFieldKey.trim()) handleAddField();
|
||||||
|
}}
|
||||||
|
disabled={addingField}
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
className="kv-btn kv-btn-primary"
|
||||||
|
disabled={!newFieldKey.trim() || addingField}
|
||||||
|
onClick={handleAddField}
|
||||||
|
>
|
||||||
|
{addingField ? 'adding…' : '+ Add field'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function FieldGroupView({
|
||||||
|
group,
|
||||||
|
edits,
|
||||||
|
collapsed,
|
||||||
|
pulse,
|
||||||
|
onToggle,
|
||||||
|
onLeafChange,
|
||||||
|
onLeafDeleteToggle,
|
||||||
|
onLanguageChange,
|
||||||
|
onJumpToLeaf,
|
||||||
|
}: {
|
||||||
|
group: RowGroup;
|
||||||
|
edits: Map<string, string | null>;
|
||||||
|
collapsed: boolean;
|
||||||
|
pulse: string | null;
|
||||||
|
onToggle: () => void;
|
||||||
|
onLeafChange: (fieldKey: string, value: string) => void;
|
||||||
|
onLeafDeleteToggle: (fieldKey: string) => void;
|
||||||
|
onLanguageChange: (
|
||||||
|
groupKey: string,
|
||||||
|
nextMap: Record<string, string>,
|
||||||
|
originalMap: Record<string, string>,
|
||||||
|
) => void;
|
||||||
|
onJumpToLeaf: (fieldKey: string) => void;
|
||||||
|
}) {
|
||||||
|
if (!group.isCluster) {
|
||||||
|
return (
|
||||||
|
<>
|
||||||
|
{group.rows.map((row) => (
|
||||||
|
<RowView
|
||||||
|
key={row.fieldKey}
|
||||||
|
row={row}
|
||||||
|
edits={edits}
|
||||||
|
pulse={pulse}
|
||||||
|
onLeafChange={onLeafChange}
|
||||||
|
onLeafDeleteToggle={onLeafDeleteToggle}
|
||||||
|
onLanguageChange={onLanguageChange}
|
||||||
|
onJumpToLeaf={onJumpToLeaf}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<div className={`kv-file-group ${collapsed ? 'collapsed' : ''}`}>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-file-group-header"
|
||||||
|
onClick={onToggle}
|
||||||
|
title={collapsed ? 'expand' : 'collapse'}
|
||||||
|
>
|
||||||
|
<span className="kv-file-group-chevron">{collapsed ? '▸' : '▾'}</span>
|
||||||
|
<span className="kv-file-group-name">{group.prefix}</span>
|
||||||
|
<span className="kv-file-group-count">{group.rows.length}</span>
|
||||||
|
</button>
|
||||||
|
{!collapsed && (
|
||||||
|
<div className="kv-file-group-body">
|
||||||
|
{group.rows.map((row) => (
|
||||||
|
<RowView
|
||||||
|
key={row.fieldKey}
|
||||||
|
row={row}
|
||||||
|
edits={edits}
|
||||||
|
pulse={pulse}
|
||||||
|
onLeafChange={onLeafChange}
|
||||||
|
onLeafDeleteToggle={onLeafDeleteToggle}
|
||||||
|
onLanguageChange={onLanguageChange}
|
||||||
|
onJumpToLeaf={onJumpToLeaf}
|
||||||
|
/>
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function RowView({
|
||||||
|
row,
|
||||||
|
edits,
|
||||||
|
pulse,
|
||||||
|
onLeafChange,
|
||||||
|
onLeafDeleteToggle,
|
||||||
|
onLanguageChange,
|
||||||
|
onJumpToLeaf,
|
||||||
|
}: {
|
||||||
|
row: Row;
|
||||||
|
edits: Map<string, string | null>;
|
||||||
|
pulse: string | null;
|
||||||
|
onLeafChange: (fieldKey: string, value: string) => void;
|
||||||
|
onLeafDeleteToggle: (fieldKey: string) => void;
|
||||||
|
onLanguageChange: (
|
||||||
|
groupKey: string,
|
||||||
|
nextMap: Record<string, string>,
|
||||||
|
originalMap: Record<string, string>,
|
||||||
|
) => void;
|
||||||
|
onJumpToLeaf: (fieldKey: string) => void;
|
||||||
|
}) {
|
||||||
|
const pulsed = pulse === row.fieldKey;
|
||||||
|
if (row.kind === 'leaf') {
|
||||||
|
const pendingDelete = edits.get(row.fieldKey) === null;
|
||||||
|
const draft = edits.has(row.fieldKey) ? (edits.get(row.fieldKey) ?? '') : row.value;
|
||||||
|
const dirty = edits.has(row.fieldKey);
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={`kv-file-row ${dirty ? 'dirty' : ''} ${pendingDelete ? 'pending-delete' : ''} ${pulsed ? 'pulsed' : ''}`}
|
||||||
|
data-row-key={row.fieldKey}
|
||||||
|
>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-file-row-label"
|
||||||
|
onClick={() => onJumpToLeaf(row.fieldKey)}
|
||||||
|
title="open in single-leaf editor"
|
||||||
|
>
|
||||||
|
{row.fieldKey}
|
||||||
|
</button>
|
||||||
|
<div className="kv-file-row-value">
|
||||||
|
{pendingDelete ? (
|
||||||
|
<div className="kv-file-row-deleted-note">marked for deletion</div>
|
||||||
|
) : (
|
||||||
|
<LeafValueEditor
|
||||||
|
type={row.fieldType}
|
||||||
|
value={draft}
|
||||||
|
onChange={(v) => onLeafChange(row.fieldKey, v)}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-file-row-delete"
|
||||||
|
onClick={() => onLeafDeleteToggle(row.fieldKey)}
|
||||||
|
title={pendingDelete ? 'undo deletion' : 'delete this field'}
|
||||||
|
aria-label={pendingDelete ? 'undo deletion' : 'delete this field'}
|
||||||
|
>
|
||||||
|
{pendingDelete ? '↺' : '×'}
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<LanguageGroupRow
|
||||||
|
row={row}
|
||||||
|
edits={edits}
|
||||||
|
pulsed={pulsed}
|
||||||
|
onLanguageChange={onLanguageChange}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function LanguageGroupRow({
|
||||||
|
row,
|
||||||
|
edits,
|
||||||
|
pulsed,
|
||||||
|
onLanguageChange,
|
||||||
|
}: {
|
||||||
|
row: LanguageRow;
|
||||||
|
edits: Map<string, string | null>;
|
||||||
|
pulsed: boolean;
|
||||||
|
onLanguageChange: (
|
||||||
|
groupKey: string,
|
||||||
|
nextMap: Record<string, string>,
|
||||||
|
originalMap: Record<string, string>,
|
||||||
|
) => void;
|
||||||
|
}) {
|
||||||
|
// Apply pending edits on top of the original language map.
|
||||||
|
const effectiveMap: Record<string, string> = { ...row.languageMap };
|
||||||
|
for (const [k, v] of edits) {
|
||||||
|
if (!k.startsWith(`${row.fieldKey}/`)) continue;
|
||||||
|
const lang = k.slice(row.fieldKey.length + 1);
|
||||||
|
if (v === null) delete effectiveMap[lang];
|
||||||
|
else effectiveMap[lang] = v;
|
||||||
|
}
|
||||||
|
|
||||||
|
const entries = Object.entries(effectiveMap);
|
||||||
|
const dirty = [...edits.keys()].some((k) => k.startsWith(`${row.fieldKey}/`));
|
||||||
|
|
||||||
|
const updateLang = (lang: string, text: string) => {
|
||||||
|
onLanguageChange(row.fieldKey, { ...effectiveMap, [lang]: text }, row.languageMap);
|
||||||
|
};
|
||||||
|
|
||||||
|
const renameLang = (oldLang: string, newLang: string) => {
|
||||||
|
if (oldLang === newLang) return;
|
||||||
|
const next = { ...effectiveMap };
|
||||||
|
const text = next[oldLang];
|
||||||
|
delete next[oldLang];
|
||||||
|
next[newLang] = text;
|
||||||
|
onLanguageChange(row.fieldKey, next, row.languageMap);
|
||||||
|
};
|
||||||
|
|
||||||
|
const removeLang = (lang: string) => {
|
||||||
|
const next = { ...effectiveMap };
|
||||||
|
delete next[lang];
|
||||||
|
onLanguageChange(row.fieldKey, next, row.languageMap);
|
||||||
|
};
|
||||||
|
|
||||||
|
const addLang = () => {
|
||||||
|
const used = new Set(entries.map(([k]) => k));
|
||||||
|
const defaults = ['eng', 'jpn', 'zho', 'kor', 'fra', 'deu', 'spa', 'fre'];
|
||||||
|
const defaultLang = defaults.find((l) => !used.has(l)) || 'eng';
|
||||||
|
onLanguageChange(row.fieldKey, { ...effectiveMap, [defaultLang]: '' }, row.languageMap);
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={`kv-file-row kv-file-row-lang ${dirty ? 'dirty' : ''} ${pulsed ? 'pulsed' : ''}`}
|
||||||
|
data-row-key={row.fieldKey}
|
||||||
|
>
|
||||||
|
<div className="kv-file-row-label-static">{row.fieldKey}</div>
|
||||||
|
<div className="kv-file-row-value">
|
||||||
|
<div className="kv-file-lang-entries">
|
||||||
|
{entries.map(([lang, text]) => (
|
||||||
|
<div key={lang} className="kv-file-lang-entry">
|
||||||
|
<LanguageCombobox
|
||||||
|
value={lang}
|
||||||
|
onChange={(newLang) => renameLang(lang, newLang)}
|
||||||
|
/>
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={text}
|
||||||
|
title={formatLanguageOption(lang)}
|
||||||
|
onChange={(e) => updateLang(lang, e.target.value)}
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-btn-icon kv-file-lang-remove"
|
||||||
|
onClick={() => removeLang(lang)}
|
||||||
|
title="remove this language"
|
||||||
|
>
|
||||||
|
×
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
))}
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-btn kv-file-lang-add"
|
||||||
|
onClick={addLang}
|
||||||
|
>
|
||||||
|
+ Add language
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// ──────────────────────────────────────────────────────────────────────────
|
||||||
|
// Row model — kept local because it's only used here and is small.
|
||||||
|
// ──────────────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
function buildRows(
|
||||||
|
flat: Record<string, string>,
|
||||||
|
schema: ReturnType<typeof useSchema>['schema'],
|
||||||
|
): Row[] {
|
||||||
|
const languageGroupKeys = new Set<string>();
|
||||||
|
if (schema) {
|
||||||
|
for (const [k, info] of Object.entries(schema.fields)) {
|
||||||
|
if (info.key_hint === 'language-code') languageGroupKeys.add(k);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const langMaps: Record<string, Record<string, string>> = {};
|
||||||
|
const rows: Row[] = [];
|
||||||
|
|
||||||
|
for (const [key, value] of Object.entries(flat)) {
|
||||||
|
const slash = key.lastIndexOf('/');
|
||||||
|
if (slash > 0) {
|
||||||
|
const prefix = key.slice(0, slash);
|
||||||
|
const tail = key.slice(slash + 1);
|
||||||
|
if (languageGroupKeys.has(prefix) && LANG_CODE_RE.test(tail)) {
|
||||||
|
if (!langMaps[prefix]) langMaps[prefix] = {};
|
||||||
|
langMaps[prefix][tail] = value;
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
const info = schema?.fields[key];
|
||||||
|
rows.push({
|
||||||
|
kind: 'leaf',
|
||||||
|
fieldKey: key,
|
||||||
|
value,
|
||||||
|
// Schema decides when it can; otherwise heuristic on (path, value).
|
||||||
|
fieldType: schemaToFieldType(info) ?? detectType(key, value),
|
||||||
|
info,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const [groupKey, langMap] of Object.entries(langMaps)) {
|
||||||
|
rows.push({
|
||||||
|
kind: 'language-group',
|
||||||
|
fieldKey: groupKey,
|
||||||
|
languageMap: langMap,
|
||||||
|
info: schema?.fields[groupKey],
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
rows.sort((a, b) => a.fieldKey.localeCompare(b.fieldKey));
|
||||||
|
return rows;
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Escape characters that would break a CSS attribute selector value. */
|
||||||
|
function cssEscape(s: string): string {
|
||||||
|
return s.replace(/["\\]/g, (c) => `\\${c}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function groupRows(rows: Row[]): RowGroup[] {
|
||||||
|
const out: RowGroup[] = [];
|
||||||
|
for (const row of rows) {
|
||||||
|
const prefix = row.fieldKey.split('/', 1)[0];
|
||||||
|
const last = out[out.length - 1];
|
||||||
|
if (last && last.prefix === prefix) {
|
||||||
|
last.rows.push(row);
|
||||||
|
} else {
|
||||||
|
out.push({ prefix, isCluster: false, rows: [row] });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for (const g of out) g.isCluster = g.rows.length > 1;
|
||||||
|
return out;
|
||||||
|
}
|
||||||
@@ -0,0 +1,160 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { KVAPI } from '../api/kvApi';
|
||||||
|
import { useSchema } from '../api/schemaApi';
|
||||||
|
import { ALL_TYPES, describeType, detectType } from './typeHeuristics';
|
||||||
|
import { lookupSchemaForKey } from './schemaLookup';
|
||||||
|
import { LeafValueEditor } from './leafRenderers';
|
||||||
|
import { FieldType } from './types';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
// Full Redis key path of the selected leaf.
|
||||||
|
selectedKey: string;
|
||||||
|
// Bumped after a delete or external change to force refetch.
|
||||||
|
reloadToken: number;
|
||||||
|
onSaved: () => void;
|
||||||
|
onDeleted: () => void;
|
||||||
|
onError: (msg: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function KVLeafEditor({ selectedKey, reloadToken, onSaved, onDeleted, onError }: Props) {
|
||||||
|
const [original, setOriginal] = useState<string>('');
|
||||||
|
const [draft, setDraft] = useState<string>('');
|
||||||
|
const [exists, setExists] = useState<boolean>(true);
|
||||||
|
const [redisType, setRedisType] = useState<string>('');
|
||||||
|
const [loading, setLoading] = useState(true);
|
||||||
|
const [saving, setSaving] = useState(false);
|
||||||
|
|
||||||
|
// Manual override for the inferred type. "auto" defers to schema → heuristic.
|
||||||
|
const [typeOverride, setTypeOverride] = useState<FieldType>('auto');
|
||||||
|
const { schema } = useSchema();
|
||||||
|
const schemaLookup = lookupSchemaForKey(schema, selectedKey);
|
||||||
|
// Schema is authoritative when it has an opinion; heuristics fill the gap.
|
||||||
|
const detected = schemaLookup.fieldType ?? detectType(selectedKey, original);
|
||||||
|
const effectiveType = typeOverride === 'auto' ? detected : typeOverride;
|
||||||
|
|
||||||
|
// Reset override when the selected key changes — the previous override
|
||||||
|
// doesn't apply to a different leaf.
|
||||||
|
useEffect(() => { setTypeOverride('auto'); }, [selectedKey]);
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
let cancelled = false;
|
||||||
|
setLoading(true);
|
||||||
|
KVAPI.get(selectedKey)
|
||||||
|
.then((r) => {
|
||||||
|
if (cancelled) return;
|
||||||
|
setOriginal(r.value);
|
||||||
|
setDraft(r.value);
|
||||||
|
setExists(r.exists);
|
||||||
|
setRedisType(r.type);
|
||||||
|
})
|
||||||
|
.catch((e) => onError(e.message))
|
||||||
|
.finally(() => { if (!cancelled) setLoading(false); });
|
||||||
|
return () => { cancelled = true; };
|
||||||
|
}, [selectedKey, reloadToken, onError]);
|
||||||
|
|
||||||
|
const dirty = draft !== original;
|
||||||
|
const editable = exists ? redisType === 'string' || redisType === '' : true;
|
||||||
|
|
||||||
|
const save = async () => {
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
await KVAPI.put(selectedKey, draft);
|
||||||
|
setOriginal(draft);
|
||||||
|
setExists(true);
|
||||||
|
setRedisType('string');
|
||||||
|
onSaved();
|
||||||
|
} catch (e: any) {
|
||||||
|
onError(e.message);
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
const del = async () => {
|
||||||
|
if (!confirm(`Delete ${selectedKey}?`)) return;
|
||||||
|
setSaving(true);
|
||||||
|
try {
|
||||||
|
await KVAPI.del(selectedKey);
|
||||||
|
onDeleted();
|
||||||
|
} catch (e: any) {
|
||||||
|
onError(e.message);
|
||||||
|
} finally {
|
||||||
|
setSaving(false);
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
if (loading) {
|
||||||
|
return <div className="kv-pane-status">loading…</div>;
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!editable) {
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf">
|
||||||
|
<KeyHeader keyPath={selectedKey} />
|
||||||
|
<div className="kv-pane-status">
|
||||||
|
Key has Redis type <code>{redisType}</code> — only string-typed values are editable here.
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf">
|
||||||
|
<KeyHeader keyPath={selectedKey} />
|
||||||
|
|
||||||
|
<div className="kv-leaf-meta">
|
||||||
|
<label className="kv-leaf-label">Type</label>
|
||||||
|
<select
|
||||||
|
className="kv-leaf-type"
|
||||||
|
value={typeOverride}
|
||||||
|
onChange={(e) => setTypeOverride(e.target.value as FieldType)}
|
||||||
|
>
|
||||||
|
{ALL_TYPES.map((t) => (
|
||||||
|
<option key={t} value={t}>
|
||||||
|
{t === 'auto' ? `auto (${describeType(detected)})` : describeType(t)}
|
||||||
|
</option>
|
||||||
|
))}
|
||||||
|
</select>
|
||||||
|
{!exists && <span className="kv-leaf-hint">new key — save to create</span>}
|
||||||
|
</div>
|
||||||
|
|
||||||
|
<LeafValueEditor type={effectiveType} value={draft} onChange={setDraft} />
|
||||||
|
|
||||||
|
<div className="kv-leaf-actions">
|
||||||
|
<button
|
||||||
|
className="kv-btn kv-btn-primary"
|
||||||
|
onClick={save}
|
||||||
|
disabled={!dirty || saving}
|
||||||
|
>{saving ? 'saving…' : exists ? 'Save' : 'Create'}</button>
|
||||||
|
<button
|
||||||
|
className="kv-btn"
|
||||||
|
onClick={() => setDraft(original)}
|
||||||
|
disabled={!dirty || saving}
|
||||||
|
>Reset</button>
|
||||||
|
{exists && (
|
||||||
|
<button
|
||||||
|
className="kv-btn kv-btn-danger"
|
||||||
|
onClick={del}
|
||||||
|
disabled={saving}
|
||||||
|
>Delete</button>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function KeyHeader({ keyPath }: { keyPath: string }) {
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf-key">
|
||||||
|
<code className="kv-leaf-key-text">{keyPath}</code>
|
||||||
|
<button
|
||||||
|
className="kv-btn-icon"
|
||||||
|
onClick={() => navigator.clipboard.writeText(keyPath)}
|
||||||
|
title="copy key"
|
||||||
|
>⧉</button>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Leaf value rendering moved to leafRenderers.tsx (LeafValueEditor)
|
||||||
|
// so the file-detail view can reuse it per row.
|
||||||
@@ -0,0 +1,339 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { KVAPI } from '../api/kvApi';
|
||||||
|
|
||||||
|
const FILE_BRANCH_RE = /^file:[^/]+\/$/;
|
||||||
|
const fileNameCache = new Map<string, string | null>();
|
||||||
|
const fileNamePromises = new Map<string, Promise<void>>();
|
||||||
|
const fileNameSubscribers = new Set<() => void>();
|
||||||
|
|
||||||
|
function notifyFileNameSubscribers() {
|
||||||
|
for (const cb of fileNameSubscribers) cb();
|
||||||
|
}
|
||||||
|
|
||||||
|
function ensureFileName(branchPath: string): void {
|
||||||
|
if (fileNameCache.has(branchPath) || fileNamePromises.has(branchPath)) return;
|
||||||
|
const p = KVAPI.get(branchPath + 'fileName')
|
||||||
|
.then((r) => {
|
||||||
|
fileNameCache.set(branchPath, r.exists && r.value ? r.value : null);
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
fileNameCache.set(branchPath, null);
|
||||||
|
})
|
||||||
|
.finally(() => {
|
||||||
|
fileNamePromises.delete(branchPath);
|
||||||
|
notifyFileNameSubscribers();
|
||||||
|
});
|
||||||
|
fileNamePromises.set(branchPath, p);
|
||||||
|
}
|
||||||
|
|
||||||
|
function useFileName(branchPath: string | null): string | null {
|
||||||
|
const [, force] = useState(0);
|
||||||
|
useEffect(() => {
|
||||||
|
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return;
|
||||||
|
ensureFileName(branchPath);
|
||||||
|
const cb = () => force((n) => n + 1);
|
||||||
|
fileNameSubscribers.add(cb);
|
||||||
|
return () => {
|
||||||
|
fileNameSubscribers.delete(cb);
|
||||||
|
};
|
||||||
|
}, [branchPath]);
|
||||||
|
if (!branchPath || !FILE_BRANCH_RE.test(branchPath)) return null;
|
||||||
|
return fileNameCache.get(branchPath) ?? null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function isFileBranch(path: string): boolean {
|
||||||
|
return FILE_BRANCH_RE.test(path);
|
||||||
|
}
|
||||||
|
|
||||||
|
function shortHashFromFileBranch(path: string): string {
|
||||||
|
// path = "file:<hash>/" → last 6 chars of hash
|
||||||
|
const hash = path.slice('file:'.length, -1);
|
||||||
|
if (hash.length <= 12) return hash;
|
||||||
|
return `…${hash.slice(-6)}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
// A node in the lazy tree. branch-typed nodes have an associated path that
|
||||||
|
// ends with the delimiter and lazy-load their children on first expand.
|
||||||
|
type NodeKind = 'branch' | 'leaf';
|
||||||
|
|
||||||
|
interface TreeNode {
|
||||||
|
kind: NodeKind;
|
||||||
|
// Display label: the segment of the path under the parent.
|
||||||
|
label: string;
|
||||||
|
// Full key path. For branches, ends with the delimiter.
|
||||||
|
path: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BranchState {
|
||||||
|
loaded: boolean;
|
||||||
|
loading: boolean;
|
||||||
|
error?: string;
|
||||||
|
children: TreeNode[];
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sub-segment of a path used as the visible label for a branch / leaf.
|
||||||
|
// Splits on either ":" or "/" so file:abc/title/eng becomes file: > abc > title > eng.
|
||||||
|
function labelOf(path: string, parentPath: string): string {
|
||||||
|
const rest = path.slice(parentPath.length);
|
||||||
|
// The "rest" includes a trailing delimiter for branches; strip it for display.
|
||||||
|
return rest.replace(/[/:]+$/, '');
|
||||||
|
}
|
||||||
|
|
||||||
|
// Hide internal Redis structures from the tree. The editor is for managing
|
||||||
|
// file metadata; everything else is meta-core's own bookkeeping and noise
|
||||||
|
// for the user. Hidden:
|
||||||
|
// - `meta:` — event streams (file:events, meta:events, etc.)
|
||||||
|
// - `cid:` — reverse index (cid:<algo>:<value> → uuid), derived
|
||||||
|
// from each file's cid_* fields; the editor surfaces
|
||||||
|
// these per-file under their parent root
|
||||||
|
// - `meta-core:` — internal service state (mounts, watcher config,
|
||||||
|
// leader info), edited via dedicated UI pages, not
|
||||||
|
// the KV tree
|
||||||
|
// - bare top-level leaves under a colon-namespace like file:__index__
|
||||||
|
// and file:events (Redis sets / streams that aren't user data).
|
||||||
|
function isHiddenNode(node: TreeNode): boolean {
|
||||||
|
if (node.kind === 'branch') {
|
||||||
|
if (node.path === 'meta:' || node.path === 'cid:' || node.path === 'meta-core:') {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (node.kind === 'leaf' && /^[^/]+:[^/]+$/.test(node.path)) return true;
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
// Selected leaf path (full key) — highlighted.
|
||||||
|
selectedKey: string | null;
|
||||||
|
// When set, the tree forces these branches open (used for auto-expand on
|
||||||
|
// search hit or deep-link).
|
||||||
|
expandedPaths: Set<string>;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
onToggleBranch: (path: string, willOpen: boolean) => void;
|
||||||
|
// Bumping this triggers a full reload of root and any expanded branches.
|
||||||
|
reloadToken: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function KVTree({
|
||||||
|
selectedKey,
|
||||||
|
expandedPaths,
|
||||||
|
onSelectLeaf,
|
||||||
|
onSelectBranch,
|
||||||
|
onToggleBranch,
|
||||||
|
reloadToken,
|
||||||
|
}: Props) {
|
||||||
|
// Map of branch path → load state. The empty key "" represents the root.
|
||||||
|
const [branches, setBranches] = useState<Record<string, BranchState>>({});
|
||||||
|
|
||||||
|
// Load (or reload) a branch's children from the API. The root is special-
|
||||||
|
// cased: we list at prefix="" with delimiter=":" to surface the namespaces.
|
||||||
|
const loadBranch = async (path: string) => {
|
||||||
|
setBranches((b) => ({
|
||||||
|
...b,
|
||||||
|
[path]: { loaded: false, loading: true, children: b[path]?.children ?? [] },
|
||||||
|
}));
|
||||||
|
try {
|
||||||
|
const isRoot = path === '';
|
||||||
|
const delim = isRoot ? ':' : '/';
|
||||||
|
const resp = await KVAPI.treeAll(path, delim, 5000);
|
||||||
|
const branchNodes: TreeNode[] = resp.branches.map((p) => ({
|
||||||
|
kind: 'branch',
|
||||||
|
label: labelOf(p, path) + delim,
|
||||||
|
path: p,
|
||||||
|
}));
|
||||||
|
const leafNodes: TreeNode[] = resp.leaves.map((p) => ({
|
||||||
|
kind: 'leaf',
|
||||||
|
label: labelOf(p, path),
|
||||||
|
path: p,
|
||||||
|
}));
|
||||||
|
const children = [...branchNodes, ...leafNodes];
|
||||||
|
setBranches((b) => ({
|
||||||
|
...b,
|
||||||
|
[path]: { loaded: true, loading: false, children },
|
||||||
|
}));
|
||||||
|
} catch (err: any) {
|
||||||
|
setBranches((b) => ({
|
||||||
|
...b,
|
||||||
|
[path]: { loaded: false, loading: false, children: [], error: err.message },
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
// On reloadToken change, refetch root and every currently-expanded branch.
|
||||||
|
useEffect(() => {
|
||||||
|
loadBranch('');
|
||||||
|
for (const p of expandedPaths) {
|
||||||
|
if (p !== '') loadBranch(p);
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [reloadToken]);
|
||||||
|
|
||||||
|
// Whenever expandedPaths gains a path we haven't loaded yet, lazy-load it.
|
||||||
|
useEffect(() => {
|
||||||
|
for (const p of expandedPaths) {
|
||||||
|
if (p === '') continue;
|
||||||
|
const state = branches[p];
|
||||||
|
if (!state || (!state.loaded && !state.loading)) {
|
||||||
|
loadBranch(p);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// eslint-disable-next-line react-hooks/exhaustive-deps
|
||||||
|
}, [expandedPaths]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-tree">
|
||||||
|
<Branch
|
||||||
|
path=""
|
||||||
|
state={branches['']}
|
||||||
|
depth={0}
|
||||||
|
expandedPaths={expandedPaths}
|
||||||
|
branches={branches}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
onToggleBranch={(p, open) => {
|
||||||
|
onToggleBranch(p, open);
|
||||||
|
if (open && (!branches[p] || !branches[p].loaded)) loadBranch(p);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BranchProps {
|
||||||
|
path: string;
|
||||||
|
state: BranchState | undefined;
|
||||||
|
depth: number;
|
||||||
|
expandedPaths: Set<string>;
|
||||||
|
branches: Record<string, BranchState>;
|
||||||
|
selectedKey: string | null;
|
||||||
|
onSelectLeaf: (key: string) => void;
|
||||||
|
onSelectBranch: (path: string) => void;
|
||||||
|
onToggleBranch: (path: string, willOpen: boolean) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function Branch({
|
||||||
|
state, depth, expandedPaths, branches,
|
||||||
|
selectedKey, onSelectLeaf, onSelectBranch, onToggleBranch,
|
||||||
|
}: BranchProps) {
|
||||||
|
if (!state) {
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (state.loading && state.children.length === 0) {
|
||||||
|
return <div className="kv-tree-loading" style={{ paddingLeft: depth * 14 + 18 }}>loading…</div>;
|
||||||
|
}
|
||||||
|
if (state.error) {
|
||||||
|
return <div className="kv-tree-error" style={{ paddingLeft: depth * 14 + 18 }}>error: {state.error}</div>;
|
||||||
|
}
|
||||||
|
const visibleChildren = state.children.filter((c) => !isHiddenNode(c));
|
||||||
|
if (visibleChildren.length === 0) {
|
||||||
|
return <div className="kv-tree-empty" style={{ paddingLeft: depth * 14 + 18 }}>(empty)</div>;
|
||||||
|
}
|
||||||
|
return (
|
||||||
|
<ul className="kv-tree-list">
|
||||||
|
{visibleChildren.map((node) => {
|
||||||
|
if (node.kind === 'branch') {
|
||||||
|
const open = expandedPaths.has(node.path);
|
||||||
|
const childState = branches[node.path];
|
||||||
|
return (
|
||||||
|
<li key={node.path}>
|
||||||
|
<BranchRow
|
||||||
|
node={node}
|
||||||
|
open={open}
|
||||||
|
depth={depth}
|
||||||
|
onToggle={() => {
|
||||||
|
onToggleBranch(node.path, !open);
|
||||||
|
onSelectBranch(node.path);
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
{open && (
|
||||||
|
<Branch
|
||||||
|
path={node.path}
|
||||||
|
state={childState}
|
||||||
|
depth={depth + 1}
|
||||||
|
expandedPaths={expandedPaths}
|
||||||
|
branches={branches}
|
||||||
|
selectedKey={selectedKey}
|
||||||
|
onSelectLeaf={onSelectLeaf}
|
||||||
|
onSelectBranch={onSelectBranch}
|
||||||
|
onToggleBranch={onToggleBranch}
|
||||||
|
/>
|
||||||
|
)}
|
||||||
|
</li>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
const isSelected = node.path === selectedKey;
|
||||||
|
return (
|
||||||
|
<li key={node.path}>
|
||||||
|
<div
|
||||||
|
className={`kv-tree-row kv-tree-leaf${isSelected ? ' selected' : ''}`}
|
||||||
|
style={{ paddingLeft: depth * 14 + 4 }}
|
||||||
|
onClick={() => onSelectLeaf(node.path)}
|
||||||
|
>
|
||||||
|
<span className="kv-tree-leaf-dot">•</span>
|
||||||
|
<span className="kv-tree-label">{node.label}</span>
|
||||||
|
</div>
|
||||||
|
</li>
|
||||||
|
);
|
||||||
|
})}
|
||||||
|
</ul>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
interface BranchRowProps {
|
||||||
|
node: TreeNode;
|
||||||
|
open: boolean;
|
||||||
|
depth: number;
|
||||||
|
onToggle: () => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
function BranchRow({ node, open, depth, onToggle }: BranchRowProps) {
|
||||||
|
// Surface the actual filename next to file:<hash>/ branches so the tree
|
||||||
|
// reads like a file list at that level instead of a wall of hashes.
|
||||||
|
const fileName = useFileName(isFileBranch(node.path) ? node.path : null);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div
|
||||||
|
className={`kv-tree-row kv-tree-branch${open ? ' open' : ''}`}
|
||||||
|
style={{ paddingLeft: depth * 14 + 4 }}
|
||||||
|
onClick={onToggle}
|
||||||
|
title={isFileBranch(node.path) ? node.path : undefined}
|
||||||
|
>
|
||||||
|
<span className="kv-tree-toggle">{open ? '▾' : '▸'}</span>
|
||||||
|
{fileName ? (
|
||||||
|
<>
|
||||||
|
<span className="kv-tree-label kv-tree-filename">{fileName}</span>
|
||||||
|
<span className="kv-tree-hash-pill">{shortHashFromFileBranch(node.path)}</span>
|
||||||
|
</>
|
||||||
|
) : (
|
||||||
|
<span className="kv-tree-label">{node.label}</span>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Helper: given a key path with both ":" and "/" separators, return the list
|
||||||
|
// of ancestor branch paths that need to be expanded to reveal it.
|
||||||
|
export function ancestorBranches(key: string): string[] {
|
||||||
|
const out: string[] = [];
|
||||||
|
let i = 0;
|
||||||
|
while (i < key.length) {
|
||||||
|
const next = nextSep(key, i);
|
||||||
|
if (next < 0) break;
|
||||||
|
out.push(key.slice(0, next + 1));
|
||||||
|
i = next + 1;
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
|
function nextSep(s: string, from: number): number {
|
||||||
|
let best = -1;
|
||||||
|
for (const d of [':', '/']) {
|
||||||
|
const idx = s.indexOf(d, from);
|
||||||
|
if (idx >= 0 && (best < 0 || idx < best)) best = idx;
|
||||||
|
}
|
||||||
|
return best;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Re-export for convenience in the shell
|
||||||
|
export type { TreeNode };
|
||||||
@@ -0,0 +1,57 @@
|
|||||||
|
import { useState } from 'react';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
fields: string[];
|
||||||
|
onChange: (fields: string[]) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
// Pill list of field names that the value-search uses. Each pill is removable;
|
||||||
|
// the inline input adds a new field on Enter or comma. Persisting to
|
||||||
|
// localStorage is the parent's responsibility.
|
||||||
|
export function SearchFieldChips({ fields, onChange }: Props) {
|
||||||
|
const [input, setInput] = useState('');
|
||||||
|
|
||||||
|
const add = (raw: string) => {
|
||||||
|
const next = raw.trim().replace(/,+$/, '');
|
||||||
|
if (!next) return;
|
||||||
|
if (fields.includes(next)) return;
|
||||||
|
onChange([...fields, next]);
|
||||||
|
setInput('');
|
||||||
|
};
|
||||||
|
|
||||||
|
const remove = (f: string) => onChange(fields.filter((x) => x !== f));
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-chips">
|
||||||
|
<span className="kv-chips-label">search in:</span>
|
||||||
|
{fields.map((f) => (
|
||||||
|
<span key={f} className="kv-chip">
|
||||||
|
<code>{f}</code>
|
||||||
|
<button
|
||||||
|
className="kv-chip-x"
|
||||||
|
onClick={() => remove(f)}
|
||||||
|
title={`remove ${f}`}
|
||||||
|
aria-label={`remove ${f}`}
|
||||||
|
>×</button>
|
||||||
|
</span>
|
||||||
|
))}
|
||||||
|
<input
|
||||||
|
className="kv-chip-input"
|
||||||
|
type="text"
|
||||||
|
value={input}
|
||||||
|
placeholder="+ field"
|
||||||
|
onChange={(e) => {
|
||||||
|
const v = e.target.value;
|
||||||
|
if (v.endsWith(',')) add(v);
|
||||||
|
else setInput(v);
|
||||||
|
}}
|
||||||
|
onKeyDown={(e) => {
|
||||||
|
if (e.key === 'Enter') { e.preventDefault(); add(input); }
|
||||||
|
if (e.key === 'Backspace' && input === '' && fields.length > 0) {
|
||||||
|
remove(fields[fields.length - 1]);
|
||||||
|
}
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
import { useEffect } from 'react';
|
||||||
|
import { ToastEntry } from './types';
|
||||||
|
|
||||||
|
interface Props {
|
||||||
|
toasts: ToastEntry[];
|
||||||
|
onDismiss: (id: number) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function ToastStack({ toasts, onDismiss }: Props) {
|
||||||
|
return (
|
||||||
|
<div className="kv-toast-stack">
|
||||||
|
{toasts.map((t) => (
|
||||||
|
<Toast key={t.id} entry={t} onDismiss={() => onDismiss(t.id)} />
|
||||||
|
))}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function Toast({ entry, onDismiss }: { entry: ToastEntry; onDismiss: () => void }) {
|
||||||
|
useEffect(() => {
|
||||||
|
if (entry.kind === 'error') return;
|
||||||
|
const timer = setTimeout(onDismiss, 4500);
|
||||||
|
return () => clearTimeout(timer);
|
||||||
|
}, [entry.kind, onDismiss]);
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className={`kv-toast kv-toast-${entry.kind}`}>
|
||||||
|
<div className="kv-toast-row">
|
||||||
|
<span className="kv-toast-text">{entry.text}</span>
|
||||||
|
<button className="kv-toast-close" onClick={onDismiss} aria-label="dismiss">×</button>
|
||||||
|
</div>
|
||||||
|
{entry.details && entry.details.length > 0 && (
|
||||||
|
<ul className="kv-toast-details">
|
||||||
|
{entry.details.slice(0, 8).map((d, i) => <li key={i}>{d}</li>)}
|
||||||
|
{entry.details.length > 8 && <li>… {entry.details.length - 8} more</li>}
|
||||||
|
</ul>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let nextId = 1;
|
||||||
|
export function makeToast(kind: ToastEntry['kind'], text: string, details?: string[]): ToastEntry {
|
||||||
|
return { id: nextId++, kind, text, details };
|
||||||
|
}
|
||||||
@@ -0,0 +1,295 @@
|
|||||||
|
import { useEffect, useState } from 'react';
|
||||||
|
import { LanguageCombobox } from '../components/LanguageCombobox';
|
||||||
|
import { FieldType } from './types';
|
||||||
|
|
||||||
|
interface ValueRendererProps {
|
||||||
|
value: string;
|
||||||
|
onChange: (v: string) => void;
|
||||||
|
}
|
||||||
|
|
||||||
|
interface LeafValueEditorProps extends ValueRendererProps {
|
||||||
|
type: FieldType;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Type-dispatched leaf value editor. Used by both the single-leaf editor
|
||||||
|
* (KVLeafEditor) and the per-row editor in the file-detail view.
|
||||||
|
*/
|
||||||
|
export function LeafValueEditor({ type, value, onChange }: LeafValueEditorProps) {
|
||||||
|
switch (type) {
|
||||||
|
case 'bool':
|
||||||
|
return (
|
||||||
|
<select
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={value === 'true' ? 'true' : value === 'false' ? 'false' : ''}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
>
|
||||||
|
<option value="">(unset)</option>
|
||||||
|
<option value="true">true</option>
|
||||||
|
<option value="false">false</option>
|
||||||
|
</select>
|
||||||
|
);
|
||||||
|
case 'number':
|
||||||
|
return (
|
||||||
|
<input
|
||||||
|
type="number"
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
case 'datetime':
|
||||||
|
return <DateTimeLeaf value={value} onChange={onChange} />;
|
||||||
|
case 'lang':
|
||||||
|
return <LanguageCombobox value={value} onChange={onChange} />;
|
||||||
|
case 'url':
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf-url">
|
||||||
|
<input
|
||||||
|
type="url"
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
/>
|
||||||
|
{value && (
|
||||||
|
<a className="kv-leaf-url-open" href={value} target="_blank" rel="noreferrer">
|
||||||
|
open ↗
|
||||||
|
</a>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
case 'json':
|
||||||
|
return (
|
||||||
|
<textarea
|
||||||
|
className="kv-leaf-textarea kv-leaf-mono"
|
||||||
|
rows={12}
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
spellCheck={false}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
case 'text':
|
||||||
|
return (
|
||||||
|
<textarea
|
||||||
|
className="kv-leaf-textarea"
|
||||||
|
rows={8}
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
case 'hash':
|
||||||
|
return <CidLeaf value={value} onChange={onChange} />;
|
||||||
|
case 'auto':
|
||||||
|
case 'string':
|
||||||
|
default:
|
||||||
|
return (
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
interface CidFileInfo {
|
||||||
|
exists: boolean;
|
||||||
|
contentType?: string;
|
||||||
|
size?: number;
|
||||||
|
filePath?: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* CID leaf editor. Shows the value as an editable monospace input plus a
|
||||||
|
* content-typed preview when meta-core can resolve the CID to a file on disk
|
||||||
|
* (`/api/file/{cid}/info` → image/video/audio rendering, else download-only).
|
||||||
|
*/
|
||||||
|
function CidLeaf({ value, onChange }: ValueRendererProps) {
|
||||||
|
const [info, setInfo] = useState<CidFileInfo | null>(null);
|
||||||
|
const cid = value.trim();
|
||||||
|
const fileUrl = cid ? `/file/${cid}` : '';
|
||||||
|
const downloadName = info?.filePath?.split('/').pop() || cid;
|
||||||
|
|
||||||
|
useEffect(() => {
|
||||||
|
if (!cid) {
|
||||||
|
setInfo(null);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
let alive = true;
|
||||||
|
fetch(`/api/file/${cid}/info`)
|
||||||
|
.then((r) => (r.ok ? r.json() : { exists: false }))
|
||||||
|
.then((data: CidFileInfo) => {
|
||||||
|
if (alive) setInfo(data);
|
||||||
|
})
|
||||||
|
.catch(() => {
|
||||||
|
if (alive) setInfo({ exists: false });
|
||||||
|
});
|
||||||
|
return () => {
|
||||||
|
alive = false;
|
||||||
|
};
|
||||||
|
}, [cid]);
|
||||||
|
|
||||||
|
const renderPreview = () => {
|
||||||
|
if (!info?.exists || !info.contentType || !cid) return null;
|
||||||
|
if (info.contentType.startsWith('image/')) {
|
||||||
|
return <img src={fileUrl} alt="" className="kv-leaf-cid-preview" loading="lazy" />;
|
||||||
|
}
|
||||||
|
if (info.contentType.startsWith('video/')) {
|
||||||
|
return (
|
||||||
|
<video
|
||||||
|
src={fileUrl}
|
||||||
|
className="kv-leaf-cid-preview kv-leaf-cid-preview-media"
|
||||||
|
controls
|
||||||
|
preload="metadata"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
if (info.contentType.startsWith('audio/')) {
|
||||||
|
return (
|
||||||
|
<audio
|
||||||
|
src={fileUrl}
|
||||||
|
className="kv-leaf-cid-preview-media"
|
||||||
|
controls
|
||||||
|
preload="metadata"
|
||||||
|
/>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
};
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf-cid">
|
||||||
|
{renderPreview()}
|
||||||
|
<div className="kv-leaf-cid-controls">
|
||||||
|
<input
|
||||||
|
type="text"
|
||||||
|
className="kv-leaf-input kv-leaf-mono"
|
||||||
|
value={value}
|
||||||
|
onChange={(e) => onChange(e.target.value)}
|
||||||
|
spellCheck={false}
|
||||||
|
/>
|
||||||
|
<button
|
||||||
|
type="button"
|
||||||
|
className="kv-btn-icon"
|
||||||
|
onClick={() => navigator.clipboard.writeText(value)}
|
||||||
|
title="copy CID"
|
||||||
|
>
|
||||||
|
⧉
|
||||||
|
</button>
|
||||||
|
</div>
|
||||||
|
{info?.exists && cid && (
|
||||||
|
<div className="kv-leaf-cid-actions">
|
||||||
|
<a
|
||||||
|
href={fileUrl}
|
||||||
|
download={downloadName}
|
||||||
|
className="kv-btn kv-btn-primary"
|
||||||
|
title={`Download ${downloadName}${info.size ? ` (${formatBytes(info.size)})` : ''}`}
|
||||||
|
>
|
||||||
|
⬇ Download
|
||||||
|
</a>
|
||||||
|
<a href={fileUrl} className="kv-btn" target="_blank" rel="noreferrer">
|
||||||
|
open ↗
|
||||||
|
</a>
|
||||||
|
<span className="kv-leaf-cid-meta">
|
||||||
|
{info.contentType}
|
||||||
|
{info.size != null ? ` · ${formatBytes(info.size)}` : ''}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
)}
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Timestamp shapes we round-trip through the picker. */
|
||||||
|
type TimestampShape = 'iso-date' | 'iso-datetime' | 'unix-s' | 'unix-ms';
|
||||||
|
|
||||||
|
function detectShape(raw: string): TimestampShape | null {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (!trimmed) return null;
|
||||||
|
if (/^\d{4}-\d{2}-\d{2}$/.test(trimmed)) return 'iso-date';
|
||||||
|
if (/^\d+$/.test(trimmed)) {
|
||||||
|
const n = Number.parseInt(trimmed, 10);
|
||||||
|
if (n >= 1_000_000_000 && n <= 4_102_444_800) return 'unix-s';
|
||||||
|
if (n >= 1_000_000_000_000 && n <= 4_102_444_800_000) return 'unix-ms';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
if (!Number.isNaN(Date.parse(trimmed))) return 'iso-datetime';
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function rawToDate(raw: string, shape: TimestampShape): Date | null {
|
||||||
|
const trimmed = raw.trim();
|
||||||
|
if (!trimmed) return null;
|
||||||
|
if (shape === 'unix-s') return new Date(Number.parseInt(trimmed, 10) * 1000);
|
||||||
|
if (shape === 'unix-ms') return new Date(Number.parseInt(trimmed, 10));
|
||||||
|
const d = new Date(trimmed);
|
||||||
|
return Number.isNaN(d.getTime()) ? null : d;
|
||||||
|
}
|
||||||
|
|
||||||
|
function dateToRaw(d: Date, shape: TimestampShape): string {
|
||||||
|
switch (shape) {
|
||||||
|
case 'unix-s':
|
||||||
|
return String(Math.floor(d.getTime() / 1000));
|
||||||
|
case 'unix-ms':
|
||||||
|
return String(d.getTime());
|
||||||
|
case 'iso-date':
|
||||||
|
return d.toISOString().slice(0, 10);
|
||||||
|
case 'iso-datetime':
|
||||||
|
return d.toISOString();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Datetime leaf editor. Preserves the original storage shape — ISO date stays
|
||||||
|
* ISO date, unix seconds stay unix seconds, etc. Unknown shapes fall back to
|
||||||
|
* iso-datetime.
|
||||||
|
*/
|
||||||
|
function DateTimeLeaf({ value, onChange }: ValueRendererProps) {
|
||||||
|
const shape: TimestampShape = detectShape(value) ?? 'iso-datetime';
|
||||||
|
const date = rawToDate(value, shape);
|
||||||
|
const inputType = shape === 'iso-date' ? 'date' : 'datetime-local';
|
||||||
|
|
||||||
|
const inputValue = (() => {
|
||||||
|
if (!date || Number.isNaN(date.getTime())) return '';
|
||||||
|
if (inputType === 'date') return date.toISOString().slice(0, 10);
|
||||||
|
// datetime-local wants YYYY-MM-DDTHH:mm in the user's local timezone.
|
||||||
|
const local = new Date(date.getTime() - date.getTimezoneOffset() * 60000);
|
||||||
|
return local.toISOString().slice(0, 16);
|
||||||
|
})();
|
||||||
|
|
||||||
|
return (
|
||||||
|
<div className="kv-leaf-datetime">
|
||||||
|
<input
|
||||||
|
type={inputType}
|
||||||
|
className="kv-leaf-input"
|
||||||
|
value={inputValue}
|
||||||
|
onChange={(e) => {
|
||||||
|
const v = e.target.value;
|
||||||
|
if (!v) {
|
||||||
|
onChange('');
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
const parsed = new Date(v);
|
||||||
|
if (Number.isNaN(parsed.getTime())) return;
|
||||||
|
onChange(dateToRaw(parsed, shape));
|
||||||
|
}}
|
||||||
|
/>
|
||||||
|
<span className="kv-leaf-datetime-shape" title="Storage format preserved on save">
|
||||||
|
{shape}
|
||||||
|
</span>
|
||||||
|
</div>
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function formatBytes(n: number): string {
|
||||||
|
if (n < 1024) return `${n} B`;
|
||||||
|
const units = ['KB', 'MB', 'GB', 'TB'];
|
||||||
|
let v = n / 1024;
|
||||||
|
let i = 0;
|
||||||
|
while (v >= 1024 && i < units.length - 1) {
|
||||||
|
v /= 1024;
|
||||||
|
i++;
|
||||||
|
}
|
||||||
|
return `${v.toFixed(v >= 10 ? 0 : 1)} ${units[i]}`;
|
||||||
|
}
|
||||||
@@ -0,0 +1,90 @@
|
|||||||
|
import { SchemaFieldInfo, SchemaResponse } from '../api/schemaApi';
|
||||||
|
import { FieldType } from './types';
|
||||||
|
|
||||||
|
export interface SchemaLookup {
|
||||||
|
/** The matched schema entry, if any. */
|
||||||
|
info?: SchemaFieldInfo;
|
||||||
|
/** Suggested FieldType for the leaf editor, or null when schema can't decide. */
|
||||||
|
fieldType: FieldType | null;
|
||||||
|
/**
|
||||||
|
* True when the leaf is one variant under a language-keyed group
|
||||||
|
* (e.g. plot/eng). The schema entry returned is the group's, not the leaf's.
|
||||||
|
*/
|
||||||
|
isLanguageVariant: boolean;
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Resolve the logical schema entry for a full Redis key.
|
||||||
|
*
|
||||||
|
* - "file:<hash>/releasedate" → schema.fields["releasedate"]
|
||||||
|
* - "file:<hash>/plot/eng" → schema.fields["plot"] (language-keyed group)
|
||||||
|
* - non-file keys → undefined (schema is per-file today)
|
||||||
|
*/
|
||||||
|
export function lookupSchemaForKey(
|
||||||
|
schema: SchemaResponse | null,
|
||||||
|
key: string,
|
||||||
|
): SchemaLookup {
|
||||||
|
if (!schema) {
|
||||||
|
return { fieldType: null, isLanguageVariant: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const propertyPath = stripFilePrefix(key);
|
||||||
|
if (!propertyPath) {
|
||||||
|
return { fieldType: null, isLanguageVariant: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
const direct = schema.fields[propertyPath];
|
||||||
|
if (direct) {
|
||||||
|
return {
|
||||||
|
info: direct,
|
||||||
|
fieldType: schemaToFieldType(direct),
|
||||||
|
isLanguageVariant: false,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
// Language-keyed collapse: tail is a 2/3-letter lang code under a known group.
|
||||||
|
const lastSlash = propertyPath.lastIndexOf('/');
|
||||||
|
if (lastSlash > 0) {
|
||||||
|
const tail = propertyPath.slice(lastSlash + 1);
|
||||||
|
if (/^[a-z]{2,3}$/.test(tail)) {
|
||||||
|
const group = schema.fields[propertyPath.slice(0, lastSlash)];
|
||||||
|
if (group?.key_hint === 'language-code') {
|
||||||
|
return {
|
||||||
|
info: group,
|
||||||
|
fieldType: schemaToFieldType(group),
|
||||||
|
isLanguageVariant: true,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return { fieldType: null, isLanguageVariant: false };
|
||||||
|
}
|
||||||
|
|
||||||
|
export function schemaToFieldType(info: SchemaFieldInfo | undefined): FieldType | null {
|
||||||
|
if (!info) return null;
|
||||||
|
if (info.hint === 'cid') return 'hash';
|
||||||
|
if (info.hint === 'timestamp') return 'datetime';
|
||||||
|
switch (info.type) {
|
||||||
|
case 'bool':
|
||||||
|
return 'bool';
|
||||||
|
case 'int':
|
||||||
|
case 'float':
|
||||||
|
return 'number';
|
||||||
|
case 'json-object':
|
||||||
|
case 'json-array':
|
||||||
|
return 'json';
|
||||||
|
case 'string':
|
||||||
|
case 'mixed':
|
||||||
|
case 'undefined':
|
||||||
|
default:
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function stripFilePrefix(key: string): string {
|
||||||
|
if (!key.startsWith('file:')) return '';
|
||||||
|
const slash = key.indexOf('/');
|
||||||
|
if (slash < 0) return '';
|
||||||
|
return key.slice(slash + 1);
|
||||||
|
}
|
||||||
@@ -0,0 +1,59 @@
|
|||||||
|
import { FieldType } from './types';
|
||||||
|
|
||||||
|
// Path-pattern table. The leftmost segment of the path that matches wins.
|
||||||
|
// Order matters: more specific patterns should appear first.
|
||||||
|
const PATH_PATTERNS: Array<[RegExp, FieldType]> = [
|
||||||
|
[/(^|\/)(language|lang|audioLang|subLang)$/i, 'lang'],
|
||||||
|
[/(^|\/)(cid_|hash$|sha\d|midhash|md5|crc32)/i, 'hash'],
|
||||||
|
[/(^|\/)(duration|bitrate|fps|width|height|count|size|sizeByte|increment|episode|season|year|runtime|trackId|index)$/i, 'number'],
|
||||||
|
[/(^|\/)(enabled|active|disabled|isHidden|hidden|favorite|watched)$/i, 'bool'],
|
||||||
|
[/(^|\/)(url|uri|href|link|posterUrl|backdropUrl|imageUrl)$/i, 'url'],
|
||||||
|
[/(^|\/)(.+_at|.+At|.+Date|.+Time|timestamp|createdAt|updatedAt)$/i, 'datetime'],
|
||||||
|
[/(^|\/)(description|overview|notes?|comment|summary|nfo)$/i, 'text'],
|
||||||
|
];
|
||||||
|
|
||||||
|
function detectFromPath(path: string): FieldType | null {
|
||||||
|
for (const [re, t] of PATH_PATTERNS) {
|
||||||
|
if (re.test(path)) return t;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function detectFromValue(v: string): FieldType {
|
||||||
|
if (v === '') return 'string';
|
||||||
|
if (v === 'true' || v === 'false') return 'bool';
|
||||||
|
if (/^-?\d+$/.test(v)) return 'number';
|
||||||
|
if (/^-?\d*\.\d+$/.test(v)) return 'number';
|
||||||
|
if (/^https?:\/\//i.test(v)) return 'url';
|
||||||
|
const trimmed = v.trim();
|
||||||
|
if ((trimmed.startsWith('{') && trimmed.endsWith('}')) || (trimmed.startsWith('[') && trimmed.endsWith(']'))) {
|
||||||
|
try { JSON.parse(trimmed); return 'json'; } catch {}
|
||||||
|
}
|
||||||
|
if (v.includes('\n') || v.length > 200) return 'text';
|
||||||
|
return 'string';
|
||||||
|
}
|
||||||
|
|
||||||
|
// Combine path and value heuristics. Path wins when it matches; otherwise
|
||||||
|
// fall back to value shape.
|
||||||
|
export function detectType(path: string, value: string): FieldType {
|
||||||
|
return detectFromPath(path) ?? detectFromValue(value);
|
||||||
|
}
|
||||||
|
|
||||||
|
export const ALL_TYPES: FieldType[] = [
|
||||||
|
'auto', 'string', 'text', 'number', 'bool', 'json', 'url', 'lang', 'datetime', 'hash',
|
||||||
|
];
|
||||||
|
|
||||||
|
export function describeType(t: FieldType): string {
|
||||||
|
switch (t) {
|
||||||
|
case 'auto': return 'auto';
|
||||||
|
case 'string': return 'string';
|
||||||
|
case 'text': return 'text (multiline)';
|
||||||
|
case 'number': return 'number';
|
||||||
|
case 'bool': return 'bool';
|
||||||
|
case 'json': return 'json';
|
||||||
|
case 'url': return 'url';
|
||||||
|
case 'lang': return 'language code';
|
||||||
|
case 'datetime': return 'datetime';
|
||||||
|
case 'hash': return 'hash / cid';
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,19 @@
|
|||||||
|
// Inferred type for a leaf value. "auto" defers to the heuristic detector.
|
||||||
|
export type FieldType =
|
||||||
|
| 'auto'
|
||||||
|
| 'string'
|
||||||
|
| 'number'
|
||||||
|
| 'bool'
|
||||||
|
| 'json'
|
||||||
|
| 'url'
|
||||||
|
| 'text'
|
||||||
|
| 'lang'
|
||||||
|
| 'datetime'
|
||||||
|
| 'hash';
|
||||||
|
|
||||||
|
export interface ToastEntry {
|
||||||
|
id: number;
|
||||||
|
kind: 'info' | 'success' | 'error';
|
||||||
|
text: string;
|
||||||
|
details?: string[];
|
||||||
|
}
|
||||||
+2
-2
@@ -1,10 +1,10 @@
|
|||||||
import { StrictMode } from 'react';
|
import { StrictMode } from 'react';
|
||||||
import { createRoot } from 'react-dom/client';
|
import { createRoot } from 'react-dom/client';
|
||||||
import './index.css';
|
import './index.css';
|
||||||
import App from './App.tsx';
|
import KVApp from './kv/KVApp.tsx';
|
||||||
|
|
||||||
createRoot(document.getElementById('root')!).render(
|
createRoot(document.getElementById('root')!).render(
|
||||||
<StrictMode>
|
<StrictMode>
|
||||||
<App />
|
<KVApp />
|
||||||
</StrictMode>,
|
</StrictMode>,
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -1,155 +0,0 @@
|
|||||||
// API Response Types
|
|
||||||
export interface FileMetadata {
|
|
||||||
// File info
|
|
||||||
fileName?: string;
|
|
||||||
extension?: string;
|
|
||||||
byteSize?: number;
|
|
||||||
mimeType?: string;
|
|
||||||
fileType?: string;
|
|
||||||
|
|
||||||
// Filename parsing
|
|
||||||
title?: string;
|
|
||||||
titles?: string[];
|
|
||||||
originaltitle?: string;
|
|
||||||
showtitle?: string;
|
|
||||||
season?: number;
|
|
||||||
episode?: number;
|
|
||||||
movieYear?: number;
|
|
||||||
videoType?: string;
|
|
||||||
|
|
||||||
// General media metadata
|
|
||||||
year?: number;
|
|
||||||
imdbid?: string;
|
|
||||||
tmdbid?: string;
|
|
||||||
anidbid?: string;
|
|
||||||
premiered?: string;
|
|
||||||
genre?: string[];
|
|
||||||
releasedate?: string;
|
|
||||||
plot?: string;
|
|
||||||
director?: string;
|
|
||||||
rating?: number;
|
|
||||||
runtime?: number;
|
|
||||||
studio?: string;
|
|
||||||
art?: any;
|
|
||||||
actor?: any[];
|
|
||||||
aired?: string;
|
|
||||||
customrating?: string;
|
|
||||||
dateadded?: string;
|
|
||||||
sorttitle?: string;
|
|
||||||
mpaa?: string;
|
|
||||||
aspectratio?: string;
|
|
||||||
collectionnumber?: number;
|
|
||||||
criticrating?: number;
|
|
||||||
|
|
||||||
// Hashes
|
|
||||||
cid_crc32?: string;
|
|
||||||
cid_md5?: string;
|
|
||||||
cid_sha1?: string;
|
|
||||||
'cid_sha2-256'?: string;
|
|
||||||
'cid_sha3-256'?: string;
|
|
||||||
cid_sha3_384?: string;
|
|
||||||
|
|
||||||
// Video info
|
|
||||||
video?: {
|
|
||||||
codec?: string;
|
|
||||||
width?: number;
|
|
||||||
height?: number;
|
|
||||||
bitrate?: number;
|
|
||||||
fps?: number;
|
|
||||||
};
|
|
||||||
|
|
||||||
// Audio info
|
|
||||||
audio?: Array<{
|
|
||||||
codec?: string;
|
|
||||||
channels?: number;
|
|
||||||
language?: string;
|
|
||||||
bitrate?: number;
|
|
||||||
}>;
|
|
||||||
|
|
||||||
// Subtitles
|
|
||||||
subtitles?: Array<{
|
|
||||||
language?: string;
|
|
||||||
codec?: string;
|
|
||||||
}>;
|
|
||||||
|
|
||||||
// Other
|
|
||||||
extra?: any;
|
|
||||||
anime?: any;
|
|
||||||
fileinfo?: any;
|
|
||||||
trackers?: string[];
|
|
||||||
|
|
||||||
// Allow any additional properties
|
|
||||||
[key: string]: any;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SearchResult {
|
|
||||||
hashId: string;
|
|
||||||
metadata: FileMetadata;
|
|
||||||
score?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SearchResponse {
|
|
||||||
results: SearchResult[];
|
|
||||||
count: number;
|
|
||||||
total?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface UpdateResult {
|
|
||||||
status: string;
|
|
||||||
hashId: string;
|
|
||||||
message?: string;
|
|
||||||
error?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface BatchUpdateResponse {
|
|
||||||
status: string;
|
|
||||||
total: number;
|
|
||||||
success: number;
|
|
||||||
errors: number;
|
|
||||||
results: Array<{
|
|
||||||
hashId: string;
|
|
||||||
status: string;
|
|
||||||
error?: string;
|
|
||||||
}>;
|
|
||||||
}
|
|
||||||
|
|
||||||
// Plugin Schema Types
|
|
||||||
export type SchemaFieldType = 'string' | 'number' | 'boolean' | 'select' | 'text' | 'array' | 'json' | 'cid' | 'languageString';
|
|
||||||
|
|
||||||
export interface SelectOption {
|
|
||||||
value: string;
|
|
||||||
label: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface SchemaField {
|
|
||||||
label: string;
|
|
||||||
type?: SchemaFieldType;
|
|
||||||
readonly?: boolean;
|
|
||||||
indexed?: boolean;
|
|
||||||
options?: SelectOption[];
|
|
||||||
hint?: string;
|
|
||||||
link?: string;
|
|
||||||
pattern?: string;
|
|
||||||
min?: number;
|
|
||||||
max?: number;
|
|
||||||
placeholder?: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PluginSchema {
|
|
||||||
id: string;
|
|
||||||
name: string;
|
|
||||||
version: string;
|
|
||||||
description?: string;
|
|
||||||
priority?: number;
|
|
||||||
color?: string;
|
|
||||||
dependencies: string[];
|
|
||||||
active: boolean;
|
|
||||||
metadataSchema?: Record<string, SchemaField>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PluginsResponse {
|
|
||||||
plugins: PluginSchema[];
|
|
||||||
executionOrder: string[];
|
|
||||||
activeCount: number;
|
|
||||||
totalCount: number;
|
|
||||||
}
|
|
||||||
@@ -1,53 +0,0 @@
|
|||||||
/**
|
|
||||||
* Format bytes to human readable format
|
|
||||||
*/
|
|
||||||
export function formatBytes(bytes: number | undefined): string {
|
|
||||||
if (!bytes) return 'N/A';
|
|
||||||
|
|
||||||
const sizes = ['B', 'KB', 'MB', 'GB', 'TB'];
|
|
||||||
if (bytes === 0) return '0 B';
|
|
||||||
|
|
||||||
const i = Math.floor(Math.log(bytes) / Math.log(1024));
|
|
||||||
return Math.round((bytes / Math.pow(1024, i)) * 100) / 100 + ' ' + sizes[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Format duration in seconds to HH:MM:SS
|
|
||||||
*/
|
|
||||||
export function formatDuration(seconds: number | undefined): string {
|
|
||||||
if (!seconds) return 'N/A';
|
|
||||||
|
|
||||||
const hours = Math.floor(seconds / 3600);
|
|
||||||
const minutes = Math.floor((seconds % 3600) / 60);
|
|
||||||
const secs = Math.floor(seconds % 60);
|
|
||||||
|
|
||||||
if (hours > 0) {
|
|
||||||
return `${hours}:${minutes.toString().padStart(2, '0')}:${secs.toString().padStart(2, '0')}`;
|
|
||||||
}
|
|
||||||
return `${minutes}:${secs.toString().padStart(2, '0')}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Truncate hash to first and last 6 characters
|
|
||||||
*/
|
|
||||||
export function truncateHash(hash: string | undefined): string {
|
|
||||||
if (!hash) return 'N/A';
|
|
||||||
if (hash.length <= 16) return hash;
|
|
||||||
return `${hash.slice(0, 6)}...${hash.slice(-6)}`;
|
|
||||||
}
|
|
||||||
|
|
||||||
/**
|
|
||||||
* Get display name for a file from metadata
|
|
||||||
*/
|
|
||||||
export function getDisplayName(metadata: any): string {
|
|
||||||
if (metadata.title) {
|
|
||||||
if (metadata.season && metadata.episode) {
|
|
||||||
return `${metadata.title} S${String(metadata.season).padStart(2, '0')}E${String(metadata.episode).padStart(2, '0')}`;
|
|
||||||
}
|
|
||||||
if (metadata.movieYear) {
|
|
||||||
return `${metadata.title} (${metadata.movieYear})`;
|
|
||||||
}
|
|
||||||
return metadata.title;
|
|
||||||
}
|
|
||||||
return metadata.fileName || 'Unknown';
|
|
||||||
}
|
|
||||||
@@ -0,0 +1 @@
|
|||||||
|
{"root":["./src/main.tsx","./src/vite-env.d.ts","./src/api/kvApi.ts","./src/api/schemaApi.ts","./src/api/snapshotApi.ts","./src/components/LanguageCombobox.tsx","./src/components/SnapshotPanel.tsx","./src/kv/FilteredKVTree.tsx","./src/kv/KVApp.tsx","./src/kv/KVBranchEditor.tsx","./src/kv/KVFileEditor.tsx","./src/kv/KVLeafEditor.tsx","./src/kv/KVTree.tsx","./src/kv/SearchFieldChips.tsx","./src/kv/Toast.tsx","./src/kv/leafRenderers.tsx","./src/kv/schemaLookup.ts","./src/kv/typeHeuristics.ts","./src/kv/types.ts"],"version":"5.9.3"}
|
||||||
@@ -3,10 +3,13 @@ module github.com/metazla/meta-core
|
|||||||
go 1.21
|
go 1.21
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1
|
||||||
github.com/google/uuid v1.6.0
|
github.com/google/uuid v1.6.0
|
||||||
github.com/gorilla/mux v1.8.1
|
github.com/gorilla/mux v1.8.1
|
||||||
|
github.com/mr-tron/base58 v1.3.0
|
||||||
github.com/redis/go-redis/v9 v9.4.0
|
github.com/redis/go-redis/v9 v9.4.0
|
||||||
golang.org/x/net v0.20.0
|
golang.org/x/crypto v0.31.0
|
||||||
|
golang.org/x/net v0.21.0
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
|||||||
@@ -4,13 +4,21 @@ github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
|
|||||||
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
|
||||||
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
|
||||||
|
github.com/decred/dcrd/crypto/blake256 v1.1.0 h1:zPMNGQCm0g4QTY27fOCorQW7EryeQ/U0x++OzVrdms8=
|
||||||
|
github.com/decred/dcrd/crypto/blake256 v1.1.0/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo=
|
||||||
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1 h1:5RVFMOWjMyRy8cARdy79nAmgYw3hK/4HUq48LQ6Wwqo=
|
||||||
|
github.com/decred/dcrd/dcrec/secp256k1/v4 v4.4.1/go.mod h1:ZXNYxsqcloTdSy/rNShjYzMhyjf0LaoftYK0p+A3h40=
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
|
||||||
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY=
|
||||||
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ=
|
||||||
|
github.com/mr-tron/base58 v1.3.0 h1:K6Y13R2h+dku0wOqKtecgRnBUBPrZzLZy5aIj8lCcJI=
|
||||||
|
github.com/mr-tron/base58 v1.3.0/go.mod h1:2BuubE67DCSWwVfx37JWNG8emOC0sHEU4/HpcYgCLX8=
|
||||||
github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
|
github.com/redis/go-redis/v9 v9.4.0 h1:Yzoz33UZw9I/mFhx4MNrB6Fk+XHO1VukNcCa1+lwyKk=
|
||||||
github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
github.com/redis/go-redis/v9 v9.4.0/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
|
||||||
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo=
|
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
||||||
golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY=
|
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
||||||
|
golang.org/x/net v0.21.0 h1:AQyQV4dYCvJ7vGmJyKki9+PBdyvhkSd8EIx/qb0AYv4=
|
||||||
|
golang.org/x/net v0.21.0/go.mod h1:bIjVDfnllIU7BJ2DNgfnXvpSvtn8VRwhlsaeUTyUS44=
|
||||||
|
|||||||
@@ -0,0 +1,30 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
|
||||||
|
// stranded midhash-rooted entries into their UUID roots. See
|
||||||
|
// storage.MigrateDualRoots for the data-flow detail and
|
||||||
|
// docs/api-mediated-access.md for the bug background.
|
||||||
|
//
|
||||||
|
// Returns {fixed: N} on success. Safe to call repeatedly — entries already
|
||||||
|
// migrated are skipped on the next pass.
|
||||||
|
func (s *Server) handleMigrateDualRoots(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fixed, err := s.storage.MigrateDualRootsWithTimeout()
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||||
|
"success": true,
|
||||||
|
"fixed": fixed,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"net/http"
|
||||||
|
|
||||||
|
"github.com/gorilla/mux"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleGetMetaByCID handles GET /api/meta/{cid}.
|
||||||
|
//
|
||||||
|
// Resolves any known CID — midhash256, sha256, IPFS, btih, or anything else
|
||||||
|
// in the reverse index — to the full metadata document for that file.
|
||||||
|
// Returns:
|
||||||
|
// - 200 with the document on a hit
|
||||||
|
// - 404 if the CID isn't registered
|
||||||
|
// - 503 if storage is down
|
||||||
|
//
|
||||||
|
// CIDs in the URL are bare multibase CIDv1 strings ("bafk…") — the midhash,
|
||||||
|
// a sibling sha2-256, an IPFS CID, a btih, anything registered in the
|
||||||
|
// reverse index. A CID is self-describing (its algorithm is the multicodec),
|
||||||
|
// so there is no <algo>: prefix.
|
||||||
|
//
|
||||||
|
// This endpoint is the public read API for the CID-resolution layer and
|
||||||
|
// is auth-bypassed alongside /api/file/{cid} so that meta-share peers and
|
||||||
|
// IPFS-style clients can use it without going through Authelia.
|
||||||
|
func (s *Server) handleGetMetaByCID(w http.ResponseWriter, r *http.Request) {
|
||||||
|
cid := mux.Vars(r)["cid"]
|
||||||
|
if cid == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "cid is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
uuid, err := s.storage.GetByCID(cid)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if uuid == "" {
|
||||||
|
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
doc, err := s.storage.GetMetadataDocument(uuid)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if doc == nil {
|
||||||
|
// Reverse index pointed at a UUID with no remaining keys — stale
|
||||||
|
// alias, probably mid-delete race. Treat as not-found.
|
||||||
|
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "no metadata for this CID", false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Response envelope mirrors GetMetadataDocument JSON tags (metadata /
|
||||||
|
// cids / canonical_cid / duplicates), with cid added so the caller can
|
||||||
|
// correlate when they fan out N lookups. canonical_cid is derived by
|
||||||
|
// rank from the cids key-set on read — it is not a stored field.
|
||||||
|
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||||
|
"cid": cid,
|
||||||
|
"metadata": doc.Flat,
|
||||||
|
"cids": doc.CIDs,
|
||||||
|
"canonical_cid": doc.Canonical,
|
||||||
|
"duplicates": doc.Duplicates,
|
||||||
|
})
|
||||||
|
}
|
||||||
+138
-5
@@ -63,10 +63,48 @@ type DataPathResponse struct {
|
|||||||
Exists bool `json:"exists"`
|
Exists bool `json:"exists"`
|
||||||
}
|
}
|
||||||
|
|
||||||
// ErrorResponse is the response for errors
|
// ErrorResponse is the typed error envelope returned on every 4xx/5xx.
|
||||||
|
//
|
||||||
|
// Shape and slug vocabulary documented in docs/api-mediated-access.md
|
||||||
|
// ("Error envelope"). Stable slug strings let clients switch on the failure
|
||||||
|
// category without parsing the message; `retryable` tells callers whether
|
||||||
|
// a backoff-and-retry has any chance of succeeding.
|
||||||
type ErrorResponse struct {
|
type ErrorResponse struct {
|
||||||
Error string `json:"error"`
|
Error string `json:"error"`
|
||||||
Message string `json:"message,omitempty"`
|
Message string `json:"message,omitempty"`
|
||||||
|
Retryable bool `json:"retryable"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stable error-slug strings. Pinned in the doc; consumers may match on
|
||||||
|
// these. Do not rename without a deprecation cycle.
|
||||||
|
const (
|
||||||
|
ErrAliasCollision = "alias_collision"
|
||||||
|
ErrUnknownRoot = "unknown_root"
|
||||||
|
ErrUnknownCID = "unknown_cid"
|
||||||
|
ErrSchemaViolation = "schema_violation"
|
||||||
|
ErrStorageUnavailable = "storage_unavailable"
|
||||||
|
ErrInternal = "internal"
|
||||||
|
)
|
||||||
|
|
||||||
|
// defaultSlugFor maps an HTTP status code to the most likely error slug.
|
||||||
|
// Handlers that want to override (e.g. /api/meta/{cid} → "unknown_cid"
|
||||||
|
// instead of "unknown_root" on 404) call writeErrorSlug directly.
|
||||||
|
func defaultSlugFor(status int) (slug string, retryable bool) {
|
||||||
|
switch status {
|
||||||
|
case http.StatusConflict:
|
||||||
|
return ErrAliasCollision, false
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return ErrUnknownRoot, false
|
||||||
|
case http.StatusBadRequest:
|
||||||
|
return ErrSchemaViolation, false
|
||||||
|
case http.StatusServiceUnavailable:
|
||||||
|
return ErrStorageUnavailable, true
|
||||||
|
case http.StatusInternalServerError:
|
||||||
|
return ErrInternal, true
|
||||||
|
default:
|
||||||
|
// 4xx → not retryable, 5xx → retryable. Conservative default.
|
||||||
|
return ErrInternal, status >= 500
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
var startTime = time.Now()
|
var startTime = time.Now()
|
||||||
@@ -169,6 +207,10 @@ func (s *Server) handleGetMeta(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Resolve CID → UUID via the reverse index so reads find data wherever
|
||||||
|
// the watcher minted the root, regardless of which CID the caller knows.
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
metadata, err := s.storage.GetMetadataFlat(hashID)
|
metadata, err := s.storage.GetMetadataFlat(hashID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(w, http.StatusInternalServerError, err.Error())
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
@@ -203,6 +245,10 @@ func (s *Server) handlePutMeta(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// CID → UUID resolution so writes land on the canonical root, not on
|
||||||
|
// a parallel midhash-rooted entry the watcher's UUID never sees.
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
var metadata map[string]string
|
var metadata map[string]string
|
||||||
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
||||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||||
@@ -235,6 +281,8 @@ func (s *Server) handleDeleteMeta(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
deleted, err := s.storage.DeleteMetadata(hashID)
|
deleted, err := s.storage.DeleteMetadata(hashID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(w, http.StatusInternalServerError, err.Error())
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
@@ -340,6 +388,63 @@ func (s *Server) handleHeadData(w http.ResponseWriter, r *http.Request) {
|
|||||||
w.WriteHeader(http.StatusOK)
|
w.WriteHeader(http.StatusOK)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// handleGetFileByCIDInfo handles GET /api/file/{cid}/info.
|
||||||
|
// Returns lightweight metadata for the file (existence, content type, size,
|
||||||
|
// relative path) without streaming any bytes. Used by the editor to decide
|
||||||
|
// whether to render a preview (image/video) or a download-only button.
|
||||||
|
func (s *Server) handleGetFileByCIDInfo(w http.ResponseWriter, r *http.Request) {
|
||||||
|
vars := mux.Vars(r)
|
||||||
|
cid := vars["cid"]
|
||||||
|
|
||||||
|
type infoResp struct {
|
||||||
|
Exists bool `json:"exists"`
|
||||||
|
ContentType string `json:"contentType,omitempty"`
|
||||||
|
Size int64 `json:"size,omitempty"`
|
||||||
|
FilePath string `json:"filePath,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
|
||||||
|
if cid == "" {
|
||||||
|
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
relPath, err := s.storage.LookupPathByCID(cid)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if relPath == "" {
|
||||||
|
_ = json.NewEncoder(w).Encode(infoResp{Exists: false})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
fullPath := filepath.Join(s.config.FilesPath, relPath)
|
||||||
|
fi, err := os.Stat(fullPath)
|
||||||
|
if err != nil {
|
||||||
|
_ = json.NewEncoder(w).Encode(infoResp{Exists: false, FilePath: relPath})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ext := strings.ToLower(filepath.Ext(fullPath))
|
||||||
|
contentType, ok := contentTypeByExt[ext]
|
||||||
|
if !ok {
|
||||||
|
contentType = "application/octet-stream"
|
||||||
|
}
|
||||||
|
|
||||||
|
_ = json.NewEncoder(w).Encode(infoResp{
|
||||||
|
Exists: true,
|
||||||
|
ContentType: contentType,
|
||||||
|
Size: fi.Size(),
|
||||||
|
FilePath: relPath,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
// handleGetFileByCID handles GET /file/{cid}
|
// handleGetFileByCID handles GET /file/{cid}
|
||||||
// Serves a file by looking up its CID in poster/backdrop metadata
|
// Serves a file by looking up its CID in poster/backdrop metadata
|
||||||
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
|
func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
|
||||||
@@ -364,7 +469,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
if relPath == "" {
|
if relPath == "" {
|
||||||
writeError(w, http.StatusNotFound, "file not found for CID")
|
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file not found for CID", false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -374,7 +479,7 @@ func (s *Server) handleGetFileByCID(w http.ResponseWriter, r *http.Request) {
|
|||||||
// Check if file exists
|
// Check if file exists
|
||||||
fileInfo, err := os.Stat(fullPath)
|
fileInfo, err := os.Stat(fullPath)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(w, http.StatusNotFound, "file does not exist on disk")
|
writeErrorSlug(w, http.StatusNotFound, ErrUnknownCID, "file does not exist on disk", false)
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -465,11 +570,26 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
|||||||
json.NewEncoder(w).Encode(data)
|
json.NewEncoder(w).Encode(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
// writeError writes an error response
|
// writeError writes the typed error envelope. The slug + retryable bit
|
||||||
|
// are inferred from the status (see defaultSlugFor); call writeErrorSlug
|
||||||
|
// directly when a more specific slug applies.
|
||||||
func writeError(w http.ResponseWriter, status int, message string) {
|
func writeError(w http.ResponseWriter, status int, message string) {
|
||||||
|
slug, retryable := defaultSlugFor(status)
|
||||||
writeJSON(w, status, ErrorResponse{
|
writeJSON(w, status, ErrorResponse{
|
||||||
Error: http.StatusText(status),
|
Error: slug,
|
||||||
Message: message,
|
Message: message,
|
||||||
|
Retryable: retryable,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeErrorSlug writes the typed envelope with an explicit slug and
|
||||||
|
// retryable bit. Use this for endpoint-specific slugs (alias_collision,
|
||||||
|
// unknown_cid) where the default mapping from HTTP status would be wrong.
|
||||||
|
func writeErrorSlug(w http.ResponseWriter, status int, slug, message string, retryable bool) {
|
||||||
|
writeJSON(w, status, ErrorResponse{
|
||||||
|
Error: slug,
|
||||||
|
Message: message,
|
||||||
|
Retryable: retryable,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -489,6 +609,11 @@ func (s *Server) handlePatchMeta(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// PATCH is the hot write path for meta-sort. Resolve so plugin output
|
||||||
|
// merges into the watcher's UUID root rather than minting a parallel
|
||||||
|
// midhash-rooted entry.
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
var metadata map[string]string
|
var metadata map[string]string
|
||||||
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
if err := json.NewDecoder(r.Body).Decode(&metadata); err != nil {
|
||||||
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||||
@@ -530,6 +655,8 @@ func (s *Server) handleGetProperty(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
value, err := s.storage.GetProperty(hashID, key)
|
value, err := s.storage.GetProperty(hashID, key)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
writeError(w, http.StatusInternalServerError, err.Error())
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
@@ -575,6 +702,8 @@ func (s *Server) handlePutProperty(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
// Read body
|
// Read body
|
||||||
buf := make([]byte, 1024*1024) // 1MB max
|
buf := make([]byte, 1024*1024) // 1MB max
|
||||||
n, err := r.Body.Read(buf)
|
n, err := r.Body.Read(buf)
|
||||||
@@ -629,6 +758,8 @@ func (s *Server) handleDeleteProperty(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
if err := s.storage.DeleteProperty(hashID, key); err != nil {
|
if err := s.storage.DeleteProperty(hashID, key); err != nil {
|
||||||
writeError(w, http.StatusInternalServerError, err.Error())
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
return
|
return
|
||||||
@@ -664,6 +795,8 @@ func (s *Server) handleAddToSet(w http.ResponseWriter, r *http.Request) {
|
|||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
|
hashID = s.storage.ResolveRoot(hashID)
|
||||||
|
|
||||||
// Read body
|
// Read body
|
||||||
buf := make([]byte, 1024*1024) // 1MB max
|
buf := make([]byte, 1024*1024) // 1MB max
|
||||||
n, err := r.Body.Read(buf)
|
n, err := r.Body.Read(buf)
|
||||||
|
|||||||
@@ -0,0 +1,277 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base64"
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/identity"
|
||||||
|
)
|
||||||
|
|
||||||
|
// identityStatusResponse is the shape of GET /api/identity. The private key
|
||||||
|
// is intentionally absent — it is only ever returned once, by
|
||||||
|
// POST /api/identity/generate, immediately after creation.
|
||||||
|
type identityStatusResponse struct {
|
||||||
|
HasIdentity bool `json:"hasIdentity"`
|
||||||
|
UID string `json:"uid,omitempty"`
|
||||||
|
Curve string `json:"curve,omitempty"`
|
||||||
|
CreatedAt int64 `json:"createdAt,omitempty"`
|
||||||
|
// AccountCount is the total number of accounts in the keystore. Lets
|
||||||
|
// multi-account clients (meta-watch) tell "single identity" from "pick a
|
||||||
|
// profile" without a second call. Older single-identity clients ignore it.
|
||||||
|
AccountCount int `json:"accountCount,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// identityAccountInfo is one entry in GET /api/identity/accounts. Never carries
|
||||||
|
// the private key — that is only ever returned once, by generate.
|
||||||
|
type identityAccountInfo struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Curve string `json:"curve"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityAccountsResponse struct {
|
||||||
|
Accounts []identityAccountInfo `json:"accounts"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityGenerateResponse struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Curve string `json:"curve"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
PrivateKeyHex string `json:"privateKeyHex"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityImportRequest struct {
|
||||||
|
PrivateKeyHex string `json:"privateKeyHex"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityImportResponse struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Curve string `json:"curve"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityDeleteRequest struct {
|
||||||
|
Confirm bool `json:"confirm"`
|
||||||
|
UID string `json:"uid"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identitySignRequest struct {
|
||||||
|
BytesB64 string `json:"bytesB64"`
|
||||||
|
// UID selects which account signs. Optional — when omitted and exactly one
|
||||||
|
// account exists, that account is used (single-identity back-compat).
|
||||||
|
UID string `json:"uid,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identitySignResponse struct {
|
||||||
|
SigB64 string `json:"sigB64"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type identityAEADKeyResponse struct {
|
||||||
|
AEADKeyB64 string `json:"aeadKeyB64"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveAccount picks the account an operation targets. A non-empty uid must
|
||||||
|
// match an existing account. An empty uid falls back to the sole account
|
||||||
|
// (single-identity back-compat); with zero or many accounts and no uid it
|
||||||
|
// returns a non-zero status for the caller to surface. status == 0 means ok.
|
||||||
|
func (s *Server) resolveAccount(uid string) (id *identity.Identity, status int, slug, msg string) {
|
||||||
|
dir := s.config.IdentityAccountsDir()
|
||||||
|
uid = strings.TrimSpace(uid)
|
||||||
|
if uid != "" {
|
||||||
|
got, err := identity.LoadByUID(dir, uid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, http.StatusBadRequest, "bad_uid", err.Error()
|
||||||
|
}
|
||||||
|
if got == nil {
|
||||||
|
return nil, http.StatusNotFound, "no_identity", "no identity for the given uid"
|
||||||
|
}
|
||||||
|
return got, 0, "", ""
|
||||||
|
}
|
||||||
|
list, err := identity.List(dir)
|
||||||
|
if err != nil {
|
||||||
|
return nil, http.StatusInternalServerError, "list_failed", err.Error()
|
||||||
|
}
|
||||||
|
switch len(list) {
|
||||||
|
case 0:
|
||||||
|
return nil, http.StatusNotFound, "no_identity", "no identity is configured"
|
||||||
|
case 1:
|
||||||
|
return list[0], 0, "", ""
|
||||||
|
default:
|
||||||
|
return nil, http.StatusBadRequest, "uid_required", "multiple accounts exist; specify uid"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityGet(w http.ResponseWriter, r *http.Request) {
|
||||||
|
list, err := identity.List(s.config.IdentityAccountsDir())
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if len(list) == 0 {
|
||||||
|
writeJSON(w, http.StatusOK, identityStatusResponse{HasIdentity: false})
|
||||||
|
return
|
||||||
|
}
|
||||||
|
// Back-compat: report the oldest account as "the" identity. Multi-account
|
||||||
|
// clients enumerate via GET /api/identity/accounts instead.
|
||||||
|
def := list[0]
|
||||||
|
writeJSON(w, http.StatusOK, identityStatusResponse{
|
||||||
|
HasIdentity: true,
|
||||||
|
UID: def.UID,
|
||||||
|
Curve: def.Curve,
|
||||||
|
CreatedAt: def.CreatedAt,
|
||||||
|
AccountCount: len(list),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityAccounts(w http.ResponseWriter, r *http.Request) {
|
||||||
|
list, err := identity.List(s.config.IdentityAccountsDir())
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
out := make([]identityAccountInfo, 0, len(list))
|
||||||
|
for _, id := range list {
|
||||||
|
out = append(out, identityAccountInfo{UID: id.UID, Curve: id.Curve, CreatedAt: id.CreatedAt})
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, identityAccountsResponse{Accounts: out})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityGenerate(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, err := identity.Generate()
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := identity.SaveAccount(s.config.IdentityAccountsDir(), id); err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusCreated, identityGenerateResponse{
|
||||||
|
UID: id.UID,
|
||||||
|
Curve: id.Curve,
|
||||||
|
CreatedAt: id.CreatedAt,
|
||||||
|
PrivateKeyHex: id.PrivateKeyHex,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityImport(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body identityImportRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if strings.TrimSpace(body.PrivateKeyHex) == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "privateKeyHex is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
id, err := identity.Import(strings.TrimSpace(body.PrivateKeyHex))
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dir := s.config.IdentityAccountsDir()
|
||||||
|
if identity.ExistsByUID(dir, id.UID) {
|
||||||
|
writeErrorSlug(w, http.StatusConflict, "identity_exists",
|
||||||
|
"an account with this key already exists", false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if err := identity.SaveAccount(dir, id); err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusCreated, identityImportResponse{
|
||||||
|
UID: id.UID,
|
||||||
|
Curve: id.Curve,
|
||||||
|
CreatedAt: id.CreatedAt,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityDelete(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body identityDeleteRequest
|
||||||
|
// Tolerate empty body — confirm flag in a query param is also acceptable
|
||||||
|
// for shells where sending a JSON body with DELETE is awkward.
|
||||||
|
_ = json.NewDecoder(r.Body).Decode(&body)
|
||||||
|
if !body.Confirm && r.URL.Query().Get("confirm") != "true" {
|
||||||
|
writeErrorSlug(w, http.StatusBadRequest, "confirm_required",
|
||||||
|
"identity deletion requires {\"confirm\": true} or ?confirm=true", false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
dir := s.config.IdentityAccountsDir()
|
||||||
|
uid := strings.TrimSpace(body.UID)
|
||||||
|
if uid == "" {
|
||||||
|
uid = strings.TrimSpace(r.URL.Query().Get("uid"))
|
||||||
|
}
|
||||||
|
if uid == "" {
|
||||||
|
// Back-compat: with no uid, delete the sole account. Ambiguous (and so
|
||||||
|
// refused) once more than one account exists.
|
||||||
|
list, err := identity.List(dir)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
switch len(list) {
|
||||||
|
case 0:
|
||||||
|
w.WriteHeader(http.StatusNoContent) // idempotent
|
||||||
|
return
|
||||||
|
case 1:
|
||||||
|
uid = list[0].UID
|
||||||
|
default:
|
||||||
|
writeErrorSlug(w, http.StatusBadRequest, "uid_required",
|
||||||
|
"multiple accounts exist; specify uid to delete", false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := identity.DeleteByUID(dir, uid); err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
w.WriteHeader(http.StatusNoContent)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentitySign(w http.ResponseWriter, r *http.Request) {
|
||||||
|
var body identitySignRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&body); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
uid := body.UID
|
||||||
|
if strings.TrimSpace(uid) == "" {
|
||||||
|
uid = r.URL.Query().Get("uid")
|
||||||
|
}
|
||||||
|
id, status, slug, msg := s.resolveAccount(uid)
|
||||||
|
if status != 0 {
|
||||||
|
writeErrorSlug(w, status, slug, msg, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
raw, err := base64.StdEncoding.DecodeString(body.BytesB64)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "bytesB64 is not valid base64")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
sig, err := identity.Sign(id, raw)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, identitySignResponse{
|
||||||
|
SigB64: base64.StdEncoding.EncodeToString(sig),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (s *Server) handleIdentityAEADKey(w http.ResponseWriter, r *http.Request) {
|
||||||
|
id, status, slug, msg := s.resolveAccount(r.URL.Query().Get("uid"))
|
||||||
|
if status != 0 {
|
||||||
|
writeErrorSlug(w, status, slug, msg, false)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
key, err := identity.DeriveAEADKey(id)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, identityAEADKeyResponse{
|
||||||
|
AEADKeyB64: base64.StdEncoding.EncodeToString(key),
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -1,12 +1,14 @@
|
|||||||
package api
|
package api
|
||||||
|
|
||||||
import (
|
import (
|
||||||
|
"encoding/json"
|
||||||
"net/http"
|
"net/http"
|
||||||
"net/url"
|
"net/url"
|
||||||
"strconv"
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
|
|
||||||
"github.com/gorilla/mux"
|
"github.com/gorilla/mux"
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
)
|
)
|
||||||
|
|
||||||
// KVInfoResponse is the response for GET /api/kv/info
|
// KVInfoResponse is the response for GET /api/kv/info
|
||||||
@@ -184,3 +186,287 @@ func (s *Server) handleKVGetKey(w http.ResponseWriter, r *http.Request) {
|
|||||||
|
|
||||||
writeJSON(w, http.StatusOK, response)
|
writeJSON(w, http.StatusOK, response)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// KVTreeResponse is the response for GET /api/kv/tree
|
||||||
|
type KVTreeResponse struct {
|
||||||
|
Prefix string `json:"prefix"`
|
||||||
|
Delimiter string `json:"delimiter"`
|
||||||
|
Branches []string `json:"branches"`
|
||||||
|
Leaves []string `json:"leaves"`
|
||||||
|
Cursor string `json:"cursor"`
|
||||||
|
HasMore bool `json:"hasMore"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KVSearchResponse is the response for GET /api/kv/search
|
||||||
|
type KVSearchResponse struct {
|
||||||
|
Contains string `json:"contains"`
|
||||||
|
Keys []string `json:"keys"`
|
||||||
|
Truncated bool `json:"truncated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KVFindResponse is the response for GET /api/kv/find
|
||||||
|
type KVFindResponse struct {
|
||||||
|
Contains string `json:"contains"`
|
||||||
|
Fields []string `json:"fields"`
|
||||||
|
Matches []storage.FindMatch `json:"matches"`
|
||||||
|
Truncated bool `json:"truncated"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KVValueResponse is the response for GET /api/kv/value
|
||||||
|
type KVValueResponse struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Type string `json:"type"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
Exists bool `json:"exists"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KVValuePutRequest is the body for PUT /api/kv/value
|
||||||
|
type KVValuePutRequest struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// KVValueDeleteResponse is the response for DELETE /api/kv/value
|
||||||
|
type KVValueDeleteResponse struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Deleted bool `json:"deleted"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVTree handles GET /api/kv/tree?prefix=&delimiter=/&cursor=0&max=500
|
||||||
|
//
|
||||||
|
// Lists the immediate children at prefix. When delimiter is set, keys with a
|
||||||
|
// further delimiter are collapsed into branches. Pass the returned cursor back
|
||||||
|
// to continue when hasMore is true.
|
||||||
|
func (s *Server) handleKVTree(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
prefix := q.Get("prefix")
|
||||||
|
delimiter := q.Get("delimiter")
|
||||||
|
cursor := q.Get("cursor")
|
||||||
|
|
||||||
|
maxResults := 500
|
||||||
|
if v := q.Get("max"); v != "" {
|
||||||
|
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||||
|
maxResults = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
leaves, branches, nextCursor, err := s.storage.ScanKeys(prefix, delimiter, cursor, maxResults)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVTreeResponse{
|
||||||
|
Prefix: prefix,
|
||||||
|
Delimiter: delimiter,
|
||||||
|
Branches: branches,
|
||||||
|
Leaves: leaves,
|
||||||
|
Cursor: nextCursor,
|
||||||
|
HasMore: nextCursor != "0",
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVFind handles GET /api/kv/find?contains=&fields=a,b,c&limit=200
|
||||||
|
//
|
||||||
|
// Value-search: scans every key whose path ends with /<field> for each
|
||||||
|
// configured field, then returns those whose value contains the substring
|
||||||
|
// (case-insensitive). Used as the editor's "filter the view" action.
|
||||||
|
func (s *Server) handleKVFind(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
contains := q.Get("contains")
|
||||||
|
if contains == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "contains is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fieldsParam := q.Get("fields")
|
||||||
|
if fieldsParam == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "fields is required (comma-separated)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fields := strings.Split(fieldsParam, ",")
|
||||||
|
for i := range fields {
|
||||||
|
fields[i] = strings.TrimSpace(fields[i])
|
||||||
|
}
|
||||||
|
|
||||||
|
limit := 200
|
||||||
|
if v := q.Get("limit"); v != "" {
|
||||||
|
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||||
|
limit = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
matches, truncated, err := s.storage.FindByValue(contains, fields, limit)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if matches == nil {
|
||||||
|
matches = []storage.FindMatch{}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVFindResponse{
|
||||||
|
Contains: contains,
|
||||||
|
Fields: fields,
|
||||||
|
Matches: matches,
|
||||||
|
Truncated: truncated,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVSearch handles GET /api/kv/search?contains=&limit=500
|
||||||
|
func (s *Server) handleKVSearch(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
contains := q.Get("contains")
|
||||||
|
if contains == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "contains is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
limit := 500
|
||||||
|
if v := q.Get("limit"); v != "" {
|
||||||
|
if n, err := strconv.Atoi(v); err == nil && n > 0 {
|
||||||
|
limit = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
keys, truncated, err := s.storage.SearchKeys(contains, limit)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVSearchResponse{
|
||||||
|
Contains: contains,
|
||||||
|
Keys: keys,
|
||||||
|
Truncated: truncated,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVValueGet handles GET /api/kv/value?key=...
|
||||||
|
func (s *Server) handleKVValueGet(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
key := r.URL.Query().Get("key")
|
||||||
|
if key == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "key is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
kind, value, exists, err := s.storage.GetKeyInfo(key)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVValueResponse{
|
||||||
|
Key: key,
|
||||||
|
Type: kind,
|
||||||
|
Value: value,
|
||||||
|
Exists: exists,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVValuePut handles PUT /api/kv/value with body {key, value}.
|
||||||
|
// Maintains the file index when the key matches file:<hashId>/...
|
||||||
|
func (s *Server) handleKVValuePut(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
var req KVValuePutRequest
|
||||||
|
if err := json.NewDecoder(r.Body).Decode(&req); err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, "invalid JSON body")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if req.Key == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "key is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := s.storage.SetRawValue(req.Key, req.Value); err != nil {
|
||||||
|
writeError(w, http.StatusConflict, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if hashID := extractFileHashID(req.Key); hashID != "" {
|
||||||
|
if err := s.storage.IndexAdd(hashID); err != nil {
|
||||||
|
// Non-fatal: the value was written; index entry is best-effort.
|
||||||
|
// Surface in the log path of writeJSON via response header.
|
||||||
|
w.Header().Set("X-KV-Index-Warning", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVValueResponse{
|
||||||
|
Key: req.Key,
|
||||||
|
Type: "string",
|
||||||
|
Value: req.Value,
|
||||||
|
Exists: true,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleKVValueDelete handles DELETE /api/kv/value?key=...
|
||||||
|
// When the key matches file:<hashId>/..., removes the file index entry if no
|
||||||
|
// sibling keys remain after the delete.
|
||||||
|
func (s *Server) handleKVValueDelete(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
key := r.URL.Query().Get("key")
|
||||||
|
if key == "" {
|
||||||
|
writeError(w, http.StatusBadRequest, "key is required")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
deleted, err := s.storage.DeleteRawKey(key)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
if deleted {
|
||||||
|
if hashID := extractFileHashID(key); hashID != "" {
|
||||||
|
if _, err := s.storage.IndexRemoveIfEmpty(hashID); err != nil {
|
||||||
|
w.Header().Set("X-KV-Index-Warning", err.Error())
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, KVValueDeleteResponse{
|
||||||
|
Key: key,
|
||||||
|
Deleted: deleted,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractFileHashID returns the hashID for a key shaped like file:<hashId>/...
|
||||||
|
// Returns "" when the key does not follow that convention.
|
||||||
|
func extractFileHashID(key string) string {
|
||||||
|
if !strings.HasPrefix(key, "file:") {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
rest := strings.TrimPrefix(key, "file:")
|
||||||
|
slash := strings.Index(rest, "/")
|
||||||
|
if slash <= 0 {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return rest[:slash]
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,45 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleSchemaGet returns the live per-field schema derived from meta:events.
|
||||||
|
func (s *Server) handleSchemaGet(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if s.schemaIndexer == nil {
|
||||||
|
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
snap := s.schemaIndexer.Snapshot()
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(snap)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSchemaRescan re-emits a set event for every existing property via the
|
||||||
|
// meta publisher, waits for the indexer to drain, and returns the refreshed
|
||||||
|
// schema. Equivalent to a full SCAN-based rebuild without locking Redis.
|
||||||
|
func (s *Server) handleSchemaRescan(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if s.schemaIndexer == nil {
|
||||||
|
http.Error(w, "schema indexer not running", http.StatusServiceUnavailable)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
count, err := s.RepublishMetadata()
|
||||||
|
if err != nil {
|
||||||
|
http.Error(w, err.Error(), http.StatusInternalServerError)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
s.schemaIndexer.WaitForDrain(10 * time.Second)
|
||||||
|
|
||||||
|
snap := s.schemaIndexer.Snapshot()
|
||||||
|
snap.Source = "rescan"
|
||||||
|
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
_ = json.NewEncoder(w).Encode(map[string]any{
|
||||||
|
"published_events": count,
|
||||||
|
"schema": snap,
|
||||||
|
})
|
||||||
|
}
|
||||||
@@ -11,8 +11,10 @@ import (
|
|||||||
"github.com/metazla/meta-core/internal/config"
|
"github.com/metazla/meta-core/internal/config"
|
||||||
"github.com/metazla/meta-core/internal/discovery"
|
"github.com/metazla/meta-core/internal/discovery"
|
||||||
"github.com/metazla/meta-core/internal/events"
|
"github.com/metazla/meta-core/internal/events"
|
||||||
|
"github.com/metazla/meta-core/internal/identity"
|
||||||
"github.com/metazla/meta-core/internal/leader"
|
"github.com/metazla/meta-core/internal/leader"
|
||||||
"github.com/metazla/meta-core/internal/mounts"
|
"github.com/metazla/meta-core/internal/mounts"
|
||||||
|
"github.com/metazla/meta-core/internal/schema"
|
||||||
"github.com/metazla/meta-core/internal/storage"
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
"github.com/metazla/meta-core/internal/watcher"
|
"github.com/metazla/meta-core/internal/watcher"
|
||||||
"github.com/metazla/meta-core/internal/watchers"
|
"github.com/metazla/meta-core/internal/watchers"
|
||||||
@@ -29,6 +31,7 @@ type Server struct {
|
|||||||
mountsManager *mounts.Manager
|
mountsManager *mounts.Manager
|
||||||
mountsHandlers *mounts.Handlers
|
mountsHandlers *mounts.Handlers
|
||||||
mountPoller *mounts.Poller
|
mountPoller *mounts.Poller
|
||||||
|
mountStatsPoller *mounts.StatsPoller
|
||||||
watcherDispatcher *watcher.Dispatcher
|
watcherDispatcher *watcher.Dispatcher
|
||||||
fileWatcher *watcher.Watcher
|
fileWatcher *watcher.Watcher
|
||||||
watcherHandlers *watcher.Handlers
|
watcherHandlers *watcher.Handlers
|
||||||
@@ -36,6 +39,7 @@ type Server struct {
|
|||||||
watchersPoller *watchers.Poller
|
watchersPoller *watchers.Poller
|
||||||
watchersHandlers *watchers.Handlers
|
watchersHandlers *watchers.Handlers
|
||||||
metaPublisher *events.MetaPublisher
|
metaPublisher *events.MetaPublisher
|
||||||
|
schemaIndexer *schema.Indexer
|
||||||
webdavHandler *webdav.Handler
|
webdavHandler *webdav.Handler
|
||||||
router *mux.Router
|
router *mux.Router
|
||||||
server *http.Server
|
server *http.Server
|
||||||
@@ -71,6 +75,11 @@ func NewServer(
|
|||||||
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
log.Printf("[API] Warning: failed to initialize file watcher: %v", err)
|
||||||
} else {
|
} else {
|
||||||
s.fileWatcher = fileWatcher
|
s.fileWatcher = fileWatcher
|
||||||
|
// Wire storage so the watcher can persist (path,size,mtime,cid)
|
||||||
|
// tuples to Redis as it computes midhashes.
|
||||||
|
if stor != nil {
|
||||||
|
fileWatcher.SetStorageClient(stor)
|
||||||
|
}
|
||||||
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
s.watcherHandlers = watcher.NewHandlers(fileWatcher, s.watcherDispatcher)
|
||||||
// Create scan function from watcher for mounts
|
// Create scan function from watcher for mounts
|
||||||
scanFunc = fileWatcher.ScanMountPath
|
scanFunc = fileWatcher.ScanMountPath
|
||||||
@@ -88,6 +97,20 @@ func NewServer(
|
|||||||
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
|
||||||
// Pass republish callback to handlers (handles nil metaPublisher case)
|
// Pass republish callback to handlers (handles nil metaPublisher case)
|
||||||
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
|
s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
|
||||||
|
|
||||||
|
// Hydrate the in-memory state registry from Redis tuples so
|
||||||
|
// the first scan tick after boot can skip ComputeMidHash256
|
||||||
|
// for files whose (size, mtime) match.
|
||||||
|
configs := watchersManager.List()
|
||||||
|
roots := make([]watcher.WatcherRoot, 0, len(configs))
|
||||||
|
for _, c := range configs {
|
||||||
|
roots = append(roots, watcher.WatcherRoot{ID: c.ID, Root: c.Path})
|
||||||
|
}
|
||||||
|
if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
|
||||||
|
log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
|
||||||
|
} else {
|
||||||
|
log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -103,11 +126,25 @@ func NewServer(
|
|||||||
// Create and configure mount poller
|
// Create and configure mount poller
|
||||||
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
|
s.mountPoller = mounts.NewPoller(mountsManager, scanFunc)
|
||||||
mountsManager.SetPoller(s.mountPoller)
|
mountsManager.SetPoller(s.mountPoller)
|
||||||
|
|
||||||
|
// IO stats poller — queries the local rclone RC API every few seconds
|
||||||
|
// and exposes live throughput per mount via the same status JSON.
|
||||||
|
// Stats are daemon-global (rclone limitation); see stats_poller.go.
|
||||||
|
s.mountStatsPoller = mounts.NewStatsPoller(mountsManager)
|
||||||
|
mountsManager.SetStatsPoller(s.mountStatsPoller)
|
||||||
}
|
}
|
||||||
|
|
||||||
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
|
// Initialize WebDAV handler (caching is handled by nginx proxy_cache)
|
||||||
s.webdavHandler = webdav.NewHandler(cfg)
|
s.webdavHandler = webdav.NewHandler(cfg)
|
||||||
|
|
||||||
|
// One-time migration: fold a pre-multi-account identity.json into the
|
||||||
|
// per-account keystore so existing single-identity hosts keep their uid.
|
||||||
|
if migrated, err := identity.MigrateLegacy(cfg.IdentityFilePath(), cfg.IdentityAccountsDir()); err != nil {
|
||||||
|
log.Printf("[API] Warning: identity migration failed: %v", err)
|
||||||
|
} else if migrated {
|
||||||
|
log.Printf("[API] Migrated legacy identity.json into multi-account keystore")
|
||||||
|
}
|
||||||
|
|
||||||
s.setupRoutes()
|
s.setupRoutes()
|
||||||
return s
|
return s
|
||||||
}
|
}
|
||||||
@@ -126,15 +163,40 @@ func (s *Server) setupRoutes() {
|
|||||||
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
s.router.HandleFunc("/api/metadata/search", s.handleSearchMetadata).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
s.router.HandleFunc("/api/metadata/batch", s.handleBatchUpdate).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
s.router.HandleFunc("/api/metadata/clear", s.handleClearMetadata).Methods("POST")
|
||||||
|
|
||||||
|
// User Data Layer signing identity (see internal/identity). Auth-gated
|
||||||
|
// at the perimeter — do NOT add to nginx-hash-lock unauth bypass.
|
||||||
|
s.router.HandleFunc("/api/identity", s.handleIdentityGet).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/identity/accounts", s.handleIdentityAccounts).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/identity/generate", s.handleIdentityGenerate).Methods("POST")
|
||||||
|
s.router.HandleFunc("/api/identity/import", s.handleIdentityImport).Methods("POST")
|
||||||
|
s.router.HandleFunc("/api/identity", s.handleIdentityDelete).Methods("DELETE")
|
||||||
|
s.router.HandleFunc("/api/identity/sign", s.handleIdentitySign).Methods("POST")
|
||||||
|
s.router.HandleFunc("/api/identity/aead-key", s.handleIdentityAEADKey).Methods("GET")
|
||||||
|
|
||||||
|
// Snapshot (export / import / wipe)
|
||||||
|
s.router.HandleFunc("/api/snapshot/export", s.handleSnapshotExport).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/snapshot/import", s.handleSnapshotImport).Methods("POST")
|
||||||
|
s.router.HandleFunc("/api/snapshot/wipe", s.handleSnapshotWipe).Methods("POST")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataGetProperty).Methods("GET")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
s.router.HandleFunc("/api/metadata/{hashId}/property", s.handleMetadataUpdateProperty).Methods("PUT")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
s.router.HandleFunc("/api/metadata/{hashId}", s.handleGetMetadataByHashId).Methods("GET")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
|
s.router.HandleFunc("/api/metadata/{hashId}", s.handleUpdateMetadataByHashId).Methods("PUT")
|
||||||
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
|
s.router.HandleFunc("/api/metadata/{hashId}", s.handleDeleteMetadataByHashId).Methods("DELETE")
|
||||||
|
|
||||||
|
// Schema inference
|
||||||
|
s.router.HandleFunc("/api/schema", s.handleSchemaGet).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/schema/rescan", s.handleSchemaRescan).Methods("POST")
|
||||||
|
|
||||||
// KV Browser API routes
|
// KV Browser API routes
|
||||||
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
|
s.router.HandleFunc("/api/kv/info", s.handleKVInfo).Methods("GET")
|
||||||
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
|
s.router.HandleFunc("/api/kv/keys", s.handleKVKeys).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/kv/tree", s.handleKVTree).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/kv/search", s.handleKVSearch).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/kv/find", s.handleKVFind).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/kv/value", s.handleKVValueGet).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/kv/value", s.handleKVValuePut).Methods("PUT")
|
||||||
|
s.router.HandleFunc("/api/kv/value", s.handleKVValueDelete).Methods("DELETE")
|
||||||
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
|
s.router.HandleFunc("/api/kv/key/{key:.*}", s.handleKVGetKey).Methods("GET")
|
||||||
|
|
||||||
// Metadata operations - base endpoints
|
// Metadata operations - base endpoints
|
||||||
@@ -157,9 +219,27 @@ func (s *Server) setupRoutes() {
|
|||||||
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
|
s.router.HandleFunc("/data/{hash}", s.handleHeadData).Methods("HEAD")
|
||||||
|
|
||||||
// File operations (by CID)
|
// File operations (by CID)
|
||||||
|
s.router.HandleFunc("/api/file/{cid}/info", s.handleGetFileByCIDInfo).Methods("GET")
|
||||||
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
|
s.router.HandleFunc("/file/{cid}", s.handleGetFileByCID).Methods("GET")
|
||||||
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
|
s.router.HandleFunc("/file/cid", s.handleComputeFileCID).Methods("POST")
|
||||||
|
|
||||||
|
// Public metadata-by-CID resolver. Reverse-index lookup → full document.
|
||||||
|
// Auth-bypassed alongside /api/file/{cid} so meta-share peers can query
|
||||||
|
// without going through Authelia.
|
||||||
|
s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
|
||||||
|
|
||||||
|
// Admin: reunify stranded midhash-rooted entries into their UUID roots
|
||||||
|
// (the dual-root pattern caused by historical writes that bypassed CID
|
||||||
|
// resolution).
|
||||||
|
s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
|
||||||
|
|
||||||
|
// SSE event streams. Mediated mirror of the Redis Streams; consumers
|
||||||
|
// reach them over HTTP instead of touching Redis directly. Auth-bypass
|
||||||
|
// applies but exposure should be inside-only (do NOT proxy through
|
||||||
|
// Caddy publicly). See docs/api-mediated-access.md "Auth".
|
||||||
|
s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
|
||||||
|
s.router.HandleFunc("/api/events/meta", s.handleEventsMeta).Methods("GET")
|
||||||
|
|
||||||
// Service discovery
|
// Service discovery
|
||||||
// Both /services and /api/services are supported for consistency with other services
|
// Both /services and /api/services are supported for consistency with other services
|
||||||
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
|
s.router.HandleFunc("/services", s.handleListServices).Methods("GET")
|
||||||
@@ -234,6 +314,13 @@ func (s *Server) Start() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Start IO stats poller alongside the mount poller. Cheap (one /proc read
|
||||||
|
// per CIFS/NFS mount per cycle, plus one RC call shared by all rclone
|
||||||
|
// mounts), so always-on rather than gated by config.
|
||||||
|
if s.mountStatsPoller != nil {
|
||||||
|
s.mountStatsPoller.Start()
|
||||||
|
}
|
||||||
|
|
||||||
// Start meta publisher (if storage is connected)
|
// Start meta publisher (if storage is connected)
|
||||||
// Publishes keyspace notifications to meta:events stream
|
// Publishes keyspace notifications to meta:events stream
|
||||||
if s.storage != nil && s.storage.IsConnected() {
|
if s.storage != nil && s.storage.IsConnected() {
|
||||||
@@ -241,6 +328,15 @@ func (s *Server) Start() error {
|
|||||||
if err := s.metaPublisher.Start(); err != nil {
|
if err := s.metaPublisher.Start(); err != nil {
|
||||||
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
|
log.Printf("[API] Warning: failed to start meta publisher: %v", err)
|
||||||
} else {
|
} else {
|
||||||
|
// Schema indexer consumes the same stream to derive a live
|
||||||
|
// per-field schema (type/hint/breakdown). Start it before the
|
||||||
|
// republish flood so it captures the bootstrap events.
|
||||||
|
s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
|
||||||
|
if err := s.schemaIndexer.Start(); err != nil {
|
||||||
|
log.Printf("[API] Warning: failed to start schema indexer: %v", err)
|
||||||
|
s.schemaIndexer = nil
|
||||||
|
}
|
||||||
|
|
||||||
// Republish events for all existing metadata
|
// Republish events for all existing metadata
|
||||||
go func() {
|
go func() {
|
||||||
count, err := s.RepublishMetadata()
|
count, err := s.RepublishMetadata()
|
||||||
@@ -272,6 +368,18 @@ func (s *Server) Stop() error {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Stop IO stats poller
|
||||||
|
if s.mountStatsPoller != nil {
|
||||||
|
s.mountStatsPoller.Stop()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop schema indexer before the publisher so the consumer drains cleanly
|
||||||
|
if s.schemaIndexer != nil {
|
||||||
|
if err := s.schemaIndexer.Stop(); err != nil {
|
||||||
|
log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Stop meta publisher
|
// Stop meta publisher
|
||||||
if s.metaPublisher != nil {
|
if s.metaPublisher != nil {
|
||||||
if err := s.metaPublisher.Stop(); err != nil {
|
if err := s.metaPublisher.Stop(); err != nil {
|
||||||
|
|||||||
@@ -0,0 +1,167 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/snapshot"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Bound the upload size so a malicious client can't OOM the server.
|
||||||
|
// 2 GiB is well above any realistic export.
|
||||||
|
const maxImportBytes = 2 << 30
|
||||||
|
|
||||||
|
// pluginOutputDir is where plugin-produced artifacts live on disk
|
||||||
|
// (subtitles, posters, etc). Mounted from the host into the meta-core
|
||||||
|
// container at $FILES_PATH/plugin.
|
||||||
|
func (s *Server) pluginOutputDir() string {
|
||||||
|
return filepath.Join(s.config.FilesPath, "plugin")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotExport streams a ZIP snapshot of all metadata.
|
||||||
|
// GET /api/snapshot/export?include=hash-index,plugin-output
|
||||||
|
func (s *Server) handleSnapshotExport(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The server has a 30s WriteTimeout (server.go:222). Long exports
|
||||||
|
// blow past that, leaving a truncated ZIP without a central directory.
|
||||||
|
// Clear the deadline for this handler so the export can run as long as
|
||||||
|
// needed. ResponseController is the standard way to do this.
|
||||||
|
rc := http.NewResponseController(w)
|
||||||
|
_ = rc.SetWriteDeadline(time.Time{})
|
||||||
|
|
||||||
|
include := splitCSVParam(r.URL.Query().Get("include"))
|
||||||
|
opts := snapshot.ExportOptions{
|
||||||
|
IncludePluginOutput: include["plugin-output"],
|
||||||
|
PluginOutputDir: s.pluginOutputDir(),
|
||||||
|
}
|
||||||
|
|
||||||
|
filename := fmt.Sprintf("metamesh-snapshot-%s.zip", time.Now().UTC().Format("20060102-150405"))
|
||||||
|
w.Header().Set("Content-Type", "application/zip")
|
||||||
|
w.Header().Set("Content-Disposition", fmt.Sprintf(`attachment; filename="%s"`, filename))
|
||||||
|
// We stream — no Content-Length.
|
||||||
|
|
||||||
|
exporter := snapshot.NewExporter(s.storage)
|
||||||
|
if _, err := exporter.Export(w, opts); err != nil {
|
||||||
|
// Headers may already be sent; log and bail.
|
||||||
|
// Client will see a truncated zip, which is the best signal we can give.
|
||||||
|
fmt.Printf("[Snapshot] export failed: %v\n", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotImport applies an uploaded ZIP snapshot.
|
||||||
|
// POST /api/snapshot/import?mode=replace|merge&conflict=mine|source&dry_run=true
|
||||||
|
// Body: application/zip (raw) OR multipart/form-data with field "snapshot"
|
||||||
|
func (s *Server) handleSnapshotImport(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Long imports can blow past the server's 30s ReadTimeout/WriteTimeout
|
||||||
|
// when the snapshot is large. Clear both for this handler.
|
||||||
|
rc := http.NewResponseController(w)
|
||||||
|
_ = rc.SetReadDeadline(time.Time{})
|
||||||
|
_ = rc.SetWriteDeadline(time.Time{})
|
||||||
|
|
||||||
|
q := r.URL.Query()
|
||||||
|
opts := snapshot.ImportOptions{
|
||||||
|
Mode: snapshot.Mode(q.Get("mode")),
|
||||||
|
Conflict: snapshot.Conflict(q.Get("conflict")),
|
||||||
|
DryRun: strings.EqualFold(q.Get("dry_run"), "true"),
|
||||||
|
PluginOutputDir: s.pluginOutputDir(),
|
||||||
|
}
|
||||||
|
if opts.Mode == "" {
|
||||||
|
opts.Mode = snapshot.ModeMerge
|
||||||
|
}
|
||||||
|
if opts.Mode == snapshot.ModeMerge && opts.Conflict == "" {
|
||||||
|
opts.Conflict = snapshot.ConflictSource
|
||||||
|
}
|
||||||
|
|
||||||
|
body, err := readImportBody(r)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
importer := snapshot.NewImporter(s.storage)
|
||||||
|
result, err := importer.Import(bytes.NewReader(body), int64(len(body)), opts)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusBadRequest, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
writeJSON(w, http.StatusOK, result)
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleSnapshotWipe clears Redis metadata.
|
||||||
|
// POST /api/snapshot/wipe?scope=metadata
|
||||||
|
func (s *Server) handleSnapshotWipe(w http.ResponseWriter, r *http.Request) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
scopes := splitCSVParam(r.URL.Query().Get("scope"))
|
||||||
|
if len(scopes) == 0 {
|
||||||
|
writeError(w, http.StatusBadRequest, "scope query param is required (e.g. scope=metadata)")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
scope := snapshot.WipeScope{
|
||||||
|
Metadata: scopes["metadata"],
|
||||||
|
}
|
||||||
|
if !scope.Metadata {
|
||||||
|
writeError(w, http.StatusBadRequest, "no supported scope requested; scope=metadata is the only accepted value")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
wiper := snapshot.NewWiper(s.storage)
|
||||||
|
res, err := wiper.Wipe(scope)
|
||||||
|
if err != nil {
|
||||||
|
writeError(w, http.StatusInternalServerError, err.Error())
|
||||||
|
return
|
||||||
|
}
|
||||||
|
writeJSON(w, http.StatusOK, res)
|
||||||
|
}
|
||||||
|
|
||||||
|
// readImportBody accepts either a raw application/zip body or a multipart
|
||||||
|
// upload with a field named "snapshot". Buffers up to maxImportBytes.
|
||||||
|
func readImportBody(r *http.Request) ([]byte, error) {
|
||||||
|
r.Body = http.MaxBytesReader(nil, r.Body, maxImportBytes)
|
||||||
|
|
||||||
|
ct := r.Header.Get("Content-Type")
|
||||||
|
if strings.HasPrefix(ct, "multipart/form-data") {
|
||||||
|
if err := r.ParseMultipartForm(32 << 20); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse multipart: %w", err)
|
||||||
|
}
|
||||||
|
f, _, err := r.FormFile("snapshot")
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("missing 'snapshot' file field: %w", err)
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
return io.ReadAll(f)
|
||||||
|
}
|
||||||
|
|
||||||
|
return io.ReadAll(r.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitCSVParam parses "a,b,c" into a set. Empty values yield an empty set.
|
||||||
|
func splitCSVParam(v string) map[string]bool {
|
||||||
|
out := map[string]bool{}
|
||||||
|
for _, part := range strings.Split(v, ",") {
|
||||||
|
p := strings.TrimSpace(part)
|
||||||
|
if p != "" {
|
||||||
|
out[p] = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,280 @@
|
|||||||
|
package api
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"log"
|
||||||
|
"net/http"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SSE event stream endpoints for the file:events and meta:events Redis Streams.
|
||||||
|
//
|
||||||
|
// Wire contract (see docs/api-mediated-access.md):
|
||||||
|
// - One SSE event per Redis Stream entry.
|
||||||
|
// - `id:` is the opaque Redis Stream entry ID (`<ms>-<seq>`); consumers
|
||||||
|
// MUST treat it as a black-box string they echo back via Last-Event-ID.
|
||||||
|
// - `event:` is the `type` field from the underlying entry.
|
||||||
|
// - `data:` is the JSON-encoded rest of the entry (all fields except `type`).
|
||||||
|
// - Heartbeats are SSE comment lines (`:keep-alive\n\n`) every 30s of silence.
|
||||||
|
// - On reconnect with Last-Event-ID, resume from the next entry. If the
|
||||||
|
// cursor has been trimmed by retention, emit one `event: gap` payload
|
||||||
|
// before resuming from the oldest available entry.
|
||||||
|
|
||||||
|
const (
|
||||||
|
sseHeartbeatInterval = 30 * time.Second
|
||||||
|
|
||||||
|
// sseReadBlock controls how long each XREAD blocks before returning.
|
||||||
|
// Much shorter than sseHeartbeatInterval so the handler stays responsive
|
||||||
|
// to client disconnects and the heartbeat check runs frequently enough
|
||||||
|
// that the keep-alive fires within a few seconds of its nominal cadence.
|
||||||
|
sseReadBlock = 2 * time.Second
|
||||||
|
|
||||||
|
// sseReadCount caps how many entries one XREAD returns. Small enough
|
||||||
|
// to keep per-flush memory low, large enough to drain bursts cheaply.
|
||||||
|
sseReadCount = 100
|
||||||
|
)
|
||||||
|
|
||||||
|
// handleEventsFiles streams the file:events Redis Stream as SSE.
|
||||||
|
func (s *Server) handleEventsFiles(w http.ResponseWriter, r *http.Request) {
|
||||||
|
s.streamSSE(w, r, "file:events")
|
||||||
|
}
|
||||||
|
|
||||||
|
// handleEventsMeta streams the meta:events Redis Stream as SSE.
|
||||||
|
func (s *Server) handleEventsMeta(w http.ResponseWriter, r *http.Request) {
|
||||||
|
s.streamSSE(w, r, "meta:events")
|
||||||
|
}
|
||||||
|
|
||||||
|
// streamSSE is the shared SSE pump shared by both endpoints. One handler per
|
||||||
|
// stream — there is no fan-out; each subscriber holds its own XREAD loop.
|
||||||
|
// That's fine at our scale (≤20 subscribers; one per consumer service).
|
||||||
|
func (s *Server) streamSSE(w http.ResponseWriter, r *http.Request, stream string) {
|
||||||
|
if !s.storage.IsConnected() {
|
||||||
|
w.Header().Set("Retry-After", "5")
|
||||||
|
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
flusher, ok := w.(http.Flusher)
|
||||||
|
if !ok {
|
||||||
|
writeError(w, http.StatusInternalServerError, "streaming unsupported")
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// The server-wide WriteTimeout (30s on Server.server in server.go) is
|
||||||
|
// fine for ordinary handlers but lethal for SSE — it would close every
|
||||||
|
// connection at exactly 30s, racing the heartbeat. Disable it
|
||||||
|
// per-request via http.NewResponseController. SSE endpoints never need
|
||||||
|
// a write deadline; the client disconnect path already cleans up.
|
||||||
|
rc := http.NewResponseController(w)
|
||||||
|
if err := rc.SetWriteDeadline(time.Time{}); err != nil {
|
||||||
|
// Older transports (or nested response writers) may not support
|
||||||
|
// this. Log and continue — the 30s cap will still apply, but the
|
||||||
|
// handler is still useful for short-lived subscriptions.
|
||||||
|
log.Printf("[SSE %s] could not disable write deadline: %v", stream, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// SSE response headers. X-Accel-Buffering disables nginx response
|
||||||
|
// buffering; the hash-lock sidecar uses nginx, so this matters.
|
||||||
|
w.Header().Set("Content-Type", "text/event-stream")
|
||||||
|
w.Header().Set("Cache-Control", "no-cache")
|
||||||
|
w.Header().Set("Connection", "keep-alive")
|
||||||
|
w.Header().Set("X-Accel-Buffering", "no")
|
||||||
|
w.WriteHeader(http.StatusOK)
|
||||||
|
flusher.Flush()
|
||||||
|
|
||||||
|
client := s.storage.GetRedisClient()
|
||||||
|
if client == nil {
|
||||||
|
// Storage was just disconnected — close cleanly.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx := r.Context()
|
||||||
|
cursor := resolveStartCursor(ctx, client, stream, r.Header.Get("Last-Event-ID"), w, flusher)
|
||||||
|
|
||||||
|
lastWrite := time.Now()
|
||||||
|
for {
|
||||||
|
if err := ctx.Err(); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Bounded BLOCK lets us re-check ctx and emit heartbeats. The
|
||||||
|
// context's deadline isn't propagated into XREAD's BLOCK arg —
|
||||||
|
// that's a poll-style timeout on Redis itself — so we keep BLOCK
|
||||||
|
// short.
|
||||||
|
readCtx, cancel := context.WithTimeout(ctx, sseReadBlock+2*time.Second)
|
||||||
|
result, err := client.XRead(readCtx, &redis.XReadArgs{
|
||||||
|
Streams: []string{stream, cursor},
|
||||||
|
Block: sseReadBlock,
|
||||||
|
Count: sseReadCount,
|
||||||
|
}).Result()
|
||||||
|
cancel()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
// redis.Nil = BLOCK timeout with no entries. Anything else
|
||||||
|
// while the client is still connected is a real failure.
|
||||||
|
if errors.Is(err, redis.Nil) || isContextTimeout(err) {
|
||||||
|
// fall through to heartbeat check
|
||||||
|
} else if errors.Is(err, context.Canceled) || ctx.Err() != nil {
|
||||||
|
return
|
||||||
|
} else {
|
||||||
|
log.Printf("[SSE %s] XREAD error: %v", stream, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, str := range result {
|
||||||
|
for _, msg := range str.Messages {
|
||||||
|
writeSSEEntry(w, flusher, msg)
|
||||||
|
cursor = msg.ID
|
||||||
|
lastWrite = time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Heartbeat if quiet for >= the interval. Comment line; SSE
|
||||||
|
// clients ignore it per spec.
|
||||||
|
if time.Since(lastWrite) >= sseHeartbeatInterval {
|
||||||
|
if _, err := fmt.Fprint(w, ":keep-alive\n\n"); err != nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
flusher.Flush()
|
||||||
|
lastWrite = time.Now()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// resolveStartCursor decides where to start reading.
|
||||||
|
// - No Last-Event-ID → start at "$" (only new entries).
|
||||||
|
// - Last-Event-ID provided AND still in stream → resume from it.
|
||||||
|
// - Last-Event-ID provided BUT trimmed → emit one synthetic `gap` event
|
||||||
|
// and resume from the oldest available entry's predecessor "0", so the
|
||||||
|
// next XREAD picks up everything still in the stream.
|
||||||
|
//
|
||||||
|
// The "still in stream" check is cheap: one XRANGE for the first entry.
|
||||||
|
func resolveStartCursor(ctx context.Context, client *redis.Client, stream, lastID string, w http.ResponseWriter, flusher http.Flusher) string {
|
||||||
|
if lastID == "" {
|
||||||
|
return "$"
|
||||||
|
}
|
||||||
|
|
||||||
|
// One-entry XRANGE to find the oldest ID currently in the stream.
|
||||||
|
xCtx, cancel := context.WithTimeout(ctx, 2*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
entries, err := client.XRangeN(xCtx, stream, "-", "+", 1).Result()
|
||||||
|
if err != nil {
|
||||||
|
// Treat any error as "stream not available right now" and resume
|
||||||
|
// from the cursor — XREAD will either find it or block until new
|
||||||
|
// entries arrive.
|
||||||
|
log.Printf("[SSE %s] XRANGE probe failed: %v (resuming from cursor)", stream, err)
|
||||||
|
return lastID
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(entries) == 0 {
|
||||||
|
// Stream is empty. Cursor may be from a previous incarnation; safest
|
||||||
|
// is to resume from the cursor — XREAD against an empty stream
|
||||||
|
// returns nothing and blocks until new entries arrive.
|
||||||
|
return lastID
|
||||||
|
}
|
||||||
|
|
||||||
|
oldest := entries[0].ID
|
||||||
|
if compareStreamIDs(lastID, oldest) >= 0 {
|
||||||
|
// Cursor is at or past the oldest — fine to resume.
|
||||||
|
return lastID
|
||||||
|
}
|
||||||
|
|
||||||
|
// Cursor was trimmed. Emit a gap event so consumers that care can react.
|
||||||
|
gap := map[string]string{
|
||||||
|
"requested": lastID,
|
||||||
|
"resumeFrom": oldest,
|
||||||
|
"reason": "retention",
|
||||||
|
}
|
||||||
|
payload, _ := json.Marshal(gap)
|
||||||
|
fmt.Fprintf(w, "event: gap\ndata: %s\n\n", string(payload))
|
||||||
|
flusher.Flush()
|
||||||
|
|
||||||
|
// Return "0" so the next XREAD picks up entries starting from the
|
||||||
|
// stream's true minimum — including `oldest` itself.
|
||||||
|
return "0"
|
||||||
|
}
|
||||||
|
|
||||||
|
// writeSSEEntry serialises one Redis Stream entry as an SSE event.
|
||||||
|
//
|
||||||
|
// `event:` comes from the entry's `type` field (no prefix munging). `data:`
|
||||||
|
// is a JSON object containing all OTHER fields. Field values from the Redis
|
||||||
|
// stream are interface{} (typically strings); we encode whatever the stream
|
||||||
|
// holds.
|
||||||
|
func writeSSEEntry(w http.ResponseWriter, flusher http.Flusher, msg redis.XMessage) {
|
||||||
|
eventType := "message"
|
||||||
|
data := make(map[string]interface{}, len(msg.Values))
|
||||||
|
for k, v := range msg.Values {
|
||||||
|
if k == "type" {
|
||||||
|
if s, ok := v.(string); ok {
|
||||||
|
eventType = s
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
data[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
payload, err := json.Marshal(data)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[SSE] failed to marshal entry %s: %v", msg.ID, err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Spec: each event ends with a blank line. id/event/data lines are
|
||||||
|
// terminated with single \n.
|
||||||
|
fmt.Fprintf(w, "id: %s\n", msg.ID)
|
||||||
|
fmt.Fprintf(w, "event: %s\n", eventType)
|
||||||
|
fmt.Fprintf(w, "data: %s\n\n", string(payload))
|
||||||
|
flusher.Flush()
|
||||||
|
}
|
||||||
|
|
||||||
|
// compareStreamIDs compares two Redis Stream IDs of the form "<ms>-<seq>".
|
||||||
|
// Returns -1/0/+1. Missing or malformed IDs sort as zero parts.
|
||||||
|
func compareStreamIDs(a, b string) int {
|
||||||
|
aMs, aSeq := parseStreamID(a)
|
||||||
|
bMs, bSeq := parseStreamID(b)
|
||||||
|
switch {
|
||||||
|
case aMs < bMs:
|
||||||
|
return -1
|
||||||
|
case aMs > bMs:
|
||||||
|
return 1
|
||||||
|
case aSeq < bSeq:
|
||||||
|
return -1
|
||||||
|
case aSeq > bSeq:
|
||||||
|
return 1
|
||||||
|
default:
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseStreamID(id string) (int64, int64) {
|
||||||
|
parts := strings.SplitN(id, "-", 2)
|
||||||
|
ms, _ := strconv.ParseInt(parts[0], 10, 64)
|
||||||
|
var seq int64
|
||||||
|
if len(parts) == 2 {
|
||||||
|
seq, _ = strconv.ParseInt(parts[1], 10, 64)
|
||||||
|
}
|
||||||
|
return ms, seq
|
||||||
|
}
|
||||||
|
|
||||||
|
// isContextTimeout returns true for the context deadline error wrapped by
|
||||||
|
// go-redis when XREAD's read context fires. We treat it the same as
|
||||||
|
// redis.Nil — i.e. an empty poll cycle, not a fatal error.
|
||||||
|
func isContextTimeout(err error) bool {
|
||||||
|
if err == nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if errors.Is(err, context.DeadlineExceeded) {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
// go-redis sometimes wraps this in its own error type. String match is
|
||||||
|
// a safety net.
|
||||||
|
return strings.Contains(err.Error(), "context deadline exceeded")
|
||||||
|
}
|
||||||
@@ -0,0 +1,136 @@
|
|||||||
|
// Package cid centralizes CID-format helpers: decoding a multibase-base32
|
||||||
|
// CIDv1 into its multicodec parts, ranking by utility for external
|
||||||
|
// presentation, and picking which CID is the canonical externally-visible
|
||||||
|
// identifier of a file.
|
||||||
|
//
|
||||||
|
// Sibling CIDs are stored as a bare-CID key-set (`cids/<cid> = "true"`); a
|
||||||
|
// CID is self-describing — its algorithm is recoverable from the CIDv1
|
||||||
|
// codec / multihash code, so there is no `<algorithm>:` prefix and no
|
||||||
|
// per-algorithm field name. Ranking and selection parse the multicodec
|
||||||
|
// rather than a string prefix.
|
||||||
|
package cid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base32"
|
||||||
|
"encoding/binary"
|
||||||
|
"fmt"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Multicodec / multihash codes used for ranking. Source of truth for the
|
||||||
|
// digest codes is meta-hash/src/lib/hash-compute/MultiHashData.ts and
|
||||||
|
// METADATA_KEYS.md §2.
|
||||||
|
const (
|
||||||
|
CodecRaw = 0x55 // raw — IPFS content-addressed block
|
||||||
|
CodecDagPB = 0x70 // dag-pb — IPFS content-addressed block
|
||||||
|
CodeSha1 = 0x11 // multihash sha1
|
||||||
|
CodeSha256 = 0x12 // multihash sha2-256
|
||||||
|
CodeSha3_384 = 0x15 // multihash sha3-384
|
||||||
|
CodeSha3_256 = 0x16 // multihash sha3-256
|
||||||
|
CodeMd5 = 0xd5 // multihash md5
|
||||||
|
CodeCrc32 = 0x0132 // multihash crc32
|
||||||
|
CodeMidhash256 = 0x1000 // custom — SHA-256(size‖middle 1MB)
|
||||||
|
CodeBtPiecesRoot = 0xb702 // bittorrent v2 pieces root
|
||||||
|
CodeBtihV2 = 0x10B7 // custom — BitTorrent v2 info hash (BEP 52)
|
||||||
|
CodeBtihV1File = 0x1001 // custom — per-file BitTorrent v1 CID
|
||||||
|
CodeBtihV2File = 0x1002 // custom — per-file BitTorrent v2 CID
|
||||||
|
)
|
||||||
|
|
||||||
|
// Decode parses a multibase-base32 CIDv1 string ("b…") into its content
|
||||||
|
// codec and multihash function code. It is the inverse of the encoder in
|
||||||
|
// internal/watcher/midhash.go. Returns an error for anything that isn't a
|
||||||
|
// CIDv1 in base32lower (e.g. a CIDv0 "Qm…" or a non-CID string).
|
||||||
|
func Decode(cidStr string) (codec uint64, mhCode uint64, err error) {
|
||||||
|
if len(cidStr) < 2 || cidStr[0] != 'b' {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
|
||||||
|
}
|
||||||
|
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
|
||||||
|
}
|
||||||
|
version, n := binary.Uvarint(raw)
|
||||||
|
if n <= 0 {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: bad version varint", cidStr)
|
||||||
|
}
|
||||||
|
raw = raw[n:]
|
||||||
|
if version != 1 {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
|
||||||
|
}
|
||||||
|
codec, n = binary.Uvarint(raw)
|
||||||
|
if n <= 0 {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: bad codec varint", cidStr)
|
||||||
|
}
|
||||||
|
raw = raw[n:]
|
||||||
|
mhCode, n = binary.Uvarint(raw)
|
||||||
|
if n <= 0 {
|
||||||
|
return 0, 0, fmt.Errorf("cid %q: bad multihash-code varint", cidStr)
|
||||||
|
}
|
||||||
|
return codec, mhCode, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Rank returns a score for a bare CID. Higher = more preferred as the
|
||||||
|
// public-facing "canonical" identifier for a file. Unknown / undecodable
|
||||||
|
// CIDs rank 0 so a new digest type can't accidentally outrank an
|
||||||
|
// established one until it's classified here.
|
||||||
|
//
|
||||||
|
// We rank by the **multihash code** (the algorithm), with the custom
|
||||||
|
// BitTorrent CID codecs checked alongside it. This is deliberate: MetaMesh's
|
||||||
|
// two CID encoders disagree on the *content codec* — meta-hash/the watcher
|
||||||
|
// set codec == algorithm code, while the fullhash plugin and gateway wrap
|
||||||
|
// every digest with the raw codec (0x55) and put the algorithm in the
|
||||||
|
// multihash code. The multihash code is the algorithm in all of them, so it
|
||||||
|
// is the only reliable discriminator. (The BT info hash is given a distinct
|
||||||
|
// multihash code — btih-v2 0x10B7 — so it can't be confused with a plain
|
||||||
|
// sha2-256, see METADATA_KEYS.md §14.13.)
|
||||||
|
//
|
||||||
|
// Ordering rationale (utility-weighted, NOT cryptographic-strength):
|
||||||
|
// - dag-pb (chunked IPFS file) wins: best retrieval over the IPFS network.
|
||||||
|
// - sha2-256 / sha3 digests follow: standard content hashes (a raw IPFS
|
||||||
|
// block of the file IS its sha2-256, so they share this tier).
|
||||||
|
// - BitTorrent infohashes (btih v2, per-file v1/v2) outrank midhash256 for
|
||||||
|
// swarm interop, not strength.
|
||||||
|
// - midhash256 is the floor among real options: internally meaningful,
|
||||||
|
// externally invisible.
|
||||||
|
// - md5 / crc32 / plain sha1 / pieces-root / unknown rank 0.
|
||||||
|
func Rank(cidStr string) int {
|
||||||
|
codec, mh, err := Decode(cidStr)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
if codec == CodecDagPB {
|
||||||
|
return 40
|
||||||
|
}
|
||||||
|
// BitTorrent: distinct CID codecs (per-file v1/v2) or the btih-v2
|
||||||
|
// multihash code, in either encoder convention.
|
||||||
|
if codec == CodeBtihV1File || codec == CodeBtihV2File || codec == CodeBtihV2 ||
|
||||||
|
mh == CodeBtihV1File || mh == CodeBtihV2File || mh == CodeBtihV2 {
|
||||||
|
return 20
|
||||||
|
}
|
||||||
|
switch mh {
|
||||||
|
case CodeSha256, CodeSha3_256, CodeSha3_384:
|
||||||
|
return 30
|
||||||
|
case CodeMidhash256:
|
||||||
|
return 10
|
||||||
|
default:
|
||||||
|
// md5, crc32, plain sha1, pieces-root, unknown.
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Better picks the higher-ranked of two CIDs. On ties, returns the
|
||||||
|
// lexicographically smaller one — gives the reconciler a deterministic
|
||||||
|
// answer so the chosen canonical doesn't churn when peers add the same
|
||||||
|
// digest under a different ordering.
|
||||||
|
func Better(a, b string) string {
|
||||||
|
ra, rb := Rank(a), Rank(b)
|
||||||
|
if ra > rb {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
if rb > ra {
|
||||||
|
return b
|
||||||
|
}
|
||||||
|
if a <= b {
|
||||||
|
return a
|
||||||
|
}
|
||||||
|
return b
|
||||||
|
}
|
||||||
@@ -0,0 +1,130 @@
|
|||||||
|
package cid
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/base32"
|
||||||
|
"encoding/binary"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
// buildCID assembles a multibase-base32 CIDv1 from a content codec, a
|
||||||
|
// multihash code, and a digest length (digest bytes are zero — only the
|
||||||
|
// header is under test). Mirrors the encoder in watcher/midhash.go and the
|
||||||
|
// TS CID.createV1(code, create(code, digest)) path.
|
||||||
|
func buildCID(codec, mhCode uint64, digestLen int) string {
|
||||||
|
var b []byte
|
||||||
|
b = binary.AppendUvarint(b, 1) // CIDv1
|
||||||
|
b = binary.AppendUvarint(b, codec)
|
||||||
|
b = binary.AppendUvarint(b, mhCode)
|
||||||
|
b = binary.AppendUvarint(b, uint64(digestLen))
|
||||||
|
b = append(b, make([]byte, digestLen)...)
|
||||||
|
enc := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(b)
|
||||||
|
return "b" + strings.ToLower(enc)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Real CIDs produced by @metazla/meta-hash (from its main.test.ts). For
|
||||||
|
// digest CIDs meta-hash sets codec == multihash code == algorithm code.
|
||||||
|
const (
|
||||||
|
cidCrc32 = "bagzafmqcathsbnrk"
|
||||||
|
cidMd5 = "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa"
|
||||||
|
cidSha1 = "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra"
|
||||||
|
cidSha256 = "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua"
|
||||||
|
cidSha3256 = "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq"
|
||||||
|
cidSha3384 = "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestDecode_Fixtures(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
cid string
|
||||||
|
wantMh uint64
|
||||||
|
}{
|
||||||
|
{"crc32", cidCrc32, CodeCrc32},
|
||||||
|
{"md5", cidMd5, CodeMd5},
|
||||||
|
{"sha1", cidSha1, CodeSha1},
|
||||||
|
{"sha2-256", cidSha256, CodeSha256},
|
||||||
|
{"sha3-256", cidSha3256, CodeSha3_256},
|
||||||
|
{"sha3-384", cidSha3384, CodeSha3_384},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
codec, mh, err := Decode(tc.cid)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("Decode(%s %q) error: %v", tc.name, tc.cid, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
// meta-hash digest CIDs use codec == multihash code.
|
||||||
|
if codec != tc.wantMh || mh != tc.wantMh {
|
||||||
|
t.Errorf("Decode(%s) = codec 0x%x mh 0x%x, want both 0x%x", tc.name, codec, mh, tc.wantMh)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDecode_Rejects(t *testing.T) {
|
||||||
|
for _, in := range []string{"", "Qmabc", "x", "bzzzz!!"} {
|
||||||
|
if _, _, err := Decode(in); err == nil {
|
||||||
|
t.Errorf("Decode(%q) expected error, got nil", in)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRank_Tiers(t *testing.T) {
|
||||||
|
ipfsDagPB := buildCID(CodecDagPB, CodeSha256, 32) // chunked IPFS file
|
||||||
|
rawSha256 := buildCID(CodecRaw, CodeSha256, 32) // fullhash-style sha2-256
|
||||||
|
btihV2meta := buildCID(CodeBtihV2, CodeBtihV2, 32) // meta-hash btih-v2 (codec==mh)
|
||||||
|
btihV2raw := buildCID(CodecRaw, CodeBtihV2, 32) // fullhash btih-v2 (raw codec)
|
||||||
|
btV1File := buildCID(CodeBtihV1File, CodeBtihV1File, 20) // gateway per-file btih v1
|
||||||
|
midhashRaw := buildCID(CodecRaw, CodeMidhash256, 32) // fullhash midhash (raw codec)
|
||||||
|
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
cid string
|
||||||
|
want int
|
||||||
|
}{
|
||||||
|
{"ipfs-dagpb", ipfsDagPB, 40},
|
||||||
|
{"sha2-256(fixture, codec==mh)", cidSha256, 30},
|
||||||
|
{"sha2-256(raw codec)", rawSha256, 30},
|
||||||
|
{"sha3-256", cidSha3256, 30},
|
||||||
|
{"sha3-384", cidSha3384, 30},
|
||||||
|
{"btih-v2(codec==mh)", btihV2meta, 20},
|
||||||
|
{"btih-v2(raw codec)", btihV2raw, 20},
|
||||||
|
{"btih-v1-file", btV1File, 20},
|
||||||
|
{"midhash256(raw codec)", midhashRaw, 10},
|
||||||
|
{"sha1", cidSha1, 0},
|
||||||
|
{"md5", cidMd5, 0},
|
||||||
|
{"crc32", cidCrc32, 0},
|
||||||
|
{"garbage", "not-a-cid", 0},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := Rank(tc.cid); got != tc.want {
|
||||||
|
t.Errorf("Rank(%s) = %d, want %d", tc.name, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestRank_StrictOrdering(t *testing.T) {
|
||||||
|
ipfs := Rank(buildCID(CodecDagPB, CodeSha256, 32))
|
||||||
|
sha := Rank(cidSha256)
|
||||||
|
btih := Rank(buildCID(CodeBtihV2, CodeBtihV2, 32))
|
||||||
|
mid := Rank(buildCID(CodeMidhash256, CodeMidhash256, 32))
|
||||||
|
unknown := Rank(cidMd5)
|
||||||
|
if !(ipfs > sha && sha > btih && btih > mid && mid > unknown) {
|
||||||
|
t.Fatalf("rank ordering broken: ipfs=%d sha=%d btih=%d mid=%d unknown=%d",
|
||||||
|
ipfs, sha, btih, mid, unknown)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestBetter(t *testing.T) {
|
||||||
|
dagpb := buildCID(CodecDagPB, CodeSha256, 32)
|
||||||
|
// dag-pb (IPFS) beats a sha2-256 digest, regardless of arg order.
|
||||||
|
if got := Better(dagpb, cidSha256); got != dagpb {
|
||||||
|
t.Errorf("Better(dagpb, sha256) = %q, want dagpb", got)
|
||||||
|
}
|
||||||
|
if got := Better(cidSha256, dagpb); got != dagpb {
|
||||||
|
t.Errorf("Better(sha256, dagpb) = %q, want dagpb (order shouldn't matter)", got)
|
||||||
|
}
|
||||||
|
// Tie → lexicographically smaller wins, deterministic.
|
||||||
|
a, b := "bafyaaa", "bafyaab"
|
||||||
|
if Better(a, b) != a || Better(b, a) != a {
|
||||||
|
t.Errorf("tie-break not deterministic for %q vs %q", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -17,6 +17,7 @@ type Config struct {
|
|||||||
ServiceVersion string // Service version (default: "1.0.0")
|
ServiceVersion string // Service version (default: "1.0.0")
|
||||||
APIPort int // Service HTTP port (for leader info)
|
APIPort int // Service HTTP port (for leader info)
|
||||||
BaseURL string // Base URL for stable service discovery
|
BaseURL string // Base URL for stable service discovery
|
||||||
|
PublicURL string // Browser-facing URL for the service-bar nav; overrides BaseURL when set (e.g. a debug-direct port with no Caddy in front)
|
||||||
|
|
||||||
// Redis configuration
|
// Redis configuration
|
||||||
RedisPort int // Redis port (default: 6379)
|
RedisPort int // Redis port (default: 6379)
|
||||||
@@ -50,6 +51,7 @@ func Load() *Config {
|
|||||||
ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"),
|
ServiceVersion: getEnv("SERVICE_VERSION", "1.0.0"),
|
||||||
APIPort: getEnvInt("API_PORT", 8180),
|
APIPort: getEnvInt("API_PORT", 8180),
|
||||||
BaseURL: getEnv("BASE_URL", ""),
|
BaseURL: getEnv("BASE_URL", ""),
|
||||||
|
PublicURL: getEnv("META_CORE_PUBLIC_URL", ""),
|
||||||
RedisPort: getEnvInt("REDIS_PORT", 6379),
|
RedisPort: getEnvInt("REDIS_PORT", 6379),
|
||||||
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
|
HTTPPort: getEnvInt("META_CORE_HTTP_PORT", 9000),
|
||||||
HTTPHost: getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
|
HTTPHost: getEnv("META_CORE_HTTP_HOST", "127.0.0.1"),
|
||||||
@@ -104,6 +106,27 @@ func (c *Config) WatchersFilePath() string {
|
|||||||
return c.MetaCorePath + "/watchers.json"
|
return c.MetaCorePath + "/watchers.json"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// IdentityDir returns the directory holding the User Data Layer signing
|
||||||
|
// identity (created by the dashboard's Identity menu — never auto-generated
|
||||||
|
// at startup).
|
||||||
|
func (c *Config) IdentityDir() string {
|
||||||
|
return c.MetaCorePath + "/identity"
|
||||||
|
}
|
||||||
|
|
||||||
|
// IdentityFilePath returns the path to the legacy single-identity JSON. It is
|
||||||
|
// only read once, by the multi-account migration on startup; new accounts live
|
||||||
|
// under IdentityAccountsDir.
|
||||||
|
func (c *Config) IdentityFilePath() string {
|
||||||
|
return c.IdentityDir() + "/identity.json"
|
||||||
|
}
|
||||||
|
|
||||||
|
// IdentityAccountsDir returns the directory holding one JSON file per account
|
||||||
|
// in the multi-account keystore ({uid}.json). meta-core is a pure multi-key
|
||||||
|
// signer — display name / avatar live in the User Data Layer, not here.
|
||||||
|
func (c *Config) IdentityAccountsDir() string {
|
||||||
|
return c.IdentityDir() + "/accounts"
|
||||||
|
}
|
||||||
|
|
||||||
func getEnv(key, defaultValue string) string {
|
func getEnv(key, defaultValue string) string {
|
||||||
if value := os.Getenv(key); value != "" {
|
if value := os.Getenv(key); value != "" {
|
||||||
return value
|
return value
|
||||||
@@ -132,3 +155,20 @@ func getEnvBool(key string, defaultValue bool) bool {
|
|||||||
func (c *Config) DefaultWatchPath() string {
|
func (c *Config) DefaultWatchPath() string {
|
||||||
return c.FilesPath + "/watch"
|
return c.FilesPath + "/watch"
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// DefaultWatcherPaths returns every directory that should be watched out of
|
||||||
|
// the box on a fresh install. The watcher seeds these on first start.
|
||||||
|
//
|
||||||
|
// - /files/watch — user-curated media. The main library.
|
||||||
|
// - /files/plugin — plugin output (TMDB posters/backdrops, extracted
|
||||||
|
// subtitles, etc.). Indexed so each artefact gets a
|
||||||
|
// midhash256 alias and CID-based references from
|
||||||
|
// other files (e.g. `backdrop=<cid>`) resolve through
|
||||||
|
// the standard reverse index instead of needing a
|
||||||
|
// path companion field.
|
||||||
|
func (c *Config) DefaultWatcherPaths() []string {
|
||||||
|
return []string{
|
||||||
|
c.FilesPath + "/watch",
|
||||||
|
c.FilesPath + "/plugin",
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -87,9 +87,13 @@ func (s *Service) buildServiceInfo() *ServiceInfo {
|
|||||||
hostname, _ := os.Hostname()
|
hostname, _ := os.Hostname()
|
||||||
ip := getLocalIP()
|
ip := getLocalIP()
|
||||||
|
|
||||||
// Use BASE_URL for external access, fall back to internal IP
|
// Browser-facing nav URL: prefer META_CORE_PUBLIC_URL (the reachable URL
|
||||||
|
// when there's no Caddy perimeter in front — e.g. a debug-direct port),
|
||||||
|
// then BASE_URL (the Caddy perimeter URL), then the internal IP.
|
||||||
var baseUrl string
|
var baseUrl string
|
||||||
if s.config.BaseURL != "" {
|
if s.config.PublicURL != "" {
|
||||||
|
baseUrl = s.config.PublicURL
|
||||||
|
} else if s.config.BaseURL != "" {
|
||||||
baseUrl = s.config.BaseURL
|
baseUrl = s.config.BaseURL
|
||||||
} else {
|
} else {
|
||||||
baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
|
baseUrl = fmt.Sprintf("http://%s:%d", ip, s.config.APIPort)
|
||||||
|
|||||||
@@ -14,6 +14,12 @@ import (
|
|||||||
const (
|
const (
|
||||||
// MetaEventsStream is the Redis stream name for metadata events
|
// MetaEventsStream is the Redis stream name for metadata events
|
||||||
MetaEventsStream = "meta:events"
|
MetaEventsStream = "meta:events"
|
||||||
|
|
||||||
|
// MetaEventsMaxLen is the approximate cap on stream length. Bounded
|
||||||
|
// retention is a contract requirement for the SSE wire — see
|
||||||
|
// docs/api-mediated-access.md "Backing-store retention and lifecycle".
|
||||||
|
// Cheap (~) trim keeps growth bounded without serialising writes.
|
||||||
|
MetaEventsMaxLen = 100_000
|
||||||
)
|
)
|
||||||
|
|
||||||
// MetaPublisher subscribes to Redis keyspace notifications
|
// MetaPublisher subscribes to Redis keyspace notifications
|
||||||
@@ -37,7 +43,12 @@ func NewMetaPublisher(client *redis.Client) *MetaPublisher {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// Start begins subscribing to keyspace notifications
|
// Start begins subscribing to keyspace notifications.
|
||||||
|
//
|
||||||
|
// The stream is NOT truncated on startup — the SSE wire contract requires
|
||||||
|
// Last-Event-ID resumption to survive a meta-core restart, which means
|
||||||
|
// entries must persist across restart. Growth is bounded instead by
|
||||||
|
// MAXLEN ~ MetaEventsMaxLen on every XAdd (see publishEvent).
|
||||||
func (p *MetaPublisher) Start() error {
|
func (p *MetaPublisher) Start() error {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
if p.running {
|
if p.running {
|
||||||
@@ -47,16 +58,6 @@ func (p *MetaPublisher) Start() error {
|
|||||||
p.running = true
|
p.running = true
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
// Clear the stream on startup - consumers will re-bootstrap from Redis keys
|
|
||||||
// This ensures the stream only contains events from the current session
|
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
|
||||||
defer cancel()
|
|
||||||
if err := p.client.Del(ctx, MetaEventsStream).Err(); err != nil {
|
|
||||||
log.Printf("[MetaPublisher] Warning: failed to clear stream on startup: %v", err)
|
|
||||||
} else {
|
|
||||||
log.Printf("[MetaPublisher] Cleared %s stream for fresh start", MetaEventsStream)
|
|
||||||
}
|
|
||||||
|
|
||||||
p.wg.Add(1)
|
p.wg.Add(1)
|
||||||
go p.subscribeLoop()
|
go p.subscribeLoop()
|
||||||
|
|
||||||
@@ -133,9 +134,13 @@ func (p *MetaPublisher) publishEvent(channel, operation string) {
|
|||||||
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
|
"ts": fmt.Sprintf("%d", time.Now().UnixMilli()),
|
||||||
}
|
}
|
||||||
|
|
||||||
// Add to stream (no MaxLen - stream is cleared on meta-core restart)
|
// Approximate MAXLEN trim keeps the stream bounded across restarts
|
||||||
|
// (see Start). Cap is intentionally generous; most consumers reconnect
|
||||||
|
// well within retention.
|
||||||
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
|
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
|
||||||
Stream: MetaEventsStream,
|
Stream: MetaEventsStream,
|
||||||
|
MaxLen: MetaEventsMaxLen,
|
||||||
|
Approx: true,
|
||||||
Values: fields,
|
Values: fields,
|
||||||
}).Result()
|
}).Result()
|
||||||
|
|
||||||
@@ -154,6 +159,8 @@ func (p *MetaPublisher) publishEventDirect(operation, key string) {
|
|||||||
|
|
||||||
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
|
_, err := p.client.XAdd(p.ctx, &redis.XAddArgs{
|
||||||
Stream: MetaEventsStream,
|
Stream: MetaEventsStream,
|
||||||
|
MaxLen: MetaEventsMaxLen,
|
||||||
|
Approx: true,
|
||||||
Values: fields,
|
Values: fields,
|
||||||
}).Result()
|
}).Result()
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,341 @@
|
|||||||
|
// Package identity manages the user-controlled signing identity used by the
|
||||||
|
// User Data Layer (see docs/project-architecture/user-data-layer.md).
|
||||||
|
//
|
||||||
|
// The keypair is secp256k1 (D1: wallet forward-compat). The uid is the
|
||||||
|
// multibase base58btc encoding of the 33-byte compressed pubkey, prefixed
|
||||||
|
// with "z" per multibase convention.
|
||||||
|
//
|
||||||
|
// Storage is plaintext JSON at {META_CORE_PATH}/identity/identity.json,
|
||||||
|
// matching the trust model of the rest of /meta-core. Signing happens
|
||||||
|
// in-process; meta-watch calls POST /api/identity/sign rather than holding
|
||||||
|
// the private key itself.
|
||||||
|
package identity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/rand"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"encoding/json"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"sort"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/decred/dcrd/dcrec/secp256k1/v4"
|
||||||
|
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
|
||||||
|
"github.com/mr-tron/base58"
|
||||||
|
"golang.org/x/crypto/hkdf"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
CurveSecp256k1 = "secp256k1"
|
||||||
|
|
||||||
|
// HKDF parameters for deriving the AEAD key from the private key.
|
||||||
|
// Stable across versions — changing these is a breaking change for
|
||||||
|
// encrypted records on disk.
|
||||||
|
aeadKDFSalt = "metamesh-uds-v1"
|
||||||
|
aeadKDFInfo = "aead-key"
|
||||||
|
aeadKeyLen = 32
|
||||||
|
)
|
||||||
|
|
||||||
|
// Identity is the persisted form of a user's keypair.
|
||||||
|
//
|
||||||
|
// PrivateKeyHex is the 32-byte secp256k1 scalar, lower-case hex, no prefix.
|
||||||
|
// PublicKeyHex is the 33-byte compressed pubkey, lower-case hex, no prefix.
|
||||||
|
type Identity struct {
|
||||||
|
UID string `json:"uid"`
|
||||||
|
Curve string `json:"curve"`
|
||||||
|
PublicKeyHex string `json:"publicKeyHex"`
|
||||||
|
PrivateKeyHex string `json:"privateKeyHex"`
|
||||||
|
CreatedAt int64 `json:"createdAt"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Generate creates a fresh secp256k1 identity.
|
||||||
|
func Generate() (*Identity, error) {
|
||||||
|
priv, err := secp256k1.GeneratePrivateKey()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("generate keypair: %w", err)
|
||||||
|
}
|
||||||
|
return identityFromPrivKey(priv), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import parses an externally-provided 32-byte secp256k1 scalar and derives
|
||||||
|
// the matching pubkey. Accepts hex with or without a "0x" prefix.
|
||||||
|
func Import(privateKeyHex string) (*Identity, error) {
|
||||||
|
if len(privateKeyHex) >= 2 && (privateKeyHex[:2] == "0x" || privateKeyHex[:2] == "0X") {
|
||||||
|
privateKeyHex = privateKeyHex[2:]
|
||||||
|
}
|
||||||
|
raw, err := hex.DecodeString(privateKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("invalid hex: %w", err)
|
||||||
|
}
|
||||||
|
if len(raw) != 32 {
|
||||||
|
return nil, fmt.Errorf("private key must be 32 bytes, got %d", len(raw))
|
||||||
|
}
|
||||||
|
priv := secp256k1.PrivKeyFromBytes(raw)
|
||||||
|
// Reject the zero scalar (secp256k1 lib does not reject it on its own).
|
||||||
|
pubBytes := priv.PubKey().SerializeCompressed()
|
||||||
|
if isAllZero(raw) || isAllZero(pubBytes) {
|
||||||
|
return nil, errors.New("private key is the curve identity (zero); refusing")
|
||||||
|
}
|
||||||
|
return identityFromPrivKey(priv), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func identityFromPrivKey(priv *secp256k1.PrivateKey) *Identity {
|
||||||
|
pubBytes := priv.PubKey().SerializeCompressed()
|
||||||
|
return &Identity{
|
||||||
|
UID: uidFromCompressedPubkey(pubBytes),
|
||||||
|
Curve: CurveSecp256k1,
|
||||||
|
PublicKeyHex: hex.EncodeToString(pubBytes),
|
||||||
|
PrivateKeyHex: hex.EncodeToString(priv.Serialize()),
|
||||||
|
CreatedAt: time.Now().UTC().Unix(),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// uidFromCompressedPubkey encodes the pubkey as a multibase base58btc
|
||||||
|
// string ("z" prefix). The encoding is deterministic and matches MetaMesh
|
||||||
|
// CID conventions.
|
||||||
|
func uidFromCompressedPubkey(pub []byte) string {
|
||||||
|
return "z" + base58.Encode(pub)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Load reads the identity file. Returns (nil, nil) if it does not exist —
|
||||||
|
// callers distinguish "no identity" from real errors.
|
||||||
|
func Load(path string) (*Identity, error) {
|
||||||
|
data, err := os.ReadFile(path)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("read identity file: %w", err)
|
||||||
|
}
|
||||||
|
var id Identity
|
||||||
|
if err := json.Unmarshal(data, &id); err != nil {
|
||||||
|
return nil, fmt.Errorf("parse identity file: %w", err)
|
||||||
|
}
|
||||||
|
if id.Curve != CurveSecp256k1 {
|
||||||
|
return nil, fmt.Errorf("unsupported curve on disk: %q", id.Curve)
|
||||||
|
}
|
||||||
|
return &id, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Save writes the identity atomically (tmp + rename), matching the pattern
|
||||||
|
// used by discovery/service.go.
|
||||||
|
func Save(path string, id *Identity) error {
|
||||||
|
if id == nil {
|
||||||
|
return errors.New("nil identity")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(filepath.Dir(path), 0700); err != nil {
|
||||||
|
return fmt.Errorf("mkdir identity dir: %w", err)
|
||||||
|
}
|
||||||
|
data, err := json.MarshalIndent(id, "", " ")
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("marshal identity: %w", err)
|
||||||
|
}
|
||||||
|
tmp := path + ".tmp"
|
||||||
|
if err := os.WriteFile(tmp, data, 0600); err != nil {
|
||||||
|
return fmt.Errorf("write tmp identity: %w", err)
|
||||||
|
}
|
||||||
|
if err := os.Rename(tmp, path); err != nil {
|
||||||
|
return fmt.Errorf("rename tmp identity: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete removes the identity file. Idempotent.
|
||||||
|
func Delete(path string) error {
|
||||||
|
if err := os.Remove(path); err != nil && !errors.Is(err, os.ErrNotExist) {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exists reports whether an identity file is present.
|
||||||
|
func Exists(path string) bool {
|
||||||
|
_, err := os.Stat(path)
|
||||||
|
return err == nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// --- Multi-account keystore -------------------------------------------------
|
||||||
|
//
|
||||||
|
// Accounts live as one file per keypair at {dir}/{uid}.json. The uid is a
|
||||||
|
// multibase base58btc string, so it is filename-safe; validUID still refuses
|
||||||
|
// anything that could escape the accounts directory.
|
||||||
|
|
||||||
|
// validUID accepts the alphanumeric multibase/base58btc shape we mint for uids
|
||||||
|
// and rejects path separators or traversal sequences.
|
||||||
|
func validUID(uid string) bool {
|
||||||
|
if uid == "" || len(uid) > 128 || uid != filepath.Base(uid) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for _, r := range uid {
|
||||||
|
switch {
|
||||||
|
case r >= 'a' && r <= 'z':
|
||||||
|
case r >= 'A' && r <= 'Z':
|
||||||
|
case r >= '0' && r <= '9':
|
||||||
|
default:
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func accountFilePath(dir, uid string) (string, error) {
|
||||||
|
if !validUID(uid) {
|
||||||
|
return "", fmt.Errorf("invalid uid: %q", uid)
|
||||||
|
}
|
||||||
|
return filepath.Join(dir, uid+".json"), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SaveAccount writes an identity into the multi-account store. It does not
|
||||||
|
// check for prior existence — callers wanting append-only semantics check
|
||||||
|
// ExistsByUID first.
|
||||||
|
func SaveAccount(dir string, id *Identity) error {
|
||||||
|
if id == nil {
|
||||||
|
return errors.New("nil identity")
|
||||||
|
}
|
||||||
|
path, err := accountFilePath(dir, id.UID)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return Save(path, id)
|
||||||
|
}
|
||||||
|
|
||||||
|
// LoadByUID reads one account. Returns (nil, nil) when absent.
|
||||||
|
func LoadByUID(dir, uid string) (*Identity, error) {
|
||||||
|
path, err := accountFilePath(dir, uid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
return Load(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// ExistsByUID reports whether an account file exists for uid.
|
||||||
|
func ExistsByUID(dir, uid string) bool {
|
||||||
|
path, err := accountFilePath(dir, uid)
|
||||||
|
if err != nil {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return Exists(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteByUID removes one account. Idempotent.
|
||||||
|
func DeleteByUID(dir, uid string) error {
|
||||||
|
path, err := accountFilePath(dir, uid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
return Delete(path)
|
||||||
|
}
|
||||||
|
|
||||||
|
// List returns every account, sorted by CreatedAt ascending (then UID, for a
|
||||||
|
// stable order among same-second creations). Returns nil when the directory
|
||||||
|
// does not exist yet.
|
||||||
|
func List(dir string) ([]*Identity, error) {
|
||||||
|
entries, err := os.ReadDir(dir)
|
||||||
|
if err != nil {
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return nil, fmt.Errorf("read accounts dir: %w", err)
|
||||||
|
}
|
||||||
|
var out []*Identity
|
||||||
|
for _, e := range entries {
|
||||||
|
if e.IsDir() || !strings.HasSuffix(e.Name(), ".json") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
id, err := Load(filepath.Join(dir, e.Name()))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
if id != nil {
|
||||||
|
out = append(out, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Slice(out, func(i, j int) bool {
|
||||||
|
if out[i].CreatedAt != out[j].CreatedAt {
|
||||||
|
return out[i].CreatedAt < out[j].CreatedAt
|
||||||
|
}
|
||||||
|
return out[i].UID < out[j].UID
|
||||||
|
})
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MigrateLegacy folds a pre-multi-account identity.json into the accounts
|
||||||
|
// store as an entry, then removes the legacy file. Idempotent and a no-op when
|
||||||
|
// the legacy file is absent. Returns true when a migration ran.
|
||||||
|
func MigrateLegacy(legacyPath, accountsDir string) (bool, error) {
|
||||||
|
id, err := Load(legacyPath)
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
if id == nil {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
// Don't clobber an account already present for this uid.
|
||||||
|
if !ExistsByUID(accountsDir, id.UID) {
|
||||||
|
if err := SaveAccount(accountsDir, id); err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if err := Delete(legacyPath); err != nil {
|
||||||
|
return false, fmt.Errorf("remove legacy identity after migration: %w", err)
|
||||||
|
}
|
||||||
|
return true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sign produces an ECDSA-secp256k1 signature over SHA-256(bytes). The output
|
||||||
|
// is DER-encoded — the wire format we commit to for User Data Layer records.
|
||||||
|
func Sign(id *Identity, bytes []byte) ([]byte, error) {
|
||||||
|
if id == nil {
|
||||||
|
return nil, errors.New("no identity loaded")
|
||||||
|
}
|
||||||
|
raw, err := hex.DecodeString(id.PrivateKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode private key: %w", err)
|
||||||
|
}
|
||||||
|
if len(raw) != 32 {
|
||||||
|
return nil, fmt.Errorf("private key length %d", len(raw))
|
||||||
|
}
|
||||||
|
priv := secp256k1.PrivKeyFromBytes(raw)
|
||||||
|
digest := sha256.Sum256(bytes)
|
||||||
|
sig := ecdsa.Sign(priv, digest[:])
|
||||||
|
return sig.Serialize(), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeriveAEADKey derives the symmetric key used to encrypt private-tier
|
||||||
|
// User Data Layer records. The KDF parameters (salt, info, length) are
|
||||||
|
// fixed for the v1 wire format.
|
||||||
|
func DeriveAEADKey(id *Identity) ([]byte, error) {
|
||||||
|
if id == nil {
|
||||||
|
return nil, errors.New("no identity loaded")
|
||||||
|
}
|
||||||
|
raw, err := hex.DecodeString(id.PrivateKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("decode private key: %w", err)
|
||||||
|
}
|
||||||
|
r := hkdf.New(sha256.New, raw, []byte(aeadKDFSalt), []byte(aeadKDFInfo))
|
||||||
|
out := make([]byte, aeadKeyLen)
|
||||||
|
if _, err := io.ReadFull(r, out); err != nil {
|
||||||
|
return nil, fmt.Errorf("hkdf: %w", err)
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func isAllZero(b []byte) bool {
|
||||||
|
for _, x := range b {
|
||||||
|
if x != 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
// Ensure rand.Reader is referenced so dependency-stripped builds don't
|
||||||
|
// accidentally remove the crypto/rand import that secp256k1.GeneratePrivateKey
|
||||||
|
// relies on transitively.
|
||||||
|
var _ = rand.Reader
|
||||||
@@ -0,0 +1,308 @@
|
|||||||
|
package identity
|
||||||
|
|
||||||
|
import (
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
"github.com/decred/dcrd/dcrec/secp256k1/v4"
|
||||||
|
"github.com/decred/dcrd/dcrec/secp256k1/v4/ecdsa"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestGenerateUIDIsDeterministicFromPubkey(t *testing.T) {
|
||||||
|
id, err := Generate()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("generate: %v", err)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(id.UID, "z") {
|
||||||
|
t.Fatalf("uid missing multibase 'z' prefix: %q", id.UID)
|
||||||
|
}
|
||||||
|
if id.Curve != CurveSecp256k1 {
|
||||||
|
t.Fatalf("curve = %q", id.Curve)
|
||||||
|
}
|
||||||
|
if len(id.PrivateKeyHex) != 64 {
|
||||||
|
t.Fatalf("privkey hex len = %d", len(id.PrivateKeyHex))
|
||||||
|
}
|
||||||
|
if len(id.PublicKeyHex) != 66 {
|
||||||
|
t.Fatalf("compressed pubkey hex len = %d", len(id.PublicKeyHex))
|
||||||
|
}
|
||||||
|
|
||||||
|
// Reimporting yields the same uid.
|
||||||
|
imported, err := Import(id.PrivateKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("import: %v", err)
|
||||||
|
}
|
||||||
|
if imported.UID != id.UID {
|
||||||
|
t.Fatalf("uid changed after import roundtrip: %q vs %q", id.UID, imported.UID)
|
||||||
|
}
|
||||||
|
if imported.PublicKeyHex != id.PublicKeyHex {
|
||||||
|
t.Fatalf("pubkey changed after import roundtrip")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestImportRejectsBadInput(t *testing.T) {
|
||||||
|
if _, err := Import(""); err == nil {
|
||||||
|
t.Fatal("empty hex should error")
|
||||||
|
}
|
||||||
|
if _, err := Import("zz"); err == nil {
|
||||||
|
t.Fatal("non-hex should error")
|
||||||
|
}
|
||||||
|
if _, err := Import("00"); err == nil {
|
||||||
|
t.Fatal("wrong-length hex should error")
|
||||||
|
}
|
||||||
|
if _, err := Import(strings.Repeat("00", 32)); err == nil {
|
||||||
|
t.Fatal("zero scalar should error")
|
||||||
|
}
|
||||||
|
// Hex with 0x prefix should work.
|
||||||
|
id1, err := Generate()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
id2, err := Import("0x" + id1.PrivateKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("0x-prefixed hex: %v", err)
|
||||||
|
}
|
||||||
|
if id1.UID != id2.UID {
|
||||||
|
t.Fatal("uid mismatch through 0x prefix")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSaveLoadRoundtrip(t *testing.T) {
|
||||||
|
dir := t.TempDir()
|
||||||
|
path := filepath.Join(dir, "identity.json")
|
||||||
|
|
||||||
|
if Exists(path) {
|
||||||
|
t.Fatal("file should not exist yet")
|
||||||
|
}
|
||||||
|
if got, _ := Load(path); got != nil {
|
||||||
|
t.Fatal("Load on missing file should return nil")
|
||||||
|
}
|
||||||
|
|
||||||
|
id, err := Generate()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := Save(path, id); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !Exists(path) {
|
||||||
|
t.Fatal("file should exist after Save")
|
||||||
|
}
|
||||||
|
|
||||||
|
loaded, err := Load(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if loaded == nil || loaded.UID != id.UID || loaded.PrivateKeyHex != id.PrivateKeyHex {
|
||||||
|
t.Fatalf("roundtrip mismatch: %+v vs %+v", id, loaded)
|
||||||
|
}
|
||||||
|
|
||||||
|
// File should be 0600 (private key on disk).
|
||||||
|
info, err := os.Stat(path)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if info.Mode().Perm() != 0600 {
|
||||||
|
t.Errorf("identity file mode = %o, want 0600", info.Mode().Perm())
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := Delete(path); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if Exists(path) {
|
||||||
|
t.Fatal("file should not exist after Delete")
|
||||||
|
}
|
||||||
|
// Delete is idempotent.
|
||||||
|
if err := Delete(path); err != nil {
|
||||||
|
t.Fatalf("Delete on missing file should be nil: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestSignAndVerify(t *testing.T) {
|
||||||
|
id, err := Generate()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
msg := []byte("hello user data layer")
|
||||||
|
sig, err := Sign(id, msg)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(sig) == 0 {
|
||||||
|
t.Fatal("empty sig")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Verify externally with the public key, matching what meta-watch will do.
|
||||||
|
pubBytes, err := hex.DecodeString(id.PublicKeyHex)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
pub, err := secp256k1.ParsePubKey(pubBytes)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
parsedSig, err := ecdsa.ParseDERSignature(sig)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("parse DER sig: %v", err)
|
||||||
|
}
|
||||||
|
digest := sha256.Sum256(msg)
|
||||||
|
if !parsedSig.Verify(digest[:], pub) {
|
||||||
|
t.Fatal("signature did not verify")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flipping a byte must break verification.
|
||||||
|
tampered := append([]byte{}, msg...)
|
||||||
|
tampered[0] ^= 0x01
|
||||||
|
digest2 := sha256.Sum256(tampered)
|
||||||
|
if parsedSig.Verify(digest2[:], pub) {
|
||||||
|
t.Fatal("signature verified on tampered message")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDeriveAEADKeyIsDeterministic(t *testing.T) {
|
||||||
|
id, err := Generate()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k1, err := DeriveAEADKey(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
k2, err := DeriveAEADKey(id)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if !bytesEqual(k1, k2) {
|
||||||
|
t.Fatal("AEAD key derivation is non-deterministic")
|
||||||
|
}
|
||||||
|
if len(k1) != aeadKeyLen {
|
||||||
|
t.Fatalf("AEAD key len = %d, want %d", len(k1), aeadKeyLen)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Different identity → different key.
|
||||||
|
other, _ := Generate()
|
||||||
|
k3, _ := DeriveAEADKey(other)
|
||||||
|
if bytesEqual(k1, k3) {
|
||||||
|
t.Fatal("two distinct identities derived the same AEAD key")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccountStoreRoundtrip(t *testing.T) {
|
||||||
|
dir := filepath.Join(t.TempDir(), "accounts")
|
||||||
|
|
||||||
|
// Empty store.
|
||||||
|
list, err := List(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("List on missing dir: %v", err)
|
||||||
|
}
|
||||||
|
if len(list) != 0 {
|
||||||
|
t.Fatalf("expected empty store, got %d", len(list))
|
||||||
|
}
|
||||||
|
|
||||||
|
a, _ := Generate()
|
||||||
|
b, _ := Generate()
|
||||||
|
if err := SaveAccount(dir, a); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if err := SaveAccount(dir, b); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if !ExistsByUID(dir, a.UID) || !ExistsByUID(dir, b.UID) {
|
||||||
|
t.Fatal("both accounts should exist")
|
||||||
|
}
|
||||||
|
|
||||||
|
list, err = List(dir)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if len(list) != 2 {
|
||||||
|
t.Fatalf("expected 2 accounts, got %d", len(list))
|
||||||
|
}
|
||||||
|
|
||||||
|
got, err := LoadByUID(dir, a.UID)
|
||||||
|
if err != nil || got == nil || got.PrivateKeyHex != a.PrivateKeyHex {
|
||||||
|
t.Fatalf("LoadByUID roundtrip failed: %+v err=%v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Missing uid → (nil, nil).
|
||||||
|
if got, err := LoadByUID(dir, "zdoesnotexist"); err != nil || got != nil {
|
||||||
|
t.Fatalf("LoadByUID on missing should be (nil,nil): %+v %v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Delete one; the other remains. Delete is idempotent.
|
||||||
|
if err := DeleteByUID(dir, a.UID); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
if ExistsByUID(dir, a.UID) {
|
||||||
|
t.Fatal("account should be gone after delete")
|
||||||
|
}
|
||||||
|
if err := DeleteByUID(dir, a.UID); err != nil {
|
||||||
|
t.Fatalf("delete should be idempotent: %v", err)
|
||||||
|
}
|
||||||
|
list, _ = List(dir)
|
||||||
|
if len(list) != 1 || list[0].UID != b.UID {
|
||||||
|
t.Fatalf("expected only b remaining, got %+v", list)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestAccountFilePathRejectsTraversal(t *testing.T) {
|
||||||
|
for _, bad := range []string{"", "../escape", "a/b", "z..", strings.Repeat("z", 200)} {
|
||||||
|
if _, err := accountFilePath("/tmp/x", bad); err == nil {
|
||||||
|
t.Fatalf("expected rejection of uid %q", bad)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// A real uid is accepted.
|
||||||
|
id, _ := Generate()
|
||||||
|
if _, err := accountFilePath("/tmp/x", id.UID); err != nil {
|
||||||
|
t.Fatalf("real uid rejected: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMigrateLegacy(t *testing.T) {
|
||||||
|
base := t.TempDir()
|
||||||
|
legacy := filepath.Join(base, "identity.json")
|
||||||
|
accounts := filepath.Join(base, "accounts")
|
||||||
|
|
||||||
|
// No legacy file → no-op.
|
||||||
|
if migrated, err := MigrateLegacy(legacy, accounts); err != nil || migrated {
|
||||||
|
t.Fatalf("expected no-op migration, got migrated=%v err=%v", migrated, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
id, _ := Generate()
|
||||||
|
if err := Save(legacy, id); err != nil {
|
||||||
|
t.Fatal(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
migrated, err := MigrateLegacy(legacy, accounts)
|
||||||
|
if err != nil || !migrated {
|
||||||
|
t.Fatalf("expected migration to run: migrated=%v err=%v", migrated, err)
|
||||||
|
}
|
||||||
|
if Exists(legacy) {
|
||||||
|
t.Fatal("legacy file should be removed after migration")
|
||||||
|
}
|
||||||
|
got, err := LoadByUID(accounts, id.UID)
|
||||||
|
if err != nil || got == nil || got.UID != id.UID {
|
||||||
|
t.Fatalf("migrated account not found: %+v err=%v", got, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Second run is a clean no-op (legacy already gone).
|
||||||
|
if migrated, err := MigrateLegacy(legacy, accounts); err != nil || migrated {
|
||||||
|
t.Fatalf("second migration should no-op: migrated=%v err=%v", migrated, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func bytesEqual(a, b []byte) bool {
|
||||||
|
if len(a) != len(b) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
for i := range a {
|
||||||
|
if a[i] != b[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
@@ -75,7 +75,11 @@ func (p *LeaderInfoProvider) buildLeaderInfo() *LeaderLockInfo {
|
|||||||
Hostname: hostname,
|
Hostname: hostname,
|
||||||
BaseUrl: baseUrl,
|
BaseUrl: baseUrl,
|
||||||
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
|
ApiUrl: fmt.Sprintf("http://%s:%d", ip, p.config.HTTPPort),
|
||||||
RedisUrl: fmt.Sprintf("redis://%s:%d", ip, p.config.RedisPort),
|
// RedisUrl deliberately empty — api-mediated-access PR D removes
|
||||||
|
// direct Redis exposure. Consumers route metadata I/O through
|
||||||
|
// meta-core's HTTP API (/meta/{hash}*) and event streams through
|
||||||
|
// SSE (/api/events/{files,meta}).
|
||||||
|
RedisUrl: "",
|
||||||
WebdavUrl: webdavUrl,
|
WebdavUrl: webdavUrl,
|
||||||
WebdavUrlInternal: webdavUrlInternal,
|
WebdavUrlInternal: webdavUrlInternal,
|
||||||
Timestamp: time.Now().UnixMilli(),
|
Timestamp: time.Now().UnixMilli(),
|
||||||
|
|||||||
@@ -292,13 +292,35 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
|||||||
json.NewEncoder(w).Encode(data)
|
json.NewEncoder(w).Encode(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeError writes the typed error envelope (see api/handlers.go for the
|
||||||
|
// canonical definition + slug vocabulary). Kept local to avoid an import
|
||||||
|
// cycle with the api package.
|
||||||
func writeError(w http.ResponseWriter, status int, message string) {
|
func writeError(w http.ResponseWriter, status int, message string) {
|
||||||
writeJSON(w, status, map[string]string{
|
slug, retryable := slugForStatus(status)
|
||||||
"error": http.StatusText(status),
|
writeJSON(w, status, map[string]interface{}{
|
||||||
|
"error": slug,
|
||||||
"message": message,
|
"message": message,
|
||||||
|
"retryable": retryable,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func slugForStatus(status int) (string, bool) {
|
||||||
|
switch status {
|
||||||
|
case http.StatusConflict:
|
||||||
|
return "alias_collision", false
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return "unknown_root", false
|
||||||
|
case http.StatusBadRequest:
|
||||||
|
return "schema_violation", false
|
||||||
|
case http.StatusServiceUnavailable:
|
||||||
|
return "storage_unavailable", true
|
||||||
|
case http.StatusInternalServerError:
|
||||||
|
return "internal", true
|
||||||
|
default:
|
||||||
|
return "internal", status >= 500
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
func parseIntOrDefault(s string, defaultVal int) (int, error) {
|
func parseIntOrDefault(s string, defaultVal int) (int, error) {
|
||||||
var result int
|
var result int
|
||||||
if s == "" {
|
if s == "" {
|
||||||
|
|||||||
+93
-12
@@ -24,6 +24,7 @@ type Manager struct {
|
|||||||
filesPath string
|
filesPath string
|
||||||
mountsDir string
|
mountsDir string
|
||||||
poller *Poller
|
poller *Poller
|
||||||
|
statsPoller *StatsPoller
|
||||||
}
|
}
|
||||||
|
|
||||||
// NewManager creates a new mount manager
|
// NewManager creates a new mount manager
|
||||||
@@ -39,14 +40,55 @@ func NewManager(cfg *config.Config) (*Manager, error) {
|
|||||||
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
return nil, fmt.Errorf("failed to create mount directories: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// One-shot migration for legacy NFS entries — disable rather than fail to
|
||||||
|
// start, so operators can fix their config at their own pace.
|
||||||
|
if err := m.migrateLegacyMounts(); err != nil {
|
||||||
|
log.Printf("[Mounts] legacy migration warning: %v", err)
|
||||||
|
}
|
||||||
|
|
||||||
return m, nil
|
return m, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// migrateLegacyMounts disables any mount entries with type="nfs". The native
|
||||||
|
// NFS handler was removed when the project consolidated on rclone; an NFS
|
||||||
|
// entry left enabled would loop the watcher with mount errors.
|
||||||
|
func (m *Manager) migrateLegacyMounts() error {
|
||||||
|
mountsFile, err := m.readConfig()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
dirty := false
|
||||||
|
for i := range mountsFile.Mounts {
|
||||||
|
mnt := &mountsFile.Mounts[i]
|
||||||
|
if string(mnt.Type) == "nfs" {
|
||||||
|
if mnt.Enabled || mnt.DesiredMounted {
|
||||||
|
log.Printf("[Mounts] WARNING: disabling legacy NFS mount %q (%s) — NFS support was removed; please re-add via SMB or a pre-configured rclone remote", mnt.Name, mnt.ID)
|
||||||
|
mnt.Enabled = false
|
||||||
|
mnt.DesiredMounted = false
|
||||||
|
dirty = true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if dirty {
|
||||||
|
return m.writeConfig(mountsFile)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
// SetPoller sets the poller for the manager
|
// SetPoller sets the poller for the manager
|
||||||
func (m *Manager) SetPoller(poller *Poller) {
|
func (m *Manager) SetPoller(poller *Poller) {
|
||||||
m.poller = poller
|
m.poller = poller
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// SetStatsPoller sets the IO stats poller for the manager. ListMounts and
|
||||||
|
// GetMount will merge the latest sampled stats into MountStatus.IOStats when
|
||||||
|
// set; remains nil-safe when stats are disabled.
|
||||||
|
func (m *Manager) SetStatsPoller(sp *StatsPoller) {
|
||||||
|
m.statsPoller = sp
|
||||||
|
}
|
||||||
|
|
||||||
// ensureDirs creates required directories
|
// ensureDirs creates required directories
|
||||||
func (m *Manager) ensureDirs() error {
|
func (m *Manager) ensureDirs() error {
|
||||||
dirs := []string{
|
dirs := []string{
|
||||||
@@ -197,6 +239,15 @@ func (m *Manager) ListMounts() ([]MountStatus, error) {
|
|||||||
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
pollingStatus := m.poller.GetPollingStatus(statuses[i].ID)
|
||||||
statuses[i].PollingActive = pollingStatus.Active
|
statuses[i].PollingActive = pollingStatus.Active
|
||||||
statuses[i].LastPolledScan = pollingStatus.LastScan
|
statuses[i].LastPolledScan = pollingStatus.LastScan
|
||||||
|
statuses[i].Scanning = pollingStatus.Scanning
|
||||||
|
statuses[i].CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||||
|
statuses[i].LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||||
|
statuses[i].NextScanAt = pollingStatus.NextScanAt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if m.statsPoller != nil {
|
||||||
|
for i := range statuses {
|
||||||
|
statuses[i].IOStats = m.statsPoller.GetStats(statuses[i].ID)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -251,6 +302,13 @@ func (m *Manager) GetMount(id string) (*MountStatus, error) {
|
|||||||
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
pollingStatus := m.poller.GetPollingStatus(mount.ID)
|
||||||
status.PollingActive = pollingStatus.Active
|
status.PollingActive = pollingStatus.Active
|
||||||
status.LastPolledScan = pollingStatus.LastScan
|
status.LastPolledScan = pollingStatus.LastScan
|
||||||
|
status.Scanning = pollingStatus.Scanning
|
||||||
|
status.CurrentScanStartedAt = pollingStatus.CurrentScanStartedAt
|
||||||
|
status.LastScanDurationMs = pollingStatus.LastScanDurationMs
|
||||||
|
status.NextScanAt = pollingStatus.NextScanAt
|
||||||
|
}
|
||||||
|
if m.statsPoller != nil {
|
||||||
|
status.IOStats = m.statsPoller.GetStats(mount.ID)
|
||||||
}
|
}
|
||||||
|
|
||||||
return status, nil
|
return status, nil
|
||||||
@@ -266,16 +324,19 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
return nil, fmt.Errorf("mount name is required")
|
return nil, fmt.Errorf("mount name is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
if req.Type != MountTypeNFS && req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
// "nfs" used to be valid; the native handler is gone. Reject loudly so the
|
||||||
return nil, fmt.Errorf("valid mount type (nfs, smb, rclone) is required")
|
// caller knows their integration needs to migrate to a Samba re-export or
|
||||||
|
// switch to a supported protocol.
|
||||||
|
if req.Type == MountType("nfs") {
|
||||||
|
return nil, fmt.Errorf("NFS mounts are no longer supported — please use SMB or a pre-configured rclone remote")
|
||||||
|
}
|
||||||
|
|
||||||
|
if req.Type != MountTypeSMB && req.Type != MountTypeRclone {
|
||||||
|
return nil, fmt.Errorf("valid mount type (smb, rclone) is required")
|
||||||
}
|
}
|
||||||
|
|
||||||
// Validate type-specific fields
|
// Validate type-specific fields
|
||||||
switch req.Type {
|
switch req.Type {
|
||||||
case MountTypeNFS:
|
|
||||||
if req.NFSServer == "" || req.NFSPath == "" {
|
|
||||||
return nil, fmt.Errorf("NFS server and path are required")
|
|
||||||
}
|
|
||||||
case MountTypeSMB:
|
case MountTypeSMB:
|
||||||
if req.SMBServer == "" || req.SMBShare == "" {
|
if req.SMBServer == "" || req.SMBShare == "" {
|
||||||
return nil, fmt.Errorf("SMB server and share are required")
|
return nil, fmt.Errorf("SMB server and share are required")
|
||||||
@@ -309,8 +370,11 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
enabled = *req.Enabled
|
enabled = *req.Enabled
|
||||||
}
|
}
|
||||||
|
|
||||||
// Default polling settings
|
// Polling defaults on for both supported types — neither SMB-via-rclone nor
|
||||||
pollingEnabled := false
|
// generic rclone delivers kernel inotify, so a periodic re-scan is the only
|
||||||
|
// way to pick up upstream changes. Explicit PollingEnabled in the request
|
||||||
|
// wins.
|
||||||
|
pollingEnabled := true
|
||||||
if req.PollingEnabled != nil {
|
if req.PollingEnabled != nil {
|
||||||
pollingEnabled = *req.PollingEnabled
|
pollingEnabled = *req.PollingEnabled
|
||||||
}
|
}
|
||||||
@@ -326,16 +390,15 @@ func (m *Manager) CreateMount(req *CreateMountRequest) (*MountStatus, error) {
|
|||||||
Enabled: enabled,
|
Enabled: enabled,
|
||||||
DesiredMounted: enabled, // Auto-mount if enabled
|
DesiredMounted: enabled, // Auto-mount if enabled
|
||||||
MountPath: mountPath,
|
MountPath: mountPath,
|
||||||
Options: req.Options,
|
CacheMaxSize: req.CacheMaxSize,
|
||||||
|
CacheMaxAge: req.CacheMaxAge,
|
||||||
|
DirCacheTime: req.DirCacheTime,
|
||||||
PollingEnabled: pollingEnabled,
|
PollingEnabled: pollingEnabled,
|
||||||
PollingIntervalMs: pollingIntervalMs,
|
PollingIntervalMs: pollingIntervalMs,
|
||||||
}
|
}
|
||||||
|
|
||||||
// Type-specific fields
|
// Type-specific fields
|
||||||
switch req.Type {
|
switch req.Type {
|
||||||
case MountTypeNFS:
|
|
||||||
mount.NFSServer = req.NFSServer
|
|
||||||
mount.NFSPath = req.NFSPath
|
|
||||||
case MountTypeSMB:
|
case MountTypeSMB:
|
||||||
mount.SMBServer = req.SMBServer
|
mount.SMBServer = req.SMBServer
|
||||||
mount.SMBShare = req.SMBShare
|
mount.SMBShare = req.SMBShare
|
||||||
@@ -414,6 +477,24 @@ func (m *Manager) UpdateMount(id string, updates map[string]interface{}) (*Mount
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// VFS cache knobs — strings round-tripped to rclone unchanged, so we
|
||||||
|
// don't validate format here; rclone rejects bad values at mount time.
|
||||||
|
if val, ok := updates["cacheMaxSize"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.CacheMaxSize = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if val, ok := updates["cacheMaxAge"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.CacheMaxAge = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if val, ok := updates["dirCacheTime"]; ok {
|
||||||
|
if s, ok := val.(string); ok {
|
||||||
|
mount.DirCacheTime = s
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
if err := m.writeConfig(mountsFile); err != nil {
|
if err := m.writeConfig(mountsFile); err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
|||||||
+166
-29
@@ -9,6 +9,25 @@ import (
|
|||||||
// ScanFunc is a function type for scanning a mount path
|
// ScanFunc is a function type for scanning a mount path
|
||||||
type ScanFunc func(path string) (int, error)
|
type ScanFunc func(path string) (int, error)
|
||||||
|
|
||||||
|
// IdleFactor controls the adaptive scheduler's duty cycle. The next scan is
|
||||||
|
// delayed by max(pollingIntervalMs, lastScanDurationMs * IdleFactor), which
|
||||||
|
// pins steady-state work to roughly 1/(IdleFactor+1) of wall-clock time. With
|
||||||
|
// IdleFactor=9 a 60s scan triggers a 9-min rest (≈10% duty cycle), keeping
|
||||||
|
// large remote shares (CORN, etc.) from monopolising the network/CPU.
|
||||||
|
const IdleFactor = 9
|
||||||
|
|
||||||
|
// InitialScanDelay is how long a freshly-started poller waits before its first
|
||||||
|
// scan. Avoids hammering the system at container startup when many mounts come
|
||||||
|
// up at once.
|
||||||
|
const InitialScanDelay = 2 * time.Second
|
||||||
|
|
||||||
|
// ReconcileInterval drives the periodic resync loop that keeps poller
|
||||||
|
// goroutines aligned with mount state. NotifyMountChanged covers config edits,
|
||||||
|
// but the kernel mount itself is brought up asynchronously by mount-watcher.sh
|
||||||
|
// — without periodic reconciliation, the poller would never observe the
|
||||||
|
// transition from `mounted=false` to `mounted=true` and never start scanning.
|
||||||
|
const ReconcileInterval = 10 * time.Second
|
||||||
|
|
||||||
// mountPoller manages polling for a single mount
|
// mountPoller manages polling for a single mount
|
||||||
type mountPoller struct {
|
type mountPoller struct {
|
||||||
mountID string
|
mountID string
|
||||||
@@ -20,7 +39,11 @@ type mountPoller struct {
|
|||||||
// PollingStatus holds the polling state for a mount
|
// PollingStatus holds the polling state for a mount
|
||||||
type PollingStatus struct {
|
type PollingStatus struct {
|
||||||
Active bool `json:"active"`
|
Active bool `json:"active"`
|
||||||
LastScan int64 `json:"lastScan,omitempty"`
|
LastScan int64 `json:"lastScan,omitempty"` // ms; completion timestamp
|
||||||
|
Scanning bool `json:"scanning,omitempty"` // true while a scan is in flight
|
||||||
|
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
|
||||||
|
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the previous scan
|
||||||
|
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of next adaptive scan
|
||||||
}
|
}
|
||||||
|
|
||||||
// Poller manages per-mount polling goroutines
|
// Poller manages per-mount polling goroutines
|
||||||
@@ -32,6 +55,7 @@ type Poller struct {
|
|||||||
status map[string]*PollingStatus
|
status map[string]*PollingStatus
|
||||||
|
|
||||||
running bool
|
running bool
|
||||||
|
reconcileStop chan struct{}
|
||||||
mu sync.RWMutex
|
mu sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -54,15 +78,35 @@ func (p *Poller) Start() error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
p.running = true
|
p.running = true
|
||||||
|
p.reconcileStop = make(chan struct{})
|
||||||
|
|
||||||
log.Println("[Poller] Starting mount polling service")
|
log.Println("[Poller] Starting mount polling service")
|
||||||
|
|
||||||
// Initial sync with mount configurations
|
// Initial sync with mount configurations
|
||||||
go p.syncWithMounts()
|
go p.syncWithMounts()
|
||||||
|
|
||||||
|
// Periodic reconciliation — picks up mount-watcher.sh transitions
|
||||||
|
// (mounted/unmounted) that NotifyMountChanged doesn't observe.
|
||||||
|
go p.reconcileLoop(p.reconcileStop)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// reconcileLoop runs syncWithMounts on a fixed cadence so the poller catches
|
||||||
|
// mount state changes initiated outside of CreateMount/UpdateMount/etc.
|
||||||
|
func (p *Poller) reconcileLoop(stop <-chan struct{}) {
|
||||||
|
ticker := time.NewTicker(ReconcileInterval)
|
||||||
|
defer ticker.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-ticker.C:
|
||||||
|
p.syncWithMounts()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Stop stops all polling goroutines
|
// Stop stops all polling goroutines
|
||||||
func (p *Poller) Stop() error {
|
func (p *Poller) Stop() error {
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
@@ -75,6 +119,11 @@ func (p *Poller) Stop() error {
|
|||||||
|
|
||||||
log.Println("[Poller] Stopping mount polling service")
|
log.Println("[Poller] Stopping mount polling service")
|
||||||
|
|
||||||
|
if p.reconcileStop != nil {
|
||||||
|
close(p.reconcileStop)
|
||||||
|
p.reconcileStop = nil
|
||||||
|
}
|
||||||
|
|
||||||
// Stop all pollers
|
// Stop all pollers
|
||||||
for id, poller := range p.pollers {
|
for id, poller := range p.pollers {
|
||||||
p.stopPoller(poller)
|
p.stopPoller(poller)
|
||||||
@@ -127,7 +176,7 @@ func (p *Poller) syncWithMounts() {
|
|||||||
existingPoller, exists := p.pollers[mount.ID]
|
existingPoller, exists := p.pollers[mount.ID]
|
||||||
|
|
||||||
if shouldPoll {
|
if shouldPoll {
|
||||||
// Calculate interval
|
// Calculate interval (used as the floor for adaptive scheduling)
|
||||||
interval := mount.PollingIntervalMs
|
interval := mount.PollingIntervalMs
|
||||||
if interval < MinPollingIntervalMs {
|
if interval < MinPollingIntervalMs {
|
||||||
interval = DefaultPollingIntervalMs
|
interval = DefaultPollingIntervalMs
|
||||||
@@ -135,7 +184,7 @@ func (p *Poller) syncWithMounts() {
|
|||||||
|
|
||||||
if !exists {
|
if !exists {
|
||||||
// Start new poller
|
// Start new poller
|
||||||
log.Printf("[Poller] Starting poller for mount %s (interval: %dms)", mount.Name, interval)
|
log.Printf("[Poller] Starting poller for mount %s (interval floor: %dms, idle factor: %dx)", mount.Name, interval, IdleFactor)
|
||||||
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
p.startPollerForMount(mount.ID, mount.MountPath, interval)
|
||||||
}
|
}
|
||||||
} else if exists {
|
} else if exists {
|
||||||
@@ -156,7 +205,13 @@ func (p *Poller) syncWithMounts() {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// startPollerForMount starts a polling goroutine for a mount
|
// startPollerForMount starts the adaptive polling goroutine for a mount.
|
||||||
|
//
|
||||||
|
// The loop runs scans sequentially: scan → measure duration → sleep
|
||||||
|
// max(intervalMs, duration*IdleFactor) → repeat. This makes scans
|
||||||
|
// self-debouncing (no overlap) and bounds steady-state work to ~10% duty
|
||||||
|
// cycle on slow shares, while keeping fresh checks on small mounts via the
|
||||||
|
// configured interval floor.
|
||||||
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int) {
|
||||||
poller := &mountPoller{
|
poller := &mountPoller{
|
||||||
mountID: mountID,
|
mountID: mountID,
|
||||||
@@ -170,17 +225,40 @@ func (p *Poller) startPollerForMount(mountID, mountPath string, intervalMs int)
|
|||||||
}
|
}
|
||||||
p.status[mountID].Active = true
|
p.status[mountID].Active = true
|
||||||
|
|
||||||
|
floor := time.Duration(intervalMs) * time.Millisecond
|
||||||
|
|
||||||
go func() {
|
go func() {
|
||||||
ticker := time.NewTicker(time.Duration(intervalMs) * time.Millisecond)
|
// Initial delay so a freshly-mounted share doesn't get hammered the
|
||||||
defer ticker.Stop()
|
// instant the poller starts.
|
||||||
|
nextDelay := InitialScanDelay
|
||||||
|
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-poller.stopChan:
|
case <-poller.stopChan:
|
||||||
return
|
return
|
||||||
case <-ticker.C:
|
case <-time.After(nextDelay):
|
||||||
p.runPollScan(mountID, mountPath)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Re-check stop after the wake-up to avoid running a scan during shutdown.
|
||||||
|
select {
|
||||||
|
case <-poller.stopChan:
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
duration, err := p.executeScan(mountID, mountPath)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Adaptive backoff: idle for IdleFactor × last duration, with the
|
||||||
|
// configured interval as a floor so tiny mounts still scan often.
|
||||||
|
nextDelay = duration * IdleFactor
|
||||||
|
if nextDelay < floor {
|
||||||
|
nextDelay = floor
|
||||||
|
}
|
||||||
|
p.setNextScanAt(mountID, time.Now().Add(nextDelay))
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
}
|
}
|
||||||
@@ -199,54 +277,109 @@ func (p *Poller) stopPoller(poller *mountPoller) {
|
|||||||
// Update status
|
// Update status
|
||||||
if status, exists := p.status[poller.mountID]; exists {
|
if status, exists := p.status[poller.mountID]; exists {
|
||||||
status.Active = false
|
status.Active = false
|
||||||
|
status.NextScanAt = 0
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// runPollScan executes a scan for a mount
|
// executeScan runs scanFunc with timing + status tracking. Both the adaptive
|
||||||
func (p *Poller) runPollScan(mountID, mountPath string) {
|
// loop and manual TriggerScan funnel through here so the UI sees a consistent
|
||||||
|
// "scanning / last duration / last scan" picture regardless of which path
|
||||||
|
// kicked it off. Returns the scan duration so the caller can compute the next
|
||||||
|
// adaptive delay.
|
||||||
|
func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
|
||||||
if p.scanFunc == nil {
|
if p.scanFunc == nil {
|
||||||
return
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
startMs := start.UnixMilli()
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.Scanning = true
|
||||||
|
status.CurrentScanStartedAt = startMs
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
fileCount, err := p.scanFunc(mountPath)
|
fileCount, err := p.scanFunc(mountPath)
|
||||||
if err != nil {
|
|
||||||
log.Printf("[Poller] Scan failed for mount %s: %v", mountID, err)
|
|
||||||
return
|
|
||||||
}
|
|
||||||
|
|
||||||
// Update status
|
end := time.Now()
|
||||||
|
duration := end.Sub(start)
|
||||||
|
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
if status, exists := p.status[mountID]; exists {
|
if status, exists := p.status[mountID]; exists {
|
||||||
status.LastScan = NowMS()
|
status.Scanning = false
|
||||||
|
status.CurrentScanStartedAt = 0
|
||||||
|
status.LastScan = end.UnixMilli()
|
||||||
|
status.LastScanDurationMs = duration.Milliseconds()
|
||||||
}
|
}
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
log.Printf("[Poller] Poll scan complete for mount %s: %d files", mountID, fileCount)
|
if err == nil {
|
||||||
|
log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
|
||||||
}
|
}
|
||||||
|
|
||||||
// TriggerScan triggers an immediate scan for a mount
|
return duration, err
|
||||||
|
}
|
||||||
|
|
||||||
|
// setNextScanAt records the projected start time of the next adaptive scan
|
||||||
|
// (used purely for the UI countdown — the loop's own timer is authoritative).
|
||||||
|
func (p *Poller) setNextScanAt(mountID string, at time.Time) {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.NextScanAt = at.UnixMilli()
|
||||||
|
}
|
||||||
|
|
||||||
|
// runPollScan is kept for backwards compatibility with any external caller —
|
||||||
|
// new code should go through executeScan via the adaptive loop.
|
||||||
|
func (p *Poller) runPollScan(mountID, mountPath string) {
|
||||||
|
_, _ = p.executeScan(mountID, mountPath)
|
||||||
|
}
|
||||||
|
|
||||||
|
// TriggerScan triggers an immediate scan for a mount (the "Scan Now" button).
|
||||||
|
// Runs synchronously; concurrent calls with the adaptive loop are possible
|
||||||
|
// but rare in practice — both paths update the same status atomically.
|
||||||
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
func (p *Poller) TriggerScan(mountID, mountPath string) (int, error) {
|
||||||
if p.scanFunc == nil {
|
if p.scanFunc == nil {
|
||||||
return 0, nil
|
return 0, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
start := time.Now()
|
||||||
|
startMs := start.UnixMilli()
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
status, exists := p.status[mountID]
|
||||||
|
if !exists {
|
||||||
|
status = &PollingStatus{}
|
||||||
|
p.status[mountID] = status
|
||||||
|
}
|
||||||
|
status.Scanning = true
|
||||||
|
status.CurrentScanStartedAt = startMs
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
fileCount, err := p.scanFunc(mountPath)
|
fileCount, err := p.scanFunc(mountPath)
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
end := time.Now()
|
||||||
}
|
duration := end.Sub(start)
|
||||||
|
|
||||||
// Update status
|
|
||||||
p.mu.Lock()
|
p.mu.Lock()
|
||||||
if status, exists := p.status[mountID]; exists {
|
if status, exists := p.status[mountID]; exists {
|
||||||
status.LastScan = NowMS()
|
status.Scanning = false
|
||||||
} else {
|
status.CurrentScanStartedAt = 0
|
||||||
p.status[mountID] = &PollingStatus{
|
status.LastScan = end.UnixMilli()
|
||||||
LastScan: NowMS(),
|
status.LastScanDurationMs = duration.Milliseconds()
|
||||||
}
|
|
||||||
}
|
}
|
||||||
p.mu.Unlock()
|
p.mu.Unlock()
|
||||||
|
|
||||||
return fileCount, nil
|
return fileCount, err
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetPollingStatus returns the polling status for a mount
|
// GetPollingStatus returns the polling status for a mount
|
||||||
@@ -258,6 +391,10 @@ func (p *Poller) GetPollingStatus(mountID string) *PollingStatus {
|
|||||||
return &PollingStatus{
|
return &PollingStatus{
|
||||||
Active: status.Active,
|
Active: status.Active,
|
||||||
LastScan: status.LastScan,
|
LastScan: status.LastScan,
|
||||||
|
Scanning: status.Scanning,
|
||||||
|
CurrentScanStartedAt: status.CurrentScanStartedAt,
|
||||||
|
LastScanDurationMs: status.LastScanDurationMs,
|
||||||
|
NextScanAt: status.NextScanAt,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
return &PollingStatus{}
|
return &PollingStatus{}
|
||||||
|
|||||||
@@ -0,0 +1,70 @@
|
|||||||
|
package mounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"net/http"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StatsSample is a raw cumulative-counter snapshot taken at one instant in
|
||||||
|
// time. The poller keeps the previous sample per mount and computes rates
|
||||||
|
// (Bps, transfers/sec) from the delta to the latest sample.
|
||||||
|
//
|
||||||
|
// Post-consolidation: rclone is the only sample source, since all mount types
|
||||||
|
// execute through the local rclone daemon.
|
||||||
|
type StatsSample struct {
|
||||||
|
TakenAt time.Time
|
||||||
|
Source string // "rclone"
|
||||||
|
BytesRead int64
|
||||||
|
BytesWritten int64
|
||||||
|
ReadOps int64
|
||||||
|
WriteOps int64
|
||||||
|
// File-level transfer count from rclone's RC API. Distinct from ReadOps
|
||||||
|
// (which is unused for rclone — kernel ops counters are not exposed
|
||||||
|
// through FUSE).
|
||||||
|
Transfers int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// rcloneCoreStats matches the RC API response from POST /core/stats.
|
||||||
|
type rcloneCoreStats struct {
|
||||||
|
Bytes int64 `json:"bytes"`
|
||||||
|
TotalBytes int64 `json:"totalBytes"`
|
||||||
|
Transfers int64 `json:"transfers"`
|
||||||
|
TotalTransfers int64 `json:"totalTransfers"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SampleRclone queries the local rclone RC API for the cumulative bytes +
|
||||||
|
// transfers counters. The numbers are global across all active mounts on this
|
||||||
|
// host — rclone tracks stats per daemon, not per mount. The poller reuses one
|
||||||
|
// sample for every mounted entry on a tick.
|
||||||
|
func SampleRclone(ctx context.Context) (*StatsSample, error) {
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodPost, "http://127.0.0.1:5572/core/stats", nil)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("build rclone stats request: %w", err)
|
||||||
|
}
|
||||||
|
req.SetBasicAuth("admin", "admin")
|
||||||
|
|
||||||
|
client := &http.Client{Timeout: 3 * time.Second}
|
||||||
|
resp, err := client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("call rclone /core/stats: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode != http.StatusOK {
|
||||||
|
return nil, fmt.Errorf("rclone /core/stats returned %d", resp.StatusCode)
|
||||||
|
}
|
||||||
|
|
||||||
|
var stats rcloneCoreStats
|
||||||
|
if err := json.NewDecoder(resp.Body).Decode(&stats); err != nil {
|
||||||
|
return nil, fmt.Errorf("decode rclone stats: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
return &StatsSample{
|
||||||
|
Source: "rclone",
|
||||||
|
TakenAt: time.Now(),
|
||||||
|
BytesRead: stats.Bytes,
|
||||||
|
Transfers: stats.Transfers,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,185 @@
|
|||||||
|
package mounts
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"sync"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// StatsInterval drives the IO-stats sampling cadence. 5 s is the usual sweet
|
||||||
|
// spot — short enough that throughput numbers feel live, long enough that
|
||||||
|
// /proc reads aren't a load source themselves.
|
||||||
|
const StatsInterval = 5 * time.Second
|
||||||
|
|
||||||
|
// StatsPoller samples cumulative counters from the local rclone RC API and
|
||||||
|
// exposes computed rates (bytes/sec, transfers/sec) via GetStats.
|
||||||
|
//
|
||||||
|
// Post-consolidation note: every mount type executes through rclone, so
|
||||||
|
// /proc/fs/cifs/Stats and /proc/self/mountstats are no longer relevant —
|
||||||
|
// rclone is the only counter source. core/stats is daemon-global across all
|
||||||
|
// active mounts; the same sample is applied to each mount's status. If
|
||||||
|
// per-mount byte counters become important, the path forward is rclone stats
|
||||||
|
// groups (out of scope for this consolidation).
|
||||||
|
type StatsPoller struct {
|
||||||
|
manager *Manager
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
lastSamples map[string]*StatsSample // mountID -> previous raw sample
|
||||||
|
latestStats map[string]*MountIOStats
|
||||||
|
stopChan chan struct{}
|
||||||
|
running bool
|
||||||
|
pollInterval time.Duration
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewStatsPoller constructs a StatsPoller backed by the given manager.
|
||||||
|
func NewStatsPoller(manager *Manager) *StatsPoller {
|
||||||
|
return &StatsPoller{
|
||||||
|
manager: manager,
|
||||||
|
lastSamples: make(map[string]*StatsSample),
|
||||||
|
latestStats: make(map[string]*MountIOStats),
|
||||||
|
pollInterval: StatsInterval,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start launches the sampling loop. Idempotent.
|
||||||
|
func (p *StatsPoller) Start() {
|
||||||
|
p.mu.Lock()
|
||||||
|
if p.running {
|
||||||
|
p.mu.Unlock()
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.running = true
|
||||||
|
p.stopChan = make(chan struct{})
|
||||||
|
stop := p.stopChan
|
||||||
|
p.mu.Unlock()
|
||||||
|
|
||||||
|
log.Printf("[StatsPoller] Starting IO stats sampler (interval: %s)", p.pollInterval)
|
||||||
|
go p.loop(stop)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop terminates the sampling loop. Idempotent.
|
||||||
|
func (p *StatsPoller) Stop() {
|
||||||
|
p.mu.Lock()
|
||||||
|
defer p.mu.Unlock()
|
||||||
|
if !p.running {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
p.running = false
|
||||||
|
if p.stopChan != nil {
|
||||||
|
close(p.stopChan)
|
||||||
|
p.stopChan = nil
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetStats returns the latest computed IO stats for a mount, or nil if no
|
||||||
|
// stats have been observed yet.
|
||||||
|
func (p *StatsPoller) GetStats(mountID string) *MountIOStats {
|
||||||
|
p.mu.RLock()
|
||||||
|
defer p.mu.RUnlock()
|
||||||
|
s, ok := p.latestStats[mountID]
|
||||||
|
if !ok {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Return a copy so the caller can't mutate our cache.
|
||||||
|
cp := *s
|
||||||
|
return &cp
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *StatsPoller) loop(stop <-chan struct{}) {
|
||||||
|
// Take the first sample immediately so rates start populating on the
|
||||||
|
// second tick instead of waiting two intervals.
|
||||||
|
p.tick()
|
||||||
|
|
||||||
|
t := time.NewTicker(p.pollInterval)
|
||||||
|
defer t.Stop()
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-stop:
|
||||||
|
return
|
||||||
|
case <-t.C:
|
||||||
|
p.tick()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (p *StatsPoller) tick() {
|
||||||
|
mounts, err := p.manager.listMountsWithoutPollingStatus()
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[StatsPoller] listMounts failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
// Sample rclone once per tick — its RC API gives global stats across all
|
||||||
|
// active rclone mounts on this host. Both supported mount types (smb,
|
||||||
|
// rclone) execute through the same rclone daemon, so the same sample is
|
||||||
|
// applied to every mounted entry.
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
sample, err := SampleRclone(ctx)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[StatsPoller] rclone sample failed: %v", err)
|
||||||
|
return
|
||||||
|
}
|
||||||
|
if sample == nil {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, m := range mounts {
|
||||||
|
if !m.Mounted {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
p.mu.Lock()
|
||||||
|
prev := p.lastSamples[m.ID]
|
||||||
|
p.lastSamples[m.ID] = sample
|
||||||
|
stats := computeStats(prev, sample)
|
||||||
|
p.latestStats[m.ID] = stats
|
||||||
|
p.mu.Unlock()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// computeStats turns a (previous, current) raw-sample pair into the rate-
|
||||||
|
// based MountIOStats the API exposes. With no previous sample the rates are
|
||||||
|
// zero — operators will see cumulative counters but Bps/IOPS only on the
|
||||||
|
// second poll.
|
||||||
|
func computeStats(prev, cur *StatsSample) *MountIOStats {
|
||||||
|
out := &MountIOStats{
|
||||||
|
Source: cur.Source,
|
||||||
|
BytesRead: cur.BytesRead,
|
||||||
|
BytesWritten: cur.BytesWritten,
|
||||||
|
ReadOps: cur.ReadOps,
|
||||||
|
WriteOps: cur.WriteOps,
|
||||||
|
LastSampleAt: cur.TakenAt.UnixMilli(),
|
||||||
|
}
|
||||||
|
if prev == nil {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
dt := cur.TakenAt.Sub(prev.TakenAt).Seconds()
|
||||||
|
if dt <= 0 {
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
out.IntervalMs = int64(dt * 1000)
|
||||||
|
|
||||||
|
// Counters can reset to zero (e.g. CIFS module reload, remount). Treat
|
||||||
|
// negative deltas as "no movement" rather than letting the rate go
|
||||||
|
// negative or implausibly huge.
|
||||||
|
if d := cur.BytesRead - prev.BytesRead; d > 0 {
|
||||||
|
out.ReadBps = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.BytesWritten - prev.BytesWritten; d > 0 {
|
||||||
|
out.WriteBps = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.ReadOps - prev.ReadOps; d > 0 {
|
||||||
|
out.ReadIops = float64(d) / dt
|
||||||
|
}
|
||||||
|
if d := cur.WriteOps - prev.WriteOps; d > 0 {
|
||||||
|
out.WriteIops = float64(d) / dt
|
||||||
|
}
|
||||||
|
if cur.Source == "rclone" {
|
||||||
|
if d := cur.Transfers - prev.Transfers; d > 0 {
|
||||||
|
out.TransfersPerSec = float64(d) / dt
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
+64
-14
@@ -2,11 +2,19 @@ package mounts
|
|||||||
|
|
||||||
import "time"
|
import "time"
|
||||||
|
|
||||||
// MountType represents the type of mount
|
// MountType represents the type of mount.
|
||||||
|
//
|
||||||
|
// All types execute through the local rclone daemon. "smb" is a UX-level alias
|
||||||
|
// for rclone's :smb: backend — operators fill in SMB-specific fields and the
|
||||||
|
// mount-watcher synthesises the rclone remote spec at mount time. "rclone" is
|
||||||
|
// the generic escape hatch for any pre-defined remote in rclone.conf.
|
||||||
|
//
|
||||||
|
// Native NFS/CIFS handlers were removed in favour of the rclone-only path —
|
||||||
|
// kernel inotify wasn't reliable on those mounts anyway and the watcher polls
|
||||||
|
// regardless.
|
||||||
type MountType string
|
type MountType string
|
||||||
|
|
||||||
const (
|
const (
|
||||||
MountTypeNFS MountType = "nfs"
|
|
||||||
MountTypeSMB MountType = "smb"
|
MountTypeSMB MountType = "smb"
|
||||||
MountTypeRclone MountType = "rclone"
|
MountTypeRclone MountType = "rclone"
|
||||||
)
|
)
|
||||||
@@ -17,6 +25,15 @@ const (
|
|||||||
MinPollingIntervalMs = 5000 // 5 seconds
|
MinPollingIntervalMs = 5000 // 5 seconds
|
||||||
)
|
)
|
||||||
|
|
||||||
|
// VFS cache defaults — applied per-mount when the caller leaves the field
|
||||||
|
// blank. Values are strings so they round-trip rclone's parsers (SizeSuffix,
|
||||||
|
// Duration) untouched.
|
||||||
|
const (
|
||||||
|
DefaultCacheMaxSize = "50G"
|
||||||
|
DefaultCacheMaxAge = "24h"
|
||||||
|
DefaultDirCacheTime = "5m"
|
||||||
|
)
|
||||||
|
|
||||||
// MountConfig represents a mount configuration
|
// MountConfig represents a mount configuration
|
||||||
type MountConfig struct {
|
type MountConfig struct {
|
||||||
// Common fields
|
// Common fields
|
||||||
@@ -26,23 +43,26 @@ type MountConfig struct {
|
|||||||
Enabled bool `json:"enabled"`
|
Enabled bool `json:"enabled"`
|
||||||
DesiredMounted bool `json:"desiredMounted"`
|
DesiredMounted bool `json:"desiredMounted"`
|
||||||
MountPath string `json:"mountPath"`
|
MountPath string `json:"mountPath"`
|
||||||
Options string `json:"options,omitempty"`
|
|
||||||
|
|
||||||
// NFS-specific fields
|
// SMB-specific fields (also valid for type=smb routed through rclone-smb)
|
||||||
NFSServer string `json:"nfsServer,omitempty"`
|
|
||||||
NFSPath string `json:"nfsPath,omitempty"`
|
|
||||||
|
|
||||||
// SMB-specific fields
|
|
||||||
SMBServer string `json:"smbServer,omitempty"`
|
SMBServer string `json:"smbServer,omitempty"`
|
||||||
SMBShare string `json:"smbShare,omitempty"`
|
SMBShare string `json:"smbShare,omitempty"`
|
||||||
SMBUsername string `json:"smbUsername,omitempty"`
|
SMBUsername string `json:"smbUsername,omitempty"`
|
||||||
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
SMBPasswordObscured string `json:"smbPasswordObscured,omitempty"`
|
||||||
SMBDomain string `json:"smbDomain,omitempty"`
|
SMBDomain string `json:"smbDomain,omitempty"`
|
||||||
|
|
||||||
// rclone-specific fields
|
// rclone-specific fields (escape hatch — references a pre-defined remote)
|
||||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||||
RclonePath string `json:"rclonePath,omitempty"`
|
RclonePath string `json:"rclonePath,omitempty"`
|
||||||
|
|
||||||
|
// VFS cache configuration — applied to the rclone mount call. All three
|
||||||
|
// fields are optional; blanks fall back to Default* constants above.
|
||||||
|
// CacheMaxSize uses rclone's SizeSuffix ("50G", "10M"), CacheMaxAge and
|
||||||
|
// DirCacheTime use rclone's Duration ("24h", "30m").
|
||||||
|
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||||
|
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||||
|
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||||
|
|
||||||
// Polling configuration
|
// Polling configuration
|
||||||
PollingEnabled bool `json:"pollingEnabled"`
|
PollingEnabled bool `json:"pollingEnabled"`
|
||||||
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
PollingIntervalMs int `json:"pollingIntervalMs,omitempty"` // Default: 30000
|
||||||
@@ -56,6 +76,36 @@ type MountStatus struct {
|
|||||||
LastChecked int64 `json:"lastChecked"`
|
LastChecked int64 `json:"lastChecked"`
|
||||||
PollingActive bool `json:"pollingActive,omitempty"`
|
PollingActive bool `json:"pollingActive,omitempty"`
|
||||||
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
LastPolledScan int64 `json:"lastPolledScan,omitempty"`
|
||||||
|
// Progress tracking — surface the poller's adaptive scan state to the UI.
|
||||||
|
Scanning bool `json:"scanning,omitempty"`
|
||||||
|
CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; non-zero only while Scanning
|
||||||
|
LastScanDurationMs int64 `json:"lastScanDurationMs,omitempty"` // duration of the most recently completed scan
|
||||||
|
NextScanAt int64 `json:"nextScanAt,omitempty"` // ms; planned start of the next adaptive scan
|
||||||
|
// Live IO stats — sampled by StatsPoller from /proc/fs/cifs/Stats,
|
||||||
|
// /proc/self/mountstats, or rclone's RC API depending on mount type.
|
||||||
|
IOStats *MountIOStats `json:"ioStats,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// MountIOStats captures throughput + operation rate for a single mount.
|
||||||
|
// Cumulative counters (BytesRead, ReadOps) come straight from the source;
|
||||||
|
// rates (ReadBps, ReadIops) are computed by StatsPoller as the delta between
|
||||||
|
// the last two samples divided by the interval.
|
||||||
|
type MountIOStats struct {
|
||||||
|
Source string `json:"source"` // "rclone" — only source after the rclone-only consolidation
|
||||||
|
BytesRead int64 `json:"bytesRead"`
|
||||||
|
BytesWritten int64 `json:"bytesWritten,omitempty"`
|
||||||
|
ReadBps float64 `json:"readBps"`
|
||||||
|
WriteBps float64 `json:"writeBps,omitempty"`
|
||||||
|
ReadOps int64 `json:"readOps,omitempty"`
|
||||||
|
WriteOps int64 `json:"writeOps,omitempty"`
|
||||||
|
// ReadIops and WriteIops are op counters per second — number of read/write
|
||||||
|
// requests issued by the kernel against the share. Distinct from
|
||||||
|
// TransfersPerSec which is a file-level metric only relevant to rclone.
|
||||||
|
ReadIops float64 `json:"readIops,omitempty"`
|
||||||
|
WriteIops float64 `json:"writeIops,omitempty"`
|
||||||
|
TransfersPerSec float64 `json:"transfersPerSec,omitempty"` // rclone-only
|
||||||
|
LastSampleAt int64 `json:"lastSampleAt"` // ms
|
||||||
|
IntervalMs int64 `json:"intervalMs,omitempty"` // gap between the two samples used to compute rates
|
||||||
}
|
}
|
||||||
|
|
||||||
// MountsFile represents the mounts.json file structure
|
// MountsFile represents the mounts.json file structure
|
||||||
@@ -75,11 +125,6 @@ type CreateMountRequest struct {
|
|||||||
Name string `json:"name"`
|
Name string `json:"name"`
|
||||||
Type MountType `json:"type"`
|
Type MountType `json:"type"`
|
||||||
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
Enabled *bool `json:"enabled,omitempty"` // Pointer to detect if set
|
||||||
Options string `json:"options,omitempty"`
|
|
||||||
|
|
||||||
// NFS
|
|
||||||
NFSServer string `json:"nfsServer,omitempty"`
|
|
||||||
NFSPath string `json:"nfsPath,omitempty"`
|
|
||||||
|
|
||||||
// SMB
|
// SMB
|
||||||
SMBServer string `json:"smbServer,omitempty"`
|
SMBServer string `json:"smbServer,omitempty"`
|
||||||
@@ -92,6 +137,11 @@ type CreateMountRequest struct {
|
|||||||
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
RcloneRemote string `json:"rcloneRemote,omitempty"`
|
||||||
RclonePath string `json:"rclonePath,omitempty"`
|
RclonePath string `json:"rclonePath,omitempty"`
|
||||||
|
|
||||||
|
// VFS cache (optional — falls back to Default* constants when unset)
|
||||||
|
CacheMaxSize string `json:"cacheMaxSize,omitempty"`
|
||||||
|
CacheMaxAge string `json:"cacheMaxAge,omitempty"`
|
||||||
|
DirCacheTime string `json:"dirCacheTime,omitempty"`
|
||||||
|
|
||||||
// Polling
|
// Polling
|
||||||
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
PollingEnabled *bool `json:"pollingEnabled,omitempty"`
|
||||||
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
PollingIntervalMs *int `json:"pollingIntervalMs,omitempty"`
|
||||||
|
|||||||
@@ -0,0 +1,103 @@
|
|||||||
|
package schema
|
||||||
|
|
||||||
|
import (
|
||||||
|
"encoding/json"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// cidRegex matches the CID shapes produced by @metazla/meta-hash:
|
||||||
|
// - CIDv1 multibase-base32: every algorithm (crc32, md5, sha1, sha2-256,
|
||||||
|
// sha3-256, sha3-384, midhash256, btih-v2, …) is serialized via
|
||||||
|
// CID.createV1(code, digest).toString(), which always yields a lowercase
|
||||||
|
// base32 string starting with "ba" (the "b" is the multibase tag, the
|
||||||
|
// "a" is the CIDv1 version byte under base32-encoding-of-0x01).
|
||||||
|
// - CIDv0 base58btc: "Qm" + 44 chars, kept for compatibility with other
|
||||||
|
// services that may still emit v0.
|
||||||
|
//
|
||||||
|
// Minimum CIDv1 length 16 chars matches the shortest real output we observed
|
||||||
|
// (crc32: "bafk3eaqerjhiwhy").
|
||||||
|
var cidRegex = regexp.MustCompile(`^(ba[a-z2-7]{14,}|Qm[1-9A-HJ-NP-Za-km-z]{44})$`)
|
||||||
|
|
||||||
|
// ClassifyValue returns the (primitive, hint) for a Redis value.
|
||||||
|
// exists=false (key missing) and JSON null literal both map to PrimUndefined.
|
||||||
|
// Empty string is a valid string.
|
||||||
|
func ClassifyValue(value string, exists bool) (Primitive, Hint) {
|
||||||
|
if !exists {
|
||||||
|
return PrimUndefined, ""
|
||||||
|
}
|
||||||
|
trimmed := strings.TrimSpace(value)
|
||||||
|
if trimmed == "null" {
|
||||||
|
return PrimUndefined, ""
|
||||||
|
}
|
||||||
|
if trimmed == "true" || trimmed == "false" {
|
||||||
|
return PrimBool, ""
|
||||||
|
}
|
||||||
|
if _, err := strconv.ParseInt(trimmed, 10, 64); err == nil {
|
||||||
|
if isTimestampInt(trimmed) {
|
||||||
|
return PrimInt, HintTimestamp
|
||||||
|
}
|
||||||
|
return PrimInt, ""
|
||||||
|
}
|
||||||
|
if _, err := strconv.ParseFloat(trimmed, 64); err == nil {
|
||||||
|
return PrimFloat, ""
|
||||||
|
}
|
||||||
|
if len(trimmed) > 0 {
|
||||||
|
switch trimmed[0] {
|
||||||
|
case '{':
|
||||||
|
var v map[string]any
|
||||||
|
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||||
|
return PrimJSONObj, ""
|
||||||
|
}
|
||||||
|
case '[':
|
||||||
|
var v []any
|
||||||
|
if json.Unmarshal([]byte(trimmed), &v) == nil {
|
||||||
|
return PrimJSONArr, ""
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if cidRegex.MatchString(value) {
|
||||||
|
return PrimString, HintCID
|
||||||
|
}
|
||||||
|
if isTimestampString(value) {
|
||||||
|
return PrimString, HintTimestamp
|
||||||
|
}
|
||||||
|
return PrimString, ""
|
||||||
|
}
|
||||||
|
|
||||||
|
// isTimestampInt accepts unix-seconds in [2001-09, 2100] or unix-millis in the
|
||||||
|
// same range. Anything else is treated as a plain int.
|
||||||
|
func isTimestampInt(s string) bool {
|
||||||
|
n, err := strconv.ParseInt(s, 10, 64)
|
||||||
|
if err != nil || n <= 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
if n >= 1_000_000_000 && n <= 4_102_444_800 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if n >= 1_000_000_000_000 && n <= 4_102_444_800_000 {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
func isTimestampString(s string) bool {
|
||||||
|
if _, err := time.Parse(time.RFC3339, s); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
if _, err := time.Parse("2006-01-02", s); err == nil {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
// breakdownLabel returns the most specific label for a classified value.
|
||||||
|
// Hints take precedence over primitives.
|
||||||
|
func breakdownLabel(prim Primitive, hint Hint) string {
|
||||||
|
if hint != "" {
|
||||||
|
return string(hint)
|
||||||
|
}
|
||||||
|
return string(prim)
|
||||||
|
}
|
||||||
@@ -0,0 +1,355 @@
|
|||||||
|
package schema
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"log"
|
||||||
|
"regexp"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
"sync/atomic"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/events"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
var langCodeRegex = regexp.MustCompile(`^[a-z]{2,3}$`)
|
||||||
|
|
||||||
|
// fieldState is the per-field accumulator. perKey lets us undo a previous
|
||||||
|
// contribution when the same Redis key is re-SET or DELed.
|
||||||
|
type fieldState struct {
|
||||||
|
counts map[string]int
|
||||||
|
primitives map[Primitive]int
|
||||||
|
hintCounts map[Hint]int
|
||||||
|
typedCount int
|
||||||
|
undefinedCount int
|
||||||
|
languageKeyed bool
|
||||||
|
perKey map[string]labelPair
|
||||||
|
}
|
||||||
|
|
||||||
|
type labelPair struct {
|
||||||
|
prim Primitive
|
||||||
|
hint Hint
|
||||||
|
}
|
||||||
|
|
||||||
|
func newFieldState() *fieldState {
|
||||||
|
return &fieldState{
|
||||||
|
counts: make(map[string]int),
|
||||||
|
primitives: make(map[Primitive]int),
|
||||||
|
hintCounts: make(map[Hint]int),
|
||||||
|
perKey: make(map[string]labelPair),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Indexer maintains a live field-schema by consuming meta:events.
|
||||||
|
type Indexer struct {
|
||||||
|
client *redis.Client
|
||||||
|
keyPrefix string
|
||||||
|
|
||||||
|
mu sync.RWMutex
|
||||||
|
fields map[string]*fieldState
|
||||||
|
|
||||||
|
processed uint64 // total events processed since start
|
||||||
|
|
||||||
|
ctx context.Context
|
||||||
|
cancel context.CancelFunc
|
||||||
|
wg sync.WaitGroup
|
||||||
|
runMu sync.Mutex
|
||||||
|
running bool
|
||||||
|
}
|
||||||
|
|
||||||
|
// NewIndexer creates an Indexer. keyPrefix should match storage.Client.GetPrefix()
|
||||||
|
// so the indexer can build absolute Redis keys for GET calls.
|
||||||
|
func NewIndexer(client *redis.Client, keyPrefix string) *Indexer {
|
||||||
|
ctx, cancel := context.WithCancel(context.Background())
|
||||||
|
return &Indexer{
|
||||||
|
client: client,
|
||||||
|
keyPrefix: keyPrefix,
|
||||||
|
fields: make(map[string]*fieldState),
|
||||||
|
ctx: ctx,
|
||||||
|
cancel: cancel,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Start begins consuming the meta:events stream from the earliest available ID.
|
||||||
|
func (i *Indexer) Start() error {
|
||||||
|
i.runMu.Lock()
|
||||||
|
if i.running {
|
||||||
|
i.runMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
i.running = true
|
||||||
|
i.runMu.Unlock()
|
||||||
|
|
||||||
|
i.wg.Add(1)
|
||||||
|
go i.consumeLoop()
|
||||||
|
|
||||||
|
log.Println("[SchemaIndexer] Started - consuming meta:events stream")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Stop terminates the consumer goroutine.
|
||||||
|
func (i *Indexer) Stop() error {
|
||||||
|
i.runMu.Lock()
|
||||||
|
if !i.running {
|
||||||
|
i.runMu.Unlock()
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
i.running = false
|
||||||
|
i.runMu.Unlock()
|
||||||
|
|
||||||
|
i.cancel()
|
||||||
|
i.wg.Wait()
|
||||||
|
log.Println("[SchemaIndexer] Stopped")
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ProcessedCount returns the total number of stream events processed.
|
||||||
|
// Used by WaitForDrain to detect when a rescan flood has been absorbed.
|
||||||
|
func (i *Indexer) ProcessedCount() uint64 {
|
||||||
|
return atomic.LoadUint64(&i.processed)
|
||||||
|
}
|
||||||
|
|
||||||
|
// WaitForDrain blocks until ProcessedCount has been stable for `stableFor` or
|
||||||
|
// the timeout elapses. Suitable for use right after RepublishMetadata.
|
||||||
|
func (i *Indexer) WaitForDrain(timeout time.Duration) {
|
||||||
|
deadline := time.Now().Add(timeout)
|
||||||
|
last := i.ProcessedCount()
|
||||||
|
stableSince := time.Now()
|
||||||
|
for time.Now().Before(deadline) {
|
||||||
|
time.Sleep(100 * time.Millisecond)
|
||||||
|
now := i.ProcessedCount()
|
||||||
|
if now != last {
|
||||||
|
last = now
|
||||||
|
stableSince = time.Now()
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if time.Since(stableSince) >= 300*time.Millisecond {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Indexer) consumeLoop() {
|
||||||
|
defer i.wg.Done()
|
||||||
|
lastID := "0"
|
||||||
|
for {
|
||||||
|
select {
|
||||||
|
case <-i.ctx.Done():
|
||||||
|
return
|
||||||
|
default:
|
||||||
|
}
|
||||||
|
|
||||||
|
res, err := i.client.XRead(i.ctx, &redis.XReadArgs{
|
||||||
|
Streams: []string{events.MetaEventsStream, lastID},
|
||||||
|
Block: 2 * time.Second,
|
||||||
|
Count: 500,
|
||||||
|
}).Result()
|
||||||
|
|
||||||
|
if err != nil {
|
||||||
|
if err == redis.Nil || i.ctx.Err() != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
log.Printf("[SchemaIndexer] XRead error: %v", err)
|
||||||
|
time.Sleep(1 * time.Second)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, stream := range res {
|
||||||
|
for _, msg := range stream.Messages {
|
||||||
|
lastID = msg.ID
|
||||||
|
i.processEvent(msg.Values)
|
||||||
|
atomic.AddUint64(&i.processed, 1)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Indexer) processEvent(values map[string]any) {
|
||||||
|
op, _ := values["type"].(string)
|
||||||
|
key, _ := values["key"].(string)
|
||||||
|
if key == "" {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fieldPath, keyHint, ok := parseFieldPath(key)
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
switch op {
|
||||||
|
case "set":
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
|
||||||
|
val, err := i.client.Get(ctx, i.keyPrefix+key).Result()
|
||||||
|
cancel()
|
||||||
|
exists := err != redis.Nil
|
||||||
|
if err != nil && err != redis.Nil {
|
||||||
|
// Treat transient errors as "value disappeared" — nothing to record.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
prim, hint := ClassifyValue(val, exists)
|
||||||
|
i.upsert(fieldPath, key, keyHint, prim, hint)
|
||||||
|
case "del", "expire":
|
||||||
|
i.remove(fieldPath, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseFieldPath turns "file:<rootID>/<propertyPath>" into the logical field
|
||||||
|
// path used as the map key. rootID is opaque to the indexer — currently a
|
||||||
|
// UUIDv7, historically a midhash256:abc token; both shapes parse identically
|
||||||
|
// because the split is on the first '/' after "file:".
|
||||||
|
//
|
||||||
|
// When the last segment of propertyPath matches ^[a-z]{2,3}$ it is treated as
|
||||||
|
// a language code: the field name collapses to the prefix and key_hint is set.
|
||||||
|
func parseFieldPath(key string) (string, KeyHint, bool) {
|
||||||
|
if !strings.HasPrefix(key, "file:") {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
rest := strings.TrimPrefix(key, "file:")
|
||||||
|
slash := strings.Index(rest, "/")
|
||||||
|
if slash < 0 {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
property := rest[slash+1:]
|
||||||
|
if property == "" {
|
||||||
|
return "", "", false
|
||||||
|
}
|
||||||
|
if last := strings.LastIndex(property, "/"); last >= 0 {
|
||||||
|
tail := property[last+1:]
|
||||||
|
if langCodeRegex.MatchString(tail) {
|
||||||
|
return property[:last], KeyHintLanguageCode, true
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return property, "", true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Indexer) upsert(fieldPath, redisKey string, keyHint KeyHint, prim Primitive, hint Hint) {
|
||||||
|
i.mu.Lock()
|
||||||
|
defer i.mu.Unlock()
|
||||||
|
|
||||||
|
fs, ok := i.fields[fieldPath]
|
||||||
|
if !ok {
|
||||||
|
fs = newFieldState()
|
||||||
|
i.fields[fieldPath] = fs
|
||||||
|
}
|
||||||
|
if keyHint != "" {
|
||||||
|
fs.languageKeyed = true
|
||||||
|
}
|
||||||
|
|
||||||
|
if old, ok := fs.perKey[redisKey]; ok {
|
||||||
|
decContribution(fs, old)
|
||||||
|
}
|
||||||
|
|
||||||
|
fs.perKey[redisKey] = labelPair{prim: prim, hint: hint}
|
||||||
|
incContribution(fs, labelPair{prim: prim, hint: hint})
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Indexer) remove(fieldPath, redisKey string) {
|
||||||
|
i.mu.Lock()
|
||||||
|
defer i.mu.Unlock()
|
||||||
|
|
||||||
|
fs, ok := i.fields[fieldPath]
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
old, ok := fs.perKey[redisKey]
|
||||||
|
if !ok {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
delete(fs.perKey, redisKey)
|
||||||
|
decContribution(fs, old)
|
||||||
|
if len(fs.perKey) == 0 {
|
||||||
|
delete(i.fields, fieldPath)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func incContribution(fs *fieldState, lp labelPair) {
|
||||||
|
label := breakdownLabel(lp.prim, lp.hint)
|
||||||
|
fs.counts[label]++
|
||||||
|
if lp.prim == PrimUndefined {
|
||||||
|
fs.undefinedCount++
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fs.typedCount++
|
||||||
|
fs.primitives[lp.prim]++
|
||||||
|
if lp.hint != "" {
|
||||||
|
fs.hintCounts[lp.hint]++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func decContribution(fs *fieldState, lp labelPair) {
|
||||||
|
label := breakdownLabel(lp.prim, lp.hint)
|
||||||
|
fs.counts[label]--
|
||||||
|
if fs.counts[label] <= 0 {
|
||||||
|
delete(fs.counts, label)
|
||||||
|
}
|
||||||
|
if lp.prim == PrimUndefined {
|
||||||
|
fs.undefinedCount--
|
||||||
|
return
|
||||||
|
}
|
||||||
|
fs.typedCount--
|
||||||
|
fs.primitives[lp.prim]--
|
||||||
|
if fs.primitives[lp.prim] <= 0 {
|
||||||
|
delete(fs.primitives, lp.prim)
|
||||||
|
}
|
||||||
|
if lp.hint != "" {
|
||||||
|
fs.hintCounts[lp.hint]--
|
||||||
|
if fs.hintCounts[lp.hint] <= 0 {
|
||||||
|
delete(fs.hintCounts, lp.hint)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Snapshot returns the current schema. Safe to call concurrently with consumption.
|
||||||
|
func (i *Indexer) Snapshot() *SchemaResponse {
|
||||||
|
i.mu.RLock()
|
||||||
|
defer i.mu.RUnlock()
|
||||||
|
|
||||||
|
out := &SchemaResponse{
|
||||||
|
Fields: make(map[string]*FieldSchema, len(i.fields)),
|
||||||
|
GeneratedAt: time.Now().UTC(),
|
||||||
|
Source: "live",
|
||||||
|
}
|
||||||
|
for fp, fs := range i.fields {
|
||||||
|
out.Fields[fp] = computeFieldSchema(fs)
|
||||||
|
}
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func computeFieldSchema(fs *fieldState) *FieldSchema {
|
||||||
|
out := &FieldSchema{
|
||||||
|
Breakdown: make(map[string]int, len(fs.counts)),
|
||||||
|
}
|
||||||
|
for k, v := range fs.counts {
|
||||||
|
out.Breakdown[k] = v
|
||||||
|
}
|
||||||
|
|
||||||
|
nPrim := len(fs.primitives)
|
||||||
|
hasUndef := fs.undefinedCount > 0
|
||||||
|
|
||||||
|
switch {
|
||||||
|
case nPrim == 0 && hasUndef:
|
||||||
|
out.Type = PrimUndefined
|
||||||
|
case nPrim == 1 && !hasUndef:
|
||||||
|
for p := range fs.primitives {
|
||||||
|
out.Type = p
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
out.Type = PrimMixed
|
||||||
|
}
|
||||||
|
|
||||||
|
// Strict hint promotion: exactly one hint observed, and it covers every
|
||||||
|
// typed (non-undefined) value. Hints are never set on mixed fields.
|
||||||
|
if out.Type != PrimMixed && len(fs.hintCounts) == 1 && fs.typedCount > 0 {
|
||||||
|
for h, c := range fs.hintCounts {
|
||||||
|
if c == fs.typedCount {
|
||||||
|
out.Hint = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if fs.languageKeyed {
|
||||||
|
out.KeyHint = KeyHintLanguageCode
|
||||||
|
}
|
||||||
|
|
||||||
|
return out
|
||||||
|
}
|
||||||
@@ -0,0 +1,48 @@
|
|||||||
|
package schema
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// Primitive is the coarse value type used by the schema indexer.
|
||||||
|
// PrimMixed appears only as an aggregate label on a field, never per value.
|
||||||
|
type Primitive string
|
||||||
|
|
||||||
|
const (
|
||||||
|
PrimString Primitive = "string"
|
||||||
|
PrimInt Primitive = "int"
|
||||||
|
PrimFloat Primitive = "float"
|
||||||
|
PrimBool Primitive = "bool"
|
||||||
|
PrimJSONObj Primitive = "json-object"
|
||||||
|
PrimJSONArr Primitive = "json-array"
|
||||||
|
PrimUndefined Primitive = "undefined"
|
||||||
|
PrimMixed Primitive = "mixed"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Hint is a semantic refinement on top of a primitive (e.g. cid, timestamp).
|
||||||
|
// A hint is promoted onto a field only when every non-undefined value matches it.
|
||||||
|
type Hint string
|
||||||
|
|
||||||
|
const (
|
||||||
|
HintCID Hint = "cid"
|
||||||
|
HintTimestamp Hint = "timestamp"
|
||||||
|
)
|
||||||
|
|
||||||
|
// KeyHint describes the shape of the subkey dimension when a field is keyed
|
||||||
|
// by a structured discriminator (e.g. title/eng, title/fre → language-code).
|
||||||
|
type KeyHint string
|
||||||
|
|
||||||
|
const KeyHintLanguageCode KeyHint = "language-code"
|
||||||
|
|
||||||
|
// FieldSchema is the per-field entry returned by GET /api/schema.
|
||||||
|
type FieldSchema struct {
|
||||||
|
Type Primitive `json:"type"`
|
||||||
|
Hint Hint `json:"hint,omitempty"`
|
||||||
|
KeyHint KeyHint `json:"key_hint,omitempty"`
|
||||||
|
Breakdown map[string]int `json:"breakdown"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// SchemaResponse is the top-level payload for GET /api/schema.
|
||||||
|
type SchemaResponse struct {
|
||||||
|
Fields map[string]*FieldSchema `json:"fields"`
|
||||||
|
GeneratedAt time.Time `json:"generated_at"`
|
||||||
|
Source string `json:"source"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,143 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"errors"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
|
||||||
|
// Trailing slash because ZIP entry names are forward-slash relative paths.
|
||||||
|
const PluginOutputArchivePrefix = "cache/plugin-output/"
|
||||||
|
|
||||||
|
// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
|
||||||
|
// file under it into zw at PluginOutputArchivePrefix + relPath.
|
||||||
|
//
|
||||||
|
// Returns the number of files added. A missing srcDir is not an error: it
|
||||||
|
// just means no plugin has produced output yet, so the section stays empty.
|
||||||
|
func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
|
||||||
|
info, err := os.Stat(srcDir)
|
||||||
|
if errors.Is(err, os.ErrNotExist) {
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("stat %s: %w", srcDir, err)
|
||||||
|
}
|
||||||
|
if !info.IsDir() {
|
||||||
|
return 0, fmt.Errorf("%s is not a directory", srcDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
count := 0
|
||||||
|
err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
|
||||||
|
if walkErr != nil {
|
||||||
|
return walkErr
|
||||||
|
}
|
||||||
|
if d.IsDir() {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
rel, err := filepath.Rel(srcDir, path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
|
||||||
|
|
||||||
|
w, err := zw.Create(entry)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
f, err := os.Open(path)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer f.Close()
|
||||||
|
if _, err := io.Copy(w, f); err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
count++
|
||||||
|
return nil
|
||||||
|
})
|
||||||
|
if err != nil {
|
||||||
|
return count, fmt.Errorf("walk %s: %w", srcDir, err)
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
|
||||||
|
// the ZIP and writes it under dstDir. Existing files are overwritten when
|
||||||
|
// overwrite=true; otherwise they're left alone (treats existing files as the
|
||||||
|
// "mine wins" choice — same toggle the metadata import uses).
|
||||||
|
//
|
||||||
|
// Path traversal is rejected: any entry whose relative path resolves outside
|
||||||
|
// dstDir is skipped with a warning.
|
||||||
|
func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
|
||||||
|
if dstDir == "" {
|
||||||
|
return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
|
||||||
|
}
|
||||||
|
if err := os.MkdirAll(dstDir, 0o755); err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
cleanRoot, err := filepath.Abs(dstDir)
|
||||||
|
if err != nil {
|
||||||
|
return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if strings.HasSuffix(f.Name, "/") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
|
||||||
|
dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
|
||||||
|
|
||||||
|
// Reject anything that escapes the dst root.
|
||||||
|
if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
|
||||||
|
log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
|
||||||
|
if !overwrite {
|
||||||
|
if _, err := os.Stat(dst); err == nil {
|
||||||
|
skipped++
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||||
|
return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err := writeZipFile(f, dst); err != nil {
|
||||||
|
return written, skipped, err
|
||||||
|
}
|
||||||
|
written++
|
||||||
|
}
|
||||||
|
return written, skipped, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeZipFile(f *zip.File, dst string) error {
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("open zip entry %s: %w", f.Name, err)
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
|
||||||
|
out, err := os.Create(dst)
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("create %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
defer out.Close()
|
||||||
|
|
||||||
|
if _, err := io.Copy(out, rc); err != nil {
|
||||||
|
return fmt.Errorf("write %s: %w", dst, err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,132 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"os"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ExportOptions controls what an export includes.
|
||||||
|
type ExportOptions struct {
|
||||||
|
IncludePluginOutput bool
|
||||||
|
// PluginOutputDir is the absolute path on disk that holds plugin-produced
|
||||||
|
// artifacts (typically $FILES_PATH/plugin). Required when
|
||||||
|
// IncludePluginOutput is true.
|
||||||
|
PluginOutputDir string
|
||||||
|
// SourceHost is recorded in the manifest so imports can attribute
|
||||||
|
// where the snapshot came from (defaults to OS hostname).
|
||||||
|
SourceHost string
|
||||||
|
}
|
||||||
|
|
||||||
|
// Exporter writes snapshot ZIPs from a live Redis store.
|
||||||
|
type Exporter struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewExporter(stor *storage.Client) *Exporter {
|
||||||
|
return &Exporter{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Export streams a ZIP archive to w. The archive layout is:
|
||||||
|
//
|
||||||
|
// manifest.json
|
||||||
|
// index.json
|
||||||
|
// metadata/{cid}.json (one per known CID)
|
||||||
|
//
|
||||||
|
// Optional sections (hash-index, plugin-output) are gated by ExportOptions
|
||||||
|
// but are not yet implemented — they live on filesystems meta-core does not
|
||||||
|
// own, and need cross-service collection. The flag is plumbed for forward
|
||||||
|
// compatibility but does nothing today.
|
||||||
|
func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
|
||||||
|
if e.storage == nil || !e.storage.IsConnected() {
|
||||||
|
return Manifest{}, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
hashIDs, err := e.storage.GetAllHashIDs()
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
zw := zip.NewWriter(w)
|
||||||
|
defer zw.Close()
|
||||||
|
|
||||||
|
// 1. manifest.json — written last after we know FileCount, but we need
|
||||||
|
// the entry order to put it first in the archive for readers. Solve
|
||||||
|
// by buffering in memory: it's tiny.
|
||||||
|
host := opts.SourceHost
|
||||||
|
if host == "" {
|
||||||
|
if h, err := os.Hostname(); err == nil {
|
||||||
|
host = h
|
||||||
|
}
|
||||||
|
}
|
||||||
|
manifest := Manifest{
|
||||||
|
SchemaVersion: ManifestSchemaVersion,
|
||||||
|
GeneratedAt: time.Now().UTC(),
|
||||||
|
SourceHost: host,
|
||||||
|
FileCount: len(hashIDs),
|
||||||
|
Includes: Includes{
|
||||||
|
Metadata: true,
|
||||||
|
PluginOutput: opts.IncludePluginOutput,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if err := writeJSON(zw, ManifestFile, manifest); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write manifest: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. index.json — flat list of CIDs. Redundant with metadata/ but
|
||||||
|
// cheap and useful for readers that only need the keyset.
|
||||||
|
if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write index: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. metadata/{cid}.json — one nested document per CID.
|
||||||
|
for _, hashID := range hashIDs {
|
||||||
|
flat, err := e.storage.GetMetadataFlat(hashID)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(flat) == 0 {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
nested := Reconstruct(flat)
|
||||||
|
entry := MetadataDir + hashID + ".json"
|
||||||
|
if err := writeJSON(zw, entry, nested); err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
|
||||||
|
// streams each file into the ZIP. Heavy (~120 MB on a populated dev
|
||||||
|
// stack), so it's gated by the ExportOptions toggle.
|
||||||
|
if opts.IncludePluginOutput {
|
||||||
|
if opts.PluginOutputDir == "" {
|
||||||
|
return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
|
||||||
|
}
|
||||||
|
n, err := addPluginOutput(zw, opts.PluginOutputDir)
|
||||||
|
if err != nil {
|
||||||
|
return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
|
||||||
|
}
|
||||||
|
log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
|
||||||
|
len(hashIDs), host, opts.IncludePluginOutput)
|
||||||
|
return manifest, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func writeJSON(zw *zip.Writer, name string, body interface{}) error {
|
||||||
|
w, err := zw.Create(name)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
enc := json.NewEncoder(w)
|
||||||
|
enc.SetIndent("", " ")
|
||||||
|
return enc.Encode(body)
|
||||||
|
}
|
||||||
@@ -0,0 +1,195 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"regexp"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Flat keys use "/" as the separator. Property paths inside Redis are
|
||||||
|
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
|
||||||
|
const flatSep = "/"
|
||||||
|
|
||||||
|
// numericKey matches a path segment that is a non-negative decimal integer.
|
||||||
|
// Used to detect array slots (".../video/0/codec").
|
||||||
|
var numericKey = regexp.MustCompile(`^\d+$`)
|
||||||
|
|
||||||
|
// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
|
||||||
|
// Keys are property paths (no leading slash, no file:{cid}/ prefix).
|
||||||
|
//
|
||||||
|
// Sibling keys whose path segments are all sequential integers starting at 0
|
||||||
|
// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
|
||||||
|
// are coerced from string to int/float/bool/null when they parse cleanly so
|
||||||
|
// the exported JSON is human-readable; on import we round-trip back to strings.
|
||||||
|
func Reconstruct(flat map[string]string) interface{} {
|
||||||
|
root := map[string]interface{}{}
|
||||||
|
for path, value := range flat {
|
||||||
|
parts := strings.Split(path, flatSep)
|
||||||
|
setPath(root, parts, parseValue(value))
|
||||||
|
}
|
||||||
|
return convertNumericMapsToArrays(root)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
|
||||||
|
// flat property map whose values are all strings (Redis-ready).
|
||||||
|
//
|
||||||
|
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
|
||||||
|
// values are stored as "" to preserve the field's presence — this matches
|
||||||
|
// what flattenMetadata in the TS implementation does.
|
||||||
|
func Flatten(value interface{}) map[string]string {
|
||||||
|
out := map[string]string{}
|
||||||
|
flattenInto(value, "", out)
|
||||||
|
return out
|
||||||
|
}
|
||||||
|
|
||||||
|
func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
|
||||||
|
if len(parts) == 0 {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
cur := root
|
||||||
|
for i, part := range parts {
|
||||||
|
if i == len(parts)-1 {
|
||||||
|
cur[part] = leaf
|
||||||
|
return
|
||||||
|
}
|
||||||
|
next, ok := cur[part].(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
next = map[string]interface{}{}
|
||||||
|
cur[part] = next
|
||||||
|
}
|
||||||
|
cur = next
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func convertNumericMapsToArrays(v interface{}) interface{} {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case map[string]interface{}:
|
||||||
|
// Recurse first so child arrays are detected before parent.
|
||||||
|
for k, child := range t {
|
||||||
|
t[k] = convertNumericMapsToArrays(child)
|
||||||
|
}
|
||||||
|
if !looksLikeArray(t) {
|
||||||
|
return t
|
||||||
|
}
|
||||||
|
// Sort numerically and emit as []interface{}.
|
||||||
|
idx := make([]int, 0, len(t))
|
||||||
|
for k := range t {
|
||||||
|
n, _ := strconv.Atoi(k)
|
||||||
|
idx = append(idx, n)
|
||||||
|
}
|
||||||
|
sort.Ints(idx)
|
||||||
|
arr := make([]interface{}, len(idx))
|
||||||
|
for i, n := range idx {
|
||||||
|
arr[i] = t[strconv.Itoa(n)]
|
||||||
|
}
|
||||||
|
return arr
|
||||||
|
case []interface{}:
|
||||||
|
for i := range t {
|
||||||
|
t[i] = convertNumericMapsToArrays(t[i])
|
||||||
|
}
|
||||||
|
return t
|
||||||
|
default:
|
||||||
|
return v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// looksLikeArray returns true when every key in m is a non-negative integer
|
||||||
|
// and the keys form a contiguous range starting at 0. Empty maps return false
|
||||||
|
// — we can't tell whether the original was {} or [], so prefer {}.
|
||||||
|
func looksLikeArray(m map[string]interface{}) bool {
|
||||||
|
if len(m) == 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen := make(map[int]bool, len(m))
|
||||||
|
for k := range m {
|
||||||
|
if !numericKey.MatchString(k) {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
n, err := strconv.Atoi(k)
|
||||||
|
if err != nil || n < 0 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
seen[n] = true
|
||||||
|
}
|
||||||
|
for i := 0; i < len(m); i++ {
|
||||||
|
if !seen[i] {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func flattenInto(v interface{}, path string, out map[string]string) {
|
||||||
|
switch t := v.(type) {
|
||||||
|
case nil:
|
||||||
|
out[path] = ""
|
||||||
|
case map[string]interface{}:
|
||||||
|
if len(t) == 0 && path != "" {
|
||||||
|
// Empty objects don't have a Redis representation; skip.
|
||||||
|
return
|
||||||
|
}
|
||||||
|
for k, child := range t {
|
||||||
|
next := k
|
||||||
|
if path != "" {
|
||||||
|
next = path + flatSep + k
|
||||||
|
}
|
||||||
|
flattenInto(child, next, out)
|
||||||
|
}
|
||||||
|
case []interface{}:
|
||||||
|
for i, child := range t {
|
||||||
|
next := strconv.Itoa(i)
|
||||||
|
if path != "" {
|
||||||
|
next = path + flatSep + next
|
||||||
|
}
|
||||||
|
flattenInto(child, next, out)
|
||||||
|
}
|
||||||
|
case string:
|
||||||
|
out[path] = t
|
||||||
|
case bool:
|
||||||
|
if t {
|
||||||
|
out[path] = "true"
|
||||||
|
} else {
|
||||||
|
out[path] = "false"
|
||||||
|
}
|
||||||
|
case float64:
|
||||||
|
// JSON numbers always decode to float64; emit ints without decimals.
|
||||||
|
if t == float64(int64(t)) {
|
||||||
|
out[path] = strconv.FormatInt(int64(t), 10)
|
||||||
|
} else {
|
||||||
|
out[path] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||||
|
}
|
||||||
|
default:
|
||||||
|
out[path] = fmt.Sprintf("%v", t)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseValue mirrors the TS reconstruct logic: convert obvious primitives
|
||||||
|
// back to their typed form for nicer exported JSON, leave anything ambiguous
|
||||||
|
// as a string so we don't lose data (e.g. "5.1" channel layouts).
|
||||||
|
func parseValue(s string) interface{} {
|
||||||
|
if s == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
switch s {
|
||||||
|
case "true":
|
||||||
|
return true
|
||||||
|
case "false":
|
||||||
|
return false
|
||||||
|
case "null":
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// Only treat as number when the string is a clean integer or decimal,
|
||||||
|
// no leading zeros / trailing junk. Matches the TS regex.
|
||||||
|
if isCleanNumber(s) {
|
||||||
|
if n, err := strconv.ParseFloat(s, 64); err == nil {
|
||||||
|
return n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||||
|
|
||||||
|
func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }
|
||||||
@@ -0,0 +1,262 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"archive/zip"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"log"
|
||||||
|
"strings"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Mode controls how an import reconciles existing CIDs.
|
||||||
|
type Mode string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ModeReplace deletes every existing property of an imported CID and
|
||||||
|
// writes the snapshot's values verbatim. Untouched CIDs are left alone.
|
||||||
|
ModeReplace Mode = "replace"
|
||||||
|
// ModeMerge keeps unmentioned existing properties and only touches the
|
||||||
|
// fields present in the snapshot. Conflict resolution within merge is
|
||||||
|
// controlled by Conflict.
|
||||||
|
ModeMerge Mode = "merge"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Conflict picks a winner for fields present on both sides during merge.
|
||||||
|
type Conflict string
|
||||||
|
|
||||||
|
const (
|
||||||
|
// ConflictMine keeps the existing (target) value when both sides have one.
|
||||||
|
ConflictMine Conflict = "mine"
|
||||||
|
// ConflictSource overwrites the existing value with the snapshot's value.
|
||||||
|
ConflictSource Conflict = "source"
|
||||||
|
)
|
||||||
|
|
||||||
|
// ImportOptions controls how Import applies a snapshot.
|
||||||
|
type ImportOptions struct {
|
||||||
|
Mode Mode
|
||||||
|
Conflict Conflict
|
||||||
|
DryRun bool
|
||||||
|
// PluginOutputDir is the on-disk destination for cache/plugin-output/*
|
||||||
|
// entries. When empty, plugin-output is silently skipped on import.
|
||||||
|
PluginOutputDir string
|
||||||
|
}
|
||||||
|
|
||||||
|
// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
|
||||||
|
type FileResult struct {
|
||||||
|
HashID string `json:"hashId"`
|
||||||
|
Added int `json:"added"` // new property keys written
|
||||||
|
Updated int `json:"updated"` // existing property keys whose value changed
|
||||||
|
Unchanged int `json:"unchanged"` // existing property keys whose value matched
|
||||||
|
Kept int `json:"kept"` // existing property keys preserved by ConflictMine
|
||||||
|
Deleted int `json:"deleted"` // existing property keys removed (replace mode)
|
||||||
|
Error string `json:"error,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// PluginOutputResult summarizes what happened to cache/plugin-output entries.
|
||||||
|
type PluginOutputResult struct {
|
||||||
|
Written int `json:"written"`
|
||||||
|
Skipped int `json:"skipped"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// ImportResult is the response body for the import endpoint.
|
||||||
|
type ImportResult struct {
|
||||||
|
Mode Mode `json:"mode"`
|
||||||
|
Conflict Conflict `json:"conflict,omitempty"`
|
||||||
|
DryRun bool `json:"dryRun"`
|
||||||
|
TotalFiles int `json:"totalFiles"`
|
||||||
|
FilesOK int `json:"filesOk"`
|
||||||
|
FilesFailed int `json:"filesFailed"`
|
||||||
|
Files []FileResult `json:"files,omitempty"`
|
||||||
|
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Importer applies snapshot ZIPs to a live Redis store.
|
||||||
|
type Importer struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewImporter(stor *storage.Client) *Importer {
|
||||||
|
return &Importer{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
|
||||||
|
// *bytes.Reader after buffering the upload) and applies it to Redis.
|
||||||
|
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
|
||||||
|
if i.storage == nil || !i.storage.IsConnected() {
|
||||||
|
return nil, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
|
||||||
|
return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
|
||||||
|
}
|
||||||
|
if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
|
||||||
|
return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
|
||||||
|
}
|
||||||
|
|
||||||
|
zr, err := zip.NewReader(r, size)
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("open zip: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
var manifest Manifest
|
||||||
|
if err := readJSON(zr, ManifestFile, &manifest); err != nil {
|
||||||
|
return nil, fmt.Errorf("read manifest: %w", err)
|
||||||
|
}
|
||||||
|
if manifest.SchemaVersion > ManifestSchemaVersion {
|
||||||
|
return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
|
||||||
|
manifest.SchemaVersion, ManifestSchemaVersion)
|
||||||
|
}
|
||||||
|
if manifest.SchemaVersion < ManifestSchemaVersion {
|
||||||
|
return nil, fmt.Errorf("snapshot schemaVersion %d is no longer supported (current: %d); v1 used content-hash root keys which are incompatible with the UUID-rooted schema",
|
||||||
|
manifest.SchemaVersion, ManifestSchemaVersion)
|
||||||
|
}
|
||||||
|
|
||||||
|
result := &ImportResult{
|
||||||
|
Mode: opts.Mode,
|
||||||
|
Conflict: opts.Conflict,
|
||||||
|
DryRun: opts.DryRun,
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
|
||||||
|
if hashID == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
fr := i.applyOne(f, hashID, opts)
|
||||||
|
result.TotalFiles++
|
||||||
|
if fr.Error != "" {
|
||||||
|
result.FilesFailed++
|
||||||
|
} else {
|
||||||
|
result.FilesOK++
|
||||||
|
}
|
||||||
|
result.Files = append(result.Files, fr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Plugin-output extraction. Only applied on real imports (not dry-run).
|
||||||
|
// If the snapshot doesn't contain a cache/plugin-output/ section, this is
|
||||||
|
// a no-op. If the destination is unset, we skip silently — the manifest's
|
||||||
|
// pluginOutput flag tells the caller it was on the source side either way.
|
||||||
|
if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
|
||||||
|
// Existing files are kept under ConflictMine / replace-doesn't-apply,
|
||||||
|
// overwritten under ConflictSource. Replace mode behaves like
|
||||||
|
// "overwrite" for cache files (it's the destructive choice for
|
||||||
|
// metadata; mirror that intent here).
|
||||||
|
overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
|
||||||
|
written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
|
||||||
|
if err != nil {
|
||||||
|
return result, fmt.Errorf("extract plugin-output: %w", err)
|
||||||
|
}
|
||||||
|
result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
|
||||||
|
log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
|
||||||
|
opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
|
||||||
|
fr := FileResult{HashID: hashID}
|
||||||
|
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("open entry: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
|
||||||
|
var nested interface{}
|
||||||
|
if err := json.NewDecoder(rc).Decode(&nested); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("decode json: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
incoming := Flatten(nested)
|
||||||
|
|
||||||
|
existing, err := i.storage.GetMetadataFlat(hashID)
|
||||||
|
if err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("read existing: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
if existing == nil {
|
||||||
|
existing = map[string]string{}
|
||||||
|
}
|
||||||
|
|
||||||
|
switch opts.Mode {
|
||||||
|
case ModeReplace:
|
||||||
|
// Diff for reporting.
|
||||||
|
toWrite := map[string]string{}
|
||||||
|
for k, v := range incoming {
|
||||||
|
toWrite[k] = v
|
||||||
|
if cur, ok := existing[k]; ok {
|
||||||
|
if cur == v {
|
||||||
|
fr.Unchanged++
|
||||||
|
} else {
|
||||||
|
fr.Updated++
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
fr.Added++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
for k := range existing {
|
||||||
|
if _, ok := incoming[k]; !ok {
|
||||||
|
fr.Deleted++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !opts.DryRun {
|
||||||
|
if _, err := i.storage.DeleteMetadata(hashID); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("delete existing: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("write: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
case ModeMerge:
|
||||||
|
toWrite := map[string]string{}
|
||||||
|
for k, v := range incoming {
|
||||||
|
cur, present := existing[k]
|
||||||
|
switch {
|
||||||
|
case !present:
|
||||||
|
toWrite[k] = v
|
||||||
|
fr.Added++
|
||||||
|
case cur == v:
|
||||||
|
fr.Unchanged++
|
||||||
|
case opts.Conflict == ConflictSource:
|
||||||
|
toWrite[k] = v
|
||||||
|
fr.Updated++
|
||||||
|
case opts.Conflict == ConflictMine:
|
||||||
|
fr.Kept++
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !opts.DryRun && len(toWrite) > 0 {
|
||||||
|
if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
|
||||||
|
fr.Error = fmt.Sprintf("merge: %v", err)
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return fr
|
||||||
|
}
|
||||||
|
|
||||||
|
func readJSON(zr *zip.Reader, name string, out interface{}) error {
|
||||||
|
for _, f := range zr.File {
|
||||||
|
if f.Name != name {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
rc, err := f.Open()
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
defer rc.Close()
|
||||||
|
return json.NewDecoder(rc).Decode(out)
|
||||||
|
}
|
||||||
|
return fmt.Errorf("entry %q not found", name)
|
||||||
|
}
|
||||||
@@ -0,0 +1,53 @@
|
|||||||
|
// Package snapshot implements export/import/wipe of MetaMesh metadata.
|
||||||
|
//
|
||||||
|
// The on-disk format is a ZIP archive containing one nested-JSON file per
|
||||||
|
// indexed root. Schema is defined by ManifestSchemaVersion; readers must
|
||||||
|
// reject archives whose schemaVersion exceeds what they know.
|
||||||
|
//
|
||||||
|
// v2 (current): root identifiers are UUIDv7 (Crockford Base32). Entries
|
||||||
|
// under metadata/<root>.json carry the flat string properties; the cids
|
||||||
|
// and duplicates Redis SETs are NOT serialized yet — they are rebuilt
|
||||||
|
// transparently on import because SetMetadataFlat / MergeMetadataFlat
|
||||||
|
// auto-register reverse-index aliases for any cid_*/midhash256 field they
|
||||||
|
// see (see internal/storage/cid_resolution.go).
|
||||||
|
//
|
||||||
|
// v1 (legacy): roots were content-hash-typed tokens like "midhash256:abc".
|
||||||
|
// v1 archives can no longer be imported — they would resurrect the
|
||||||
|
// privileged-hashID schema this layer was built to retire.
|
||||||
|
package snapshot
|
||||||
|
|
||||||
|
import "time"
|
||||||
|
|
||||||
|
// ManifestSchemaVersion is the on-disk format version. Bump when the layout
|
||||||
|
// of metadata/{root}.json or the manifest changes in a non-additive way.
|
||||||
|
const ManifestSchemaVersion = 2
|
||||||
|
|
||||||
|
// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
|
||||||
|
const MetadataDir = "metadata/"
|
||||||
|
|
||||||
|
// ManifestFile is the path inside the ZIP that holds the manifest JSON.
|
||||||
|
const ManifestFile = "manifest.json"
|
||||||
|
|
||||||
|
// IndexFile is the path inside the ZIP that holds the file index (CID list).
|
||||||
|
const IndexFile = "index.json"
|
||||||
|
|
||||||
|
// Manifest is the metadata header stored at the root of every snapshot ZIP.
|
||||||
|
type Manifest struct {
|
||||||
|
SchemaVersion int `json:"schemaVersion"`
|
||||||
|
GeneratedAt time.Time `json:"generatedAt"`
|
||||||
|
SourceHost string `json:"sourceHost,omitempty"`
|
||||||
|
MetaCoreVer string `json:"metaCoreVersion,omitempty"`
|
||||||
|
FileCount int `json:"fileCount"`
|
||||||
|
Includes Includes `json:"includes"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Includes records which optional sections are present in a snapshot.
|
||||||
|
type Includes struct {
|
||||||
|
Metadata bool `json:"metadata"`
|
||||||
|
PluginOutput bool `json:"pluginOutput,omitempty"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexFileBody is the structure of index.json inside the ZIP.
|
||||||
|
type IndexFileBody struct {
|
||||||
|
HashIDs []string `json:"hashIds"`
|
||||||
|
}
|
||||||
@@ -0,0 +1,42 @@
|
|||||||
|
package snapshot
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
|
)
|
||||||
|
|
||||||
|
// WipeScope picks which classes of data to clear.
|
||||||
|
type WipeScope struct {
|
||||||
|
Metadata bool // file:* keys + file:__index__
|
||||||
|
}
|
||||||
|
|
||||||
|
// WipeResult reports what was cleared.
|
||||||
|
type WipeResult struct {
|
||||||
|
MetadataDeleted int64 `json:"metadataDeleted"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wiper clears metadata from Redis.
|
||||||
|
type Wiper struct {
|
||||||
|
storage *storage.Client
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewWiper(stor *storage.Client) *Wiper {
|
||||||
|
return &Wiper{storage: stor}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Wipe applies the requested scope. Returns the count of files removed.
|
||||||
|
func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
|
||||||
|
if w.storage == nil || !w.storage.IsConnected() {
|
||||||
|
return nil, fmt.Errorf("storage not connected")
|
||||||
|
}
|
||||||
|
res := &WipeResult{}
|
||||||
|
if scope.Metadata {
|
||||||
|
n, err := w.storage.ClearAllMetadata()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("clear metadata: %w", err)
|
||||||
|
}
|
||||||
|
res.MetadataDeleted = n
|
||||||
|
}
|
||||||
|
return res, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,426 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/metazla/meta-core/internal/cid"
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
// This file implements the CID-resolution layer that lives on top of the
|
||||||
|
// flat metadata storage in client.go. The short version:
|
||||||
|
//
|
||||||
|
// - Roots are opaque UUIDv7s — file:<uuid>/<property> for every property.
|
||||||
|
// The UUID is an internal implementation detail; the mental model is
|
||||||
|
// "N CIDs → one record of keys→metadata".
|
||||||
|
// - Every known CID for a file is registered as a reverse-index entry:
|
||||||
|
// cid:<bareCid> → <uuid> (Redis STRING)
|
||||||
|
// and as a member of the per-root bare-CID key-set:
|
||||||
|
// file:<uuid>/cids/<bareCid> = "true" (flat key, sentinel value)
|
||||||
|
// - A CID is self-describing: its algorithm is recoverable from the CIDv1
|
||||||
|
// multicodec (see internal/cid), so there is no per-algorithm field name
|
||||||
|
// and no <algo>: token prefix. The canonical externally-visible CID is
|
||||||
|
// *derived by rank* from the key-set on read (GetMetadataDocument), never
|
||||||
|
// stored.
|
||||||
|
//
|
||||||
|
// Auto-alias hooks in SetProperty/SetMetadataFlat/MergeMetadataFlat catch
|
||||||
|
// any write of a cids/<cid> field and register the reverse-index entry, so
|
||||||
|
// writers that just emit the key-set members don't need to know about the
|
||||||
|
// reverse index at all. The hook helpers are *Locked — they assume c.mu is
|
||||||
|
// already held by the caller, avoiding nested RLock (which can deadlock
|
||||||
|
// against a waiting writer).
|
||||||
|
|
||||||
|
// CIDIndexPrefix is the Redis key prefix for the reverse index. Combined
|
||||||
|
// with a bare CID, it forms the full key: "cid:bafk…".
|
||||||
|
const CIDIndexPrefix = "cid:"
|
||||||
|
|
||||||
|
// CIDsKeyPrefix is the field-name prefix for the bare-CID key-set. Sibling
|
||||||
|
// CIDs live as flat keys file:<uuid>/cids/<cid> = "true"; the set IS the
|
||||||
|
// keyset, the value is the sentinel. Read by GetMetadataDocument (hoisted
|
||||||
|
// into the typed CIDs array) and by DeleteRoot/DeleteMetadata (to find the
|
||||||
|
// reverse-index entries that need cleanup).
|
||||||
|
const CIDsKeyPrefix = "cids/"
|
||||||
|
|
||||||
|
// DuplicatesField is the per-root SET key holding additional file paths
|
||||||
|
// whose content hashes to the same CIDs as this root. The watcher writes
|
||||||
|
// here when it sees a new path for content that's already indexed; meta-dup
|
||||||
|
// reads from it.
|
||||||
|
const DuplicatesField = "duplicates"
|
||||||
|
|
||||||
|
// buildCIDIndexKey returns the absolute Redis key for a reverse-index entry.
|
||||||
|
// cidStr is a bare multibase CIDv1 ("bafk…").
|
||||||
|
func (c *Client) buildCIDIndexKey(cidStr string) string {
|
||||||
|
return c.buildKey(CIDIndexPrefix + cidStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// Mint creates a new root with filePath/sizeByte/mtimeNano and registers it
|
||||||
|
// in the file index. Returns the new UUID. The caller is expected to follow
|
||||||
|
// up with AddAlias for at least one CID — a root without any alias is
|
||||||
|
// reachable only via the index, which works but defeats the point.
|
||||||
|
func (c *Client) Mint(filePath string, size, mtimeNano int64) (string, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return "", fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
uuid, err := NewUUID()
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("uuid: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
prefix := c.buildKeyPrefix(uuid)
|
||||||
|
pipe := c.client.Pipeline()
|
||||||
|
pipe.Set(ctx, prefix+"filePath", filePath, 0)
|
||||||
|
pipe.Set(ctx, prefix+"sizeByte", strconv.FormatInt(size, 10), 0)
|
||||||
|
pipe.Set(ctx, prefix+"mtimeNano", strconv.FormatInt(mtimeNano, 10), 0)
|
||||||
|
pipe.SAdd(ctx, c.buildIndexKey(), uuid)
|
||||||
|
if _, err := pipe.Exec(ctx); err != nil {
|
||||||
|
return "", fmt.Errorf("mint pipeline: %w", err)
|
||||||
|
}
|
||||||
|
return uuid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddAlias registers cidStr as a sibling CID of uuid: it writes the reverse
|
||||||
|
// index cid:<cidStr> → uuid and the key-set member cids/<cidStr> = "true".
|
||||||
|
// Idempotent: registering the same CID twice is a no-op.
|
||||||
|
func (c *Client) AddAlias(uuid, cidStr string) error {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
return c.addAliasLocked(ctx, uuid, cidStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// addAliasLocked is the lock-free body of AddAlias. The caller must already
|
||||||
|
// hold c.mu (read or write). Split out so the alias-maintenance hooks
|
||||||
|
// invoked from SetProperty / SetMetadataFlat / MergeMetadataFlat can reuse
|
||||||
|
// the work without re-entering the lock — nested RLock can deadlock against
|
||||||
|
// a waiting writer.
|
||||||
|
func (c *Client) addAliasLocked(ctx context.Context, uuid, cidStr string) error {
|
||||||
|
if uuid == "" || cidStr == "" {
|
||||||
|
return fmt.Errorf("uuid and cid are required")
|
||||||
|
}
|
||||||
|
|
||||||
|
keysetKey := c.buildKeyPrefix(uuid) + CIDsKeyPrefix + cidStr
|
||||||
|
|
||||||
|
// Self-pointing alias guard. If the caller is trying to register
|
||||||
|
// cid:<cid> → <cid> (i.e. the root IS the cid value), that's the
|
||||||
|
// dual-root failure mode in disguise — it means somebody wrote
|
||||||
|
// /meta/<cid> without first resolving via ResolveRoot. Refuse the alias
|
||||||
|
// write; leave any existing alias alone. Still record the key-set
|
||||||
|
// membership so the field-level write succeeds.
|
||||||
|
if uuid == cidStr {
|
||||||
|
return c.client.Set(ctx, keysetKey, "true", 0).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// Guard against the dual-root pattern: if cid:<cid> already points at a
|
||||||
|
// DIFFERENT uuid, don't silently overwrite. Leave the existing alias and
|
||||||
|
// just record the key-set membership so it's still discoverable via the
|
||||||
|
// root → cids enumeration.
|
||||||
|
indexKey := c.buildCIDIndexKey(cidStr)
|
||||||
|
existing, err := c.client.Get(ctx, indexKey).Result()
|
||||||
|
if err != nil && err != redis.Nil {
|
||||||
|
return fmt.Errorf("read existing alias: %w", err)
|
||||||
|
}
|
||||||
|
if existing != "" && existing != uuid {
|
||||||
|
return c.client.Set(ctx, keysetKey, "true", 0).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
pipe := c.client.Pipeline()
|
||||||
|
pipe.Set(ctx, indexKey, uuid, 0)
|
||||||
|
pipe.Set(ctx, keysetKey, "true", 0)
|
||||||
|
if _, err := pipe.Exec(ctx); err != nil {
|
||||||
|
return fmt.Errorf("add alias pipeline: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// ResolveRoot maps a hash supplied by a caller (a meta-sort write, an editor
|
||||||
|
// lookup, an SSE consumer) to the actual storage root key.
|
||||||
|
//
|
||||||
|
// Callers send a bare CID (e.g. the midhash "bafk…" the watcher publishes on
|
||||||
|
// file:events, or any sibling CID). Roots are kept opaque and every CID is
|
||||||
|
// aliased into the reverse index:
|
||||||
|
//
|
||||||
|
// cid:bafk… → 01KSBHM*
|
||||||
|
//
|
||||||
|
// Without resolution, every /meta/<cid> write creates a parallel cid-rooted
|
||||||
|
// entry that the watcher's UUID never sees.
|
||||||
|
//
|
||||||
|
// Behaviour:
|
||||||
|
// - empty hash → empty (caller's problem)
|
||||||
|
// - hash matches a reverse-index entry → return UUID
|
||||||
|
// - otherwise return hash unchanged (legacy / explicit-UUID writes)
|
||||||
|
//
|
||||||
|
// Errors are converted to "no resolution" — if Redis is down or the lookup
|
||||||
|
// fails we let the caller proceed with the bare hash and surface the Redis
|
||||||
|
// failure on the actual operation.
|
||||||
|
func (c *Client) ResolveRoot(hash string) string {
|
||||||
|
if hash == "" {
|
||||||
|
return hash
|
||||||
|
}
|
||||||
|
uuid, err := c.GetByCID(hash)
|
||||||
|
if err != nil || uuid == "" {
|
||||||
|
return hash
|
||||||
|
}
|
||||||
|
return uuid
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetByCID resolves a bare CID to the UUID of its root. Returns ("", nil)
|
||||||
|
// when the CID is unknown; the empty string is the "not found" signal rather
|
||||||
|
// than a typed error so callers can branch without errors.As.
|
||||||
|
func (c *Client) GetByCID(cidStr string) (string, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return "", fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
if cidStr == "" {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
uuid, err := c.client.Get(ctx, c.buildCIDIndexKey(cidStr)).Result()
|
||||||
|
if err == redis.Nil {
|
||||||
|
return "", nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", fmt.Errorf("get cid index: %w", err)
|
||||||
|
}
|
||||||
|
return uuid, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// AddDuplicatePath records that path is another physical location whose
|
||||||
|
// content matches uuid. Returns true if path was newly added (not already
|
||||||
|
// in the set). The watcher calls this when it sees a file whose midhash
|
||||||
|
// matches an existing root but at a different filePath.
|
||||||
|
func (c *Client) AddDuplicatePath(uuid, path string) (bool, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
if uuid == "" || path == "" {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
added, err := c.client.SAdd(ctx, c.buildKeyPrefix(uuid)+DuplicatesField, path).Result()
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("sadd duplicates: %w", err)
|
||||||
|
}
|
||||||
|
return added > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// cidsForRootLocked scans the bare-CID key-set for a root and returns its
|
||||||
|
// members (bare CIDs, with the cids/ prefix stripped). Caller must hold c.mu.
|
||||||
|
func (c *Client) cidsForRootLocked(ctx context.Context, uuid string) ([]string, error) {
|
||||||
|
prefix := c.buildKeyPrefix(uuid) + CIDsKeyPrefix
|
||||||
|
var out []string
|
||||||
|
var cursor uint64
|
||||||
|
for {
|
||||||
|
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scan cids: %w", err)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
out = append(out, strings.TrimPrefix(k, prefix))
|
||||||
|
}
|
||||||
|
cursor = next
|
||||||
|
if cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteRoot removes everything keyed off uuid: every file:<uuid>/* property
|
||||||
|
// (including the cids/<cid> key-set), every reverse-index entry for those
|
||||||
|
// CIDs, and the index membership. Idempotent — a uuid with no remaining keys
|
||||||
|
// is a successful no-op.
|
||||||
|
func (c *Client) DeleteRoot(uuid string) error {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
if uuid == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
prefix := c.buildKeyPrefix(uuid)
|
||||||
|
|
||||||
|
// 1. Read the cids key-set first — once we delete file:<uuid>/* we lose
|
||||||
|
// the list of reverse-index entries to clean up.
|
||||||
|
cids, err := c.cidsForRootLocked(ctx, uuid)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
// 2. Delete every reverse-index entry in one shot.
|
||||||
|
if len(cids) > 0 {
|
||||||
|
keys := make([]string, 0, len(cids))
|
||||||
|
for _, cidStr := range cids {
|
||||||
|
keys = append(keys, c.buildCIDIndexKey(cidStr))
|
||||||
|
}
|
||||||
|
if err := c.client.Del(ctx, keys...).Err(); err != nil {
|
||||||
|
return fmt.Errorf("del cid index: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 3. Scan and delete every file:<uuid>/* key (cids/<cid> keys included).
|
||||||
|
var cursor uint64
|
||||||
|
for {
|
||||||
|
keys, next, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("scan root keys: %w", err)
|
||||||
|
}
|
||||||
|
if len(keys) > 0 {
|
||||||
|
if err := c.client.Del(ctx, keys...).Err(); err != nil {
|
||||||
|
return fmt.Errorf("del root keys: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = next
|
||||||
|
if cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// 4. Remove from the file index set.
|
||||||
|
if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
|
||||||
|
return fmt.Errorf("srem index: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeAddAliasFromFieldLocked inspects a (field, value) pair written into a
|
||||||
|
// root. If it's a key-set member — field name starts with "cids/" — it
|
||||||
|
// registers the implied CID in the reverse index. Writers emit cids/<cid>
|
||||||
|
// members without knowing about the reverse index; this hook keeps it in
|
||||||
|
// sync transparently.
|
||||||
|
//
|
||||||
|
// Caller must hold c.mu. Errors are intentionally swallowed at the call site
|
||||||
|
// (logged, not returned): an out-of-sync reverse index is recoverable via a
|
||||||
|
// sweep, while a failed property write is not.
|
||||||
|
func (c *Client) maybeAddAliasFromFieldLocked(ctx context.Context, uuid, field, value string) error {
|
||||||
|
cidStr := cidFromKeysetField(field)
|
||||||
|
if cidStr == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return c.addAliasLocked(ctx, uuid, cidStr)
|
||||||
|
}
|
||||||
|
|
||||||
|
// maybeAddAliasesFromMetadataLocked bulk-applies maybeAddAliasFromFieldLocked
|
||||||
|
// over every entry in a metadata map. Used by SetMetadataFlat /
|
||||||
|
// MergeMetadataFlat after the underlying pipeline write succeeds. Caller must
|
||||||
|
// hold c.mu.
|
||||||
|
func (c *Client) maybeAddAliasesFromMetadataLocked(ctx context.Context, uuid string, metadata map[string]string) {
|
||||||
|
for k := range metadata {
|
||||||
|
_ = c.maybeAddAliasFromFieldLocked(ctx, uuid, k, metadata[k])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// cidFromKeysetField returns the bare CID encoded in a cids/<cid> field name,
|
||||||
|
// or "" if the field isn't a key-set member.
|
||||||
|
func cidFromKeysetField(field string) string {
|
||||||
|
if !strings.HasPrefix(field, CIDsKeyPrefix) {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return strings.TrimPrefix(field, CIDsKeyPrefix)
|
||||||
|
}
|
||||||
|
|
||||||
|
// MetadataDocument is the complete state of one file as the public API
|
||||||
|
// returns it: every flat string field (the cids/<cid> members included),
|
||||||
|
// plus the CIDs hoisted into a typed array, the canonical CID derived by
|
||||||
|
// rank, and the duplicates set. Distinct from GetMetadataFlat (which only
|
||||||
|
// returns the raw string keys).
|
||||||
|
type MetadataDocument struct {
|
||||||
|
Flat map[string]string `json:"metadata"`
|
||||||
|
CIDs []string `json:"cids"`
|
||||||
|
Canonical string `json:"canonical_cid,omitempty"`
|
||||||
|
Duplicates []string `json:"duplicates"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetMetadataDocument returns the full document for a root: every string
|
||||||
|
// property under file:<uuid>/* (cids/<cid> members included in Flat), the
|
||||||
|
// CIDs hoisted into a typed array, the canonical CID computed by rank, and
|
||||||
|
// the duplicates SET. Returns nil if the root has no keys at all (deleted or
|
||||||
|
// never existed).
|
||||||
|
func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
if uuid == "" {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
flat, err := c.getMetadataFlatInternal(ctx, uuid)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
prefix := c.buildKeyPrefix(uuid)
|
||||||
|
|
||||||
|
// Hoist the cids/<cid> key-set members into a typed array and derive the
|
||||||
|
// canonical CID by rank. The members remain in Flat too — that's how
|
||||||
|
// flat-map consumers (meta-share ingest) see them.
|
||||||
|
cids := make([]string, 0)
|
||||||
|
canonical := ""
|
||||||
|
for k := range flat {
|
||||||
|
if !strings.HasPrefix(k, CIDsKeyPrefix) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
cidStr := strings.TrimPrefix(k, CIDsKeyPrefix)
|
||||||
|
cids = append(cids, cidStr)
|
||||||
|
if canonical == "" {
|
||||||
|
canonical = cidStr
|
||||||
|
} else {
|
||||||
|
canonical = cid.Better(canonical, cidStr)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
dups, err := c.client.SMembers(ctx, prefix+DuplicatesField).Result()
|
||||||
|
if err != nil && err != redis.Nil {
|
||||||
|
return nil, fmt.Errorf("smembers duplicates: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if len(flat) == 0 && len(cids) == 0 && len(dups) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
return &MetadataDocument{
|
||||||
|
Flat: flat,
|
||||||
|
CIDs: cids,
|
||||||
|
Canonical: canonical,
|
||||||
|
Duplicates: dups,
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
+608
-43
@@ -4,6 +4,8 @@ import (
|
|||||||
"context"
|
"context"
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"sort"
|
||||||
|
"strconv"
|
||||||
"strings"
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
@@ -190,6 +192,10 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
|
|||||||
return fmt.Errorf("sadd index failed: %w", err)
|
return fmt.Errorf("sadd index failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Auto-register reverse-index aliases for any cid_* or midhash256 fields
|
||||||
|
// in the payload. See cid_resolution.go for the rules.
|
||||||
|
c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -261,11 +267,17 @@ func (c *Client) SetProperty(hashID, property, value string) error {
|
|||||||
return fmt.Errorf("sadd index failed: %w", err)
|
return fmt.Errorf("sadd index failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Auto-register reverse-index alias if this is a CID field.
|
||||||
|
_ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value)
|
||||||
|
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DeleteMetadata deletes all metadata for a file
|
// DeleteMetadata deletes all metadata for a file
|
||||||
// Uses SCAN + DEL for flat keys: file:{hashId}/*
|
// Uses SCAN + DEL for flat keys: file:{hashId}/*
|
||||||
|
// Also unmaps every reverse-index entry registered for this root, so a
|
||||||
|
// subsequent GET /api/meta/<old-cid> correctly returns 404 instead of
|
||||||
|
// pointing at a UUID with no keys.
|
||||||
func (c *Client) DeleteMetadata(hashID string) (int64, error) {
|
func (c *Client) DeleteMetadata(hashID string) (int64, error) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
defer c.mu.RUnlock()
|
defer c.mu.RUnlock()
|
||||||
@@ -279,6 +291,22 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) {
|
|||||||
|
|
||||||
prefix := c.buildKeyPrefix(hashID)
|
prefix := c.buildKeyPrefix(hashID)
|
||||||
|
|
||||||
|
// Read the cids key-set first — once file:<uuid>/* is gone we've lost the
|
||||||
|
// list of reverse-index keys that need cleanup.
|
||||||
|
cids, err := c.cidsForRootLocked(ctx, hashID)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if len(cids) > 0 {
|
||||||
|
idxKeys := make([]string, 0, len(cids))
|
||||||
|
for _, cidStr := range cids {
|
||||||
|
idxKeys = append(idxKeys, c.buildCIDIndexKey(cidStr))
|
||||||
|
}
|
||||||
|
if err := c.client.Del(ctx, idxKeys...).Err(); err != nil {
|
||||||
|
return 0, fmt.Errorf("del cid index: %w", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
// Scan and delete all keys with this prefix
|
// Scan and delete all keys with this prefix
|
||||||
var deletedCount int64
|
var deletedCount int64
|
||||||
var cursor uint64
|
var cursor uint64
|
||||||
@@ -319,54 +347,74 @@ func (c *Client) CountFiles() (int, error) {
|
|||||||
return len(hashIDs), nil
|
return len(hashIDs), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// LookupPathByCID searches all file metadata for a matching CID
|
// LookupPathByCID resolves a bare CID to its file path on disk via the
|
||||||
// Returns the file path if found in poster/posterPath or backdrop/backdropPath
|
// reverse index. O(1) lookup followed by a single GET of file:<uuid>/filePath.
|
||||||
func (c *Client) LookupPathByCID(cid string) (string, error) {
|
//
|
||||||
c.mu.RLock()
|
// Any registered CID resolves — the midhash, a sibling sha2-256, an IPFS
|
||||||
defer c.mu.RUnlock()
|
// CID, a btih, etc. (they all alias to the same root). The bare CID is what
|
||||||
|
// the editor passes to /api/file/{cid}/info and what plugins write (e.g.
|
||||||
if c.client == nil {
|
// `backdrop=<bare-cid>`).
|
||||||
return "", fmt.Errorf("not connected")
|
//
|
||||||
}
|
// Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is
|
||||||
|
// resolved through this same path: the plugin output directory is a
|
||||||
if cid == "" {
|
// watcher root (see config.DefaultWatcherPaths), so every plugin artefact
|
||||||
return "", nil
|
// gets a midhash alias and resolves through the reverse index like any
|
||||||
}
|
// other file. There is no path-companion sidecar field.
|
||||||
|
func (c *Client) LookupPathByCID(cidStr string) (string, error) {
|
||||||
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
uuid, err := c.GetByCID(cidStr)
|
||||||
defer cancel()
|
|
||||||
|
|
||||||
// Get all hash IDs first
|
|
||||||
hashIDs, err := c.getAllHashIDsInternal(ctx)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return "", err
|
return "", err
|
||||||
}
|
}
|
||||||
|
if uuid == "" {
|
||||||
// For each hash, check poster and backdrop CIDs using flat keys
|
|
||||||
for _, hashID := range hashIDs {
|
|
||||||
prefix := c.buildKeyPrefix(hashID)
|
|
||||||
|
|
||||||
// Check poster
|
|
||||||
posterCID, err := c.client.Get(ctx, prefix+"poster").Result()
|
|
||||||
if err == nil && posterCID == cid {
|
|
||||||
posterPath, err := c.client.Get(ctx, prefix+"posterPath").Result()
|
|
||||||
if err == nil && posterPath != "" {
|
|
||||||
return posterPath, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Check backdrop
|
|
||||||
backdropCID, err := c.client.Get(ctx, prefix+"backdrop").Result()
|
|
||||||
if err == nil && backdropCID == cid {
|
|
||||||
backdropPath, err := c.client.Get(ctx, prefix+"backdropPath").Result()
|
|
||||||
if err == nil && backdropPath != "" {
|
|
||||||
return backdropPath, nil
|
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
return "", nil
|
return "", nil
|
||||||
}
|
}
|
||||||
|
fp, err := c.GetProperty(uuid, "filePath")
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return normalizeFilesRelativePath(fp), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can
|
||||||
|
// safely filepath.Join(FilesPath, …) without double-prefixing. Different
|
||||||
|
// plugins store paths inconsistently — poster/backdrop use FilesPath-relative
|
||||||
|
// ("/plugin/…") while filePath is FilesPath-absolute ("/files/watch/…").
|
||||||
|
func normalizeFilesRelativePath(p string) string {
|
||||||
|
return strings.TrimPrefix(p, "/files")
|
||||||
|
}
|
||||||
|
|
||||||
|
// getMetadataFlatInternal mirrors GetMetadataFlat without re-acquiring the lock.
|
||||||
|
// Callers must already hold c.mu.
|
||||||
|
func (c *Client) getMetadataFlatInternal(ctx context.Context, hashID string) (map[string]string, error) {
|
||||||
|
prefix := c.buildKeyPrefix(hashID)
|
||||||
|
result := make(map[string]string)
|
||||||
|
|
||||||
|
var cursor uint64
|
||||||
|
for {
|
||||||
|
keys, nextCursor, err := c.client.Scan(ctx, cursor, prefix+"*", 1000).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("scan failed: %w", err)
|
||||||
|
}
|
||||||
|
if len(keys) > 0 {
|
||||||
|
values, err := c.client.MGet(ctx, keys...).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("mget failed: %w", err)
|
||||||
|
}
|
||||||
|
for i, key := range keys {
|
||||||
|
if values[i] == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
field := strings.TrimPrefix(key, prefix)
|
||||||
|
result[field] = values[i].(string)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
cursor = nextCursor
|
||||||
|
if cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
// getAllHashIDsInternal is an internal version that doesn't acquire locks
|
// getAllHashIDsInternal is an internal version that doesn't acquire locks
|
||||||
// (caller must hold the lock)
|
// (caller must hold the lock)
|
||||||
@@ -413,6 +461,10 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
|
|||||||
return len(metadata), fmt.Errorf("sadd index failed: %w", err)
|
return len(metadata), fmt.Errorf("sadd index failed: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Auto-register reverse-index aliases for any cid_* or midhash256 fields
|
||||||
|
// in the payload. See cid_resolution.go for the rules.
|
||||||
|
c.maybeAddAliasesFromMetadataLocked(ctx, hashID, metadata)
|
||||||
|
|
||||||
return len(metadata), nil
|
return len(metadata), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -535,6 +587,102 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
|
|||||||
return true, c.client.Set(ctx, key, newValue, 0).Err()
|
return true, c.client.Set(ctx, key, newValue, 0).Err()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// FileTuple holds the (uuid, path, size, mtime, midhash) record for one
|
||||||
|
// file. Used by the watcher hydrator at startup to avoid re-mid-hashing
|
||||||
|
// files whose tuple matches what's already in Redis.
|
||||||
|
//
|
||||||
|
// HashID is the root identifier (UUIDv7 in the current model). Hydration
|
||||||
|
// uses HashID + (FilePath, Size, MtimeNano) to seed the in-memory state so
|
||||||
|
// the next scan can skip re-mid-hashing files whose size+mtime are unchanged.
|
||||||
|
type FileTuple struct {
|
||||||
|
HashID string
|
||||||
|
FilePath string
|
||||||
|
Size int64
|
||||||
|
MtimeNano int64
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetTuplesForAllFiles returns one FileTuple per indexed root, fetched in
|
||||||
|
// batched MGETs (no SCAN per root). Skips files missing any of filePath,
|
||||||
|
// sizeByte, or mtimeNano. Sub-second on local Redis for ~10K files.
|
||||||
|
//
|
||||||
|
// The midhash is intentionally NOT fetched here: it now lives only as a
|
||||||
|
// cids/<cid> key-set member (no named field), and the skip-rehash decision
|
||||||
|
// is purely size+mtime — so reading it would cost a SCAN per file for no
|
||||||
|
// benefit.
|
||||||
|
func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("smembers index: %w", err)
|
||||||
|
}
|
||||||
|
if len(hashIDs) == 0 {
|
||||||
|
return nil, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// Build the key list: 4 keys per root, in a fixed order so we can decode
|
||||||
|
// the MGET response by position. fields[i] for hashIDs[j] sits at
|
||||||
|
// keys[j*4+i].
|
||||||
|
const fieldsPerFile = 3
|
||||||
|
fields := [fieldsPerFile]string{"filePath", "sizeByte", "mtimeNano"}
|
||||||
|
keys := make([]string, 0, len(hashIDs)*fieldsPerFile)
|
||||||
|
for _, h := range hashIDs {
|
||||||
|
prefix := c.buildKeyPrefix(h)
|
||||||
|
for _, f := range fields {
|
||||||
|
keys = append(keys, prefix+f)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// Chunked MGET — Redis caps args per command; 1000 is a safe batch.
|
||||||
|
const batch = 1000
|
||||||
|
values := make([]interface{}, 0, len(keys))
|
||||||
|
for start := 0; start < len(keys); start += batch {
|
||||||
|
end := start + batch
|
||||||
|
if end > len(keys) {
|
||||||
|
end = len(keys)
|
||||||
|
}
|
||||||
|
chunk, err := c.client.MGet(ctx, keys[start:end]...).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("mget chunk [%d:%d]: %w", start, end, err)
|
||||||
|
}
|
||||||
|
values = append(values, chunk...)
|
||||||
|
}
|
||||||
|
|
||||||
|
out := make([]FileTuple, 0, len(hashIDs))
|
||||||
|
for i, h := range hashIDs {
|
||||||
|
base := i * fieldsPerFile
|
||||||
|
filePath, _ := values[base].(string)
|
||||||
|
sizeStr, _ := values[base+1].(string)
|
||||||
|
mtimeStr, _ := values[base+2].(string)
|
||||||
|
if filePath == "" || sizeStr == "" || mtimeStr == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
size, err := strconv.ParseInt(sizeStr, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
mtime, err := strconv.ParseInt(mtimeStr, 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
out = append(out, FileTuple{
|
||||||
|
HashID: h,
|
||||||
|
FilePath: filePath,
|
||||||
|
Size: size,
|
||||||
|
MtimeNano: mtime,
|
||||||
|
})
|
||||||
|
}
|
||||||
|
return out, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetMemoryInfo returns Redis memory usage information
|
// GetMemoryInfo returns Redis memory usage information
|
||||||
func (c *Client) GetMemoryInfo() (string, error) {
|
func (c *Client) GetMemoryInfo() (string, error) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
@@ -620,6 +768,378 @@ func (c *Client) ClearAllMetadata() (int64, error) {
|
|||||||
return deletedCount, nil
|
return deletedCount, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// ScanKeys lists the immediate children at the given prefix, S3-style.
|
||||||
|
//
|
||||||
|
// Walks the keyspace under prefix using Redis SCAN. When delimiter is set,
|
||||||
|
// keys with a delimiter occurrence after the prefix collapse into a branch
|
||||||
|
// path (the prefix up to and including that first delimiter); the rest are
|
||||||
|
// returned as leaves. When delimiter is empty all matching keys are leaves.
|
||||||
|
//
|
||||||
|
// Continues SCAN-ing until either the keyspace is exhausted (nextCursor == "0")
|
||||||
|
// or leaves+branches reaches maxResults — whichever comes first. The caller
|
||||||
|
// passes nextCursor back to continue. Results are deduped and sorted.
|
||||||
|
func (c *Client) ScanKeys(prefix, delimiter, cursorStr string, maxResults int) ([]string, []string, string, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, nil, "", fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
if maxResults <= 0 {
|
||||||
|
maxResults = 500
|
||||||
|
}
|
||||||
|
|
||||||
|
var cursor uint64
|
||||||
|
if cursorStr != "" {
|
||||||
|
if v, err := strconv.ParseUint(cursorStr, 10, 64); err == nil {
|
||||||
|
cursor = v
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
fullPrefix := c.buildKey(prefix)
|
||||||
|
pattern := escapeGlob(fullPrefix) + "*"
|
||||||
|
|
||||||
|
leafSet := make(map[string]struct{})
|
||||||
|
branchSet := make(map[string]struct{})
|
||||||
|
|
||||||
|
for {
|
||||||
|
// SCAN COUNT is a hint; ask for a larger batch when grouping by
|
||||||
|
// delimiter since many keys collapse into one branch.
|
||||||
|
scanCount := int64(maxResults * 4)
|
||||||
|
if scanCount > 5000 {
|
||||||
|
scanCount = 5000
|
||||||
|
}
|
||||||
|
keys, next, err := c.client.Scan(ctx, cursor, pattern, scanCount).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, nil, "", fmt.Errorf("scan failed: %w", err)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
userKey := strings.TrimPrefix(k, c.prefix)
|
||||||
|
suffix := strings.TrimPrefix(userKey, prefix)
|
||||||
|
if delimiter != "" {
|
||||||
|
if idx := strings.Index(suffix, delimiter); idx >= 0 {
|
||||||
|
branchSet[prefix+suffix[:idx+len(delimiter)]] = struct{}{}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
}
|
||||||
|
leafSet[userKey] = struct{}{}
|
||||||
|
}
|
||||||
|
cursor = next
|
||||||
|
if cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if len(leafSet)+len(branchSet) >= maxResults {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
leaves := make([]string, 0, len(leafSet))
|
||||||
|
for k := range leafSet {
|
||||||
|
leaves = append(leaves, k)
|
||||||
|
}
|
||||||
|
branches := make([]string, 0, len(branchSet))
|
||||||
|
for k := range branchSet {
|
||||||
|
branches = append(branches, k)
|
||||||
|
}
|
||||||
|
sort.Strings(leaves)
|
||||||
|
sort.Strings(branches)
|
||||||
|
return leaves, branches, strconv.FormatUint(cursor, 10), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindMatch is a single value-search hit.
|
||||||
|
type FindMatch struct {
|
||||||
|
Key string `json:"key"`
|
||||||
|
Value string `json:"value"`
|
||||||
|
EntryPath string `json:"entryPath"`
|
||||||
|
Field string `json:"field"`
|
||||||
|
}
|
||||||
|
|
||||||
|
// FindByValue scans every key whose path ends with /<field> for each field
|
||||||
|
// in fields, MGETs them, and returns the keys whose value contains the
|
||||||
|
// substring (case-insensitive). Returns truncated when limit was hit before
|
||||||
|
// the scan exhausted.
|
||||||
|
//
|
||||||
|
// EntryPath is the parent prefix up to (and including) the slash before
|
||||||
|
// field — i.e. the file branch path. Useful for revealing the matched entry
|
||||||
|
// in the tree.
|
||||||
|
func (c *Client) FindByValue(contains string, fields []string, limit int) ([]FindMatch, bool, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
if contains == "" || len(fields) == 0 {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 200
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 60*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
needle := strings.ToLower(contains)
|
||||||
|
var matches []FindMatch
|
||||||
|
truncated := false
|
||||||
|
|
||||||
|
for _, field := range fields {
|
||||||
|
field = strings.TrimSpace(field)
|
||||||
|
if field == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if len(matches) >= limit {
|
||||||
|
truncated = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
// Walk all keys ending with /field via SCAN MATCH "*/<field>".
|
||||||
|
pattern := "*/" + escapeGlob(field)
|
||||||
|
if c.prefix != "" {
|
||||||
|
pattern = escapeGlob(c.prefix) + pattern
|
||||||
|
}
|
||||||
|
var keysToFetch []string
|
||||||
|
var cursor uint64
|
||||||
|
for {
|
||||||
|
keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("scan failed: %w", err)
|
||||||
|
}
|
||||||
|
keysToFetch = append(keysToFetch, keys...)
|
||||||
|
cursor = next
|
||||||
|
if cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
if len(keysToFetch) > 20000 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
// MGET in chunks (Redis limits the number of args per command).
|
||||||
|
for chunkStart := 0; chunkStart < len(keysToFetch); chunkStart += 500 {
|
||||||
|
chunkEnd := chunkStart + 500
|
||||||
|
if chunkEnd > len(keysToFetch) {
|
||||||
|
chunkEnd = len(keysToFetch)
|
||||||
|
}
|
||||||
|
chunk := keysToFetch[chunkStart:chunkEnd]
|
||||||
|
values, err := c.client.MGet(ctx, chunk...).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("mget failed: %w", err)
|
||||||
|
}
|
||||||
|
for i, v := range values {
|
||||||
|
if v == nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
sv, _ := v.(string)
|
||||||
|
if !strings.Contains(strings.ToLower(sv), needle) {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
userKey := strings.TrimPrefix(chunk[i], c.prefix)
|
||||||
|
entryPath := strings.TrimSuffix(userKey, field)
|
||||||
|
matches = append(matches, FindMatch{
|
||||||
|
Key: userKey,
|
||||||
|
Value: sv,
|
||||||
|
EntryPath: entryPath,
|
||||||
|
Field: field,
|
||||||
|
})
|
||||||
|
if len(matches) >= limit {
|
||||||
|
truncated = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if truncated {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
sort.Slice(matches, func(i, j int) bool { return matches[i].Key < matches[j].Key })
|
||||||
|
return matches, truncated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SearchKeys returns up to limit keys whose full path contains the substring.
|
||||||
|
// truncated is true when the search stopped at limit before exhausting Redis.
|
||||||
|
func (c *Client) SearchKeys(contains string, limit int) ([]string, bool, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
if contains == "" {
|
||||||
|
return nil, false, nil
|
||||||
|
}
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 500
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
pattern := escapeGlob(c.prefix) + "*" + escapeGlob(contains) + "*"
|
||||||
|
|
||||||
|
results := make([]string, 0, limit)
|
||||||
|
truncated := false
|
||||||
|
var cursor uint64
|
||||||
|
for {
|
||||||
|
keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
|
||||||
|
if err != nil {
|
||||||
|
return nil, false, fmt.Errorf("scan failed: %w", err)
|
||||||
|
}
|
||||||
|
for _, k := range keys {
|
||||||
|
if len(results) >= limit {
|
||||||
|
truncated = true
|
||||||
|
break
|
||||||
|
}
|
||||||
|
results = append(results, strings.TrimPrefix(k, c.prefix))
|
||||||
|
}
|
||||||
|
cursor = next
|
||||||
|
if truncated || cursor == 0 {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
sort.Strings(results)
|
||||||
|
return results, truncated, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// GetKeyInfo returns the Redis type ("string", "set", "hash", ...) and, for
|
||||||
|
// string-typed keys, the value. exists is false when the key is absent.
|
||||||
|
func (c *Client) GetKeyInfo(key string) (kind string, value string, exists bool, err error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return "", "", false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
full := c.buildKey(key)
|
||||||
|
t, err := c.client.Type(ctx, full).Result()
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, fmt.Errorf("type failed: %w", err)
|
||||||
|
}
|
||||||
|
if t == "none" {
|
||||||
|
return "", "", false, nil
|
||||||
|
}
|
||||||
|
if t != "string" {
|
||||||
|
return t, "", true, nil
|
||||||
|
}
|
||||||
|
v, err := c.client.Get(ctx, full).Result()
|
||||||
|
if err == redis.Nil {
|
||||||
|
return "", "", false, nil
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return "", "", false, fmt.Errorf("get failed: %w", err)
|
||||||
|
}
|
||||||
|
return "string", v, true, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// SetRawValue writes a string value at the exact key. Refuses to overwrite a
|
||||||
|
// key whose existing Redis type is not "string".
|
||||||
|
func (c *Client) SetRawValue(key, value string) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
full := c.buildKey(key)
|
||||||
|
t, err := c.client.Type(ctx, full).Result()
|
||||||
|
if err != nil {
|
||||||
|
return fmt.Errorf("type failed: %w", err)
|
||||||
|
}
|
||||||
|
if t != "none" && t != "string" {
|
||||||
|
return fmt.Errorf("key has incompatible type %q (only string is editable)", t)
|
||||||
|
}
|
||||||
|
return c.client.Set(ctx, full, value, 0).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// DeleteRawKey deletes the exact key. Returns true if the key existed.
|
||||||
|
func (c *Client) DeleteRawKey(key string) (bool, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
n, err := c.client.Del(ctx, c.buildKey(key)).Result()
|
||||||
|
if err != nil {
|
||||||
|
return false, err
|
||||||
|
}
|
||||||
|
return n > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexAdd adds a hash ID to the file index set. Idempotent.
|
||||||
|
func (c *Client) IndexAdd(hashID string) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
return c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err()
|
||||||
|
}
|
||||||
|
|
||||||
|
// IndexRemoveIfEmpty removes hashID from the index set when no
|
||||||
|
// file:<hashID>/* keys remain. Returns true if the index entry was removed.
|
||||||
|
func (c *Client) IndexRemoveIfEmpty(hashID string) (bool, error) {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return false, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
prefix := c.buildKeyPrefix(hashID)
|
||||||
|
keys, _, err := c.client.Scan(ctx, 0, prefix+"*", 1).Result()
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("scan failed: %w", err)
|
||||||
|
}
|
||||||
|
if len(keys) > 0 {
|
||||||
|
return false, nil
|
||||||
|
}
|
||||||
|
removed, err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Result()
|
||||||
|
if err != nil {
|
||||||
|
return false, fmt.Errorf("srem failed: %w", err)
|
||||||
|
}
|
||||||
|
return removed > 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// escapeGlob escapes Redis SCAN MATCH glob metacharacters so a literal string
|
||||||
|
// can be embedded inside a pattern. Redis uses *, ?, [, \ as metacharacters.
|
||||||
|
func escapeGlob(s string) string {
|
||||||
|
var b strings.Builder
|
||||||
|
b.Grow(len(s))
|
||||||
|
for _, r := range s {
|
||||||
|
switch r {
|
||||||
|
case '*', '?', '[', ']', '\\':
|
||||||
|
b.WriteByte('\\')
|
||||||
|
}
|
||||||
|
b.WriteRune(r)
|
||||||
|
}
|
||||||
|
return b.String()
|
||||||
|
}
|
||||||
|
|
||||||
// GetRedisClient returns the underlying redis client for advanced operations
|
// GetRedisClient returns the underlying redis client for advanced operations
|
||||||
func (c *Client) GetRedisClient() *redis.Client {
|
func (c *Client) GetRedisClient() *redis.Client {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
@@ -685,6 +1205,51 @@ func (c *Client) ClearStream(stream string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// StreamEntry represents one entry from a Redis stream.
|
||||||
|
type StreamEntry struct {
|
||||||
|
ID string
|
||||||
|
Values map[string]interface{}
|
||||||
|
}
|
||||||
|
|
||||||
|
// XRevRange reads stream entries from newest to oldest.
|
||||||
|
// start defaults to "+" (newest) and stop defaults to "-" (oldest) when empty.
|
||||||
|
// If count > 0, at most that many entries are returned.
|
||||||
|
func (c *Client) XRevRange(stream, start, stop string, count int64) ([]StreamEntry, error) {
|
||||||
|
c.mu.RLock()
|
||||||
|
defer c.mu.RUnlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return nil, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
if start == "" {
|
||||||
|
start = "+"
|
||||||
|
}
|
||||||
|
if stop == "" {
|
||||||
|
stop = "-"
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
var msgs []redis.XMessage
|
||||||
|
var err error
|
||||||
|
if count > 0 {
|
||||||
|
msgs, err = c.client.XRevRangeN(ctx, stream, start, stop, count).Result()
|
||||||
|
} else {
|
||||||
|
msgs, err = c.client.XRevRange(ctx, stream, start, stop).Result()
|
||||||
|
}
|
||||||
|
if err != nil {
|
||||||
|
return nil, fmt.Errorf("xrevrange failed: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries := make([]StreamEntry, 0, len(msgs))
|
||||||
|
for _, m := range msgs {
|
||||||
|
entries = append(entries, StreamEntry{ID: m.ID, Values: m.Values})
|
||||||
|
}
|
||||||
|
return entries, nil
|
||||||
|
}
|
||||||
|
|
||||||
// GetStreamLength returns the length of a Redis stream
|
// GetStreamLength returns the length of a Redis stream
|
||||||
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
func (c *Client) GetStreamLength(stream string) (int64, error) {
|
||||||
c.mu.RLock()
|
c.mu.RLock()
|
||||||
|
|||||||
@@ -0,0 +1,41 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
)
|
||||||
|
|
||||||
|
// MigrateDualRoots was a one-shot repair for the "dual-root" pattern in the
|
||||||
|
// legacy storage model: /meta/{hash} write handlers that didn't resolve
|
||||||
|
// CID → UUID first created a parallel midhash-rooted entry alongside the
|
||||||
|
// watcher's UUID root.
|
||||||
|
//
|
||||||
|
// That pattern is now structurally impossible:
|
||||||
|
// - writes resolve via ResolveRoot before landing (see handlers), and
|
||||||
|
// - CIDs are stored as a bare-CID key-set (cids/<cid>) keyed off opaque
|
||||||
|
// UUID roots, with the reverse index cid:<cid> → uuid as the single
|
||||||
|
// resolution path.
|
||||||
|
//
|
||||||
|
// The legacy on-disk shape this migration targeted (midhash-rooted entries,
|
||||||
|
// <algo>:<value> tokens, named midhash256 fields) no longer exists, and the
|
||||||
|
// cut-over wipes Redis, so there is nothing to migrate. The function is kept
|
||||||
|
// as a no-op so the admin endpoint and its wrappers continue to compile and
|
||||||
|
// respond.
|
||||||
|
func (c *Client) MigrateDualRoots(ctx context.Context) (int, error) {
|
||||||
|
if c.client == nil {
|
||||||
|
return 0, fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// MigrateDualRootsWithTimeout is a convenience wrapper used by the API
|
||||||
|
// handler and the startup hook.
|
||||||
|
func (c *Client) MigrateDualRootsWithTimeout() (int, error) {
|
||||||
|
return c.MigrateDualRoots(context.Background())
|
||||||
|
}
|
||||||
|
|
||||||
|
// MigrateDualRootsReport returns a human-readable summary. With the dual-root
|
||||||
|
// pattern retired (see MigrateDualRoots), this always reports nothing to do.
|
||||||
|
func (c *Client) MigrateDualRootsReport(ctx context.Context) (string, int, error) {
|
||||||
|
return "dual-root migration is obsolete under the bare-CID key-set model; nothing to do", 0, nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,83 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/redis/go-redis/v9"
|
||||||
|
)
|
||||||
|
|
||||||
|
// SchemaVersion is the Redis-layout version this build of meta-core expects.
|
||||||
|
// Distinct from the snapshot manifest version (which is on-disk format);
|
||||||
|
// this number describes the in-memory Redis schema — primarily, "are roots
|
||||||
|
// UUIDs or content-hash tokens, and do reverse-index keys cid:* exist?"
|
||||||
|
//
|
||||||
|
// Bump only when a Redis-layout change can't be detected from the data
|
||||||
|
// itself or healed by the auto-alias hooks. Each bump is a flag day in
|
||||||
|
// alpha: operators wipe and restart.
|
||||||
|
const SchemaVersion = 2
|
||||||
|
|
||||||
|
// SchemaVersionKey is the sentinel under which we record SchemaVersion in
|
||||||
|
// Redis. Checked on every meta-core boot.
|
||||||
|
const SchemaVersionKey = "meta-core:schema-version"
|
||||||
|
|
||||||
|
// EnsureSchemaVersion verifies the Redis store matches the expected schema
|
||||||
|
// version. Called once at startup, before any other reads/writes.
|
||||||
|
//
|
||||||
|
// Behavior:
|
||||||
|
// - sentinel present and matches expected → OK
|
||||||
|
// - sentinel missing AND file:__index__ empty → fresh store, stamp sentinel
|
||||||
|
// - sentinel missing AND file:__index__ non-empty → legacy data, refuse
|
||||||
|
// - sentinel present but mismatched → refuse, point operator at clean:all
|
||||||
|
//
|
||||||
|
// Refusal is hard-fail; the caller is expected to log.Fatalf and exit.
|
||||||
|
// The error message names the wipe remediation because in alpha there
|
||||||
|
// is no automated migration — operators bring up a clean store.
|
||||||
|
func (c *Client) EnsureSchemaVersion(expected int) error {
|
||||||
|
c.mu.Lock()
|
||||||
|
defer c.mu.Unlock()
|
||||||
|
|
||||||
|
if c.client == nil {
|
||||||
|
return fmt.Errorf("not connected")
|
||||||
|
}
|
||||||
|
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second)
|
||||||
|
defer cancel()
|
||||||
|
|
||||||
|
key := c.buildKey(SchemaVersionKey)
|
||||||
|
have, err := c.client.Get(ctx, key).Result()
|
||||||
|
if err != nil && err != redis.Nil {
|
||||||
|
return fmt.Errorf("get schema version: %w", err)
|
||||||
|
}
|
||||||
|
|
||||||
|
if err == redis.Nil {
|
||||||
|
// Sentinel absent. Fresh store iff there's no file:__index__ membership.
|
||||||
|
n, sErr := c.client.SCard(ctx, c.buildIndexKey()).Result()
|
||||||
|
if sErr != nil && sErr != redis.Nil {
|
||||||
|
return fmt.Errorf("scard file:__index__: %w", sErr)
|
||||||
|
}
|
||||||
|
if n > 0 {
|
||||||
|
return fmt.Errorf(
|
||||||
|
"redis has %d indexed roots but no %q sentinel — this is alpha "+
|
||||||
|
"data from before the UUID-rooted schema landed. Run "+
|
||||||
|
"`pnpm run clean:all` (or `docker exec metacore-app redis-cli FLUSHDB`) "+
|
||||||
|
"to wipe and restart, or downgrade meta-core to a compatible build",
|
||||||
|
n, SchemaVersionKey)
|
||||||
|
}
|
||||||
|
if err := c.client.Set(ctx, key, strconv.Itoa(expected), 0).Err(); err != nil {
|
||||||
|
return fmt.Errorf("set schema version: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if have == strconv.Itoa(expected) {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return fmt.Errorf(
|
||||||
|
"redis schema version is %q, this build of meta-core expects %d. "+
|
||||||
|
"Run `pnpm run clean:all` to wipe and restart, or use a meta-core "+
|
||||||
|
"build that matches the existing data",
|
||||||
|
have, expected)
|
||||||
|
}
|
||||||
@@ -0,0 +1,62 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/google/uuid"
|
||||||
|
)
|
||||||
|
|
||||||
|
// NewUUID returns a new UUIDv7 encoded as 26-char Crockford Base32 (ULID
|
||||||
|
// layout). Used as the root key for file metadata entries.
|
||||||
|
//
|
||||||
|
// Why this format over canonical 8-4-4-4-12 hex:
|
||||||
|
// - 26 chars vs 36 — shorter Redis keys, less prefix space waste
|
||||||
|
// - Time-sortable prefix — KEYS file:01JKR* returns entries minted in a
|
||||||
|
// given time window, restoring some of the debug-ability lost vs the
|
||||||
|
// previous midhash256:abc rooting
|
||||||
|
// - Case-insensitive — no ambiguity when copying from logs
|
||||||
|
// - Crockford alphabet skips I/L/O/U, so eyeballed inspection of a key
|
||||||
|
// can't be confused for an ambient digit/letter typo
|
||||||
|
func NewUUID() (string, error) {
|
||||||
|
u, err := uuid.NewV7()
|
||||||
|
if err != nil {
|
||||||
|
return "", err
|
||||||
|
}
|
||||||
|
return encodeCrockford(u[:]), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
const crockfordAlphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||||
|
|
||||||
|
// encodeCrockford encodes a 16-byte value as 26 Crockford Base32 chars.
|
||||||
|
// 128 bits packs into 130 bits (26 × 5); the top 2 bits of the first char
|
||||||
|
// carry the high bits of byte 0, the remaining 3 are implicit zero. This
|
||||||
|
// matches the ULID encoding so the output is interchangeable with ULIDs in
|
||||||
|
// any tool that accepts them.
|
||||||
|
func encodeCrockford(b []byte) string {
|
||||||
|
out := make([]byte, 26)
|
||||||
|
out[0] = crockfordAlphabet[(b[0]&0xE0)>>5]
|
||||||
|
out[1] = crockfordAlphabet[b[0]&0x1F]
|
||||||
|
out[2] = crockfordAlphabet[(b[1]&0xF8)>>3]
|
||||||
|
out[3] = crockfordAlphabet[((b[1]&0x07)<<2)|((b[2]&0xC0)>>6)]
|
||||||
|
out[4] = crockfordAlphabet[(b[2]&0x3E)>>1]
|
||||||
|
out[5] = crockfordAlphabet[((b[2]&0x01)<<4)|((b[3]&0xF0)>>4)]
|
||||||
|
out[6] = crockfordAlphabet[((b[3]&0x0F)<<1)|((b[4]&0x80)>>7)]
|
||||||
|
out[7] = crockfordAlphabet[(b[4]&0x7C)>>2]
|
||||||
|
out[8] = crockfordAlphabet[((b[4]&0x03)<<3)|((b[5]&0xE0)>>5)]
|
||||||
|
out[9] = crockfordAlphabet[b[5]&0x1F]
|
||||||
|
out[10] = crockfordAlphabet[(b[6]&0xF8)>>3]
|
||||||
|
out[11] = crockfordAlphabet[((b[6]&0x07)<<2)|((b[7]&0xC0)>>6)]
|
||||||
|
out[12] = crockfordAlphabet[(b[7]&0x3E)>>1]
|
||||||
|
out[13] = crockfordAlphabet[((b[7]&0x01)<<4)|((b[8]&0xF0)>>4)]
|
||||||
|
out[14] = crockfordAlphabet[((b[8]&0x0F)<<1)|((b[9]&0x80)>>7)]
|
||||||
|
out[15] = crockfordAlphabet[(b[9]&0x7C)>>2]
|
||||||
|
out[16] = crockfordAlphabet[((b[9]&0x03)<<3)|((b[10]&0xE0)>>5)]
|
||||||
|
out[17] = crockfordAlphabet[b[10]&0x1F]
|
||||||
|
out[18] = crockfordAlphabet[(b[11]&0xF8)>>3]
|
||||||
|
out[19] = crockfordAlphabet[((b[11]&0x07)<<2)|((b[12]&0xC0)>>6)]
|
||||||
|
out[20] = crockfordAlphabet[(b[12]&0x3E)>>1]
|
||||||
|
out[21] = crockfordAlphabet[((b[12]&0x01)<<4)|((b[13]&0xF0)>>4)]
|
||||||
|
out[22] = crockfordAlphabet[((b[13]&0x0F)<<1)|((b[14]&0x80)>>7)]
|
||||||
|
out[23] = crockfordAlphabet[(b[14]&0x7C)>>2]
|
||||||
|
out[24] = crockfordAlphabet[((b[14]&0x03)<<3)|((b[15]&0xE0)>>5)]
|
||||||
|
out[25] = crockfordAlphabet[b[15]&0x1F]
|
||||||
|
return string(out)
|
||||||
|
}
|
||||||
@@ -0,0 +1,74 @@
|
|||||||
|
package storage
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNewUUID_Shape(t *testing.T) {
|
||||||
|
id, err := NewUUID()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUUID: %v", err)
|
||||||
|
}
|
||||||
|
if len(id) != 26 {
|
||||||
|
t.Errorf("expected 26 chars, got %d (%q)", len(id), id)
|
||||||
|
}
|
||||||
|
const alphabet = "0123456789ABCDEFGHJKMNPQRSTVWXYZ"
|
||||||
|
for i, c := range id {
|
||||||
|
if !strings.ContainsRune(alphabet, c) {
|
||||||
|
t.Errorf("char %d (%q) not in Crockford alphabet (uuid=%q)", i, c, id)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewUUID_Unique(t *testing.T) {
|
||||||
|
const n = 1000
|
||||||
|
seen := make(map[string]struct{}, n)
|
||||||
|
for i := 0; i < n; i++ {
|
||||||
|
id, err := NewUUID()
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("NewUUID at i=%d: %v", i, err)
|
||||||
|
}
|
||||||
|
if _, dup := seen[id]; dup {
|
||||||
|
t.Fatalf("collision after %d mints: %q", i+1, id)
|
||||||
|
}
|
||||||
|
seen[id] = struct{}{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNewUUID_TimeSortable(t *testing.T) {
|
||||||
|
// UUIDv7 has a millisecond timestamp prefix; consecutive mints in the
|
||||||
|
// same millisecond may not be strictly ordered (random tail), but
|
||||||
|
// mints across milliseconds are. Just check that an early ID
|
||||||
|
// lexicographically sorts before a later one when there's a real
|
||||||
|
// time gap. Doesn't test the no-gap case — the v7 spec only
|
||||||
|
// guarantees ordering across millisecond boundaries.
|
||||||
|
a, _ := NewUUID()
|
||||||
|
time.Sleep(2 * time.Millisecond)
|
||||||
|
b, _ := NewUUID()
|
||||||
|
if a >= b {
|
||||||
|
t.Errorf("expected %q < %q (UUIDv7 should be roughly time-sortable)", a, b)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestCIDFromKeysetField(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
field string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"cids/bafkmidhash", "bafkmidhash"},
|
||||||
|
{"cids/baejbeisha256", "baejbeisha256"},
|
||||||
|
{"cids/bafyipfs", "bafyipfs"},
|
||||||
|
{"title", ""}, // non-CID field
|
||||||
|
{"tmdb/title", ""}, // non-CID nested
|
||||||
|
{"cid_sha2-256", ""}, // legacy flat field — no longer recognized
|
||||||
|
{"midhash256", ""}, // legacy named field — no longer recognized
|
||||||
|
{"cids", ""}, // prefix without a member
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := cidFromKeysetField(tc.field); got != tc.want {
|
||||||
|
t.Errorf("cidFromKeysetField(%q) = %q, want %q", tc.field, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -3,6 +3,8 @@ package watcher
|
|||||||
import (
|
import (
|
||||||
"fmt"
|
"fmt"
|
||||||
"log"
|
"log"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
"sync"
|
"sync"
|
||||||
|
|
||||||
"github.com/metazla/meta-core/internal/storage"
|
"github.com/metazla/meta-core/internal/storage"
|
||||||
@@ -11,6 +13,11 @@ import (
|
|||||||
const (
|
const (
|
||||||
// EventsStream is the Redis Stream name for file events
|
// EventsStream is the Redis Stream name for file events
|
||||||
EventsStream = "file:events"
|
EventsStream = "file:events"
|
||||||
|
|
||||||
|
// EventsStreamMaxLen is the approximate cap on stream length. Bounded
|
||||||
|
// retention is a contract requirement for the SSE wire — see
|
||||||
|
// docs/api-mediated-access.md "Backing-store retention and lifecycle".
|
||||||
|
EventsStreamMaxLen = 100_000
|
||||||
)
|
)
|
||||||
|
|
||||||
// Dispatcher sends file events to Redis Stream
|
// Dispatcher sends file events to Redis Stream
|
||||||
@@ -68,8 +75,10 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
|
|||||||
fields["watcherId"] = event.WatcherID
|
fields["watcherId"] = event.WatcherID
|
||||||
}
|
}
|
||||||
|
|
||||||
// Publish to stream (no MaxLen - stream is cleared on reset)
|
// Approximate MAXLEN trim keeps the stream bounded. The deliberate
|
||||||
_, err := client.XAdd(EventsStream, 0, fields)
|
// `EmitReset → ClearStream` path is still the only way the stream gets
|
||||||
|
// truncated to zero (signalled to consumers via the `reset` event).
|
||||||
|
_, err := client.XAdd(EventsStream, EventsStreamMaxLen, fields)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
|
log.Printf("[Dispatcher] Failed to publish to stream: %v", err)
|
||||||
}
|
}
|
||||||
@@ -108,6 +117,85 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// GetRecentEvents reads recent events from the Redis stream, newest first.
|
||||||
|
// If sinceMS > 0, only events strictly newer than that timestamp are returned.
|
||||||
|
// limit caps the number of returned events (defaults to 100 when <= 0).
|
||||||
|
func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
|
||||||
|
d.mu.RLock()
|
||||||
|
client := d.storageClient
|
||||||
|
d.mu.RUnlock()
|
||||||
|
|
||||||
|
if client == nil || !client.IsConnected() {
|
||||||
|
return []FileEvent{}, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit <= 0 {
|
||||||
|
limit = 100
|
||||||
|
}
|
||||||
|
|
||||||
|
start := "+"
|
||||||
|
stop := "-"
|
||||||
|
if sinceMS > 0 {
|
||||||
|
// Exclusive lower bound: skip entries at or before sinceMS.
|
||||||
|
stop = fmt.Sprintf("(%d-0", sinceMS)
|
||||||
|
}
|
||||||
|
|
||||||
|
entries, err := client.XRevRange(EventsStream, start, stop, int64(limit))
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
|
||||||
|
events := make([]FileEvent, 0, len(entries))
|
||||||
|
for _, entry := range entries {
|
||||||
|
events = append(events, parseStreamEntry(entry))
|
||||||
|
}
|
||||||
|
return events, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// parseStreamEntry converts a Redis stream entry back into a FileEvent.
|
||||||
|
// All fields are stored as strings by XADD, so numerics need parsing.
|
||||||
|
func parseStreamEntry(entry storage.StreamEntry) FileEvent {
|
||||||
|
event := FileEvent{}
|
||||||
|
|
||||||
|
if v, ok := entry.Values["type"].(string); ok {
|
||||||
|
event.Type = FileEventType(v)
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["path"].(string); ok {
|
||||||
|
event.Path = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["timestamp"].(string); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
event.Timestamp = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["size"].(string); ok {
|
||||||
|
if n, err := strconv.ParseInt(v, 10, 64); err == nil {
|
||||||
|
event.Size = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["midhash256"].(string); ok {
|
||||||
|
event.MidHash256 = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["oldPath"].(string); ok {
|
||||||
|
event.OldPath = v
|
||||||
|
}
|
||||||
|
if v, ok := entry.Values["watcherId"].(string); ok {
|
||||||
|
event.WatcherID = v
|
||||||
|
}
|
||||||
|
|
||||||
|
// Fall back to the stream ID's millisecond prefix if the timestamp
|
||||||
|
// field is missing or unparseable — every Redis stream ID starts with ms.
|
||||||
|
if event.Timestamp == 0 && entry.ID != "" {
|
||||||
|
if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
|
||||||
|
if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {
|
||||||
|
event.Timestamp = n
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
return event
|
||||||
|
}
|
||||||
|
|
||||||
// GetStreamLength returns the current stream length
|
// GetStreamLength returns the current stream length
|
||||||
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
func (d *Dispatcher) GetStreamLength() (int64, error) {
|
||||||
d.mu.RLock()
|
d.mu.RLock()
|
||||||
|
|||||||
@@ -2,6 +2,7 @@ package watcher
|
|||||||
|
|
||||||
import (
|
import (
|
||||||
"encoding/json"
|
"encoding/json"
|
||||||
|
"log"
|
||||||
"net/http"
|
"net/http"
|
||||||
"strconv"
|
"strconv"
|
||||||
|
|
||||||
@@ -32,8 +33,16 @@ func (h *Handlers) RegisterRoutes(r *mux.Router) {
|
|||||||
r.HandleFunc("/api/scan/status", h.handleScanStatusDeprecated).Methods("GET")
|
r.HandleFunc("/api/scan/status", h.handleScanStatusDeprecated).Methods("GET")
|
||||||
}
|
}
|
||||||
|
|
||||||
// handlePoll handles GET /api/events/poll
|
// handlePoll handles GET /api/events/poll.
|
||||||
|
//
|
||||||
|
// DEPRECATED. The long-poll variant over file:events has been superseded by
|
||||||
|
// the SSE endpoint at /api/events/files. New consumers should use SSE. This
|
||||||
|
// handler stays for one release with a Deprecation header before removal —
|
||||||
|
// see docs/api-mediated-access.md PR A scope.
|
||||||
func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
||||||
|
w.Header().Set("Deprecation", "true")
|
||||||
|
w.Header().Set("Link", `</api/events/files>; rel="successor-version"`)
|
||||||
|
w.Header().Set("Sunset", "after next release")
|
||||||
// Get since parameter
|
// Get since parameter
|
||||||
sinceStr := r.URL.Query().Get("since")
|
sinceStr := r.URL.Query().Get("since")
|
||||||
sinceMS := int64(0)
|
sinceMS := int64(0)
|
||||||
@@ -52,7 +61,14 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
events := h.watcher.GetRecentEvents(sinceMS, limit)
|
// Read events from the Redis `file:events` stream rather than the
|
||||||
|
// in-memory buffer. The buffer is fed by the unused debouncer path,
|
||||||
|
// so it is always empty in practice — the stream is the source of truth.
|
||||||
|
events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)
|
||||||
|
if err != nil {
|
||||||
|
log.Printf("[watcher.handlePoll] failed to read stream: %v", err)
|
||||||
|
events = []FileEvent{}
|
||||||
|
}
|
||||||
|
|
||||||
writeJSON(w, http.StatusOK, EventsListResponse{
|
writeJSON(w, http.StatusOK, EventsListResponse{
|
||||||
Events: events,
|
Events: events,
|
||||||
@@ -84,9 +100,31 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) {
|
|||||||
json.NewEncoder(w).Encode(data)
|
json.NewEncoder(w).Encode(data)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// writeError writes the typed error envelope (see api/handlers.go for the
|
||||||
|
// canonical definition + slug vocabulary). Kept local to avoid an import
|
||||||
|
// cycle with the api package.
|
||||||
func writeError(w http.ResponseWriter, status int, message string) {
|
func writeError(w http.ResponseWriter, status int, message string) {
|
||||||
writeJSON(w, status, map[string]string{
|
slug, retryable := slugForStatus(status)
|
||||||
"error": http.StatusText(status),
|
writeJSON(w, status, map[string]interface{}{
|
||||||
|
"error": slug,
|
||||||
"message": message,
|
"message": message,
|
||||||
|
"retryable": retryable,
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func slugForStatus(status int) (string, bool) {
|
||||||
|
switch status {
|
||||||
|
case http.StatusConflict:
|
||||||
|
return "alias_collision", false
|
||||||
|
case http.StatusNotFound:
|
||||||
|
return "unknown_root", false
|
||||||
|
case http.StatusBadRequest:
|
||||||
|
return "schema_violation", false
|
||||||
|
case http.StatusServiceUnavailable:
|
||||||
|
return "storage_unavailable", true
|
||||||
|
case http.StatusInternalServerError:
|
||||||
|
return "internal", true
|
||||||
|
default:
|
||||||
|
return "internal", status >= 500
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user