From 94ad0c240bdbe31a56e8d03202f11227ff72275f Mon Sep 17 00:00:00 2001 From: Pierre Date: Mon, 25 May 2026 23:47:30 +0200 Subject: [PATCH] bugfix and new internal API --- Makefile | 14 + README.md | 708 ++++++--------- docs/api-mediated-access.md | 198 ++++- docs/meta-core-architecture.md | 1321 +++++++++------------------- docs/metadata-storage-structure.md | 745 +++++----------- docs/uuid-rooted-metadata.md | 4 +- editor/README.md | 178 ++-- editor/src/kv/KVTree.tsx | 22 +- internal/config/config.go | 17 + internal/mounts/handlers.go | 28 +- internal/storage/cid_resolution.go | 46 +- internal/storage/client.go | 39 +- internal/watcher/handlers.go | 28 +- internal/watchers/handlers.go | 28 +- internal/watchers/manager.go | 83 +- internal/watchers/poller.go | 22 +- 16 files changed, 1361 insertions(+), 2120 deletions(-) diff --git a/Makefile b/Makefile index d39848b..036db51 100644 --- a/Makefile +++ b/Makefile @@ -64,10 +64,24 @@ clean: rm -f coverage.out coverage.html # Build Docker image +# +# We tag THREE names: +# - meta-core:$(VERSION) — the canonical versioned tag +# - meta-core:latest — convenience for `docker run meta-core` +# - meta-core:local — the tag every sub-stack docker-compose.yml uses +# (meta-share federation, meta-gateway dev stacks). +# Without this, `make docker` produces an image +# the dev stacks can't see by name, and previously +# -tagged stale `:local` images linger, silently +# breaking downstream API fields (e.g. /urls +# missing webdavUrlInternal → meta-gateway's +# poster auto-store fails). See README.md → +# Quick start → Build for the audit recipe. docker: @echo "Building Docker image..." docker build -t $(DOCKER_IMAGE):$(VERSION) . docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):latest + docker tag $(DOCKER_IMAGE):$(VERSION) $(DOCKER_IMAGE):local # Run locally (for development) run: build diff --git a/README.md b/README.md index e3754ac..35d8737 100644 --- a/README.md +++ b/README.md @@ -1,520 +1,374 @@ # meta-core -A Go-based sidecar binary that provides unified leader election, Redis management, and HTTP API for MetaMesh services. +Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE +metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack. ## Overview -meta-core runs alongside each MetaMesh service (meta-sort, meta-fuse, meta-stremio) as a sidecar process managed by supervisord. It provides the foundational infrastructure that enables MetaMesh's distributed architecture without requiring external coordination services like etcd or Consul. +meta-core runs as its own container (`metacore-app`). Other MetaMesh services +(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by +reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis +is not exposed outside the container — all metadata I/O is mediated by the +HTTP API. -### Key Capabilities +### Key capabilities | Feature | Description | |---------|-------------| -| **Leader Election** | flock-based leader election (Go binary only runs as leader) | -| **Redis Management** | Leader spawns Redis with AOF+RDB persistence, auto-restart on crash | -| **HTTP API** | Language-agnostic REST interface for metadata and service discovery | -| **Service Discovery** | File-based registry with heartbeats and stale detection | -| **Metadata Storage** | Flat key-value schema with connection pooling and batch operations | -| **File Watching** | Directory scanning, MidHash256 computation, event dispatch | -| **Mount Management** | SMB/rclone configuration, health monitoring | -| **WebDAV Server** | File access with nginx proxy_cache | -| **Event Publishing** | Redis keyspace notifications to meta:events stream | +| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. | +| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. | +| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. | +| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. | +| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. | +| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. | +| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. | +| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. | +| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. | -### Design Characteristics +### Design characteristics -- **No external dependencies** - Uses filesystem locks (works on NFS/CIFS shared volumes) -- **Automatic failover** - Lock releases on process death, followers detect and re-elect -- **Thread-safe** - RWMutex protection on all shared state -- **Graceful shutdown** - Signal handling with reverse startup order -- **Static binary** - CGO disabled, single binary deployment +- **Single-leader** — flock on a shared volume; no external consensus. +- **HTTP-only externally** — Redis is bound to the container, never published. +- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the + on-disk Redis layout predates the current build (alpha clean-wipe policy; + see `docs/uuid-rooted-metadata.md`). +- **Static binary** — CGO disabled, single Go binary plus a thin Alpine + image with `redis-server`, `rclone`, nginx, and supervisord. ## Architecture -meta-core runs as a **standalone container** that other services connect to via HTTP API: - ``` -┌───────────────────────────────────────────────────────────────────────┐ -│ meta-core container (standalone) │ -│ │ -│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │ -│ │ Role │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │ -│ │ Provider │ │ Server │ │ Server │ │ /meta, /urls, │ │ -│ │ │ │ │ │ │ │ /health, /services │ │ -│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │ -│ │ -│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │ -└──────────────────────────────────────────────┬────────────────────────┘ - │ - ┌──────────────────────────────────────┴──────────────────────────┐ - │ /meta-core (shared volume) │ - │ ├── locks/kv-leader.info (leader URLs) │ - │ ├── db/redis/ (Redis data) │ - │ ├── services/ (hostname-based .json) │ - │ └── mounts/ (mount configurations) │ - └─────────────────────────────────────────────────────────────────┘ - ▲ ▲ ▲ - ┌────────────┘ │ └────────────┐ - │ │ │ -┌───────────────┐ ┌───────────────┐ ┌───────────────┐ -│ meta-sort │ │ meta-fuse │ │ meta-stremio │ -│ (reads info, │ │ (reads info, │ │ (reads info, │ -│ calls HTTP) │ │ calls HTTP) │ │ calls HTTP) │ -└───────────────┘ └───────────────┘ └───────────────┘ +┌──────────────────────────────────────────────────────────────────────────┐ +│ metacore-app (standalone container) │ +│ │ +│ supervisord │ +│ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │ +│ │ writes kv-leader.info, exec supervisord) │ +│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │ +│ ├─ nginx (TLS termination, proxy_cache for /webdav) │ +│ ├─ rclone rcd (mount daemon, RC API) │ +│ ├─ mount-watcher.sh (rclone mount lifecycle) │ +│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │ +└──────────────────────────────────────────────────────────────────────────┘ + │ + ┌──────────────────────┴──────────────────────┐ + │ /meta-core (shared volume) │ + │ locks/kv-leader.{lock,info} │ + │ db/redis/ │ + │ services/*.json │ + │ mounts/{mounts.json, errors/} │ + │ watchers.json │ + │ cache/ (nginx proxy_cache) │ + └─────────────────────────────────────────────┘ + ▲ + ┌──────────────────────────────┼────────────────────────────────┐ + │ │ │ +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ meta-sort │ │ meta-fuse │ │ meta-stremio│ +│ │ │ │ │ │ +│ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │ +│ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│ +└─────────────┘ └─────────────┘ └─────────────┘ ``` -## Quick Start +## Quick start ### Build ```bash -# Build binary -make build - -# Build Docker image -make docker - -# Run tests -make test +make build # build static binary to bin/meta-core +make docker # build container image as meta-core:1.0.0 + :latest + :local +make test # run Go unit tests ``` -### Run +`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The +`:local` tag is the one every sub-stack docker-compose.yml references +(meta-share 10-peer federation, meta-gateway dev stacks); without it the +sub-stacks can't see a freshly-built image by name. + +> **Source-change → recreate.** `make docker` rebuilds and re-tags the +> image, but **already-running containers stay on the old image SHA** — +> `docker compose up -d` won't recreate them unless the compose file +> itself changed. After a meta-core source change, force-recreate the +> downstream containers so they pick up the new binary: +> +> ```bash +> make docker +> docker compose -f packages/meta-gateway/dev/docker-compose.yml \ +> up -d --force-recreate metagateway-generic-core +> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \ +> up -d --force-recreate metagateway-torznab-core +> docker compose -f packages/meta-share/dev/docker-compose.yml \ +> --profile share-10node up -d --force-recreate metashare-core metashare-core-test +> ``` +> +> The symptom of a missed recreate is missing API fields on `/urls` +> (e.g. `webdavUrlInternal` absent → meta-gateway's +> `store_poster_for_record` silently fails and torznab posters never +> land). Audit recipe: +> +> ```bash +> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA +> docker ps --filter ancestor=meta-core:local --format '{{.Names}}' +> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate. +> ``` + +### Dev container + +In the MetaMesh dev stack, the container is brought up by `dev/docker-compose.yml`: ```bash -# With environment variables -META_CORE_PATH=/meta-core \ -FILES_PATH=/files \ -SERVICE_NAME=meta-sort \ -META_CORE_HTTP_PORT=9000 \ -./bin/meta-core +cd dev && ./scripts/start.sh ``` -### Docker - -```bash -docker build -t meta-core . -docker run -v meta-core:/meta-core -v files:/files meta-core -``` +The user-facing URL is `https://metacore-dev.localhost:8083`; the direct-backend +HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`. ## Configuration -| Environment Variable | Default | Description | -|---------------------|---------|-------------| -| `META_CORE_PATH` | `/meta-core` | Shared volume for locks/db | -| `FILES_PATH` | `/files` | Shared volume for media files | -| `SERVICE_NAME` | `meta-core` | Service identifier | -| `SERVICE_VERSION` | `1.0.0` | Service version | -| `API_PORT` | `8180` | Main service HTTP port | -| `BASE_URL` | - | Stable service URL | -| `REDIS_PORT` | `6379` | Redis port (leader only) | -| `META_CORE_HTTP_PORT` | `9000` | HTTP API port | -| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP API bind address | -| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Health check interval | -| `HEARTBEAT_INTERVAL_MS` | `30000` | Service heartbeat interval | -| `STALE_THRESHOLD_MS` | `60000` | Stale service threshold | -| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning | +Environment variables consumed by `internal/config/config.go`: -## API Reference +| Variable | Default | Description | +|---|---|---| +| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). | +| `FILES_PATH` | `/files` | Files volume root. | +| `SERVICE_NAME` | `meta-core` | Identity in the service registry. | +| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. | +| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. | +| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). | +| `REDIS_PORT` | `6379` | Local Redis port. | +| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. | +| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind. | +| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. | +| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. | +| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. | +| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. | +| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. | +| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. | +| `DEBOUNCE_MS` | `30000` | File-event debounce window. | +| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. | -### Health & Status +Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and +no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service +identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock +retry interval (`ELECTION_RETRY_SECS`, default `5`). + +## API reference + +All endpoints below are served by `internal/api/server.go`. Examples assume +the in-container Go port `:9000`; from the host use +`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083` +(debug-direct). + +### Health, leader, URLs ```bash -# Health check -curl http://localhost:9000/health -# {"status":"ok","redis":true,"timestamp":"..."} - -# Detailed status -curl http://localhost:9000/status -# {"status":"ok","serviceName":"meta-core","version":"1.0.0",...} - -# Current leader -curl http://localhost:9000/leader -# {"host":"meta-sort-dev","api":"redis://10.0.1.50:6379","http":"http://10.0.1.50:8180",...} +GET /health # storage + role +GET /status # version, uptime, etc. +GET /leader # current leader info (hostname, PID, timestamps) +GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty) +GET /services # registered services + heartbeats +GET /api/services # alias for /services +GET /services/{name} # one service +GET /services/cleanup/stats ``` -### Metadata Operations +### Metadata — primary surface + +These are the routes other services use. `{hash}` may be a UUID root or +any registered CID alias; CID aliases resolve to the underlying UUID +before reads/writes (`internal/storage/cid_resolution.go`). ```bash -# List all file hashes -curl http://localhost:9000/meta -# {"hashIds":["midhash256:abc123",...],"count":42} - -# Get metadata for a file -curl http://localhost:9000/meta/{hash} -# {"hashId":"midhash256:abc123","metadata":{"title":"Movie","year":"2024",...}} - -# Update metadata -curl -X PUT http://localhost:9000/meta/{hash} \ - -H "Content-Type: application/json" \ - -d '{"title":"New Title"}' - -# Delete metadata -curl -X DELETE http://localhost:9000/meta/{hash} +GET /meta # list of root IDs +GET /meta/{hash} # full document (flat keys → nested JSON) +PUT /meta/{hash} # replace document +PATCH /meta/{hash} # merge into document +DELETE /meta/{hash} # delete root + all CID aliases +GET /meta/{hash}/{key...} # single property (key may contain "/") +PUT /meta/{hash}/{key...} # set single property +DELETE /meta/{hash}/{key...} # delete single property +POST /meta/{hash}/_add/{key...} # add value to set-valued property ``` -### Data Operations +CID-addressed access (public; auth-bypassed in the Caddy perimeter): ```bash -# Get file path -curl http://localhost:9000/data/{hash}/path -# {"hashId":"midhash256:abc123","path":"/files/movies/Movie.mkv","exists":true} - -# Check if file exists -curl -I http://localhost:9000/data/{hash} -# HTTP/1.1 200 OK (or 404) +GET /api/meta/{cid} # resolve CID → metadata document +GET /api/file/{cid}/info # CID resolution metadata +GET /file/{cid} # serve file bytes (range-aware) +POST /file/cid # compute CID for an uploaded file +HEAD /data/{hash} # existence + size +GET /data/{hash}/path # absolute path on /files ``` -### File Operations (by CID) +(There is no `/data/{hash}/stream`; bulk reads go through `/webdav/*` or +the `/file/{cid}` reverse-lookup path.) + +### Editor / KV browser surface ```bash -# Serve file by CID (looks up poster/backdrop CIDs in metadata) -curl http://localhost:9000/file/{cid} --output poster.jpg -# Returns raw file bytes with appropriate Content-Type header +GET /api/metadata/hash-ids +GET /api/metadata/list +POST /api/metadata/search +POST /api/metadata/batch +POST /api/metadata/clear +GET /api/metadata/{hashId} +PUT /api/metadata/{hashId} +DELETE /api/metadata/{hashId} +GET /api/metadata/{hashId}/property +PUT /api/metadata/{hashId}/property -# Example: fetch a poster image -curl -v http://localhost:9000/file/bafkreih5aznjvttude6c3wbvqeebb6rlx5wkbzyppv7garber7ndsuxku4 -# Content-Type: image/jpeg -# [binary image data] +GET /api/kv/info +GET /api/kv/keys +GET /api/kv/tree +GET /api/kv/search +GET /api/kv/find +GET /api/kv/value +PUT /api/kv/value +DELETE /api/kv/value +GET /api/kv/key/{key...} + +GET /api/schema +POST /api/schema/rescan + +GET /api/snapshot/export +POST /api/snapshot/import +POST /api/snapshot/wipe ``` -The `/file/{cid}` endpoint searches all file metadata for matching `poster` or `backdrop` CIDs and serves the corresponding file from disk. Supports range requests for partial content retrieval. +### SSE event streams -### Service Discovery +The HTTP-mirror of the Redis Streams. External services never read Redis +directly — they subscribe here with `Last-Event-ID` for resume. See +[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire +contract (one event per stream entry, opaque IDs, heartbeats, gap +signalling). ```bash -# List all services -curl http://localhost:9000/services -# {"services":[{"name":"meta-sort",...}],"count":3} - -# Get specific service -curl http://localhost:9000/services/meta-sort -# {"name":"meta-sort","version":"2.0.0","api":"http://...","status":"running",...} +GET /api/events/files # file:events stream (watcher events) +GET /api/events/meta # meta:events stream (metadata mutations) ``` -### Mount Management +### Mounts ```bash -# List all mounts -curl http://localhost:9000/api/mounts/list -# {"mounts":[{"id":"smb-nas","type":"smb","path":"/files/corn/nas",...}]} - -# Add a new mount -curl -X POST http://localhost:9000/api/mounts/add \ - -H "Content-Type: application/json" \ - -d '{"type":"smb","path":"/files/corn/nas","config":{...}}' - -# Remove a mount -curl -X DELETE http://localhost:9000/api/mounts/smb-nas +GET /api/mounts +POST /api/mounts +GET /api/mounts/{id} +PUT /api/mounts/{id} # also PATCH +DELETE /api/mounts/{id} +POST /api/mounts/{id}/mount +POST /api/mounts/{id}/unmount +POST /api/mounts/{id}/safe-unmount +POST /api/mounts/{id}/scan +GET /api/mounts/rclone/remotes ``` -### File Watching +### Watchers ```bash -# Get scan status -curl http://localhost:9000/api/scan/status -# {"scanning":false,"lastScan":"2024-01-15T10:30:00Z","filesFound":12847} - -# Trigger rescan -curl -X POST http://localhost:9000/api/scan/trigger - -# Stream file events (SSE) -curl http://localhost:9000/api/events/stream -# data: {"type":"add","path":"movies/Movie.mkv","midhash256":"bafkr..."} +GET /api/watchers +POST /api/watchers +GET /api/watchers/{id} +PUT /api/watchers/{id} +DELETE /api/watchers/{id} +POST /api/watchers/{id}/scan +POST /api/watchers/{id}/reset +POST /api/watchers/scan-all +POST /api/watchers/reset-all ``` -### KV Browser +The legacy `/api/scan/trigger` and `/api/scan/status` still respond but +return a deprecation pointer at the `/api/watchers/*` routes +(`internal/watcher/handlers.go:32-33`). + +### Admin / WebDAV ```bash -# Storage statistics -curl http://localhost:9000/api/kv/info -# {"totalKeys":50000,"memoryUsed":"256MB",...} - -# List keys with cursor pagination -curl "http://localhost:9000/api/kv/keys?cursor=0&count=100" - -# Get specific key value -curl http://localhost:9000/api/kv/key/file:bafkr.../title +POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries +GET/PUT/DELETE /webdav/... # mounted on /files ``` -### Metadata Editor +No `/metrics` endpoint is exposed; observability is via container logs and +the `/health` / `/status` JSON. -```bash -# Get all hash IDs -curl http://localhost:9000/api/metadata/hash-ids -# {"hashIds":["bafkr...","bafks..."]} +## Leader gate -# Paginated metadata listing -curl "http://localhost:9000/api/metadata/list?page=0&pageSize=50" +Leader election is a bash flock loop in `docker/leader-election.sh`, not a +Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the +script writes the leader API URL to `kv-leader.info` and `exec`s +supervisord. When the process tree dies, the kernel releases the lock and +any standby instance can take over. There is no follower path inside the +Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API. -# Search metadata -curl -X POST http://localhost:9000/api/metadata/search \ - -H "Content-Type: application/json" \ - -d '{"query":"inception","fields":["title"]}' - -# Batch update -curl -X POST http://localhost:9000/api/metadata/batch \ - -H "Content-Type: application/json" \ - -d '{"operations":[{"hashId":"bafkr...","updates":{"title":"New Title"}}]}' - -# Clear all metadata -curl -X POST http://localhost:9000/api/metadata/clear -``` - -### WebDAV Access - -```bash -# List directory (returns JSON) -curl http://localhost:9000/webdav/ -# [{"name":"watch","isDir":true},{"name":"plugin","isDir":true}] - -# Get file -curl http://localhost:9000/webdav/watch/movie.mkv --output movie.mkv - -# Upload file -curl -X PUT http://localhost:9000/webdav/plugin/output.txt \ - -H "Content-Type: application/octet-stream" \ - --data-binary @output.txt - -# Create directory -curl -X MKCOL http://localhost:9000/webdav/newdir/ - -# Delete file/directory -curl -X DELETE http://localhost:9000/webdav/oldfile.txt -``` - -## Leader Election - -meta-core uses POSIX file locking (flock) for distributed leader election: - -1. Each instance tries to acquire exclusive lock on `/meta-core/locks/kv-leader.lock` -2. Winner becomes **leader**, spawns Redis, writes info to `/meta-core/locks/kv-leader.info` -3. Losers become **followers**, read leader info, connect to leader's Redis -4. Lock automatically releases when process dies (no stale locks) - -### Election Flow - -``` -Instance 1 Instance 2 Instance 3 - │ │ │ - ├── Try flock ──────────┼───────────────────────┤ - │ │ │ - [WIN] [LOSE] [LOSE] - LEADER FOLLOWER FOLLOWER - │ │ │ - Start Redis Read leader.info Read leader.info - │ │ │ - Write leader.info Connect to Redis Connect to Redis - │ │ │ - [Health loop 5s] [Health loop 5s] [Health loop 5s] - - Update timestamp - Re-read leader - Re-read leader - - Check Redis alive - Detect changes - Detect changes -``` - -### Role Callbacks - -The election system provides callbacks for role transitions: - -```go -election.OnBecomeLeader(func(info LeaderLockInfo) { - // Start Redis, initialize resources -}) -election.OnBecomeFollower(func(info LeaderLockInfo) { - // Connect to leader's Redis -}) -election.OnLeaderLost(func() { - // Handle leader failure, attempt re-election -}) -``` - -### Lock Info Format - -The `kv-leader.info` file contains a plain text API URL: +`kv-leader.info` is a single plaintext line containing the API base URL: ``` http://10.0.1.50:9000 ``` -Services read this file to discover the leader, then call the `/urls` API endpoint to get full connection details (Redis URL, WebDAV URL, base URL, etc.). +Other services read this file, then call `GET /urls` for full discovery. -## Redis Management +The dev compose runs a single `metacore-app`; the flock primitive is +retained as a correctness anchor (one process owns Redis and the +writable state under `/meta-core`), not as live failover. -The leader is responsible for spawning and managing the Redis server. +## Metadata storage shape (one-line summary) -### Redis Configuration +Roots are opaque UUIDv7 strings; every property is a separate Redis +STRING at `file:/`; every known CID for a file is a +reverse-index entry `cid:: → `. Full details in +[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md) +and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md). +The authoritative registry of field semantics and value formats lives in +the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md). -```bash -redis-server --port 6379 --bind 0.0.0.0 \ - --dir /meta-core/db/redis \ - --appendonly yes --appendfilename appendonly.aof \ - --dbfilename dump.rdb --save 60 1 \ - --loglevel warning -``` - -### Persistence Strategy - -| Method | Configuration | Purpose | -|--------|--------------|---------| -| **AOF** | `appendonly yes` | Write-ahead log for durability | -| **RDB** | `save 60 1` | Snapshot after 60s if ≥1 key changed | - -### Failure Handling - -- **Redis crash**: Auto-detected via health check, automatically restarted -- **Leader crash**: Lock released, followers detect via stale timestamp, re-election occurs -- **Graceful shutdown**: SIGTERM sent to Redis (10s timeout), then SIGKILL if needed - -## Metadata Storage - -meta-core uses a flat key-value schema in Redis for storing file metadata. - -### Storage Format - -``` -Key: /file/{hashId}/{property} -Value: string - -Examples: -/file/midhash256:abc123/title → "Movie Title" -/file/midhash256:abc123/year → "2024" -/file/midhash256:abc123/genres → "action,drama" -/file/midhash256:abc123/filePath → "movies/Movie.mkv" -``` - -### Storage Client Features - -| Feature | Details | -|---------|---------| -| Connection Pool | 10 connections | -| Timeouts | 5s dial, 30s read/write | -| Batch Operations | Pipelined writes for performance | -| Key Scanning | SCAN-based iteration (non-blocking) | -| CID Lookup | Find file paths by poster/backdrop CID | - -## Service Discovery - -Services register by writing JSON to `/meta-core/services/{name}.json`. - -### Registration Format - -```json -{ - "name": "meta-sort", - "version": "2.0.0", - "api": "http://10.0.1.50:8180", - "status": "running", - "pid": 12345, - "hostname": "meta-sort-dev", - "startedAt": "2024-01-01T00:00:00Z", - "lastHeartbeat": "2024-01-01T00:01:00Z", - "capabilities": ["meta-core"], - "endpoints": { - "health": "http://10.0.1.50:9000/health", - "meta": "http://10.0.1.50:9000/meta" - } -} -``` - -### Heartbeat & Stale Detection - -| Parameter | Default | Purpose | -|-----------|---------|---------| -| `HEARTBEAT_INTERVAL_MS` | 30000 | How often services update `lastHeartbeat` | -| `STALE_THRESHOLD_MS` | 60000 | Mark service "stale" if heartbeat older than this | - -Services are automatically marked as `stale` when discovered if their `lastHeartbeat` exceeds the threshold. On graceful shutdown, services remove their registration file. - -## Development - -```bash -# Format code -make fmt - -# Run linter -make lint - -# Run tests with coverage -make test-cover - -# Clean build artifacts -make clean -``` - -## Internal Packages +## Internal packages | Package | Purpose | |---------|---------| -| `cmd/meta-core` | Main entry point, startup/shutdown orchestration | -| `internal/config` | Environment-based configuration with defaults | -| `internal/leader` | Election system (`election.go`) and Redis manager (`redis.go`) | -| `internal/storage` | Redis client wrapper with flat key-value operations | -| `internal/discovery` | Service registration, heartbeat loop, discovery | -| `internal/api` | HTTP server, router, and all endpoint handlers | -| `internal/events` | Redis keyspace notification publisher to meta:events | -| `internal/watcher` | File system scanning, MidHash256 computation, event dispatch | -| `internal/watchers` | Polling-based watcher configuration management | -| `internal/mounts` | SMB/rclone mount configuration and health monitoring | -| `internal/webdav` | WebDAV protocol handler with caching integration | +| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. | +| `internal/config` | Env-driven configuration, path helpers. | +| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). | +| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). | +| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. | +| `internal/discovery` | Service registry + dead-service cleaner. | +| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). | +| `internal/api` | HTTP server + all handlers + SSE event endpoints. | +| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). | +| `internal/snapshot` | Snapshot export / import / wipe. | +| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. | +| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). | +| `internal/mounts` | rclone mount manager, lifecycle handlers, stats poller. | +| `internal/webdav` | WebDAV handler (mounted at `/webdav/*`; nginx handles caching upstream). | -### Startup Sequence +### Startup sequence ``` 1. Load configuration from environment -2. Create storage client (Redis wrapper) -3. Initialize leader election -4. Register role transition callbacks -5. Start election (acquire lock or become follower) -6. Start service discovery (register + heartbeat loop) -7. Initialize dead service cleaner -8. Initialize file watcher (if enabled) -9. Initialize mount manager -10. Start HTTP API server (WebDAV caching handled by nginx) -11. Wait for SIGINT/SIGTERM -12. Shutdown in reverse order -``` - -## Integration - -### As Sidecar in Docker - -Add to existing service's supervisord config: - -```ini -[program:meta-core] -command=/usr/local/bin/meta-core -priority=10 -autostart=true -autorestart=true -``` - -### Client Usage (TypeScript) - -```typescript -const response = await fetch('http://localhost:9000/meta/' + hashId); -const { metadata } = await response.json(); -``` - -### Client Usage (Python) - -```python -import requests -response = requests.get(f'http://localhost:9000/meta/{hash_id}') -metadata = response.json()['metadata'] +2. Build LeaderInfoProvider (no election — the bash gate already won) +3. Connect to local Redis (retry up to 30× / 1s) +4. EnsureSchemaVersion — abort if the existing Redis layout is stale +5. Start service discovery + dead-service cleaner +6. Construct API server (initialises watcher, mounts, WebDAV) +7. Start API server; on first connect, start MetaPublisher + schema Indexer + and republish existing metadata onto meta:events +8. Wait for SIGINT/SIGTERM, shutdown in reverse order ``` ## Dependencies -| Dependency | Version | Purpose | -|------------|---------|---------| -| `github.com/gorilla/mux` | v1.8.1 | HTTP router | -| `github.com/redis/go-redis/v9` | v9.4.0 | Redis client | -| `golang.org/x/net` | v0.20.0 | WebDAV support | -| `github.com/google/uuid` | v1.6.0 | UUID generation | -| Go stdlib | - | context, encoding/json, net, os, sync, syscall | +| Dependency | Purpose | +|------------|---------| +| `github.com/gorilla/mux` | HTTP router | +| `github.com/redis/go-redis/v9` | Redis client | +| `github.com/google/uuid` | UUIDv7 minting | +| `golang.org/x/net` | WebDAV support | +| Go stdlib | context, net/http, os, sync, syscall | -### Runtime Requirements - -- **Go 1.21+** for building -- **Redis** binary available in PATH (leader spawns it) -- **Shared filesystem** accessible by all instances (for flock) +Runtime requirements: Go 1.21+ at build time; the container ships Redis, +rclone, nginx, and supervisord alongside the binary. ## License diff --git a/docs/api-mediated-access.md b/docs/api-mediated-access.md index ae3009b..31abb54 100644 --- a/docs/api-mediated-access.md +++ b/docs/api-mediated-access.md @@ -2,11 +2,17 @@ ## Status -**Proposed** — design accepted; implementation pending. Builds on -[uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes meta-core -is the single writer to the metadata keyspace. That assumption is currently -violated by meta-sort, and this doc is the plan to put it back on solid -ground. +**Landed** — all four PRs (A/B/C/D) deployed and verified end-to-end on the +dev stack. Redis is unreachable from outside `metacore-app`; every metadata +read, write, and event flows through meta-core's HTTP+SSE surface. Builds +on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes +meta-core is the single writer to the metadata keyspace — an assumption +that's now actually enforced rather than aspirational. + +See "Implementation outcome" at the bottom for the delta between this doc's +plan and what shipped (notably the write-path CID resolution and the +dual-root migration sweep, which weren't in the original plan but turned +out to be load-bearing). ## Motivation @@ -441,6 +447,10 @@ Closing the network access (PR D) is the moment the lock becomes a lock; everything before it is opt-in migration that consumers can adopt at their own pace. +**All four PRs have landed.** What follows is the original plan as +written; see "Implementation outcome" at the bottom for the delta +between plan and ship. + ### PR A — SSE endpoints in meta-core Adds `GET /api/events/files` and `GET /api/events/meta`. Implements the @@ -584,28 +594,34 @@ Stable `error` slugs the client should switch on: | `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. | | `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. | -Today's handlers return `{error, message}` strings; PR A is the natural -place to introduce the typed shape (along with the new SSE endpoints -which will need their own error responses anyway). Going through a -typed envelope from the start avoids the retrofit cost. +**Implemented.** Every 4xx/5xx response across `internal/api`, +`internal/mounts`, `internal/watcher`, and `internal/watchers` now +emits the typed envelope. The slug-mapping helper is duplicated across +those packages (one ~15-line `slugForStatus` per package) to avoid an +import cycle on the api package; consolidating it into a shared +`internal/httperr` package is a worthwhile follow-up but not load-bearing. ## Remaining open questions -The audit closed everything that was answerable. What's left: +Mostly answered post-implementation: -- **Deprecation timeline for `/api/metadata/{hashId}`.** It stays during - the migration (editor uses it), but once we've consolidated on - `/meta/{hash}` for service-to-service writes the editor could migrate - too and we'd delete the duplicate surface. Not urgent; revisit - post-PR D when everything else is stable. +- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The + editor's React UI continues to call it; the surface itself was untouched + by this work. Worth a follow-up to consolidate the editor onto + `/meta/{hash}` and delete the parallel routes. -- **What does `/api/events/files` emit during the watcher's initial - bootstrap scan?** When meta-core starts on a populated library, the - watcher discovers N files in quick succession and publishes N `add` - events. A consumer subscribing for the first time will receive that - whole flood. Probably fine — consumers are designed to handle bursts - — but worth a smoke test once PR A lands to confirm meta-share's - ingest doesn't trip on the cardinality. +- **Initial bootstrap flood on `/api/events/files`.** Resolved in + practice: meta-fuse uses `initialCursor: '0-0'` to deliberately + replay every entry on each restart and the StreamingStateBuilder + handles the resulting burst (~2.5k property events on a 50-file + library) without issue. meta-sort persists its cursor so only sees + the flood once per fresh deploy. meta-stremio filters to "interesting + fields" before triggering cache invalidation, so the burst is mostly + no-ops. + +- **meta-dup SSE port.** Still pending — meta-dup is in no active + compose file. When revived, copy meta-fuse's pattern wholesale + (SSEEventClient + MetaCoreApiClient). Tracked separately. ## Non-goals @@ -619,7 +635,137 @@ The audit closed everything that was answerable. What's left: - **Introducing new transport, broker, proxy, or sidecar.** Everything builds on what's deployed today. - **Migrating meta-fuse / meta-stremio / meta-dup direct Redis reads - via XSCAN/GET** (separate from event-stream reads). These are - read-only and don't break schema invariants, but they will fail once - PR D closes Redis access. PR C is the migration window for them — - the same SSE adoption work covers it. + via XSCAN/GET** (separate from event-stream reads). ~~Was a non-goal~~ + ended up being required for PR D's network closure to be safe. + meta-fuse and meta-stremio migrated; meta-dup deferred (not deployed). + See PR C / "Implementation outcome" for details. + +## Implementation outcome + +What actually shipped vs. what this doc planned. + +### Delta against the original plan + +- **Read-path migration for meta-fuse and meta-stremio is now in scope + and done.** The original "out of scope" carve-out was wrong: closing + the Redis network port doesn't just break stream consumers, it also + breaks every `kvClient.get` / `redis_storage.get_all_videos` call. + Both services grew their own `MetaCoreApiClient` (TS for meta-fuse, + Python for meta-stremio) and now route reads through `/meta/{hash}` + + `/meta` + `/api/file/{cid}/info`. + - **meta-stremio's HTTP mode auto-recovers from transient meta-core + outages.** `is_connected` re-probes meta-core's `/health` on every + call (cheap; a stateless GET), so a metacore-app restart no longer + latches the addon into a permanent disconnected state. The + background reconnect thread is only needed for legacy Redis mode + where the socket itself is sticky. +- **Hard `redisUrl` guard is gated by `ALLOW_LEGACY_REDIS_URL=1`.** All + three consumer LeaderClients (TS x2 + Python) throw on seeing the + field by default. Setting the env var downgrades to a warn — escape + hatch for a deliberate temporary rollback to a meta-core that still + publishes the field. +- **Write-path CID resolution** turned out to be a hard prerequisite + for the lockdown to work in practice. Documented separately below. +- **PR D ordering matched the doc** (consumer migration first, then + publisher stops emitting, then network closure). The `meta-core` + docker-network alias and the `6380:6379` host port mapping are both + removed; sibling services reach the HTTP/SSE/WebDAV surface via the + canonical `metacore-app` hostname. +- **meta-dup is the one service that didn't migrate.** It's in no + active compose file. The TS port is straightforward when needed: copy + meta-fuse's pattern (SSEEventClient for streams, MetaCoreApiClient + for reads). + +### Write-path CID resolution (`storage.ResolveRoot`) + +Not in the original plan but load-bearing. + +**The problem.** meta-sort's PATCH-via-HTTP migration (PR B) routed +every write through `/meta/{hash}`. But `{hash}` is the midhash256 CID +that meta-sort received via file:events, not the watcher's minted UUID. +With no resolution step in the handler, every write went to +`file:/*` — a parallel root the watcher's UUID never saw. The +auto-alias hook on `cid_midhash256=` then OVERWROTE the +watcher's `cid:midhash256:` → UUID alias to point at the +midhash itself, leaving the UUID stranded with only the watcher's +sparse fields (filePath, sizeByte, mtimeNano, duplicates) and the +midhash root carrying the rich plugin output. This is the exact +"dual roots per file" failure the doc's Motivation section listed as +the prior bug — it just moved from "convention violation" to "API +contract violation." + +**The fix.** Three layers in `internal/storage/cid_resolution.go`: + +1. **`ResolveRoot(hash)`** — at the entry of every `/meta/{hash}` + read/write handler, look up `cid:midhash256:` in the reverse + index. If found, use the UUID as the storage root. Falls through to + the bare hash for direct-UUID writes / legacy entries. +2. **Self-pointing alias guard in `addAliasLocked`** — refuse to + register `cid::` → ``. The dual-root bug's chicken-and- + egg is: meta-sort writes first, registers self-pointing alias, + watcher then can't recover. Refusing the self-write keeps the + watcher's legitimate alias intact. +3. **`MigrateDualRoots` sweep** (`POST /api/admin/migrate-dual-roots`). + One-shot fixer for already-stranded entries: walks every UUID's + `midhash256` field to build the orphan index, merges each stranded + `file:/*` into the matching UUID via `MergeMetadataFlat`, + deletes the stranded root, and re-points the alias. + +Verified on the dev stack: 48 stranded entries were reunited; the +editor now shows every file with its full plugin metadata under one +UUID; future writes stay unified. + +### Typed error envelope + +Done across all four handler packages (`api`, `mounts`, `watcher`, +`watchers`). Each has its own `writeError` returning +`{error: , message, retryable}`. The slug-mapping helper is +duplicated in each to avoid an import cycle on the `api` package. + +### What's NOT done + +- **meta-dup SSE + read migration.** Not deployed, no test harness. + When revived, copy meta-fuse's pattern. +- **Editor `/api/metadata/{hashId}` consolidation.** The duplicate + editor-internal API surface is untouched; could be retired in a + follow-up by porting the editor UI to `/meta/{hash}`. +- **Consolidated `internal/httperr` package** for the typed envelope + helpers (currently duplicated 4× to avoid the import cycle). Code + hygiene; functionally a no-op. + +### Verification done + +- All four service dashboards load cleanly with no console errors. +- meta-sort SSE consumer + HTTP write path verified end-to-end by + delete + inject + reprocess of Sintel. +- meta-fuse SSE consumer rebuilds VFS to 48 files on each boot. +- meta-stremio catalog API now surfaces the full library (47 videos, + 1 movie + 5 series + 46 episodes — was 0/0/0 before the Python + read-path migration because LeaderStorage broke when it tried Redis). +- `redis-cli` from any container other than `metacore-app` produces a + connection refused; `cid:midhash256:` aliases all point to UUIDs + (no self-pointing); `file:__index__` has only UUID-style hashIds. +- ALLOW_LEGACY_REDIS_URL=1 escape hatch present in all three consumer + LeaderClients (TS x2 + Python). + +### Known test-suite gaps (post-implementation bats run) + +Running `docker exec meta-test-runner /app/test/test.sh` after the +lockdown shipped reveals a number of failing suites, almost all of +which are pre-existing test/environment drift unrelated to this work: + +| Failure | Root cause | Relation to lockdown | +|---|---|---| +| meta-stremio (~35 tests), stremio-tmdb (6 tests) | Tests call `/manifest.json` directly; dev `.env` sets `STREMIO_HASH_API_SEED=dev-stremio-secret` which puts the addon behind a path-token prefix | Unrelated — pre-existing env mismatch | +| meta-fuse FUSE-mount (3 tests) | Tests use `docker exec meta-fuse` but the container is named `metafuse-app` (Yundera prod-parity rename) | Unrelated — predates this work | +| meta-fuse WebDAV PROPFIND (4 tests) | Tests use Caddy URL with WebDAV basic-auth credentials that no longer match the hash-lock perimeter | Unrelated — predates this work | +| meta-sort "API integration tests" (1 test) | Test runs `docker exec meta-sort pnpm test`; container is `metasort-app` | Unrelated — predates this work | +| meta-core "create and delete mount config" (1 test) | Test creates an NFS mount; production code dropped NFS support ("NFS mounts are no longer supported") | Unrelated — predates this work | +| service-discovery (9 tests) | Cascading from meta-stremio breakage (test expects unauth `/api/services` on stremio) | Unrelated to lockdown directly | +| meta-share / share-1c (multiple) | Tests assert `"phase":"1f"` in `/health`; meta-share has moved past that phase | Unrelated — meta-share evolution | +| flat-keys: "filePath field exists" (1 test) | Test SRANDMEMBERs file:__index__; if it lands on a UUID with no filePath (e.g. a manual test write) it fails | Was hit transiently after my interactive testing left stub entries; resolved by cleanup | + +No failure traceable to a real regression from this work was found. +The dev-environment test harness needs an unrelated refresh to catch +up with the prod-parity container renames, the NFS removal, the +STREMIO_HASH_API_SEED default, and meta-share's current phase. diff --git a/docs/meta-core-architecture.md b/docs/meta-core-architecture.md index 71c86b4..97912b3 100644 --- a/docs/meta-core-architecture.md +++ b/docs/meta-core-architecture.md @@ -2,1058 +2,551 @@ ## Document Purpose -This document describes the architectural design of **meta-core**, the unified data and metadata access layer for MetaMesh. Meta-core is a language-agnostic sidecar binary that runs alongside each service container, providing KV storage, leader election, service discovery, and file access APIs. This document covers the sidecar pattern, leader election mechanism, API design, data model, and integration patterns. +This document describes the architectural design of **meta-core**, the +standalone container that owns Redis, the leader-info file, the HTTP+SSE +metadata surface, and a WebDAV file server. It covers the leader gate, +storage data model, API design, the SSE event surface, and the integration +contract with other MetaMesh services. + +For the on-disk Redis layout, see +[metadata-storage-structure.md](metadata-storage-structure.md). For the +external-access design (Redis lockdown, SSE wire contract, auth rules), +see [api-mediated-access.md](api-mediated-access.md). For the UUID-rooted +storage rationale, see [uuid-rooted-metadata.md](uuid-rooted-metadata.md). --- ## System Overview -MetaMesh consists of three services that require shared access to metadata and files: +MetaMesh is a collection of services that need a shared, ordered, durable +view of file metadata: | Service | Language | Role | -|---------|----------|------| -| **meta-sort** | TypeScript | Watches folders, extracts metadata, writes to KV | -| **meta-fuse** | TypeScript/Rust | Serves virtual filesystem from KV metadata | -| **meta-stremio** | Python | Stremio addon for HLS streaming | -| **meta-dup** | TypeScript | Duplicate detection service | +|---|---|---| +| **meta-sort** | TypeScript | Watches folders, runs plugins, writes metadata via meta-core's HTTP API. | +| **meta-fuse** | TypeScript / Rust | Serves the virtual filesystem from metadata. | +| **meta-stremio** | Python | Stremio addon for HLS streaming. | +| **meta-dup** | TypeScript | Duplicate detection. | +| **meta-share** | Rust | Decentralised metadata sharing (libp2p / Kamilata). | -### Problem Statement +meta-core sits below all of these as the single owner of: -Without meta-core, each service implements its own: -- Leader election logic (616 lines TS + 301 lines Python + 277 lines TS) -- Redis client configuration -- Service discovery mechanism -- File path resolution +1. **Redis** — the metadata store and event source. +2. **The leader-info file** — the one place services look to discover meta-core's URLs. +3. **The HTTP+SSE surface** — the only way external services touch the data plane. +4. **A WebDAV server** — file access for plugins and other services. -This leads to: -1. **Code duplication** across languages -2. **Inconsistent behavior** between implementations -3. **Tight coupling** to Redis internals -4. **Complex maintenance** when adding new services +### Core design principle -### Core Design Principle +> External services never speak Redis. Every read, write, and event flows +> through meta-core's HTTP+SSE surface. -**Provide a single, language-agnostic interface for all data and metadata operations, abstracting storage implementation from service logic.** - -This design enables: -- **Language independence**: Any service language can use HTTP API -- **Single source of truth**: One leader manages all storage -- **Clean abstraction**: Services don't know about Redis, flock, or file paths -- **Operational simplicity**: One binary to deploy, monitor, and debug +This is the contract that PR D of [api-mediated-access.md](api-mediated-access.md) +landed and that the rest of this document assumes. --- -## Architecture Design +## Architecture -### Standalone Container Pattern +### Standalone container topology -Meta-core runs as a **standalone container** that other services connect to via HTTP API: +meta-core runs as a dedicated container (`metacore-app`). The dev compose +runs a single instance; the legacy multi-instance topology (sibling +followers) was deprecated. The flock primitive is retained as the +correctness anchor — it guarantees that at most one process owns Redis and +the writable state under `/meta-core`. ``` -┌───────────────────────────────────────────────────────────────────────┐ -│ meta-core container (standalone) │ +┌────────────────────────────────────────────────────────────────────────┐ +│ metacore-app (standalone container) │ │ │ -│ ┌──────────────┐ ┌─────────┐ ┌──────────┐ ┌────────────────────┐ │ -│ │ Leader │ │ Redis │ │ WebDAV │ │ HTTP API (:9000) │ │ -│ │ Election │ │ Server │ │ Server │ │ /meta, /urls, │ │ -│ │ (flock) │ │ │ │ │ │ /health, /services │ │ -│ └──────────────┘ └─────────┘ └──────────┘ └────────────────────┘ │ -│ │ -│ Writes: /meta-core/locks/kv-leader.info (URLs for all services) │ -└──────────────────────────────────────────────┬────────────────────────┘ - │ - ┌──────────────────────────┼────────────────────────┐ - │ /meta-core (shared volume) │ - │ ▼ │ - │ ┌─────────────────────────────────────────────┐ │ - │ │ locks/ │ │ - │ │ kv-leader.lock (flock marker) │ │ - │ │ kv-leader.info (leader JSON) │ │ - │ │ db/ │ │ - │ │ redis/ (RDB + AOF) │ │ - │ │ services/ │ │ - │ │ meta-sort-*.json (service registry) │ │ - │ │ meta-fuse-*.json │ │ - │ │ meta-stremio-*.json │ │ - │ │ mounts/ │ │ - │ │ mounts.json (mount configurations) │ │ - │ └─────────────────────────────────────────────┘ │ - └──────────────────────────────────────────────────┘ - ▲ ▲ - ┌────────────────────┘ └────────────────────┐ - │ │ -┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ -│ meta-sort │ │ meta-fuse │ │ meta-stremio │ │ meta-dup │ -│ container │ │ container │ │ container │ │ container │ -│ │ │ │ │ │ │ │ -│ Reads leader │ │ Reads leader │ │ Reads leader │ │ Reads leader │ -│ .info, calls │ │ .info, calls │ │ .info, calls │ │ .info, calls │ -│ HTTP API │ │ HTTP API │ │ HTTP API │ │ HTTP API │ -└─────────────────┘ └─────────────────┘ └─────────────────┘ └─────────────────┘ - - ┌──────────────────────────────────────────────┐ - │ /files (shared volume) │ - │ │ - │ watch/ (host media, read-only) │ - │ test/ (test media) │ - │ plugin/ (plugin output, read-write) │ - │ corn/ (SMB/rclone mounts) │ - └──────────────────────────────────────────────┘ +│ supervisord │ +│ ├─ leader-election.sh (flock → write kv-leader.info → exec self) │ +│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │ +│ ├─ nginx (TLS termination, proxy_cache for /webdav) │ +│ ├─ rclone rcd (mount daemon) │ +│ ├─ mount-watcher.sh (rclone mount lifecycle) │ +│ └─ meta-core (Go) ──► HTTP + SSE + WebDAV on :9000 │ +└────────────────────────────────────────────────────────────────────────┘ + │ + ┌────────────────────────────┴────────────────────────────┐ + │ /meta-core (shared volume) │ + │ locks/kv-leader.{lock,info} │ + │ db/redis/ │ + │ services/*.json │ + │ mounts/{mounts.json, errors/} │ + │ watchers.json │ + │ cache/ (nginx proxy_cache) │ + └─────────────────────────────────────────────────────────┘ + ▲ + ┌────────────────────────────┼────────────────────────────┐ + │ │ │ +┌─────────────┐ ┌─────────────┐ ┌─────────────┐ +│ meta-sort │ HTTP / SSE │ meta-fuse │ HTTP / SSE │ meta-stremio│ +│ ├────────────►│ ├─────────────►│ │ +│ │ │ │ │ │ +└─────────────┘ └─────────────┘ └─────────────┘ ``` -### Multi-Container Topology +### `/files` shared volume -Meta-core runs as a **standalone container**. Other services connect to it via HTTP API: +Independently of `/meta-core`, every service mounts the `/files` volume +read-only (write access scoped to specific subtrees: `/files/plugin`, +`/files/corn` mounts). meta-core exposes this volume over WebDAV at +`/webdav/*` so plugins running in their own containers don't have to mount +it themselves. ``` - ┌─────────────────────────────────────┐ - │ meta-core container │ - │ ★ LEADER │ - │ │ - │ ┌─────────┐ ┌──────────────────┐ │ - │ │ Redis │ │ HTTP API (:9000) │ │ - │ │ Server │ │ WebDAV Server │ │ - │ └─────────┘ └──────────────────┘ │ - └────────────────────┬────────────────┘ - │ - ┌────────────────────────────┼────────────────────────────┐ - │ │ │ - ▼ ▼ ▼ -┌─────────────────────┐ ┌─────────────────────┐ ┌─────────────────────┐ -│ meta-sort │ │ meta-fuse │ │ meta-stremio │ -│ container │ │ container │ │ container │ -│ │ │ │ │ │ -│ ┌───────────────┐ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ -│ │ LeaderClient │ │ │ │ LeaderClient │ │ │ │ LeaderClient │ │ -│ │ (reads info, │ │ │ │ (reads info, │ │ │ │ (reads info, │ │ -│ │ calls HTTP) │ │ │ │ calls HTTP) │ │ │ │ calls HTTP) │ │ -│ └───────────────┘ │ │ └───────────────┘ │ │ └───────────────┘ │ -│ ▲ │ │ ▲ │ │ ▲ │ -│ │ │ │ │ │ │ │ │ -│ ▼ │ │ ▼ │ │ ▼ │ -│ ┌───────────────┐ │ │ ┌───────────────┐ │ │ ┌───────────────┐ │ -│ │ meta-sort │ │ │ │ meta-fuse │ │ │ │ meta-stremio │ │ -│ │ service │ │ │ │ service │ │ │ │ service │ │ -│ └───────────────┘ │ │ └───────────────┘ │ │ └───────────────┘ │ -└─────────────────────┘ └─────────────────────┘ └─────────────────────┘ - │ │ │ - └────────────────────────┼────────────────────────┘ - ▼ - ┌──────────────────────────────┐ - │ /meta-core (shared volume) │ - │ /files (shared volume) │ - └──────────────────────────────┘ +/files + ├── watch/ (host media, read-only) + ├── test/ (test fixtures) + ├── plugin/ (plugin output, read+write) + └── corn/ (rclone-managed mounts) ``` -The flock-based leader election primitive is described below. Multi-instance -meta-core deployments (with a sibling failover instance) were used in earlier -dev stacks but are deprecated; the dev compose now runs a single -`metacore-app`. The election primitive itself is retained — it is what gates -which process owns Redis and the writeable `/meta-core` state, and it remains -correct under a single-instance deployment (the lock is held by that one -process for its lifetime). - --- -## Leader Election +## Leader Gate -### Mechanism: flock-based Consensus +### Mechanism: bash + flock -Meta-core uses **POSIX file locking (flock)** on a shared filesystem for leader election. The election is performed by `leader-election.sh` which starts supervisord when the lock is acquired. The Go binary only ever runs as leader (followers loop in bash and never start the binary). This approach requires no external consensus system (etcd, Consul, Zookeeper). +Leader election is implemented in `docker/leader-election.sh`, not in Go. +The script: + +1. Opens `${META_CORE_PATH}/locks/kv-leader.lock` on file descriptor 200. +2. Tries `flock -n 200`. On success it transitions to LEADER, writes the + API URL to `${META_CORE_PATH}/locks/kv-leader.info`, and `exec`s + supervisord — replacing itself in the process tree. When supervisord + exits, the kernel releases the flock automatically. +3. On failure (someone else holds the lock) it sleeps `ELECTION_RETRY_SECS` + (default `5s`) and retries. Followers loop in bash and never start the + Go binary. ``` -Leader Election Flow: - - Container startup - │ - ▼ - ┌──────────────────┐ - │ leader-election │ - │ .sh │ - │ Try flock() on │ - │ kv-leader.lock │ - └────────┬─────────┘ - │ - ┌───────┴───────┐ - │ │ - acquired blocked - │ │ - ▼ ▼ -┌─────────────┐ ┌──────────────┐ -│ Write │ │ Sleep and │ -│ leader.info │ │ retry flock │ -│ │ │ │ -│ exec │ └──────────────┘ -│ supervisord │ -└──────┬──────┘ +Container startup │ ▼ -┌──────────────┐ -│ meta-core │ -│ (Go binary) │ -│ │ -│ Spawns Redis │ -│ Starts API │ -└──────────────┘ +┌──────────────────┐ +│ leader-election │ +│ .sh │ +│ flock(kv-leader │ +│ .lock) │ +└────────┬─────────┘ + │ + ┌────┴────┐ + │ │ + acquired blocked + │ │ + ▼ ▼ +┌─────────┐ ┌──────────────┐ +│ Write │ │ sleep N, │ +│ kv- │ │ retry │ +│ leader │ └──────────────┘ +│ .info │ +│ │ +│ exec │ +│ super- │ +│ visord │ +└─────────┘ + │ + ▼ +┌──────────────────────────────┐ +│ supervisord starts: │ +│ redis → nginx → rclone → │ +│ meta-core Go binary │ +└──────────────────────────────┘ ``` -### Lock File Structure +### Implications for the Go binary -**`/meta-core/locks/kv-leader.lock`** -- Binary flock marker (empty file) -- Held exclusively by leader process -- Automatically released on process death (kernel handles cleanup) +The Go binary at `cmd/meta-core/main.go` is invoked only after the bash +gate wins. Therefore: + +- `internal/leader/election.go` does **not** run an election loop. It + contains `LeaderInfoProvider`, which builds the leader info struct from + local hostname + IP + config and returns it on demand. There are no + `OnBecomeLeader` / `OnBecomeFollower` / `OnLeaderLost` callbacks; those + states do not exist inside the Go process. +- `LeaderLockInfo.RedisUrl` is intentionally empty in the response from + `/urls` — see `internal/leader/election.go:82`. PR D of + api-mediated-access.md removed direct Redis exposure; consumers route + metadata I/O through meta-core's HTTP API and SSE event streams. + +### Lock-info file + +`/meta-core/locks/kv-leader.info` is a single plaintext line: -**`/meta-core/locks/kv-leader.info`** ``` http://10.0.1.50:9000 ``` -Plain text file containing the leader's API URL. Services read this to discover the leader, then call the `/urls` API endpoint for full connection details (Redis URL, WebDAV URL, etc.). +Other services read this file, then call `GET /urls` for full discovery +(hostname, baseUrl, apiUrl, webdavUrl, webdavUrlInternal). -### Failover Behavior +### Failure handling -When the leader process dies: +- **meta-core process dies** → supervisord restarts it. The bash flock + loop does not need to participate — supervisord owns the lock through + its lifetime. +- **Container dies** → kernel releases the flock. If a standby + `metacore-app` is provisioned, its bash loop will acquire on next tick. + The dev compose runs a single instance; failover is "restart the + container." +- **Redis crash inside the container** → supervisord restarts redis. The + Go binary's connect loop retries up to 30 times (1s delay) at startup; + at steady state, individual operations fail and clients retry. -1. **Kernel releases flock** automatically (no stale lock possible) -2. **Followers detect** via: - - File watch on `kv-leader.info` - - Health check failure to leader HTTP endpoint -3. **First follower acquires lock** becomes new leader -4. **New leader spawns Redis** using existing data in `/meta-core/db/redis/` -5. **Other followers reconnect** to new leader +--- -``` -Timeline: Leader Failure and Recovery +## Schema-Version Gate -t=0 t=1 t=2 t=3 t=4 - │ │ │ │ │ - ▼ ▼ ▼ ▼ ▼ -Leader Leader Flock Follower New leader -running crashes released acquires spawns Redis - flock & serves -``` - -### Leader Responsibilities - -The leader meta-core instance: - -1. **Spawns Redis** as a child process on port 6379 -2. **Writes leader.info** with connection details -3. **Accepts all write operations** (metadata updates) -4. **Serves read operations** (metadata queries, file paths) -5. **Manages service registry** (heartbeats, stale detection) - -### Follower Responsibilities - -Follower meta-core instances: - -1. **Monitor leader.info** for leader changes -2. **Proxy requests to leader** for writes -3. **Cache metadata locally** for fast reads (optional) -4. **Attempt leader acquisition** on leader failure - -### WebDAV Caching Architecture - -Meta-core provides a WebDAV server at `/webdav/` with nginx proxy_cache for file caching: - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ nginx (:80) │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ -│ │ HTTP Request │─►│ proxy_cache │─►│ Go WebDAV Handler │ │ -│ │ │ │ │ │ (:9000) │ │ -│ │ GET/HEAD │ │ Cached ─────►│ │ │ │ -│ │ PUT/DELETE │ │ Bypass ─────►│ │ │ │ -│ │ MKCOL │ │ │ │ │ │ -│ └──────────────┘ └──────────────┘ └──────────────────────┐ │ -│ │ │ │ -│ /meta-core/cache/ │ ▼ │ -│ (nginx managed) │ ┌────────────────────┐│ -│ │ │ File System ││ -│ │ │ (/files) ││ -│ │ │ - watch/ ││ -│ │ │ - plugin/ ││ -│ │ │ - corn/ ││ -│ │ └────────────────────┘│ -└─────────────────────────────────────────────────────────────────┘ -``` - -Cache features: -- **proxy_cache**: nginx handles caching of GET/HEAD requests with 1-hour TTL -- **Slice module**: Large files cached in 1MB chunks for efficient range requests -- **TTL-based expiry**: Entries expire after 1 hour of inactivity -- **X-Cache-Status header**: Returns MISS/HIT/EXPIRED for debugging -- **Write bypass**: PUT/DELETE/MKCOL operations bypass cache -- **Directory Listing**: Returns JSON for programmatic access - -### File Watching System - -Meta-core scans directories and emits events for file changes: - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ File Watcher │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ -│ │ Directory │─►│ State │─►│ Event Dispatcher │ │ -│ │ Scanner │ │ Registry │ │ │ │ -│ │ │ │ │ │ - Debouncing │ │ -│ │ - Recursive │ │ - File hash │ │ - Buffering │ │ -│ │ - MidHash256 │ │ - Timestamps │ │ - SSE streaming │ │ -│ └──────────────┘ └──────────────┘ └────────────────────────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────────────────┐ │ -│ │ meta:events stream │ │ -│ │ (add/change/delete/ │ │ -│ │ rename/reset) │ │ -│ └──────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -Components: -- **Directory Scanner**: Recursively scans file trees, computes MidHash256 -- **State Registry**: Tracks file states for change detection -- **Debouncer**: Batches rapid events for efficiency -- **Event Dispatcher**: Routes events to subscribers with buffering - -Events are published to the `meta:events` Redis stream. - -### Mount Management - -Meta-core manages remote mounts via a single rclone backend. Two user-facing -mount types are supported: `smb` (synthesised on-the-fly into rclone's `:smb:` -backend) and `rclone` (references a pre-defined remote in `rclone.conf`). -Native `mount.cifs` and `mount.nfs` were removed in favour of one code path. - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Mount Manager │ -│ │ -│ ┌──────────────┐ ┌──────────────┐ ┌────────────────────────┐ │ -│ │ Config │─►│ Validator │─►│ Mount Watcher │ │ -│ │ (mounts.json)│ │ │ │ │ │ -│ │ │ │ - SMB params │ │ - Calls rclone RC │ │ -│ │ - id │ │ - rclone cfg │ │ /mount/mount │ │ -│ │ - type │ │ - cache size │ │ - ReadOnly: true │ │ -│ │ - path │ │ │ │ - Polls every 5s │ │ -│ │ - cache* │ │ │ │ - Error tracking │ │ -│ └──────────────┘ └──────────────┘ └────────────────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -Features: -- **Configuration**: Mount configs stored in `/meta-core/mounts/mounts.json` -- **Read-only**: All mounts are mounted read-only by construction (no API knob) -- **VFS cache**: Per-mount `cacheMaxSize` (default `50G`), `cacheMaxAge` - (default `24h`), `dirCacheTime` (default `5m`) propagate to rclone's - `vfsOpt`. The cache directory itself is daemon-global — rclone keeps - per-remote subdirectories beneath it. -- **Health Monitoring**: Polls `/proc/mounts` every 5s for accessibility -- **Error Handling**: Tracks mount errors in `/meta-core/mounts/errors/{id}.error` -- **Stats**: `/api/mounts` returns daemon-global rclone counters (bytes/sec, - transfers/sec) — same numbers across all mounts since rclone tracks per- - daemon, not per-mount. +Before serving any traffic, `cmd/meta-core/main.go:54-61` calls +`storage.EnsureSchemaVersion`. The sentinel lives in Redis at +`meta-core:schema-version` (`internal/storage/schema_version.go`). The +gate refuses to boot when the existing Redis layout predates the +UUID-rooted schema, so legacy `file:midhash256:*` data can't leak into +the current code paths. In alpha there is no automated migration — +operators wipe with `pnpm run clean:all` and restart. --- ## Data Model -### Two-Concept Model - -Meta-core manages two distinct but related concepts: +### Two concepts: metadata vs. file bytes ``` -┌─────────────────────────────────────────────────────────────────┐ -│ METADATA (KV Store) │ -│ │ -│ Key: midhash256 CID │ -│ Value: { │ -│ "title": "Movie Name", │ -│ "year": 2024, │ -│ "path": "movies/Movie.2024.mkv", ◄── relative to /files │ -│ "size": 15032385536, │ -│ "duration": 7200, │ -│ "codec": "hevc", │ -│ ... │ -│ } │ -│ │ -│ ✓ Can exist WITHOUT local file (remote/deleted) │ -│ ✓ Searchable by any field │ -│ ✓ Stored in Redis hashes │ -└─────────────────────────────────────────────────────────────────┘ +┌────────────────────────────────────────────────────────────────────┐ +│ METADATA (Redis) │ +│ │ +│ Root: opaque UUIDv7 (Crockford Base32; ULID layout, 26 chars) │ +│ Storage: flat string keys │ +│ file:/ → string value │ +│ file:/cids → SET of CID tokens │ +│ file:/canonical_cid → preferred CID │ +│ file:/duplicates → SET of paths │ +│ │ +│ Aliases (reverse index, one Redis STRING each): │ +│ cid:: → │ +│ │ +│ Index: file:__index__ → SET of all UUIDs │ +│ │ +│ Mintable without a local file (remote / deleted entries OK). │ +│ Resolvable by any registered CID in O(1). │ +└────────────────────────────────────────────────────────────────────┘ -┌─────────────────────────────────────────────────────────────────┐ -│ DATA (Files) │ -│ │ -│ Location: /files volume (shared across containers) │ -│ Access: Via path from metadata │ -│ │ -│ ✓ Always has metadata (at minimum: hash, path, title) │ -│ ✓ Accessed by path or streamed via API │ -│ ✓ Never duplicated or moved │ -└─────────────────────────────────────────────────────────────────┘ +┌────────────────────────────────────────────────────────────────────┐ +│ FILE BYTES │ +│ │ +│ Location: /files volume (shared across containers). │ +│ Access: path resolution via metadata, served via /webdav/*, │ +│ /file/{cid} (reverse-lookup), or /data/{hash}. │ +│ Path stored as filePath in the metadata, relative to /files. │ +└────────────────────────────────────────────────────────────────────┘ ``` -### Path Resolution +### Why UUID-rooted? -File paths in metadata are **relative to the /files volume**: +Roots were previously content-hash tokens (`file:midhash256:abc/...`), +which privileged one hash algorithm. UUID-rooted storage treats every CID +symmetrically as an alias, makes `GET /api/meta/` O(1) for any +registered CID, and lets new digest types be added without rewriting root +keys. See [uuid-rooted-metadata.md](uuid-rooted-metadata.md) for the +design rationale and [metadata-storage-structure.md](metadata-storage-structure.md) +for the on-disk layout. -``` -Metadata path: "movies/Movie.2024.mkv" -Container path: "/files/movies/Movie.2024.mkv" -Host path: "${DATA_WATCH_PATH}/movies/Movie.2024.mkv" -``` +### Path resolution -This enables: -- **Multi-host deployments**: Same metadata works across machines -- **Volume remapping**: Change mount points without updating metadata -- **Portable backups**: Metadata export/import works across systems +`filePath` values stored in metadata are absolute paths under the `/files` +volume (e.g. `/files/watch/movies/Movie.2024.mkv`). The `/files` prefix is +itself a per-host detail — moving the volume only requires re-mounting it +at the same path inside the container. --- ## API Design -### Endpoint Overview +### Endpoint catalogue -Meta-core exposes an HTTP API on `localhost:9000` (configurable): +The router is in `internal/api/server.go`. Endpoint set, by surface: ``` -Health & Status - GET /health → Health check and role info - GET /status → Detailed status and metrics +Bootstrap / discovery + GET /health + GET /status + GET /leader + GET /urls + GET /services (alias: /api/services) + GET /services/{name} + GET /services/cleanup/stats -Metadata Operations - GET /meta/{hash} → Get metadata by hash - PUT /meta/{hash} → Create/update metadata - DELETE /meta/{hash} → Delete metadata - GET /meta?search=... → Search metadata +Metadata (primary surface for other services) + GET /meta + GET /meta/{hash} + PUT /meta/{hash} + PATCH /meta/{hash} + DELETE /meta/{hash} + GET /meta/{hash}/{key...} + PUT /meta/{hash}/{key...} + DELETE /meta/{hash}/{key...} + POST /meta/{hash}/_add/{key...} -Data Operations - GET /data/{hash}/path → Get file path - GET /data/{hash}/stream → Stream file content - HEAD /data/{hash} → Check if file exists +CID-addressed (public, auth-bypassed at the perimeter) + GET /api/meta/{cid} + GET /api/file/{cid}/info + GET /file/{cid} + POST /file/cid + HEAD /data/{hash} + GET /data/{hash}/path -Service Discovery - GET /services → List registered services - POST /services/{name} → Register/heartbeat service - DELETE /services/{name} → Deregister service +Editor / KV / schema / snapshot + GET /api/metadata/hash-ids + GET /api/metadata/list + POST /api/metadata/search + POST /api/metadata/batch + POST /api/metadata/clear + {GET,PUT,DELETE} /api/metadata/{hashId} + {GET,PUT} /api/metadata/{hashId}/property + GET /api/kv/{info,keys,tree,search,find,value} + PUT /api/kv/value + DELETE /api/kv/value + GET /api/kv/key/{key...} + GET /api/schema + POST /api/schema/rescan + GET /api/snapshot/export + POST /api/snapshot/import + POST /api/snapshot/wipe -Mount Management - GET /api/mounts/list → List all mounts - POST /api/mounts/add → Add new mount - DELETE /api/mounts/{id} → Remove mount +SSE event streams (HTTP-mirror of Redis Streams) + GET /api/events/files + GET /api/events/meta -File Watching - GET /api/scan/status → Scan status - POST /api/scan/trigger → Trigger rescan - GET /api/events/stream → Stream file events (SSE) +Mounts (rclone-only) + GET /api/mounts + POST /api/mounts + GET /api/mounts/{id} + PUT /api/mounts/{id} (also PATCH) + DELETE /api/mounts/{id} + POST /api/mounts/{id}/{mount,unmount,safe-unmount,scan} + GET /api/mounts/rclone/remotes -KV Browser - GET /api/kv/info → Storage statistics - GET /api/kv/keys → Key listing with cursor - GET /api/kv/key/{key} → Get key value +Watchers + {GET,POST} /api/watchers + {GET,PUT,DELETE} /api/watchers/{id} + POST /api/watchers/{id}/{scan,reset} + POST /api/watchers/{scan-all,reset-all} + (deprecated: /api/scan/trigger, /api/scan/status — see watcher/handlers.go:32) -Metadata Editor - GET /api/metadata/hash-ids → All hash IDs - GET /api/metadata/list → Paginated listing - POST /api/metadata/search → Advanced search - POST /api/metadata/batch → Batch updates - POST /api/metadata/clear → Clear all metadata +Admin + POST /api/admin/migrate-dual-roots WebDAV - GET/PUT/DELETE /webdav/* → File operations - MKCOL/COPY/MOVE /webdav/* → Directory operations - PROPFIND /webdav/* → Property queries + /webdav/... (GET, PUT, DELETE, MKCOL, COPY, MOVE, PROPFIND) ``` -### Health Endpoint +No `/metrics` endpoint is exposed. Observability is via container logs + +the `/health` and `/status` JSON. -``` -GET /health +### Auth perimeter -Response: -{ - "status": "healthy", - "leader": { - "hostname": "meta-sort-dev", - "http_url": "http://10.0.1.50:9000", - "redis_url": "redis://10.0.1.50:6379" - }, - "uptime_seconds": 3600, - "version": "1.0.0" -} -``` +In the dev and prod stacks Caddy + an nginx-hash-lock sidecar enforce +OIDC auth in front of meta-core. The following paths bypass auth (so +peers and unauthenticated tooling can reach them): -### Metadata CRUD +- `/health` +- `/webdav/*` +- `/meta/*` +- `/file/cid` +- `/api/file/{cid}` and `/api/meta/{cid}` +- (`/api/events/files`, `/api/events/meta` are also auth-bypassed but + **should not** be exposed publicly via Caddy; they're inside-only.) -**Get Metadata** -``` -GET /meta/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku +See `docs/api-mediated-access.md` "Auth" for the rationale. -Response: -{ - "hash": "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku", - "path": "movies/Inception.2010.mkv", - "title": "Inception", - "year": 2010, - "duration": 8880, - "size": 15032385536, - ... -} -``` +### SSE event streams — wire contract -**Update Metadata** -``` -PUT /meta/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku -Content-Type: application/json +`/api/events/files` and `/api/events/meta` are SSE mirrors of the +`file:events` and `meta:events` Redis Streams. The contract +(`internal/api/sse_events.go`): -{ - "title": "Inception", - "year": 2010, - "custom.rating": 9.5 -} +- One SSE event per Redis Stream entry. +- `id:` is the opaque `-` Redis Stream entry ID — clients echo + it back via `Last-Event-ID` on reconnect. +- `event:` is the `type` field of the underlying entry. +- `data:` is the JSON of the rest of the entry. +- Heartbeats: SSE comment `:keep-alive\n\n` every 30s of silence. +- On reconnect with `Last-Event-ID`, the handler resumes from the next + entry. If retention has trimmed past the cursor, the handler emits one + `event: gap` payload before resuming from the oldest available entry. -Response: -{ - "success": true, - "hash": "bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku" -} -``` +External services use SSE with `Last-Event-ID` as the resume primitive. +A Redis consumer-group (XREADGROUP / XACK) exists *internally* — the +`MetaPublisher` uses one to drain keyspace notifications — but it is not +the external contract. -**Search Metadata** -``` -GET /meta?title=inception&year=2010&limit=10&offset=0 +### WebDAV URLs -Response: -{ - "results": [ - { "hash": "bafkrei...", "title": "Inception", ... } - ], - "total": 1, - "limit": 10, - "offset": 0 -} -``` +`/urls` exposes two WebDAV URLs (`internal/leader/election.go`): -### Data Access +| Field | Purpose | Construction | +|---|---|---| +| `webdavUrl` | External access | `{BASE_URL}/webdav` (or `http://{ip}:{API_PORT}/webdav` if `BASE_URL` is unset) | +| `webdavUrlInternal` | Container-to-container | `http://{hostname}:{META_CORE_HTTP_PORT}/webdav` | -**Get File Path** -``` -GET /data/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/path +Container plugins should use `webdavUrlInternal`. Browsers / external +clients use `webdavUrl` (which goes through nginx + Caddy). -Response: -{ - "path": "/files/movies/Inception.2010.mkv", - "exists": true, - "size": 15032385536 -} -``` - -**Stream File Content** -``` -GET /data/bafkreihdwdcefgh4dqkjv67uzcmw7ojee6xedzdetojuzjevtenxquvyku/stream -Range: bytes=0-1048575 - -Response: -HTTP/1.1 206 Partial Content -Content-Type: application/octet-stream -Content-Range: bytes 0-1048575/15032385536 -Content-Length: 1048576 - -[binary data] -``` - -### Service Discovery - -**Register Service** -``` -POST /services/meta-fuse -Content-Type: application/json - -{ - "hostname": "meta-fuse-dev", - "http_url": "http://10.0.1.51:8181", - "version": "2.0.0", - "capabilities": ["vfs", "webdav"] -} - -Response: -{ - "success": true, - "ttl_seconds": 60 -} -``` - -**List Services** -``` -GET /services - -Response: -{ - "services": [ - { - "name": "meta-sort", - "hostname": "meta-sort-dev", - "http_url": "http://10.0.1.50:8180", - "status": "healthy", - "last_heartbeat": "2024-01-15T10:30:00Z" - }, - { - "name": "meta-fuse", - "hostname": "meta-fuse-dev", - "http_url": "http://10.0.1.51:8181", - "status": "healthy", - "last_heartbeat": "2024-01-15T10:30:05Z" - } - ] -} -``` +WebDAV caching is handled by nginx's `proxy_cache` (see +`docker/nginx.conf`), not by a Go cache layer. There is no +`internal/cache` package. --- ## Implementation -### Technology Choice: Go +### Why Go -Meta-core is implemented in Go for the following reasons: +| Requirement | Go advantage | +|---|---| +| Language-agnostic | HTTP API, no runtime dependency on callers | +| Single binary | Static compilation | +| Low resource | ~10-20 MB resident | +| HTTP server | stdlib `net/http` + gorilla/mux | +| Redis client | go-redis | +| WebDAV | `golang.org/x/net/webdav` | -| Requirement | Go Advantage | -|-------------|--------------| -| Language agnostic | HTTP API, no runtime dependencies | -| Single binary | Static compilation, no interpreter | -| Low resource usage | Small memory footprint (~10-20MB) | -| File locking | stdlib `syscall.Flock` | -| HTTP server | stdlib `net/http` | -| Redis client | go-redis (mature, full-featured) | -| Cross-platform | Compile for Linux, macOS, Windows | - -### Binary Structure +### Repo layout ``` -meta-core -├── cmd/ -│ └── meta-core/ -│ └── main.go # Entry point +packages/meta-core/ +├── cmd/meta-core/main.go entry point ├── internal/ -│ ├── api/ # HTTP server and handlers -│ │ ├── server.go # HTTP server setup -│ │ ├── health.go # Health endpoints -│ │ ├── meta.go # Metadata endpoints -│ │ ├── data.go # Data endpoints -│ │ ├── services.go # Service discovery endpoints -│ │ ├── mounts.go # Mount management endpoints -│ │ └── kv.go # KV browser endpoints -│ ├── config/ # Configuration loading -│ │ └── config.go -│ ├── discovery/ # Service registration -│ │ └── discovery.go -│ ├── events/ # Keyspace notification publisher -│ │ └── publisher.go -│ ├── leader/ # Role provider and leader info -│ │ ├── election.go # flock-based election -│ │ └── redis.go # Redis process management -│ ├── mounts/ # SMB/rclone management -│ │ ├── config.go # Mount configuration -│ │ └── health.go # Mount health monitoring -│ ├── storage/ # Redis client wrapper -│ │ └── client.go -│ ├── watcher/ # File scanning and events -│ │ ├── scanner.go # Directory scanner -│ │ ├── dispatcher.go # Event dispatcher -│ │ └── debouncer.go # Event debouncing -│ ├── watchers/ # Polling-based watchers -│ │ └── config.go -│ └── webdav/ # WebDAV protocol handler -│ └── handler.go -├── test/ +│ ├── api/ HTTP server + handlers + SSE +│ ├── cid/ CID parsing + algorithm ranking +│ ├── config/ env-driven configuration +│ ├── discovery/ service registry + dead-service cleaner +│ ├── events/ meta:events stream publisher +│ ├── leader/ LeaderInfoProvider (no election logic) +│ ├── mounts/ rclone mount manager + handlers + stats +│ ├── schema/ live per-field schema indexer +│ ├── snapshot/ snapshot export / import / wipe +│ ├── storage/ Redis wrapper + UUID + CID reverse index + schema gate +│ ├── watcher/ scanner + dispatcher + midhash + state +│ ├── watchers/ polling-based watcher configs +│ └── webdav/ /webdav/* handler +├── docker/ +│ ├── entrypoint.sh +│ ├── leader-election.sh the actual leader election +│ ├── mount-watcher.sh +│ ├── nginx.conf +│ └── supervisord.conf +├── editor/ React metadata browser (Vite) +├── dashboard/ React status dashboard (Vite) ├── docs/ -├── Makefile └── go.mod ``` -### Process Management - -Meta-core manages the Redis process as a child: - -```go -// Leader spawns Redis -cmd := exec.Command("redis-server", - "--port", "6379", - "--dir", "/meta-core/db/redis", - "--save", "60", "1", // RDB snapshot - "--appendonly", "yes", // AOF persistence -) -cmd.Start() - -// Monitor Redis health -go func() { - for { - if err := redisClient.Ping(); err != nil { - log.Error("Redis unhealthy, restarting...") - cmd.Process.Kill() - cmd.Start() - } - time.Sleep(5 * time.Second) - } -}() -``` - -### Container Integration - -Meta-core runs via supervisord alongside the main service: - -```ini -# /etc/supervisor/conf.d/meta-core.conf - -[program:meta-core] -command=/usr/local/bin/meta-core -directory=/meta-core -autostart=true -autorestart=true -priority=1 -startsecs=2 -stdout_logfile=/var/log/meta-core.log -stderr_logfile=/var/log/meta-core.err - -[program:main-service] -command=/app/start.sh -autostart=true -autorestart=true -priority=10 -startsecs=5 -``` - -### Configuration - -Environment variables configure meta-core: - -```bash -# Core paths -META_CORE_PATH=/meta-core # Shared volume for locks/db -FILES_PATH=/files # Shared volume for media files - -# Network -META_CORE_HTTP_PORT=9000 # HTTP API port -META_CORE_REDIS_PORT=6379 # Redis port (leader only) - -# Behavior -META_CORE_SERVICE_NAME=meta-sort # Service identity for discovery -META_CORE_LEADER_TIMEOUT=30 # Seconds before leader considered dead -META_CORE_HEARTBEAT_INTERVAL=10 # Service heartbeat frequency -``` - ---- - -## Client Integration - -### TypeScript Client - -Services use a simple HTTP client to interact with meta-core: - -```typescript -// meta-core-client.ts - -class MetaCoreClient { - private baseUrl = 'http://localhost:9000'; - - async getMeta(hash: string): Promise { - const res = await fetch(`${this.baseUrl}/meta/${hash}`); - if (res.status === 404) return null; - return res.json(); - } - - async setMeta(hash: string, meta: Partial): Promise { - await fetch(`${this.baseUrl}/meta/${hash}`, { - method: 'PUT', - headers: { 'Content-Type': 'application/json' }, - body: JSON.stringify(meta), - }); - } - - async getDataPath(hash: string): Promise { - const res = await fetch(`${this.baseUrl}/data/${hash}/path`); - if (res.status === 404) return null; - const data = await res.json(); - return data.path; - } - - async search(query: SearchQuery): Promise { - const params = new URLSearchParams(query as any); - const res = await fetch(`${this.baseUrl}/meta?${params}`); - return res.json(); - } - - async isHealthy(): Promise { - try { - const res = await fetch(`${this.baseUrl}/health`); - return res.ok; - } catch { - return false; - } - } -} - -export const metaCore = new MetaCoreClient(); -``` - -### Python Client - -```python -# meta_core_client.py - -import requests -from typing import Optional, Dict, Any - -class MetaCoreClient: - def __init__(self, base_url: str = "http://localhost:9000"): - self.base_url = base_url - - def get_meta(self, hash: str) -> Optional[Dict[str, Any]]: - res = requests.get(f"{self.base_url}/meta/{hash}") - if res.status_code == 404: - return None - return res.json() - - def set_meta(self, hash: str, meta: Dict[str, Any]) -> None: - requests.put( - f"{self.base_url}/meta/{hash}", - json=meta - ) - - def get_data_path(self, hash: str) -> Optional[str]: - res = requests.get(f"{self.base_url}/data/{hash}/path") - if res.status_code == 404: - return None - return res.json()["path"] - - def search(self, **query) -> Dict[str, Any]: - res = requests.get(f"{self.base_url}/meta", params=query) - return res.json() - - def is_healthy(self) -> bool: - try: - res = requests.get(f"{self.base_url}/health") - return res.ok - except: - return False - -meta_core = MetaCoreClient() -``` - ---- - -## Search Capabilities - -### Current Implementation - -Basic search uses Redis SCAN with pattern matching: +### Startup sequence (`cmd/meta-core/main.go`) ``` -GET /meta?title=inception - -→ SCAN 0 MATCH "file:*" COUNT 100 -→ For each key: HGET key title → filter matches +1. Load configuration from environment +2. Build LeaderInfoProvider (no election; bash gate already won) +3. Connect to local Redis (retry up to 30× / 1s) +4. EnsureSchemaVersion (abort if Redis layout is stale) +5. Start service discovery + dead-service cleaner +6. NewServer + Server.Start (boots watcher, mounts, WebDAV, API) +7. On first connection: start MetaPublisher + schema Indexer, + republish existing metadata onto meta:events +8. Wait for SIGINT/SIGTERM +9. Shutdown in reverse order ``` -This works for small collections but scales poorly. +### Process management -### Enhanced Search (Future) - -For advanced search, meta-core supports Redis Search module: - -``` -# Create search index -FT.CREATE idx:files ON HASH PREFIX 1 "file:" SCHEMA - title TEXT WEIGHT 5.0 - series_title TEXT WEIGHT 3.0 - year NUMERIC SORTABLE - media_type TAG - duration NUMERIC SORTABLE - -# Search query -FT.SEARCH idx:files "@title:inception @year:[2010 2010]" -``` - -Search query syntax: - -``` -GET /meta?q=title:inception AND year:2010 -GET /meta?q=media_type:episode AND series_title:"Breaking Bad" -GET /meta?q=duration:[3600 7200] # 1-2 hours -``` +Redis is not spawned by the Go binary; supervisord starts it before +meta-core (`docker/supervisord.conf`). The Go binary connects to +`localhost:6379` and retries until Redis is reachable. --- ## Failure Modes -### Leader Crash +### Container restart -1. **Detection**: Followers detect via health check failure (5s timeout) -2. **Election**: Fastest follower acquires flock -3. **Recovery**: New leader spawns Redis with existing data -4. **Reconnection**: Other followers read new leader.info, reconnect -5. **Downtime**: 5-10 seconds typical +Supervisord restarts crashed processes (meta-core, redis, nginx, rclone) +individually. The bash flock is held by supervisord's process tree, so +internal crashes don't trigger re-election. -### Redis Crash +### Redis crash -1. **Detection**: Leader detects ping failure -2. **Recovery**: Leader restarts Redis process -3. **Data**: Recovered from RDB snapshot + AOF replay -4. **Downtime**: 1-2 seconds typical +Supervisord restarts redis. The Go binary's connect retry handles the +window. Persistence is RDB + AOF on the `/meta-core` volume, so no data is +lost across restarts. -### Network Partition +### Container crash -1. **Followers isolated**: Continue serving cached reads, writes fail -2. **Leader isolated**: Continues operating, followers elect new leader -3. **Resolution**: On reconnection, followers sync with true leader -4. **Risk**: Brief split-brain if partition during election (mitigated by flock atomicity) +Kernel releases the flock; a standby container (if any) can take over. +The dev compose runs a single instance, so the recovery path is "restart +the container." -### Shared Volume Unavailable +### Schema mismatch -1. **Impact**: All meta-core instances fail -2. **Behavior**: Services operate in degraded mode (no metadata) -3. **Recovery**: Automatic when volume restored +`EnsureSchemaVersion` refuses to boot. Operators wipe with +`pnpm run clean:all` (or equivalent) and restart. This is an alpha-stage +hard-fail; there is no automated migration path. ---- +### Shared volume unavailable -## Observability - -### Metrics Endpoint - -``` -GET /metrics - -# Prometheus format -meta_core_role{service="meta-sort"} 1 # 1=leader, 0=follower -meta_core_uptime_seconds{service="meta-sort"} 3600 -meta_core_requests_total{endpoint="/meta"} 15420 -meta_core_request_duration_seconds{endpoint="/meta",quantile="0.99"} 0.005 -meta_core_redis_commands_total 892341 -meta_core_files_total 12847 -meta_core_metadata_bytes 45678901 -``` - -### Logging - -Structured JSON logging: - -```json -{ - "time": "2024-01-15T10:30:00Z", - "level": "info", - "msg": "Acquired leader lock", - "service": "meta-sort", - "pid": 12345 -} -``` - -Log levels: `debug`, `info`, `warn`, `error` - -### Health Probes - -For Kubernetes/Docker health checks: - -```yaml -# docker-compose.yml -healthcheck: - test: ["CMD", "curl", "-f", "http://localhost:9000/health"] - interval: 10s - timeout: 5s - retries: 3 -``` - ---- - -## Security Considerations - -### Network Isolation - -- Meta-core listens on `localhost:9000` only (not exposed externally) -- Redis listens on `localhost:6379` only -- Inter-container communication uses Docker network - -### File Access - -- Meta-core runs as non-root user -- Read-only access to /files volume (except meta-sort) -- No shell access or command execution - -### Input Validation - -- Hash parameters validated as valid CID format -- Path traversal prevented (paths must be relative, no `..`) -- JSON body size limited (10MB max) - ---- - -## WebDAV URL Configuration - -Meta-core exposes two WebDAV URLs via the `/urls` API to support both internal and external file access: - -| Field | Purpose | Example | -|-------|---------|---------| -| `webdavUrl` | External access (via nginx/HTTPS) | `https://media.example.com/webdav` | -| `webdavUrlInternal` | Internal container-to-container | `http://meta-core-1:9000/webdav` | - -### URL Composition - -- **External** (`webdavUrl`): `{BASE_URL}/webdav` - - Uses `BASE_URL` env var if set, otherwise falls back to `http://{ip}:{API_PORT}` - -- **Internal** (`webdavUrlInternal`): `http://{hostname}:{META_CORE_HTTP_PORT}/webdav` - - Uses container hostname (Docker DNS resolvable) - - Port 9000 is the direct Go WebDAV server - -### Usage Patterns - -- **Container plugins** should use `webdavUrlInternal` for file access (direct connection, lower latency) -- **External clients** should use `webdavUrl` for browser/remote access (may include SSL termination) +All meta-core operations fail. Other services see meta-core as unhealthy +via `/health`; metadata I/O and SSE event streams stop. --- ## Configuration Reference -### Environment Variables +### Environment variables | Variable | Default | Description | -|----------|---------|-------------| -| `META_CORE_PATH` | `/meta-core` | Path to shared infrastructure volume | -| `FILES_PATH` | `/files` | Path to shared files volume | -| `META_CORE_HTTP_PORT` | `9000` | HTTP API port | -| `META_CORE_REDIS_PORT` | `6379` | Redis port (leader only) | -| `META_CORE_SERVICE_NAME` | hostname | Service identity | -| `META_CORE_LEADER_TIMEOUT` | `30` | Leader health timeout (seconds) | -| `META_CORE_HEARTBEAT_INTERVAL` | `10` | Heartbeat frequency (seconds) | -| `META_CORE_LOG_LEVEL` | `info` | Log verbosity | -| `META_CORE_CACHE_ENABLED` | `false` | Enable local read cache | -| `META_CORE_CACHE_TTL` | `60` | Cache TTL (seconds) | -| `ENABLE_FILE_WATCHER` | `true` | Enable file scanning | +|---|---|---| +| `META_CORE_PATH` | `/meta-core` | Shared volume root. | +| `FILES_PATH` | `/files` | Files volume root. | +| `SERVICE_NAME` | `meta-core` | Identity in the service registry. | +| `SERVICE_VERSION` | `1.0.0` | Reported in `/status`. | +| `API_PORT` | `8180` | External port baked into `baseUrl`. | +| `BASE_URL` | _empty_ | Overrides constructed `baseUrl` (HTTPS perimeter). | +| `REDIS_PORT` | `6379` | Local Redis port. | +| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. | +| `META_CORE_HTTP_HOST` | `127.0.0.1` | HTTP bind address. | +| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. | +| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. | +| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is stale. | +| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. | +| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. | +| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. | +| `DEBOUNCE_MS` | `30000` | File-event debounce window. | +| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher. | +| `ELECTION_RETRY_SECS` | `5` | Bash flock retry interval (consumed by `docker/leader-election.sh`, not the Go binary). | -### File Paths +### File paths under `/meta-core` | Path | Purpose | -|------|---------| -| `/meta-core/locks/kv-leader.lock` | Leader election flock file | -| `/meta-core/locks/kv-leader.info` | Leader connection info | -| `/meta-core/db/redis/dump.rdb` | Redis RDB snapshot | -| `/meta-core/db/redis/appendonly.aof` | Redis AOF log | -| `/meta-core/services/*.json` | Service registry files | -| `/meta-core/mounts/mounts.json` | Mount configurations | -| `/meta-core/cache/` | nginx proxy_cache directory | -| `/meta-core/watchers.json` | Watcher configurations | +|---|---| +| `locks/kv-leader.lock` | Flock target. | +| `locks/kv-leader.info` | Leader API URL (plaintext). | +| `db/redis/{dump.rdb, appendonly.aof}` | Redis persistence. | +| `services/*.json` | Service registry. | +| `mounts/mounts.json` | rclone mount configuration. | +| `mounts/errors/{id}.error` | Mount error logs. | +| `watchers.json` | Polling watcher configuration. | +| `cache/` | nginx `proxy_cache` directory. | --- ## Summary -Meta-core provides a unified, language-agnostic interface for MetaMesh's data and metadata operations. By consolidating leader election, storage management, and service discovery into a single sidecar binary, it: - -1. **Eliminates code duplication** across TypeScript and Python services -2. **Simplifies service implementation** to pure business logic -3. **Provides operational visibility** through consistent health/metrics -4. **Enables future evolution** by abstracting storage implementation - -The flock-based leader election ensures exactly-one-leader semantics without external dependencies, while the HTTP API enables integration from any programming language. +meta-core consolidates leader gating, metadata storage, event mirroring, +and file access for MetaMesh into one container with a typed HTTP+SSE +surface. The bash flock gate guarantees single-leader semantics without +an external consensus system. UUID-rooted storage with CID aliasing +treats all content hashes symmetrically and makes any-CID resolution +O(1). API-mediated access ensures no other service ever needs to speak +Redis. diff --git a/docs/metadata-storage-structure.md b/docs/metadata-storage-structure.md index 577f567..200667c 100644 --- a/docs/metadata-storage-structure.md +++ b/docs/metadata-storage-structure.md @@ -1,595 +1,272 @@ -# Redis Metadata Storage Architecture +# Redis Metadata Storage Structure -This document describes how MetaMesh stores file metadata in Redis, the key structure, plugin metadata contributions, and special cases like FFmpeg stream handling. +How meta-core lays out file metadata in Redis. This document covers the +on-disk shape (UUID-rooted flat string keys, CID reverse index, +per-root sets) and the schema-version gate. It does **not** enumerate +field semantics — that's the job of the repo-root +[`METADATA_KEYS.md`](../../../METADATA_KEYS.md), the authoritative key +registry for everyone who writes to or reads from metadata. -## Overview +For the design rationale (why UUIDs instead of content hashes, why a +reverse index), see [`uuid-rooted-metadata.md`](uuid-rooted-metadata.md). +For the external-access contract (HTTP-only, SSE for events), see +[`api-mediated-access.md`](api-mediated-access.md). -MetaMesh uses Redis as its primary metadata storage, storing file metadata as **flat string keys**. Each property is stored as a separate Redis key rather than using Redis Hashes. This design enables: +## Quick summary -- **Redis keyspace notifications** for field-level change detection -- **Pattern-based subscriptions** (e.g., `file:*/tmdb/*` for TMDB updates) -- **meta:events stream** integration for real-time updates across services -- Individual field updates without affecting other properties -- Fast VFS lookups using content-based hash IDs +- **Roots are UUIDv7** encoded as 26-char Crockford Base32 (ULID layout), + not content hashes. Examples in this doc use the placeholder + `01JKR8XW5T4QMV7AHNJ2DEFGHK`. +- **Properties are flat Redis STRING keys**, one per leaf: + `file:/` → string value. *Not* Redis Hashes. +- **CIDs are reverse-index aliases**: `cid::` → `` + (a Redis STRING per known CID). +- **Per-root sets**: `file:/cids` (every CID), `file:/duplicates` + (alternate paths with the same content), plus `file:__index__` (every UUID). +- **A schema-version sentinel** (`meta-core:schema-version`) refuses to + boot against legacy data. -## Key Structure +Implementation lives in `internal/storage/client.go`, +`internal/storage/cid_resolution.go`, `internal/storage/uuid.go`, and +`internal/storage/schema_version.go`. -### Flat Key Storage +## Key structure -Each file property is stored as a separate Redis STRING key: +### Roots ``` -Key Pattern: file:{hashId}/{property/path} -Key Type: STRING -Value: String value +file:/ → Redis STRING (always) ``` -**Example Redis commands:** -```bash -# Store file metadata (each property is a separate key) -SET "file:bafkr4ih5kapbjzq.../cid_midhash256" "bafkr4ih5kapbjzq..." -SET "file:bafkr4ih5kapbjzq.../filePath" "/files/watch/movie.mkv" -SET "file:bafkr4ih5kapbjzq.../fileinfo/duration" "7200.5" -SET "file:bafkr4ih5kapbjzq.../fileinfo/formatName" "matroska,webm" +UUIDs are produced by `storage.NewUUID()` (`internal/storage/uuid.go`): +UUIDv7 → 26-char Crockford Base32 (skips I/L/O/U; ULID-compatible). The +time-sortable prefix lets you `KEYS file:01JKR8*` to find roots minted +in a given window, which restored some of the debug-ability lost when +midhash256 stopped being the root. -# Retrieve single field -GET "file:bafkr4ih5kapbjzq.../fileinfo/duration" - -# Retrieve all metadata for a file (scan by prefix) -SCAN 0 MATCH "file:bafkr4ih5kapbjzq.../*" COUNT 1000 -``` - -### Why Flat Keys? - -The flat key structure (vs. Redis Hashes) enables: - -1. **Keyspace Notifications**: Redis publishes events when individual keys change -2. **Pattern Subscriptions**: Services can subscribe to `__keyspace@0__:file:*/tmdb/*` -3. **Selective Updates**: Update one field without touching others -4. **Stream Integration**: The `meta:events` stream captures field-level changes - -### File Index - -A Redis Set tracks all known file hash IDs: +Properties use `/` as the path separator. Nested objects in source code +are flattened to flat keys on write and reconstructed on read. Examples: ``` -Key: file:__index__ -Type: Set -Value: {hashId1, hashId2, hashId3, ...} +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath = "/files/watch/movies/Inception.2010.mkv" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/sizeByte = "15032385536" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/mtimeNano = "1729872000000000000" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/title = "Inception" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration = "8878.5" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/streamdetails/video/0/codec = "h264" +file:01JKR8XW5T4QMV7AHNJ2DEFGHK/tmdb/poster = "bafkreih..." ``` -This enables fast bulk queries like "get all files" without scanning keys. +### Reverse-index aliases (CIDs) + +Every CID known for a file is registered as its own Redis STRING: + +``` +cid:: → +``` + +Examples: + +``` +cid:midhash256:bafkreih5kapbjzq... → 01JKR8XW5T4QMV7AHNJ2DEFGHK +cid:sha256:abc... → 01JKR8XW5T4QMV7AHNJ2DEFGHK +cid:ipfs:bafy... → 01JKR8XW5T4QMV7AHNJ2DEFGHK +``` + +The reverse index makes `GET /api/meta/` an O(1) lookup for any +registered CID, independent of which hash algorithm "primary" — there is +no primary. The auto-alias hooks in `SetProperty` and `SetMetadataFlat` +(see `internal/storage/cid_resolution.go`) catch any write of a `cid_*` +or `midhash256` field and register the alias automatically, so plugins +don't need to know about the reverse index. + +### Per-root sets + +``` +file:/cids → Redis SET of ":" tokens +file:/canonical_cid → STRING — the highest-ranked CID for advertising externally +file:/duplicates → Redis SET of alternate paths with same content +``` + +`canonical_cid` is reconciled by `reconcileCanonicalCIDLocked` whenever +the `cids` set changes; ranking is done by `internal/cid/rank.go`. + +### Global index + +``` +file:__index__ → Redis SET of all UUIDs +``` + +`internal/storage/client.go::GetAllHashIDs` reads this set to enumerate +every file without scanning the keyspace. ```bash -# Get all hash IDs -SMEMBERS "file:__index__" - -# Count total files -SCARD "file:__index__" - -# Check if a file exists -SISMEMBER "file:__index__" "bafkr4ih5kapbjzq..." +SMEMBERS file:__index__ # all UUIDs +SCARD file:__index__ # total file count +SISMEMBER file:__index__ ``` -### Keyspace Notifications - -Redis keyspace notifications are enabled to detect metadata changes. This requires the `notify-keyspace-events` configuration: - -```bash -# Required Redis config (set by meta-core) -CONFIG SET notify-keyspace-events "K$" - -# K = Keyspace events (published on __keyspace@__:) -# $ = String commands (SET, etc.) -``` - -Services can subscribe to keyspace events for real-time updates: - -```bash -# Subscribe to all file key changes -PSUBSCRIBE '__keyspace@0__:file:*' - -# Subscribe to specific field pattern (e.g., TMDB updates) -PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*' -``` - -### meta:events Stream - -The `meta:events` stream provides reliable event delivery for file system events from meta-core: +### Schema-version sentinel ``` -Stream Key: meta:events -Entry Fields: - - type: add | change | delete | rename | reset - - path: File path relative to /files - - midhash256: Content hash (for add/change events) - - oldPath: Previous path (for rename events) - - timestamp: Event timestamp +meta-core:schema-version → STRING (currently "2") ``` -Consumer groups enable reliable processing with acknowledgment: +Set at first boot, checked on every subsequent boot +(`internal/storage/schema_version.go`). If the sentinel is missing but +`file:__index__` is non-empty, the gate refuses to boot (legacy data). +If the sentinel is present but doesn't match `SchemaVersion`, the gate +also refuses to boot. Alpha-stage policy: operators wipe and restart; +there is no automated migration. -```bash -# Create consumer group -XGROUP CREATE meta:events mygroup 0 MKSTREAM +## Why flat STRING keys? -# Read new events (blocking) -XREADGROUP GROUP mygroup myconsumer BLOCK 5000 STREAMS meta:events > +The store does not use Redis Hashes. The reasons are: -# Acknowledge processed events -XACK meta:events mygroup -``` +1. **Field-level keyspace notifications.** Redis publishes + `__keyspace@0__:` on every SET/DEL of a STRING key. With Hashes + you'd only get one notification on the hash itself. meta-core's + `MetaPublisher` (`internal/events/meta_publisher.go`) consumes these + notifications and publishes structured events to the `meta:events` + Redis Stream. +2. **Selective updates without read-modify-write.** A plugin writing + `tmdb/poster` doesn't have to read the rest of the document. +3. **Compatibility with the CID alias hooks.** Field writes can trigger + alias-maintenance work in `SetProperty` without the plugin caring. -## Primary Hash ID +## Reads and writes -The primary identifier for each file is `cid_midhash256` - an IPFS-compatible CID generated from: +The Go API in `internal/storage/client.go`: -1. SHA-256 hash of the middle 1MB of the file -2. File size in bytes -3. IPFS CID v1 encoding (multicodec 0x1000) +| Method | Behaviour | +|---|---| +| `GetMetadataFlat(uuid)` | `SCAN` for `file:/*`, `MGET` the values, return `map[string]string`. | +| `SetMetadataFlat(uuid, m)` | Pipeline `SET` per field; `SADD file:__index__`; auto-register any `cid_*` aliases. | +| `MergeMetadataFlat(uuid, m)` | Like `SetMetadataFlat` but doesn't pre-clear; used by `PATCH /meta/{hash}`. | +| `GetProperty(uuid, field)` | `GET file:/`. | +| `SetProperty(uuid, field, value)` | `SET file:/`; auto-register alias if the field is a CID. | +| `DeleteRoot(uuid)` | `SCAN` + `DEL` for `file:/*`, unmap every reverse-index entry. | +| `Mint(filePath, size, mtimeNano)` | `NewUUID()` + write `filePath/sizeByte/mtimeNano` + add to index. | +| `AddAlias(uuid, ":")` | Set `cid::` → ``, add to `file:/cids`, reconcile `canonical_cid`. | +| `ResolveCID(":")` | `GET cid::` → uuid. | -**Example:** `bafkr4ih5kapbjzqvylwj7kx7zmkpxn6qj5xw...` +## Reconstruction (flat → nested) -This provides: -- Content-based addressing (same file = same ID) -- Fast computation (only reads 1MB) -- IPFS compatibility for future distributed storage - -### Collision Handling - -If two different files produce the same midhash (rare), the system falls back to full SHA-256: - -```typescript -// Normal case -hashId = midHash256; // "bafkr4ih5kapbjzq..." - -// Collision detected (different filePath with same midhash) -hashId = metadata['cid_sha2-256']; // Full hash fallback -``` - -## Metadata Flattening - -Nested objects are flattened into path-based keys for efficient Redis storage. - -### Flattening Rules - -```typescript -// Input (nested object) -{ - title: "Inception", - video: { - codec: "h265", - width: "1920" - }, - streams: [ - { language: "eng" }, - { language: "jpn" } - ] -} - -// Output (flattened for Redis) -title = "Inception" -video/codec = "h265" -video/width = "1920" -streams/0/language = "eng" -streams/1/language = "jpn" -``` - -**Key rules:** -- Nested objects use `/` as path separator -- Arrays use numeric indices (0, 1, 2...) -- All values stored as strings -- `null`/`undefined` stored as empty strings -- Numbers and booleans converted via `String()` - -### Reconstruction - -When reading, the flattened structure is reconstructed into nested objects: - -```typescript -const metadata = await kvClient.getMetadataFlat(hashId); -// Returns: { title: "Inception", video: { codec: "h265", width: "1920" }, ... } -``` - -## Plugin Metadata Structure - -Each plugin writes metadata using a prefixed key pattern: +When the HTTP API returns a document, it reconstructs the flattened +storage back into a nested JSON object: ``` -{plugin-id}/{field} # Simple field -{plugin-id}/{group}/{field} # Grouped field -{plugin-id}/{group}/{index}/{field} # Indexed field (for arrays) +flat nested +───────────────────────────────────── ─────────────────────────── +title = "Inception" title: "Inception" +fileinfo/duration = "8878.5" fileinfo: { duration: "8878.5", +fileinfo/streamdetails/ streamdetails: { video: { 0: { + video/0/codec = "h264" codec: "h264", ... } }, ... } } +streams/0/language = "eng" streams: [ { language: "eng" }, ... ] ``` -### Plugin Contribution Flow +Rules: + +- `/` is the path separator. +- Numeric path segments (`0`, `1`, ...) become array indices on read. +- All values are strings on the wire; type interpretation is the + caller's job (and is documented per-field in `METADATA_KEYS.md`). +- `null`/`undefined` round-trip as empty strings. + +## Plugin contribution flow + +Plugins don't talk to Redis. They write metadata via meta-core's HTTP +API (`PUT /meta/{hash}/{key...}` or `PATCH /meta/{hash}`). Each plugin +writes under a prefix: ``` -File Discovered - ↓ -[FileProcessorPiscina] - ├─ Compute midhash256 - ├─ Store preliminary metadata - └─ Dispatch plugin tasks - ↓ -[Plugin Processing] - ├─ file-info → fileType, mimeType, sizeByte - ├─ ffmpeg → fileinfo/duration, fileinfo/streamdetails/... - ├─ filename-parser → titles/eng, season, episode, videoType - ├─ anime-detector → anime, titles/jpn, titles/jpl - ├─ tmdb → tmdbid, plot/eng, genres - └─ full-hash → cid_sha2-256, cid_sha1 - ↓ -[Plugin Callbacks] - └─ Store metadata via flat keys (SET per property) +{plugin-id}/ single value +{plugin-id}// grouped +{plugin-id}/// indexed (array) ``` -## Core Metadata Types +See [`METADATA_KEYS.md`](../../../METADATA_KEYS.md) for the full, +authoritative list of fields each plugin writes (content identification, +file stats, video classification, anime detection, subtitle references, +Jellyfin metadata, FFmpeg stream details, etc.) and the value formats +(`int-string`, `lang3`, `path-relfiles`, `csv-set`, ...). This doc +deliberately does not duplicate that registry — it would drift. -### HashMeta - Content Identification +## Event streams -``` -cid_crc32 = "bafkr..." # CRC32 (multicodec 0x0132) -cid_md5 = "bafkr..." # MD5 (multicodec 0xd5) -cid_sha1 = "bafkr..." # SHA-1 (multicodec 0x11) -cid_sha2-256 = "bafkr..." # SHA-256 (multicodec 0x12) -cid_midhash256 = "bafkr..." # Middle-hash (PRIMARY ID, multicodec 0x1000) -``` +Two Redis Streams are produced inside meta-core: -### FileStatMeta - File Properties +| Stream | Producer | Contents | +|---|---|---| +| `file:events` | `internal/watcher` dispatcher | `add` / `change` / `delete` / `rename` / `reset` events for files on the `/files` volume. | +| `meta:events` | `internal/events.MetaPublisher` | Field-level metadata mutations derived from Redis keyspace notifications. | -``` -fileType = "video" # video, audio, image, text -mimeType = "video/x-matroska" # MIME type -sizeByte = "4294967296" # File size in bytes -``` +External consumers do **not** read these streams directly. They subscribe +via SSE: -### VideoMeta - Classification +- `GET /api/events/files` — mirror of `file:events` +- `GET /api/events/meta` — mirror of `meta:events` -``` -videoType = "movie" # movie, episode, unknown -originalTitle = "Inception" -titles/eng = "Inception" # Localized titles -titles/jpn = "インセプション" -season = "1" -episode = "5" -movieYear = "2010" -``` +The SSE wire contract uses `Last-Event-ID` for resume (the opaque +`-` Redis Stream entry ID). Heartbeats are SSE comments +(`:keep-alive\n\n`) every 30s of silence. If retention has trimmed past +a client's cursor, the handler emits one `event: gap` payload before +resuming. See `internal/api/sse_events.go` and +[`api-mediated-access.md`](api-mediated-access.md) for the full +contract. -### AnimeMeta - Anime Detection - -``` -anime = "true" # "true" or "false" -titles/jpn = "進撃の巨人" # Japanese title -titles/jpl = "Shingeki no Kyojin" # Romaji -``` - -### SubtitleMeta - Subtitle References - -``` -subtitles/eng/0 = "bafkr..." # Reference to subtitle file CID -subtitles/eng/1 = "bafks..." -subtitles/jpn/0 = "bafkt..." -``` - -### JellyfinMeta - Comprehensive Metadata - -``` -imdbid = "tt1375666" -tmdbid = "27205" -anidbid = "12345" -plot/eng = "A movie about..." -genre/0 = "Action" -genre/1 = "Sci-Fi" -rating = "8.8" -mpaa = "PG-13" -releasedate = "2010-07-16" -art/poster = "bafkr..." # Reference to poster image CID -art/fanart = "bafks..." # Reference to backdrop image CID -``` - -## FFmpeg Plugin - Special Case - -The FFmpeg plugin extracts detailed stream information with a specific nested structure. - -### Stream Details Structure - -``` -fileinfo/duration = "7200.5" -fileinfo/formatName = "matroska,webm" -fileinfo/streamdetails/video/0/codec = "h264" -fileinfo/streamdetails/video/0/width = "1920" -fileinfo/streamdetails/video/0/height = "1080" -fileinfo/streamdetails/video/0/aspect = "16:9" -fileinfo/streamdetails/video/0/bitrate = "8000000" -fileinfo/streamdetails/audio/0/codec = "aac" -fileinfo/streamdetails/audio/0/language = "eng" -fileinfo/streamdetails/audio/0/channels = "6" -fileinfo/streamdetails/audio/1/codec = "dts" -fileinfo/streamdetails/audio/1/language = "fra" -fileinfo/streamdetails/audio/1/channels = "6" -fileinfo/streamdetails/subtitle/0/language = "eng" -fileinfo/streamdetails/subtitle/0/format = "subrip" -fileinfo/streamdetails/subtitle/1/language = "jpn" -fileinfo/streamdetails/subtitle/1/format = "ass" -fileinfo/streamdetails/embeddedimage/0/type = "cover" -``` - -### Stream Type Keys - -| Stream Type | Key Pattern | Description | -|------------|-------------|-------------| -| Video | `fileinfo/streamdetails/video/{n}/...` | Video streams (0, 1, 2...) | -| Audio | `fileinfo/streamdetails/audio/{n}/...` | Audio tracks | -| Subtitle | `fileinfo/streamdetails/subtitle/{n}/...` | Embedded subtitles | -| Embedded Image | `fileinfo/streamdetails/embeddedimage/{n}/...` | Cover art, thumbnails | - -### Common Stream Fields - -**Video streams:** -- `codec` - h264, hevc, av1, vp9, etc. -- `width`, `height` - Resolution -- `aspect` - Aspect ratio -- `bitrate` - Video bitrate -- `profile` - Codec profile (main, high, etc.) -- `level` - Codec level - -**Audio streams:** -- `codec` - aac, dts, ac3, flac, etc. -- `language` - ISO 639-2 code (eng, jpn, fra) -- `channels` - Channel count (2, 6, 8) -- `bitrate` - Audio bitrate -- `samplerate` - Sample rate - -**Subtitle streams:** -- `language` - ISO 639-2 code -- `format` - subrip, ass, pgs, vobsub - -## Real-Time Notifications - -MetaMesh provides multiple mechanisms for real-time metadata change notifications: - -### 1. meta:events Stream (Recommended) - -The `meta:events` Redis stream provides reliable, ordered event delivery with consumer groups: - -```bash -# File system events from meta-core watcher -XREAD STREAMS meta:events 0 - -# Event types: add, change, delete, rename, reset -``` - -Benefits: -- **Reliable delivery** via consumer groups and acknowledgment -- **Replay capability** for catching up after restarts -- **Ordered processing** with stream IDs - -### 2. Keyspace Notifications - -For field-level change detection, subscribe to Redis keyspace notifications: - -```bash -# All file metadata changes -PSUBSCRIBE '__keyspace@0__:file:*' - -# Specific plugin updates (e.g., TMDB enrichment) -PSUBSCRIBE '__keyspace@0__:file:*/tmdb/*' -``` - -### Cache Invalidation - -The WebDAV cache in meta-core integrates with keyspace notifications for automatic invalidation: - -``` -┌─────────────────────────────────────────────────────────────────┐ -│ Cache Invalidation Flow │ -│ │ -│ Redis Keyspace Cache │ -│ ┌──────────────┐ Invalidator │ -│ │ SET file:*/ │ ──────────► ┌──────────────┐ │ -│ │ DEL file:*/ │ subscribe │ Listens to │ │ -│ └──────────────┘ │ __keyspace@ │ │ -│ │ 0__:file:* │ │ -│ └──────┬───────┘ │ -│ │ │ -│ ▼ │ -│ ┌──────────────┐ │ -│ │ Remove from │ │ -│ │ LRU cache │ │ -│ └──────────────┘ │ -└─────────────────────────────────────────────────────────────────┘ -``` - -Flow: -1. Cache invalidator subscribes to `__keyspace@0__:file:*` -2. When metadata changes (SET/DEL), invalidator receives notification -3. Invalidator removes the corresponding file from the WebDAV cache -4. Next access fetches fresh copy from source filesystem - -This ensures cached files stay synchronized with metadata changes. - -### 3. Pub/Sub Channels (Legacy) - -Batch notification channels are still supported: - -``` -meta-sort:file:batch # Batch file updates (every 5 seconds) -meta-sort:scan:reset # Full rescan notification -``` - -```json -{ - "timestamp": 1704067200000, - "changes": [ - { "action": "add", "hashId": "bafkr4ih5..." }, - { "action": "update", "hashId": "bafkr4ih5..." }, - { "action": "remove", "hashId": "bafkr4ih5..." } - ] -} -``` - -## Complete Example - -Here's a complete metadata example for a movie file: - -```bash -# Query all keys for a file -redis-cli --scan --pattern 'file:bafkr4ih5kapbjzq.../*' - -# Example flat keys stored in Redis: -# file:bafkr4ih5kapbjzq.../cid_midhash256 = "bafkr4ih5kapbjzq..." -# file:bafkr4ih5kapbjzq.../filePath = "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv" -# file:bafkr4ih5kapbjzq.../fileType = "video" -# file:bafkr4ih5kapbjzq.../fileinfo/duration = "8878.5" -# file:bafkr4ih5kapbjzq.../titles/eng = "Inception" -# ... - -# When reconstructed by the API (nested object representation) -{ - # Content identification - "cid_midhash256": "bafkr4ih5kapbjzq...", - "cid_sha2-256": "bafks7ij2kbpck...", - "filePath": "/files/watch/movies/Inception.2010.1080p.BluRay.x264.mkv", - - # File properties - "fileType": "video", - "mimeType": "video/x-matroska", - "sizeByte": "4294967296", - - # Classification - "videoType": "movie", - "originalTitle": "Inception", - "movieYear": "2010", - - # Localized titles - "titles": { - "eng": "Inception", - "fra": "Inception", - "jpn": "インセプション" - }, - - # FFmpeg stream details - "fileinfo": { - "duration": "8878.5", - "formatName": "matroska,webm", - "streamdetails": { - "video": { - "0": { - "codec": "h264", - "width": "1920", - "height": "1080", - "aspect": "16:9", - "bitrate": "8000000" - } - }, - "audio": { - "0": { "codec": "dts", "language": "eng", "channels": "6" }, - "1": { "codec": "ac3", "language": "fra", "channels": "6" } - }, - "subtitle": { - "0": { "language": "eng", "format": "subrip" }, - "1": { "language": "fra", "format": "subrip" } - } - } - }, - - # External metadata (TMDB) - "tmdbid": "27205", - "imdbid": "tt1375666", - "plot": { - "eng": "A thief who steals corporate secrets through dream-sharing technology..." - }, - "genre": ["Action", "Science Fiction", "Adventure"], - "rating": "8.4", - "releasedate": "2010-07-16", - - # Media assets - "art": { - "poster": "bafkr4poster...", - "fanart": "bafkr4fanart..." - }, - - # Detection flags - "anime": "false" -} -``` - -## Key Implementation Files - -| Component | File Path | -|-----------|-----------| -| Redis Client | `packages/meta-sort/packages/meta-sort-core/src/kv/RedisClient.ts` | -| Metadata Utils | `packages/meta-sort/packages/meta-sort-core/src/kv/MetadataUtils.ts` | -| KV Interface | `packages/meta-sort/packages/meta-sort-core/src/kv/IKVClient.ts` | -| File Processor | `packages/meta-sort/packages/meta-sort-core/src/logic/fileProcessor/FileProcessorPiscina.ts` | -| Pub/Sub Pipeline | `packages/meta-sort/packages/meta-sort-core/src/logic/pipeline/StreamingPipeline.ts` | -| Type Definitions | `packages/meta-interface/src/lib/metadata-type/*.ts` | -| meta-core Storage | `packages/meta-core/internal/storage/client.go` | -| meta-core Events | `packages/meta-core/internal/events/publisher.go` | -| meta-core Cache | `packages/meta-core/internal/cache/` | +Internally, meta-core uses a Redis consumer group (XREADGROUP / XACK) +inside `MetaPublisher` to drain keyspace notifications, but this is not +the external surface — external services use SSE. ## Debugging ```bash -# List all file hash IDs -docker exec meta-core-dev redis-cli SMEMBERS 'file:__index__' +# Roots +docker exec metacore-app redis-cli SMEMBERS 'file:__index__' +docker exec metacore-app redis-cli SCARD 'file:__index__' +docker exec metacore-app redis-cli SRANDMEMBER 'file:__index__' -# Count total files -docker exec meta-core-dev redis-cli SCARD 'file:__index__' +# Full document for one root (flat scan) +docker exec metacore-app redis-cli --scan --pattern 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/*' -# Get a sample hash ID -docker exec meta-core-dev redis-cli SRANDMEMBER 'file:__index__' +# One property +docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/fileinfo/duration' -# List all keys for a specific file (flat key scan) -docker exec meta-core-dev redis-cli --scan --pattern 'file:bafkr4ih5.../*' +# Confirm keys are STRING (not HASH) +docker exec metacore-app redis-cli TYPE 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/filePath' +# string -# Get specific field (flat key) -docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../fileinfo/duration' +# Reverse-index lookups +docker exec metacore-app redis-cli GET 'cid:midhash256:bafkreih...' # → uuid +docker exec metacore-app redis-cli SMEMBERS 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/cids' +docker exec metacore-app redis-cli GET 'file:01JKR8XW5T4QMV7AHNJ2DEFGHK/canonical_cid' -# Get filePath for a file -docker exec meta-core-dev redis-cli GET 'file:bafkr4ih5.../filePath' +# Schema-version sentinel +docker exec metacore-app redis-cli GET 'meta-core:schema-version' -# Verify keys are STRING type (not HASH) -docker exec meta-core-dev redis-cli TYPE 'file:bafkr4ih5.../filePath' -# Should return: string +# Streams (server-side; external consumers should use SSE) +docker exec metacore-app redis-cli XRANGE 'file:events' - + COUNT 10 +docker exec metacore-app redis-cli XRANGE 'meta:events' - + COUNT 10 -# Monitor metadata changes via stream -docker exec meta-core-dev redis-cli XREAD STREAMS 'meta:events' 0 - -# Monitor keyspace notifications (requires notify-keyspace-events enabled) -docker exec meta-core-dev redis-cli PSUBSCRIBE '__keyspace@0__:file:*' - -# meta-core API debugging -# Cache status -curl http://localhost:9000/api/cache/status - -# Clear cache -curl -X POST http://localhost:9000/api/cache/clear - -# Watch events stream (SSE) -curl http://localhost:9000/api/events/stream - -# KV browser - storage info -curl http://localhost:9000/api/kv/info - -# Scan status -curl http://localhost:9000/api/scan/status +# HTTP equivalents — preferred +curl -k 'https://metacore-dev.localhost:8083/api/meta/' # CID → document +curl -k 'https://metacore-dev.localhost:8083/meta' # list root IDs +curl -k 'https://metacore-dev.localhost:8083/meta/' # document by UUID +curl -k 'https://metacore-dev.localhost:8083/api/events/meta' # live SSE +curl -k 'https://metacore-dev.localhost:8083/api/kv/info' # KV stats ``` -## meta:events Stream +## Key implementation files -The `meta:events` stream captures metadata change events for real-time notifications: - -```bash -# View recent events -docker exec meta-core-dev redis-cli XRANGE 'meta:events' - + COUNT 10 - -# Read events from a consumer group -docker exec meta-core-dev redis-cli XREADGROUP GROUP mygroup myconsumer STREAMS 'meta:events' > -``` - -Stream events contain: -- `type`: Event type (add, change, delete, rename, reset) -- `path`: File path -- `midhash256`: Content hash (for add/change) -- `timestamp`: Event timestamp +| Component | Path | +|---|---| +| Redis client + flat-key ops | `internal/storage/client.go` | +| UUIDv7 minting | `internal/storage/uuid.go` | +| CID reverse index + aliases + canonical-CID reconciliation | `internal/storage/cid_resolution.go` | +| Schema-version gate | `internal/storage/schema_version.go` | +| Dual-root migration (legacy clean-up) | `internal/storage/dual_root_migration.go` | +| Keyspace-notification → `meta:events` publisher | `internal/events/meta_publisher.go` | +| Watcher → `file:events` dispatcher | `internal/watcher/dispatcher.go` | +| SSE event mirror | `internal/api/sse_events.go` | +| Field semantics (authoritative registry) | [`/METADATA_KEYS.md`](../../../METADATA_KEYS.md) | diff --git a/docs/uuid-rooted-metadata.md b/docs/uuid-rooted-metadata.md index 3453764..754b6af 100644 --- a/docs/uuid-rooted-metadata.md +++ b/docs/uuid-rooted-metadata.md @@ -2,9 +2,7 @@ ## Status -**Proposed** — design under discussion. Not yet implemented. Supersedes the -"midhash256 as primary key" model currently described in -[metadata-storage-structure.md](metadata-storage-structure.md). +**Status:** Implemented (2026 alpha clean-wipe gate landed). Kept for design rationale. ## Motivation diff --git a/editor/README.md b/editor/README.md index d66fc08..4850a9f 100644 --- a/editor/README.md +++ b/editor/README.md @@ -1,140 +1,76 @@ -# @metazla/meta-editor +# @meta-core/editor -User-friendly metadata editor for MetaMesh. Search and edit file metadata directly through a web interface. +React + Vite metadata browser/editor that ships inside the meta-core +container. Walks the UUID-rooted Redis keyspace, edits leaf values, +imports/exports snapshots, and consults the live schema indexer. -## Features +## What it talks to -- **Search**: Find files by title, filename, or hash ID -- **Edit**: Modify all metadata fields including title, season, episode, year, Jellyfin metadata, and more -- **Batch Operations**: Update multiple files simultaneously -- **Real-time**: Direct integration with etcd via REST API -- **User-friendly**: Clean, intuitive interface with form validation +The editor is a pure SPA — it has no server of its own. It calls the +meta-core HTTP API directly under three prefixes (see `src/api/`): -## Development +- `/api/kv/*` — slash-tree KV browser (`info`, `tree`, `search`, `find`, + `value`, `key/{key...}`). This is the UUID-exposing surface — keys look + like `file:/` and `cid::`. +- `/api/schema` — live per-field schema (type hints, value breakdowns), + produced by meta-core's schema indexer that consumes `meta:events`. +- `/api/snapshot/*` — snapshot export / import / wipe. -### Prerequisites +In production the editor is served by meta-core's nginx (the Vite build +output is copied into the image at the same path) under +`https://metacore-dev.localhost:8083/editor/`. The Vite `base` is set to +`/editor/` accordingly. -- Node.js 21.6.2+ -- pnpm 10.19.0+ -- Meta-mesh API server running (provides the `/api/metadata/*` endpoints) - -### Running Locally - -```bash -# From the root directory -pnpm run start:editor - -# Or from the package directory -cd packages/meta-editor -pnpm run dev -``` - -The editor will be available at `http://localhost:5173` with API proxying to `http://localhost:3000`. - -### Building - -```bash -# From the root directory -pnpm run build:editor - -# Or from the package directory -cd packages/meta-editor -pnpm run build -``` - -The built files will be in `dist/` directory. - -## Docker Deployment - -In the Docker development environment, the editor is automatically available at: +## Project layout ``` -http://localhost/editor -``` - -The nginx configuration serves the built files from `/app/packages/meta-editor/dist/`. - -## API Integration - -The editor communicates with the UnifiedAPIServer (meta-mesh) via REST API: - -- `POST /api/metadata/search` - Search for files -- `GET /api/metadata/:hashId` - Get file metadata -- `PUT /api/metadata/:hashId` - Update complete metadata -- `PUT /api/metadata/:hashId/property` - Update specific property -- `POST /api/metadata/batch` - Batch update multiple files - -See `src/api/metadataApi.ts` for the API client implementation. - -## Usage - -### Searching for Files - -1. Select search type: "Search by Title/Filename" or "Search by Hash ID" -2. Enter your search query -3. Click "Search" -4. Results will appear in the left sidebar - -### Editing Metadata - -1. Select a file from the search results -2. Modify any fields in the editor form -3. Click "Save Changes" to persist updates -4. Use "Reset" to discard changes - -### Supported Metadata Fields - -**Basic Information:** -- Title, Original Title, Show Title -- Video Type (Movie, TV, Anime, Other) -- Year, Season, Episode - -**Jellyfin Metadata:** -- Plot, IMDb ID, TMDb ID, AniDB ID -- Director, Studio, Rating, Runtime -- Genres, Premiered Date, Release Date - -**File Information:** -- Filename, File Size, MIME Type -- Hash values (SHA-256, MD5, etc.) - read-only - -## Project Structure - -``` -packages/meta-editor/ -├── src/ -│ ├── api/ # API client for metadata operations -│ ├── components/ # React components -│ │ ├── SearchBar.tsx # Search interface -│ │ ├── FileList.tsx # Search results list -│ │ └── MetadataEditor.tsx # Main editing form -│ ├── types/ # TypeScript type definitions -│ ├── utils/ # Utility functions -│ ├── App.tsx # Main application component -│ ├── main.tsx # Entry point -│ └── index.css # Global styles +packages/meta-core/editor/ ├── index.html ├── vite.config.ts ├── tsconfig.json -└── package.json +├── package.json @meta-core/editor +└── src/ + ├── main.tsx entry + ├── api/ + │ ├── kvApi.ts /api/kv client + │ ├── schemaApi.ts /api/schema client + cache + │ └── snapshotApi.ts /api/snapshot client + ├── kv/ KV browser + leaf/branch editors + ├── components/ shared widgets (language combobox, snapshot panel, ...) + └── index.css ``` -## Technology Stack +## Local development -- **React 18** - UI framework -- **TypeScript 5.7** - Type safety -- **Vite 6** - Build tool and dev server -- **Native CSS** - Styling (no CSS framework) +Prerequisites: Node 21.6.2+, pnpm pinned in the root `package.json`. -## Contributing +```bash +cd packages/meta-core/editor +pnpm install +pnpm dev # vite dev server on http://localhost:5173 +``` -When adding new features: +The Vite dev server proxies `/api/*` to `http://localhost:3000` (see +`vite.config.ts`). For local development against a running meta-core +container, change that target to wherever meta-core is reachable on your +host — e.g. `http://localhost:18083` for the dev compose's direct-backend +port. -1. Add type definitions to `src/types/index.ts` -2. Update API client in `src/api/metadataApi.ts` -3. Create/update components as needed -4. Update this README with new features +No root-level `start:editor` or `build:editor` scripts exist; run pnpm +commands from this directory. -## License +## Build -Part of the MetaMesh project. +```bash +pnpm build # tsc -b && vite build → dist/ +``` + +The container build copies `dist/` into the nginx web root so the editor +is served at `/editor/`. + +## Tech + +- React 18, TypeScript 5 +- Vite 5 (`base: '/editor/'`) +- Native CSS — no UI framework +- `iso-639-3` for language code rendering diff --git a/editor/src/kv/KVTree.tsx b/editor/src/kv/KVTree.tsx index 8b73bf8..fb66fd3 100644 --- a/editor/src/kv/KVTree.tsx +++ b/editor/src/kv/KVTree.tsx @@ -79,12 +79,24 @@ function labelOf(path: string, parentPath: string): string { return rest.replace(/[/:]+$/, ''); } -// Hide internal Redis structures from the tree: -// - the `meta:` namespace (event streams, internal state) -// - bare top-level leaves under a colon-namespace like file:__index__ and -// file:events (Redis sets / streams that aren't user data). +// Hide internal Redis structures from the tree. The editor is for managing +// file metadata; everything else is meta-core's own bookkeeping and noise +// for the user. Hidden: +// - `meta:` — event streams (file:events, meta:events, etc.) +// - `cid:` — reverse index (cid:: → uuid), derived +// from each file's cid_* fields; the editor surfaces +// these per-file under their parent root +// - `meta-core:` — internal service state (mounts, watcher config, +// leader info), edited via dedicated UI pages, not +// the KV tree +// - bare top-level leaves under a colon-namespace like file:__index__ +// and file:events (Redis sets / streams that aren't user data). function isHiddenNode(node: TreeNode): boolean { - if (node.kind === 'branch' && node.path === 'meta:') return true; + if (node.kind === 'branch') { + if (node.path === 'meta:' || node.path === 'cid:' || node.path === 'meta-core:') { + return true; + } + } if (node.kind === 'leaf' && /^[^/]+:[^/]+$/.test(node.path)) return true; return false; } diff --git a/internal/config/config.go b/internal/config/config.go index 9b7e30f..8cf51d4 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -132,3 +132,20 @@ func getEnvBool(key string, defaultValue bool) bool { func (c *Config) DefaultWatchPath() string { return c.FilesPath + "/watch" } + +// DefaultWatcherPaths returns every directory that should be watched out of +// the box on a fresh install. The watcher seeds these on first start. +// +// - /files/watch — user-curated media. The main library. +// - /files/plugin — plugin output (TMDB posters/backdrops, extracted +// subtitles, etc.). Indexed so each artefact gets a +// midhash256 alias and CID-based references from +// other files (e.g. `backdrop=`) resolve through +// the standard reverse index instead of needing a +// path companion field. +func (c *Config) DefaultWatcherPaths() []string { + return []string{ + c.FilesPath + "/watch", + c.FilesPath + "/plugin", + } +} diff --git a/internal/mounts/handlers.go b/internal/mounts/handlers.go index 9f2ba13..b5da499 100644 --- a/internal/mounts/handlers.go +++ b/internal/mounts/handlers.go @@ -292,13 +292,35 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) { json.NewEncoder(w).Encode(data) } +// writeError writes the typed error envelope (see api/handlers.go for the +// canonical definition + slug vocabulary). Kept local to avoid an import +// cycle with the api package. func writeError(w http.ResponseWriter, status int, message string) { - writeJSON(w, status, map[string]string{ - "error": http.StatusText(status), - "message": message, + slug, retryable := slugForStatus(status) + writeJSON(w, status, map[string]interface{}{ + "error": slug, + "message": message, + "retryable": retryable, }) } +func slugForStatus(status int) (string, bool) { + switch status { + case http.StatusConflict: + return "alias_collision", false + case http.StatusNotFound: + return "unknown_root", false + case http.StatusBadRequest: + return "schema_violation", false + case http.StatusServiceUnavailable: + return "storage_unavailable", true + case http.StatusInternalServerError: + return "internal", true + default: + return "internal", status >= 500 + } +} + func parseIntOrDefault(s string, defaultVal int) (int, error) { var result int if s == "" { diff --git a/internal/storage/cid_resolution.go b/internal/storage/cid_resolution.go index faef938..c50e93e 100644 --- a/internal/storage/cid_resolution.go +++ b/internal/storage/cid_resolution.go @@ -462,39 +462,15 @@ func (c *Client) GetMetadataDocument(uuid string) (*MetadataDocument, error) { }, nil } -// lookupSidecarPathByCID is the fallback that LookupPathByCID falls into -// when the token isn't a known root alias. Used to resolve poster/backdrop -// CIDs, which are stored as values pointing at sidecar files (posterPath / -// backdropPath) rather than registered as aliases of the file itself. +// lookupSidecarPathByCID — REMOVED. // -// O(n) — walks every root. Acceptable because (a) sidecar lookups are rare -// vs. file CID lookups and (b) the alternative is registering sidecars in -// the reverse index, which conflates "this CID is the file" with "this CID -// is an image associated with the file." -func (c *Client) lookupSidecarPathByCID(token string) (string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - hashIDs, err := c.getAllHashIDsInternal(ctx) - if err != nil { - return "", err - } - - for _, hashID := range hashIDs { - meta, err := c.getMetadataFlatInternal(ctx, hashID) - if err != nil || len(meta) == 0 { - continue - } - if meta["poster"] == token { - if p := meta["posterPath"]; p != "" { - return normalizeFilesRelativePath(p), nil - } - } - if meta["backdrop"] == token { - if p := meta["backdropPath"]; p != "" { - return normalizeFilesRelativePath(p), nil - } - } - } - return "", nil -} +// This function used to walk every metadata entry looking for poster/backdrop +// matches and return their `posterPath`/`backdropPath` companion fields. It +// was a workaround for plugin output (TMDB images, etc.) not being indexed +// by the watcher. +// +// Now that the plugin output directory is a first-class watcher root +// (see config.DefaultWatcherPaths), every plugin artefact has its own +// midhash256 alias and resolves through the standard reverse index. Storing +// paths in metadata broke the CID-as-identity contract; the watcher +// indexing fixes the root cause. diff --git a/internal/storage/client.go b/internal/storage/client.go index f378a4b..c0850e3 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -347,33 +347,40 @@ func (c *Client) CountFiles() (int, error) { return len(hashIDs), nil } -// LookupPathByCID resolves a CID to its file path on disk. +// LookupPathByCID resolves a CID to its file path on disk via the reverse +// index. O(1) lookup followed by a single GET of file:/filePath. // -// Primary path: the reverse index (cid: → uuid). O(1) lookup followed -// by a single GET of file:/filePath. +// Accepts both prefixed tokens ("midhash256:bafk…", "sha256:…", +// "ipfs:bafy…") and bare midhash256 CIDs — the bare form is what the +// editor passes to /api/file/{cid}/info because that's how plugins +// write the value (e.g. `backdrop=`). For bare tokens we +// try the midhash256 prefix as a fallback. // -// Fallback path: poster/backdrop CIDs aren't registered as aliases (they -// refer to sidecar images, not the main file content), so unknown tokens -// fall through to a scan of poster/backdrop fields. O(n) but rare in -// practice — sidecar lookups are infrequent compared to file CID lookups. -// -// Accepts CID tokens in prefixed form ("midhash256:bafk…", "sha256:…", -// "ipfs:bafy…"). Bare CIDs without an algorithm prefix won't resolve via -// the reverse index — callers should pass the prefixed form. +// Plugin output (TMDB posters/backdrops, extracted subtitles, etc.) is +// resolved through this same path: the plugin output directory is a +// watcher root (see config.DefaultWatcherPaths), so every plugin +// artefact gets a midhash256 alias and resolves through the reverse +// index like any other file. There is no path-companion sidecar field. func (c *Client) LookupPathByCID(token string) (string, error) { uuid, err := c.GetByCID(token) if err != nil { return "", err } - if uuid != "" { - fp, err := c.GetProperty(uuid, "filePath") + if uuid == "" && !strings.Contains(token, ":") { + // Bare CID — most likely a midhash256 written by a plugin. + uuid, err = c.GetByCID("midhash256:" + token) if err != nil { return "", err } - return normalizeFilesRelativePath(fp), nil } - // Not a file alias — try poster/backdrop sidecars. - return c.lookupSidecarPathByCID(token) + if uuid == "" { + return "", nil + } + fp, err := c.GetProperty(uuid, "filePath") + if err != nil { + return "", err + } + return normalizeFilesRelativePath(fp), nil } // normalizeFilesRelativePath strips a leading "/files/" prefix so handlers can diff --git a/internal/watcher/handlers.go b/internal/watcher/handlers.go index dcf2cb7..9238d6c 100644 --- a/internal/watcher/handlers.go +++ b/internal/watcher/handlers.go @@ -100,9 +100,31 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) { json.NewEncoder(w).Encode(data) } +// writeError writes the typed error envelope (see api/handlers.go for the +// canonical definition + slug vocabulary). Kept local to avoid an import +// cycle with the api package. func writeError(w http.ResponseWriter, status int, message string) { - writeJSON(w, status, map[string]string{ - "error": http.StatusText(status), - "message": message, + slug, retryable := slugForStatus(status) + writeJSON(w, status, map[string]interface{}{ + "error": slug, + "message": message, + "retryable": retryable, }) } + +func slugForStatus(status int) (string, bool) { + switch status { + case http.StatusConflict: + return "alias_collision", false + case http.StatusNotFound: + return "unknown_root", false + case http.StatusBadRequest: + return "schema_violation", false + case http.StatusServiceUnavailable: + return "storage_unavailable", true + case http.StatusInternalServerError: + return "internal", true + default: + return "internal", status >= 500 + } +} diff --git a/internal/watchers/handlers.go b/internal/watchers/handlers.go index 986851b..24dd382 100644 --- a/internal/watchers/handlers.go +++ b/internal/watchers/handlers.go @@ -242,9 +242,31 @@ func writeJSON(w http.ResponseWriter, status int, data interface{}) { json.NewEncoder(w).Encode(data) } +// writeError writes the typed error envelope (see api/handlers.go for the +// canonical definition + slug vocabulary). Kept local to avoid an import +// cycle with the api package. func writeError(w http.ResponseWriter, status int, message string) { - writeJSON(w, status, map[string]string{ - "error": http.StatusText(status), - "message": message, + slug, retryable := slugForStatus(status) + writeJSON(w, status, map[string]interface{}{ + "error": slug, + "message": message, + "retryable": retryable, }) } + +func slugForStatus(status int) (string, bool) { + switch status { + case http.StatusConflict: + return "alias_collision", false + case http.StatusNotFound: + return "unknown_root", false + case http.StatusBadRequest: + return "schema_violation", false + case http.StatusServiceUnavailable: + return "storage_unavailable", true + case http.StatusInternalServerError: + return "internal", true + default: + return "internal", status >= 500 + } +} diff --git a/internal/watchers/manager.go b/internal/watchers/manager.go index f2c205a..2896084 100644 --- a/internal/watchers/manager.go +++ b/internal/watchers/manager.go @@ -55,47 +55,86 @@ func (m *Manager) SetOnChanged(cb func()) { m.onChanged = cb } -// loadOrCreateDefault loads existing config or creates default watcher +// loadOrCreateDefault loads existing config or creates default watchers. +// Also reconciles after load: any path in DefaultWatcherPaths that isn't +// yet present gets added. This catches deployments whose watchers.json +// pre-dates the addition of a new default (e.g. /files/plugin), without +// requiring operators to manually edit JSON. func (m *Manager) loadOrCreateDefault() error { - // Try to load existing file if _, err := os.Stat(m.watchersFile); err == nil { - return m.load() + if err := m.load(); err != nil { + return err + } + return m.ensureDefaultWatchers() } - // File doesn't exist, create default watcher for /files/watch - log.Printf("[Watchers] No existing config, creating default watcher for %s", m.config.DefaultWatchPath()) - return m.createDefaultWatcher() + log.Printf("[Watchers] No existing config, creating defaults") + return m.createDefaultWatchers() } -// createDefaultWatcher creates the default /files/watch watcher and folder -func (m *Manager) createDefaultWatcher() error { +// ensureDefaultWatchers makes sure every path in DefaultWatcherPaths has a +// corresponding watcher entry. Existing entries (matched by path) are left +// untouched — only missing ones are added. +func (m *Manager) ensureDefaultWatchers() error { m.mu.Lock() defer m.mu.Unlock() - defaultPath := m.config.DefaultWatchPath() + existing := make(map[string]bool, len(m.watchers)) + for _, w := range m.watchers { + existing[w.Path] = true + } - // Create the watch folder if it doesn't exist - if err := os.MkdirAll(defaultPath, 0755); err != nil { - log.Printf("[Watchers] Warning: failed to create default watch folder %s: %v", defaultPath, err) - // Continue anyway - folder might be created later or mounted + added := false + for _, p := range m.config.DefaultWatcherPaths() { + if existing[p] { + continue + } + if err := m.addDefaultWatcherUnsafe(p); err != nil { + log.Printf("[Watchers] Warning: failed to add default watcher %s: %v", p, err) + continue + } + added = true + } + if !added { + return nil + } + return m.saveUnsafe() +} + +// createDefaultWatchers seeds the watcher config with every entry in +// DefaultWatcherPaths. Used on first start when no config file exists. +func (m *Manager) createDefaultWatchers() error { + m.mu.Lock() + defer m.mu.Unlock() + + for _, p := range m.config.DefaultWatcherPaths() { + if err := m.addDefaultWatcherUnsafe(p); err != nil { + log.Printf("[Watchers] Warning: failed to seed default watcher %s: %v", p, err) + } + } + return m.saveUnsafe() +} + +// addDefaultWatcherUnsafe creates the on-disk folder (if missing) and adds +// a watcher entry. Caller must hold m.mu. +func (m *Manager) addDefaultWatcherUnsafe(path string) error { + if err := os.MkdirAll(path, 0755); err != nil { + log.Printf("[Watchers] Warning: failed to create folder %s: %v", path, err) + // Continue — folder might be mounted later } else { - log.Printf("[Watchers] Created default watch folder: %s", defaultPath) + log.Printf("[Watchers] Created folder: %s", path) } - // Create default watcher id := uuid.New().String()[:8] - watcher := &WatcherConfig{ + m.watchers[id] = &WatcherConfig{ ID: id, - Path: defaultPath, + Path: path, IntervalMs: DefaultIntervalMs, Enabled: true, CreatedAt: NowMS(), } - m.watchers[id] = watcher - log.Printf("[Watchers] Created default watcher: %s -> %s", id, defaultPath) - - // Save config - return m.saveUnsafe() + log.Printf("[Watchers] Created default watcher: %s -> %s", id, path) + return nil } // load loads watchers from file diff --git a/internal/watchers/poller.go b/internal/watchers/poller.go index 100fab8..d014332 100644 --- a/internal/watchers/poller.go +++ b/internal/watchers/poller.go @@ -68,14 +68,20 @@ func (p *Poller) Start() error { log.Println("[WatcherPoller] Starting file watcher polling service") - // Clear file:events stream on startup to prevent duplicate events from accumulating - // across container restarts. Since in-memory state is empty, all files will be - // re-emitted as "add" events - we need a fresh stream to match. - if p.dispatcher != nil { - if err := p.dispatcher.EmitReset("startup"); err != nil { - log.Printf("[WatcherPoller] Warning: failed to clear stream on startup: %v", err) - } - } + // No startup reset. The file:events stream is bounded by MAXLEN + // (api-mediated-access PR A) and SSE consumers resume from their + // persisted Last-Event-ID, so a startup wipe would only orphan + // consumer state without buying anything. The historical reason for + // wiping — "in-memory state is empty, all files re-emit as add" — + // stopped being true when HydrateStateFromStorage landed: state is + // hydrated from Redis before this runs, so a startup EmitReset would + // tell consumers to throw out their world without sending the + // re-emission they need to rebuild it. + // + // Operator-triggered resets (POST /api/watchers/{id}/reset) still + // work — that path goes through TriggerReset, which clears both the + // stream AND the watcher's in-memory state, so the next scan tick + // emits add events for every file. // Initial sync with watcher configurations go p.syncWithWatchers()