Compare commits
20
Commits
ec1b227631
...
v2.0.0
| Author | SHA1 | Date | |
|---|---|---|---|
 Worph
|
84af298792 | ||
|
Manielle
|
b0adefb97b | ||
|
Worph
|
5369dc4b9c | ||
|
Worph
|
fb4bdad61b | ||
|
Worph
|
dc5cbdfb9b | ||
|
Worph
|
c53146e514 | ||
|
Worph
|
bb04a3f4dc | ||
|
Worph
|
36eb3543b9 | ||
|
Worph
|
155fd83392 | ||
|
Worph
|
cf63a8da68 | ||
|
Worph
|
27ed45999b | ||
|
Worph
|
9254f043a4 | ||
|
Worph
|
ebc894fa42 | ||
|
Worph
|
515ec04369 | ||
|
Worph
|
0403a6f3bc | ||
|
Worph
|
355c666183 | ||
|
Worph
|
0cccceccd9 | ||
|
Worph
|
40b553016c | ||
|
Worph
|
4894810fd4 | ||
|
Worph
|
ac21315255 |
@@ -0,0 +1,90 @@
|
||||
# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
|
||||
# folder exists, and GitHub never reads it.
|
||||
#
|
||||
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
||||
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
||||
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
||||
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
||||
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
||||
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
||||
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
||||
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
||||
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
|
||||
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
|
||||
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
|
||||
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Tag for the image (optional, defaults to sha)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
REGISTRY: ipcr-push-85-17-246-67.nip.io
|
||||
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Docker CLI
|
||||
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
|
||||
|
||||
- name: Trust the Mesh Router CA
|
||||
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
|
||||
env:
|
||||
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
buildkitd-config-inline: |
|
||||
[registry."${{ env.REGISTRY }}"]
|
||||
ca = ["/tmp/ipcr-ca.pem"]
|
||||
|
||||
- name: Log in to IPCR
|
||||
env:
|
||||
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
|
||||
run: |
|
||||
mkdir -p ~/.docker
|
||||
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
@@ -45,6 +45,9 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
+10
-9
@@ -6,20 +6,21 @@
|
||||
# - Bash-based leader election script (flock on shared volume)
|
||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||
# - nginx (dashboard, WebDAV, API proxy)
|
||||
# - Redis (managed by supervisord on leader)
|
||||
# - SQLite metadata store, embedded in the Go binary: truth /meta-core/db/meta-core.db
|
||||
# (signed claims, backed up) + cache /meta-core/db-cache/cache.db (rebuildable)
|
||||
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
|
||||
# - supervisord (process management - runs on leader only)
|
||||
#
|
||||
# Architecture:
|
||||
# - leader-election.sh handles flock, starts supervisord on leader
|
||||
# - Followers remain dormant, retry flock every 5s
|
||||
# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
|
||||
# - Only leader runs meta-core, nginx, rclone, mount-watcher
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 1: Build Go binary
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM golang:1.21-alpine AS go-builder
|
||||
FROM golang:1.26-alpine AS go-builder
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
@@ -74,7 +75,7 @@ ARG RCLONE_VERSION=1.73.5
|
||||
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
|
||||
# all remote mounts go through rclone now.
|
||||
RUN apk add --no-cache \
|
||||
redis \
|
||||
sqlite \
|
||||
nginx \
|
||||
nginx-mod-http-dav-ext \
|
||||
supervisor \
|
||||
@@ -104,7 +105,8 @@ RUN chmod +x /usr/local/bin/meta-core
|
||||
# Create directories
|
||||
RUN mkdir -p \
|
||||
/meta-core/locks \
|
||||
/meta-core/db/redis \
|
||||
/meta-core/db \
|
||||
/meta-core/db-cache \
|
||||
/meta-core/services \
|
||||
/meta-core/mounts/errors \
|
||||
/files \
|
||||
@@ -140,13 +142,12 @@ ENV META_CORE_PATH=/meta-core \
|
||||
SERVICE_VERSION=1.0.0 \
|
||||
META_CORE_HTTP_PORT=9000 \
|
||||
META_CORE_HTTP_HOST=0.0.0.0 \
|
||||
REDIS_PORT=6379 \
|
||||
WATCH_INTERVAL_MS=1000
|
||||
|
||||
# Expose ports
|
||||
# 80: nginx (dashboard + API proxy + WebDAV)
|
||||
# 6379: Redis (for other services)
|
||||
EXPOSE 80 6379
|
||||
# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port:
|
||||
# it is a file inside this container, reachable only through the HTTP API.
|
||||
EXPOSE 80
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# Variables
|
||||
BINARY := meta-core
|
||||
VERSION ?= 1.0.0
|
||||
VERSION ?= 2.0.0
|
||||
BUILD_DIR := bin
|
||||
DOCKER_IMAGE := meta-core
|
||||
|
||||
|
||||
@@ -1,39 +1,50 @@
|
||||
# meta-core
|
||||
|
||||
Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
|
||||
metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
|
||||
Standalone Go service that owns the metadata store (an embedded SQLite file),
|
||||
the HTTP+SSE metadata surface, the
|
||||
User Data Layer store and signing keystore, and a WebDAV file server for the
|
||||
MetaMesh dev/prod stack.
|
||||
|
||||
## Overview
|
||||
|
||||
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
|
||||
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
|
||||
reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
|
||||
is not exposed outside the container — all metadata I/O is mediated by the
|
||||
HTTP API.
|
||||
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
|
||||
meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
|
||||
its `metamesh.core` advertise carries the full `/urls` payload) and then call its
|
||||
HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded
|
||||
in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API
|
||||
(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
|
||||
|
||||
### Key capabilities
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
|
||||
| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
|
||||
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
|
||||
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
|
||||
| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. |
|
||||
| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). |
|
||||
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
|
||||
| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field_v` over (field name, value) of every record key, plus an FTS5 trigram table; the smallest part drives, the other filters are probed per candidate — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). |
|
||||
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. |
|
||||
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
|
||||
| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
|
||||
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
|
||||
| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
|
||||
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
|
||||
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
|
||||
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
|
||||
| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
|
||||
| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
|
||||
| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
|
||||
|
||||
### Design characteristics
|
||||
|
||||
- **Single-leader** — flock on a shared volume; no external consensus.
|
||||
- **HTTP-only externally** — Redis is bound to the container, never published.
|
||||
- **Single writer** — flock on the `/meta-core` volume; no external consensus.
|
||||
- **HTTP-only externally** — there is no database port; debug read-only with
|
||||
`docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …`
|
||||
(dev stacks: `/meta-core-db/meta-core.db`).
|
||||
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
|
||||
on-disk Redis layout predates the current build (alpha clean-wipe policy;
|
||||
on-disk store layout predates the current build (alpha clean-wipe policy;
|
||||
see `docs/uuid-rooted-metadata.md`).
|
||||
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
|
||||
image with `redis-server`, `rclone`, nginx, and supervisord.
|
||||
- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a
|
||||
thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI
|
||||
(for debugging).
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -41,34 +52,31 @@ HTTP API.
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ metacore-app (standalone container) │
|
||||
│ │
|
||||
│ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
|
||||
│ then exec supervisord) │
|
||||
│ supervisord │
|
||||
│ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
|
||||
│ │ writes kv-leader.info, exec supervisord) │
|
||||
│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
|
||||
│ ├─ nginx (TLS termination, proxy_cache for /webdav) │
|
||||
│ ├─ rclone rcd (mount daemon, RC API) │
|
||||
│ ├─ nginx :80 (dashboard, /editor, API proxy, │
|
||||
│ │ proxy_cache for /webdav, rclone GUI /rclone) │
|
||||
│ ├─ rclone rcd (mount daemon, RC API on :5572) │
|
||||
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
|
||||
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
|
||||
│ ─► embedded SQLite (db/meta-core.db) │
|
||||
│ ─► UDP advertise/listen 239.255.99.1:9099 │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌──────────────────────┴──────────────────────┐
|
||||
│ /meta-core (shared volume) │
|
||||
│ locks/kv-leader.{lock,info} │
|
||||
│ db/redis/ │
|
||||
│ services/*.json │
|
||||
│ /meta-core (meta-core's own volume) │
|
||||
│ locks/kv-leader.lock │
|
||||
│ db/meta-core.db (+ -wal, -shm) │
|
||||
│ identity/accounts/*.json │
|
||||
│ mounts/{mounts.json, errors/} │
|
||||
│ watchers.json │
|
||||
│ cache/ (nginx proxy_cache) │
|
||||
└─────────────────────────────────────────────┘
|
||||
▲
|
||||
┌──────────────────────────────┼────────────────────────────────┐
|
||||
│ │ │
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ meta-sort │ │ meta-fuse │ │ meta-stremio│
|
||||
│ │ │ │ │ │
|
||||
│ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │
|
||||
│ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│
|
||||
└─────────────┘ └─────────────┘ └─────────────┘
|
||||
|
||||
meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
|
||||
│ 1. hear the metamesh.core advertise on UDP multicast (carries /urls)
|
||||
└─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume
|
||||
```
|
||||
|
||||
## Quick start
|
||||
@@ -82,8 +90,9 @@ make test # run Go unit tests
|
||||
```
|
||||
|
||||
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
|
||||
`:local` tag is the one every sub-stack docker-compose.yml references
|
||||
(meta-share 10-peer federation, meta-gateway dev stacks); without it the
|
||||
`:local` tag is the one the sub-stack docker-compose files reference
|
||||
(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
|
||||
dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
|
||||
sub-stacks can't see a freshly-built image by name.
|
||||
|
||||
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
|
||||
@@ -94,12 +103,14 @@ sub-stacks can't see a freshly-built image by name.
|
||||
>
|
||||
> ```bash
|
||||
> make docker
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metagateway-generic-core
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
|
||||
> up -d --force-recreate metagateway-torznab-core
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
|
||||
> up -d --force-recreate metagateway-hub-metacore
|
||||
> docker compose -f packages/meta-share/dev/docker-compose.yml \
|
||||
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
|
||||
> docker compose -f packages/meta-read/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metaread-dev-core
|
||||
> docker compose -f packages/meta-listen/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metalisten-dev-core
|
||||
> ```
|
||||
>
|
||||
> The symptom of a missed recreate is missing API fields on `/urls`
|
||||
@@ -109,7 +120,7 @@ sub-stacks can't see a freshly-built image by name.
|
||||
>
|
||||
> ```bash
|
||||
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
|
||||
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
|
||||
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}' # repeat for :latest
|
||||
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
|
||||
> ```
|
||||
|
||||
@@ -126,36 +137,43 @@ HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables consumed by `internal/config/config.go`:
|
||||
Environment variables (from `internal/config/config.go` unless noted):
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
|
||||
| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. |
|
||||
| `FILES_PATH` | `/files` | Files volume root. |
|
||||
| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
|
||||
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
|
||||
| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
|
||||
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
|
||||
| `REDIS_PORT` | `6379` | Local Redis port. |
|
||||
| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
|
||||
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
|
||||
| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
|
||||
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
|
||||
| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
|
||||
| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). |
|
||||
| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). |
|
||||
| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
|
||||
| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
|
||||
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
|
||||
| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
|
||||
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
|
||||
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
|
||||
| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
|
||||
| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
|
||||
| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
|
||||
| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
|
||||
| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
|
||||
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
|
||||
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
|
||||
| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
|
||||
| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
|
||||
| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
|
||||
| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
|
||||
|
||||
Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
|
||||
no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
|
||||
identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
|
||||
retry interval (`ELECTION_RETRY_SECS`, default `5`).
|
||||
`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
|
||||
`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
|
||||
`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
|
||||
them (leftovers of the file-based registry); setting them has no effect. There
|
||||
is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES`
|
||||
(gone with Redis and the in-memory search index; search results now include
|
||||
`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` /
|
||||
`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
|
||||
configs created via `/api/watchers`.
|
||||
|
||||
## API reference
|
||||
|
||||
All endpoints below are served by `internal/api/server.go`. Examples assume
|
||||
All endpoints below are registered in `internal/api/server.go` (mounts,
|
||||
watcher and watchers routes in their own packages' `handlers.go`). Examples assume
|
||||
the in-container Go port `:9000`; from the host use
|
||||
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
|
||||
(debug-direct).
|
||||
@@ -163,16 +181,47 @@ the in-container Go port `:9000`; from the host use
|
||||
### Health, leader, URLs
|
||||
|
||||
```bash
|
||||
GET /health # storage + role
|
||||
GET /health # storage + role (`redis: bool` kept for compatibility = store open)
|
||||
GET /status # version, uptime, etc.
|
||||
GET /leader # current leader info (hostname, PID, timestamps)
|
||||
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
|
||||
GET /services # registered services + heartbeats
|
||||
GET /api/services # alias for /services
|
||||
GET /services/{name} # one service
|
||||
GET /services/cleanup/stats
|
||||
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility)
|
||||
GET /neighbors # neighbours heard over UDP (beacon v2)
|
||||
GET /api/neighbors # same, under /api
|
||||
GET /api/stats # dashboard counters (see below)
|
||||
```
|
||||
|
||||
`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
|
||||
are **gone** — the registration-and-heartbeat registry they served was replaced
|
||||
by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
|
||||
with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
|
||||
than a 404; that is what made the dashboard fail with
|
||||
`SyntaxError: Unexpected token '<'` until 1.0.21.
|
||||
|
||||
#### `GET /api/stats`
|
||||
|
||||
Every headline counter in one request, split by cost:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"records": 88709, // count of idx:roots/* — metadata roots, file or not
|
||||
"redisKeys": 3115447, // kv row count (name kept for compatibility; cached 60 s)
|
||||
"redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility)
|
||||
"identities": 3, // signing accounts on disk
|
||||
"files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep
|
||||
"udlUsers": 5, // accounts holding User Data Layer records
|
||||
"sweptAt": 1789402401970,
|
||||
"sweeping": false
|
||||
}
|
||||
```
|
||||
|
||||
`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and
|
||||
`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
|
||||
box), so they are served from a single-flight cache refreshed in the background
|
||||
with a 60s TTL — the walk is **never** on the request path, because the
|
||||
dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this
|
||||
over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
|
||||
across records that share a file.
|
||||
|
||||
### Metadata — primary surface
|
||||
|
||||
These are the routes other services use. `{hash}` may be a UUID root or
|
||||
@@ -198,6 +247,8 @@ GET /api/meta/{cid} # resolve CID → metadata document
|
||||
GET /api/file/{cid}/info # CID resolution metadata
|
||||
GET /file/{cid} # serve file bytes (range-aware)
|
||||
POST /file/cid # compute CID for an uploaded file
|
||||
POST /api/files/register # mint + alias one file under /files/plugin by midhash
|
||||
# (the watcher's path for one file; works with the watcher OFF)
|
||||
HEAD /data/{hash} # existence + size
|
||||
GET /data/{hash}/path # absolute path on /files
|
||||
```
|
||||
@@ -209,6 +260,7 @@ the `/file/{cid}` reverse-lookup path.)
|
||||
|
||||
```bash
|
||||
GET /api/metadata/hash-ids
|
||||
GET /api/files/tuples
|
||||
GET /api/metadata/list
|
||||
POST /api/metadata/search
|
||||
POST /api/metadata/batch
|
||||
@@ -221,10 +273,10 @@ PUT /api/metadata/{hashId}/property
|
||||
|
||||
GET /api/kv/info
|
||||
GET /api/kv/keys
|
||||
GET /api/kv/tree
|
||||
GET /api/kv/tree # cursor is an opaque next-key ("0" when exhausted)
|
||||
GET /api/kv/search
|
||||
GET /api/kv/find
|
||||
GET /api/kv/value
|
||||
GET /api/kv/value # type is always "string" for an existing key
|
||||
PUT /api/kv/value
|
||||
DELETE /api/kv/value
|
||||
GET /api/kv/key/{key...}
|
||||
@@ -239,7 +291,8 @@ POST /api/snapshot/wipe
|
||||
|
||||
### SSE event streams
|
||||
|
||||
The HTTP-mirror of the Redis Streams. External services never read Redis
|
||||
The HTTP-mirror of the store's event streams (Redis Streams until 2026-10;
|
||||
IDs keep the `<ms>-<seq>` form). External services never read the store
|
||||
directly — they subscribe here with `Last-Event-ID` for resume. See
|
||||
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
|
||||
contract (one event per stream entry, opaque IDs, heartbeats, gap
|
||||
@@ -248,6 +301,44 @@ signalling).
|
||||
```bash
|
||||
GET /api/events/files # file:events stream (watcher events)
|
||||
GET /api/events/meta # meta:events stream (metadata mutations)
|
||||
GET /api/events/poll # DEPRECATED long-poll over file:events — use SSE
|
||||
```
|
||||
|
||||
### Identity (signing keystore)
|
||||
|
||||
Reached through meta-search's identity proxy (and directly by meta-gateway,
|
||||
meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
|
||||
|
||||
```bash
|
||||
GET /api/identity # status (never the private key)
|
||||
GET /api/identity/accounts
|
||||
POST /api/identity/generate # the only call that returns a new private key
|
||||
POST /api/identity/import
|
||||
POST /api/identity/challenge # one-shot challenge for proof-of-possession
|
||||
POST /api/identity/reveal # needs proof-of-possession
|
||||
DELETE /api/identity # needs proof-of-possession; reports purged UDL counts
|
||||
POST /api/identity/sign
|
||||
POST /api/identity/sign-batch
|
||||
GET /api/identity/aead-key
|
||||
```
|
||||
|
||||
### User Data Layer
|
||||
|
||||
Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
|
||||
writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
|
||||
cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec:<uid>/<cid>/<key>` → JSON `{record,version,ts,value?}`; index sets as one key per member).
|
||||
|
||||
```bash
|
||||
GET /api/udl/record?uid&cid&key
|
||||
PUT /api/udl/record
|
||||
PUT /api/udl/records # batch
|
||||
GET /api/udl/user/{uid}/key/{key}
|
||||
GET /api/udl/user/{uid}/cid/{cid}
|
||||
GET /api/udl/user/{uid}/all
|
||||
GET /api/udl/user/{uid}/count
|
||||
GET /api/udl/cid/{cid}/users
|
||||
GET /api/udl/cid/{cid}/aggregate # meta-watch Aggregate wire shape
|
||||
GET /api/udl/users/stats
|
||||
```
|
||||
|
||||
### Mounts
|
||||
@@ -283,46 +374,43 @@ The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
|
||||
return a deprecation pointer at the `/api/watchers/*` routes
|
||||
(`internal/watcher/handlers.go:32-33`).
|
||||
|
||||
### Admin / WebDAV
|
||||
### WebDAV
|
||||
|
||||
```bash
|
||||
POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
|
||||
POST /api/admin/sweep-literature-echoes[?apply=true]
|
||||
# strip query echoes off literature cards: anilistid on site cards,
|
||||
# workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
|
||||
# Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
|
||||
GET/PUT/DELETE /webdav/... # mounted on /files
|
||||
```
|
||||
|
||||
The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`,
|
||||
`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were
|
||||
removed with the 2026-10 SQLite switch; they had been run once on
|
||||
watch.nsl.sh before the migration.
|
||||
|
||||
No `/metrics` endpoint is exposed; observability is via container logs and
|
||||
the `/health` / `/status` JSON.
|
||||
|
||||
## Leader gate
|
||||
## DB-file mutex
|
||||
|
||||
Leader election is a bash flock loop in `docker/leader-election.sh`, not a
|
||||
Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
|
||||
script writes the leader API URL to `kv-leader.info` and `exec`s
|
||||
supervisord. When the process tree dies, the kernel releases the lock and
|
||||
any standby instance can take over. There is no follower path inside the
|
||||
Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
|
||||
`docker/leader-election.sh` is a bash flock loop, not a Go-level state
|
||||
machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
|
||||
script `exec`s supervisord; a second container mounting the same volume
|
||||
retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
|
||||
dies, the kernel releases the lock. There is no follower path inside the Go
|
||||
binary.
|
||||
|
||||
`kv-leader.info` is a single plaintext line containing the API base URL:
|
||||
|
||||
```
|
||||
http://10.0.1.50:9000
|
||||
```
|
||||
|
||||
Other services read this file, then call `GET /urls` for full discovery.
|
||||
|
||||
The dev compose runs a single `metacore-app`; the flock primitive is
|
||||
retained as a correctness anchor (one process owns Redis and the
|
||||
writable state under `/meta-core`), not as live failover.
|
||||
The lock is a **mutex, not service discovery** — it only stops two
|
||||
writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info`
|
||||
file it used to publish is gone: siblings locate meta-core over UDP
|
||||
(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
|
||||
`internal/leader` keeps its name for history, but only builds the `/urls`
|
||||
payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
|
||||
and the UDP announce share.
|
||||
|
||||
## Metadata storage shape (one-line summary)
|
||||
|
||||
Roots are opaque UUIDv7 strings; every property is a separate Redis
|
||||
STRING at `file:<uuid>/<property>`; every known CID for a file is a
|
||||
reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
|
||||
Roots are opaque UUIDv7 strings; every property is a separate row
|
||||
of the `kv` table at `file:<uuid>/<property>`; every known CID is a key-set member
|
||||
`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
|
||||
(the canonical CID is derived by rank on read, never stored). Full details in
|
||||
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
|
||||
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
|
||||
The authoritative registry of field semantics and value formats lives in
|
||||
@@ -332,15 +420,16 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
|
||||
| Package | Purpose |
|
||||
|---------|---------|
|
||||
| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
|
||||
| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. |
|
||||
| `internal/config` | Env-driven configuration, path helpers. |
|
||||
| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
|
||||
| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
|
||||
| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
|
||||
| `internal/discovery` | Service registry + dead-service cleaner. |
|
||||
| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
|
||||
| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
|
||||
| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
|
||||
| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
|
||||
| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
|
||||
| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). |
|
||||
| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field_v` + FTS (`search.go`), derived counters (`onChange`), UDL store (`udl.go`, `udl_admin.go`), stats. |
|
||||
| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
|
||||
| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
|
||||
| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. |
|
||||
| `internal/schema` | Per-field schema, folded from counters the write path keeps exact (`idx:schema/…`). |
|
||||
| `internal/snapshot` | Snapshot export / import / wipe. |
|
||||
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
|
||||
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
|
||||
@@ -352,13 +441,17 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
```
|
||||
1. Load configuration from environment
|
||||
2. Build LeaderInfoProvider (no election — the bash gate already won)
|
||||
3. Connect to local Redis (retry up to 30× / 1s)
|
||||
4. EnsureSchemaVersion — abort if the existing Redis layout is stale
|
||||
5. Start service discovery + dead-service cleaner
|
||||
6. Construct API server (initialises watcher, mounts, WebDAV)
|
||||
7. Start API server; on first connect, start MetaPublisher + schema Indexer
|
||||
and republish existing metadata onto meta:events
|
||||
8. Wait for SIGINT/SIGTERM, shutdown in reverse order
|
||||
3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
|
||||
4. EnsureSchemaVersion — abort if the existing store layout is stale
|
||||
5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
|
||||
6. Construct API server (identity keystore migration, watcher, watchers,
|
||||
mounts, WebDAV)
|
||||
7. EnsureCounters — one-time build of the derived counters (record count,
|
||||
per-field schema) on a store that predates them; blocking, ~1 min at 169k
|
||||
records, then never again
|
||||
8. Start API server + pollers (no republish: consumers that want the whole
|
||||
state ask /api/events/meta for `0-0` and get a snapshot)
|
||||
9. Wait for SIGINT/SIGTERM, shutdown in reverse order
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
@@ -366,13 +459,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
| Dependency | Purpose |
|
||||
|------------|---------|
|
||||
| `github.com/gorilla/mux` | HTTP router |
|
||||
| `github.com/redis/go-redis/v9` | Redis client |
|
||||
| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) |
|
||||
| `github.com/google/uuid` | UUIDv7 minting |
|
||||
| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
|
||||
| `golang.org/x/net` | WebDAV support |
|
||||
| Go stdlib | context, net/http, os, sync, syscall |
|
||||
|
||||
Runtime requirements: Go 1.21+ at build time; the container ships Redis,
|
||||
rclone, nginx, and supervisord alongside the binary.
|
||||
Runtime requirements: Go 1.21+ at build time; the container ships
|
||||
rclone, nginx, and supervisord alongside the binary, plus the React dashboard
|
||||
and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+98
-42
@@ -1,22 +1,24 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/api"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
"github.com/metazla/meta-core/internal/discovery"
|
||||
"github.com/metazla/meta-core/internal/leader"
|
||||
"github.com/metazla/meta-core/internal/meshdisco"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// Version is set at build time
|
||||
var Version = "1.0.0"
|
||||
var Version = "2.0.0"
|
||||
|
||||
func main() {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
@@ -29,51 +31,93 @@ func main() {
|
||||
// Create leader info provider (Go binary only runs as leader - followers loop in bash)
|
||||
leaderProvider := leader.NewLeaderInfoProvider(cfg)
|
||||
|
||||
// Connect to local Redis (supervisord starts Redis before meta-core)
|
||||
// Retry connection since Redis may still be loading its dataset
|
||||
redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
|
||||
log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
|
||||
// Open the two embedded SQLite files: truth (signed claims, backed up)
|
||||
// and the cache (everything derived, rebuildable). No database process.
|
||||
if err := os.MkdirAll(filepath.Dir(cfg.CachePath()), 0o755); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to create the cache folder: %v", err)
|
||||
}
|
||||
storageClient := storage.NewClient()
|
||||
if err := storageClient.Connect(cfg.DBPath(), cfg.CachePath()); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to open the metadata store: %v", err)
|
||||
}
|
||||
|
||||
storageClient := storage.NewClient("")
|
||||
maxRetries := 30
|
||||
retryDelay := 1 // seconds
|
||||
var err error
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
err = storageClient.Connect(redisURL)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err)
|
||||
time.Sleep(time.Duration(retryDelay) * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err)
|
||||
}
|
||||
log.Println("[meta-core] Connected to Redis")
|
||||
|
||||
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
|
||||
// the existing Redis store predates the UUID-rooted schema, so legacy
|
||||
// data can't silently leak into the new code paths. See
|
||||
// docs/uuid-rooted-metadata.md.
|
||||
// Schema-version gate: refuses a v1.x store (which the default truth path
|
||||
// still points at) and any unknown layout. No migration code here — see
|
||||
// tools/claims-migrate.
|
||||
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
|
||||
log.Fatalf("[meta-core] %v", err)
|
||||
}
|
||||
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
|
||||
log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion)
|
||||
|
||||
// Create and start service discovery
|
||||
disc := discovery.NewService(cfg)
|
||||
if err := disc.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
|
||||
// Bring the cache level with truth (replay after a crash, full rebuild
|
||||
// when the cache folder is empty), then start the apply loop.
|
||||
started := time.Now()
|
||||
if err := storageClient.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] %v", err)
|
||||
}
|
||||
log.Printf("[meta-core] Cache ready in %s", time.Since(started).Round(time.Millisecond))
|
||||
|
||||
// Derived counters (record count, per-field schema) are kept exact by the
|
||||
// write path. A store that predates them gets a one-time build here,
|
||||
// before anything else writes.
|
||||
started = time.Now()
|
||||
if built, err := storageClient.EnsureCounters(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err)
|
||||
} else if built {
|
||||
log.Printf("[meta-core] Built derived counters in %s", time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
|
||||
// Create and start dead service cleaner
|
||||
cleaner := discovery.NewCleaner(cfg)
|
||||
if err := cleaner.Start(); err != nil {
|
||||
log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
|
||||
// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
|
||||
// as metamesh.core — carrying the /urls block services locate us by — plus
|
||||
// its own metamesh.service/<name> nav entry. The payload is rebuilt from
|
||||
// leaderProvider on every tick, which keeps it and GET /urls from drifting.
|
||||
var mesh *meshdisco.Node
|
||||
if cfg.EnableUDPDiscovery {
|
||||
hostname, _ := os.Hostname()
|
||||
mesh = meshdisco.New(meshdisco.Config{
|
||||
Name: cfg.ServiceName,
|
||||
Instance: hostname,
|
||||
Version: Version,
|
||||
Group: cfg.DiscoveryGroup,
|
||||
Port: cfg.DiscoveryPort,
|
||||
Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
|
||||
Payload: func() (string, []meshdisco.Resource) {
|
||||
service := meshdisco.Resource{
|
||||
ID: "service",
|
||||
Caps: []string{meshdisco.CapServicePrefix + cfg.ServiceName},
|
||||
Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
|
||||
}
|
||||
info := leaderProvider.LeaderInfo()
|
||||
if info == nil {
|
||||
return "starting", []meshdisco.Resource{service}
|
||||
}
|
||||
data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
|
||||
Hostname: info.Hostname,
|
||||
BaseUrl: info.BaseUrl,
|
||||
ApiUrl: info.ApiUrl,
|
||||
WebdavUrl: info.WebdavUrl,
|
||||
WebdavUrlInternal: info.WebdavUrlInternal,
|
||||
}})
|
||||
core := meshdisco.Resource{
|
||||
ID: "core",
|
||||
Caps: []string{meshdisco.CapCore},
|
||||
Endpoints: map[string]string{"http": info.ApiUrl},
|
||||
Data: data,
|
||||
}
|
||||
return "running", []meshdisco.Resource{core, service}
|
||||
},
|
||||
})
|
||||
if err := mesh.Start(); err != nil {
|
||||
log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
|
||||
mesh = nil
|
||||
}
|
||||
} else {
|
||||
log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
|
||||
}
|
||||
|
||||
// Create and start API server
|
||||
apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
|
||||
apiServer := api.NewServer(cfg, leaderProvider, storageClient)
|
||||
apiServer.SetMeshNode(mesh)
|
||||
if err := apiServer.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to start API server: %v", err)
|
||||
}
|
||||
@@ -90,12 +134,10 @@ func main() {
|
||||
log.Printf("[meta-core] Error stopping API server: %v", err)
|
||||
}
|
||||
|
||||
if err := cleaner.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping service cleaner: %v", err)
|
||||
if mesh != nil {
|
||||
if err := mesh.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
|
||||
}
|
||||
|
||||
if err := disc.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping service discovery: %v", err)
|
||||
}
|
||||
|
||||
if err := storageClient.Close(); err != nil {
|
||||
@@ -112,3 +154,17 @@ func waitForShutdown() {
|
||||
<-sigChan
|
||||
}
|
||||
|
||||
// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
|
||||
// the three-way fallback the file registry used (discovery/service.go):
|
||||
// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
|
||||
// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
|
||||
// the container IP.
|
||||
func coreBaseURL(cfg *config.Config) string {
|
||||
if cfg.PublicURL != "" {
|
||||
return cfg.PublicURL
|
||||
}
|
||||
if cfg.BaseURL != "" {
|
||||
return cfg.BaseURL
|
||||
}
|
||||
return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
|
||||
}
|
||||
|
||||
Vendored
+1
-1
@@ -21,7 +21,7 @@
|
||||
min-height: 100vh;
|
||||
}
|
||||
</style>
|
||||
<script type="module" crossorigin src="/assets/index-BW9EQGCP.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -1,155 +1,53 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useEffect } from 'react';
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
|
||||
// of defining the custom element.
|
||||
import '../meta-service-menu.js';
|
||||
|
||||
interface ServiceInfo {
|
||||
name: string;
|
||||
hostname: string;
|
||||
baseUrl: string;
|
||||
status: string;
|
||||
lastHeartbeat: string;
|
||||
/**
|
||||
* ServiceNav — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
|
||||
* a framework-free custom element shared by every meta-* UI (see
|
||||
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
|
||||
* that existed in four near-identical copies and polled meta-core's
|
||||
* /api/services; the element polls this service's own /api/neighbors instead,
|
||||
* so the nav still renders when meta-core is down.
|
||||
*
|
||||
* The element lives in shadow DOM and cannot see this app's CSS variables, so
|
||||
* the four --mm-nav-* tokens below map our palette onto it.
|
||||
*/
|
||||
|
||||
declare global {
|
||||
// eslint-disable-next-line @typescript-eslint/no-namespace
|
||||
namespace JSX {
|
||||
interface IntrinsicElements {
|
||||
'meta-service-menu': React.DetailedHTMLProps<
|
||||
React.HTMLAttributes<HTMLElement>,
|
||||
HTMLElement
|
||||
> & { current?: string; endpoint?: string; label?: string };
|
||||
}
|
||||
|
||||
interface ServicesResponse {
|
||||
services: ServiceInfo[];
|
||||
current: string;
|
||||
}
|
||||
|
||||
const serviceIcons: Record<string, string> = {
|
||||
'meta-core': '🗄️',
|
||||
'meta-sort': '📁',
|
||||
'meta-fuse': '🗂️',
|
||||
'meta-stremio': '🎬',
|
||||
'meta-orbit': '🌐',
|
||||
'default': '📦'
|
||||
};
|
||||
|
||||
function formatServiceName(name: string): string {
|
||||
return name.split('-').map(word =>
|
||||
word.charAt(0).toUpperCase() + word.slice(1)
|
||||
).join(' ');
|
||||
}
|
||||
|
||||
function ServiceNav() {
|
||||
const [services, setServices] = useState<ServiceInfo[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const currentService = 'meta-core';
|
||||
|
||||
useEffect(() => {
|
||||
const fetchServices = async () => {
|
||||
try {
|
||||
const response = await fetch('/services');
|
||||
if (response.ok) {
|
||||
const data: ServicesResponse = await response.json();
|
||||
setServices(data.services || []);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch services:', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchServices();
|
||||
const interval = setInterval(fetchServices, 30000);
|
||||
return () => clearInterval(interval);
|
||||
// Import above is for the side effect of defining the element; this
|
||||
// keeps bundlers from tree-shaking it away in production builds.
|
||||
}, []);
|
||||
|
||||
if (loading) return null;
|
||||
|
||||
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
if (sortedServices.length === 0) return null;
|
||||
|
||||
return (
|
||||
<nav className="services-nav">
|
||||
<span className="services-nav-label">Services:</span>
|
||||
<div className="services-nav-items">
|
||||
{sortedServices.map(service => {
|
||||
const icon = serviceIcons[service.name] || serviceIcons.default;
|
||||
const isActive = service.name === currentService;
|
||||
return (
|
||||
<a
|
||||
key={service.name}
|
||||
href={isActive ? '#' : service.baseUrl}
|
||||
className={`service-link${isActive ? ' active' : ''}`}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
if (!isActive) {
|
||||
window.location.href = service.baseUrl;
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="service-icon">{icon}</span>
|
||||
<span>{formatServiceName(service.name)}</span>
|
||||
<span className="service-status"></span>
|
||||
</a>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<>
|
||||
<meta-service-menu current="meta-core" />
|
||||
<style>{`
|
||||
.services-nav {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.services-nav-label {
|
||||
color: #888;
|
||||
font-size: 0.8rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.services-nav-items {
|
||||
display: flex;
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.service-link {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
padding: 0.35rem 0.6rem;
|
||||
border-radius: 6px;
|
||||
text-decoration: none;
|
||||
font-size: 0.8rem;
|
||||
font-weight: 500;
|
||||
transition: all 0.2s;
|
||||
color: #ccc;
|
||||
background: #0f3460;
|
||||
border: 1px solid #1a4a7a;
|
||||
}
|
||||
|
||||
.service-link:hover {
|
||||
background: #1a4a7a;
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
text-decoration: none;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.service-link.active {
|
||||
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.service-link .service-icon {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.service-link .service-status {
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: #4ecdc4;
|
||||
}
|
||||
|
||||
@media (max-width: 800px) {
|
||||
.services-nav-label {
|
||||
display: none;
|
||||
}
|
||||
meta-service-menu {
|
||||
--mm-nav-fg: var(--text-primary, #e0e0e0);
|
||||
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
|
||||
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
|
||||
--mm-nav-accent: var(--accent-primary, #4ecdc4);
|
||||
}
|
||||
`}</style>
|
||||
</nav>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
// JSON fetch that fails with a sentence instead of a parser error.
|
||||
//
|
||||
// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
|
||||
// path it does not have a proxy rule for answers 200 text/html with the SPA
|
||||
// shell. A caller that does `res.ok && res.json()` on such a path gets
|
||||
// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
|
||||
// no clue which request produced it — which is exactly how a route deleted
|
||||
// from the Go router (`/services`, removed in favour of /api/neighbors) sat
|
||||
// broken on the dashboard.
|
||||
//
|
||||
// So: check the content type, and name the URL in every error.
|
||||
|
||||
export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(url, init);
|
||||
const ct = res.headers.get('content-type') || '';
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`${url} → HTTP ${res.status}`);
|
||||
}
|
||||
if (!ct.includes('application/json')) {
|
||||
// The SPA fallback is the overwhelmingly likely cause; say so.
|
||||
throw new Error(
|
||||
`${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
|
||||
`missing and nginx served the dashboard shell instead)`
|
||||
);
|
||||
}
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Replaces the eight divergent "services bar" implementations (four copies of
|
||||
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
|
||||
* ones) with a single framework-free custom element. It drops into the React
|
||||
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
|
||||
* as an ES module.
|
||||
*
|
||||
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
|
||||
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
|
||||
* make a fetch to a neighbour a dead end. Links are plain navigations.
|
||||
*
|
||||
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
|
||||
* `scripts/check-mirrors.sh`. Edit one → edit all.
|
||||
*
|
||||
* Shadow DOM is not optional here. The eight UIs use four incompatible token
|
||||
* vocabularies, and two of them define *different colours under the same
|
||||
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
|
||||
* #161b22 in meta-dup). meta-search and meta-share are also the only
|
||||
* light-mode-capable UIs, so a hardcoded dark palette would break them.
|
||||
* The element therefore styles itself from four tokens with dark defaults:
|
||||
*
|
||||
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
|
||||
*
|
||||
* Each host maps its own variables onto those, e.g.
|
||||
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
|
||||
*
|
||||
* Attributes:
|
||||
* current name of the host service, so it can be marked and not linked
|
||||
* endpoint override the default "/api/neighbors"
|
||||
* label button tooltip (default "Services")
|
||||
*
|
||||
* Property:
|
||||
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
|
||||
* translate internal hostnames into browser-reachable ones
|
||||
* (meta-gateway does this for its dev/prod split).
|
||||
*/
|
||||
|
||||
const ICONS = {
|
||||
"meta-core": "⚙️",
|
||||
"meta-sort": "📁",
|
||||
"meta-fuse": "🗂️",
|
||||
"meta-stremio": "🎬",
|
||||
"meta-dup": "🔍",
|
||||
"meta-search": "🔎",
|
||||
"meta-share": "🔗",
|
||||
"meta-gateway": "🌉",
|
||||
"meta-watch": "📺",
|
||||
"meta-listen": "🎧",
|
||||
"meta-read": "📖",
|
||||
};
|
||||
const DEFAULT_ICON = "📦";
|
||||
const POLL_MS = 30000;
|
||||
|
||||
function titleCase(name) {
|
||||
return String(name || "")
|
||||
.split("-")
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
const TEMPLATE = `
|
||||
<style>
|
||||
:host {
|
||||
--_fg: var(--mm-nav-fg, #e8eaed);
|
||||
--_bg: var(--mm-nav-bg, #1a1a2e);
|
||||
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
|
||||
--_accent: var(--mm-nav-accent, #4ecdc4);
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
button.trigger {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
width: 2.1rem;
|
||||
height: 2.1rem;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
color: var(--_fg);
|
||||
background: transparent;
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 8px;
|
||||
font: inherit;
|
||||
transition: border-color 0.15s, background 0.15s;
|
||||
}
|
||||
button.trigger:hover { border-color: var(--_accent); }
|
||||
button.trigger:focus-visible {
|
||||
outline: 2px solid var(--_accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
|
||||
.bars { display: block; width: 15px; }
|
||||
.bars span {
|
||||
display: block;
|
||||
height: 2px;
|
||||
margin: 3px 0;
|
||||
border-radius: 2px;
|
||||
background: currentColor;
|
||||
}
|
||||
.panel {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
right: 0;
|
||||
z-index: 1000;
|
||||
min-width: 13rem;
|
||||
max-height: 70vh;
|
||||
overflow-y: auto;
|
||||
padding: 0.3rem;
|
||||
background: var(--_bg);
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
.panel[hidden] { display: none; }
|
||||
a.item, .item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
padding: 0.45rem 0.6rem;
|
||||
border-radius: 6px;
|
||||
color: var(--_fg);
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
}
|
||||
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
|
||||
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
|
||||
.item.dead { opacity: 0.55; cursor: not-allowed; }
|
||||
.icon { font-size: 1rem; line-height: 1; }
|
||||
.dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
margin-left: auto;
|
||||
border-radius: 50%;
|
||||
background: var(--_accent);
|
||||
}
|
||||
.empty {
|
||||
padding: 0.6rem;
|
||||
opacity: 0.7;
|
||||
font-size: 0.85em;
|
||||
white-space: nowrap;
|
||||
}
|
||||
/* Respect a host that opts out of motion. */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
button.trigger { transition: none; }
|
||||
}
|
||||
</style>
|
||||
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
|
||||
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
|
||||
</button>
|
||||
<div class="panel" role="menu" hidden></div>
|
||||
`;
|
||||
|
||||
class MetaServiceMenu extends HTMLElement {
|
||||
constructor() {
|
||||
super();
|
||||
this.attachShadow({ mode: "open" });
|
||||
this.shadowRoot.innerHTML = TEMPLATE;
|
||||
this._btn = this.shadowRoot.querySelector("button.trigger");
|
||||
this._panel = this.shadowRoot.querySelector(".panel");
|
||||
this._neighbors = [];
|
||||
this._timer = null;
|
||||
this._open = false;
|
||||
|
||||
this._onDocClick = (e) => {
|
||||
if (!this._open) return;
|
||||
if (e.composedPath().includes(this)) return;
|
||||
this._setOpen(false);
|
||||
};
|
||||
this._onKey = (e) => {
|
||||
if (e.key === "Escape" && this._open) {
|
||||
this._setOpen(false);
|
||||
this._btn.focus();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
this._btn.title = this.getAttribute("label") || "Services";
|
||||
this._btn.setAttribute("aria-label", this._btn.title);
|
||||
this._btn.addEventListener("click", () => this._setOpen(!this._open));
|
||||
document.addEventListener("click", this._onDocClick);
|
||||
document.addEventListener("keydown", this._onKey);
|
||||
|
||||
this.refresh();
|
||||
this._timer = setInterval(() => this.refresh(), POLL_MS);
|
||||
}
|
||||
|
||||
disconnectedCallback() {
|
||||
if (this._timer) clearInterval(this._timer);
|
||||
this._timer = null;
|
||||
document.removeEventListener("click", this._onDocClick);
|
||||
document.removeEventListener("keydown", this._onKey);
|
||||
}
|
||||
|
||||
get endpoint() {
|
||||
return this.getAttribute("endpoint") || "/api/neighbors";
|
||||
}
|
||||
|
||||
get current() {
|
||||
return this.getAttribute("current") || "";
|
||||
}
|
||||
|
||||
async refresh() {
|
||||
try {
|
||||
const res = await fetch(this.endpoint, {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!res.ok) throw new Error(String(res.status));
|
||||
const data = await res.json();
|
||||
// `services` is the legacy alias some backends still emit.
|
||||
this._neighbors = data.neighbors || data.services || [];
|
||||
} catch {
|
||||
// Silent: the menu is navigation, not a health indicator. Keep whatever
|
||||
// we last had rather than blanking the list on one failed poll.
|
||||
}
|
||||
this._render();
|
||||
}
|
||||
|
||||
_setOpen(open) {
|
||||
this._open = open;
|
||||
this._panel.hidden = !open;
|
||||
this._btn.setAttribute("aria-expanded", String(open));
|
||||
if (open) this.refresh();
|
||||
}
|
||||
|
||||
_render() {
|
||||
const panel = this._panel;
|
||||
panel.textContent = "";
|
||||
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const n of this._neighbors) {
|
||||
const name = n.name || "";
|
||||
if (!name || seen.has(name)) continue;
|
||||
// Beacon v2 neighbours include plugins; the nav lists services only.
|
||||
// (A row without `caps` comes from a pre-v2 backend — keep it.)
|
||||
const isService = (c) => String(c).startsWith("metamesh.service/");
|
||||
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
|
||||
seen.add(name);
|
||||
rows.push(n);
|
||||
}
|
||||
// Always show the host itself, even before its own echo comes back.
|
||||
if (this.current && !seen.has(this.current)) {
|
||||
rows.push({ name: this.current, baseUrl: "" });
|
||||
}
|
||||
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
|
||||
|
||||
if (rows.length === 0) {
|
||||
const empty = document.createElement("div");
|
||||
empty.className = "empty";
|
||||
empty.textContent = "No services found";
|
||||
panel.appendChild(empty);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const n of rows) {
|
||||
const isActive = n.name === this.current;
|
||||
let url = n.baseUrl || "";
|
||||
if (typeof this.rewriteUrl === "function") {
|
||||
try {
|
||||
url = this.rewriteUrl(n.name, url, n) || "";
|
||||
} catch {
|
||||
/* a bad hook must not blank the menu */
|
||||
}
|
||||
}
|
||||
|
||||
// A neighbour with no reachable URL is shown but not linked — better
|
||||
// than a link that goes nowhere.
|
||||
const el = document.createElement(isActive || !url ? "span" : "a");
|
||||
el.className =
|
||||
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
|
||||
if (el.tagName === "A") {
|
||||
el.href = url;
|
||||
el.setAttribute("role", "menuitem");
|
||||
}
|
||||
if (!isActive && !url) el.title = "No reachable URL advertised";
|
||||
|
||||
const icon = document.createElement("span");
|
||||
icon.className = "icon";
|
||||
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = titleCase(n.name);
|
||||
|
||||
el.append(icon, label);
|
||||
// Beacon v2 omits `status` when running.
|
||||
if ((n.status || "running") === "running" || isActive) {
|
||||
const dot = document.createElement("span");
|
||||
dot.className = "dot";
|
||||
dot.title = n.status || "running";
|
||||
el.appendChild(dot);
|
||||
}
|
||||
panel.appendChild(el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get("meta-service-menu")) {
|
||||
customElements.define("meta-service-menu", MetaServiceMenu);
|
||||
}
|
||||
|
||||
export default MetaServiceMenu;
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { getJSON } from '../lib/api';
|
||||
|
||||
interface HealthStatus {
|
||||
status: string;
|
||||
@@ -6,6 +7,22 @@ interface HealthStatus {
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface FileStats {
|
||||
count: number;
|
||||
totalSize: number;
|
||||
}
|
||||
|
||||
interface Stats {
|
||||
records: number;
|
||||
redisKeys: number;
|
||||
redisMemory: string;
|
||||
identities: number;
|
||||
files: FileStats | null;
|
||||
udlUsers: number | null;
|
||||
sweptAt: number;
|
||||
sweeping: boolean;
|
||||
}
|
||||
|
||||
interface WatcherStatus {
|
||||
id: string;
|
||||
path: string;
|
||||
@@ -22,13 +39,6 @@ interface WatchersResponse {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface ServiceInfo {
|
||||
name: string;
|
||||
api: string;
|
||||
capabilities: string[];
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
const cardStyle: React.CSSProperties = {
|
||||
background: '#16213e',
|
||||
borderRadius: '8px',
|
||||
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
|
||||
|
||||
const gridStyle: React.CSSProperties = {
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
|
||||
// 200px, not 220: five tiles must fit one row inside the 1200px container
|
||||
// (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
|
||||
gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
|
||||
gap: '1rem',
|
||||
marginBottom: '1rem',
|
||||
};
|
||||
|
||||
const statStyle: React.CSSProperties = {
|
||||
fontSize: '2rem',
|
||||
fontWeight: 'bold',
|
||||
color: '#4ade80',
|
||||
lineHeight: 1.2,
|
||||
};
|
||||
|
||||
const labelStyle: React.CSSProperties = {
|
||||
marginBottom: '0.75rem',
|
||||
color: '#888',
|
||||
fontSize: '0.85rem',
|
||||
textTransform: 'uppercase',
|
||||
letterSpacing: '0.04em',
|
||||
};
|
||||
|
||||
const noteStyle: React.CSSProperties = {
|
||||
color: '#6b7a99',
|
||||
fontSize: '0.8rem',
|
||||
marginTop: '0.4rem',
|
||||
};
|
||||
|
||||
const numberFmt = new Intl.NumberFormat();
|
||||
|
||||
function formatCount(n: number | null | undefined): string {
|
||||
return n === null || n === undefined ? '—' : numberFmt.format(n);
|
||||
}
|
||||
|
||||
// Binary units: this counts bytes on disk, and every other tool the operator
|
||||
// compares against (du, docker, the mount page) reports GiB.
|
||||
function formatBytes(bytes: number | null | undefined): string {
|
||||
if (bytes === null || bytes === undefined) return '—';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit++;
|
||||
}
|
||||
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function Tile({
|
||||
label,
|
||||
value,
|
||||
note,
|
||||
pending,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
note?: string;
|
||||
pending?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div style={cardStyle}>
|
||||
<div style={labelStyle}>{label}</div>
|
||||
<div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
|
||||
{value}
|
||||
</div>
|
||||
{note && <div style={noteStyle}>{note}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Dashboard() {
|
||||
const [health, setHealth] = useState<HealthStatus | null>(null);
|
||||
const [stats, setStats] = useState<Stats | null>(null);
|
||||
const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
|
||||
const [services, setServices] = useState<ServiceInfo[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
// Fetch health
|
||||
const healthRes = await fetch('/health');
|
||||
if (healthRes.ok) {
|
||||
setHealth(await healthRes.json());
|
||||
}
|
||||
// Settled, not all: one dead endpoint must not blank every tile.
|
||||
const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
|
||||
getJSON<HealthStatus>('/health'),
|
||||
getJSON<Stats>('/api/stats'),
|
||||
getJSON<WatchersResponse>('/api/watchers'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
|
||||
// Fetch watchers status
|
||||
const watchersRes = await fetch('/api/watchers');
|
||||
if (watchersRes.ok) {
|
||||
setWatchersData(await watchersRes.json());
|
||||
}
|
||||
if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
|
||||
if (statsRes.status === 'fulfilled') setStats(statsRes.value);
|
||||
if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
|
||||
|
||||
// Fetch services
|
||||
const servicesRes = await fetch('/services');
|
||||
if (servicesRes.ok) {
|
||||
const data = await servicesRes.json();
|
||||
setServices(data.services || []);
|
||||
}
|
||||
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(String(err));
|
||||
}
|
||||
const failures = [healthRes, statsRes, watchersRes]
|
||||
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
|
||||
.map((r) => String(r.reason?.message ?? r.reason));
|
||||
setError(failures.length ? failures.join(' · ') : null);
|
||||
};
|
||||
|
||||
fetchData();
|
||||
const interval = setInterval(fetchData, 5000);
|
||||
return () => clearInterval(interval);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(interval);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleTriggerScan = async () => {
|
||||
@@ -95,49 +162,102 @@ export default function Dashboard() {
|
||||
}
|
||||
};
|
||||
|
||||
// Aggregate watcher stats
|
||||
const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
|
||||
const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
|
||||
const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
|
||||
const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
|
||||
const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
|
||||
const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
|
||||
const lastScan =
|
||||
watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
|
||||
|
||||
// Records we hold metadata for but have no local file: gateway/cid-rooted
|
||||
// entries, plus anything whose file went away. The interesting number on a
|
||||
// client box, where it should be most of them.
|
||||
const fileless =
|
||||
stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
|
||||
|
||||
const sweepPending = !!stats && !stats.files;
|
||||
const sweptNote = (() => {
|
||||
if (!stats) return undefined;
|
||||
if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
|
||||
if (stats.sweeping) return 'refreshing…';
|
||||
if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
|
||||
return undefined;
|
||||
})();
|
||||
|
||||
return (
|
||||
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
|
||||
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
|
||||
<h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
|
||||
|
||||
{error && (
|
||||
<div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
|
||||
Error: {error}
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={gridStyle}>
|
||||
{/* Health Card */}
|
||||
<Tile
|
||||
label="Files discovered"
|
||||
value={formatCount(stats?.files?.count)}
|
||||
note={sweptNote}
|
||||
pending={sweepPending}
|
||||
/>
|
||||
<Tile
|
||||
label="Library size"
|
||||
value={formatBytes(stats?.files?.totalSize)}
|
||||
note={sweptNote}
|
||||
pending={sweepPending}
|
||||
/>
|
||||
<Tile
|
||||
label="Metadata entries"
|
||||
value={formatCount(stats?.records)}
|
||||
note={
|
||||
fileless === null
|
||||
? 'records in the index'
|
||||
: `${formatCount(fileless)} with no local file`
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Store keys"
|
||||
value={formatCount(stats?.redisKeys)}
|
||||
note={stats?.redisMemory ? `${stats.redisMemory} on disk` : undefined}
|
||||
/>
|
||||
<Tile
|
||||
label="Identities"
|
||||
value={formatCount(stats?.identities)}
|
||||
note={
|
||||
stats?.udlUsers !== null && stats?.udlUsers !== undefined
|
||||
? `${formatCount(stats.udlUsers)} with user data`
|
||||
: 'signing accounts'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
|
||||
<div style={labelStyle}>Service health</div>
|
||||
{health ? (
|
||||
<>
|
||||
<div style={statStyle}>
|
||||
{health.status === 'ok' ? 'Healthy' : health.status}
|
||||
</div>
|
||||
<p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
|
||||
<p style={noteStyle}>
|
||||
Metadata store: {health.redis ? 'Open' : 'Unavailable'}
|
||||
</p>
|
||||
</>
|
||||
) : (
|
||||
<p>Loading...</p>
|
||||
<p style={noteStyle}>Loading…</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Watchers Status Card */}
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
|
||||
<div style={labelStyle}>File watchers</div>
|
||||
{watchersData ? (
|
||||
<>
|
||||
<div style={statStyle}>{totalFiles}</div>
|
||||
<p>Files discovered</p>
|
||||
<p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
|
||||
<p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
|
||||
<div style={statStyle}>
|
||||
{activeWatchers} / {watchersData.count}
|
||||
</div>
|
||||
<p style={noteStyle}>active watchers</p>
|
||||
<p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
|
||||
{lastScan > 0 && (
|
||||
<p>Last scan: {new Date(lastScan).toLocaleString()}</p>
|
||||
<p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
|
||||
)}
|
||||
<button
|
||||
onClick={handleTriggerScan}
|
||||
@@ -155,27 +275,7 @@ export default function Dashboard() {
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<p>Loading...</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Services Card */}
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
|
||||
<div style={statStyle}>{services.length}</div>
|
||||
<p>Services registered</p>
|
||||
{services.length > 0 && (
|
||||
<ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
|
||||
{services.map((svc) => (
|
||||
<li key={svc.name} style={{ marginBottom: '0.5rem' }}>
|
||||
<strong>{svc.name}</strong>
|
||||
<br />
|
||||
<span style={{ color: '#888', fontSize: '0.9rem' }}>
|
||||
{svc.capabilities?.join(', ') || 'No capabilities'}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<p style={noteStyle}>Loading…</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -179,11 +179,10 @@ async function fetchAccounts(): Promise<Account[]> {
|
||||
// Resolve an account's display name from the User Data Layer, or null.
|
||||
//
|
||||
// Never throws and never reports an error upward: a name is decoration on top
|
||||
// of the uid, and the account list must render identically whether Redis is up
|
||||
// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
|
||||
// — meta-core stores the plaintext beside the signed record and verifies
|
||||
// nothing — so the uid stays on screen as the real identity, and the length cap
|
||||
// bounds what a hostile writer can do to the layout.
|
||||
// of the uid, and the account list must render identically whether the store is up
|
||||
// (identity itself is file-backed, the UDL is not). The name is the user's
|
||||
// signed MMUR2 claim (meta-core verified it on write); the uid stays on screen
|
||||
// as the real identity, and the length cap bounds the layout.
|
||||
async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
const url =
|
||||
`/api/udl/record?uid=${encodeURIComponent(uid)}` +
|
||||
@@ -192,12 +191,11 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) return null; // 503 when storage is down, 500, …
|
||||
const data = (await res.json()) as { exists?: boolean; value?: unknown };
|
||||
// Key off `value`, not `exists`: clearing a name writes a tombstone, which
|
||||
// leaves the cell existing with no plaintext. Private-tier cells look the
|
||||
// same, which is exactly right — there is nothing to show for either.
|
||||
if (typeof data.value !== 'string') return null;
|
||||
const name = data.value.trim();
|
||||
const data = (await res.json()) as { exists?: boolean; claim?: { value?: unknown } | null };
|
||||
// A retraction (null) or ciphertext ({b64}) has nothing to show.
|
||||
const value = data.claim?.value;
|
||||
if (typeof value !== 'string') return null;
|
||||
const name = value.trim();
|
||||
return name ? name.slice(0, 64) : null;
|
||||
} catch {
|
||||
return null;
|
||||
@@ -209,7 +207,7 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
// so per-row calls would walk the keyspace once per account.
|
||||
//
|
||||
// Never throws, for the same reason fetchDisplayName does not: the counts are
|
||||
// context on top of the uid, and the list must render identically when Redis is
|
||||
// context on top of the uid, and the list must render identically when the store is
|
||||
// down — accounts are file-backed, their data is not.
|
||||
//
|
||||
// Returns null on failure rather than an empty map. The difference is
|
||||
@@ -603,7 +601,7 @@ function AccountCard({
|
||||
records === 1 ? 'record' : 'records'
|
||||
} — likes, ratings, watch progress across ${titles.toLocaleString()} ${
|
||||
titles === 1 ? 'title' : 'titles'
|
||||
} — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
|
||||
} — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '}
|
||||
This cannot be undone and there is no backup.
|
||||
</li>
|
||||
<li>
|
||||
@@ -655,12 +653,12 @@ export default function Identity() {
|
||||
return;
|
||||
}
|
||||
if (gen !== reqGen.current) return;
|
||||
// Render the list before resolving names, so a down Redis delays nothing.
|
||||
// Render the list before resolving names, so a down store delays nothing.
|
||||
setAccounts(list);
|
||||
setLoading(false);
|
||||
|
||||
// Names and counts both come after the list is on screen, and neither can
|
||||
// fail the page: a down Redis costs decoration, not the account list.
|
||||
// fail the page: a down store costs decoration, not the account list.
|
||||
const [settled, counts] = await Promise.all([
|
||||
Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
|
||||
fetchStats(),
|
||||
|
||||
@@ -9,8 +9,8 @@ set -e
|
||||
|
||||
# Create required directories
|
||||
mkdir -p /meta-core/locks
|
||||
mkdir -p /meta-core/db/redis
|
||||
mkdir -p /meta-core/services
|
||||
mkdir -p /meta-core/db
|
||||
mkdir -p /meta-core/db-cache
|
||||
mkdir -p /meta-core/mounts/errors
|
||||
mkdir -p /files
|
||||
mkdir -p /var/log/supervisor
|
||||
|
||||
@@ -2,9 +2,14 @@
|
||||
# =============================================================================
|
||||
# Leader Election Script for meta-core
|
||||
#
|
||||
# Uses flock for leader election. Only the leader runs the full application
|
||||
# stack (supervisord with Redis, meta-core, nginx, rclone, mount-watcher).
|
||||
# Followers remain dormant, periodically retrying to acquire the lock.
|
||||
# The flock here is a MUTEX, not service discovery. It is the only thing
|
||||
# stopping two containers that mount the same /meta-core volume from putting
|
||||
# two writers on one database file (/meta-core/db/meta-core.db). Sibling
|
||||
# services no longer read anything from this directory — they locate meta-core
|
||||
# over UDP (beacon v2, see docs/project-architecture/beacon-v2.md), so the
|
||||
# kv-leader.info file
|
||||
# this script used to publish is gone. Keep the lock; it costs nothing and the
|
||||
# corruption it prevents is unrecoverable.
|
||||
#
|
||||
# Environment variables:
|
||||
# META_CORE_PATH - Path to meta-core data directory (default: /meta-core)
|
||||
@@ -20,7 +25,6 @@ META_CORE_HTTP_PORT="${META_CORE_HTTP_PORT:-9000}"
|
||||
ELECTION_RETRY_SECS="${ELECTION_RETRY_SECS:-5}"
|
||||
|
||||
LOCK_FILE="${META_CORE_PATH}/locks/kv-leader.lock"
|
||||
INFO_FILE="${META_CORE_PATH}/locks/kv-leader.info"
|
||||
SUPERVISORD_CONF="/etc/supervisor/conf.d/supervisord.conf"
|
||||
|
||||
# Get the container's IP address
|
||||
@@ -40,35 +44,13 @@ get_local_ip() {
|
||||
echo "$ip"
|
||||
}
|
||||
|
||||
# Write leader info file (plain text API URL)
|
||||
write_leader_info() {
|
||||
local ip=$(get_local_ip)
|
||||
local api_url="http://${ip}:${META_CORE_HTTP_PORT}"
|
||||
|
||||
# Write atomically using temp file + rename
|
||||
local temp_file="${INFO_FILE}.tmp"
|
||||
echo "$api_url" > "$temp_file"
|
||||
mv "$temp_file" "$INFO_FILE"
|
||||
|
||||
echo "[election] Wrote leader info: $api_url"
|
||||
}
|
||||
|
||||
# Clean up leader info file on shutdown
|
||||
cleanup() {
|
||||
echo "[election] Cleaning up..."
|
||||
rm -f "$INFO_FILE"
|
||||
exit 0
|
||||
}
|
||||
|
||||
# Set up signal handlers for graceful shutdown
|
||||
trap cleanup SIGTERM SIGINT
|
||||
|
||||
# Ensure lock directory exists
|
||||
mkdir -p "$(dirname "$LOCK_FILE")"
|
||||
|
||||
echo "[election] Starting leader election..."
|
||||
echo "[election] Lock file: $LOCK_FILE"
|
||||
echo "[election] Info file: $INFO_FILE"
|
||||
|
||||
# Open lock file for flock (file descriptor 200)
|
||||
exec 200>"$LOCK_FILE"
|
||||
@@ -79,9 +61,6 @@ while true; do
|
||||
if flock -n 200; then
|
||||
echo "[election] Acquired flock - transitioning to LEADER"
|
||||
|
||||
# Write leader info before starting services
|
||||
write_leader_info
|
||||
|
||||
# exec supervisord - replaces this process
|
||||
# When supervisord exits, the flock is released automatically
|
||||
echo "[election] Starting supervisord..."
|
||||
|
||||
+4
-8
@@ -123,14 +123,10 @@ http {
|
||||
# =========================================================================
|
||||
# Service discovery
|
||||
# =========================================================================
|
||||
location /services {
|
||||
proxy_pass http://meta-core-api/services;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
location /services/ {
|
||||
proxy_pass http://meta-core-api/services/;
|
||||
# meta-discovery v1 neighbours (supersedes /services, which stays
|
||||
# while consumers migrate).
|
||||
location /neighbors {
|
||||
proxy_pass http://meta-core-api/neighbors;
|
||||
proxy_http_version 1.1;
|
||||
proxy_set_header Host $host;
|
||||
}
|
||||
|
||||
+3
-17
@@ -16,26 +16,13 @@ serverurl=unix:///var/run/supervisor.sock
|
||||
supervisor.rpcinterface_factory = supervisor.rpcinterface:make_main_rpcinterface
|
||||
|
||||
# =============================================================================
|
||||
# Redis: Key-value store (managed by supervisord, started before meta-core)
|
||||
# Previously spawned by Go code, now managed by bash-based leader election
|
||||
# =============================================================================
|
||||
[program:redis]
|
||||
command=/usr/bin/redis-server --port %(ENV_REDIS_PORT)s --bind 0.0.0.0 --protected-mode no --dir /meta-core/db/redis --appendonly yes --appendfilename appendonly.aof --dbfilename dump.rdb --save 60 1 --loglevel warning --notify-keyspace-events Kg$
|
||||
priority=1
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
stdout_logfile_maxbytes=0
|
||||
stderr_logfile=/dev/stderr
|
||||
stderr_logfile_maxbytes=0
|
||||
startsecs=2
|
||||
|
||||
# =============================================================================
|
||||
# meta-core: HTTP API, Mount API, File Watcher (connects to local Redis)
|
||||
# meta-core: HTTP API, Mount API, File Watcher, metadata store (embedded SQLite)
|
||||
# =============================================================================
|
||||
[program:meta-core]
|
||||
command=/usr/local/bin/meta-core
|
||||
priority=10
|
||||
; Room to finish in-flight writes and checkpoint the WAL on shutdown.
|
||||
stopwaitsecs=60
|
||||
autostart=true
|
||||
autorestart=true
|
||||
stdout_logfile=/dev/stdout
|
||||
@@ -50,7 +37,6 @@ environment=
|
||||
SERVICE_VERSION="%(ENV_SERVICE_VERSION)s",
|
||||
META_CORE_HTTP_PORT="%(ENV_META_CORE_HTTP_PORT)s",
|
||||
META_CORE_HTTP_HOST="%(ENV_META_CORE_HTTP_HOST)s",
|
||||
REDIS_PORT="%(ENV_REDIS_PORT)s",
|
||||
WATCH_INTERVAL_MS="%(ENV_WATCH_INTERVAL_MS)s",
|
||||
API_PORT="80",
|
||||
BASE_URL="%(ENV_BASE_URL)s"
|
||||
|
||||
+263
-34
@@ -1,4 +1,14 @@
|
||||
# API-Mediated Access (Redis Lockdown)
|
||||
# API-Mediated Access (Redis Lockdown → Signed Claims)
|
||||
|
||||
> **meta-core v2.0.0 — the claims model.** The implementation basis is
|
||||
> [`docs/implementation/metacore-claims-model.md`](../../../docs/implementation/metacore-claims-model.md)
|
||||
> (meta-root). Where this file and that one disagree, that one wins. The
|
||||
> *access* contract below still holds unchanged — HTTP only, events over
|
||||
> SSE, no database reachable from outside `metacore-app`. The *write*
|
||||
> contract changed: every metadata write is a **signed claim** posted to
|
||||
> `POST /api/claims`, and the legacy unsigned write endpoints answer
|
||||
> `410 Gone`. See ["Write contract — signed claims"](#write-contract--signed-claims-v200)
|
||||
> below.
|
||||
|
||||
## Status
|
||||
|
||||
@@ -9,6 +19,16 @@ on [uuid-rooted-metadata.md](uuid-rooted-metadata.md), which assumes
|
||||
meta-core is the single writer to the metadata keyspace — an assumption
|
||||
that's now actually enforced rather than aspirational.
|
||||
|
||||
> **2026-10: meta-core no longer uses Redis at all.** Its store is now an
|
||||
> embedded SQLite file (`/meta-core/db/meta-core.db`, see
|
||||
> [metadata-storage-structure.md](metadata-storage-structure.md)) and the
|
||||
> `file:events` / `meta:events` streams are rows of that store. The lockdown
|
||||
> below is now structural — there is no database port anywhere — and the
|
||||
> SSE wire contract was kept byte-for-byte, so no consumer changed. The
|
||||
> rest of this doc is the design record of the Redis-era lockdown; the
|
||||
> sections that describe *current* behaviour (SSE retention, reconnect,
|
||||
> error handling) have been updated in place.
|
||||
|
||||
See "Implementation outcome" at the bottom for the delta between this doc's
|
||||
plan and what shipped (notably the write-path CID resolution and the
|
||||
dual-root migration sweep, which weren't in the original plan but turned
|
||||
@@ -59,20 +79,23 @@ we don't fully control.
|
||||
## The architecture in one picture
|
||||
|
||||
```
|
||||
meta-core (only Redis writer + reader)
|
||||
meta-core v2.0.0 (only store writer + reader)
|
||||
┌──────────────────────────────────────┐
|
||||
│ HTTP API surface: │
|
||||
│ POST/PUT/PATCH /api/metadata/* │ ◄── writes
|
||||
│ GET /api/meta/{cid} │ ◄── reads (by CID)
|
||||
│ GET /api/metadata/* │ ◄── reads (by id)
|
||||
│ POST /api/claims │ ◄── writes (signed claims only)
|
||||
│ POST /api/merge │ ◄── stateless merge, no write
|
||||
│ GET /api/meta/{cid} │ ◄── reads (by CID, merged)
|
||||
│ GET /api/meta/{cid}/claims │ ◄── reads (raw signed claims)
|
||||
│ GET /api/metadata/* │ ◄── reads (by rid, merged)
|
||||
│ GET /api/file/{cid} │ ◄── content
|
||||
│ GET /api/events/files │ ◄── SSE stream
|
||||
│ GET /api/events/meta │ ◄── SSE stream
|
||||
│ GET /api/events/meta │ ◄── SSE stream (merged changes)
|
||||
│ │ │
|
||||
│ ▼ │
|
||||
│ Redis (file:*, cid:*, streams) │
|
||||
│ ◄── container-local, no exposed │
|
||||
│ port, no docker-network alias │
|
||||
│ ▼ verify sig → truth → apply │
|
||||
│ truth /meta-core/db/meta-core.db │
|
||||
│ cache /meta-core/db-cache/cache.db │
|
||||
│ ◄── embedded SQLite, no port, │
|
||||
│ no docker-network alias │
|
||||
└──────────────────────────────────────┘
|
||||
▲ ▲
|
||||
│ HTTP │ HTTP SSE
|
||||
@@ -85,9 +108,18 @@ we don't fully control.
|
||||
```
|
||||
|
||||
**The single-writer rule.** meta-core has exclusive write authority over
|
||||
the `file:*`, `cid:*`, `file:__index__`, and `meta-core:*` keyspaces. No
|
||||
other service can write — not because of an honoured convention, but
|
||||
because they can't reach Redis at all.
|
||||
its store — in v2.0.0 the two SQLite databases (truth and cache); in the
|
||||
Redis era the `file:*`, `cid:*`, `file:__index__` (later `idx:*`), and
|
||||
`meta-core:*` keyspaces. No other service can write the store — not because
|
||||
of an honoured convention, but because they can't reach it at all.
|
||||
|
||||
**What changed in v2.0.0 is who may *author* metadata.** meta-core is still
|
||||
the only process that touches the database, but it no longer accepts
|
||||
unsigned values: a writer submits signed votes (claims), meta-core verifies
|
||||
each signature and stores the vote. meta-core **never signs a claim and
|
||||
holds no writer key** — a store that could sign for everyone would prove
|
||||
nothing. The signature is the write authentication; there are no
|
||||
per-writer tokens.
|
||||
|
||||
**Stream reads also mediated.** Events go out through SSE endpoints on
|
||||
meta-core's HTTP surface. Consumers don't see the underlying Redis
|
||||
@@ -98,6 +130,147 @@ without breaking every consumer.
|
||||
|
||||
**Redis is private.** No host port mapping, no docker-network alias, no
|
||||
URL in `kv-leader.info`. Discovery returns only meta-core's HTTP base.
|
||||
(Since 2026-10 there is no Redis at all; the SQLite files are reachable only
|
||||
from inside `metacore-app`.)
|
||||
|
||||
## Write contract — signed claims (v2.0.0)
|
||||
|
||||
Every metadata value is a **signed vote**: *signer `uid` says `key = value`
|
||||
about content `cid`* — `claim(uid, cid, key) → value + ts + sig`. The
|
||||
record a reader sees is the **merged view** that meta-core precomputes from
|
||||
the votes; a reader's own votes always win in their view. Full model,
|
||||
schema and merge rules: the
|
||||
[implementation basis](../../../docs/implementation/metacore-claims-model.md)
|
||||
§1–§7.
|
||||
|
||||
### Endpoint table
|
||||
|
||||
| | Endpoint | Behaviour |
|
||||
|---|---|---|
|
||||
| **Write** | `POST /api/claims` | A batch of signed claims (a JSON array, or `{"claims": [...]}`). **The only metadata write path.** |
|
||||
| Removed | `PUT/PATCH/DELETE /meta/{hash}`, `PUT/DELETE /meta/{hash}/{key...}`, `POST /meta/{hash}/_add/{key...}`, `PUT/DELETE /api/metadata/{hashId}`, `PUT /api/metadata/{hashId}/property`, `POST /api/metadata/batch` | `410 Gone`, with a pointer to `POST /api/claims` |
|
||||
| Kept | `PUT/GET/DELETE /api/kv/value` | Service scratch KV (`btmagnet:*`, `searchcov:*`), plain rows of the cache database's `kv` table (any key outside the reserved prefixes `file:`, `cid:`, `idx:`, `dup:`, `ev:`, `evhwm:`, `meta:`, `meta-core:`) — losing it only costs a refetch |
|
||||
| Kept | `POST /api/files/register`, the file watcher | Fill `local_file` (box-local `filePath`, size, mtime, midhash — unsigned) and `cid_group` |
|
||||
| UDL facade | `/api/udl/*` (same paths) | Writes go through the claims write path; reads come from the claims (per-signer index `claim_by_pid`). Records are MMUR2 claims. Shapes: see "UDL facade shapes" below |
|
||||
| Read | `GET /api/meta/{cid}`, `GET /api/metadata/{rid}`, `GET /file/{cid}`, `GET /meta/{hash}`, `GET /api/metadata/hash-ids`, `GET /api/metadata/list`, `POST /api/metadata/search`, `GET /api/files/tuples`, `GET /api/schema`, `GET /api/stats` | Unchanged paths and response shapes, served from the merged view (+ `local_file`); optional `?as=<uid>`. They carry no `nsrc` |
|
||||
| **New** | `GET /api/meta/{cid}/claims` | `{rid, cids, claims}`: the record's rid, the cids of its group and every live signed claim on them (all signers, tombstones excluded). `?key=` keeps one exact key, `?uid=` one signer. `404 unknown_cid` when no record holds the cid |
|
||||
| **New** | `GET /api/metadata/{rid}/claims` | Same body and filters, addressed by rid (a cid is also accepted); `404` for an unknown record. What meta-search ingest and the gateway's corpus search use |
|
||||
| **New** | `POST /api/merge` | Stateless: body `{claims, as?}`; verifies each claim (invalid ones dropped and counted, never a batch error), merges with the store's merge rule, overlays `as`'s own votes (in the request and held by this core for the same cids). Returns `{fields, nsrc, rejected}` — merged `key → value`, `key → agreeing signers`, rejected count. `413` above 10 000 claims. Writes nothing |
|
||||
| Events | `GET /api/events/meta` | Unchanged `{type: set\|del, key, ts}` shape, now emitted for changes of the **merged** view, `key` still the literal `file:<rid>/<field>`; `GET /api/events/files` unchanged. Both streams are rows of the cache `kv` table (`ev:meta:events/…`, `ev:file:events/…`), ~100k entries each |
|
||||
| Admin | `POST /api/metadata/clear` | Wipes both databases (truth and cache) |
|
||||
| Removed | `/api/snapshot/export`, `/api/snapshot/import`, `/api/snapshot/wipe` | No snapshot format in v2.0.0 |
|
||||
|
||||
Endpoints the claims model does not touch keep their behaviour: discovery
|
||||
(`/health`, `/status`, `/leader`, `/urls`, `/neighbors`), the account-key
|
||||
identity service (`/api/identity/*` — its removal is a separate ticket;
|
||||
v2.0.0 adds the `format` field to `sign` / `sign-batch` — `"compact"` or
|
||||
`"der"`, default compact for `"MMUR2\n"` payloads and DER otherwise — and
|
||||
makes `DELETE /api/identity` sign and store a retraction of every live
|
||||
claim of the account before removing it),
|
||||
`/api/file/{cid}/info`, `/file/cid`, `/data/{hash}*`, mounts, watchers and
|
||||
WebDAV.
|
||||
|
||||
### The claim on the wire (JSON form)
|
||||
|
||||
```json
|
||||
{ "cid": "bafy…", "uid": "zQ3s…", "key": "title",
|
||||
"value": "The Matrix", // or {"b64": "…"} for bytes, or null
|
||||
"ts": 1793001234567,
|
||||
"sig": "base64(64 bytes)" }
|
||||
```
|
||||
|
||||
| Field | Meaning |
|
||||
|---|---|
|
||||
| `cid` | The subject: a content CID as the writer knows it (bare CID string, as in `cids/*`). Never a rid — rids are local and never signed |
|
||||
| `uid` | The signer: `"z" + base58btc(33-byte compressed secp256k1 pubkey)` |
|
||||
| `key` | The field (`METADATA_KEYS.md`); a key-set member is inside the key (`genres/Drama`) |
|
||||
| `value` | Text = public plaintext; `{"b64": …}` = bytes, a private (ciphertext) envelope; `null` = retraction |
|
||||
| `ts` | Milliseconds since epoch. It **is** the version: only increases per `(uid, cid, key)` |
|
||||
| `sig` | MMUR2: secp256k1 ECDSA, compact 64 bytes, over SHA-256 of `"MMUR2\n" ‖ canonical CBOR [cid, uid, key, value, ts]` |
|
||||
|
||||
A batch (the `POST /api/claims` body) is a JSON array of these (or
|
||||
`{"claims": [...]}`). Kamilata and
|
||||
the gateway wire carry the same five fields plus `sig`; the signed bytes
|
||||
never change on any hop, and relays (meta-search, a client persisting a
|
||||
gateway hit) forward claims verbatim — they never re-sign.
|
||||
|
||||
### Per-claim statuses
|
||||
|
||||
`POST /api/claims` answers with one status per claim:
|
||||
|
||||
| Status | Meaning |
|
||||
|---|---|
|
||||
| `accepted` | Verified and stored (or a retraction stored as a tombstone) |
|
||||
| `unchanged` | The stored row for `(uid, cid, key)` already has this `ts` and `sig` — **or the same value under any `ts`** (a signer re-asserting what it holds): no verification, no write, and the stored row keeps its older `ts` |
|
||||
| `stale` | The stored `ts` is ≥ the incoming `ts`; ignored (blocks replay of old votes) |
|
||||
| `bad_signature` | The MMUR2 signature does not verify against the `uid`'s public key |
|
||||
| `bad_request` | The claim is malformed |
|
||||
|
||||
The response is `200` with a JSON array of `{status, reason?}`, one entry
|
||||
per claim, in input order, even when some are refused. A batch above 10 000
|
||||
claims answers `413 too_many_claims`; a body that is not claims JSON
|
||||
`400 invalid_json`. The
|
||||
removed write routes answer `410 {"error":"gone","use":"POST /api/claims"}`.
|
||||
`POST /api/claims`, `POST /api/merge` and the two claims read routes are
|
||||
container-to-container routes, not routed through Caddy.
|
||||
|
||||
Rules: every claim is signed — ciphertext and retractions included; an
|
||||
unsigned or badly signed claim is refused. A retraction (`value: null`) is
|
||||
kept as a tombstone, so a peer cannot replay the vote it retracted. Values
|
||||
are stored exactly as signed; normalisation (trim + case-fold, no NFC)
|
||||
happens only inside the merge.
|
||||
|
||||
Accepted claims commit to the truth database in one group-committed
|
||||
transaction; an apply loop then folds them, in commit order, into the
|
||||
cache (the merged `file:<rid>/<field>` rows of its v1-layout `kv` table,
|
||||
FTS, counters, events). `POST /api/claims` answers only once the cache has
|
||||
applied the batch, so the writer reads its own write; another reader may
|
||||
briefly see the previous merged value — the cache is behind, never wrong.
|
||||
|
||||
### `?as=<uid>` — the viewer's overlay
|
||||
|
||||
Every merged read accepts an optional `?as=<uid>`. The merge then puts that
|
||||
signer's own live text vote first for each key (read-time overlay found
|
||||
through the per-signer index, never stored in the shared merged view). Without
|
||||
`?as=` the read returns the shared merge: most distinct signers → oldest
|
||||
`ts` → byte-wise smallest value; ciphertext never merges.
|
||||
|
||||
### Raw claims and client-side merging
|
||||
|
||||
The claims read routes (`GET /api/metadata/{rid}/claims` for meta-search
|
||||
ingest and the gateway's corpus hits, `GET /api/meta/{cid}/claims` by cid)
|
||||
return the raw signed votes of every signer, not a merged map. A client that receives
|
||||
claims from the swarm or a gateway displays them by calling its **local**
|
||||
meta-core's `POST /api/merge` (one merge implementation, in Go), and
|
||||
persists a hit by posting the claims **as received** to `POST /api/claims`.
|
||||
|
||||
### UDL facade shapes
|
||||
|
||||
Claims in the JSON form above (`internal/api/udl.go`):
|
||||
|
||||
| Route | Request | Response |
|
||||
|---|---|---|
|
||||
| `PUT /api/udl/record` | one claim | `200 {accepted: true, status}` for `accepted` / `unchanged`; `409 stale_version` for `stale`; `400` (status as slug) for `bad_signature` / `bad_request` |
|
||||
| `PUT /api/udl/records` | `{records: [claim, …]}` | `200 {results: [{key, accepted, status, error?}, …]}`, input order; `accepted` = `accepted` or `unchanged` |
|
||||
| `GET /api/udl/record?uid=&cid=&key=` | — | `{exists, claim}` — the stored claim, tombstone included; `exists` false when absent or a tombstone |
|
||||
| `GET /api/udl/user/{uid}/key/{key}`, `/user/{uid}/cid/{cid}` | — | `{entries: [claim, …]}` |
|
||||
| `GET /api/udl/user/{uid}/all?after=&limit=` | — | `{entries: [claim, …], nextCid}` |
|
||||
| `GET /api/udl/user/{uid}/count` | — | `{cids}` |
|
||||
| `GET /api/udl/cid/{cid}/users` | — | `{users: [uid, …]}` |
|
||||
| `GET /api/udl/cid/{cid}/aggregate?keys=` | — | `{aggregates: [{key, count, avg, trues, sample_size}]}` |
|
||||
| `GET /api/udl/users/stats` | — | `{users: {uid: {records, cids, keys}}}` for **every signer** with live claims — plugin, feeder, gateway and service keys included. `/api/stats` `udlUsers` is the size of this map |
|
||||
|
||||
### What a writer needs
|
||||
|
||||
Each writing container (every meta-sort plugin, every self-publishing
|
||||
feeder, gateway core, meta-sort, meta-share, meta-search) signs with its
|
||||
**own** key; users sign in the browser. meta-core holds none of these keys.
|
||||
Key configuration per signer: implementation basis §9.
|
||||
|
||||
Box-local file facts (`filePath`, size, `mtimeNano`, midhash) are not
|
||||
claims: the watcher and `POST /api/files/register` write them into the
|
||||
cache's `local_file` table, they are joined into the record view unsigned,
|
||||
and they never appear in `/api/meta/{cid}/claims` or on the swarm.
|
||||
|
||||
## Eventing — HTTP Server-Sent Events
|
||||
|
||||
@@ -243,6 +416,36 @@ After PR A, the cursor contract is **"durable across meta-core restart
|
||||
and bounded by `MAXLEN ~ N`"** — matching what consumers already assume
|
||||
when they persist a Last-Event-ID.
|
||||
|
||||
**Current backing store (since 2026-10).** The streams are rows of
|
||||
meta-core's SQLite kv table — `ev:<stream>/<ms13>-<seq10>` → JSON of the
|
||||
entry's fields, plus a high-water mark `evhwm:<stream>` that seeds the next
|
||||
ID (so a `ClearStream` reset never reissues an ID an old cursor treats as
|
||||
seen). The contract above holds unchanged:
|
||||
|
||||
- Entries persist across restart (they are in the database file).
|
||||
- Retention is ~100k entries per stream, trimmed every 10k appends.
|
||||
- IDs keep the Redis `<ms>-<seq>` form, so cursors persisted by meta-sort /
|
||||
meta-share / meta-stremio before the switch still compare.
|
||||
- `meta:events` entries (`{type: set|del, key, ts}`) describe changes of
|
||||
the **merged** view (v2.0.0): the apply loop emits `set`/`del` (keyed by
|
||||
rid + key) only when a claim actually changed a merged value, in the same
|
||||
cache transaction as that change. A group merge (two cid groups linked by
|
||||
a `cids/*` vote) emits `del` for every key of the losing rid and `set` for
|
||||
the survivor. *History:* v1.x appended them in the same transaction as the
|
||||
`file:` key change; before 2026-10 a Redis keyspace-notification hop
|
||||
(PSUBSCRIBE in `MetaPublisher`) could drop events when it lagged.
|
||||
- **Cache rebuild (v2.0.0).** Events live in the cache database. A cache
|
||||
rebuild (missing cache folder, or catch-up after an unclean shutdown)
|
||||
emits no per-key events; SSE consumers get `event: gap` and take a
|
||||
snapshot, as they do for a trimmed cursor. Event IDs stay time-based, so
|
||||
they keep rising across a rebuild.
|
||||
- **`Last-Event-ID: 0` / `0-0` on `/api/events/meta` returns a snapshot**
|
||||
(since v1.2.0): one `set` per existing record key, then the live tail from
|
||||
the head captured before the snapshot. Only the last snapshot event carries
|
||||
an `id:`. This replaced the boot-time republish of every field, which a
|
||||
~100k-entry stream could never hold. A trimmed (non-zero) cursor still gets
|
||||
the `gap` event and a replay of what is retained.
|
||||
|
||||
### Heartbeats
|
||||
|
||||
Server writes an SSE comment line (`:keep-alive\n\n`) every 30 seconds
|
||||
@@ -254,10 +457,13 @@ per the SSE spec.
|
||||
|
||||
Per the SSE spec the client (or `EventSource`) auto-reconnects after a
|
||||
network error, sending `Last-Event-ID`. The server resumes from the next
|
||||
entry after that ID by translating to Redis Stream `XREAD … STREAMS file:events <id>`.
|
||||
entry after that ID (a range read of the stream's rows; under Redis this
|
||||
was `XREAD … STREAMS file:events <id>`). With no `Last-Event-ID` the stream
|
||||
starts at the head, resolved once at connect; `0` / `0-0` go through the
|
||||
gap path and replay from the beginning.
|
||||
|
||||
**Gap handling.** If the requested ID has been trimmed out of the Redis
|
||||
Stream by retention, the server emits one synthetic event before resuming
|
||||
**Gap handling.** If the requested ID is older than the oldest retained
|
||||
entry, the server emits one synthetic event before resuming
|
||||
from the oldest available entry:
|
||||
|
||||
```
|
||||
@@ -271,9 +477,10 @@ react. Consumers that don't care ignore it.
|
||||
|
||||
### Error handling
|
||||
|
||||
- **Redis unreachable on connect**: 503 with `Retry-After: 5`.
|
||||
- **Redis fails mid-stream**: close the SSE connection. Client reconnects
|
||||
with `Last-Event-ID`; the cycle resumes when Redis recovers.
|
||||
- **Store unavailable on connect**: 503 with `Retry-After: 5`.
|
||||
- **Store read fails mid-stream**: close the SSE connection. Client
|
||||
reconnects with `Last-Event-ID`; the cycle resumes when meta-core is back
|
||||
(a meta-core restart is the common case — SSE clients just reconnect).
|
||||
- **Client disconnects**: handler goroutine exits cleanly; no per-client
|
||||
state on meta-core.
|
||||
|
||||
@@ -307,7 +514,8 @@ inside-only until a concrete external consumer appears.
|
||||
### Implementation footprint
|
||||
|
||||
The SSE handler itself is ~80 lines of Go in meta-core: one handler per
|
||||
stream, each running an `XREAD … BLOCK 30s` loop and writing
|
||||
stream, each running a read-after-cursor loop (an `XREAD … BLOCK 30s` loop
|
||||
in the Redis era) and writing
|
||||
SSE-formatted lines to the response. Go's `net/http` natively supports
|
||||
streaming responses with `http.Flusher.Flush()`. Standard SSE clients
|
||||
exist for every language we use: Node `eventsource`, Rust
|
||||
@@ -484,6 +692,10 @@ durability across a meta-core restart).
|
||||
|
||||
### PR B — meta-sort writes via HTTP
|
||||
|
||||
> *History.* PR B moved writes onto the unsigned `/meta/{hash}` family.
|
||||
> In v2.0.0 those routes answer `410 Gone`; writers sign claims and post
|
||||
> them to `POST /api/claims` (see "Write contract — signed claims").
|
||||
|
||||
Swap `RedisClient` for an `HttpKVClient` implementation of the existing
|
||||
`IKVClient` interface in `packages/meta-sort-core`. All writes
|
||||
(`setMetadataFlat`, `setMetadataProperty`, batch updates) become HTTP
|
||||
@@ -562,17 +774,22 @@ metacore-app produces a connection refused. The lock is real.
|
||||
| Reads from `file:*` direct? | Forbidden by network topology | Same lock, same reason |
|
||||
| Event streams direct? | No — mediated through SSE | Kubernetes-style. Costs one HTTP hop; buys schema evolution + threat-model fit |
|
||||
| Enforcement mechanism? | Network isolation (primary) + docs + code review (secondary) | No ACLs, no proxy, no lint |
|
||||
| Backing store for streams? | Redis Streams (unchanged) | SSE wraps it; could be swapped later without consumer impact |
|
||||
| Backing store for streams? | Redis Streams at the time; SQLite kv rows since 2026-10 | SSE wraps it; the 2026-10 swap proved it — no consumer changed |
|
||||
| Consumer groups / horizontal scale? | Not supported via SSE | Not needed today; revisit if scale ever requires it |
|
||||
| Storage portability? | Not a goal but a free side-effect | Will not be exercised in alpha |
|
||||
| **Canonical write API path?** | **`/meta/{hash}` family** | Has GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
|
||||
| **SSE event-id contract?** | **Opaque to consumers** | Today it's the Redis Stream entry ID (`<ms>-<seq>`); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
|
||||
| **Canonical write API path?** | **`POST /api/claims` (v2.0.0)** — signed claims only; every unsigned write route answers `410 Gone` | *History (v1.x):* the **`/meta/{hash}` family** was the canonical write surface. It had GET/PUT/**PATCH**/DELETE plus `/_add/{key}` and per-property `{key:.*}` operations. `PATCH` calls `MergeMetadataFlat` — matches the partial-update semantics PR B needs. `/api/metadata/{hashId}` stays as the editor-internal surface (no PATCH, no per-property routes) but is not the migration target. |
|
||||
| **SSE event-id contract?** | **Opaque to consumers** | It is `<ms>-<seq>` (the Redis Stream entry form, kept after the 2026-10 SQLite switch); consumers MUST treat as opaque so the backing store can be swapped without breaking the wire contract. Documented in the SSE endpoint comment and in this doc. |
|
||||
| **Typed error envelope?** | **`{error, message, retryable}` with stable `error` slug** | See "Error envelope" subsection below. |
|
||||
| **Plugin direct-Redis risk?** | **Confirmed: no plugin uses Redis directly** | Grep across `packages/plugins/` for `ioredis` / `redis.createClient` / `new Redis()` returned empty. Plugins POST results back to meta-sort via `/api/plugins/callback`; meta-sort then writes. PR B picks up the plugin write path transparently. |
|
||||
| **meta-sort non-`file:*` writes?** | **Confirmed: none in production** | Full audit of `kvClient.*` callers in `packages/meta-sort/packages/meta-sort-core/src/` shows only: `setMetadataFlat`, `deleteMetadataFlat`, `getMetadataFlat`, `getMetadata`, `getAllHashIds`, `health`, plus the `file:events` stream consumer (`initStreamConsumer` / `startStreamConsumer` / `processPendingEntries`). The generic `kv.set(key, value)` / `kv.setProperty(key, value)` methods on the interface exist for plugin sandboxes and tests but are not invoked by production code with non-`file:*` keys. PR B + PR C cover the full meta-sort surface. |
|
||||
|
||||
## Error envelope
|
||||
|
||||
(`410 Gone` from the removed v1.x write routes carries this envelope with a
|
||||
message pointing to `POST /api/claims`. Per-claim outcomes of
|
||||
`POST /api/claims` are statuses inside the response, not envelopes — see
|
||||
"Per-claim statuses" above.)
|
||||
|
||||
Standard shape for all 4xx / 5xx responses from the metadata API:
|
||||
|
||||
```json
|
||||
@@ -591,7 +808,11 @@ Stable `error` slugs the client should switch on:
|
||||
| `unknown_root` | 404 | no | The hashId / UUID doesn't exist. Caller probably has a stale reference. |
|
||||
| `unknown_cid` | 404 | no | The CID isn't in the reverse index. No retry helps. |
|
||||
| `schema_violation` | 400 | no | Body shape rejected (missing required field, invalid value). Caller bug. |
|
||||
| `storage_unavailable` | 503 | yes | Redis transient. Caller should retry with backoff. |
|
||||
| `invalid_json` | 400 | no | `POST /api/claims` / `POST /api/merge` body is not claims JSON. Caller bug. |
|
||||
| `too_many_claims` | 413 | no | More than 10 000 claims in one `POST /api/claims` or `POST /api/merge`. Split the batch. |
|
||||
| `stale_version` | 409 | yes | `PUT /api/udl/record`: an equal or newer `ts` is stored for this signer, cid and key. Re-read the record, sign with a newer `ts`, retry. |
|
||||
| `gone` | 410 | no | A removed unsigned write route. Body is `{"error":"gone","use":"POST /api/claims"}` (no `message` / `retryable`). |
|
||||
| `storage_unavailable` | 503 | yes | Store transient (Redis at the time; the embedded SQLite store since 2026-10). Caller should retry with backoff. |
|
||||
| `internal` | 500 | yes (cautiously) | Unhandled server-side error. Retry once; escalate if persistent. |
|
||||
|
||||
**Implemented.** Every 4xx/5xx response across `internal/api`,
|
||||
@@ -605,10 +826,9 @@ import cycle on the api package; consolidating it into a shared
|
||||
|
||||
Mostly answered post-implementation:
|
||||
|
||||
- **Deprecation timeline for `/api/metadata/{hashId}`.** Still open. The
|
||||
editor's React UI continues to call it; the surface itself was untouched
|
||||
by this work. Worth a follow-up to consolidate the editor onto
|
||||
`/meta/{hash}` and delete the parallel routes.
|
||||
- **Deprecation timeline for `/api/metadata/{hashId}`.** Answered by
|
||||
v2.0.0: its `PUT`/`DELETE` (and `PUT …/property`) answer `410 Gone`
|
||||
like every unsigned write; `GET` stays as a merged read.
|
||||
|
||||
- **Initial bootstrap flood on `/api/events/files`.** Resolved in
|
||||
practice: meta-fuse uses `initialCursor: '0-0'` to deliberately
|
||||