Compare commits
24
Commits
ec1b227631
...
v2.1.0.1
| Author | SHA1 | Date | |
|---|---|---|---|
 Worph
|
a66ef51b86 | ||
|
Worph
|
f367ed037f | ||
|
Worph
|
cef9dba59e | ||
|
Worph
|
9ccaff90d8 | ||
|
Worph
|
84af298792 | ||
|
Manielle
|
b0adefb97b | ||
|
Worph
|
5369dc4b9c | ||
|
Worph
|
fb4bdad61b | ||
|
Worph
|
dc5cbdfb9b | ||
|
Worph
|
c53146e514 | ||
|
Worph
|
bb04a3f4dc | ||
|
Worph
|
36eb3543b9 | ||
|
Worph
|
155fd83392 | ||
|
Worph
|
cf63a8da68 | ||
|
Worph
|
27ed45999b | ||
|
Worph
|
9254f043a4 | ||
|
Worph
|
ebc894fa42 | ||
|
Worph
|
515ec04369 | ||
|
Worph
|
0403a6f3bc | ||
|
Worph
|
355c666183 | ||
|
Worph
|
0cccceccd9 | ||
|
Worph
|
40b553016c | ||
|
Worph
|
4894810fd4 | ||
|
Worph
|
ac21315255 |
@@ -0,0 +1,129 @@
|
||||
# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
|
||||
# folder exists, and GitHub never reads it.
|
||||
#
|
||||
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
|
||||
# push ipcr-push-85-17-246-67.ip.nsl.sh/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
|
||||
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
|
||||
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
|
||||
# The push goes to ipcr-push-85-17-246-67.ip.nsl.sh, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
|
||||
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
|
||||
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
|
||||
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
|
||||
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
|
||||
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
|
||||
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
|
||||
|
||||
name: Build and Push Docker Image
|
||||
|
||||
on:
|
||||
push:
|
||||
branches:
|
||||
- main
|
||||
tags:
|
||||
- 'v*'
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
tag:
|
||||
description: 'Tag for the image (optional, defaults to sha)'
|
||||
required: false
|
||||
type: string
|
||||
|
||||
env:
|
||||
REGISTRY: ipcr-push-85-17-246-67.ip.nsl.sh
|
||||
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write # the release step below
|
||||
packages: write
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v4
|
||||
|
||||
- name: Install Docker CLI
|
||||
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
|
||||
|
||||
- name: Trust the Mesh Router CA
|
||||
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
|
||||
env:
|
||||
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
with:
|
||||
version: latest
|
||||
buildkitd-config-inline: |
|
||||
[registry."${{ env.REGISTRY }}"]
|
||||
ca = ["/tmp/ipcr-ca.pem"]
|
||||
|
||||
- name: Log in to IPCR
|
||||
env:
|
||||
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
|
||||
run: |
|
||||
mkdir -p ~/.docker
|
||||
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
|
||||
|
||||
- name: Extract metadata (tags, labels)
|
||||
id: meta
|
||||
uses: docker/metadata-action@v5
|
||||
with:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
|
||||
|
||||
- name: Build and push Docker image
|
||||
id: build
|
||||
uses: docker/build-push-action@v5
|
||||
with:
|
||||
context: .
|
||||
push: true
|
||||
tags: ${{ steps.meta.outputs.tags }}
|
||||
labels: ${{ steps.meta.outputs.labels }}
|
||||
|
||||
# Gitea's Packages tab only lists images in Gitea's own registry, so a v* tag gets a
|
||||
# release instead, naming the image it shipped. The digest line is the one to trust:
|
||||
# ipcr 1.7.1 can drop some of the tags pushed together (see the tag list).
|
||||
- name: Publish the Gitea release
|
||||
if: startsWith(github.ref, 'refs/tags/v')
|
||||
env:
|
||||
TOKEN: ${{ secrets.GITEA_TOKEN }}
|
||||
API: ${{ github.server_url }}/api/v1/repos/${{ github.repository }}
|
||||
TAG: ${{ github.ref_name }}
|
||||
SHA: ${{ github.sha }}
|
||||
DIGEST: ${{ steps.build.outputs.digest }}
|
||||
TAGS: ${{ steps.meta.outputs.tags }}
|
||||
PULL: ipcr-watch.nsl.sh/ipns/metadec.eth/${{ env.IMAGE_NAME }}
|
||||
run: |
|
||||
node -e '
|
||||
const e = process.env, ver = e.TAG.replace(/^v/, "");
|
||||
const tags = e.TAGS.split("\n").filter(Boolean).map(t => "`" + t.split(":").pop() + "`");
|
||||
const body = [
|
||||
"**Image** (anonymous pull):",
|
||||
"```", `docker pull ${e.PULL}:${ver}`, "```",
|
||||
"Pinned by digest:",
|
||||
"```", `${e.PULL}@${e.DIGEST}`, "```",
|
||||
`Tags pushed: ${tags.join(", ")} · commit ${e.SHA.slice(0, 7)}`,
|
||||
].join("\n");
|
||||
console.log(JSON.stringify({ tag_name: e.TAG, name: e.TAG, body }));
|
||||
' > /tmp/release.json
|
||||
auth="Authorization: token $TOKEN"
|
||||
code=$(curl -s -o /tmp/existing.json -w '%{http_code}' -H "$auth" "$API/releases/tags/$TAG")
|
||||
if [ "$code" = 200 ]; then
|
||||
id=$(node -p 'require("/tmp/existing.json").id')
|
||||
method=PATCH url="$API/releases/$id"
|
||||
else
|
||||
method=POST url="$API/releases"
|
||||
fi
|
||||
code=$(curl -s -o /tmp/out.json -w '%{http_code}' -X "$method" -H "$auth" \
|
||||
-H 'Content-Type: application/json' --data @/tmp/release.json "$url")
|
||||
case "$code" in 2*) echo "release $TAG: $method $code" ;; *) cat /tmp/out.json; exit 1 ;; esac
|
||||
@@ -45,6 +45,9 @@ jobs:
|
||||
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
|
||||
tags: |
|
||||
type=semver,pattern={{version}}
|
||||
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
|
||||
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
|
||||
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
|
||||
type=semver,pattern={{major}}.{{minor}}
|
||||
type=sha,prefix=
|
||||
type=raw,value=latest,enable={{is_default_branch}}
|
||||
|
||||
+10
-9
@@ -6,20 +6,21 @@
|
||||
# - Bash-based leader election script (flock on shared volume)
|
||||
# - meta-core Go binary (HTTP API, Mount API, Watcher)
|
||||
# - nginx (dashboard, WebDAV, API proxy)
|
||||
# - Redis (managed by supervisord on leader)
|
||||
# - SQLite metadata store, embedded in the Go binary: truth /meta-core/db/meta-core.db
|
||||
# (signed claims, backed up) + cache /meta-core/db-cache/cache.db (rebuildable)
|
||||
# - rclone (single mount executor — SMB goes through rclone's :smb: backend)
|
||||
# - supervisord (process management - runs on leader only)
|
||||
#
|
||||
# Architecture:
|
||||
# - leader-election.sh handles flock, starts supervisord on leader
|
||||
# - Followers remain dormant, retry flock every 5s
|
||||
# - Only leader runs Redis, meta-core, nginx, rclone, mount-watcher
|
||||
# - Only leader runs meta-core, nginx, rclone, mount-watcher
|
||||
# =============================================================================
|
||||
|
||||
# -----------------------------------------------------------------------------
|
||||
# Stage 1: Build Go binary
|
||||
# -----------------------------------------------------------------------------
|
||||
FROM golang:1.21-alpine AS go-builder
|
||||
FROM golang:1.26-alpine AS go-builder
|
||||
|
||||
RUN apk add --no-cache git
|
||||
|
||||
@@ -74,7 +75,7 @@ ARG RCLONE_VERSION=1.73.5
|
||||
# Install runtime dependencies. Native NFS/CIFS clients intentionally absent —
|
||||
# all remote mounts go through rclone now.
|
||||
RUN apk add --no-cache \
|
||||
redis \
|
||||
sqlite \
|
||||
nginx \
|
||||
nginx-mod-http-dav-ext \
|
||||
supervisor \
|
||||
@@ -104,7 +105,8 @@ RUN chmod +x /usr/local/bin/meta-core
|
||||
# Create directories
|
||||
RUN mkdir -p \
|
||||
/meta-core/locks \
|
||||
/meta-core/db/redis \
|
||||
/meta-core/db \
|
||||
/meta-core/db-cache \
|
||||
/meta-core/services \
|
||||
/meta-core/mounts/errors \
|
||||
/files \
|
||||
@@ -140,13 +142,12 @@ ENV META_CORE_PATH=/meta-core \
|
||||
SERVICE_VERSION=1.0.0 \
|
||||
META_CORE_HTTP_PORT=9000 \
|
||||
META_CORE_HTTP_HOST=0.0.0.0 \
|
||||
REDIS_PORT=6379 \
|
||||
WATCH_INTERVAL_MS=1000
|
||||
|
||||
# Expose ports
|
||||
# 80: nginx (dashboard + API proxy + WebDAV)
|
||||
# 6379: Redis (for other services)
|
||||
EXPOSE 80 6379
|
||||
# 80: nginx (dashboard + API proxy + WebDAV). The metadata store has no port:
|
||||
# it is a file inside this container, reachable only through the HTTP API.
|
||||
EXPOSE 80
|
||||
|
||||
# Health check
|
||||
HEALTHCHECK --interval=10s --timeout=5s --start-period=10s --retries=3 \
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
# Variables
|
||||
BINARY := meta-core
|
||||
VERSION ?= 1.0.0
|
||||
VERSION ?= 2.0.0
|
||||
BUILD_DIR := bin
|
||||
DOCKER_IMAGE := meta-core
|
||||
|
||||
|
||||
@@ -1,39 +1,51 @@
|
||||
# meta-core
|
||||
|
||||
Standalone Go service that owns Redis, the leader-info file, the HTTP+SSE
|
||||
metadata surface, and a WebDAV file server for the MetaMesh dev/prod stack.
|
||||
Standalone Go service that owns the metadata store (an embedded SQLite file),
|
||||
the HTTP+SSE metadata surface, the
|
||||
User Data Layer store and signing keystore, and a WebDAV file server for the
|
||||
MetaMesh dev/prod stack.
|
||||
|
||||
## Overview
|
||||
|
||||
meta-core runs as its own container (`metacore-app`). Other MetaMesh services
|
||||
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-share) discover it by
|
||||
reading `/meta-core/locks/kv-leader.info` and then call its HTTP API. Redis
|
||||
is not exposed outside the container — all metadata I/O is mediated by the
|
||||
HTTP API.
|
||||
(meta-sort, meta-fuse, meta-stremio, meta-dup, meta-search, meta-share,
|
||||
meta-gateway, meta-watch, …) locate it over UDP multicast (beacon v2 —
|
||||
its `metamesh.core` advertise carries the full `/urls` payload) and then call its
|
||||
HTTP API. `META_CORE_URL` on a client pins it instead. The store is embedded
|
||||
in the Go binary (no database port at all) — all metadata I/O is mediated by the HTTP/SSE API
|
||||
(see [`docs/api-mediated-access.md`](docs/api-mediated-access.md)).
|
||||
|
||||
### Key capabilities
|
||||
|
||||
| Feature | Description |
|
||||
|---------|-------------|
|
||||
| **Leader gate** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner. The Go binary therefore only ever runs as leader. |
|
||||
| **Redis owner** | Supervisord starts Redis with AOF + RDB persistence inside the container; meta-core connects to it on localhost. |
|
||||
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, services, and file access by CID. |
|
||||
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the underlying Redis Streams so external services never speak Redis. |
|
||||
| **DB-file mutex** | Bash `flock` loop in `docker/leader-election.sh` only `exec`s supervisord on the winner, so two containers sharing one `/meta-core` volume can never run two writers on the database file. It is a mutex, not discovery — nothing reads the lock. |
|
||||
| **Embedded SQLite store** | One schema-free `kv(k, v)` table in `/meta-core/db/meta-core.db` (pure-Go `modernc.org/sqlite`, WAL, `synchronous=NORMAL`). One writer goroutine with group commit; a record write (fields + root index + CID aliases + FTS row + events) is atomic. Replaced Redis in 2026-10 (Redis + the in-memory search copy needed ~6.6 GB on watch.nsl.sh's 169k records; the SQLite file is ~2.5 GB on disk with a bounded page cache). |
|
||||
| **HTTP API** | Typed REST surface (gorilla/mux) for metadata, KV browsing, snapshots, schema inference, mounts, watchers, neighbours, identity, UDL, and file access by CID. |
|
||||
| **Indexed search** | `POST /api/metadata/search` builds a superset of candidates in SQL — a partial expression index `kv_field_v` over (field name, value) of every record key, plus an FTS5 trigram table; the smallest part drives, the other filters are probed per candidate — then verifies each with the unchanged Go matchers. Fresh immediately, no in-memory copy (`internal/storage/search.go`, `internal/api/search_index.go`). |
|
||||
| **SSE event streams** | `/api/events/files` and `/api/events/meta` mirror the store's `file:events` / `meta:events` streams so external services never touch the store. `meta:events` entries are written in the same transaction as the change. |
|
||||
| **WebDAV server** | `/webdav/*` exposes the `/files` volume (read+write) for cross-service file access. nginx in front of meta-core handles caching. |
|
||||
| **Service discovery** | File-based registry under `/meta-core/services/*.json` with heartbeat + dead-service cleanup. |
|
||||
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. |
|
||||
| **Service discovery** | beacon v2: UDP multicast advertise/listen on `239.255.99.1:9099`, neighbours served at `/neighbors`. The file-based registry under `/meta-core/services/*.json` it replaced is gone. |
|
||||
| **File watcher / scanner** | Recursive scan + MidHash256 computation; emits events on `file:events`. Watched roots are POSTed watcher configs (`/meta-core/watchers.json`), not env vars. |
|
||||
| **Mount management** | rclone-only mount manager (SMB rendered into `:smb:`, plus pre-defined remotes). Read-only by construction. |
|
||||
| **UUID-rooted storage** | Roots are UUIDv7 (Crockford Base32, ULID layout); CIDs are reverse-index aliases. Schema-version sentinel refuses to boot against stale data. |
|
||||
| **Identity keystore** | Multi-account secp256k1 signing keys under `/meta-core/identity/accounts/`, created from the dashboard's Identity menu (never auto-generated). A private key is returned only by generate, or by reveal against a proof-of-possession signature (delete needs the same proof); `/api/identity/sign` refuses challenge-domain payloads. |
|
||||
| **User Data Layer** | Version-gated store for per-user signed (and, for private-tier keys, encrypted) records — My List, Continue Watching, ratings — indexed by user and CID. meta-core never inspects the crypto. See [User Data Layer](../../docs/project-architecture/user-data-layer.md). |
|
||||
| **Report inbox** | `/api/reports` stores the bug reports / feedback the client apps' 🐛 buttons file (forwarded server-to-server by meta-watch and meta-read), in its own bounded `db/reports.db`; the dashboard's **Reports** page lists, reads and triages them. See [architecture: Report inbox](docs/meta-core-architecture.md#report-inbox). |
|
||||
| **Dashboard + editor** | nginx serves a React dashboard at `/` and the standalone metadata editor at `/editor/`. |
|
||||
|
||||
### Design characteristics
|
||||
|
||||
- **Single-leader** — flock on a shared volume; no external consensus.
|
||||
- **HTTP-only externally** — Redis is bound to the container, never published.
|
||||
- **Single writer** — flock on the `/meta-core` volume; no external consensus.
|
||||
- **HTTP-only externally** — there is no database port; debug read-only with
|
||||
`docker exec metacore-app sqlite3 -readonly /meta-core/db/meta-core.db …`
|
||||
(dev stacks: `/meta-core-db/meta-core.db`).
|
||||
- **Schema-gated boot** — `EnsureSchemaVersion` aborts startup when the
|
||||
on-disk Redis layout predates the current build (alpha clean-wipe policy;
|
||||
on-disk store layout predates the current build (alpha clean-wipe policy;
|
||||
see `docs/uuid-rooted-metadata.md`).
|
||||
- **Static binary** — CGO disabled, single Go binary plus a thin Alpine
|
||||
image with `redis-server`, `rclone`, nginx, and supervisord.
|
||||
- **Static binary** — CGO disabled (pure-Go SQLite), single Go binary plus a
|
||||
thin Alpine image with `rclone`, nginx, supervisord and the `sqlite` CLI
|
||||
(for debugging).
|
||||
|
||||
## Architecture
|
||||
|
||||
@@ -41,34 +53,31 @@ HTTP API.
|
||||
┌──────────────────────────────────────────────────────────────────────────┐
|
||||
│ metacore-app (standalone container) │
|
||||
│ │
|
||||
│ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
|
||||
│ then exec supervisord) │
|
||||
│ supervisord │
|
||||
│ ├─ leader-election.sh (flock /meta-core/locks/kv-leader.lock, │
|
||||
│ │ writes kv-leader.info, exec supervisord) │
|
||||
│ ├─ redis-server (AOF + RDB on /meta-core/db/redis) │
|
||||
│ ├─ nginx (TLS termination, proxy_cache for /webdav) │
|
||||
│ ├─ rclone rcd (mount daemon, RC API) │
|
||||
│ ├─ nginx :80 (dashboard, /editor, API proxy, │
|
||||
│ │ proxy_cache for /webdav, rclone GUI /rclone) │
|
||||
│ ├─ rclone rcd (mount daemon, RC API on :5572) │
|
||||
│ ├─ mount-watcher.sh (rclone mount lifecycle) │
|
||||
│ └─ meta-core (Go) ─► HTTP API + SSE + WebDAV on :9000 │
|
||||
│ ─► embedded SQLite (db/meta-core.db) │
|
||||
│ ─► UDP advertise/listen 239.255.99.1:9099 │
|
||||
└──────────────────────────────────────────────────────────────────────────┘
|
||||
│
|
||||
┌──────────────────────┴──────────────────────┐
|
||||
│ /meta-core (shared volume) │
|
||||
│ locks/kv-leader.{lock,info} │
|
||||
│ db/redis/ │
|
||||
│ services/*.json │
|
||||
│ /meta-core (meta-core's own volume) │
|
||||
│ locks/kv-leader.lock │
|
||||
│ db/meta-core.db (+ -wal, -shm) │
|
||||
│ identity/accounts/*.json │
|
||||
│ mounts/{mounts.json, errors/} │
|
||||
│ watchers.json │
|
||||
│ cache/ (nginx proxy_cache) │
|
||||
└─────────────────────────────────────────────┘
|
||||
▲
|
||||
┌──────────────────────────────┼────────────────────────────────┐
|
||||
│ │ │
|
||||
┌─────────────┐ ┌─────────────┐ ┌─────────────┐
|
||||
│ meta-sort │ │ meta-fuse │ │ meta-stremio│
|
||||
│ │ │ │ │ │
|
||||
│ reads │ HTTP+SSE │ reads │ HTTP+SSE │ reads │
|
||||
│ leader.info├──────────────►│ leader.info├───────────────►│ leader.info│
|
||||
└─────────────┘ └─────────────┘ └─────────────┘
|
||||
|
||||
meta-sort / meta-fuse / meta-stremio / meta-search / meta-share / …
|
||||
│ 1. hear the metamesh.core advertise on UDP multicast (carries /urls)
|
||||
└─ 2. HTTP + SSE (+ WebDAV) to metacore-app — never the store, never the volume
|
||||
```
|
||||
|
||||
## Quick start
|
||||
@@ -82,8 +91,9 @@ make test # run Go unit tests
|
||||
```
|
||||
|
||||
`make docker` tags the image as `:1.0.0`, `:latest`, **and** `:local`. The
|
||||
`:local` tag is the one every sub-stack docker-compose.yml references
|
||||
(meta-share 10-peer federation, meta-gateway dev stacks); without it the
|
||||
`:local` tag is the one the sub-stack docker-compose files reference
|
||||
(meta-share federation, meta-read and meta-listen dev stacks); the meta-gateway
|
||||
dev stack (`docker-compose.gateway.yml`) uses `:latest`. Without these tags the
|
||||
sub-stacks can't see a freshly-built image by name.
|
||||
|
||||
> **Source-change → recreate.** `make docker` rebuilds and re-tags the
|
||||
@@ -94,12 +104,14 @@ sub-stacks can't see a freshly-built image by name.
|
||||
>
|
||||
> ```bash
|
||||
> make docker
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metagateway-generic-core
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.torznab.yml \
|
||||
> up -d --force-recreate metagateway-torznab-core
|
||||
> docker compose -f packages/meta-gateway/dev/docker-compose.gateway.yml \
|
||||
> up -d --force-recreate metagateway-hub-metacore
|
||||
> docker compose -f packages/meta-share/dev/docker-compose.yml \
|
||||
> --profile share-10node up -d --force-recreate metashare-core metashare-core-test
|
||||
> docker compose -f packages/meta-read/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metaread-dev-core
|
||||
> docker compose -f packages/meta-listen/dev/docker-compose.yml \
|
||||
> up -d --force-recreate metalisten-dev-core
|
||||
> ```
|
||||
>
|
||||
> The symptom of a missed recreate is missing API fields on `/urls`
|
||||
@@ -109,7 +121,7 @@ sub-stacks can't see a freshly-built image by name.
|
||||
>
|
||||
> ```bash
|
||||
> docker inspect meta-core:local --format '{{.Id}}' # fresh SHA
|
||||
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}'
|
||||
> docker ps --filter ancestor=meta-core:local --format '{{.Names}}' # repeat for :latest
|
||||
> # any container whose Image SHA doesn't match the fresh SHA needs --force-recreate.
|
||||
> ```
|
||||
|
||||
@@ -126,36 +138,45 @@ HTTP port (no auth, bypasses Caddy) is `http://localhost:18083`.
|
||||
|
||||
## Configuration
|
||||
|
||||
Environment variables consumed by `internal/config/config.go`:
|
||||
Environment variables (from `internal/config/config.go` unless noted):
|
||||
|
||||
| Variable | Default | Description |
|
||||
|---|---|---|
|
||||
| `META_CORE_PATH` | `/meta-core` | Shared volume root (locks, db, services, mounts, cache). |
|
||||
| `META_CORE_PATH` | `/meta-core` | meta-core's own volume root (lock, SQLite DB, identity, mounts, watchers, cache). No other service mounts it. |
|
||||
| `FILES_PATH` | `/files` | Files volume root. |
|
||||
| `SERVICE_NAME` | `meta-core` | Identity in the service registry. |
|
||||
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status` and registry. |
|
||||
| `API_PORT` | `8180` | External port baked into the leader-info `baseUrl`. |
|
||||
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` if set (used for HTTPS perimeter). |
|
||||
| `REDIS_PORT` | `6379` | Local Redis port. |
|
||||
| `SERVICE_NAME` | `meta-core` | Name in the beacon v2 advertise. |
|
||||
| `SERVICE_VERSION` | `1.0.0` | Reported via `/status`. |
|
||||
| `API_PORT` | `8180` | External port baked into the constructed `baseUrl` when `BASE_URL` is unset. |
|
||||
| `BASE_URL` | _empty_ | Overrides the constructed `baseUrl` (the Caddy/HTTPS perimeter URL). |
|
||||
| `META_CORE_PUBLIC_URL` | _empty_ | Browser-facing nav URL in the announce; wins over `BASE_URL` (e.g. a debug-direct port with no Caddy). |
|
||||
| `META_CORE_DB_PATH` | `<META_CORE_PATH>/db/meta-core.db` | SQLite database file. Dev stacks set `/meta-core-db/meta-core.db` on a named volume (the `/meta-core` bind is 9p on Windows dev boxes and SQLite WAL is unsafe there). |
|
||||
| `META_CORE_DB_CACHE_MB` | `64` | Page cache of the writer connection (`internal/storage/client.go`). |
|
||||
| `META_CORE_DB_READER_CACHE_MB` | `16` | Page cache per reader connection. |
|
||||
| `META_CORE_DB_READERS` | `8` | Reader connection pool size. |
|
||||
| `META_CORE_REPORTS_DB_PATH` | `<META_CORE_PATH>/db/reports.db` | Report inbox database. |
|
||||
| `META_CORE_REPORTS_MAX` / `META_CORE_REPORTS_MAX_MB` | `5000` / `256` | Inbox bounds; closed reports are evicted first, then the oldest. |
|
||||
| `META_CORE_HTTP_PORT` | `9000` | Go HTTP+SSE+WebDAV port. |
|
||||
| `META_CORE_HTTP_HOST` | `0.0.0.0` | HTTP bind. |
|
||||
| `HEALTH_CHECK_INTERVAL_MS` | `5000` | Internal health-loop cadence. |
|
||||
| `HEARTBEAT_INTERVAL_MS` | `30000` | Service registry heartbeat. |
|
||||
| `STALE_THRESHOLD_MS` | `60000` | Age past which a registry entry is marked stale. |
|
||||
| `CLEANUP_INTERVAL_MS` | `600000` | Dead-service cleaner cadence. |
|
||||
| `DEAD_SERVICE_THRESHOLD_MS` | `600000` | Age past which a stale entry is deleted. |
|
||||
| `WATCH_INTERVAL_MS` | `1000` | Watcher poll cadence. |
|
||||
| `DEBOUNCE_MS` | `30000` | File-event debounce window. |
|
||||
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely. |
|
||||
| `ENABLE_FILE_WATCHER` | `true` | Disable to suppress the in-process watcher entirely (`POST /api/files/register` stays available). |
|
||||
| `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 advertise + listen. |
|
||||
| `BEACON_GROUP` / `BEACON_PORT` | `239.255.99.1` / `9099` | Multicast group and UDP port. |
|
||||
| `BEACON_INTERVAL_MS` | `10000` | Unsolicited advertise interval. |
|
||||
| `ELECTION_RETRY_SECS` | `5` | flock retry interval (read by `docker/leader-election.sh`, not the Go binary). |
|
||||
|
||||
Note: there is no `META_CORE_REDIS_PORT`, no `META_CORE_SERVICE_NAME`, and
|
||||
no `META_CORE_LEADER_TIMEOUT`. The Redis URL is discovered locally; service
|
||||
identity uses `SERVICE_NAME`; leader timeout is governed by the bash flock
|
||||
retry interval (`ELECTION_RETRY_SECS`, default `5`).
|
||||
`HEALTH_CHECK_INTERVAL_MS`, `HEARTBEAT_INTERVAL_MS`, `STALE_THRESHOLD_MS`,
|
||||
`CLEANUP_INTERVAL_MS`, `DEAD_SERVICE_THRESHOLD_MS`, `WATCH_INTERVAL_MS` and
|
||||
`DEBOUNCE_MS` are still parsed by `internal/config/config.go` but nothing reads
|
||||
them (leftovers of the file-based registry); setting them has no effect. There
|
||||
is no `REDIS_PORT` / `META_CORE_REDIS_PORT` / `META_CORE_INDEX_EXCLUDE_PREFIXES`
|
||||
(gone with Redis and the in-memory search index; search results now include
|
||||
`categories/newznab/*` like every record read) / `META_CORE_SERVICE_NAME` /
|
||||
`META_CORE_LEADER_TIMEOUT`, and no `WATCH_PATHS` — watched folders are watcher
|
||||
configs created via `/api/watchers`.
|
||||
|
||||
## API reference
|
||||
|
||||
All endpoints below are served by `internal/api/server.go`. Examples assume
|
||||
All endpoints below are registered in `internal/api/server.go` (mounts,
|
||||
watcher and watchers routes in their own packages' `handlers.go`). Examples assume
|
||||
the in-container Go port `:9000`; from the host use
|
||||
`https://metacore-dev.localhost:8083` (Caddy) or `http://localhost:18083`
|
||||
(debug-direct).
|
||||
@@ -163,16 +184,47 @@ the in-container Go port `:9000`; from the host use
|
||||
### Health, leader, URLs
|
||||
|
||||
```bash
|
||||
GET /health # storage + role
|
||||
GET /health # storage + role (`redis: bool` kept for compatibility = store open)
|
||||
GET /status # version, uptime, etc.
|
||||
GET /leader # current leader info (hostname, PID, timestamps)
|
||||
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is intentionally empty)
|
||||
GET /services # registered services + heartbeats
|
||||
GET /api/services # alias for /services
|
||||
GET /services/{name} # one service
|
||||
GET /services/cleanup/stats
|
||||
GET /urls # baseUrl, apiUrl, webdavUrl, webdavUrlInternal (redisUrl is always "", kept for compatibility)
|
||||
GET /neighbors # neighbours heard over UDP (beacon v2)
|
||||
GET /api/neighbors # same, under /api
|
||||
GET /api/stats # dashboard counters (see below)
|
||||
```
|
||||
|
||||
`/services`, `/api/services`, `/services/{name}` and `/services/cleanup/stats`
|
||||
are **gone** — the registration-and-heartbeat registry they served was replaced
|
||||
by beacon v2 (`/neighbors`). nginx answers an unknown non-`/api` path
|
||||
with the dashboard's `index.html`, so a stale caller gets `200 text/html` rather
|
||||
than a 404; that is what made the dashboard fail with
|
||||
`SyntaxError: Unexpected token '<'` until 1.0.21.
|
||||
|
||||
#### `GET /api/stats`
|
||||
|
||||
Every headline counter in one request, split by cost:
|
||||
|
||||
```jsonc
|
||||
{
|
||||
"records": 88709, // count of idx:roots/* — metadata roots, file or not
|
||||
"redisKeys": 3115447, // kv row count (name kept for compatibility; cached 60 s)
|
||||
"redisMemory": "588.74M",// DB + WAL file size (name kept for compatibility)
|
||||
"identities": 3, // signing accounts on disk
|
||||
"files": { "count": 4864, "totalSize": 64587914424 }, // null until first sweep
|
||||
"udlUsers": 5, // accounts holding User Data Layer records
|
||||
"sweptAt": 1789402401970,
|
||||
"sweeping": false
|
||||
}
|
||||
```
|
||||
|
||||
`records` and `identities` are live on every call; `redisKeys` is cached 60 s. `files` and
|
||||
`udlUsers` need a keyspace walk (~0.5s and ~3s respectively on an 88k-record
|
||||
box), so they are served from a single-flight cache refreshed in the background
|
||||
with a 60s TTL — the walk is **never** on the request path, because the
|
||||
dashboard polls and ten open tabs must not cost ten keyspace walks. Prefer this
|
||||
over `/api/kv/info`, which does one `GET` per root (6s+) and sums `sizeByte`
|
||||
across records that share a file.
|
||||
|
||||
### Metadata — primary surface
|
||||
|
||||
These are the routes other services use. `{hash}` may be a UUID root or
|
||||
@@ -198,6 +250,8 @@ GET /api/meta/{cid} # resolve CID → metadata document
|
||||
GET /api/file/{cid}/info # CID resolution metadata
|
||||
GET /file/{cid} # serve file bytes (range-aware)
|
||||
POST /file/cid # compute CID for an uploaded file
|
||||
POST /api/files/register # mint + alias one file under /files/plugin by midhash
|
||||
# (the watcher's path for one file; works with the watcher OFF)
|
||||
HEAD /data/{hash} # existence + size
|
||||
GET /data/{hash}/path # absolute path on /files
|
||||
```
|
||||
@@ -209,6 +263,7 @@ the `/file/{cid}` reverse-lookup path.)
|
||||
|
||||
```bash
|
||||
GET /api/metadata/hash-ids
|
||||
GET /api/files/tuples
|
||||
GET /api/metadata/list
|
||||
POST /api/metadata/search
|
||||
POST /api/metadata/batch
|
||||
@@ -221,10 +276,10 @@ PUT /api/metadata/{hashId}/property
|
||||
|
||||
GET /api/kv/info
|
||||
GET /api/kv/keys
|
||||
GET /api/kv/tree
|
||||
GET /api/kv/tree # cursor is an opaque next-key ("0" when exhausted)
|
||||
GET /api/kv/search
|
||||
GET /api/kv/find
|
||||
GET /api/kv/value
|
||||
GET /api/kv/value # type is always "string" for an existing key
|
||||
PUT /api/kv/value
|
||||
DELETE /api/kv/value
|
||||
GET /api/kv/key/{key...}
|
||||
@@ -239,7 +294,8 @@ POST /api/snapshot/wipe
|
||||
|
||||
### SSE event streams
|
||||
|
||||
The HTTP-mirror of the Redis Streams. External services never read Redis
|
||||
The HTTP-mirror of the store's event streams (Redis Streams until 2026-10;
|
||||
IDs keep the `<ms>-<seq>` form). External services never read the store
|
||||
directly — they subscribe here with `Last-Event-ID` for resume. See
|
||||
[`docs/api-mediated-access.md`](docs/api-mediated-access.md) for the wire
|
||||
contract (one event per stream entry, opaque IDs, heartbeats, gap
|
||||
@@ -248,6 +304,54 @@ signalling).
|
||||
```bash
|
||||
GET /api/events/files # file:events stream (watcher events)
|
||||
GET /api/events/meta # meta:events stream (metadata mutations)
|
||||
GET /api/events/poll # DEPRECATED long-poll over file:events — use SSE
|
||||
```
|
||||
|
||||
### Identity (signing keystore)
|
||||
|
||||
Reached through meta-search's identity proxy (and directly by meta-gateway,
|
||||
meta-read, meta-listen) for record signing, UDL writes and keypair sign-in.
|
||||
|
||||
```bash
|
||||
GET /api/identity # status (never the private key)
|
||||
GET /api/identity/accounts
|
||||
POST /api/identity/generate # the only call that returns a new private key
|
||||
POST /api/identity/import
|
||||
POST /api/identity/challenge # one-shot challenge for proof-of-possession
|
||||
POST /api/identity/reveal # needs proof-of-possession
|
||||
DELETE /api/identity # needs proof-of-possession; reports purged UDL counts
|
||||
POST /api/identity/sign
|
||||
POST /api/identity/sign-batch
|
||||
GET /api/identity/aead-key
|
||||
```
|
||||
|
||||
### User Data Layer
|
||||
|
||||
Records arrive signed (private-tier ones encrypted) as opaque base64 CBOR;
|
||||
writes are version-gated (stale → `409 stale_version`, or `accepted:false` per
|
||||
cell in a batch). Storage model in `internal/storage/udl.go` (`udl:rec:<uid>/<cid>/<key>` → JSON `{record,version,ts,value?}`; index sets as one key per member).
|
||||
|
||||
```bash
|
||||
GET /api/udl/record?uid&cid&key
|
||||
PUT /api/udl/record
|
||||
PUT /api/udl/records # batch
|
||||
GET /api/udl/user/{uid}/key/{key}
|
||||
GET /api/udl/user/{uid}/cid/{cid}
|
||||
GET /api/udl/user/{uid}/all
|
||||
GET /api/udl/user/{uid}/count
|
||||
GET /api/udl/cid/{cid}/users
|
||||
GET /api/udl/cid/{cid}/aggregate # meta-watch Aggregate wire shape
|
||||
GET /api/udl/users/stats
|
||||
```
|
||||
|
||||
### Reports (bug-report / feedback inbox)
|
||||
|
||||
```
|
||||
POST /api/reports # {app, note?, page?, route?, cid?, title?, uid?, build?, filedAt?, log?, …blocks} → 201 {id}
|
||||
GET /api/reports # ?app=&status=open|triaged|closed&q=&before=<id>&limit= → {reports, apps, counts, more}
|
||||
GET /api/reports/{id} # full report incl. body
|
||||
PATCH /api/reports/{id} # {status?, triage_note?}
|
||||
DELETE /api/reports/{id}
|
||||
```
|
||||
|
||||
### Mounts
|
||||
@@ -283,46 +387,43 @@ The legacy `/api/scan/trigger` and `/api/scan/status` still respond but
|
||||
return a deprecation pointer at the `/api/watchers/*` routes
|
||||
(`internal/watcher/handlers.go:32-33`).
|
||||
|
||||
### Admin / WebDAV
|
||||
### WebDAV
|
||||
|
||||
```bash
|
||||
POST /api/admin/migrate-dual-roots # reunify stranded midhash-rooted entries
|
||||
POST /api/admin/sweep-literature-echoes[?apply=true]
|
||||
# strip query echoes off literature cards: anilistid on site cards,
|
||||
# workcid/chapterNumber/volumeNumber/chapterStart/chapterEnd on any card.
|
||||
# Dry-run unless apply=true; returns {dryRun, planned, fixed, byKey}
|
||||
GET/PUT/DELETE /webdav/... # mounted on /files
|
||||
```
|
||||
|
||||
The one-shot Redis-era admin migrations (`/api/admin/migrate-dual-roots`,
|
||||
`migrate-domain-screen`, `migrate-user-cids`, `sweep-literature-echoes`) were
|
||||
removed with the 2026-10 SQLite switch; they had been run once on
|
||||
watch.nsl.sh before the migration.
|
||||
|
||||
No `/metrics` endpoint is exposed; observability is via container logs and
|
||||
the `/health` / `/status` JSON.
|
||||
|
||||
## Leader gate
|
||||
## DB-file mutex
|
||||
|
||||
Leader election is a bash flock loop in `docker/leader-election.sh`, not a
|
||||
Go-level state machine. When `flock` succeeds on `kv-leader.lock`, the
|
||||
script writes the leader API URL to `kv-leader.info` and `exec`s
|
||||
supervisord. When the process tree dies, the kernel releases the lock and
|
||||
any standby instance can take over. There is no follower path inside the
|
||||
Go binary — there is no `OnBecomeLeader` / `OnBecomeFollower` callback API.
|
||||
`docker/leader-election.sh` is a bash flock loop, not a Go-level state
|
||||
machine. When `flock` succeeds on `/meta-core/locks/kv-leader.lock`, the
|
||||
script `exec`s supervisord; a second container mounting the same volume
|
||||
retries every `ELECTION_RETRY_SECS` and stays dormant. When the process tree
|
||||
dies, the kernel releases the lock. There is no follower path inside the Go
|
||||
binary.
|
||||
|
||||
`kv-leader.info` is a single plaintext line containing the API base URL:
|
||||
|
||||
```
|
||||
http://10.0.1.50:9000
|
||||
```
|
||||
|
||||
Other services read this file, then call `GET /urls` for full discovery.
|
||||
|
||||
The dev compose runs a single `metacore-app`; the flock primitive is
|
||||
retained as a correctness anchor (one process owns Redis and the
|
||||
writable state under `/meta-core`), not as live failover.
|
||||
The lock is a **mutex, not service discovery** — it only stops two
|
||||
writers landing on one database file (`db/meta-core.db`). Nothing reads it, and the `kv-leader.info`
|
||||
file it used to publish is gone: siblings locate meta-core over UDP
|
||||
(see [beacon-v2.md](../../docs/project-architecture/beacon-v2.md)).
|
||||
`internal/leader` keeps its name for history, but only builds the `/urls`
|
||||
payload (`baseUrl`, `apiUrl`, `webdavUrl`, `webdavUrlInternal`) that `/urls`
|
||||
and the UDP announce share.
|
||||
|
||||
## Metadata storage shape (one-line summary)
|
||||
|
||||
Roots are opaque UUIDv7 strings; every property is a separate Redis
|
||||
STRING at `file:<uuid>/<property>`; every known CID for a file is a
|
||||
reverse-index entry `cid:<algorithm>:<value> → <uuid>`. Full details in
|
||||
Roots are opaque UUIDv7 strings; every property is a separate row
|
||||
of the `kv` table at `file:<uuid>/<property>`; every known CID is a key-set member
|
||||
`file:<uuid>/cids/<bareCid>` plus a reverse-index entry `cid:<bareCid> → <uuid>`
|
||||
(the canonical CID is derived by rank on read, never stored). Full details in
|
||||
[`docs/metadata-storage-structure.md`](docs/metadata-storage-structure.md)
|
||||
and design rationale in [`docs/uuid-rooted-metadata.md`](docs/uuid-rooted-metadata.md).
|
||||
The authoritative registry of field semantics and value formats lives in
|
||||
@@ -332,15 +433,17 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
|
||||
| Package | Purpose |
|
||||
|---------|---------|
|
||||
| `cmd/meta-core` | Entry point; Redis connect + schema sentinel + service wiring. |
|
||||
| `cmd/meta-core` | Entry point; store open + schema sentinel + UDP discovery + service wiring. |
|
||||
| `internal/config` | Env-driven configuration, path helpers. |
|
||||
| `internal/leader` | `LeaderInfoProvider` (builds the leader info from local hostname/IP/config; no election logic). |
|
||||
| `internal/storage` | Redis wrapper (`client.go`), UUIDv7 minting (`uuid.go`), CID reverse index (`cid_resolution.go`), schema sentinel (`schema_version.go`), dual-root migration (`dual_root_migration.go`). |
|
||||
| `internal/cid` | CID parsing + algorithm ranking for canonical-CID selection. |
|
||||
| `internal/discovery` | Service registry + dead-service cleaner. |
|
||||
| `internal/events` | Keyspace-notification → `meta:events` stream publisher (`meta_publisher.go`). |
|
||||
| `internal/api` | HTTP server + all handlers + SSE event endpoints. |
|
||||
| `internal/schema` | Live per-field schema indexer (consumes `meta:events`). |
|
||||
| `internal/leader` | `LeaderInfoProvider` — builds the `/urls` payload from hostname/IP/config; no election logic. |
|
||||
| `internal/meshdisco` | beacon v2 node (advertise, probe, neighbour map) — the Go port of the spec. |
|
||||
| `internal/kv` | Storage engine: SQLite `kv` table, single writer + group commit, reader pool, in-transaction change events (`kv.go`), event streams as rows (`stream.go`). |
|
||||
| `internal/storage` | Flat-key records (`client.go`), UUIDv7 minting, CID reverse index (`cid_resolution.go`), schema sentinel, search over `kv_field_v` + FTS (`search.go`), derived counters (`onChange`), UDL store (`udl.go`, `udl_admin.go`), stats. |
|
||||
| `internal/cid` | CID parsing + rank ladder for canonical-CID selection (`cid-rank-vectors.json`, guarded by the meta-root `scripts/check-cid-vectors.sh`). |
|
||||
| `internal/reports` | Report inbox: its own SQLite file, normalisation/bounds, list/filter/paging, prune (closed first). |
|
||||
| `internal/identity` | Multi-account secp256k1 keystore, challenge/proof-of-possession, signature verification. |
|
||||
| `internal/api` | HTTP server + all handlers + SSE event endpoints + search matchers + `/api/stats`. |
|
||||
| `internal/schema` | Per-field schema, folded from counters the write path keeps exact (`idx:schema/…`). |
|
||||
| `internal/snapshot` | Snapshot export / import / wipe. |
|
||||
| `internal/watcher` | File system scanner, MidHash256 computation, dispatcher, state registry. |
|
||||
| `internal/watchers` | Polling-based watcher configurations (manager + poller + handlers). |
|
||||
@@ -352,13 +455,17 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
```
|
||||
1. Load configuration from environment
|
||||
2. Build LeaderInfoProvider (no election — the bash gate already won)
|
||||
3. Connect to local Redis (retry up to 30× / 1s)
|
||||
4. EnsureSchemaVersion — abort if the existing Redis layout is stale
|
||||
5. Start service discovery + dead-service cleaner
|
||||
6. Construct API server (initialises watcher, mounts, WebDAV)
|
||||
7. Start API server; on first connect, start MetaPublisher + schema Indexer
|
||||
and republish existing metadata onto meta:events
|
||||
8. Wait for SIGINT/SIGTERM, shutdown in reverse order
|
||||
3. Open the SQLite store (META_CORE_DB_PATH or /meta-core/db/meta-core.db)
|
||||
4. EnsureSchemaVersion — abort if the existing store layout is stale
|
||||
5. Start beacon v2 node (unless ENABLE_UDP_DISCOVERY=false)
|
||||
6. Construct API server (identity keystore migration, watcher, watchers,
|
||||
mounts, WebDAV)
|
||||
7. EnsureCounters — one-time build of the derived counters (record count,
|
||||
per-field schema) on a store that predates them; blocking, ~1 min at 169k
|
||||
records, then never again
|
||||
8. Start API server + pollers (no republish: consumers that want the whole
|
||||
state ask /api/events/meta for `0-0` and get a snapshot)
|
||||
9. Wait for SIGINT/SIGTERM, shutdown in reverse order
|
||||
```
|
||||
|
||||
## Dependencies
|
||||
@@ -366,13 +473,14 @@ the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md).
|
||||
| Dependency | Purpose |
|
||||
|------------|---------|
|
||||
| `github.com/gorilla/mux` | HTTP router |
|
||||
| `github.com/redis/go-redis/v9` | Redis client |
|
||||
| `modernc.org/sqlite` | Embedded SQLite (pure Go, CGO off) |
|
||||
| `github.com/google/uuid` | UUIDv7 minting |
|
||||
| `github.com/decred/dcrd/dcrec/secp256k1/v4`, `golang.org/x/crypto`, `github.com/mr-tron/base58` | Identity keys, signing, AEAD key derivation |
|
||||
| `golang.org/x/net` | WebDAV support |
|
||||
| Go stdlib | context, net/http, os, sync, syscall |
|
||||
|
||||
Runtime requirements: Go 1.21+ at build time; the container ships Redis,
|
||||
rclone, nginx, and supervisord alongside the binary.
|
||||
Runtime requirements: Go 1.21+ at build time; the container ships
|
||||
rclone, nginx, and supervisord alongside the binary, plus the React dashboard
|
||||
and metadata editor (built from `dashboard/` and `editor/` in the Dockerfile).
|
||||
|
||||
## License
|
||||
|
||||
|
||||
+117
-41
@@ -1,22 +1,25 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"os"
|
||||
"os/signal"
|
||||
"path/filepath"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/api"
|
||||
"github.com/metazla/meta-core/internal/config"
|
||||
"github.com/metazla/meta-core/internal/discovery"
|
||||
"github.com/metazla/meta-core/internal/leader"
|
||||
"github.com/metazla/meta-core/internal/meshdisco"
|
||||
"github.com/metazla/meta-core/internal/reports"
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// Version is set at build time
|
||||
var Version = "1.0.0"
|
||||
var Version = "2.0.0"
|
||||
|
||||
func main() {
|
||||
log.SetFlags(log.LstdFlags | log.Lshortfile)
|
||||
@@ -29,51 +32,106 @@ func main() {
|
||||
// Create leader info provider (Go binary only runs as leader - followers loop in bash)
|
||||
leaderProvider := leader.NewLeaderInfoProvider(cfg)
|
||||
|
||||
// Connect to local Redis (supervisord starts Redis before meta-core)
|
||||
// Retry connection since Redis may still be loading its dataset
|
||||
redisURL := fmt.Sprintf("redis://localhost:%d", cfg.RedisPort)
|
||||
log.Printf("[meta-core] Connecting to Redis at %s", redisURL)
|
||||
// Open the two embedded SQLite files: truth (signed claims, backed up)
|
||||
// and the cache (everything derived, rebuildable). No database process.
|
||||
if err := os.MkdirAll(filepath.Dir(cfg.CachePath()), 0o755); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to create the cache folder: %v", err)
|
||||
}
|
||||
storageClient := storage.NewClient()
|
||||
if err := storageClient.Connect(cfg.DBPath(), cfg.CachePath()); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to open the metadata store: %v", err)
|
||||
}
|
||||
|
||||
storageClient := storage.NewClient("")
|
||||
maxRetries := 30
|
||||
retryDelay := 1 // seconds
|
||||
var err error
|
||||
for i := 0; i < maxRetries; i++ {
|
||||
err = storageClient.Connect(redisURL)
|
||||
if err == nil {
|
||||
break
|
||||
}
|
||||
log.Printf("[meta-core] Redis connection attempt %d/%d failed: %v", i+1, maxRetries, err)
|
||||
time.Sleep(time.Duration(retryDelay) * time.Second)
|
||||
}
|
||||
if err != nil {
|
||||
log.Fatalf("[meta-core] Failed to connect to Redis after %d attempts: %v", maxRetries, err)
|
||||
}
|
||||
log.Println("[meta-core] Connected to Redis")
|
||||
|
||||
// Schema-version sentinel: alpha clean-wipe gate. Refuses to boot when
|
||||
// the existing Redis store predates the UUID-rooted schema, so legacy
|
||||
// data can't silently leak into the new code paths. See
|
||||
// docs/uuid-rooted-metadata.md.
|
||||
// Schema-version gate: refuses a v1.x store (which the default truth path
|
||||
// still points at) and any unknown layout. No migration code here — see
|
||||
// tools/claims-migrate.
|
||||
if err := storageClient.EnsureSchemaVersion(storage.SchemaVersion); err != nil {
|
||||
log.Fatalf("[meta-core] %v", err)
|
||||
}
|
||||
log.Printf("[meta-core] Redis schema version OK (v%d)", storage.SchemaVersion)
|
||||
log.Printf("[meta-core] Store schema version OK (v%d)", storage.SchemaVersion)
|
||||
|
||||
// Create and start service discovery
|
||||
disc := discovery.NewService(cfg)
|
||||
if err := disc.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to start service discovery: %v", err)
|
||||
// Bring the cache level with truth (replay after a crash, full rebuild
|
||||
// when the cache folder is empty), then start the apply loop.
|
||||
started := time.Now()
|
||||
if err := storageClient.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] %v", err)
|
||||
}
|
||||
log.Printf("[meta-core] Cache ready in %s", time.Since(started).Round(time.Millisecond))
|
||||
|
||||
// Derived counters (record count, per-field schema) are kept exact by the
|
||||
// write path. A store that predates them gets a one-time build here,
|
||||
// before anything else writes.
|
||||
started = time.Now()
|
||||
if built, err := storageClient.EnsureCounters(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to build the store's derived counters: %v", err)
|
||||
} else if built {
|
||||
log.Printf("[meta-core] Built derived counters in %s", time.Since(started).Round(time.Millisecond))
|
||||
}
|
||||
|
||||
// Create and start dead service cleaner
|
||||
cleaner := discovery.NewCleaner(cfg)
|
||||
if err := cleaner.Start(); err != nil {
|
||||
log.Printf("[meta-core] Warning: failed to start service cleaner: %v", err)
|
||||
// Beacon v2 (docs/project-architecture/beacon-v2.md): advertise meta-core
|
||||
// as metamesh.core — carrying the /urls block services locate us by — plus
|
||||
// its own metamesh.service/<name> nav entry. The payload is rebuilt from
|
||||
// leaderProvider on every tick, which keeps it and GET /urls from drifting.
|
||||
var mesh *meshdisco.Node
|
||||
if cfg.EnableUDPDiscovery {
|
||||
hostname, _ := os.Hostname()
|
||||
mesh = meshdisco.New(meshdisco.Config{
|
||||
Name: cfg.ServiceName,
|
||||
Instance: hostname,
|
||||
Version: Version,
|
||||
Group: cfg.DiscoveryGroup,
|
||||
Port: cfg.DiscoveryPort,
|
||||
Interval: time.Duration(cfg.DiscoveryIntervalMS) * time.Millisecond,
|
||||
Payload: func() (string, []meshdisco.Resource) {
|
||||
service := meshdisco.Resource{
|
||||
ID: "service",
|
||||
Caps: []string{meshdisco.CapServicePrefix + cfg.ServiceName},
|
||||
Endpoints: map[string]string{"ui": coreBaseURL(cfg)},
|
||||
}
|
||||
info := leaderProvider.LeaderInfo()
|
||||
if info == nil {
|
||||
return "starting", []meshdisco.Resource{service}
|
||||
}
|
||||
data, _ := json.Marshal(map[string]meshdisco.URLs{"urls": {
|
||||
Hostname: info.Hostname,
|
||||
BaseUrl: info.BaseUrl,
|
||||
ApiUrl: info.ApiUrl,
|
||||
WebdavUrl: info.WebdavUrl,
|
||||
WebdavUrlInternal: info.WebdavUrlInternal,
|
||||
}})
|
||||
core := meshdisco.Resource{
|
||||
ID: "core",
|
||||
Caps: []string{meshdisco.CapCore},
|
||||
Endpoints: map[string]string{"http": info.ApiUrl},
|
||||
Data: data,
|
||||
}
|
||||
return "running", []meshdisco.Resource{core, service}
|
||||
},
|
||||
})
|
||||
if err := mesh.Start(); err != nil {
|
||||
log.Printf("[meta-core] Warning: UDP discovery disabled: %v", err)
|
||||
mesh = nil
|
||||
}
|
||||
} else {
|
||||
log.Println("[meta-core] UDP discovery disabled by ENABLE_UDP_DISCOVERY")
|
||||
}
|
||||
|
||||
// Report inbox: its own SQLite file beside truth. Not load-bearing — a
|
||||
// failure to open it leaves /api/reports answering 503, never the core down.
|
||||
reportStore, err := reports.Open(cfg.ReportsDBPath(), cfg.ReportsMax, int64(cfg.ReportsMaxMB)<<20)
|
||||
if err != nil {
|
||||
log.Printf("[meta-core] Warning: report inbox disabled: %v", err)
|
||||
reportStore = nil
|
||||
} else {
|
||||
log.Printf("[meta-core] Report inbox %s (max %d reports, %d MiB)", reportStore.Path(), cfg.ReportsMax, cfg.ReportsMaxMB)
|
||||
}
|
||||
|
||||
// Create and start API server
|
||||
apiServer := api.NewServer(cfg, leaderProvider, disc, cleaner, storageClient)
|
||||
apiServer := api.NewServer(cfg, leaderProvider, storageClient)
|
||||
apiServer.SetMeshNode(mesh)
|
||||
if reportStore != nil {
|
||||
apiServer.SetReports(reportStore)
|
||||
}
|
||||
if err := apiServer.Start(); err != nil {
|
||||
log.Fatalf("[meta-core] Failed to start API server: %v", err)
|
||||
}
|
||||
@@ -90,12 +148,16 @@ func main() {
|
||||
log.Printf("[meta-core] Error stopping API server: %v", err)
|
||||
}
|
||||
|
||||
if err := cleaner.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping service cleaner: %v", err)
|
||||
if mesh != nil {
|
||||
if err := mesh.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping UDP discovery: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := disc.Stop(); err != nil {
|
||||
log.Printf("[meta-core] Error stopping service discovery: %v", err)
|
||||
if reportStore != nil {
|
||||
if err := reportStore.Close(); err != nil {
|
||||
log.Printf("[meta-core] Error closing report inbox: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
if err := storageClient.Close(); err != nil {
|
||||
@@ -112,3 +174,17 @@ func waitForShutdown() {
|
||||
<-sigChan
|
||||
}
|
||||
|
||||
// coreBaseURL is the browser-facing URL announced for the nav menu. It keeps
|
||||
// the three-way fallback the file registry used (discovery/service.go):
|
||||
// META_CORE_PUBLIC_URL wins (the reachable URL when there is no Caddy
|
||||
// perimeter, e.g. a debug-direct port), then BASE_URL (the Caddy URL), then
|
||||
// the container IP.
|
||||
func coreBaseURL(cfg *config.Config) string {
|
||||
if cfg.PublicURL != "" {
|
||||
return cfg.PublicURL
|
||||
}
|
||||
if cfg.BaseURL != "" {
|
||||
return cfg.BaseURL
|
||||
}
|
||||
return fmt.Sprintf("http://%s:%d", meshdisco.LocalIPv4(), cfg.APIPort)
|
||||
}
|
||||
|
||||
Vendored
+1
-1
@@ -21,7 +21,7 @@
|
||||
min-height: 100vh;
|
||||
}
|
||||
</style>
|
||||
<script type="module" crossorigin src="/assets/index-BW9EQGCP.js"></script>
|
||||
<script type="module" crossorigin src="/assets/index-Bgo6moAc.js"></script>
|
||||
<link rel="stylesheet" crossorigin href="/assets/index-DtR_2V4s.css">
|
||||
</head>
|
||||
<body>
|
||||
|
||||
@@ -4,6 +4,7 @@ import Mounts from './pages/Mounts';
|
||||
import Watchers from './pages/Watchers';
|
||||
import Files from './pages/Files';
|
||||
import Identity from './pages/Identity';
|
||||
import Reports from './pages/Reports';
|
||||
import ServiceNav from './components/ServiceNav';
|
||||
|
||||
const navStyle: React.CSSProperties = {
|
||||
@@ -81,6 +82,12 @@ export default function App() {
|
||||
>
|
||||
Identity
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/reports"
|
||||
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
|
||||
>
|
||||
Reports
|
||||
</NavLink>
|
||||
<NavLink
|
||||
to="/editor"
|
||||
style={({ isActive }) => (isActive ? activeLinkStyle : linkStyle)}
|
||||
@@ -97,6 +104,7 @@ export default function App() {
|
||||
<Route path="/watchers" element={<Watchers />} />
|
||||
<Route path="/files" element={<Files />} />
|
||||
<Route path="/identity" element={<Identity />} />
|
||||
<Route path="/reports" element={<Reports />} />
|
||||
<Route path="/editor" element={<EditorPage />} />
|
||||
</Routes>
|
||||
</div>
|
||||
|
||||
@@ -1,155 +1,53 @@
|
||||
import { useState, useEffect } from 'react';
|
||||
import { useEffect } from 'react';
|
||||
// eslint-disable-next-line @typescript-eslint/ban-ts-comment
|
||||
// @ts-ignore -- plain ES module, no .d.ts; imported for its side effect
|
||||
// of defining the custom element.
|
||||
import '../meta-service-menu.js';
|
||||
|
||||
interface ServiceInfo {
|
||||
name: string;
|
||||
hostname: string;
|
||||
baseUrl: string;
|
||||
status: string;
|
||||
lastHeartbeat: string;
|
||||
}
|
||||
/**
|
||||
* ServiceNav — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Since meta-discovery v1 this is a thin wrapper around <meta-service-menu>,
|
||||
* a framework-free custom element shared by every meta-* UI (see
|
||||
* scripts/check-mirrors.sh). The old implementation was ~165 lines of React
|
||||
* that existed in four near-identical copies and polled meta-core's
|
||||
* /api/services; the element polls this service's own /api/neighbors instead,
|
||||
* so the nav still renders when meta-core is down.
|
||||
*
|
||||
* The element lives in shadow DOM and cannot see this app's CSS variables, so
|
||||
* the four --mm-nav-* tokens below map our palette onto it.
|
||||
*/
|
||||
|
||||
interface ServicesResponse {
|
||||
services: ServiceInfo[];
|
||||
current: string;
|
||||
}
|
||||
|
||||
const serviceIcons: Record<string, string> = {
|
||||
'meta-core': '🗄️',
|
||||
'meta-sort': '📁',
|
||||
'meta-fuse': '🗂️',
|
||||
'meta-stremio': '🎬',
|
||||
'meta-orbit': '🌐',
|
||||
'default': '📦'
|
||||
};
|
||||
|
||||
function formatServiceName(name: string): string {
|
||||
return name.split('-').map(word =>
|
||||
word.charAt(0).toUpperCase() + word.slice(1)
|
||||
).join(' ');
|
||||
declare global {
|
||||
// eslint-disable-next-line @typescript-eslint/no-namespace
|
||||
namespace JSX {
|
||||
interface IntrinsicElements {
|
||||
'meta-service-menu': React.DetailedHTMLProps<
|
||||
React.HTMLAttributes<HTMLElement>,
|
||||
HTMLElement
|
||||
> & { current?: string; endpoint?: string; label?: string };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function ServiceNav() {
|
||||
const [services, setServices] = useState<ServiceInfo[]>([]);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const currentService = 'meta-core';
|
||||
|
||||
useEffect(() => {
|
||||
const fetchServices = async () => {
|
||||
try {
|
||||
const response = await fetch('/services');
|
||||
if (response.ok) {
|
||||
const data: ServicesResponse = await response.json();
|
||||
setServices(data.services || []);
|
||||
}
|
||||
} catch (error) {
|
||||
console.error('Failed to fetch services:', error);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
};
|
||||
|
||||
fetchServices();
|
||||
const interval = setInterval(fetchServices, 30000);
|
||||
return () => clearInterval(interval);
|
||||
// Import above is for the side effect of defining the element; this
|
||||
// keeps bundlers from tree-shaking it away in production builds.
|
||||
}, []);
|
||||
|
||||
if (loading) return null;
|
||||
|
||||
const sortedServices = [...services].sort((a, b) => a.name.localeCompare(b.name));
|
||||
|
||||
if (sortedServices.length === 0) return null;
|
||||
|
||||
return (
|
||||
<nav className="services-nav">
|
||||
<span className="services-nav-label">Services:</span>
|
||||
<div className="services-nav-items">
|
||||
{sortedServices.map(service => {
|
||||
const icon = serviceIcons[service.name] || serviceIcons.default;
|
||||
const isActive = service.name === currentService;
|
||||
return (
|
||||
<a
|
||||
key={service.name}
|
||||
href={isActive ? '#' : service.baseUrl}
|
||||
className={`service-link${isActive ? ' active' : ''}`}
|
||||
onClick={(e) => {
|
||||
e.preventDefault();
|
||||
if (!isActive) {
|
||||
window.location.href = service.baseUrl;
|
||||
}
|
||||
}}
|
||||
>
|
||||
<span className="service-icon">{icon}</span>
|
||||
<span>{formatServiceName(service.name)}</span>
|
||||
<span className="service-status"></span>
|
||||
</a>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
|
||||
<>
|
||||
<meta-service-menu current="meta-core" />
|
||||
<style>{`
|
||||
.services-nav {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.5rem;
|
||||
}
|
||||
|
||||
.services-nav-label {
|
||||
color: #888;
|
||||
font-size: 0.8rem;
|
||||
white-space: nowrap;
|
||||
}
|
||||
|
||||
.services-nav-items {
|
||||
display: flex;
|
||||
gap: 0.35rem;
|
||||
}
|
||||
|
||||
.service-link {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
gap: 0.3rem;
|
||||
padding: 0.35rem 0.6rem;
|
||||
border-radius: 6px;
|
||||
text-decoration: none;
|
||||
font-size: 0.8rem;
|
||||
font-weight: 500;
|
||||
transition: all 0.2s;
|
||||
color: #ccc;
|
||||
background: #0f3460;
|
||||
border: 1px solid #1a4a7a;
|
||||
}
|
||||
|
||||
.service-link:hover {
|
||||
background: #1a4a7a;
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
text-decoration: none;
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.service-link.active {
|
||||
background: linear-gradient(135deg, rgba(78, 205, 196, 0.2), rgba(68, 160, 141, 0.2));
|
||||
border-color: rgba(78, 205, 196, 0.5);
|
||||
color: #fff;
|
||||
}
|
||||
|
||||
.service-link .service-icon {
|
||||
font-size: 0.9rem;
|
||||
}
|
||||
|
||||
.service-link .service-status {
|
||||
width: 5px;
|
||||
height: 5px;
|
||||
border-radius: 50%;
|
||||
background: #4ecdc4;
|
||||
}
|
||||
|
||||
@media (max-width: 800px) {
|
||||
.services-nav-label {
|
||||
display: none;
|
||||
}
|
||||
meta-service-menu {
|
||||
--mm-nav-fg: var(--text-primary, #e0e0e0);
|
||||
--mm-nav-bg: var(--bg-tertiary, var(--bg-secondary, #1a1a2e));
|
||||
--mm-nav-border: var(--border-color, rgba(255,255,255,0.14));
|
||||
--mm-nav-accent: var(--accent-primary, #4ecdc4);
|
||||
}
|
||||
`}</style>
|
||||
</nav>
|
||||
</>
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,28 @@
|
||||
// JSON fetch that fails with a sentence instead of a parser error.
|
||||
//
|
||||
// nginx serves the dashboard with `try_files $uri $uri/ /index.html`, so ANY
|
||||
// path it does not have a proxy rule for answers 200 text/html with the SPA
|
||||
// shell. A caller that does `res.ok && res.json()` on such a path gets
|
||||
// `SyntaxError: Unexpected token '<', "<!doctype "... is not valid JSON` and
|
||||
// no clue which request produced it — which is exactly how a route deleted
|
||||
// from the Go router (`/services`, removed in favour of /api/neighbors) sat
|
||||
// broken on the dashboard.
|
||||
//
|
||||
// So: check the content type, and name the URL in every error.
|
||||
|
||||
export async function getJSON<T>(url: string, init?: RequestInit): Promise<T> {
|
||||
const res = await fetch(url, init);
|
||||
const ct = res.headers.get('content-type') || '';
|
||||
|
||||
if (!res.ok) {
|
||||
throw new Error(`${url} → HTTP ${res.status}`);
|
||||
}
|
||||
if (!ct.includes('application/json')) {
|
||||
// The SPA fallback is the overwhelmingly likely cause; say so.
|
||||
throw new Error(
|
||||
`${url} → ${res.status} ${ct || 'no content-type'} (not JSON — the route is ` +
|
||||
`missing and nginx served the dashboard shell instead)`
|
||||
);
|
||||
}
|
||||
return (await res.json()) as T;
|
||||
}
|
||||
@@ -0,0 +1,271 @@
|
||||
// How a report reads at a glance (the inbox row headline) and in full (the
|
||||
// copy-as-text dump pasted into the Deck board).
|
||||
//
|
||||
// meta-core stores a report's body verbatim and knows nothing about it, so
|
||||
// the meaning lives here, per app. The meta-watch headline is ported from the
|
||||
// /info page that used to own reports (meta-watch ui/info.html,
|
||||
// reportHeadline) — its comments explain why each chip earned its place; the
|
||||
// short version is that every chip there is a stall or desync that once read
|
||||
// as "healthy" until it was measured. Anything an app adds that this file does
|
||||
// not know still renders: in the dump, and as a JSON section in the detail.
|
||||
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
export type Body = Record<string, any>;
|
||||
|
||||
export interface ReportSummary {
|
||||
id: number;
|
||||
ts: number;
|
||||
received: number;
|
||||
app: string;
|
||||
build?: string;
|
||||
uid?: string;
|
||||
page?: string;
|
||||
route?: string;
|
||||
cid?: string;
|
||||
title?: string;
|
||||
note?: string;
|
||||
status: 'open' | 'triaged' | 'closed';
|
||||
triage_note?: string;
|
||||
updated?: number;
|
||||
log_lines: number;
|
||||
bytes: number;
|
||||
}
|
||||
|
||||
export interface Report extends ReportSummary {
|
||||
body: Body;
|
||||
}
|
||||
|
||||
/** One headline segment; `strong` ones are what a human wrote or a failure. */
|
||||
export interface Bit {
|
||||
text: string;
|
||||
strong?: boolean;
|
||||
}
|
||||
|
||||
export function fmtDur(ms: number): string {
|
||||
if (ms == null || !isFinite(ms)) return '?';
|
||||
if (ms < 1000) return `${Math.round(ms)}ms`;
|
||||
const s = ms / 1000;
|
||||
if (s < 60) return `${s.toFixed(s < 10 ? 1 : 0)}s`;
|
||||
const m = Math.floor(s / 60);
|
||||
const r = Math.round(s % 60);
|
||||
if (m < 60) return `${m}m${r ? ` ${r}s` : ''}`;
|
||||
return `${Math.floor(m / 60)}h ${m % 60}m`;
|
||||
}
|
||||
|
||||
export function shortCid(cid: string): string {
|
||||
return cid && cid.length > 16 ? `${cid.slice(0, 8)}…${cid.slice(-6)}` : cid || '';
|
||||
}
|
||||
|
||||
const clip = (s: string, n: number) => (s.length > n ? s.slice(0, n - 1) + '…' : s);
|
||||
|
||||
/** First line of the build blob, minus the app prefix: which build filed it. */
|
||||
export function buildLabel(r: ReportSummary): string {
|
||||
const first = (r.build || '').split('\n')[0];
|
||||
return first.replace(/^meta-(watch|read|listen)( page| server)?\s+/, '').trim();
|
||||
}
|
||||
|
||||
/** The device a report came from, in one line. */
|
||||
export function deviceLine(d: any): string {
|
||||
if (!d || typeof d !== 'object') return '';
|
||||
const bits: string[] = [];
|
||||
if (d.model) bits.push(d.model);
|
||||
else if (d.platform) bits.push(d.platform);
|
||||
if (d.isTv) bits.push(d.tvForced ? 'TV (forced)' : 'TV');
|
||||
if (d.screen) bits.push(d.screen);
|
||||
if (d.cores) bits.push(`${d.cores} cores`);
|
||||
if (d.memoryGb) bits.push(`${d.memoryGb} GB`);
|
||||
if (d.gpu) bits.push(d.gpu);
|
||||
return bits.join(' · ');
|
||||
}
|
||||
|
||||
/** What the row is about: a title, a page + route, or a cid. */
|
||||
export function subject(r: ReportSummary, body?: Body): string {
|
||||
const c = (body && body.context) || {};
|
||||
if (r.page === 'search') return c.query ? `search “${c.query}”` : 'search';
|
||||
if (r.title) return r.title;
|
||||
if (r.page && r.page !== 'watch' && r.page !== 'read') return r.page;
|
||||
if (r.cid) return shortCid(r.cid);
|
||||
return r.page || '—';
|
||||
}
|
||||
|
||||
// ── Headlines ─────────────────────────────────────────────────────────────
|
||||
|
||||
function failureBit(c: any): Bit | null {
|
||||
const f = c && (c.failure || c.lastFailure);
|
||||
if (!f) return null;
|
||||
const who = c.lastFailure ? ` ${f.title || shortCid(f.cid || '')}` : '';
|
||||
const tries = Array.isArray(f.attempts) && f.attempts.length > 1 ? ` · ${f.attempts.length} tries` : '';
|
||||
return {
|
||||
text: `FAILED${who}${f.elapsedMs != null ? ' after ' + fmtDur(f.elapsedMs) : ''}: ${clip(f.message || '', 90)}${tries}`,
|
||||
strong: true,
|
||||
};
|
||||
}
|
||||
|
||||
function watchBits(r: ReportSummary, b: Body): string[] {
|
||||
const bits: string[] = [];
|
||||
const c = b.context || {};
|
||||
if (r.page === 'search') {
|
||||
if (c.state) bits.push(c.state === 'errored' ? 'STREAM ERRORED' : c.state);
|
||||
if (c.anchored != null) {
|
||||
bits.push(
|
||||
`${c.anchored} anchored · ${c.raw || 0} raw` +
|
||||
(c.shown != null && c.shown !== c.anchored ? ` · ${c.shown} after language filter` : '')
|
||||
);
|
||||
}
|
||||
if (c.elapsedMs != null) bits.push(`stream ${fmtDur(c.elapsedMs)}`);
|
||||
if (c.cacheSeeded) bits.push('painted from cache');
|
||||
const ups = Array.isArray(c.upstreams) ? c.upstreams : [];
|
||||
if (c.state !== 'idle' && c.state !== 'youtube-paste') {
|
||||
if (ups.length) {
|
||||
const recs = ups.reduce((n: number, u: any) => n + (u.records || 0), 0);
|
||||
bits.push(`${ups.length} upstream${ups.length === 1 ? '' : 's'} (${recs} records)`);
|
||||
} else bits.push('no upstream seen');
|
||||
}
|
||||
} else if (r.page === 'title') {
|
||||
if (c.episodes != null) {
|
||||
bits.push(`${c.episodes} release${c.episodes === 1 ? '' : 's'}` + (c.rawSources ? ` · ${c.rawSources} raw` : ''));
|
||||
}
|
||||
if (c.streamsOpen) bits.push(`${c.streamsOpen} stream${c.streamsOpen === 1 ? '' : 's'} open`);
|
||||
else if (c.settled != null) bits.push(c.settled ? 'settled' : 'not settled');
|
||||
if (c.placeholderCard) bits.push('placeholder hero');
|
||||
}
|
||||
if (c.lastPlayback) bits.push(`after playing ${c.lastPlayback.title || shortCid(c.lastPlayback.cid || '')}`);
|
||||
|
||||
const d = b.decode || {};
|
||||
const p = b.pipeline || {};
|
||||
if (d.underruns != null) bits.push(`${d.underruns} underrun${d.underruns === 1 ? '' : 's'}`);
|
||||
if (d.minAhead != null) bits.push(`min ${(+d.minAhead).toFixed(1)}s ahead`);
|
||||
if (d.decoded) {
|
||||
const pct = (d.dropped / d.decoded) * 100;
|
||||
bits.push(`${d.dropped} dropped (${pct.toFixed(pct >= 1 ? 1 : 2)}%)`);
|
||||
}
|
||||
if (d.capability) bits.push(d.capability);
|
||||
if (p.pctDemuxCpu != null) bits.push(`demux CPU ${p.pctDemuxCpu}%`);
|
||||
|
||||
// Subtitle verdict: a stalled overlay is invisible in every other number.
|
||||
const sub = b.subtitles;
|
||||
if (sub && sub.up) {
|
||||
const ago = sub.lastDrawAgoMs;
|
||||
if (ago == null) bits.push('subs: NO DRAWS');
|
||||
else if (ago > 4000) {
|
||||
const why =
|
||||
sub.paused === true ? ' while PAUSED'
|
||||
: sub.advancing === true ? ' while playing'
|
||||
: sub.paused === false ? ' (playing, not advancing)'
|
||||
: '';
|
||||
bits.push(`subs: STALLED ${fmtDur(ago)}${why}`);
|
||||
} else bits.push(`subs ok (${sub.draws} draws)`);
|
||||
if (sub.skipped > 0 && sub.draws > 0) {
|
||||
const share = Math.round((100 * sub.skipped) / (sub.skipped + sub.draws));
|
||||
if (share >= 15) bits.push(`subs: ${share}% of frames dropped`);
|
||||
}
|
||||
if (sub.commitMaxMs != null && sub.commitMaxMs > 250) {
|
||||
bits.push(
|
||||
`subs: commit ${sub.commitMs}/${sub.commitMaxMs}ms` +
|
||||
(sub.pruned ? ` (${sub.lines}+${sub.pruned}p lines)` : ` (${sub.lines} lines)`)
|
||||
);
|
||||
}
|
||||
if (sub.lastEventAgoMs != null && sub.lastEventAgoMs > 30000 && sub.advancing) {
|
||||
bits.push(`subs: no packets for ${fmtDur(sub.lastEventAgoMs)}`);
|
||||
}
|
||||
if (sub.canvas && sub.canvas !== 'ok') bits.push(`subs canvas ${sub.canvas}`);
|
||||
if (sub.videoW === 0 || sub.resizeDeferred) bits.push('subs: overlay resize owed (video has no size)');
|
||||
else if (sub.resizesBlocked > 0) {
|
||||
bits.push(`subs: ${sub.resizesBlocked} zero-size resize${sub.resizesBlocked > 1 ? 's' : ''} blocked`);
|
||||
}
|
||||
} else if (sub && sub.failed) {
|
||||
const dd = sub.death;
|
||||
if (!dd) bits.push('subs: fell back to WebVTT (build too old to say why)');
|
||||
else {
|
||||
bits.push(`subs: fell back to WebVTT at ${fmtDur(dd.mediaTime)} — ${dd.why}`);
|
||||
if (dd.commitFailures) bits.push(`after ${dd.commitFailures} rejected commits`);
|
||||
if (dd.strikes) bits.push(`after ${dd.strikes} stuck-loop strikes`);
|
||||
bits.push(dd.draws ? `had drawn ${dd.draws} frames (${dd.lines}+${dd.pruned}p lines)` : 'never drew a frame');
|
||||
if (dd.canvas && dd.canvas !== 'ok') bits.push(`canvas ${dd.canvas} at death`);
|
||||
if (dd.resizesBlocked) bits.push(`${dd.resizesBlocked} zero-size resizes blocked`);
|
||||
if (dd.videoW === 0) bits.push('video had no size at death');
|
||||
}
|
||||
}
|
||||
|
||||
// A/V sync: only the peak earns a chip.
|
||||
const au = b.audio;
|
||||
if (au && au.driftMaxMs != null && Math.abs(au.driftMaxMs) >= 40) {
|
||||
const at = au.driftMaxAtSec != null ? ` @ ${fmtDur(au.driftMaxAtSec * 1000)}` : '';
|
||||
bits.push(`A/V DESYNC ${au.driftMaxMs > 0 ? '+' : ''}${au.driftMaxMs}ms${at}`);
|
||||
}
|
||||
if (au && au.path === 'polyfill') bits.push(`audio ${au.codec || '?'} via polyfill`);
|
||||
const rs = b.restart;
|
||||
if (rs && rs.audioMinusVideoSec != null && rs.audioMinusVideoSec < -0.001) {
|
||||
bits.push(`restart base from AUDIO (${rs.audioMinusVideoSec.toFixed(3)}s)`);
|
||||
}
|
||||
if (b.ttmvpMs != null) bits.push(`TTMVP ${fmtDur(b.ttmvpMs)}`);
|
||||
if (b.capturedMsAgo != null && b.capturedMsAgo > 2000) bits.push(`measured ${fmtDur(b.capturedMsAgo)} before filing`);
|
||||
return bits;
|
||||
}
|
||||
|
||||
function readBits(r: ReportSummary, b: Body): string[] {
|
||||
const bits: string[] = [];
|
||||
const rd = b.reader;
|
||||
if (rd && typeof rd === 'object') {
|
||||
if (rd.page != null) bits.push(`page ${rd.page + 1}${rd.pages ? `/${rd.pages}` : ''}`);
|
||||
if (rd.flow) bits.push(rd.flow);
|
||||
if (rd.direction) bits.push(rd.direction);
|
||||
if (rd.spread) bits.push('spread');
|
||||
if (rd.strip && rd.strip.segments) bits.push(`strip ${rd.strip.active ?? '?'}/${rd.strip.segments}`);
|
||||
if (rd.via) bits.push(`via ${rd.via}`);
|
||||
if (rd.atEnd) bits.push('at end');
|
||||
}
|
||||
const c = b.context || {};
|
||||
if (r.page === 'search') {
|
||||
if (c.state) bits.push(c.state);
|
||||
if (c.count != null) bits.push(`${c.count} shown`);
|
||||
}
|
||||
return bits;
|
||||
}
|
||||
|
||||
/** The row headline: the note first (the only part a human wrote), then a
|
||||
* failure if there was one, then the app's own measurements. */
|
||||
export function headline(r: ReportSummary, body?: Body): Bit[] {
|
||||
const out: Bit[] = [];
|
||||
if (r.note) out.push({ text: `“${clip(r.note, 80)}”`, strong: true });
|
||||
if (!body) return out;
|
||||
const f = failureBit(body.context);
|
||||
if (f) out.push(f);
|
||||
const bits = r.app === 'meta-watch' ? watchBits(r, body) : r.app === 'meta-read' ? readBits(r, body) : [];
|
||||
for (const t of bits) out.push({ text: t });
|
||||
if (body.flags && typeof body.flags === 'object' && Object.keys(body.flags).length) {
|
||||
out.push({ text: 'overrides: ' + Object.entries(body.flags).map(([k, v]) => `${k}=${v}`).join(' ') });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── Full text dump ────────────────────────────────────────────────────────
|
||||
|
||||
const LIFTED = new Set(['app', 'build', 'uid', 'page', 'route', 'cid', 'title', 'note', 'log']);
|
||||
|
||||
/** Every block of a report that is not one of the lifted fields or the log,
|
||||
* in a stable order: context first, device second, then alphabetical. */
|
||||
export function blocks(body: Body): [string, any][] {
|
||||
const keys = Object.keys(body).filter((k) => !LIFTED.has(k));
|
||||
const rank = (k: string) => (k === 'context' ? 0 : k === 'device' ? 1 : 2);
|
||||
keys.sort((a, b) => rank(a) - rank(b) || a.localeCompare(b));
|
||||
return keys.map((k) => [k, body[k]]);
|
||||
}
|
||||
|
||||
/** The whole report as text, for pasting into a ticket. */
|
||||
export function dump(r: Report): string {
|
||||
const parts: string[] = [];
|
||||
parts.push(`REPORT #${r.id} · ${r.app} · ${new Date(r.ts).toISOString()} · ${r.status}` +
|
||||
(r.uid ? `\nUID ${r.uid}` : ''));
|
||||
if (r.note) parts.push('NOTE\n' + r.note);
|
||||
if (r.page || r.route) parts.push('PAGE ' + (r.page || '—') + (r.route ? '\nROUTE ' + r.route : ''));
|
||||
if (r.cid || r.title) parts.push(`CONTENT ${r.title || ''}${r.cid ? ` (${r.cid})` : ''}`.trim());
|
||||
if (r.build) parts.push(r.build);
|
||||
for (const [k, v] of blocks(r.body || {})) {
|
||||
parts.push(k.toUpperCase() + '\n' + (typeof v === 'string' ? v : JSON.stringify(v, null, 2)));
|
||||
}
|
||||
const log: string[] = Array.isArray(r.body?.log) ? r.body.log : [];
|
||||
if (log.length) parts.push(`LOG (${log.length} lines)\n` + log.join('\n'));
|
||||
if (r.triage_note) parts.push('TRIAGE\n' + r.triage_note);
|
||||
return parts.join('\n\n');
|
||||
}
|
||||
@@ -0,0 +1,308 @@
|
||||
/**
|
||||
* <meta-service-menu> — the neighbouring meta-* services, as a burger menu.
|
||||
*
|
||||
* Replaces the eight divergent "services bar" implementations (four copies of
|
||||
* a React `ServiceNav.tsx`, two mirrored `services.js`, two bespoke inline
|
||||
* ones) with a single framework-free custom element. It drops into the React
|
||||
* dashboards as a plain tag — no React dependency — and into the vanilla UIs
|
||||
* as an ES module.
|
||||
*
|
||||
* Data comes from the host service's own `GET /api/neighbors`, same-origin.
|
||||
* Never cross-origin: the dev stack's self-signed certs and the OIDC redirects
|
||||
* make a fetch to a neighbour a dead end. Links are plain navigations.
|
||||
*
|
||||
* ⚠ MIRRORED FILE. Byte-identical copies live in every meta-* UI tree; see
|
||||
* `scripts/check-mirrors.sh`. Edit one → edit all.
|
||||
*
|
||||
* Shadow DOM is not optional here. The eight UIs use four incompatible token
|
||||
* vocabularies, and two of them define *different colours under the same
|
||||
* names* (`--bg-secondary` is #12121a in meta-sort, #16213e in meta-core,
|
||||
* #161b22 in meta-dup). meta-search and meta-share are also the only
|
||||
* light-mode-capable UIs, so a hardcoded dark palette would break them.
|
||||
* The element therefore styles itself from four tokens with dark defaults:
|
||||
*
|
||||
* --mm-nav-fg, --mm-nav-bg, --mm-nav-border, --mm-nav-accent
|
||||
*
|
||||
* Each host maps its own variables onto those, e.g.
|
||||
* meta-service-menu { --mm-nav-bg: var(--bg-secondary); }
|
||||
*
|
||||
* Attributes:
|
||||
* current name of the host service, so it can be marked and not linked
|
||||
* endpoint override the default "/api/neighbors"
|
||||
* label button tooltip (default "Services")
|
||||
*
|
||||
* Property:
|
||||
* rewriteUrl optional (name, url, neighbour) => url, for hosts that must
|
||||
* translate internal hostnames into browser-reachable ones
|
||||
* (meta-gateway does this for its dev/prod split).
|
||||
*/
|
||||
|
||||
const ICONS = {
|
||||
"meta-core": "⚙️",
|
||||
"meta-sort": "📁",
|
||||
"meta-fuse": "🗂️",
|
||||
"meta-stremio": "🎬",
|
||||
"meta-dup": "🔍",
|
||||
"meta-search": "🔎",
|
||||
"meta-share": "🔗",
|
||||
"meta-gateway": "🌉",
|
||||
"meta-watch": "📺",
|
||||
"meta-listen": "🎧",
|
||||
"meta-read": "📖",
|
||||
};
|
||||
const DEFAULT_ICON = "📦";
|
||||
const POLL_MS = 30000;
|
||||
|
||||
function titleCase(name) {
|
||||
return String(name || "")
|
||||
.split("-")
|
||||
.map((w) => w.charAt(0).toUpperCase() + w.slice(1))
|
||||
.join(" ");
|
||||
}
|
||||
|
||||
const TEMPLATE = `
|
||||
<style>
|
||||
:host {
|
||||
--_fg: var(--mm-nav-fg, #e8eaed);
|
||||
--_bg: var(--mm-nav-bg, #1a1a2e);
|
||||
--_border: var(--mm-nav-border, rgba(255, 255, 255, 0.14));
|
||||
--_accent: var(--mm-nav-accent, #4ecdc4);
|
||||
position: relative;
|
||||
display: inline-block;
|
||||
font-family: inherit;
|
||||
font-size: 14px;
|
||||
line-height: 1.4;
|
||||
}
|
||||
button.trigger {
|
||||
display: inline-flex;
|
||||
align-items: center;
|
||||
justify-content: center;
|
||||
gap: 0.4rem;
|
||||
width: 2.1rem;
|
||||
height: 2.1rem;
|
||||
padding: 0;
|
||||
cursor: pointer;
|
||||
color: var(--_fg);
|
||||
background: transparent;
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 8px;
|
||||
font: inherit;
|
||||
transition: border-color 0.15s, background 0.15s;
|
||||
}
|
||||
button.trigger:hover { border-color: var(--_accent); }
|
||||
button.trigger:focus-visible {
|
||||
outline: 2px solid var(--_accent);
|
||||
outline-offset: 2px;
|
||||
}
|
||||
button.trigger[aria-expanded="true"] { border-color: var(--_accent); }
|
||||
.bars { display: block; width: 15px; }
|
||||
.bars span {
|
||||
display: block;
|
||||
height: 2px;
|
||||
margin: 3px 0;
|
||||
border-radius: 2px;
|
||||
background: currentColor;
|
||||
}
|
||||
.panel {
|
||||
position: absolute;
|
||||
top: calc(100% + 6px);
|
||||
right: 0;
|
||||
z-index: 1000;
|
||||
min-width: 13rem;
|
||||
max-height: 70vh;
|
||||
overflow-y: auto;
|
||||
padding: 0.3rem;
|
||||
background: var(--_bg);
|
||||
border: 1px solid var(--_border);
|
||||
border-radius: 10px;
|
||||
box-shadow: 0 10px 30px rgba(0, 0, 0, 0.35);
|
||||
}
|
||||
.panel[hidden] { display: none; }
|
||||
a.item, .item {
|
||||
display: flex;
|
||||
align-items: center;
|
||||
gap: 0.55rem;
|
||||
padding: 0.45rem 0.6rem;
|
||||
border-radius: 6px;
|
||||
color: var(--_fg);
|
||||
text-decoration: none;
|
||||
white-space: nowrap;
|
||||
}
|
||||
a.item:hover { background: color-mix(in srgb, var(--_accent) 18%, transparent); }
|
||||
.item.active { color: var(--_accent); font-weight: 600; cursor: default; }
|
||||
.item.dead { opacity: 0.55; cursor: not-allowed; }
|
||||
.icon { font-size: 1rem; line-height: 1; }
|
||||
.dot {
|
||||
width: 6px;
|
||||
height: 6px;
|
||||
margin-left: auto;
|
||||
border-radius: 50%;
|
||||
background: var(--_accent);
|
||||
}
|
||||
.empty {
|
||||
padding: 0.6rem;
|
||||
opacity: 0.7;
|
||||
font-size: 0.85em;
|
||||
white-space: nowrap;
|
||||
}
|
||||
/* Respect a host that opts out of motion. */
|
||||
@media (prefers-reduced-motion: reduce) {
|
||||
button.trigger { transition: none; }
|
||||
}
|
||||
</style>
|
||||
<button class="trigger" type="button" aria-haspopup="menu" aria-expanded="false">
|
||||
<span class="bars" aria-hidden="true"><span></span><span></span><span></span></span>
|
||||
</button>
|
||||
<div class="panel" role="menu" hidden></div>
|
||||
`;
|
||||
|
||||
class MetaServiceMenu extends HTMLElement {
|
||||
constructor() {
|
||||
super();
|
||||
this.attachShadow({ mode: "open" });
|
||||
this.shadowRoot.innerHTML = TEMPLATE;
|
||||
this._btn = this.shadowRoot.querySelector("button.trigger");
|
||||
this._panel = this.shadowRoot.querySelector(".panel");
|
||||
this._neighbors = [];
|
||||
this._timer = null;
|
||||
this._open = false;
|
||||
|
||||
this._onDocClick = (e) => {
|
||||
if (!this._open) return;
|
||||
if (e.composedPath().includes(this)) return;
|
||||
this._setOpen(false);
|
||||
};
|
||||
this._onKey = (e) => {
|
||||
if (e.key === "Escape" && this._open) {
|
||||
this._setOpen(false);
|
||||
this._btn.focus();
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
connectedCallback() {
|
||||
this._btn.title = this.getAttribute("label") || "Services";
|
||||
this._btn.setAttribute("aria-label", this._btn.title);
|
||||
this._btn.addEventListener("click", () => this._setOpen(!this._open));
|
||||
document.addEventListener("click", this._onDocClick);
|
||||
document.addEventListener("keydown", this._onKey);
|
||||
|
||||
this.refresh();
|
||||
this._timer = setInterval(() => this.refresh(), POLL_MS);
|
||||
}
|
||||
|
||||
disconnectedCallback() {
|
||||
if (this._timer) clearInterval(this._timer);
|
||||
this._timer = null;
|
||||
document.removeEventListener("click", this._onDocClick);
|
||||
document.removeEventListener("keydown", this._onKey);
|
||||
}
|
||||
|
||||
get endpoint() {
|
||||
return this.getAttribute("endpoint") || "/api/neighbors";
|
||||
}
|
||||
|
||||
get current() {
|
||||
return this.getAttribute("current") || "";
|
||||
}
|
||||
|
||||
async refresh() {
|
||||
try {
|
||||
const res = await fetch(this.endpoint, {
|
||||
headers: { Accept: "application/json" },
|
||||
});
|
||||
if (!res.ok) throw new Error(String(res.status));
|
||||
const data = await res.json();
|
||||
// `services` is the legacy alias some backends still emit.
|
||||
this._neighbors = data.neighbors || data.services || [];
|
||||
} catch {
|
||||
// Silent: the menu is navigation, not a health indicator. Keep whatever
|
||||
// we last had rather than blanking the list on one failed poll.
|
||||
}
|
||||
this._render();
|
||||
}
|
||||
|
||||
_setOpen(open) {
|
||||
this._open = open;
|
||||
this._panel.hidden = !open;
|
||||
this._btn.setAttribute("aria-expanded", String(open));
|
||||
if (open) this.refresh();
|
||||
}
|
||||
|
||||
_render() {
|
||||
const panel = this._panel;
|
||||
panel.textContent = "";
|
||||
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const n of this._neighbors) {
|
||||
const name = n.name || "";
|
||||
if (!name || seen.has(name)) continue;
|
||||
// Beacon v2 neighbours include plugins; the nav lists services only.
|
||||
// (A row without `caps` comes from a pre-v2 backend — keep it.)
|
||||
const isService = (c) => String(c).startsWith("metamesh.service/");
|
||||
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
|
||||
seen.add(name);
|
||||
rows.push(n);
|
||||
}
|
||||
// Always show the host itself, even before its own echo comes back.
|
||||
if (this.current && !seen.has(this.current)) {
|
||||
rows.push({ name: this.current, baseUrl: "" });
|
||||
}
|
||||
rows.sort((a, b) => String(a.name).localeCompare(String(b.name)));
|
||||
|
||||
if (rows.length === 0) {
|
||||
const empty = document.createElement("div");
|
||||
empty.className = "empty";
|
||||
empty.textContent = "No services found";
|
||||
panel.appendChild(empty);
|
||||
return;
|
||||
}
|
||||
|
||||
for (const n of rows) {
|
||||
const isActive = n.name === this.current;
|
||||
let url = n.baseUrl || "";
|
||||
if (typeof this.rewriteUrl === "function") {
|
||||
try {
|
||||
url = this.rewriteUrl(n.name, url, n) || "";
|
||||
} catch {
|
||||
/* a bad hook must not blank the menu */
|
||||
}
|
||||
}
|
||||
|
||||
// A neighbour with no reachable URL is shown but not linked — better
|
||||
// than a link that goes nowhere.
|
||||
const el = document.createElement(isActive || !url ? "span" : "a");
|
||||
el.className =
|
||||
"item" + (isActive ? " active" : "") + (!isActive && !url ? " dead" : "");
|
||||
if (el.tagName === "A") {
|
||||
el.href = url;
|
||||
el.setAttribute("role", "menuitem");
|
||||
}
|
||||
if (!isActive && !url) el.title = "No reachable URL advertised";
|
||||
|
||||
const icon = document.createElement("span");
|
||||
icon.className = "icon";
|
||||
icon.textContent = ICONS[n.name] || DEFAULT_ICON;
|
||||
|
||||
const label = document.createElement("span");
|
||||
label.textContent = titleCase(n.name);
|
||||
|
||||
el.append(icon, label);
|
||||
// Beacon v2 omits `status` when running.
|
||||
if ((n.status || "running") === "running" || isActive) {
|
||||
const dot = document.createElement("span");
|
||||
dot.className = "dot";
|
||||
dot.title = n.status || "running";
|
||||
el.appendChild(dot);
|
||||
}
|
||||
panel.appendChild(el);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!customElements.get("meta-service-menu")) {
|
||||
customElements.define("meta-service-menu", MetaServiceMenu);
|
||||
}
|
||||
|
||||
export default MetaServiceMenu;
|
||||
@@ -1,4 +1,5 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
import { getJSON } from '../lib/api';
|
||||
|
||||
interface HealthStatus {
|
||||
status: string;
|
||||
@@ -6,6 +7,22 @@ interface HealthStatus {
|
||||
timestamp: string;
|
||||
}
|
||||
|
||||
interface FileStats {
|
||||
count: number;
|
||||
totalSize: number;
|
||||
}
|
||||
|
||||
interface Stats {
|
||||
records: number;
|
||||
redisKeys: number;
|
||||
redisMemory: string;
|
||||
identities: number;
|
||||
files: FileStats | null;
|
||||
udlUsers: number | null;
|
||||
sweptAt: number;
|
||||
sweeping: boolean;
|
||||
}
|
||||
|
||||
interface WatcherStatus {
|
||||
id: string;
|
||||
path: string;
|
||||
@@ -22,13 +39,6 @@ interface WatchersResponse {
|
||||
count: number;
|
||||
}
|
||||
|
||||
interface ServiceInfo {
|
||||
name: string;
|
||||
api: string;
|
||||
capabilities: string[];
|
||||
timestamp: number;
|
||||
}
|
||||
|
||||
const cardStyle: React.CSSProperties = {
|
||||
background: '#16213e',
|
||||
borderRadius: '8px',
|
||||
@@ -38,53 +48,110 @@ const cardStyle: React.CSSProperties = {
|
||||
|
||||
const gridStyle: React.CSSProperties = {
|
||||
display: 'grid',
|
||||
gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))',
|
||||
// 200px, not 220: five tiles must fit one row inside the 1200px container
|
||||
// (1136 content - 4x16 gap = 1072 / 5 = 214 each) before wrapping.
|
||||
gridTemplateColumns: 'repeat(auto-fit, minmax(200px, 1fr))',
|
||||
gap: '1rem',
|
||||
marginBottom: '1rem',
|
||||
};
|
||||
|
||||
const statStyle: React.CSSProperties = {
|
||||
fontSize: '2rem',
|
||||
fontWeight: 'bold',
|
||||
color: '#4ade80',
|
||||
lineHeight: 1.2,
|
||||
};
|
||||
|
||||
const labelStyle: React.CSSProperties = {
|
||||
marginBottom: '0.75rem',
|
||||
color: '#888',
|
||||
fontSize: '0.85rem',
|
||||
textTransform: 'uppercase',
|
||||
letterSpacing: '0.04em',
|
||||
};
|
||||
|
||||
const noteStyle: React.CSSProperties = {
|
||||
color: '#6b7a99',
|
||||
fontSize: '0.8rem',
|
||||
marginTop: '0.4rem',
|
||||
};
|
||||
|
||||
const numberFmt = new Intl.NumberFormat();
|
||||
|
||||
function formatCount(n: number | null | undefined): string {
|
||||
return n === null || n === undefined ? '—' : numberFmt.format(n);
|
||||
}
|
||||
|
||||
// Binary units: this counts bytes on disk, and every other tool the operator
|
||||
// compares against (du, docker, the mount page) reports GiB.
|
||||
function formatBytes(bytes: number | null | undefined): string {
|
||||
if (bytes === null || bytes === undefined) return '—';
|
||||
const units = ['B', 'KiB', 'MiB', 'GiB', 'TiB', 'PiB'];
|
||||
let value = bytes;
|
||||
let unit = 0;
|
||||
while (value >= 1024 && unit < units.length - 1) {
|
||||
value /= 1024;
|
||||
unit++;
|
||||
}
|
||||
return `${value.toFixed(value >= 100 || unit === 0 ? 0 : 1)} ${units[unit]}`;
|
||||
}
|
||||
|
||||
function Tile({
|
||||
label,
|
||||
value,
|
||||
note,
|
||||
pending,
|
||||
}: {
|
||||
label: string;
|
||||
value: string;
|
||||
note?: string;
|
||||
pending?: boolean;
|
||||
}) {
|
||||
return (
|
||||
<div style={cardStyle}>
|
||||
<div style={labelStyle}>{label}</div>
|
||||
<div style={{ ...statStyle, color: pending ? '#6b7a99' : statStyle.color }}>
|
||||
{value}
|
||||
</div>
|
||||
{note && <div style={noteStyle}>{note}</div>}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
export default function Dashboard() {
|
||||
const [health, setHealth] = useState<HealthStatus | null>(null);
|
||||
const [stats, setStats] = useState<Stats | null>(null);
|
||||
const [watchersData, setWatchersData] = useState<WatchersResponse | null>(null);
|
||||
const [services, setServices] = useState<ServiceInfo[]>([]);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
|
||||
useEffect(() => {
|
||||
let cancelled = false;
|
||||
|
||||
const fetchData = async () => {
|
||||
try {
|
||||
// Fetch health
|
||||
const healthRes = await fetch('/health');
|
||||
if (healthRes.ok) {
|
||||
setHealth(await healthRes.json());
|
||||
}
|
||||
// Settled, not all: one dead endpoint must not blank every tile.
|
||||
const [healthRes, statsRes, watchersRes] = await Promise.allSettled([
|
||||
getJSON<HealthStatus>('/health'),
|
||||
getJSON<Stats>('/api/stats'),
|
||||
getJSON<WatchersResponse>('/api/watchers'),
|
||||
]);
|
||||
if (cancelled) return;
|
||||
|
||||
// Fetch watchers status
|
||||
const watchersRes = await fetch('/api/watchers');
|
||||
if (watchersRes.ok) {
|
||||
setWatchersData(await watchersRes.json());
|
||||
}
|
||||
if (healthRes.status === 'fulfilled') setHealth(healthRes.value);
|
||||
if (statsRes.status === 'fulfilled') setStats(statsRes.value);
|
||||
if (watchersRes.status === 'fulfilled') setWatchersData(watchersRes.value);
|
||||
|
||||
// Fetch services
|
||||
const servicesRes = await fetch('/services');
|
||||
if (servicesRes.ok) {
|
||||
const data = await servicesRes.json();
|
||||
setServices(data.services || []);
|
||||
}
|
||||
|
||||
setError(null);
|
||||
} catch (err) {
|
||||
setError(String(err));
|
||||
}
|
||||
const failures = [healthRes, statsRes, watchersRes]
|
||||
.filter((r): r is PromiseRejectedResult => r.status === 'rejected')
|
||||
.map((r) => String(r.reason?.message ?? r.reason));
|
||||
setError(failures.length ? failures.join(' · ') : null);
|
||||
};
|
||||
|
||||
fetchData();
|
||||
const interval = setInterval(fetchData, 5000);
|
||||
return () => clearInterval(interval);
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearInterval(interval);
|
||||
};
|
||||
}, []);
|
||||
|
||||
const handleTriggerScan = async () => {
|
||||
@@ -95,49 +162,102 @@ export default function Dashboard() {
|
||||
}
|
||||
};
|
||||
|
||||
// Aggregate watcher stats
|
||||
const totalFiles = watchersData?.watchers.reduce((sum, w) => sum + w.fileCount, 0) || 0;
|
||||
const isScanning = watchersData?.watchers.some(w => w.isScanning) || false;
|
||||
const activeWatchers = watchersData?.watchers.filter(w => w.active).length || 0;
|
||||
const lastScan = watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
|
||||
const isScanning = watchersData?.watchers.some((w) => w.isScanning) || false;
|
||||
const activeWatchers = watchersData?.watchers.filter((w) => w.active).length || 0;
|
||||
const lastScan =
|
||||
watchersData?.watchers.reduce((max, w) => Math.max(max, w.lastScan || 0), 0) || 0;
|
||||
|
||||
// Records we hold metadata for but have no local file: gateway/cid-rooted
|
||||
// entries, plus anything whose file went away. The interesting number on a
|
||||
// client box, where it should be most of them.
|
||||
const fileless =
|
||||
stats && stats.files ? Math.max(0, stats.records - stats.files.count) : null;
|
||||
|
||||
const sweepPending = !!stats && !stats.files;
|
||||
const sweptNote = (() => {
|
||||
if (!stats) return undefined;
|
||||
if (sweepPending) return stats.sweeping ? 'scanning…' : 'not scanned yet';
|
||||
if (stats.sweeping) return 'refreshing…';
|
||||
if (stats.sweptAt) return `as of ${new Date(stats.sweptAt).toLocaleTimeString()}`;
|
||||
return undefined;
|
||||
})();
|
||||
|
||||
return (
|
||||
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto' }}>
|
||||
<div style={{ padding: '2rem', maxWidth: '1200px', margin: '0 auto', overflowY: 'auto', height: '100%' }}>
|
||||
<h1 style={{ marginBottom: '2rem' }}>meta-core Dashboard</h1>
|
||||
|
||||
{error && (
|
||||
<div style={{ ...cardStyle, background: '#4a1a1a', color: '#f87171' }}>
|
||||
Error: {error}
|
||||
{error}
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div style={gridStyle}>
|
||||
{/* Health Card */}
|
||||
<Tile
|
||||
label="Files discovered"
|
||||
value={formatCount(stats?.files?.count)}
|
||||
note={sweptNote}
|
||||
pending={sweepPending}
|
||||
/>
|
||||
<Tile
|
||||
label="Library size"
|
||||
value={formatBytes(stats?.files?.totalSize)}
|
||||
note={sweptNote}
|
||||
pending={sweepPending}
|
||||
/>
|
||||
<Tile
|
||||
label="Metadata entries"
|
||||
value={formatCount(stats?.records)}
|
||||
note={
|
||||
fileless === null
|
||||
? 'records in the index'
|
||||
: `${formatCount(fileless)} with no local file`
|
||||
}
|
||||
/>
|
||||
<Tile
|
||||
label="Store keys"
|
||||
value={formatCount(stats?.redisKeys)}
|
||||
note={stats?.redisMemory ? `${stats.redisMemory} on disk` : undefined}
|
||||
/>
|
||||
<Tile
|
||||
label="Identities"
|
||||
value={formatCount(stats?.identities)}
|
||||
note={
|
||||
stats?.udlUsers !== null && stats?.udlUsers !== undefined
|
||||
? `${formatCount(stats.udlUsers)} with user data`
|
||||
: 'signing accounts'
|
||||
}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ ...gridStyle, gridTemplateColumns: 'repeat(auto-fit, minmax(300px, 1fr))' }}>
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Service Health</h3>
|
||||
<div style={labelStyle}>Service health</div>
|
||||
{health ? (
|
||||
<>
|
||||
<div style={statStyle}>
|
||||
{health.status === 'ok' ? 'Healthy' : health.status}
|
||||
</div>
|
||||
<p>Redis: {health.redis ? 'Connected' : 'Disconnected'}</p>
|
||||
<p style={noteStyle}>
|
||||
Metadata store: {health.redis ? 'Open' : 'Unavailable'}
|
||||
</p>
|
||||
</>
|
||||
) : (
|
||||
<p>Loading...</p>
|
||||
<p style={noteStyle}>Loading…</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Watchers Status Card */}
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>File Watchers</h3>
|
||||
<div style={labelStyle}>File watchers</div>
|
||||
{watchersData ? (
|
||||
<>
|
||||
<div style={statStyle}>{totalFiles}</div>
|
||||
<p>Files discovered</p>
|
||||
<p>Watchers: {activeWatchers} active / {watchersData.count} total</p>
|
||||
<p>Status: {isScanning ? 'Scanning...' : 'Idle'}</p>
|
||||
<div style={statStyle}>
|
||||
{activeWatchers} / {watchersData.count}
|
||||
</div>
|
||||
<p style={noteStyle}>active watchers</p>
|
||||
<p style={noteStyle}>Status: {isScanning ? 'Scanning…' : 'Idle'}</p>
|
||||
{lastScan > 0 && (
|
||||
<p>Last scan: {new Date(lastScan).toLocaleString()}</p>
|
||||
<p style={noteStyle}>Last scan: {new Date(lastScan).toLocaleString()}</p>
|
||||
)}
|
||||
<button
|
||||
onClick={handleTriggerScan}
|
||||
@@ -155,27 +275,7 @@ export default function Dashboard() {
|
||||
</button>
|
||||
</>
|
||||
) : (
|
||||
<p>Loading...</p>
|
||||
)}
|
||||
</div>
|
||||
|
||||
{/* Services Card */}
|
||||
<div style={cardStyle}>
|
||||
<h3 style={{ marginBottom: '1rem', color: '#888' }}>Connected Services</h3>
|
||||
<div style={statStyle}>{services.length}</div>
|
||||
<p>Services registered</p>
|
||||
{services.length > 0 && (
|
||||
<ul style={{ marginTop: '1rem', paddingLeft: '1.5rem' }}>
|
||||
{services.map((svc) => (
|
||||
<li key={svc.name} style={{ marginBottom: '0.5rem' }}>
|
||||
<strong>{svc.name}</strong>
|
||||
<br />
|
||||
<span style={{ color: '#888', fontSize: '0.9rem' }}>
|
||||
{svc.capabilities?.join(', ') || 'No capabilities'}
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
<p style={noteStyle}>Loading…</p>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
@@ -179,11 +179,10 @@ async function fetchAccounts(): Promise<Account[]> {
|
||||
// Resolve an account's display name from the User Data Layer, or null.
|
||||
//
|
||||
// Never throws and never reports an error upward: a name is decoration on top
|
||||
// of the uid, and the account list must render identically whether Redis is up
|
||||
// (identity itself is file-backed, the UDL is not). The name is also *unsigned*
|
||||
// — meta-core stores the plaintext beside the signed record and verifies
|
||||
// nothing — so the uid stays on screen as the real identity, and the length cap
|
||||
// bounds what a hostile writer can do to the layout.
|
||||
// of the uid, and the account list must render identically whether the store is up
|
||||
// (identity itself is file-backed, the UDL is not). The name is the user's
|
||||
// signed MMUR2 claim (meta-core verified it on write); the uid stays on screen
|
||||
// as the real identity, and the length cap bounds the layout.
|
||||
async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
const url =
|
||||
`/api/udl/record?uid=${encodeURIComponent(uid)}` +
|
||||
@@ -192,12 +191,11 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
try {
|
||||
const res = await fetch(url);
|
||||
if (!res.ok) return null; // 503 when storage is down, 500, …
|
||||
const data = (await res.json()) as { exists?: boolean; value?: unknown };
|
||||
// Key off `value`, not `exists`: clearing a name writes a tombstone, which
|
||||
// leaves the cell existing with no plaintext. Private-tier cells look the
|
||||
// same, which is exactly right — there is nothing to show for either.
|
||||
if (typeof data.value !== 'string') return null;
|
||||
const name = data.value.trim();
|
||||
const data = (await res.json()) as { exists?: boolean; claim?: { value?: unknown } | null };
|
||||
// A retraction (null) or ciphertext ({b64}) has nothing to show.
|
||||
const value = data.claim?.value;
|
||||
if (typeof value !== 'string') return null;
|
||||
const name = value.trim();
|
||||
return name ? name.slice(0, 64) : null;
|
||||
} catch {
|
||||
return null;
|
||||
@@ -209,7 +207,7 @@ async function fetchDisplayName(uid: string): Promise<string | null> {
|
||||
// so per-row calls would walk the keyspace once per account.
|
||||
//
|
||||
// Never throws, for the same reason fetchDisplayName does not: the counts are
|
||||
// context on top of the uid, and the list must render identically when Redis is
|
||||
// context on top of the uid, and the list must render identically when the store is
|
||||
// down — accounts are file-backed, their data is not.
|
||||
//
|
||||
// Returns null on failure rather than an empty map. The difference is
|
||||
@@ -603,7 +601,7 @@ function AccountCard({
|
||||
records === 1 ? 'record' : 'records'
|
||||
} — likes, ratings, watch progress across ${titles.toLocaleString()} ${
|
||||
titles === 1 ? 'title' : 'titles'
|
||||
} — are permanently deleted from Redis, along with every index that names this uid.`}{' '}
|
||||
} — are permanently deleted from the metadata store, along with every index that names this uid.`}{' '}
|
||||
This cannot be undone and there is no backup.
|
||||
</li>
|
||||
<li>
|
||||
@@ -655,12 +653,12 @@ export default function Identity() {
|
||||
return;
|
||||
}
|
||||
if (gen !== reqGen.current) return;
|
||||
// Render the list before resolving names, so a down Redis delays nothing.
|
||||
// Render the list before resolving names, so a down store delays nothing.
|
||||
setAccounts(list);
|
||||
setLoading(false);
|
||||
|
||||
// Names and counts both come after the list is on screen, and neither can
|
||||
// fail the page: a down Redis costs decoration, not the account list.
|
||||
// fail the page: a down store costs decoration, not the account list.
|
||||
const [settled, counts] = await Promise.all([
|
||||
Promise.allSettled(list.map((a) => fetchDisplayName(a.uid))),
|
||||
fetchStats(),
|
||||
|
||||
@@ -0,0 +1,406 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { getJSON } from '../lib/api';
|
||||
import {
|
||||
blocks,
|
||||
buildLabel,
|
||||
deviceLine,
|
||||
dump,
|
||||
headline,
|
||||
type Report,
|
||||
type ReportSummary,
|
||||
shortCid,
|
||||
subject,
|
||||
} from '../lib/reportText';
|
||||
|
||||
// The bug-report / feedback inbox. Client apps (meta-watch, meta-read, …)
|
||||
// forward what their 🐛 buttons file to POST /api/reports; this page lists,
|
||||
// reads and triages them. See internal/reports for storage and bounds.
|
||||
|
||||
type Status = 'open' | 'triaged' | 'closed';
|
||||
const STATUSES: Status[] = ['open', 'triaged', 'closed'];
|
||||
|
||||
interface ListPage {
|
||||
reports: ReportSummary[];
|
||||
apps: Record<string, number>;
|
||||
counts: Record<string, number>;
|
||||
more: boolean;
|
||||
}
|
||||
|
||||
const card: React.CSSProperties = {
|
||||
background: '#16213e',
|
||||
borderRadius: '8px',
|
||||
padding: '1rem',
|
||||
display: 'flex',
|
||||
flexDirection: 'column',
|
||||
overflow: 'hidden',
|
||||
minHeight: 0,
|
||||
};
|
||||
|
||||
const chip = (active: boolean): React.CSSProperties => ({
|
||||
padding: '0.25rem 0.7rem',
|
||||
borderRadius: '999px',
|
||||
border: '1px solid #0f3460',
|
||||
background: active ? '#0f3460' : 'transparent',
|
||||
color: active ? '#fff' : '#aaa',
|
||||
cursor: 'pointer',
|
||||
fontSize: '0.85rem',
|
||||
});
|
||||
|
||||
const button: React.CSSProperties = {
|
||||
padding: '0.35rem 0.8rem',
|
||||
borderRadius: '4px',
|
||||
border: '1px solid #0f3460',
|
||||
background: '#0f3460',
|
||||
color: '#fff',
|
||||
cursor: 'pointer',
|
||||
fontSize: '0.85rem',
|
||||
};
|
||||
|
||||
const mono: React.CSSProperties = { fontFamily: 'ui-monospace, Menlo, Consolas, monospace', fontSize: '0.8rem' };
|
||||
|
||||
const statusColor: Record<Status, string> = { open: '#b45309', triaged: '#1d4ed8', closed: '#374151' };
|
||||
|
||||
function StatusBadge({ s }: { s: Status }) {
|
||||
return (
|
||||
<span style={{ background: statusColor[s], color: '#fff', borderRadius: '3px', padding: '0.05rem 0.4rem', fontSize: '0.72rem' }}>
|
||||
{s}
|
||||
</span>
|
||||
);
|
||||
}
|
||||
|
||||
const fmtTime = (ms: number) => new Date(ms).toLocaleString();
|
||||
|
||||
export default function Reports() {
|
||||
const [app, setApp] = useState('');
|
||||
const [status, setStatus] = useState<Status | ''>('open');
|
||||
const [q, setQ] = useState('');
|
||||
const [page, setPage] = useState<ListPage | null>(null);
|
||||
const [older, setOlder] = useState<ReportSummary[]>([]);
|
||||
const [more, setMore] = useState(false);
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [selected, setSelected] = useState<number | null>(null);
|
||||
|
||||
const query = useCallback(
|
||||
(before?: number) => {
|
||||
const p = new URLSearchParams();
|
||||
if (app) p.set('app', app);
|
||||
if (status) p.set('status', status);
|
||||
if (q.trim()) p.set('q', q.trim());
|
||||
if (before) p.set('before', String(before));
|
||||
return `/api/reports?${p}`;
|
||||
},
|
||||
[app, status, q]
|
||||
);
|
||||
|
||||
// The newest page is polled; pages loaded with "older" are kept beneath it
|
||||
// until a filter changes, and then they own the "more" flag.
|
||||
const olderCount = useRef(0);
|
||||