5 Commits
Author SHA1 Message Date
Manielle eecfee2812 CI accepts four-part tags (v1.2.3.1) for fixes
Build and Push Docker Image / build (push) Successful in 3m51s
2026-10-25 14:21:59 +01:00
Worph d85dac7cf4 gitea workflow: push to IPCR's direct host (Cloudflare 100 MB cap)
Build and Push Docker Image / build (push) Successful in 1m24s
2026-10-13 10:02:20 +00:00
Worph 1dc6aacdc8 gitea workflow: publish to IPCR (metadec.eth) instead of ghcr
Build and Push Docker Image / build (push) Failing after 1m18s
2026-10-12 18:30:33 +00:00
Worph 2d5cfa7986 beacon v2 discovery: meshdisco.py on 239.255.99.1:9099, core located by metamesh.core cap, /api/neighbors ?cap=, nav lists services only
Build and Push Docker Image / build (push) Failing after 13s
2026-10-04 11:51:11 +00:00
Worph f6b1133f4a README: sync with current code 2026-10-03 11:52:01 +00:00
7 changed files with 383 additions and 171 deletions
+98
View File
@@ -0,0 +1,98 @@
# Gitea twin of .github/workflows/docker-publish.yml. Gitea runs only .gitea/workflows/ once this
# folder exists, and GitHub never reads it.
#
# Publishes to IPCR on watch.nsl.sh instead of ghcr.io:
# push ipcr-push-85-17-246-67.nip.io/metadec.eth/<repo>:<tag> docker login as metadec.eth with the
# IPCR_PUSH_TOKEN secret (the metadec.eth namespace token from the IPCR dashboard)
# pull ipcr-watch.nsl.sh/ipns/metadec.eth/metadec.eth/<repo>:<tag> anonymous
# The push goes to ipcr-push-85-17-246-67.nip.io, the box's own address, not ipcr-push-watch.nsl.sh: Cloudflare fronts
# *.watch.nsl.sh and refuses any request over 100 MB (413), which a large image layer is.
# That host's certificate is signed by the box's private Mesh Router CA, given to buildkit from
# the IPCR_CA_PEM repo variable. docker login would make the job's Docker daemon check the
# certificate without that CA, so the login is written straight into ~/.docker/config.json.
# The runner's job image (node:22-bookworm) has no docker CLI, so the static one is
# installed first; buildx installs itself. No `type=gha` cache: the runner has none.
name: Build and Push Docker Image
on:
push:
branches:
- main
tags:
- 'v*'
workflow_dispatch:
inputs:
tag:
description: 'Tag for the image (optional, defaults to sha)'
required: false
type: string
meta_core_version:
description: 'meta-core version to use (default: latest)'
required: false
type: string
default: 'latest'
env:
REGISTRY: ipcr-push-85-17-246-67.nip.io
IPCR_PULL: ipcr-watch.nsl.sh/ipns/metadec.eth
IMAGE_NAME: metadec.eth/${{ github.event.repository.name }}
jobs:
build:
runs-on: ubuntu-latest
permissions:
contents: read
packages: write
steps:
- name: Checkout repository
uses: actions/checkout@v4
- name: Install Docker CLI
run: curl -fsSL https://download.docker.com/linux/static/stable/x86_64/docker-28.5.1.tgz | tar -xz -C /usr/local/bin --strip-components=1 docker/docker
- name: Trust the Mesh Router CA
run: printf '%s\n' "$IPCR_CA_PEM" > /tmp/ipcr-ca.pem
env:
IPCR_CA_PEM: ${{ vars.IPCR_CA_PEM }}
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
with:
version: latest
buildkitd-config-inline: |
[registry."${{ env.REGISTRY }}"]
ca = ["/tmp/ipcr-ca.pem"]
- name: Log in to IPCR
env:
IPCR_PUSH_TOKEN: ${{ secrets.IPCR_PUSH_TOKEN }}
run: |
mkdir -p ~/.docker
printf '{"auths":{"%s":{"auth":"%s"}}}\n' "$REGISTRY" "$(printf 'metadec.eth:%s' "$IPCR_PUSH_TOKEN" | base64 -w0)" > ~/.docker/config.json
- name: Extract metadata (tags, labels)
id: meta
uses: docker/metadata-action@v5
with:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: |
type=semver,pattern={{version}}
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
type=semver,pattern={{major}}.{{minor}}
type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}}
type=raw,value=${{ inputs.tag }},enable=${{ inputs.tag != '' }}
- name: Build and push Docker image
uses: docker/build-push-action@v5
with:
context: .
push: true
tags: ${{ steps.meta.outputs.tags }}
labels: ${{ steps.meta.outputs.labels }}
build-args: |
META_CORE_IMAGE=${{ env.IPCR_PULL }}/metadec.eth/meta-core:${{ inputs.meta_core_version || 'latest' }}
+3
View File
@@ -50,6 +50,9 @@ jobs:
images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }} images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
tags: | tags: |
type=semver,pattern={{version}} type=semver,pattern={{version}}
# A fix to release 1.2.3 is tagged v1.2.3.1: four numbers are not semver, so
# the pattern above makes no tag for it; this one does (image tag 1.2.3.1).
type=match,pattern=v(\d+\.\d+\.\d+\.\d+),group=1
type=semver,pattern={{major}}.{{minor}} type=semver,pattern={{major}}.{{minor}}
type=sha,prefix= type=sha,prefix=
type=raw,value=latest,enable={{is_default_branch}} type=raw,value=latest,enable={{is_default_branch}}
+42 -38
View File
@@ -1,6 +1,6 @@
# Meta-Stremio # Meta-Stremio
Stremio addon for the MetaMesh stack. Reads file metadata from the shared KV store, reads files via WebDAV (or direct mount), and serves HLS with on-the-fly FFmpeg transcoding plus the standard Stremio addon protocol. Stremio addon for the MetaMesh stack. Reads file metadata from meta-core's HTTP API (no direct Redis access), reads files via meta-core's WebDAV (or a direct mount), and serves HLS with on-the-fly FFmpeg transcoding plus the standard Stremio addon protocol.
External port in the dev stack: **8182** (auth-gated, via Caddy + nginx-hash-lock). Internal container port is `7000`. Debug-direct port (bypasses the perimeter): **18182**. Container name: `metastremio-app` (backend) / `metastremio` (hash-lock proxy). External port in the dev stack: **8182** (auth-gated, via Caddy + nginx-hash-lock). Internal container port is `7000`. Debug-direct port (bypasses the perimeter): **18182**. Container name: `metastremio-app` (backend) / `metastremio` (hash-lock proxy).
@@ -18,8 +18,8 @@ External port in the dev stack: **8182** (auth-gated, via Caddy + nginx-hash-loc
│ │ │ │ │ │ │ │ │ │
│ ▼ ▼ ▼ │ │ ▼ ▼ ▼ │
│ ┌─────────────────────────────────────────────────────┐ │ │ ┌─────────────────────────────────────────────────────┐ │
│ │ Shared KV Store (Redis) │ │ │ │ meta-core (owns Redis; /urls + /meta APIs) │ │
│ │ meta-core is leader, exposes /urls + /meta APIs │ │ │ │ located over UDP multicast (beacon v2) │ │
│ └─────────────────────────────────────────────────────┘ │ │ └─────────────────────────────────────────────────────┘ │
│ │ │ │ │ │
│ ▼ │ │ ▼ │
@@ -34,7 +34,8 @@ External port in the dev stack: **8182** (auth-gated, via Caddy + nginx-hash-loc
- **HLS transcoding** — adaptive preset+CRF targeting a 60–80% transcode ratio, prefetch of N segments ahead, persistent segment cache. - **HLS transcoding** — adaptive preset+CRF targeting a 60–80% transcode ratio, prefetch of N segments ahead, persistent segment cache.
- **Stremio protocol** — manifest, catalog, meta, stream; one HLS stream per audio track; subtitles extracted to VTT. - **Stremio protocol** — manifest, catalog, meta, stream; one HLS stream per audio track; subtitles extracted to VTT.
- **Direct file serving** — range-request capable, for clients that can play the source directly. - **Direct file serving** — range-request capable, for clients that can play the source directly.
- **Storage abstractions** — `LeaderStorage` (auto-discovers Redis via meta-core), `RedisStorage` (direct URL), `direct` mode (reads from meta-core's HTTP API + SSE meta-stream). - **meta-core over HTTP** — `LeaderStorage` locates meta-core via **beacon v2** (UDP multicast `239.255.99.1:9099`; see [`beacon-v2.md`](../../docs/project-architecture/beacon-v2.md)) or a pinned `META_CORE_URL`, reads every record through meta-core's `/meta/*` API, and keeps its cache fresh from the SSE meta-stream. The WebDAV URL is taken from meta-core's announce/`/urls` — nothing is configured by hand and no `/meta-core` volume is mounted. (`RedisStorage` remains in `src/storage/` as a legacy, unused backend; meta-core no longer publishes a Redis URL.)
- **Neighbour nav** — this service announces itself on the discovery group and serves its own neighbour map at `/api/neighbors` for the dashboard's shared `<meta-service-menu>`.
- **Path-token gate for addon URLs** — when `HASH_API_SEED` is set, addon paths must be prefixed with `/s/<token>`. Dashboard/browser paths are protected separately by the upstream hash-lock + OIDC sidecar (Stremio clients can't carry cookies, so they get the path-token instead). - **Path-token gate for addon URLs** — when `HASH_API_SEED` is set, addon paths must be prefixed with `/s/<token>`. Dashboard/browser paths are protected separately by the upstream hash-lock + OIDC sidecar (Stremio clients can't carry cookies, so they get the path-token instead).
- **Language-configurable manifest** — `displayLanguage` in the per-install config (base64-in-path) selects which `titles` translation meta-sort's TMDB plugin populated. - **Language-configurable manifest** — `displayLanguage` in the per-install config (base64-in-path) selects which `titles` translation meta-sort's TMDB plugin populated.
@@ -54,23 +55,17 @@ This brings up `metastremio-app` on port 18182 (debug-direct) and the auth-gated
```bash ```bash
docker build -t meta-stremio packages/meta-stremio docker build -t meta-stremio packages/meta-stremio
# Leader mode (auto-discover Redis via meta-core) # Against an existing meta-core (found over UDP on the same Docker network,
# or pinned with META_CORE_URL)
docker run -d \ docker run -d \
-p 8182:7000 \ -p 8182:7000 \
-v /path/to/media:/files:ro \ -v /path/to/media:/files:ro \
-v /path/to/meta-core:/meta-core \ -e STORAGE_MODE=direct \
meta-stremio -e META_CORE_URL=http://metacore-app:9000 \
# Direct Redis mode
docker run -d \
-p 8182:7000 \
-v /path/to/media:/files:ro \
-e STORAGE_MODE=redis \
-e REDIS_URL=redis://your-redis:6379 \
meta-stremio meta-stremio
``` ```
The container always listens on port `7000` internally — map it to whatever you want externally. The container always listens on port `7000` internally — map it to whatever you want externally. The image bundles a `meta-core` binary (copied from `ghcr.io/worph/meta-core`, overridable with `--build-arg META_CORE_IMAGE=…`): with the image default `STORAGE_MODE=leader`, `docker/start.sh` starts it as an in-container sidecar first; any other value (the dev stack and the CasaOS app use `direct`) skips it and uses an external meta-core. Published images: `ghcr.io/worph/meta-stremio` (semver tags from `v*` git tags; in this repo `:latest` follows `main` — see `.github/workflows/docker-publish.yml`); the CasaOS app is `MetaStremio` in [`MetaAppStore`](../MetaAppStore/Apps/MetaStremio/docker-compose.yml).
### Standalone (development) ### Standalone (development)
@@ -78,7 +73,7 @@ The container always listens on port `7000` internally — map it to whatever yo
# Requires Python 3.9+ and FFmpeg. # Requires Python 3.9+ and FFmpeg.
pip install -r requirements.txt # redis, watchdog, Pillow, requests pip install -r requirements.txt # redis, watchdog, Pillow, requests
cd src cd src
python server.py python server.py # finds meta-core over UDP, or set META_CORE_URL=<meta-core API URL>
``` ```
## Configuration ## Configuration
@@ -91,18 +86,20 @@ python server.py
| `BASE_URL` | — | External URL emitted in manifests / poster URLs (e.g. `https://metastremio-dev.localhost:8182`). | | `BASE_URL` | — | External URL emitted in manifests / poster URLs (e.g. `https://metastremio-dev.localhost:8182`). |
| `MEDIA_DIR` | `/files/watch` | Media directory (consumed by `transcoder.py`). | | `MEDIA_DIR` | `/files/watch` | Media directory (consumed by `transcoder.py`). |
| `CACHE_DIR` | `/data/cache` | Transcoded segment cache. | | `CACHE_DIR` | `/data/cache` | Transcoded segment cache. |
| `STORAGE_MODE` | `leader` | `leader`, `redis`, or `direct`. | | `PUBLIC_URL` | — | Browser-facing URL announced to neighbours (nav menu); wins over `BASE_URL` for the announce. |
| `META_CORE_PATH` | `/meta-core` | Shared meta-core volume; used by `LeaderClient` for `kv-leader.info`. | | `STORAGE_MODE` | `leader` (image) | Read only by `docker/start.sh`: `leader` starts the bundled meta-core sidecar in-container, anything else (e.g. `direct`) skips it. The Python server always uses `LeaderStorage`. |
| `META_CORE_URL` | — | meta-core HTTP API (e.g. `http://metacore-app:9000`); required for `STORAGE_MODE=direct` since there is no leader file to read. | | `META_CORE_URL` | — | Pins meta-core's HTTP API (e.g. `http://metacore-app:9000`); when set, UDP discovery never overrides it. Unset → meta-core is located over UDP. |
| `FILES_PATH` | `/files` | Files volume. | | `ENABLE_UDP_DISCOVERY` | `true` | beacon v2 announce/listen. |
| `REDIS_URL` | `redis://localhost:6379` | Redis URL for `STORAGE_MODE=redis`. | | `LEADER_WAIT_TIMEOUT` | `0` | Seconds to wait for meta-core at startup (`0` = forever); on timeout, keeps retrying in the background. |
| `REDIS_PREFIX` | `meta-sort:` | Redis key prefix. | | `LEADER_RETRY_INTERVAL` | `5` | Seconds between meta-core lookup attempts. |
| `FILES_PATH` | `/files` | Files volume (local fallback when WebDAV is not configured). |
| `SCHEME` | `auto` | URL scheme for generated URLs: `http`, `https`, or `auto`. | | `SCHEME` | `auto` | URL scheme for generated URLs: `http`, `https`, or `auto`. |
| `SEGMENT_DURATION` | `4` | HLS segment length (s). | | `SEGMENT_DURATION` | `4` | HLS segment length (s). |
| `PREFETCH_SEGMENTS` | `4` | Segments to prefetch ahead. | | `PREFETCH_SEGMENTS` | `4` | Segments to prefetch ahead. |
| `META_CORE_WEBDAV_URL` | — | When set, files are read over HTTP from meta-core's WebDAV instead of the filesystem (lets meta-stremio reach SMB / rclone mounts that only exist inside meta-core). |
| `HASH_API_SEED` | — | When set, derives a 16-char path-token; all addon routes must be prefixed with `/s/<token>`. Dashboard routes bypass. | | `HASH_API_SEED` | — | When set, derives a 16-char path-token; all addon routes must be prefixed with `/s/<token>`. Dashboard routes bypass. |
`META_CORE_PATH`, `REDIS_URL` and `REDIS_PREFIX` are still set by the Dockerfile but are vestigial: meta-core is no longer found through the `/meta-core` volume, and reads go over meta-core's HTTP API rather than Redis.
## API endpoints ## API endpoints
All routes below are matched in `src/server.py`. When `HASH_API_SEED` is set, **every addon path is prefixed with `/s/<token>`**; the dashboard paths in the first table are *not* prefixed (they're protected by hash-lock/OIDC upstream). All routes below are matched in `src/server.py`. When `HASH_API_SEED` is set, **every addon path is prefixed with `/s/<token>`**; the dashboard paths in the first table are *not* prefixed (they're protected by hash-lock/OIDC upstream).
@@ -116,8 +113,11 @@ All routes below are matched in `src/server.py`. When `HASH_API_SEED` is set, **
| `/health` | GET | Liveness + storage status | | `/health` | GET | Liveness + storage status |
| `/api/stats` | GET | Library statistics | | `/api/stats` | GET | Library statistics |
| `/api/library` | GET | Full library list (videos + count) | | `/api/library` | GET | Full library list (videos + count) |
| `/api/services` | GET | Discovered MetaMesh services (proxied from `meta-core /services`) | | `/api/neighbors` | GET | This peer's UDP neighbour map (beacon v2) for the nav menu |
| `/api/services` | GET | Alias of `/api/neighbors` (kept for older dashboards) |
| `/api/languages` | GET | Languages selectable from the `/configure` page | | `/api/languages` | GET | Languages selectable from the `/configure` page |
| `/meta-service-menu.js` | GET | Shared `<meta-service-menu>` nav element loaded by the dashboard |
| `/transcode/metrics` | GET | Transcoder metrics (also listed below; exempt from the path-token) |
### Stremio addon protocol ### Stremio addon protocol
@@ -178,20 +178,22 @@ meta-stremio/
│ └── storage/ │ └── storage/
│ ├── __init__.py │ ├── __init__.py
│ ├── provider.py # StorageProvider + VideoMetadata abstract base │ ├── provider.py # StorageProvider + VideoMetadata abstract base
│ ├── redis_storage.py # Direct Redis backend │ ├── leader_storage.py # Storage backend in use: meta-core HTTP API + SSE cache invalidation
│ ├── leader_storage.py # Leader-aware Redis backend │ ├── leader_client.py # Locates meta-core (UDP or META_CORE_URL pin), exposes its /urls set
│ ├── leader_client.py # Reads /meta-core/locks/kv-leader.info + calls meta-core /urls │ ├── meshdisco.py # beacon v2 (Python port of the Go reference in meta-core)
│ ├── meta_consumer.py # SSE consumer from meta-core /meta stream │ ├── meta_consumer.py # SSE consumer from meta-core /meta stream
│ ├── meta_core_api_client.py # HTTP client for meta-core REST API │ ├── meta_core_api_client.py # HTTP client for meta-core REST API
│ └── service_registration.py # Registers meta-stremio with meta-core /services │ └── redis_storage.py # Legacy direct-Redis backend (unused)
└── www/ └── www/
├── index.html # Dashboard ├── index.html # Dashboard
└── configure.html # Language configuration page └── meta-service-menu.js # Shared nav element
``` ```
The `/configure` page is rendered inline by `server.py` (there is no `www/configure.html`).
## KV store schema ## KV store schema
meta-stremio reads from the per-file Redis hash `/file/{cid}` populated by meta-sort and its plugins. Field semantics (`videoType`, `originalTitle`, `titles`, `season`, `episode`, `movieYear`, `cid_*`, etc.) are documented in the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md) — that is the single source of truth, and matches the `@metazla/meta-interface` types. meta-stremio reads the per-file record `/file/{cid}` (via meta-core's API) populated by meta-sort and its plugins. Field semantics (`videoType`, `originalTitle`, `titles`, `season`, `episode`, `movieYear`, `cid_*`, etc.) are documented in the repo-root [`METADATA_KEYS.md`](../../METADATA_KEYS.md) — that is the single source of truth, and matches the `@metazla/meta-interface` types.
## Stream types ## Stream types
@@ -199,14 +201,15 @@ For each video, the addon advertises multiple streams:
1. **Direct file** — original bytes, served via `/direct/...` with `Range` support. Best quality, may not play on every device. 1. **Direct file** — original bytes, served via `/direct/...` with `Range` support. Best quality, may not play on every device.
2. **HLS original** — transcoded at source resolution, H.264/AAC, one stream per audio track. 2. **HLS original** — transcoded at source resolution, H.264/AAC, one stream per audio track.
3. **HLS ABR** — adaptive ladder. Quality presets are baked into `transcoder.py`. 3. **HLS ABR** — adaptive ladder (rungs above the source height are skipped). Presets are baked into `transcoder.py`: video is x264 **CRF-based** (the adaptive controller shifts preset and CRF offset at runtime), audio is AAC 128 kbps stereo. `BANDWIDTH` is only the hint written to the master playlist.
| Resolution | Video bitrate | Audio bitrate | | Rung | Base CRF | Master-playlist `BANDWIDTH` hint |
|---|---|---| |---|---|---|
| 1080p | 5000 kbps | 192 kbps | | original | 23 | — |
| 720p | 3000 kbps | 128 kbps | | 1080p | 23 | 4000 kbps |
| 480p | 1500 kbps | 128 kbps | | 720p | 24 | 2500 kbps |
| 360p | 800 kbps | 96 kbps | | 480p | 25 | 1200 kbps |
| 360p | 26 | 800 kbps |
## Adding to Stremio ## Adding to Stremio
@@ -234,8 +237,9 @@ Container lifecycle: do **not** `docker restart metastremio-app` to apply config
## Related projects ## Related projects
- **[meta-sort](../meta-sort)** — file indexer; writes the metadata meta-stremio reads. - **[meta-sort](../meta-sort)** — file indexer; writes the metadata meta-stremio reads.
- **[meta-fuse](../meta-fuse)** — virtual filesystem; reads the same KV store. - **[meta-core](../meta-core)** — owns Redis; the HTTP API + WebDAV meta-stremio reads from.
- **[meta-share](../meta-share)** — decentralised metadata sharing. - **[meta-fuse](../meta-fuse)** — virtual filesystem; reads the same records.
- **[meta-watch](../meta-watch)** — browser video client for the meta-share network (server-side zero media work, unlike this addon's transcoder).
## License ## License
+12 -5
View File
@@ -66,7 +66,7 @@ DASHBOARD_PATHS = frozenset({
# Initialize storage # Initialize storage
storage = stremio.init_storage() storage = stremio.init_storage()
# Service discovery is UDP now (meta-discovery v1), so there is no # Service discovery is UDP now (beacon v2), so there is no
# META_CORE_PATH gate: the leader client's node announces this service on the # META_CORE_PATH gate: the leader client's node announces this service on the
# same multicast group it listens on. Being discoverable and discovering are # same multicast group it listens on. Being discoverable and discovering are
# one thing. See docs/project-architecture/service-discovery.md. # one thing. See docs/project-architecture/service-discovery.md.
@@ -654,18 +654,25 @@ class Handler(BaseHTTPRequestHandler):
self.wfile.write(content.encode('utf-8')) self.wfile.write(content.encode('utf-8'))
def handle_services_api(self): def handle_services_api(self):
"""Neighbours for the dashboard nav, from this peer's own UDP map. """Neighbours for the dashboard nav, from this peer's own beacon v2 view.
No hop through meta-core any more: every service keeps its own No hop through meta-core any more: every service keeps its own
neighbour table, so the nav still renders when meta-core is down. neighbour table, so the nav still renders when meta-core is down.
`services` is kept as an alias of `neighbors` so the existing `?all=1` returns every instance, `?cap=<pattern>` filters by
dashboards keep working during the UI migration. capability (docs/project-architecture/beacon-v2.md). `services` is kept
as an alias of `neighbors` so the existing dashboards keep working.
""" """
from urllib.parse import parse_qs, urlparse
from storage.leader_client import get_leader_client from storage.leader_client import get_leader_client
try: try:
query = parse_qs(urlparse(self.path).query)
client = get_leader_client() client = get_leader_client()
neighbors = client.neighbors() if client else [] neighbors = client.neighbors(
all=bool(query.get('all')),
cap=(query.get('cap') or [None])[0],
) if client else []
return self.send_json({ return self.send_json({
'current': 'meta-stremio', 'current': 'meta-stremio',
'enabled': True, 'enabled': True,
+8 -7
View File
@@ -1,9 +1,9 @@
"""LeaderClient — locates meta-core and exposes its URLs. """LeaderClient — locates meta-core and exposes its URLs.
Since meta-discovery v1 this is a thin adapter over ``MetaCoreLocator`` Since UDP discovery (meta-discovery v1, now beacon v2) this is a thin adapter
(``meshdisco.py``): meta-core is found by UDP announce instead of by reading over ``MetaCoreLocator`` (``meshdisco.py``): meta-core is found by UDP advertise
``/meta-core/locks/kv-leader.info``, so meta-stremio no longer needs the instead of by reading ``/meta-core/locks/kv-leader.info``, so meta-stremio no
``/meta-core`` volume mounted at all. longer needs the ``/meta-core`` volume mounted at all.
The public surface is deliberately unchanged so ``LeaderStorage`` did not have The public surface is deliberately unchanged so ``LeaderStorage`` did not have
to move: ``get_leader_info``, ``get_urls``, ``get_api_url``, to move: ``get_leader_info``, ``get_urls``, ``get_api_url``,
@@ -263,10 +263,11 @@ class LeaderClient:
# -- nav menu ---------------------------------------------------------- # -- nav menu ----------------------------------------------------------
def neighbors(self) -> List[dict]: def neighbors(self, all: bool = False, cap: Optional[str] = None) -> List[dict]:
"""Neighbours for the nav menu (one row per service name).""" """Neighbours for /api/neighbors: one row per name (every instance with
``all``), optionally only nodes with a capability matching ``cap``."""
self._ensure_started() self._ensure_started()
return self._locator.neighbors() return self._locator.neighbors(all=all, cap=cap)
def self_announce(self) -> dict: def self_announce(self) -> dict:
"""This service's own announce, so the menu can show itself.""" """This service's own announce, so the menu can show itself."""
+213 -119
View File
@@ -1,13 +1,13 @@
"""meta-discovery v1 — UDP multicast service discovery. """Beacon v2 — local resource advertise / discover over UDP multicast
(239.255.99.1:9099).
Replaces the two file-based mechanisms that required a shared /meta-core Every node advertises resources tagged with capability strings
volume: ``locks/kv-leader.info`` (locating meta-core at boot) and (``metamesh.core``, ``metamesh.service/meta-stremio``,
``services/<name>-<host>.json`` (the dashboard nav registry). Both answered the ``metamesh.transport/nzb@1``) and keeps a live view of everyone else's. This
same question — where is meta-core, and who else is on this network — so one service uses it to locate meta-core (the ``metamesh.core`` resource carries its
announce packet answers both, and discovery scope becomes the docker network /urls block) and to feed the nav menu, with no shared volume.
rather than the mounted volume.
The normative spec is docs/project-architecture/service-discovery.md. The normative spec is docs/project-architecture/beacon-v2.md.
Two details this module must get right, both of which fail silently: Two details this module must get right, both of which fail silently:
@@ -30,18 +30,98 @@ import threading
import time import time
from typing import Callable, Dict, List, Optional from typing import Callable, Dict, List, Optional
PROTOCOL_VERSION = 1 PROTO = "beacon"
DEFAULT_GROUP = "239.255.77.1" VERSION = 2
DEFAULT_PORT = 9399 DEFAULT_GROUP = "239.255.99.1"
DEFAULT_PORT = 9099
DEFAULT_INTERVAL = 10.0 DEFAULT_INTERVAL = 10.0
# A neighbour unseen for interval * this is dropped. Replaces the reaper. # A node unheard for interval * this is dropped (evaluated at read time).
LIVENESS_FACTOR = 3 LIVENESS_FACTOR = 3
READ_BUFFER = 8192 READ_BUFFER = 8192
# Senders stay under this so nothing fragments.
MAX_DATAGRAM = 1400
TYPE_DISCOVERY = "discovery" TYPE_PROBE = "probe"
TYPE_ANNOUNCE = "announce" TYPE_ADVERTISE = "advertise"
ROLE_CORE = "core" TYPE_BYE = "bye"
ROLE_SERVICE = "service"
CAP_CORE = "metamesh.core"
CAP_SERVICE_PREFIX = "metamesh.service/"
CAP_ANY_SERVICE = "metamesh.service/*"
def cap_matches(pattern: str, cap: str) -> bool:
"""Does capability ``cap`` satisfy ``pattern``?
``*`` matches everything; an ``@N`` on the pattern must equal the cap's
contract (none ignores it); ``x/*`` matches any variant of ``x``; otherwise
the bases must be equal.
"""
if pattern == "*":
return True
pbase, _, pc = pattern.rpartition("@") if "@" in pattern else (pattern, "", None)
cbase, _, cc = cap.rpartition("@") if "@" in cap else (cap, "", None)
if pc is not None and cc != pc:
return False
if pbase.endswith("/*"):
prefix = pbase[:-2]
if not cbase.startswith(prefix):
return False
rest = cbase[len(prefix):]
return rest.startswith("/") and len(rest) > 1
return pbase == cbase
def resource_matches(resource: dict, pattern: str) -> bool:
return any(cap_matches(pattern, c) for c in resource.get("caps") or [])
def resource_endpoint(resource: dict, name: str) -> Optional[str]:
"""The named endpoint as an absolute URL ('/…' joins onto endpoints.http)."""
endpoints = resource.get("endpoints") or {}
v = endpoints.get(name)
if not v:
return None
if not v.startswith("/"):
return v
base = endpoints.get("http")
return base.rstrip("/") + v if base else None
def parse_message(data: bytes) -> Optional[dict]:
"""Decode a datagram; None for anything that is not well-formed beacon v2
(beacon v1, meta-discovery v1, other versions, junk)."""
try:
m = json.loads(data.decode("utf-8"))
except (ValueError, UnicodeDecodeError):
return None
if not isinstance(m, dict) or m.get("proto") != PROTO or m.get("v") != VERSION:
return None
t = m.get("type")
if t == TYPE_PROBE:
return m
if t in (TYPE_ADVERTISE, TYPE_BYE):
node = m.get("node")
if not isinstance(node, dict) or not node.get("name") or not node.get("instance"):
return None
return m
return None
def _neighbor_row(node: dict, resources: List[dict], addr: str, last_seen: float) -> dict:
caps: List[str] = []
base_url = None
for r in resources:
for c in r.get("caps") or []:
if c not in caps:
caps.append(c)
if base_url is None and resource_matches(r, CAP_ANY_SERVICE):
base_url = resource_endpoint(r, "ui")
row = dict(node)
row.update({"resources": resources, "caps": caps, "addr": addr, "lastSeen": last_seen})
if base_url:
row["baseUrl"] = base_url
return row
def _env_flag(name: str, default: bool = True) -> bool: def _env_flag(name: str, default: bool = True) -> bool:
@@ -97,33 +177,33 @@ def local_ipv4() -> str:
class MeshNode: class MeshNode:
"""Announces this service, answers probes, tracks neighbours.""" """One beacon v2 participant: advertises its resources, answers probes,
and keeps a TTL map of every other node."""
def __init__( def __init__(
self, self,
name: str, name: str,
role: str = ROLE_SERVICE,
instance: Optional[str] = None, instance: Optional[str] = None,
version: str = "", version: str = "",
group: str = DEFAULT_GROUP, group: Optional[str] = None,
port: int = DEFAULT_PORT, port: Optional[int] = None,
interval: float = DEFAULT_INTERVAL, interval: Optional[float] = None,
enabled: bool = True, enabled: bool = True,
payload: Optional[Callable[[], dict]] = None, payload: Optional[Callable[[], dict]] = None,
) -> None: ) -> None:
self.name = name self.name = name
self.role = role
self.instance = instance or socket.gethostname() self.instance = instance or socket.gethostname()
self.version = version self.version = version
self.group = group self.group = group or os.environ.get("BEACON_GROUP") or DEFAULT_GROUP
self.port = port self.port = port or int(os.environ.get("BEACON_PORT") or DEFAULT_PORT)
self.interval = interval self.interval = interval or (float(os.environ.get("BEACON_INTERVAL_MS") or 0) / 1000.0) or DEFAULT_INTERVAL
self.enabled = enabled self.enabled = enabled
# payload() -> {"status": str?, "resources": [resource, ...]}
self._payload = payload self._payload = payload
self._sock: Optional[socket.socket] = None self._sock: Optional[socket.socket] = None
self._ifaces: List[tuple] = [] self._ifaces: List[tuple] = []
self._neighbors: Dict[str, dict] = {} self._nodes: Dict[str, dict] = {}
self._lock = threading.Lock() self._lock = threading.Lock()
self._send_lock = threading.Lock() self._send_lock = threading.Lock()
self._running = False self._running = False
@@ -135,9 +215,8 @@ class MeshNode:
def start(self) -> None: def start(self) -> None:
if self._running or not self.enabled: if self._running or not self.enabled:
if not self.enabled: if not self.enabled:
print("[meshdisco] Disabled by configuration") print("[beacon] Disabled by configuration")
return return
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
try: try:
@@ -160,16 +239,16 @@ class MeshNode:
sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq) sock.setsockopt(socket.IPPROTO_IP, socket.IP_ADD_MEMBERSHIP, mreq)
joined += 1 joined += 1
except OSError as e: except OSError as e:
print(f"[meshdisco] Warning: join {self.group} on {ifname} failed: {e}") print(f"[beacon] Warning: join {self.group} on {ifname} failed: {e}")
self._sock = sock self._sock = sock
self._running = True self._running = True
print( print(
f"[meshdisco] Listening on {self.group}:{self.port} as " f"[beacon] Listening on {self.group}:{self.port} as "
f"{self.name}/{self.instance} (joined {joined} of {len(self._ifaces)} interfaces)" f"{self.name}/{self.instance} (joined {joined} of {len(self._ifaces)} interfaces)"
) )
for target in (self._read_loop, self._announce_loop): for target in (self._read_loop, self._advertise_loop):
t = threading.Thread(target=target, daemon=True) t = threading.Thread(target=target, daemon=True)
t.start() t.start()
self._threads.append(t) self._threads.append(t)
@@ -178,10 +257,10 @@ class MeshNode:
if not self._running: if not self._running:
return return
self._running = False self._running = False
# Final announce so neighbours drop us immediately rather than waiting # Bye, so neighbours drop us immediately rather than waiting out the
# out the staleness window. # staleness window.
try: try:
self._send(self._build_announce("stopping")) self._send({"proto": PROTO, "v": VERSION, "type": TYPE_BYE, "node": self._node_info()})
except OSError: except OSError:
pass pass
if self._sock: if self._sock:
@@ -193,19 +272,25 @@ class MeshNode:
# -- api --------------------------------------------------------------- # -- api ---------------------------------------------------------------
def probe(self) -> None: def probe(self, want: Optional[List[str]] = None) -> None:
"""Multicast a probe; every listener replies immediately.""" """Multicast a probe. Every node owning a resource matching one of
self._send({"v": PROTOCOL_VERSION, "type": TYPE_DISCOVERY}) ``want`` (every node, when empty) replies immediately."""
msg = {"proto": PROTO, "v": VERSION, "type": TYPE_PROBE, "from": self.instance}
if want:
msg["want"] = list(want)
self._send(msg)
def neighbors(self) -> List[dict]: def neighbors(self) -> List[dict]:
"""Every live node, sorted by (name, instance)."""
cutoff = time.time() - self.interval * LIVENESS_FACTOR cutoff = time.time() - self.interval * LIVENESS_FACTOR
with self._lock: with self._lock:
for key in [k for k, v in self._neighbors.items() if v["lastSeen"] < cutoff]: for key in [k for k, v in self._nodes.items() if v["lastSeen"] < cutoff]:
del self._neighbors[key] del self._nodes[key]
out = list(self._neighbors.values()) out = [_neighbor_row(v["node"], v["resources"], v["addr"], v["lastSeen"]) for v in self._nodes.values()]
return sorted(out, key=lambda n: n.get("name") or "") return sorted(out, key=lambda n: (n.get("name") or "", n.get("instance") or ""))
def neighbors_by_name(self) -> List[dict]: def neighbors_by_name(self) -> List[dict]:
"""One row per node name, most recently seen instance wins."""
best: Dict[str, dict] = {} best: Dict[str, dict] = {}
for nb in self.neighbors(): for nb in self.neighbors():
cur = best.get(nb.get("name") or "") cur = best.get(nb.get("name") or "")
@@ -213,55 +298,53 @@ class MeshNode:
best[nb.get("name") or ""] = nb best[nb.get("name") or ""] = nb
return sorted(best.values(), key=lambda n: n.get("name") or "") return sorted(best.values(), key=lambda n: n.get("name") or "")
def cores(self) -> List[dict]:
return [n for n in self.neighbors() if n.get("role") == ROLE_CORE and n.get("urls")]
def self_announce(self) -> dict: def self_announce(self) -> dict:
msg = self._build_announce("running") """This node as a row, so a UI can render itself without waiting."""
msg["addr"] = "" status, resources = self._get_payload()
msg["lastSeen"] = time.time() return _neighbor_row(self._node_info(status), resources, "", time.time())
return msg
def on_announce(self, cb: Callable[[dict], None]) -> None: def on_advertise(self, cb: Callable[[dict], None]) -> None:
self._listeners.append(cb) self._listeners.append(cb)
# -- internals --------------------------------------------------------- # -- internals ---------------------------------------------------------
def _build_announce(self, status: str) -> dict: def _get_payload(self) -> tuple:
msg = { if not self._payload:
"v": PROTOCOL_VERSION, return None, []
"type": TYPE_ANNOUNCE, try:
"name": self.name, p = self._payload() or {}
"instance": self.instance, except Exception:
"role": self.role, p = {}
"version": self.version, return p.get("status"), list(p.get("resources") or [])
"status": status,
def _node_info(self, status: Optional[str] = None) -> dict:
n = {"name": self.name, "instance": self.instance}
if self.version:
n["version"] = self.version
if status:
n["status"] = status
return n
def _advertise_msg(self) -> dict:
status, resources = self._get_payload()
return {
"proto": PROTO, "v": VERSION, "type": TYPE_ADVERTISE,
"node": self._node_info(status), "resources": resources,
} }
if self._payload:
try:
p = self._payload() or {}
except Exception:
p = {}
if p.get("baseUrl"):
msg["baseUrl"] = p["baseUrl"]
if p.get("status") and status == "running":
msg["status"] = p["status"]
# Only a core may carry a URLs block.
if self.role == ROLE_CORE and p.get("urls"):
msg["urls"] = p["urls"]
return msg
def _send(self, msg: dict, dest: Optional[tuple] = None) -> None: def _send(self, msg: dict, dest: Optional[tuple] = None) -> None:
sock = self._sock sock = self._sock
if sock is None: if sock is None:
return return
body = json.dumps(msg).encode("utf-8") body = json.dumps(msg).encode("utf-8")
if len(body) > MAX_DATAGRAM:
print(f"[beacon] Warning: {len(body)}-byte datagram exceeds the fragmentation-safe {MAX_DATAGRAM}")
with self._send_lock: with self._send_lock:
if dest is not None: if dest is not None:
try: try:
sock.sendto(body, dest) sock.sendto(body, dest)
except OSError as e: except OSError as e:
print(f"[meshdisco] Reply to {dest} failed: {e}") print(f"[beacon] Reply to {dest} failed: {e}")
return return
# One write per interface — the default route alone reaches a # One write per interface — the default route alone reaches a
# single network. # single network.
@@ -274,7 +357,7 @@ class MeshNode:
) )
sock.sendto(body, (self.group, self.port)) sock.sendto(body, (self.group, self.port))
except OSError as e: except OSError as e:
print(f"[meshdisco] Warning: send on {ifname} failed: {e}") print(f"[beacon] Warning: send on {ifname} failed: {e}")
def _read_loop(self) -> None: def _read_loop(self) -> None:
while self._running: while self._running:
@@ -287,54 +370,55 @@ class MeshNode:
if not self._running: if not self._running:
return return
continue continue
try: msg = parse_message(data)
msg = json.loads(data.decode("utf-8")) if msg is None:
except (ValueError, UnicodeDecodeError): continue # v1 or foreign traffic on the shared group
continue # foreign traffic on the group
if msg.get("v") != PROTOCOL_VERSION: if msg["type"] == TYPE_PROBE:
continue if msg.get("from") == self.instance:
# Drop our own multicast echo. continue
if msg.get("instance") == self.instance and msg.get("name") == self.name: ad = self._advertise_msg()
want = msg.get("want") or []
if not want or any(resource_matches(r, p) for r in ad["resources"] for p in want):
self._send(ad, dest=addr)
continue continue
if msg.get("type") == TYPE_DISCOVERY: node = msg["node"]
self._send(self._build_announce("running"), dest=addr) if node["instance"] == self.instance:
continue continue # our own multicast echo
if msg.get("type") != TYPE_ANNOUNCE or not msg.get("name"): if msg["type"] == TYPE_BYE:
continue
key = f"{msg['name']}|{msg.get('instance', '')}"
if msg.get("status") == "stopping":
with self._lock: with self._lock:
self._neighbors.pop(key, None) self._nodes.pop(node["instance"], None)
continue continue
# Source address comes from the packet, never the payload. # Source address comes from the packet, never the payload.
msg["addr"] = addr[0] entry = {"node": node, "resources": msg.get("resources") or [], "addr": addr[0], "lastSeen": time.time()}
msg["lastSeen"] = time.time()
with self._lock: with self._lock:
self._neighbors[key] = msg self._nodes[node["instance"]] = entry
row = _neighbor_row(entry["node"], entry["resources"], entry["addr"], entry["lastSeen"])
for cb in list(self._listeners): for cb in list(self._listeners):
try: try:
cb(msg) cb(row)
except Exception as e: except Exception as e:
print(f"[meshdisco] Listener raised: {e}") print(f"[beacon] Listener raised: {e}")
def _announce_loop(self) -> None: def _advertise_loop(self) -> None:
self._send(self._build_announce("running")) self._send(self._advertise_msg())
self.probe() self.probe()
while self._running: while self._running:
time.sleep(self.interval) time.sleep(self.interval)
if not self._running: if not self._running:
return return
try: try:
self._send(self._build_announce("running")) self._send(self._advertise_msg())
except OSError: except OSError:
pass pass
class MetaCoreLocator: class MetaCoreLocator:
"""Locates meta-core over UDP, replacing the kv-leader.info read.""" """Locates meta-core over beacon v2 (the first ``metamesh.core`` resource
carrying ``data.urls``) and advertises this service as
``metamesh.service/<service_name>``."""
def __init__( def __init__(
self, self,
@@ -350,44 +434,49 @@ class MetaCoreLocator:
self._node = MeshNode( self._node = MeshNode(
name=service_name, name=service_name,
role=ROLE_SERVICE,
version=version, version=version,
enabled=_env_flag("ENABLE_UDP_DISCOVERY", True) if enabled is None else enabled, enabled=_env_flag("ENABLE_UDP_DISCOVERY", True) if enabled is None else enabled,
payload=lambda: {"baseUrl": base_url or f"http://{local_ipv4()}"}, payload=lambda: {"resources": [{
"id": "service",
"caps": [CAP_SERVICE_PREFIX + service_name],
"endpoints": {"ui": base_url or f"http://{local_ipv4()}"},
}]},
) )
self._node.on_announce(self._on_announce) self._node.on_advertise(self._on_advertise)
def _on_announce(self, nb: dict) -> None: def _on_advertise(self, nb: dict) -> None:
if nb.get("role") != ROLE_CORE or not nb.get("urls"):
return
if self._pin: if self._pin:
return # pinned: the wire cannot move us return # pinned: the wire cannot move us
core = next((r for r in nb.get("resources") or [] if resource_matches(r, CAP_CORE)), None)
urls = ((core or {}).get("data") or {}).get("urls")
if not isinstance(urls, dict) or not urls.get("apiUrl"):
return
prev = self._current prev = self._current
if prev is None: if prev is None:
self._current = nb["urls"] self._current = urls
print(f"[meshdisco] meta-core discovered at {nb['urls']['apiUrl']} ({nb.get('addr')})") print(f"[beacon] meta-core discovered at {urls['apiUrl']} ({nb.get('addr')})")
for cb in list(self._change_callbacks): for cb in list(self._change_callbacks):
try: try:
cb() cb()
except Exception as e: except Exception as e:
print(f"[meshdisco] Change callback raised: {e}") print(f"[beacon] Change callback raised: {e}")
return return
if prev.get("apiUrl") == nb["urls"].get("apiUrl"): if prev.get("apiUrl") == urls.get("apiUrl"):
self._current = nb["urls"] self._current = urls
return return
# A second, different core. Do not flap — keep the first and say so, # A second, different core. Do not flap — keep the first and say so,
# because on a PCS box this means a client container can see a core it # because on a PCS box this means a client container can see a core it
# must not use. # must not use.
print( print(
f"[meshdisco] Ignoring a second meta-core at {nb['urls']['apiUrl']} " f"[beacon] Ignoring a second meta-core at {urls['apiUrl']} "
f"({nb.get('addr')}); staying with {prev.get('apiUrl')}. " f"({nb.get('addr')}); staying with {prev.get('apiUrl')}. "
f"Set META_CORE_URL to pin this explicitly." f"Set META_CORE_URL to pin this explicitly."
) )
def start(self) -> None: def start(self) -> None:
if self._pin: if self._pin:
print(f"[meshdisco] meta-core pinned to {self._pin}; discovery is advisory") print(f"[beacon] meta-core pinned to {self._pin}; discovery is advisory")
self._node.start() self._node.start()
def stop(self) -> None: def stop(self) -> None:
@@ -409,7 +498,7 @@ class MetaCoreLocator:
``timeout=None`` waits forever, which is meta-stremio's default ``timeout=None`` waits forever, which is meta-stremio's default
(LEADER_WAIT_TIMEOUT=0). Re-probes every 500ms rather than waiting out (LEADER_WAIT_TIMEOUT=0). Re-probes every 500ms rather than waiting out
an announce interval. an advertise interval.
""" """
deadline = None if timeout is None else time.time() + timeout deadline = None if timeout is None else time.time() + timeout
logged = False logged = False
@@ -418,20 +507,25 @@ class MetaCoreLocator:
if url: if url:
return url return url
if not logged: if not logged:
print("[meshdisco] Waiting for meta-core...") print("[beacon] Waiting for meta-core...")
logged = True logged = True
self._node.probe() self._node.probe([CAP_CORE])
time.sleep(0.5) time.sleep(0.5)
return None return None
def on_change(self, cb: Callable[[], None]) -> None: def on_change(self, cb: Callable[[], None]) -> None:
self._change_callbacks.append(cb) self._change_callbacks.append(cb)
def neighbors(self) -> List[dict]: def neighbors(self, all: bool = False, cap: Optional[str] = None) -> List[dict]:
return self._node.neighbors_by_name() """One row per name (every instance with ``all``), optionally only
nodes with a resource matching ``cap``."""
rows = self._node.neighbors() if all else self._node.neighbors_by_name()
if cap:
rows = [r for r in rows if any(resource_matches(res, cap) for res in r.get("resources") or [])]
return rows
def self_announce(self) -> dict: def self_announce(self) -> dict:
return self._node.self_announce() return self._node.self_announce()
def probe(self) -> None: def probe(self, want: Optional[List[str]] = None) -> None:
self._node.probe() self._node.probe(want)
+7 -2
View File
@@ -238,6 +238,10 @@ class MetaServiceMenu extends HTMLElement {
for (const n of this._neighbors) { for (const n of this._neighbors) {
const name = n.name || ""; const name = n.name || "";
if (!name || seen.has(name)) continue; if (!name || seen.has(name)) continue;
// Beacon v2 neighbours include plugins; the nav lists services only.
// (A row without `caps` comes from a pre-v2 backend — keep it.)
const isService = (c) => String(c).startsWith("metamesh.service/");
if (Array.isArray(n.caps) && !n.caps.some(isService)) continue;
seen.add(name); seen.add(name);
rows.push(n); rows.push(n);
} }
@@ -275,7 +279,7 @@ class MetaServiceMenu extends HTMLElement {
el.href = url; el.href = url;
el.setAttribute("role", "menuitem"); el.setAttribute("role", "menuitem");
} }
if (!isActive && !url) el.title = "No reachable URL announced"; if (!isActive && !url) el.title = "No reachable URL advertised";
const icon = document.createElement("span"); const icon = document.createElement("span");
icon.className = "icon"; icon.className = "icon";
@@ -285,7 +289,8 @@ class MetaServiceMenu extends HTMLElement {
label.textContent = titleCase(n.name); label.textContent = titleCase(n.name);
el.append(icon, label); el.append(icon, label);
if (n.status === "running" || isActive) { // Beacon v2 omits `status` when running.
if ((n.status || "running") === "running" || isActive) {
const dot = document.createElement("span"); const dot = document.createElement("span");
dot.className = "dot"; dot.className = "dot";
dot.title = n.status || "running"; dot.title = n.status || "running";