diff --git a/Apps/MetaCore/docker-compose.yml b/Apps/MetaCore/docker-compose.yml index 588683c..95f1512 100644 --- a/Apps/MetaCore/docker-compose.yml +++ b/Apps/MetaCore/docker-compose.yml @@ -14,8 +14,14 @@ services: # ran at 73-97% of 768m, i.e. one indexing burst away from an OOM kill that # takes the record store (and every service reading it) down with it. # 1536m puts steady state near 30%. - mem_limit: 1536m - memswap_limit: 1536m + # + # Raised 1536m -> 6g (2026-09-30): the record index costs ~40 KB of RAM per + # record, and watch.nsl.sh's working set reached 4.8 GiB (tens of + # thousands of records). A cap is a ceiling, not a reservation — a small + # library still uses a few hundred MB — so a generous one costs nothing + # until the library grows into it. + mem_limit: 6g + memswap_limit: 6g hostname: metacore-app restart: unless-stopped privileged: true @@ -49,6 +55,12 @@ services: # feeders write here over HTTP, so this one mount covers all of them. - ${DATA_ROOT:-/DATA}/AppData/metacore/plugin:/files/plugin:rw + # MetaShare's download cache, read-only. Content fetched over the swarm + # lands there; meta-core indexes it (as a watcher POSTed at runtime, not + # an env var) so cached items become ordinary local records with a real + # filePath. Harmless when MetaShare isn't installed: the dir is empty. + - ${DATA_ROOT:-/DATA}/AppData/metashare/data/meta-share/cache:/files/cache:ro + environment: - BASE_URL=https://metacore-$APP_DOMAIN - TZ=$TZ diff --git a/Apps/MetaFeederProwlarr/docker-compose.yml b/Apps/MetaFeederProwlarr/docker-compose.yml index c4bb828..e4c0a5c 100644 --- a/Apps/MetaFeederProwlarr/docker-compose.yml +++ b/Apps/MetaFeederProwlarr/docker-compose.yml @@ -44,9 +44,10 @@ services: cpu_shares: 30 container_name: metafeederprowlarr-feeder # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM - # instead of swap. Mirrors the dev sub-stack limits. - mem_limit: 512m - memswap_limit: 512m + # instead of swap. 512m -> 1g (2026-09-30): the feeder's steady state on + # watch.nsl.sh is ~660 MiB (release cache + in-flight torznab pages). + mem_limit: 1g + memswap_limit: 1g hostname: metafeederprowlarr-feeder restart: unless-stopped user: "0:0" diff --git a/Apps/MetaGateway/docker-compose.yml b/Apps/MetaGateway/docker-compose.yml index c390998..482220c 100644 --- a/Apps/MetaGateway/docker-compose.yml +++ b/Apps/MetaGateway/docker-compose.yml @@ -75,7 +75,7 @@ services: metagateway-share: # 1.0.8 = rsa-enabled libp2p (joins the public IPFS DHT — see metagateway-app) # + P2P_ANNOUNCE support so remote peers can dial this bitswap seeder. - image: ghcr.io/worph/meta-share:1.0.35 + image: ghcr.io/worph/meta-share:1.0.36 container_name: metagateway-share # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. @@ -163,8 +163,10 @@ services: container_name: metagateway-app # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. - mem_limit: 1g - memswap_limit: 1g + # 1g -> 2g (2026-09-30): the DHT-joined dispatcher runs at ~1.2 GiB on + # watch.nsl.sh, so the 1g cap was an OOM loop in waiting. + mem_limit: 2g + memswap_limit: 2g hostname: metagateway-app restart: unless-stopped cpu_shares: 40 @@ -206,6 +208,12 @@ services: # peer-id, every provider record it published was stale within seconds — # remote peers only ever got "Unexpected peer ID". 128 is the default; # raise it only together with the memory limit. + # glibc malloc: cap the per-thread arenas and hand freed memory back to + # the OS. Without them RSS ratchets upward under a burst of fan-out + # queries and never comes down, which is what the cap above then kills. + MALLOC_ARENA_MAX: "2" + MALLOC_MMAP_THRESHOLD_: "131072" + MALLOC_TRIM_THRESHOLD_: "131072" META_GATEWAY_MAX_CONNECTIONS: "128" META_GATEWAY_P2P_LISTEN: "/ip4/0.0.0.0/tcp/4002" # Setting a PUBLIC_ADDR is what flips the gateway out of local/mDNS-only @@ -287,8 +295,16 @@ services: # `PUT /api/gateway/plugins/:id` stay behind AppShield. caddy_0.1_handle: "/api/gateway/plugins" caddy_0.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_0.2_handle: "" - caddy_0.2_handle.reverse_proxy: "{{upstreams 80}}" + # Content bytes by CID, fetched by REMOTE peers with no session — the + # same carve-out MetaShare has always had. meta-read builds cover hint + # URLs from a gateway-sourced record's originUrl, i.e. THIS host; behind + # AppShield they answered 302 → login and every cover fell back to + # cross-internet bitswap. Read-only; settings/search/stats stay gated. + # Must stay ahead of the empty catch-all. + caddy_0.2_handle: "/api/file/*" + caddy_0.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_0.3_handle: "" + caddy_0.3_handle.reverse_proxy: "{{upstreams 80}}" # ---- metagateway-IP.nip.io (gateway TLS) ---- caddy_1: metagateway-${APP_PUBLIC_IP_DASH}.nip.io @@ -300,8 +316,10 @@ services: # `PUT /api/gateway/plugins/:id` stay behind AppShield. caddy_1.1_handle: "/api/gateway/plugins" caddy_1.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_1.2_handle: "" - caddy_1.2_handle.reverse_proxy: "{{upstreams 80}}" + caddy_1.2_handle: "/api/file/*" + caddy_1.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_1.3_handle: "" + caddy_1.3_handle.reverse_proxy: "{{upstreams 80}}" # ---- metagateway-IP.sslip.io (Let's Encrypt) ---- caddy_2: metagateway-${APP_PUBLIC_IP_DASH}.sslip.io @@ -312,8 +330,10 @@ services: # `PUT /api/gateway/plugins/:id` stay behind AppShield. caddy_2.1_handle: "/api/gateway/plugins" caddy_2.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_2.2_handle: "" - caddy_2.2_handle.reverse_proxy: "{{upstreams 80}}" + caddy_2.2_handle: "/api/file/*" + caddy_2.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_2.3_handle: "" + caddy_2.3_handle.reverse_proxy: "{{upstreams 80}}" networks: - metagateway - pcs diff --git a/Apps/MetaGatewayShared/docker-compose.yml b/Apps/MetaGatewayShared/docker-compose.yml index c1946ec..560af53 100644 --- a/Apps/MetaGatewayShared/docker-compose.yml +++ b/Apps/MetaGatewayShared/docker-compose.yml @@ -41,8 +41,10 @@ services: cpu_shares: 40 # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. - mem_limit: 1g - memswap_limit: 1g + # 1g -> 2g (2026-09-30): the DHT-joined dispatcher runs at ~1.2 GiB on + # watch.nsl.sh, so the 1g cap was an OOM loop in waiting. + mem_limit: 2g + memswap_limit: 2g expose: - 3000 # Public libp2p listener. This is the peer a remote MetaWatch's meta-search @@ -94,6 +96,12 @@ services: # peer-id, every provider record it published was stale within seconds — # remote peers only ever got "Unexpected peer ID". 128 is the default; # raise it only together with the memory limit. + # glibc malloc: cap the per-thread arenas and hand freed memory back to + # the OS. Without them RSS ratchets upward under a burst of fan-out + # queries and never comes down, which is what the cap above then kills. + MALLOC_ARENA_MAX: "2" + MALLOC_MMAP_THRESHOLD_: "131072" + MALLOC_TRIM_THRESHOLD_: "131072" META_GATEWAY_MAX_CONNECTIONS: "128" META_GATEWAY_P2P_LISTEN: "/ip4/0.0.0.0/tcp/4002" META_GATEWAY_PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4002" @@ -106,7 +114,10 @@ services: # ALL — no provider record, no discovery, no gateway fan-out. Observed on # watch.nsl.sh 2026-09-01: 2 connections with the four alone, 126 once a # reachable fifth was added. Costs nothing when the four work. - KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + # The four trailing /ip4/ peers are long-lived public DHT servers + # observed reachable from watch.nsl.sh — more doors in if the + # Foundation nodes refuse this IP. + KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ,/ip4/88.99.98.148/tcp/4001/p2p/12D3KooWJefeaXhgJGmxzcqFq9twQASbt9Fuc2Ez7MRgrcpt3aLu,/ip4/13.250.82.223/tcp/4001/p2p/12D3KooWCpQsdp1NGcoJr2RT81TpGc4qAiWeUrVwopzjTTYwGaSp,/ip4/159.203.79.101/tcp/4001/p2p/12D3KooWAhMxrbK9DVVsonnXZCxcZkn7Sj9QsseYN7KGZRqwESBp,/ip4/139.18.170.37/tcp/4001/p2p/12D3KooWF65X6XcJRGXpCAViN3CaBAMaTWSuqyoya4ndjHQxdjJu" TZ: "$TZ" depends_on: metagateway-card: @@ -199,8 +210,16 @@ services: # + nzbFetch + nzbIndexers (indexer HOSTNAMES, no API keys). caddy_0.1_handle: "/api/gateway/plugins" caddy_0.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_0.2_handle: "" - caddy_0.2_handle.reverse_proxy: "{{upstreams 80}}" + # Content bytes by CID, fetched by REMOTE peers with no session — the + # same carve-out MetaShare has always had. meta-read builds cover hint + # URLs from a gateway-sourced record's originUrl, i.e. THIS host; behind + # AppShield they answered 302 → login and every cover fell back to + # cross-internet bitswap. Read-only; settings/search/stats stay gated. + # Must stay ahead of the empty catch-all. + caddy_0.2_handle: "/api/file/*" + caddy_0.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_0.3_handle: "" + caddy_0.3_handle.reverse_proxy: "{{upstreams 80}}" # ---- metagateway-IP.nip.io (gateway TLS) ---- caddy_1: metagateway-${APP_PUBLIC_IP_DASH}.nip.io @@ -215,8 +234,10 @@ services: # + nzbFetch + nzbIndexers (indexer HOSTNAMES, no API keys). caddy_1.1_handle: "/api/gateway/plugins" caddy_1.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_1.2_handle: "" - caddy_1.2_handle.reverse_proxy: "{{upstreams 80}}" + caddy_1.2_handle: "/api/file/*" + caddy_1.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_1.3_handle: "" + caddy_1.3_handle.reverse_proxy: "{{upstreams 80}}" # ---- metagateway-IP.sslip.io (Let's Encrypt) ---- caddy_2: metagateway-${APP_PUBLIC_IP_DASH}.sslip.io @@ -230,8 +251,10 @@ services: # + nzbFetch + nzbIndexers (indexer HOSTNAMES, no API keys). caddy_2.1_handle: "/api/gateway/plugins" caddy_2.1_handle.reverse_proxy: "metagateway-app:3000" - caddy_2.2_handle: "" - caddy_2.2_handle.reverse_proxy: "{{upstreams 80}}" + caddy_2.2_handle: "/api/file/*" + caddy_2.2_handle.reverse_proxy: "metagateway-app:3000" + caddy_2.3_handle: "" + caddy_2.3_handle.reverse_proxy: "{{upstreams 80}}" icon: https://cdn.jsdelivr.net/gh/Worph/MetaAppStore@main/Apps/MetaGateway/icon.png networks: diff --git a/Apps/MetaRead/docker-compose.yml b/Apps/MetaRead/docker-compose.yml index 24e0195..24756a5 100644 --- a/Apps/MetaRead/docker-compose.yml +++ b/Apps/MetaRead/docker-compose.yml @@ -14,6 +14,11 @@ name: metaread # metaread-share → pure content transport (byte Range, bitswap, BT) # metaread-app → the reader UI/back-end (behind OIDC) # +# Contrast with `MetaReadShared` in this store, which ships only meta-search and +# reuses the standalone MetaCore + MetaShare apps (one profile shared with +# MetaWatch). Pick exactly one: both use compose project `metaread` and the same +# /DATA/AppData/metaread paths. +# # ⚠ **Distinct libp2p host ports.** MetaWatch already publishes 4003/4004; this # stack uses 4005/4006. Two peers cannot share a host port, and the failure is # a container that will not start rather than anything subtle — but if you run @@ -141,7 +146,7 @@ services: # that lists correctly and will not open. # --------------------------------------------------------------------------- metaread-share: - image: ghcr.io/worph/meta-share:1.0.35 + image: ghcr.io/worph/meta-share:1.0.36 container_name: metaread-share # Smaller than MetaWatch's 3 GB on purpose: a chapter is tens of megabytes # where a film is gigabytes, and this peer never holds a whole video. @@ -195,6 +200,11 @@ services: container_name: metaread-app hostname: metaread-app restart: unless-stopped + # The image's own USER is uid 10001, but the data dir is created as + # $PUID:$PGID by x-compose-app `folders`, so the stock user cannot write + # library.json, the session key or the tile cache. Pin the runtime uid to + # the one that owns the mount, as metawatch-app does. + user: "1000:1000" cpu_shares: 30 mem_limit: 512m memswap_limit: 512m diff --git a/Apps/MetaReadShared/docker-compose.yml b/Apps/MetaReadShared/docker-compose.yml new file mode 100644 index 0000000..3a749c1 --- /dev/null +++ b/Apps/MetaReadShared/docker-compose.yml @@ -0,0 +1,361 @@ +name: metaread + +# ============================================================================= +# MetaRead (shared backend) — the READING client for the MetaMesh network. +# +# This is the SHARED-BACKEND variant of MetaRead. It ships only its own +# discovery peer (meta-search) and REUSES the standalone MetaCore and MetaShare +# apps already installed on the box: +# +# metaread-search → discovery (Kamilata federated search, identity proxy) +# → talks to the standalone metacore-app over `pcs` +# metaread-app → the reader UI/back-end +# → streams bytes from the standalone metashare-app +# +# One keystore, one User Data Layer, one catalogue, one transport cache, however +# many clients: a profile is a secp256k1 key in a meta-core keystore, so a +# per-app core means a per-app identity and a library with nothing in it. +# +# Contrast with the `MetaRead` app in this store, which is SELF-CONTAINED (its +# own metaread-core + metaread-share). Pick exactly one: both use compose +# project `metaread` and the same /DATA/AppData/metaread paths, so they cannot +# be installed side by side. +# +# Deployed and validated on watch.nsl.sh (2026-09). +# +# REQUIRES, on the same box and on the `pcs` network: +# - MetaCore → provides metacore-app (identity keystore + record store) +# - MetaShare → provides metashare-app (byte transport for reading) +# Install those FIRST. Without them MetaRead cannot sign anyone in and cannot +# open anything. +# +# ⚠ **Distinct libp2p host port.** MetaWatch's discovery publishes 4004 and the +# shared MetaShare publishes 4001; this stack's discovery uses 4006. Two peers +# cannot share a host port, and the failure is a container that will not start. +# +# ── What it needs to actually show you something ───────────────────────────── +# A client is a window onto the swarm; it holds no catalogue of its own. To see +# comics you need a gateway peer running the literature feeders: +# +# **MetaFeeder · Suwayomi** — the chapters (and its own work cards), plus +# optionally **MetaFeeder · AniList** to name the works. Install them beside +# **MetaGateway** and register each feeder in the gateway's config. +# +# **MetaFeeder · Gutenberg** — Project Gutenberg (+ Open Library cards), for +# public-domain books. +# +# Without either, MetaRead runs perfectly and shows an empty library — which is +# the honest state of a mesh nobody has fed yet, not a fault. +# ============================================================================= + +services: + # --------------------------------------------------------------------------- + # metaread-search — DISCOVERY. Federated search (Kamilata Bloom routing), the + # gateway fan-out, cover recovery over bitswap, the shared prefs and the User + # Data Layer. Public libp2p on host port 4006; mDNS on `pcs` for same-host + # peers. + # + # ⚠ This is the peer that reaches MetaGateway. An empty MetaRead library with + # a healthy gateway usually means this peer never joined the DHT — check its + # peer count before suspecting the feeders. + # --------------------------------------------------------------------------- + metaread-search: + image: ghcr.io/worph/meta-search:1.2.6 + container_name: metaread-search + # 2g, not the self-contained listing's 1g: on the shared core it indexes the + # gateway box's whole catalogue (~830 MiB steady state on watch.nsl.sh). + mem_limit: 2g + memswap_limit: 2g + hostname: metaread-search + restart: unless-stopped + user: "0:0" + cpu_shares: 30 + expose: + - 3000 + ports: + - "4006:4001" + volumes: + # No /meta-core mount: this service locates meta-core over UDP + # (meta-discovery v1) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/service-discovery.md. + - ${DATA_ROOT:-/DATA}/AppData/metaread/search-data:/data:rw + environment: + # Distinct from metawatch-search on the same core, so the two discovery + # peers do not register under one name. + SERVICE_NAME: "metaread-search" + META_SEARCH_PUBLIC_URL: "https://metareadsearch-${APP_DOMAIN}" + # glibc malloc: cap the per-thread arenas and hand freed memory back to + # the OS, so RSS comes down after a fan-out burst. + MALLOC_ARENA_MAX: "2" + MALLOC_MMAP_THRESHOLD_: "131072" + MALLOC_TRIM_THRESHOLD_: "131072" + P2P_LISTEN: "/ip4/0.0.0.0/tcp/4001" + # ⚠ The IPV4 spelling, never the bare APP_PUBLIC_IP: on an IPv6-primary + # PCS that variable holds the v6 address, and `/ip4//tcp/...` is not a + # parseable multiaddr — the peer REFUSES TO START rather than announce a + # lie. + PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4006" + # Explicit /dns4/ bootstraps: the nested dnsaddr default resolves to + # wss-only in some container resolvers. mars.i.ipfs.io is the only + # surviving legacy *.i.ipfs.io bootstrapper and is here for resilience — + # the four IPFS-Foundation nodes can refuse an individual IP, and when they + # do a peer cannot join the DHT AT ALL (no provider record, no discovery, + # no gateway fan-out). Measured on watch.nsl.sh 2026-09-01: 2 connections + # with the four alone, 126 once a reachable fifth was added. + KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + META_CORE_URL: "http://metacore-app:80" + BASE_URL: "http://metaread-search:3000" + INTERNAL_BASE_URL: "http://metaread-search:3000" + TZ: "$TZ" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/health"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + # `pcs` is REQUIRED: without it this peer cannot reach metacore-app, and + # sign-in fails ("keystore unreachable") while every container reports + # healthy. + networks: + - metaread + - pcs + + # --------------------------------------------------------------------------- + # metaread-app — the reader. Zero page work server-side: a flat Range/206 byte + # proxy to the shared metashare-app, with the CBZ/EPUB/PDF decoding done in + # the browser. My List, Continue Reading and reading positions live in the + # User Data Layer. + # --------------------------------------------------------------------------- + metaread-app: + image: ghcr.io/worph/meta-read:1.0.8 + container_name: metaread-app + hostname: metaread-app + restart: unless-stopped + # The image's own USER is the system account `metaread` (uid 10001), but the + # data dir is created as $PUID:$PGID by x-compose-app `folders` — so the + # stock user cannot write library.json, the session key, the invite book or + # the tile cache, and fails with a WARN per row rather than an error. Pin the + # runtime uid to the one that owns the mount, as metawatch-app does. + user: "1000:1000" + cpu_shares: 30 + mem_limit: 512m + memswap_limit: 512m + expose: + - 3000 + volumes: + - ${DATA_ROOT:-/DATA}/AppData/metaread/data:/app/data + environment: + BASE_URL: "https://metaread-${APP_DOMAIN}" + SERVICE_NAME: "meta-read" + META_SHARE_URL: "http://metashare-app:3000" + META_SEARCH_URL: "http://metaread-search:3000" + # Browser-reachable topbar chips. Transport is the shared MetaShare app + # (its own AppShield gate); discovery is this stack's gate below. + META_SHARE_PUBLIC_URL: "https://metashare-${APP_DOMAIN}" + META_SEARCH_PUBLIC_URL: "https://metareadsearch-${APP_DOMAIN}" + TZ: "$TZ" + depends_on: + metaread-search: + condition: service_healthy + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/health"] + interval: 5s + timeout: 3s + retries: 10 + start_period: 5s + # ⚠ No AppShield in front of the reader, on purpose — Caddy talks straight + # to :3000, exactly as metawatch-app does. meta-read carries its OWN auth + # (secp256k1 keypair, signed challenge, invite-only signup) and gates + # deny-by-default. Stacking Authelia on top meant a second login, an OIDC + # redirect on every cover fetch ( cannot carry a header), and a + # service worker at the mercy of a session cookie. See rationale.md. + labels: + caddy_0: metaread-${APP_DOMAIN} + caddy_0.import: gateway_tls + caddy_0.reverse_proxy: "{{upstreams 3000}}" + + caddy_1: metaread-${APP_PUBLIC_IP_DASH}.nip.io + caddy_1.import: gateway_tls + caddy_1.reverse_proxy: "{{upstreams 3000}}" + + caddy_2: metaread-${APP_PUBLIC_IP_DASH}.sslip.io + caddy_2.reverse_proxy: "{{upstreams 3000}}" + + icon: https://cdn.jsdelivr.net/gh/Worph/MetaAppStore@main/Apps/MetaRead/icon.png + networks: + - metaread + - pcs + + # --------------------------------------------------------------------------- + # Gate for the co-located discovery peer, so the reader's meta-search chip + # resolves to something a browser can open. Diagnostics only — nothing routes + # through it. (Transport is the shared MetaShare app; it has its own.) + # `pcs` is REQUIRED — the gate resolves auth-registrar by name there. + # --------------------------------------------------------------------------- + metareadsearch: + image: ghcr.io/yundera/appshield:2.0.9 + cpu_shares: 50 + container_name: metareadsearch + hostname: metareadsearch + restart: unless-stopped + user: "0:0" + expose: + - 80 + environment: + OIDC_REGISTRAR_URL: "http://auth-registrar:9092" + REDIRECT_HOST_SUFFIXES: "${APP_DOMAIN},${APP_PUBLIC_IP_DASH}.nip.io,${APP_PUBLIC_IP_DASH}.sslip.io" + BACKEND_HOST: "metaread-search" + BACKEND_PORT: "3000" + LISTEN_PORT: "80" + labels: + caddy_0: metareadsearch-${APP_DOMAIN} + caddy_0.import: gateway_tls + caddy_0.reverse_proxy: "{{upstreams 80}}" + caddy_1: metareadsearch-${APP_PUBLIC_IP_DASH}.nip.io + caddy_1.import: gateway_tls + caddy_1.reverse_proxy: "{{upstreams 80}}" + caddy_2: metareadsearch-${APP_PUBLIC_IP_DASH}.sslip.io + caddy_2.reverse_proxy: "{{upstreams 80}}" + depends_on: + - metaread-search + networks: + - metaread + - pcs + +networks: + metaread: + name: metaread + driver: bridge + pcs: + name: ${APP_NET:-pcs} + external: true + +x-casaos: + architectures: + - amd64 + - arm64 + author: Worph Team + category: Media + # `needs-public-ip` is the reserved functional tag: this app publishes a raw + # libp2p TCP port that remote peers must dial inbound. + tags: ["media", "needs-public-ip"] + developer: Aptero Team + hostname: metaread-${APP_DOMAIN} + main: metaread-app + port_map: "443" + scheme: https + index: / + + description: + en_us: | + **A reading client for the MetaMesh network — comics and manga, books, + papers and scanned documents.** + + MetaRead is to reading what **MetaWatch** is to video: a library you browse + by *work* rather than by filename. Search returns a grid of covers, a work + opens to its chapter or edition list, and a chapter opens in a reader that + does the page decoding **in your browser** — the server is a flat byte + proxy and never unpacks an archive on your behalf. + + **What it reads.** CBZ and CBR comics, EPUB and PDF books, and the + per-page image lists that a manga aggregator serves for a chapter nobody + has downloaded yet. Reading direction, page layout and spread are per-work + and remembered. + + **Languages.** Pick the languages you read once, in the top bar, and every + view honours it — the same preference MetaWatch and the meta-share + dashboard already speak. An item that declares no language is always + shown: absence means *unknown*, never *not yours*. + + This build is the **shared-backend** variant: it ships only its own + discovery (meta-search) and **requires** the standalone MetaCore and + MetaShare apps on the same box, which it reuses for identity, state and + content transport. One core, one share — no duplicated services, and the + same profile as MetaWatch. If you want a stack that bundles everything + instead, install **MetaRead**. + + **Requires a feeder to have anything to show:** **MetaFeeder · Suwayomi** + (with **MetaFeeder · AniList**) for comics and manga, **MetaFeeder · Gutenberg** + for public-domain books. Without one it runs perfectly and shows an empty + library — the honest state of a mesh nobody has fed. + fr_fr: | + **Un client de lecture pour le réseau MetaMesh — BD et manga, livres, + articles et documents numérisés.** + + MetaRead est à la lecture ce que **MetaWatch** est à la vidéo : une + bibliothèque que l'on parcourt par *œuvre* plutôt que par nom de fichier. + Un chapitre s'ouvre dans un lecteur qui décode les pages **dans votre + navigateur** — le serveur n'est qu'un relais d'octets. + + Cette version est la variante **backend partagé** : elle n'embarque que + sa propre découverte (meta-search) et **nécessite** les apps MetaCore et + MetaShare autonomes sur la même machine, qu'elle réutilise pour l'identité, + l'état et le transport. Un seul core, un seul share — aucun doublon, et le + même profil que MetaWatch. + + **Nécessite un feeder** pour avoir quelque chose à afficher : + **MetaFeeder · Suwayomi** (avec **MetaFeeder · AniList**) pour la BD et le manga, + **MetaFeeder · Gutenberg** pour les livres du domaine public. + + tagline: + en_us: MetaRead reusing the standalone MetaCore + MetaShare apps + fr_fr: MetaRead réutilisant les apps MetaCore et MetaShare autonomes + + title: + en_us: MetaRead (shared backend) + + icon: https://cdn.jsdelivr.net/gh/Worph/MetaAppStore@main/Apps/MetaRead/icon.png + thumbnail: https://cdn.jsdelivr.net/gh/Worph/MetaAppStore@main/Apps/MetaRead/thumbnail.png + + store_app_id: metareadshared + is_uncontrolled: false + + tips: + before_install: + en_us: | + **Requires MetaCore + MetaShare.** Install both standalone apps first — + this build reuses them instead of bundling its own copies. Only + meta-search is local to this stack. + + **Conflicts with `MetaRead`.** Both use the compose project `metaread` + and the same data paths. Install one or the other, not both. + + **Ports.** This stack publishes libp2p on host port **4006** (its + discovery peer). MetaWatch uses 4004 and the shared MetaShare uses + 4001 — if you run another Meta* client, keep walking upward. + + **Install order:** + 1. **MetaGateway** — the peer that runs the feeders and announces what + they find. + 2. **MetaFeeder · Suwayomi** (+ **MetaFeeder · AniList**) and/or + **MetaFeeder · Gutenberg**, then register each in the gateway's + *Add a feeder* box. Gateway feeder discovery is **startup-only**, so + the gateway restarts itself afterwards. + 3. This app. + + **Authentication:** MetaRead has its **own** sign-in — a profile is a + secp256k1 keypair and you prove it by signing a challenge, so the secret + key never leaves the browser. There is no Authelia/OIDC layer in front + of the reader; creating a profile needs an invite link from an existing + user. The profile is the same one MetaWatch uses on this box. + + **An empty library is not a fault.** MetaRead holds no catalogue of its + own. If the grid is empty, the question is whether a feeder is attached + and whether this stack's discovery peer has joined the DHT — the + *meta-search* chip in the top bar shows its peer count. + +# Maison (Yundera dashboard) configuration. Read in preference to x-casaos. +# schema_version 2 is required for `folders`: an older Maison refuses the app +# outright rather than starting it without its directories. +x-compose-app: + schema_version: 2 + folders: + - path: /DATA/AppData/metaread/data + user: $PUID + group: $PGID + - path: /DATA/AppData/metaread/search-data + user: $PUID + group: $PGID + # Same string as the caddy_0 label, so the tile link and the route agree. + webui-host: metaread-${domain} + webui-path: "/" diff --git a/Apps/MetaReadShared/rationale.md b/Apps/MetaReadShared/rationale.md new file mode 100644 index 0000000..c05545b --- /dev/null +++ b/Apps/MetaReadShared/rationale.md @@ -0,0 +1,59 @@ +# MetaRead (shared backend) — Rationale + +## What deviation / exception is being requested + +1. `metaread-app` is **publicly routed with no AppShield sidecar**. It carries the + `caddy_*` labels itself. +2. `metaread-search` runs `user: 0:0` and publishes host port `4006/tcp` (libp2p). +3. The compose top-level `name:` is `metaread`, the same project id as the self-contained + `MetaRead` app in this store. + +## Why it is necessary + +1. meta-read has its **own** authentication: a secp256k1 keypair proven by signing a + challenge, with invite-only signup. Its router gates deny-by-default: a route not in + `open_routes()` answers 401. The open set is `/health`, `/api/config`, the + challenge/token pair, `/api/invite/check/:token` and `/api/image/:cid`. That is the "app's + own built-in auth" alternative the security checklist accepts. AppShield in front added a + second login and an OIDC redirect on every cover fetch, because `` cannot carry + an Authorization header. It also left the service worker's offline cache at the mercy of + a session cookie. +2. libp2p is not HTTP. Caddy cannot proxy it, and remote peers dial the advertised + multiaddr directly, so the port must be published and reachable inbound. The app carries + the reserved `needs-public-ip` tag. +3. The two MetaRead variants are alternatives, not companions. They share the project id + and the same `/DATA/AppData/metaread/` paths, so installing one over the other reuses the + same data. Only one may be installed at a time. + +## Security mitigations in place + +- meta-read's signature-challenge login is on by default, and signup is invite-only, so a + stranger reaching the public hostname cannot create an account. +- The `metaread-search` dashboard *is* behind an AppShield sidecar + (`ghcr.io/yundera/appshield:2.0.9`) with Authelia SSO; only the reader itself uses its own + gate. +- `metaread-app` runs unprivileged as `1000:1000` with a 512 MB hard memory cap and swap + disabled. +- The only published port is raw libp2p TCP; no HTTP port is published to the host. +- No user directories are mounted: no `/DATA/Documents`, `/DATA/Downloads`, `/DATA/Media` + or `/DATA/Gallery`. +- `cpu_shares` on every service; image tags pinned. + +## Alternatives considered and rejected + +- **AppShield in front of `metaread-app`.** Two identities for one user, plus the cover and + service-worker costs above. +- **Bundle a private MetaCore/MetaShare.** That is the self-contained `MetaRead` app. On a + box that already runs them, it doubles the memory footprint. It also gives the reader an + identity, a User Data Layer and a catalogue separate from MetaWatch's on the same box. +- **A distinct project id so both variants coexist.** They would fight over the same public + hostnames, the same `/DATA/AppData/metaread/` paths and host port 4006. + +## Data protection + +My List, Continue Reading, the session key and the invite book persist in +`/DATA/AppData/metaread/data`. The recovered cover blockstore and language prefs persist in +`/DATA/AppData/metaread/search-data`. Both are declared in `x-compose-app.folders` and +archived by Maison on uninstall. Because both MetaRead variants use those paths, switching +between them keeps the user's data. The MetaCore and MetaShare state this variant reads +belongs to those apps and is archived with them. diff --git a/Apps/MetaShare/docker-compose.yml b/Apps/MetaShare/docker-compose.yml index e86dfe2..927ff74 100644 --- a/Apps/MetaShare/docker-compose.yml +++ b/Apps/MetaShare/docker-compose.yml @@ -18,7 +18,7 @@ services: # RSA-keyed; without `rsa` the noise handshake fails and this peer never # actually reaches the DHT) + PUBLIC_ADDR support so remote archivists can # dial this seeder. - image: ghcr.io/worph/meta-share:1.0.35 + image: ghcr.io/worph/meta-share:1.0.36 container_name: metashare-app # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. diff --git a/Apps/MetaSort/docker-compose.yml b/Apps/MetaSort/docker-compose.yml index fbcb494..f96a152 100644 --- a/Apps/MetaSort/docker-compose.yml +++ b/Apps/MetaSort/docker-compose.yml @@ -9,6 +9,8 @@ services: hostname: metasort-app restart: unless-stopped cpu_shares: 20 + mem_limit: 2g + memswap_limit: 2g expose: - 80 @@ -26,6 +28,17 @@ services: # Docker socket for container plugin management - /var/run/docker.sock:/var/run/docker.sock:ro + # Config files from this app's seed tree (Apps/MetaSort/seed/config/), + # which Maison writes only when absent — an operator's edits survive + # updates. The image ships no plugins.yml, so without it a stock install + # spawns ZERO plugins; the supervisord override drops the image's bundled + # follower meta-core, which restart-loops against the standalone MetaCore + # app; the nginx conf is the image's stock one, mounted so a box can tune + # it (watch.nsl.sh 503s /api/stats, which sweeps every record). + - ${DATA_ROOT:-/DATA}/AppData/metasort/config/plugins.yml:/app/plugins.yml:ro + - ${DATA_ROOT:-/DATA}/AppData/metasort/config/supervisord-metasort.conf:/etc/supervisor/conf.d/supervisord.conf:ro + - ${DATA_ROOT:-/DATA}/AppData/metasort/config/nginx-metasort.conf:/etc/nginx/nginx.conf:ro + # Note: no /files mounts — files are accessed via WebDAV from MetaCore # (Architecture V3). MetaCore is the single source of truth for media files. @@ -41,6 +54,13 @@ services: # Container plugin configuration - CONTAINER_CALLBACK_URL=http://metasort-app/api/plugins/callback - CONTAINER_NETWORK=pcs + # meta-core URL handed to plugin containers. Port 3000, not 80: the baked + # nginx has no /meta/ location, so plugin writes through :80 would 404. + # (The code default, http://meta-sort, names no host on this network.) + - CONTAINER_META_CORE_URL=http://metasort-app:3000 + # Compose project label stamped on the plugin containers meta-sort spawns, + # so they group under one stack instead of floating loose. + - PLUGIN_STACK_NAME=metasort-plugins # Note: Runs as root for nginx, Redis, and file management user: 0:0 diff --git a/Apps/MetaSort/seed/config/nginx-metasort.conf b/Apps/MetaSort/seed/config/nginx-metasort.conf new file mode 100644 index 0000000..2156770 --- /dev/null +++ b/Apps/MetaSort/seed/config/nginx-metasort.conf @@ -0,0 +1,79 @@ +worker_processes auto; +error_log /var/log/nginx/error.log warn; +pid /var/run/nginx.pid; + +events { + worker_connections 1024; +} + +http { + include /etc/nginx/mime.types; + default_type application/octet-stream; + + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent"'; + + access_log /var/log/nginx/access.log main; + + sendfile on; + tcp_nopush on; + keepalive_timeout 65; + gzip on; + gzip_types text/plain text/css application/json application/javascript text/xml application/xml; + + # Upstream servers + upstream api { + server 127.0.0.1:3000; + } + + upstream rclone { + server 127.0.0.1:5572; + } + + server { + listen 80; + server_name _; + + # Static files (UI) + root /app/ui; + index index.html; + + # SPA routing - serve index.html for any non-file request + location / { + try_files $uri $uri/ /index.html; + } + + # API routes + location /api/ { + proxy_pass http://api; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } + + # Health check (direct to API) + location /health { + proxy_pass http://api/health; + proxy_http_version 1.1; + proxy_set_header Host $host; + } + + # rclone web UI + location /rclone/ { + proxy_pass http://rclone/; + proxy_http_version 1.1; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header Upgrade $http_upgrade; + proxy_set_header Connection "upgrade"; + + # Basic auth for rclone + auth_basic "rclone"; + auth_basic_user_file /etc/nginx/.htpasswd; + } + } +} diff --git a/Apps/MetaSort/seed/config/plugins.yml b/Apps/MetaSort/seed/config/plugins.yml new file mode 100644 index 0000000..eebd545 --- /dev/null +++ b/Apps/MetaSort/seed/config/plugins.yml @@ -0,0 +1,56 @@ +# Container plugins meta-sort spawns. Read at startup only: restart +# metasort-app after editing. +# +# Not enabled by default: tmdb and filename-parser (cache filenames are content +# hashes, so a parsed filename would overwrite a gateway record's better +# identity) and full-hash (reads a multi-GB file end to end on the peer that +# is streaming it). + +defaults: + resources: + memory: 256m + cpus: 0.5 + timeout: 60000 + +plugins: + file-info: + enabled: true + image: ghcr.io/worph/metamesh-plugin-file-info:1.0.1 + instances: 1 + resources: + memory: 128m + cpus: 0.25 + + ffmpeg: + enabled: true + image: ghcr.io/worph/metamesh-plugin-ffmpeg:1.0.0 + instances: 1 + resources: + memory: 512m + cpus: 1.0 + + subtitle: + enabled: true + image: ghcr.io/worph/metamesh-plugin-subtitle:1.0.1 + instances: 1 + resources: + memory: 128m + cpus: 0.25 + + subtitle-extractor: + enabled: true + image: ghcr.io/worph/metamesh-plugin-subtitle-extractor:1.0.1 + instances: 1 + resources: + memory: 512m + cpus: 1.0 + timeout: 300000 # one full sequential read of the video over WebDAV + + still-extractor: + enabled: true + image: ghcr.io/worph/metamesh-plugin-still-extractor:1.0.1 + instances: 1 + resources: + memory: 512m + cpus: 1.0 + timeout: 300000 # one HTTP seek + decode per candidate offset diff --git a/Apps/MetaSort/seed/config/supervisord-metasort.conf b/Apps/MetaSort/seed/config/supervisord-metasort.conf new file mode 100644 index 0000000..5e69bcf --- /dev/null +++ b/Apps/MetaSort/seed/config/supervisord-metasort.conf @@ -0,0 +1,44 @@ +# meta-sort image supervisord.conf with [program:meta-core] removed: the +# bundled follower core restart-loops against the standalone MetaCore app. + +[supervisord] +nodaemon=true +user=root +logfile=/var/log/supervisor/supervisord.log +pidfile=/var/run/supervisord.pid +loglevel=info + +[program:nginx] +command=/usr/sbin/nginx -g "daemon off;" +priority=20 +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/nginx.log +stderr_logfile=/var/log/supervisor/nginx_error.log +startsecs=2 + +[program:rclone] +command=/usr/bin/rclone rcd --rc-web-gui --rc-web-gui-no-open-browser --rc-addr=:5572 --rc-user=admin --rc-pass=admin +priority=30 +autostart=true +autorestart=true +stdout_logfile=/var/log/supervisor/rclone.log +stderr_logfile=/var/log/supervisor/rclone_error.log +startsecs=3 + +[program:meta-sort] +command=/usr/bin/node /app/backend/dist/index.js +directory=/app/backend +priority=40 +autostart=true +autorestart=true +stdout_logfile=/dev/stdout +stdout_logfile_maxbytes=0 +stderr_logfile=/dev/stderr +stderr_logfile_maxbytes=0 +startsecs=5 +environment= + REDIS_URL="redis://127.0.0.1:6379", + REDIS_PREFIX="", + API_HOST="0.0.0.0", + API_PORT="3000" diff --git a/Apps/MetaWatch/docker-compose.yml b/Apps/MetaWatch/docker-compose.yml index b471764..bcf85d3 100644 --- a/Apps/MetaWatch/docker-compose.yml +++ b/Apps/MetaWatch/docker-compose.yml @@ -152,7 +152,7 @@ services: # the 1.0.8 floor: rsa-enabled libp2p (joins the public IPFS DHT) + # P2P_ANNOUNCE so remote peers can dial this transport peer to fetch the # bytes it seeds (fetch-implies-seed). Without rsa it never reaches the DHT. - image: ghcr.io/worph/meta-share:1.0.35 + image: ghcr.io/worph/meta-share:1.0.36 container_name: metawatch-share # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. @@ -223,7 +223,7 @@ services: # My List + Continue Watching persist in /DATA/AppData/metawatch/data. # --------------------------------------------------------------------------- metawatch-app: - image: ghcr.io/worph/meta-watch:1.0.67 + image: ghcr.io/worph/meta-watch:1.0.68 container_name: metawatch-app hostname: metawatch-app restart: unless-stopped diff --git a/Apps/MetaWatchShared/docker-compose.yml b/Apps/MetaWatchShared/docker-compose.yml index 92efa42..34494fa 100644 --- a/Apps/MetaWatchShared/docker-compose.yml +++ b/Apps/MetaWatchShared/docker-compose.yml @@ -86,6 +86,10 @@ services: # The standalone MetaCore app. Port 80 is its hash-lock/appshield-free # backend port on `pcs` — NOT 9000 (that is the self-contained variant's # private core). + # Distinct from metaread-search on the same core, so the two discovery + # peers do not register under one name. + SERVICE_NAME: "metawatch-search" + META_SEARCH_PUBLIC_URL: "https://metawatchsearch-${APP_DOMAIN}" META_CORE_URL: "http://metacore-app:80" BASE_URL: "http://metawatch-search:3000" INTERNAL_BASE_URL: "http://metawatch-search:3000" @@ -111,7 +115,7 @@ services: # owns the caddy_* labels directly and Caddy proxies straight to :3000. # --------------------------------------------------------------------------- metawatch-app: - image: ghcr.io/worph/meta-watch:1.0.67 + image: ghcr.io/worph/meta-watch:1.0.68 container_name: metawatch-app hostname: metawatch-app restart: unless-stopped diff --git a/README.md b/README.md index f823944..bc815d2 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ https://github.com/Worph/MetaAppStore/archive/refs/heads/main.zip | Core | `MetaCore` (leader election, Redis, WebDAV) | | Library | `MetaSort`, `MetaFuse`, `MetaStremio` | | Network | `MetaShare` (content transport), `MetaGateway`, `MetaGatewayShared` | -| Clients | `MetaWatch`, `MetaWatchShared`, `MetaRead` | +| Clients | `MetaWatch`, `MetaWatchShared`, `MetaRead`, `MetaReadShared` | | Feeders | `MetaFeederAniList`, `MetaFeederArxiv`, `MetaFeederEuropePMC`, `MetaFeederGiphy`, `MetaFeederGutenberg`, `MetaFeederInternetArchive`, `MetaFeederJamendo`, `MetaFeederMusicBrainz`, `MetaFeederOpenSubtitles`, `MetaFeederProwlarr`, `MetaFeederSciHub`, `MetaFeederSuwayomi`, `MetaFeederTMDB`, `MetaFeederTribler`, `MetaFeederUsenet`, `MetaFeederWikiCommons`, `MetaFeederYouTube` | | Plugins | `MetaPluginAnimeDetector`, `MetaPluginFFmpeg`, `MetaPluginFileInfo`, `MetaPluginFilenameParser`, `MetaPluginFullHash`, `MetaPluginJellyfinNFO`, `MetaPluginLanguage`, `MetaPluginOpenSubtitles`, `MetaPluginStillExtractor`, `MetaPluginSubtitle`, `MetaPluginSubtitleExtractor`, `MetaPluginTMDB` — headless enrichment workers, one per `metamesh-plugin-*` image |