From 14d9ed1866fe25919aeb16dcb41a875047b8b9a0 Mon Sep 17 00:00:00 2001 From: Worph Date: Sun, 4 Oct 2026 13:24:05 +0000 Subject: [PATCH] MetaWatch/MetaGateway/MetaRead: meta-share 1.0.36 monolith -> 2.2.1 hull + bundled ipfs plugin (torrent/nzb off); MetaShare 2.2.1; meta-gateway 1.0.40; beacon v2 comments --- Apps/MetaFuse/docker-compose.yml | 4 +- Apps/MetaGateway/docker-compose.yml | 129 +++++++++++++--------- Apps/MetaGatewayShared/docker-compose.yml | 2 +- Apps/MetaRead/docker-compose.yml | 98 ++++++++++++---- Apps/MetaReadShared/docker-compose.yml | 4 +- Apps/MetaShare/docker-compose.yml | 2 +- Apps/MetaSort/docker-compose.yml | 4 +- Apps/MetaStremio/docker-compose.yml | 4 +- Apps/MetaWatch/docker-compose.yml | 124 +++++++++++++-------- Apps/MetaWatchShared/docker-compose.yml | 4 +- 10 files changed, 242 insertions(+), 133 deletions(-) diff --git a/Apps/MetaFuse/docker-compose.yml b/Apps/MetaFuse/docker-compose.yml index 7909f41..a1639df 100644 --- a/Apps/MetaFuse/docker-compose.yml +++ b/Apps/MetaFuse/docker-compose.yml @@ -30,8 +30,8 @@ services: # Shared infrastructure state (leader election, Redis dump, service discovery) # Owned by MetaCore; mounted rw here for service registration + leader reads. # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. # FUSE virtual filesystem mount point - ${DATA_ROOT:-/DATA}/MetaFuse/Library:/mnt/virtual:shared diff --git a/Apps/MetaGateway/docker-compose.yml b/Apps/MetaGateway/docker-compose.yml index 5a63fe1..a3719d8 100644 --- a/Apps/MetaGateway/docker-compose.yml +++ b/Apps/MetaGateway/docker-compose.yml @@ -66,72 +66,50 @@ services: - pcs # =========================================================================== - # metagateway-share — meta-share INGESTER role (META_CORE_URL set). Polls this - # gateway's meta-core, announces resolved records on Kamilata + rebuilds its - # Bloom filter, and seeds blobs over IPFS/bitswap so federated MetaWatch - # queries reach the gateway's records. Public libp2p on host port 4005 - # (the gateway-app owns 4002 for discovery). + # metagateway-share (+ metagateway-share-ipfs) — meta-share INGESTER role + # (META_CORE_URL set): the hull with one ipfs plugin. Polls this gateway's + # meta-core and seeds blobs over IPFS/bitswap so remote clients can fetch the + # gateway's records. No local NZB tier is bundled. Public libp2p on host port + # 4005 (the gateway-app owns 4002 for discovery). # =========================================================================== metagateway-share: - # 1.0.8 = rsa-enabled libp2p (joins the public IPFS DHT — see metagateway-app) - # + P2P_ANNOUNCE support so remote peers can dial this bitswap seeder. - image: ghcr.io/worph/meta-share:1.0.36 + # The meta-share HULL (2.x): HTTP API, material index, seed manifest, + # eviction, playback focus, meta-core ingest. Every byte-moving protocol is + # a transport plugin; this bundle ships ONE — metagateway-share-ipfs below — which + # makes it the INGESTER: seeds this gateway's resolved blobs over IPFS/bitswap so remote clients can fetch them. Torrent and local NZB tiers are not bundled + # (META_SHARE_PLUGIN_{TORRENT,NZB}_URL=off keeps the hull from seeding the + # MetaShare app's `metashare-` hostnames, which on a shared `pcs` + # would be ANOTHER hull's plugins writing to another /data). + # - 3000 HTTP API - 3001 plugin callback API (internal only) + image: ghcr.io/worph/meta-share:2.2.1 container_name: metagateway-share - # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM - # instead of swap. Mirrors the dev sub-stack limits. - mem_limit: 3g - memswap_limit: 3g + # The swarm and blockstore moved to the ipfs plugin, so the hull needs far + # less than the monolith did. + mem_limit: 1g + memswap_limit: 1g hostname: metagateway-share restart: unless-stopped user: "0:0" cpu_shares: 30 + # material.redb writes its allocator state on Drop; give SIGTERM room. + stop_grace_period: 30s expose: - 3000 - # Own public libp2p port (4005). The gateway-app owns 4002 (discovery); - # this ingester's bitswap tier serves the gateway's seeded bytes to remote - # peers over the public DHT, so it needs its own dialable host port. - ports: - - "4005:4001" + - 3001 volumes: - # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. - # Operator settings (settings.json — NNTP creds + indexer API keys). This - # is the CREDENTIALED peer of the pair: it materialises NZB postings for - # credential-less clients. Losing settings.json on an update means the - # gateway silently stops serving nzb-release bytes until re-entered. + # settings.json — network endpoints and the plugin list. - ${DATA_ROOT:-/DATA}/AppData/metagateway/share-config:/config:rw - # Blockstore + seed manifest + NZB cache (META_SHARE_DATA=/data/meta-share). - # Without it the gateway's seeded corpus is wiped on every recreate. + # The shared data volume: cache/, tmp/, seeds.json, material.redb are the + # hull's; ipfs/blocks.redb is the ipfs plugin's corner of the SAME folder + # (layout unchanged from 1.0.x, so an existing cache carries over). - ${DATA_ROOT:-/DATA}/AppData/metagateway/share-data:/data:rw environment: - P2P_LISTEN: "/ip4/0.0.0.0/tcp/4001" - # Advertise the routable address (published host port 4005) so kad - # provider records + identify carry a dialable multiaddr for remote - # bitswap fetches. Behind Docker NAT the swarm otherwise only sees - # container-internal addresses. - # ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP} — see the note on the - # gateway-app below: APP_PUBLIC_IP is the v6 address on an - # IPv6-primary PCS, and `/ip4//tcp/...` fails to parse. - PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4005" - # Public IPFS DHT bootstraps by explicit /dns4/.../tcp/4001 (the nested - # dnsaddr form resolves to wss-only in some container resolvers). rsa - # feature required to complete the noise handshake with these RSA-keyed - # (Qm...) bootstrap nodes. - # Includes mars.i.ipfs.io — the only surviving legacy *.i.ipfs.io - # bootstrapper (neptune/pluto/uranus/earth/saturn/venus are all NXDOMAIN). - # It is here for resilience: the four IPFS-Foundation bootstrappers can - # refuse an individual IP, and when they do a peer cannot join the DHT AT - # ALL — no provider record, no discovery, no gateway fan-out. Observed on - # watch.nsl.sh 2026-09-01: 2 connections with the four alone, 126 once a - # reachable fifth was added. Costs nothing when the four work. - KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" META_CORE_URL: "http://metagateway-core:9000" BASE_URL: "http://metagateway-share:3000" - # Renamed at the 1.0.26 bump (2026-08-26), as the old note here said to do: - # META_SHARE_PEER_API_URL was the pre-1.0.8 spelling and now only logs a - # deprecation warning. Do NOT set both. INTERNAL_BASE_URL: "http://metagateway-share:3000" + META_SHARE_PLUGIN_IPFS_URL: "http://metagateway-share-ipfs:3000" + META_SHARE_PLUGIN_TORRENT_URL: "off" + META_SHARE_PLUGIN_NZB_URL: "off" TZ: "$TZ" depends_on: metagateway-core: @@ -141,7 +119,51 @@ services: interval: 5s timeout: 3s retries: 10 - start_period: 5s + start_period: 10s + networks: + - metagateway + - pcs + + metagateway-share-ipfs: + # The hull's only transport plugin: the libp2p swarm (kad-DHT, identify, + # mDNS, bitswap), the blockstore, gateway discovery and the peer directory + # the hull uses to forward Usenet cids. Waits up to 60 s for the hull's + # internal /health before opening blocks.redb, so no depends_on. + image: ghcr.io/worph/meta-transport-ipfs:1.2.0 + container_name: metagateway-share-ipfs + hostname: metagateway-share-ipfs + restart: unless-stopped + user: "0:0" + cpu_shares: 30 + mem_limit: 2g + memswap_limit: 2g + stop_grace_period: 30s + expose: + - 3000 + # libp2p TCP must be host-published (kad lookups dial it directly; Caddy + # cannot proxy it). Same host port the 1.0.x monolith used. + ports: + - "4005:4001" + volumes: + - ${DATA_ROOT:-/DATA}/AppData/metagateway/share-data:/data:rw + - ${DATA_ROOT:-/DATA}/AppData/metagateway/share-ipfs-state:/state:rw + environment: + META_SHARE_HULL_URL: "http://metagateway-share:3001" + # Beacon v2: only this bundle's hull may list this plugin. + BEACON_BINDS: "metagateway-share" + # ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP}: on an IPv6-primary PCS the + # latter is a v6 address, and /ip4//… is not a parseable multiaddr. + PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4005" + # Public IPFS DHT bootstraps by explicit /dns4/…/tcp/4001, plus + # mars.i.ipfs.io in case the foundation four refuse this IP. + KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + TZ: "$TZ" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/health"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s networks: - metagateway - pcs @@ -159,7 +181,7 @@ services: # P2P listen/announce so it's discoverable + dialable off-host + the # 2026-07 discovery/filters fixes that make the meta-search fan-out # actually route `fileType:video` queries here. - image: ghcr.io/worph/meta-gateway:1.0.39 + image: ghcr.io/worph/meta-gateway:1.0.40 container_name: metagateway-app # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM # instead of swap. Mirrors the dev sub-stack limits. @@ -596,6 +618,9 @@ x-compose-app: - path: /DATA/AppData/metagateway/share-data user: $PUID group: $PGID + - path: /DATA/AppData/metagateway/share-ipfs-state + user: $PUID + group: $PGID # Same string as the caddy_0 label, so the tile link and the route agree. webui-host: metagateway-${domain} webui-path: "/" diff --git a/Apps/MetaGatewayShared/docker-compose.yml b/Apps/MetaGatewayShared/docker-compose.yml index 9e5376a..b86f6f4 100644 --- a/Apps/MetaGatewayShared/docker-compose.yml +++ b/Apps/MetaGatewayShared/docker-compose.yml @@ -33,7 +33,7 @@ services: # co-located metashare-app then seeds the resolved CIDs. # --------------------------------------------------------------------------- metagateway-app: - image: ghcr.io/worph/meta-gateway:1.0.39 + image: ghcr.io/worph/meta-gateway:1.0.40 container_name: metagateway-app hostname: metagateway-app restart: unless-stopped diff --git a/Apps/MetaRead/docker-compose.yml b/Apps/MetaRead/docker-compose.yml index fce72ab..3a2a89d 100644 --- a/Apps/MetaRead/docker-compose.yml +++ b/Apps/MetaRead/docker-compose.yml @@ -100,8 +100,8 @@ services: - "4006:4001" volumes: # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. - ${DATA_ROOT:-/DATA}/AppData/metaread/search-data:/data:rw environment: P2P_LISTEN: "/ip4/0.0.0.0/tcp/4001" @@ -136,9 +136,9 @@ services: - pcs # --------------------------------------------------------------------------- - # metaread-share — PURE CONTENT TRANSPORT. Fetches and serves bytes by CID - # over bitswap + a BitTorrent tier, and seeds whatever it fetches. Public - # libp2p on host port 4005. + # metaread-share (+ metaread-share-ipfs) — CONTENT TRANSPORT: the meta-share + # hull with one ipfs plugin. Fetches and serves bytes by CID over bitswap, and + # seeds whatever it fetches. Public libp2p on host port 4005. # # ⚠ It must be reachable by whatever holds the pages. A comic chapter from the # aggregator tier is a list of per-page URL locators redeemed HERE, so a @@ -146,36 +146,43 @@ services: # that lists correctly and will not open. # --------------------------------------------------------------------------- metaread-share: - image: ghcr.io/worph/meta-share:1.0.36 + # The meta-share HULL (2.x): HTTP API, material index, seed manifest, + # eviction, playback focus, meta-core ingest. Every byte-moving protocol is + # a transport plugin; this bundle ships ONE — metaread-share-ipfs below — which + # makes it the read cache: page bytes over bitswap + re-seed (a chapter is tens of MB, never a film). Torrent and local NZB tiers are not bundled + # (META_SHARE_PLUGIN_{TORRENT,NZB}_URL=off keeps the hull from seeding the + # MetaShare app's `metashare-` hostnames, which on a shared `pcs` + # would be ANOTHER hull's plugins writing to another /data). + # - 3000 HTTP API - 3001 plugin callback API (internal only) + image: ghcr.io/worph/meta-share:2.2.1 container_name: metaread-share - # Smaller than MetaWatch's 3 GB on purpose: a chapter is tens of megabytes - # where a film is gigabytes, and this peer never holds a whole video. - mem_limit: 1500m - memswap_limit: 1500m + # The swarm and blockstore moved to the ipfs plugin, so the hull needs far + # less than the monolith did. + mem_limit: 1g + memswap_limit: 1g hostname: metaread-share restart: unless-stopped user: "0:0" cpu_shares: 30 + # material.redb writes its allocator state on Drop; give SIGTERM room. + stop_grace_period: 30s expose: - 3000 - ports: - - "4005:4001" + - 3001 volumes: - # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # settings.json — network endpoints and the plugin list. - ${DATA_ROOT:-/DATA}/AppData/metaread/metashare-config:/config:rw - # The read cache: redb blockstore, seed manifest, sparse files. Without - # this mount every recreate re-downloads from a cold swarm and the peer - # seeds nothing back. + # The shared data volume: cache/, tmp/, seeds.json, material.redb are the + # hull's; ipfs/blocks.redb is the ipfs plugin's corner of the SAME folder + # (layout unchanged from 1.0.x, so an existing cache carries over). - ${DATA_ROOT:-/DATA}/AppData/metaread/share-data:/data:rw environment: - P2P_LISTEN: "/ip4/0.0.0.0/tcp/4001" - PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4005" - KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" META_CORE_URL: "http://metaread-core:9000" BASE_URL: "http://metaread-share:3000" INTERNAL_BASE_URL: "http://metaread-share:3000" + META_SHARE_PLUGIN_IPFS_URL: "http://metaread-share-ipfs:3000" + META_SHARE_PLUGIN_TORRENT_URL: "off" + META_SHARE_PLUGIN_NZB_URL: "off" TZ: "$TZ" depends_on: metaread-core: @@ -185,7 +192,51 @@ services: interval: 5s timeout: 3s retries: 10 - start_period: 5s + start_period: 10s + networks: + - metaread + - pcs + + metaread-share-ipfs: + # The hull's only transport plugin: the libp2p swarm (kad-DHT, identify, + # mDNS, bitswap), the blockstore, gateway discovery and the peer directory + # the hull uses to forward Usenet cids. Waits up to 60 s for the hull's + # internal /health before opening blocks.redb, so no depends_on. + image: ghcr.io/worph/meta-transport-ipfs:1.2.0 + container_name: metaread-share-ipfs + hostname: metaread-share-ipfs + restart: unless-stopped + user: "0:0" + cpu_shares: 30 + mem_limit: 1500m + memswap_limit: 1500m + stop_grace_period: 30s + expose: + - 3000 + # libp2p TCP must be host-published (kad lookups dial it directly; Caddy + # cannot proxy it). Same host port the 1.0.x monolith used. + ports: + - "4005:4001" + volumes: + - ${DATA_ROOT:-/DATA}/AppData/metaread/share-data:/data:rw + - ${DATA_ROOT:-/DATA}/AppData/metaread/share-ipfs-state:/state:rw + environment: + META_SHARE_HULL_URL: "http://metaread-share:3001" + # Beacon v2: only this bundle's hull may list this plugin. + BEACON_BINDS: "metaread-share" + # ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP}: on an IPv6-primary PCS the + # latter is a v6 address, and /ip4//… is not a parseable multiaddr. + PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4005" + # Public IPFS DHT bootstraps by explicit /dns4/…/tcp/4001, plus + # mars.i.ipfs.io in case the foundation four refuse this IP. + KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + TZ: "$TZ" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/health"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s networks: - metaread - pcs @@ -476,6 +527,9 @@ x-compose-app: - path: /DATA/AppData/metaread/share-data user: $PUID group: $PGID + - path: /DATA/AppData/metaread/share-ipfs-state + user: $PUID + group: $PGID - path: /DATA/AppData/metaread/search-data user: $PUID group: $PGID diff --git a/Apps/MetaReadShared/docker-compose.yml b/Apps/MetaReadShared/docker-compose.yml index a14cfb0..626eb36 100644 --- a/Apps/MetaReadShared/docker-compose.yml +++ b/Apps/MetaReadShared/docker-compose.yml @@ -76,8 +76,8 @@ services: - "4006:4001" volumes: # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. - ${DATA_ROOT:-/DATA}/AppData/metaread/search-data:/data:rw environment: # Distinct from metawatch-search on the same core, so the two discovery diff --git a/Apps/MetaShare/docker-compose.yml b/Apps/MetaShare/docker-compose.yml index df3e6ef..a2739bf 100644 --- a/Apps/MetaShare/docker-compose.yml +++ b/Apps/MetaShare/docker-compose.yml @@ -13,7 +13,7 @@ services: # - 3000 HTTP API + dashboard (proxied through the hash-lock + Caddy) # - 3001 plugin callback API — internal only, never published metashare-app: - image: ghcr.io/worph/meta-share:2.2.0 + image: ghcr.io/worph/meta-share:2.2.1 container_name: metashare-app # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM instead of # swap. The swarm, blockstore and torrent session moved to the plugins, so diff --git a/Apps/MetaSort/docker-compose.yml b/Apps/MetaSort/docker-compose.yml index 6b37e3e..48d2df4 100644 --- a/Apps/MetaSort/docker-compose.yml +++ b/Apps/MetaSort/docker-compose.yml @@ -19,8 +19,8 @@ services: # Shared infrastructure state (leader election, Redis dump, service discovery) # Owned by MetaCore; mounted rw here for service registration + leader reads. # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. # Application cache storage - ${DATA_ROOT:-/DATA}/AppData/metasort/cache:/data/cache diff --git a/Apps/MetaStremio/docker-compose.yml b/Apps/MetaStremio/docker-compose.yml index b587d2f..ef5ac8d 100644 --- a/Apps/MetaStremio/docker-compose.yml +++ b/Apps/MetaStremio/docker-compose.yml @@ -19,8 +19,8 @@ services: # Shared infrastructure state (leader election, Redis dump, service discovery) # Owned by MetaCore; mounted rw here for service registration + leader reads. # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. # Transcoding cache - ${DATA_ROOT:-/DATA}/AppData/metastremio/cache:/data/cache diff --git a/Apps/MetaWatch/docker-compose.yml b/Apps/MetaWatch/docker-compose.yml index 76eaffc..35bd3ff 100644 --- a/Apps/MetaWatch/docker-compose.yml +++ b/Apps/MetaWatch/docker-compose.yml @@ -89,8 +89,8 @@ services: - "4004:4001" volumes: # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. # META_SHARE_DATA (default /data/meta-share): prefs.json (language prefs) # + the redb blockstore holding poster/thumbnail blocks recovered off the # swarm. Without this mount both are on the container's writable layer and @@ -143,66 +143,49 @@ services: - pcs # --------------------------------------------------------------------------- - # metawatch-share — pure content TRANSPORT peer. Byte Range proxy for - # playback/probe/warm/playback-focus over bitswap + a BitTorrent tier; seeds - # whatever it fetches. Public libp2p on host port 4003. + # metawatch-share (+ metawatch-share-ipfs) — content TRANSPORT peer: the + # meta-share hull with one ipfs plugin. Byte Range proxy for playback/probe/ + # warm/playback-focus over bitswap; seeds whatever it fetches; Usenet cids are + # forwarded to a credentialed gateway. Public libp2p on host port 4003. # --------------------------------------------------------------------------- metawatch-share: - # 1.0.13 = nocopy library seeding + core/cache source surfacing, on top of - # the 1.0.8 floor: rsa-enabled libp2p (joins the public IPFS DHT) + - # P2P_ANNOUNCE so remote peers can dial this transport peer to fetch the - # bytes it seeds (fetch-implies-seed). Without rsa it never reaches the DHT. - image: ghcr.io/worph/meta-share:1.0.36 + # The meta-share HULL (2.x): HTTP API, material index, seed manifest, + # eviction, playback focus, meta-core ingest. Every byte-moving protocol is + # a transport plugin; this bundle ships ONE — metawatch-share-ipfs below — which + # makes it the playback cache: bitswap fetch + re-seed, and Usenet cids forwarded to a credentialed gateway. Torrent and local NZB tiers are not bundled + # (META_SHARE_PLUGIN_{TORRENT,NZB}_URL=off keeps the hull from seeding the + # MetaShare app's `metashare-` hostnames, which on a shared `pcs` + # would be ANOTHER hull's plugins writing to another /data). + # - 3000 HTTP API - 3001 plugin callback API (internal only) + image: ghcr.io/worph/meta-share:2.2.1 container_name: metawatch-share - # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM - # instead of swap. Mirrors the dev sub-stack limits. - mem_limit: 3g - memswap_limit: 3g + # The swarm and blockstore moved to the ipfs plugin, so the hull needs far + # less than the monolith did. + mem_limit: 1g + memswap_limit: 1g hostname: metawatch-share restart: unless-stopped user: "0:0" cpu_shares: 30 + # material.redb writes its allocator state on Drop; give SIGTERM room. + stop_grace_period: 30s expose: - 3000 - ports: - - "4003:4001" + - 3001 volumes: - # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. - # Operator settings written by the dashboard Settings panel (settings.json, - # e.g. optional direct-NNTP/Usenet credentials). Persisted on disk so - # dashboard edits survive container recreation — this stack has no separate - # credentialed gateway peer to forward to, so config lives here. + # settings.json — network endpoints and the plugin list. - ${DATA_ROOT:-/DATA}/AppData/metawatch/metashare-config:/config:rw - # Seeded/fetched bytes (META_SHARE_DATA, default /data/meta-share): redb - # blockstore, seed manifest, librqbit sparse files, NZB cache. This is the - # playback cache — without the mount every recreate re-downloads the lot - # from a cold swarm, and the peer seeds nothing back. + # The shared data volume: cache/, tmp/, seeds.json, material.redb are the + # hull's; ipfs/blocks.redb is the ipfs plugin's corner of the SAME folder + # (layout unchanged from 1.0.x, so an existing cache carries over). - ${DATA_ROOT:-/DATA}/AppData/metawatch/share-data:/data:rw environment: - P2P_LISTEN: "/ip4/0.0.0.0/tcp/4001" - # Advertise the routable address (published host port 4003) so remote - # peers can dial this transport peer for the bytes it seeds. Behind - # Docker NAT the swarm otherwise only sees container-internal addresses. - # Use the IPV4 spelling, NOT the bare APP_PUBLIC_IP: on an IPv6-primary - # PCS that variable holds the v6 address, and `/ip4//tcp/...` is not a - # parseable multiaddr — meta-share REFUSES TO START rather than announce - # a lie (config.rs: "PUBLIC_ADDR is not a valid multiaddr"). - PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4003" - # Public IPFS DHT bootstraps by explicit /dns4/.../tcp/4001 (the nested - # dnsaddr default resolves to wss-only in some container resolvers). - # Includes mars.i.ipfs.io — the only surviving legacy *.i.ipfs.io - # bootstrapper (neptune/pluto/uranus/earth/saturn/venus are all NXDOMAIN). - # It is here for resilience: the four IPFS-Foundation bootstrappers can - # refuse an individual IP, and when they do a peer cannot join the DHT AT - # ALL — no provider record, no discovery, no gateway fan-out. Observed on - # watch.nsl.sh 2026-09-01: 2 connections with the four alone, 126 once a - # reachable fifth was added. Costs nothing when the four work. - KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" META_CORE_URL: "http://metawatch-core:9000" BASE_URL: "http://metawatch-share:3000" INTERNAL_BASE_URL: "http://metawatch-share:3000" + META_SHARE_PLUGIN_IPFS_URL: "http://metawatch-share-ipfs:3000" + META_SHARE_PLUGIN_TORRENT_URL: "off" + META_SHARE_PLUGIN_NZB_URL: "off" TZ: "$TZ" depends_on: metawatch-core: @@ -212,7 +195,51 @@ services: interval: 5s timeout: 3s retries: 10 - start_period: 5s + start_period: 10s + networks: + - metawatch + - pcs + + metawatch-share-ipfs: + # The hull's only transport plugin: the libp2p swarm (kad-DHT, identify, + # mDNS, bitswap), the blockstore, gateway discovery and the peer directory + # the hull uses to forward Usenet cids. Waits up to 60 s for the hull's + # internal /health before opening blocks.redb, so no depends_on. + image: ghcr.io/worph/meta-transport-ipfs:1.2.0 + container_name: metawatch-share-ipfs + hostname: metawatch-share-ipfs + restart: unless-stopped + user: "0:0" + cpu_shares: 30 + mem_limit: 2g + memswap_limit: 2g + stop_grace_period: 30s + expose: + - 3000 + # libp2p TCP must be host-published (kad lookups dial it directly; Caddy + # cannot proxy it). Same host port the 1.0.x monolith used. + ports: + - "4003:4001" + volumes: + - ${DATA_ROOT:-/DATA}/AppData/metawatch/share-data:/data:rw + - ${DATA_ROOT:-/DATA}/AppData/metawatch/share-ipfs-state:/state:rw + environment: + META_SHARE_HULL_URL: "http://metawatch-share:3001" + # Beacon v2: only this bundle's hull may list this plugin. + BEACON_BINDS: "metawatch-share" + # ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP}: on an IPv6-primary PCS the + # latter is a v6 address, and /ip4//… is not a parseable multiaddr. + PUBLIC_ADDR: "/ip4/${APP_PUBLIC_IPV4}/tcp/4003" + # Public IPFS DHT bootstraps by explicit /dns4/…/tcp/4001, plus + # mars.i.ipfs.io in case the foundation four refuse this IP. + KAD_BOOTSTRAP_PEERS: "/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ" + TZ: "$TZ" + healthcheck: + test: ["CMD", "curl", "-sf", "http://localhost:3000/health"] + interval: 10s + timeout: 3s + retries: 10 + start_period: 30s networks: - metawatch - pcs @@ -508,6 +535,9 @@ x-compose-app: - path: /DATA/AppData/metawatch/share-data user: $PUID group: $PGID + - path: /DATA/AppData/metawatch/share-ipfs-state + user: $PUID + group: $PGID - path: /DATA/AppData/metawatch/data user: $PUID group: $PGID diff --git a/Apps/MetaWatchShared/docker-compose.yml b/Apps/MetaWatchShared/docker-compose.yml index d613dc7..700adf8 100644 --- a/Apps/MetaWatchShared/docker-compose.yml +++ b/Apps/MetaWatchShared/docker-compose.yml @@ -64,8 +64,8 @@ services: # The STANDALONE MetaCore's directory: this peer reads the leader info / # identity keystore written by metacore-app. Not a private core. # No /meta-core mount: this service locates meta-core over UDP - # (meta-discovery v1) instead of reading kv-leader.info off a shared - # volume. See docs/project-architecture/service-discovery.md. + # (beacon v2) instead of reading kv-leader.info off a shared + # volume. See docs/project-architecture/beacon-v2.md. # META_SHARE_DATA (default /data/meta-share): prefs.json (language prefs) # + the redb blockstore holding poster/thumbnail blocks recovered off the # swarm. Without this mount both are on the container's writable layer and