MetaShare 2.0.0: hull + meta-transport torrent/nzb/ipfs plugins

Apps/MetaShare/docker-compose.yml

@@ -1,122 +1,59 @@
 name: metashare
 
 services:
-  # Backend service. Reachable on the pcs network as `metashare-app`.
+  # Backend service (the hull). Reachable on the pcs network as `metashare-app`.
   # Public ingress goes through the `metashare` hash-lock proxy below.
   #
-  # Unlike the other meta apps, this container is a self-contained Rust
+  # meta-share is the protocol-agnostic hull since 2.0.0: HTTP API + dashboard,
-  # binary serving Axum on :3000 directly (no nginx wrapper, no supervisord).
+  # material index, seed manifest, eviction, playback focus, meta-core. The
-  # Ports 3000 and 4001 are split:
+  # byte-moving protocols run as the three transport plugins below (the
+  # `meta_feeder_sdk::transport` contract) and share its data volume:
   #  - 3000  HTTP API + dashboard (proxied through the hash-lock + Caddy)
-  #  - 4001  libp2p TCP listener — must be reachable on the public IP so
+  #  - 3001  plugin callback API — internal only, never published
-  #          other archivists' peers can dial in via kad-DHT provider records.
   metashare-app:
-    # 1.0.13 = nocopy library seeding (meta-core files seeded as filestore refs,
+    image: ghcr.io/worph/meta-share:2.0.0
-    # no second on-disk copy) + core/cache source surfacing in the dashboard, on
-    # top of the 1.0.8 floor this pin still requires: rsa-enabled libp2p
-    # (required to join the public IPFS DHT — the bootstrap.libp2p.io nodes are
-    # RSA-keyed; without `rsa` the noise handshake fails and this peer never
-    # actually reaches the DHT) + PUBLIC_ADDR support so remote archivists can
-    # dial this seeder.
-    image: ghcr.io/worph/meta-share:1.0.36
     container_name: metashare-app
-    # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM
+    # Hard RAM cap, swap disabled (memswap_limit == mem_limit) → OOM instead of
-    # instead of swap. Mirrors the dev sub-stack limits.
+    # swap. The swarm, blockstore and torrent session moved to the plugins, so
-    mem_limit: 3g
+    # the hull needs far less than the 3g the monolith did.
-    memswap_limit: 3g
+    mem_limit: 1g
+    memswap_limit: 1g
     hostname: metashare-app
     restart: unless-stopped
     cpu_shares: 20
-
+    # Room for a clean SIGTERM shutdown: redb's `material.redb` writes its
-    # Give meta-share room to shut down cleanly on SIGTERM (docker stop /
+    # allocator state on Drop, or the next open runs a full repair scan.
-    # `up -d` recreate) before docker escalates to SIGKILL. The binary catches
-    # SIGTERM, drains in-flight HTTP briefly, then unwinds so redb's Database
-    # Drop writes its allocator-state table — without which the next open runs a
-    # full multi-scan repair of blocks.redb (minutes of silent CPU before the
-    # port binds, proportional to blockstore size). 3s drain + the allocator
-    # write fits the 10s default, but a large store gets headroom here.
     stop_grace_period: 30s
 
     expose:
       - 3000   # HTTP API (Axum)
-
+      - 3001   # plugin callback API (internal)
-    # libp2p TCP must be host-published. Caddy can't proxy it (not HTTP)
-    # and other peers' kad-DHT lookups dial the public multiaddr directly.
-    ports:
-      - "4001:4001"
 
     volumes:
-      # Shared infrastructure state (leader election, Redis dump, service discovery)
+      # Operator settings (META_SHARE_CONFIG_DIR): settings.json holds the NNTP
-      # Owned by MetaCore; mounted rw here so meta-share registers under services/.
+      # credentials and network endpoints. MUST be persisted (else lost on every
-      # No /meta-core mount: this service locates meta-core over UDP
+      # image bump). The nzb and ipfs plugins mount it read-only.
-      # (meta-discovery v1) instead of reading kv-leader.info off a shared
-      # volume. See docs/project-architecture/service-discovery.md.
-      # Operator settings (META_SHARE_CONFIG_DIR, default /config): settings.json
-      # holds the NNTP credentials + per-indexer API keys entered in the dashboard.
-      # MUST be persisted — without this mount they live on the container's
-      # writable layer and are lost on every image bump / recreate, forcing the
-      # operator to re-enter their credentials after each update.
       - ${DATA_ROOT:-/DATA}/AppData/metashare/config:/config:rw
-      # Seeded bytes + the seed index (META_SHARE_DATA, default /data/meta-share):
+      # The shared data volume (META_SHARE_DATA): cache/, tmp/, seeds.json and
-      # the redb IPFS blockstore, seed manifest, librqbit sparse torrent files
+      # material.redb are the hull's; bt-temp/ (torrent), tmp/<cid> jobs (nzb)
-      # (bt-temp/), the NZB cache and the ipfs-seed cursor. Recomputable, but a
+      # and ipfs/blocks.redb (ipfs) are the plugins' corners of the SAME mount —
-      # peer whose job is "fetch implies seed" comes back seeding NOTHING after a
+      # one mount, so the tmp/ → cache/ rename(2) stays atomic. Layout unchanged
-      # recreate without this, and re-downloads everything from a cold swarm.
+      # from the monolith: existing caches carry over.
       - ${DATA_ROOT:-/DATA}/AppData/metashare/data:/data:rw
 
-    # Note: no /files mounts — meta-share resolves file paths through MetaCore's
-    # WebDAV URL (Architecture V3). META_SHARE_FILES_PATH is only used as the
-    # prefix for translating absolute filePaths back to WebDAV-relative paths.
-
     environment:
-      - P2P_LISTEN=/ip4/0.0.0.0/tcp/4001
+      - META_SHARE_PLUGIN_TORRENT_URL=http://metashare-torrent:3000
-      # Advertise the routable address (published host port 4001) so kad
+      - META_SHARE_PLUGIN_NZB_URL=http://metashare-nzb:3000
-      # provider records + identify carry a dialable multiaddr — otherwise,
+      - META_SHARE_PLUGIN_IPFS_URL=http://metashare-ipfs:3000
-      # behind Docker NAT, the swarm only sees container-internal addresses
+      # Service registration / neighbour announce.
-      # and remote archivists can't dial in to fetch seeded blocks.
-      # ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP}: on an IPv6-primary PCS
-      # APP_PUBLIC_IP holds the v6 address, and `/ip4/<v6>/tcp/...` is not a
-      # parseable multiaddr — meta-share REFUSES TO START rather than announce
-      # a lie (config.rs: "PUBLIC_ADDR is not a valid multiaddr").
-      - PUBLIC_ADDR=/ip4/${APP_PUBLIC_IPV4}/tcp/4001
-      # Public IPFS DHT bootstraps by explicit /dns4/.../tcp/4001. The default
-      # nested-dnsaddr form resolves to wss-only in some container resolvers
-      # (no ws transport → no bootstrap); these direct addresses avoid that.
-      # Includes mars.i.ipfs.io — the only surviving legacy *.i.ipfs.io
-      # bootstrapper. Resilience: the four IPFS-Foundation bootstrappers can refuse
-      # an individual IP, and when they do a peer cannot join the DHT AT ALL.
-      # Observed on watch.nsl.sh 2026-09-01: 2 connections with the four alone,
-      # 126 once a reachable fifth was added.
-      - KAD_BOOTSTRAP_PEERS=/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ
-
-      # Service registration into /meta-core/services/
       - BASE_URL=https://metashare-$APP_DOMAIN
-
+      # meta-core ingestion — the local archivist's library.
-      # meta-core ingestion — the local archivist's library. This peer pulls
-      # records from MetaCore and announces them on the Kamilata swarm.
       - META_CORE_URL=http://metacore-app:80
-
+      # Inter-peer fetch URL, announced in identify by the ipfs plugin. Same as
-      # Inter-peer fetch URL — where another archivist's meta-share reaches
+      # BASE_URL in prod (other peers fetch via the public Caddy URL).
-      # this peer's HTTP API. Same as BASE_URL for prod (other peers fetch
-      # via the public Caddy URL, which routes back through this hash-lock
-      # proxy with /api/peer/* and /api/file/* on the bypass list).
-      #
-      # Renamed at the 1.0.24 bump (2026-08-04), as the old note here said to do:
-      # META_SHARE_PEER_API_URL was the pre-1.0.8 spelling. watch.nsl.sh runs
-      # 1.0.24 with INTERNAL_BASE_URL and registers its peer_url correctly, so the
-      # deprecated name is no longer needed. Do NOT set both.
       - INTERNAL_BASE_URL=https://metashare-$APP_DOMAIN
-
-      # NOTE: KAD_NAMESPACE, TARGET_PEER_COUNT, ENABLE_MDNS, META_CORE_POLL_SECS,
-      # META_CORE_MAX_RECORDS, HTTP_LISTEN, SERVICE_NAME and RUST_LOG were all
-      # dropped — each merely restated the code default in meta-share:1.0.8.
-      # Set KAD_NAMESPACE to a non-default value to isolate a private cohort
-      # (every peer in the cohort must agree on it).
-
       - TZ=$TZ
 
-    # Runs as root because the meta-core volume is provisioned mode 0755 by
-    # MetaCore (root). Same convention as the other meta apps.
     user: 0:0
 
     healthcheck:
@@ -129,6 +66,118 @@ services:
     networks:
       - pcs
 
+  # BitTorrent tier (librqbit, streaming-only fork). Its peer listener must be
+  # host-published so the swarm can dial in (idle seeding without a dial storm).
+  metashare-torrent:
+    image: ghcr.io/worph/meta-transport-torrent:1.0.0
+    container_name: metashare-torrent
+    hostname: metashare-torrent
+    restart: unless-stopped
+    mem_limit: 1g
+    memswap_limit: 1g
+    cpu_shares: 20
+    stop_grace_period: 30s
+    expose:
+      - 3000
+    ports:
+      - "4240:4240"
+    volumes:
+      - ${DATA_ROOT:-/DATA}/AppData/metashare/data:/data:rw
+    environment:
+      - META_SHARE_HULL_URL=http://metashare-app:3001
+      - TZ=$TZ
+    user: 0:0
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:3000/health"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 30s
+    networks:
+      - pcs
+
+  # Usenet tier (NNTP, yEnc, par2/RAR). Reads the NNTP credentials from the
+  # hull's settings.json; the hull restarts it when they change.
+  metashare-nzb:
+    image: ghcr.io/worph/meta-transport-nzb:1.0.0
+    container_name: metashare-nzb
+    hostname: metashare-nzb
+    restart: unless-stopped
+    mem_limit: 1g
+    memswap_limit: 1g
+    cpu_shares: 20
+    stop_grace_period: 30s
+    expose:
+      - 3000
+    volumes:
+      - ${DATA_ROOT:-/DATA}/AppData/metashare/config:/config:ro
+      - ${DATA_ROOT:-/DATA}/AppData/metashare/data:/data:rw
+    environment:
+      - META_SHARE_HULL_URL=http://metashare-app:3001
+      - TZ=$TZ
+    user: 0:0
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:3000/health"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 30s
+    networks:
+      - pcs
+
+  # IPFS tier: the libp2p host (identify/kad/mdns/bitswap), the blockstore and
+  # bitswap ingress. libp2p TCP must be host-published — Caddy can't proxy it and
+  # other peers' kad lookups dial the public multiaddr directly.
+  metashare-ipfs:
+    image: ghcr.io/worph/meta-transport-ipfs:1.0.0
+    container_name: metashare-ipfs
+    hostname: metashare-ipfs
+    restart: unless-stopped
+    # The blockstore's redb page cache + the swarm live here now.
+    mem_limit: 2g
+    memswap_limit: 2g
+    cpu_shares: 20
+    stop_grace_period: 30s
+    expose:
+      - 3000
+    ports:
+      - "4001:4001"
+    volumes:
+      - ${DATA_ROOT:-/DATA}/AppData/metashare/config:/config:ro
+      - ${DATA_ROOT:-/DATA}/AppData/metashare/data:/data:rw
+    environment:
+      - META_SHARE_HULL_URL=http://metashare-app:3001
+      - P2P_LISTEN=/ip4/0.0.0.0/tcp/4001
+      # Advertise the routable address so kad provider records + identify carry
+      # a dialable multiaddr. ${APP_PUBLIC_IPV4}, NOT ${APP_PUBLIC_IP}: on an
+      # IPv6-primary PCS the latter is v6 and `/ip4/<v6>/…` won't parse (the
+      # plugin refuses to start rather than announce a lie).
+      - PUBLIC_ADDR=/ip4/${APP_PUBLIC_IPV4}/tcp/4001
+      # Public IPFS DHT bootstraps by explicit /dns4/…/tcp/4001 (the nested
+      # dnsaddr form resolves to wss-only in some container resolvers), plus
+      # mars.i.ipfs.io — the four IPFS-Foundation bootstrappers can refuse an
+      # individual IP (watch.nsl.sh 2026-09-01: 2 connections → 126 with a fifth).
+      - KAD_BOOTSTRAP_PEERS=/dns4/sg1.bootstrap.libp2p.io/tcp/4001/p2p/QmcZf59bWwK5XFi76CZX8cbJ4BhTzzA3gU1ZjYZcYW3dwt,/dns4/sv15.bootstrap.libp2p.io/tcp/4001/p2p/QmNnooDu7bfjPFoTZYxMNLWUQJyrVwtbZg5gBMjTezGAJN,/dns4/ny5.bootstrap.libp2p.io/tcp/4001/p2p/QmQCU2EcMqAqQPR2i9bChDtGNJchTbq5TbXJJ16u19uLTa,/dns4/am6.bootstrap.libp2p.io/tcp/4001/p2p/QmbLHAnMoJPWSCR5Zhtx6BHJX9KiKNN6tpvbUcqanj75Nb,/dns4/mars.i.ipfs.io/tcp/4001/p2p/QmaCpDMGvV2BGHeYERUEnRQAwe3N8SzbUtfsmvsqQLuvuJ
+      # Seed fallbacks when the hull's settings.json doesn't exist yet.
+      - META_CORE_URL=http://metacore-app:80
+      - INTERNAL_BASE_URL=https://metashare-$APP_DOMAIN
+      - TZ=$TZ
+    user: 0:0
+    # The hull migrates its material index out of ipfs/blocks.redb on its first
+    # 2.x boot; redb is single-process, so the plugin waits for it.
+    depends_on:
+      metashare-app:
+        condition: service_healthy
+    healthcheck:
+      test: ["CMD", "curl", "-sf", "http://localhost:3000/health"]
+      interval: 10s
+      timeout: 3s
+      retries: 10
+      start_period: 30s
+    networks:
+      - pcs
+
+
   # OIDC perimeter for the MetaShare dashboard. Self-registers as an OIDC client
   # with the platform's auth-registrar on first login (no secrets to manage).
   #