4 Commits
Author SHA1 Message Date
Worph 862f73d26d cid rank remove
Build and Push Docker Image / build (push) Failing after 13s
2026-09-09 17:58:15 +02:00
Worph b855adb822 rework domain field
Build and Push Docker Image / build (push) Failing after 14s
2026-09-02 00:05:19 +02:00
Worph 9bf2e623f9 align to metadata keys doc 2026-08-30 19:53:03 +02:00
Worph 7bdb54f9a7 audio support widening 2026-08-21 18:23:34 +02:00
8 changed files with 289 additions and 186 deletions
@@ -1,9 +1,23 @@
/**
* The `fileType` vocabulary — see METADATA_KEYS.md §1.
*
* ⚠ INVARIANT: a value belongs here only if an `(extension, mimeType)` lookup
* can decide it on its own. Nothing that requires knowing what the content
* *is* goes in this field — that is `contentKind`'s job — and nothing that
* decides *which app gets the record* depends on it — that is `domain`'s job
* (METADATA_KEYS.md §14.16).
*
* `card` is the one deliberate exception: it describes no bytes at all and is
* therefore never produced by the tables here, only stamped by a card feeder.
*/
export type SimpleFileType =
'audio'
| 'video'
| 'image'
| 'document'
| 'archive'
| 'subtitle'
| 'torrent'
| 'card'
| 'other'
| 'undefined';
| 'undefined';
@@ -0,0 +1,125 @@
/**
* `contentKind` → `domain` / `workForm` — the routing tables from
* METADATA_KEYS.md §1.
*
* `domain` answers "which application is this record destined for?" and is the
* key every client filters its catalogue on — **one value per app**, no
* alternation. `workForm` answers the other question the old `film`/`tv` split
* was secretly encoding: is this work complete in itself, or one instalment of
* a run? It is cross-domain, so a comic issue is `serial` in `literature`
* exactly as an episode is `serial` in `screen`.
*
* Both are **stored, not derived on read**: that is what lets the `contentKind`
* vocabulary stay open, because a peer that has never heard of a kind still
* routes the record correctly without having to interpret the token.
*
* ⚠ This binding is **permanent and one-way**. Add kinds freely; never re-map
* an existing kind to a different domain. Because the value is stored, two
* peers on different versions that disagree about where a kind belongs produce
* a split-brain that redeploying cannot heal — the wrong domain is already
* written into records across the mesh. If a kind has to move, mint a new kind.
* The `film`+`tv` → `screen` merge (METADATA_KEYS.md §14.17) is the one
* deliberate exception, taken once and paid for with a corpus sweep.
*
* ⚠ Mirror of the Rust table in `meta-feeder-sdk::domain`. Edit both together.
*/
export type Domain =
'screen'
| 'music'
| 'literature'
| 'science';
/** Is the work complete in itself, or one instalment of a run? */
export type WorkForm = 'standalone' | 'serial';
/**
* The one-way binding. A kind that is absent here has no domain, which means
* the record is **not independently routable** — correct for sidecars
* (subtitles, artwork) and for un-anchored hits, a producer bug for anything
* else.
*
* ⚠ `pack` is deliberately absent: a season pack and an album release are the
* same structural thing, so the kind alone cannot tell `screen` from `music`.
* The writer stamps `pack`'s domain from its own context — see
* {@link domainForContentKind}'s contract.
*
* ⚠ `podcast` / `podcastEpisode` are absent too: the `spoken` domain was
* retired unused, and the change that first *writes* those kinds picks their
* domain.
*/
export const domainByContentKind: { [contentKind: string]: Domain } = {
// screen — films and serials alike; the split lives in workFormByContentKind
movie: 'screen',
series: 'screen',
episode: 'screen',
// music
track: 'music',
album: 'music',
artist: 'music',
musicVideo: 'music',
djMix: 'music',
liveSet: 'music',
// literature
book: 'literature',
// An audiobook's identity is the book — same author, same ISBN family,
// same series as the ebook — so meta-read can group the two editions on
// one page. The player does not decide the domain.
audiobook: 'literature',
comic: 'literature',
manga: 'literature',
magazine: 'literature',
// science
paper: 'science',
};
/**
* The `workForm` twin of {@link domainByContentKind}.
*
* ⚠ `pack` is absent for the same reason: a season pack is `serial`, an album
* release is `standalone`, and only the writer knows which.
*/
export const workFormByContentKind: { [contentKind: string]: WorkForm } = {
// screen
movie: 'standalone',
series: 'serial',
episode: 'serial',
// music — an album is a closed work, not an ongoing run, so every rung of
// the music ladder is standalone.
track: 'standalone',
album: 'standalone',
artist: 'standalone',
musicVideo: 'standalone',
djMix: 'standalone',
liveSet: 'standalone',
// literature — the split that proves workForm is not a screen-only axis.
book: 'standalone',
audiobook: 'standalone',
comic: 'serial',
manga: 'serial',
magazine: 'serial',
// science
paper: 'standalone',
};
/**
* Resolve the domain a `contentKind` belongs to.
*
* Returns `undefined` for `pack` (not derivable from the kind — the writer
* supplies it), for sidecar/format kinds that route nowhere, and for any kind
* this build has never heard of. A caller that already knows the domain from
* its own context must prefer that over this table.
*/
export function domainForContentKind(contentKind: string | undefined | null): Domain | undefined {
if (!contentKind) return undefined;
return domainByContentKind[contentKind.trim()];
}
/**
* Resolve the work form a `contentKind` sits on. Same contract as
* {@link domainForContentKind}: `undefined` means "not derivable", never a
* guess. Whoever writes `contentKind` writes both of these in the same breath.
*/
export function workFormForContentKind(contentKind: string | undefined | null): WorkForm | undefined {
if (!contentKind) return undefined;
return workFormByContentKind[contentKind.trim()];
}
@@ -1,5 +1,14 @@
import {SimpleFileType} from "../SimpleFileType.js";
/**
* Extension → `fileType`. Half of the canonical classification table
* documented in METADATA_KEYS.md §1 (the other half is
* {@link mimeTypeMappings}); this file is the source of truth and the doc
* mirrors it.
*
* An extension absent from this map resolves to `other`, which makes the file
* invisible to every typed query — so a missing entry is not a cosmetic gap.
*/
export const extensionMappings: { [key: string]: SimpleFileType } = {
// Audio
'mp3': 'audio',
@@ -7,7 +16,19 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
'flac': 'audio',
'aac': 'audio',
'ogg': 'audio',
'oga': 'audio',
'opus': 'audio',
'm4a': 'audio',
// `m4b` is the audiobook container — without it `contentKind=audiobook`
// had no extension that resolved to `audio` (METADATA_KEYS.md §14.15).
'm4b': 'audio',
'wma': 'audio',
'aiff': 'audio',
'aif': 'audio',
'ape': 'audio',
'wv': 'audio',
'dsf': 'audio',
'mka': 'audio',
// Video
'mp4': 'video',
'mkv': 'video',
@@ -27,6 +48,8 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
'ts': 'video',
'vob': 'video',
'ogm': 'video',
'ogv': 'video',
'asf': 'video',
'divx': 'video',
'xvid': 'video',
// Document
@@ -37,31 +60,64 @@ export const extensionMappings: { [key: string]: SimpleFileType } = {
'pptx': 'document',
'xls': 'document',
'xlsx': 'document',
'odt': 'document',
'ods': 'document',
'odp': 'document',
'txt': 'document',
'md': 'document',
'rtf': 'document',
// Images (are document)
'jpg': 'document',
'jpeg': 'document',
'png': 'document',
'gif': 'document',
'bmp': 'document',
'tif': 'document',
'tiff': 'document',
'svg': 'document',
'webp': 'document',
// Document — literature formats. Their absence blocked meta-read outright
// (METADATA_KEYS.md §14.15).
'epub': 'document',
'mobi': 'document',
'azw': 'document',
'azw3': 'document',
'djvu': 'document',
// Images
// ⚠ These used to map to `document` ("Images (are document)"), which meant
// the `image` bucket was unreachable from this table while Rust feeders
// wrote it anyway. A consumer asking for documents must not receive
// posters — see METADATA_KEYS.md §14.15.
'jpg': 'image',
'jpeg': 'image',
'png': 'image',
'gif': 'image',
'bmp': 'image',
'tif': 'image',
'tiff': 'image',
'svg': 'image',
'webp': 'image',
'avif': 'image',
'heic': 'image',
'heif': 'image',
'ico': 'image',
// Archive
'zip': 'archive',
'rar': 'archive',
'7z': 'archive',
'tar': 'archive',
'tgz': 'archive',
'gz': 'archive',
'bz2': 'archive',
'xz': 'archive',
'zst': 'archive',
// Archive — comic containers. A `.cbz` is literally a zip, so `archive` is
// the honest byte-level answer; `contentKind=comic` + `domain=literature`
// are what route it to meta-read (METADATA_KEYS.md §14.16).
'cbz': 'archive',
'cbr': 'archive',
//subtitles
'srt': 'subtitle',
'sub': 'subtitle',
'sbv': 'subtitle',
'vtt': 'subtitle',
// ⚠ ASS/SSA is what meta-watch renders through libass — the client's
// primary subtitle format was the one this classifier did not know.
'ass': 'subtitle',
'ssa': 'subtitle',
'ttml': 'subtitle',
'dfxp': 'subtitle',
'smi': 'subtitle',
//torrent
'torrent': 'torrent',
};
};
@@ -1,11 +1,32 @@
import {SimpleFileType} from "../SimpleFileType.js";
/**
* MIME type → `fileType`. The other half of the canonical classification
* table (see {@link extensionMappings} and METADATA_KEYS.md §1).
*
* ⚠ Resolution is conservative on purpose: when a sniffed MIME maps to a
* definite value that *disagrees* with the extension's value,
* `FileTypeConfigurable.getFileType` returns `undefined` rather than guessing
* — a wrong `fileType` routes a file to a handler that cannot open it. Adding
* a mapping here therefore also creates a new way to disagree; only add ones
* whose extension counterpart agrees.
*/
export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
'text/plain': 'undefined',//text could be anything document or subtitle or other
// Audio MIME types
'audio/mpeg': 'audio',
'audio/wav': 'audio',
'audio/x-wav': 'audio',
'audio/flac': 'audio',
'audio/ogg': 'audio',
'audio/opus': 'audio',
'audio/aac': 'audio',
// ftyp brands `M4A `/`M4B ` sniff as audio/mp4 while a real video mp4
// sniffs as video/mp4, so this does not collide with the `mp4` extension.
'audio/mp4': 'audio',
'audio/x-ms-wma': 'audio',
'audio/x-aiff': 'audio',
'audio/x-ape': 'audio',
// Video MIME types
'video/mp4': 'video',
'video/mpeg': 'video',
@@ -13,8 +34,12 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
'video/x-msvideo': 'video',
'video/x-flv': 'video',
'video/x-ms-wmv': 'video',
'video/x-ms-asf': 'video',
'video/x-matroska': 'video',
'video/webm': 'video',
'video/ogg': 'video',
'video/3gpp': 'video',
'video/mp2t': 'video',
// Document MIME types
'application/pdf': 'document',
'application/msword': 'document',
@@ -23,20 +48,47 @@ export const mimeTypeMappings: { [key: string]: SimpleFileType } = {
'application/vnd.openxmlformats-officedocument.presentationml.presentation': 'document',
'application/vnd.ms-excel': 'document',
'application/vnd.openxmlformats-officedocument.spreadsheetml.sheet': 'document',
'image/jpeg': 'document',
'image/png': 'document',
'image/gif': 'document',
'application/vnd.oasis.opendocument.text': 'document',
'application/vnd.oasis.opendocument.spreadsheet': 'document',
'application/vnd.oasis.opendocument.presentation': 'document',
'application/rtf': 'document',
// Literature containers (METADATA_KEYS.md §14.15)
'application/epub+zip': 'document',
'application/x-mobipocket-ebook': 'document',
'image/vnd.djvu': 'document',
'text/markdown': 'document',
// Image MIME types — no longer folded into `document` (§14.15)
'image/jpeg': 'image',
'image/png': 'image',
'image/gif': 'image',
'image/webp': 'image',
'image/bmp': 'image',
'image/tiff': 'image',
'image/svg+xml': 'image',
'image/avif': 'image',
'image/heic': 'image',
'image/x-icon': 'image',
// Archive MIME types
'application/zip': 'archive',
'application/x-rar-compressed': 'archive',
'application/vnd.rar': 'archive',
'application/x-7z-compressed': 'archive',
'application/gzip': 'archive',
'application/x-bzip2': 'archive',
'application/x-xz': 'archive',
'application/zstd': 'archive',
'application/x-tar': 'archive',
// Comic containers sniff as their underlying zip/rar, which agrees with
// the `cbz`/`cbr` extensions — both say `archive`.
'application/vnd.comicbook+zip': 'archive',
'application/vnd.comicbook-rar': 'archive',
//torrent
'application/x-bittorrent': 'torrent',
//subtitles
'text/vtt': 'subtitle',
'text/srt': 'subtitle',
'text/ssa': 'subtitle',
'text/x-ssa': 'subtitle',
'text/sub': 'subtitle',
};
'application/x-subrip': 'subtitle',
};
@@ -13,6 +13,7 @@ export * from './tools/StringArrayOperation.js';
export * from './tools/TwoStringsOperation.js';
export * from './config/AnimeKeywords.js';
export * from './config/DomainMapping.js';
export * from './config/ExtentionsMapping.js';
export * from './config/KeywordsArray.js';
export * from './config/MimeTypeMapping.js';
@@ -1,10 +1,17 @@
/**
* Multicodec / multihash codes for every digest meta-hash can compute.
*
* Source of truth for these numbers, mirrored in:
* Source of truth for these numbers. The same codes appear in the CID utility
* RANK, which is a separate thing this file does not implement:
* - `packages/meta-core/internal/cid/rank.go`
* - the `cid_rank.rs` in each of meta-search and meta-share
* - `/cid-rank-vectors.json` (the golden fixture that pins them all together)
* - `/cid-rank-vectors.json` (the golden fixture pinning those three together)
*
* ⚠ This package used to vendor that fixture at `src/tests/cid-rank-vectors.json`
* and no test ever loaded it, so every vector added cost a commit here for
* nothing. Removed 2026-09-09. If a TypeScript rank implementation is ever
* written, re-add the path to `scripts/check-cid-vectors.sh` *and* a test that
* asserts it — a vendored fixture nobody reads is not a guard.
*
* ⚠ meta-hash encodes `codec == multihash code` (see `ComputeHash.cidFinalize`:
* `CID.createV1(code, create(code, digest))`), whereas the fullhash plugin and
@@ -1,166 +0,0 @@
{
"$schema": "cid-rank-vectors/v1",
"$comment": [
"CANONICAL golden vectors for the CID utility rank. THE single source of truth.",
"The rank decides which of a record's several CIDs is elected canonical - the cid a",
"record is advertised and addressed by. It is reimplemented in Go, 2x Rust and 3x TS;",
"they cannot share code (independent submodules, no cargo path-dep). These vectors are",
"the only thing holding them in sync: every implementation's test suite loads this file.",
"",
"This file is VENDORED into each consuming submodule (submodule CI cannot read above its",
"own tree). scripts/check-cid-vectors.sh diffs every copy against this one.",
"",
"Ranking is by UTILITY for external retrieval, not cryptographic strength.",
"Ties break on the lexicographically smaller cid, so the election is deterministic",
"across peers and does not churn.",
"",
"See METADATA_KEYS.md section 2/14.13 and docs/cid-formats.md."
],
"ladder": {
"40": "dag-pb (IPFS block)",
"30": "sha2-256 / sha3-256 / sha3-384 - real content digests",
"25": "nzb-posting - content-derived (Message-ID set digest), but not itself fetchable",
"20": "BitTorrent infohashes - swarm interop",
"10": "midhash256 - the record address",
"5": "opaque locators (nzb-release, url, card) - never outrank a real digest",
"0": "weak/unknown digests, unparseable input"
},
"tieBreak": "lexicographically smaller cid wins",
"vectors": [
{
"name": "ipfs-dagpb",
"cid": "bafybeiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 40,
"note": "dag-pb codec 0x70 - chunked IPFS file, broadest network reach"
},
{
"name": "sha2-256-raw-codec",
"cid": "bafkreiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 30,
"note": "fullhash/gateway convention: raw codec 0x55, mh names the algo"
},
{
"name": "sha2-256-codec-eq-mh",
"cid": "baejbeibku6ua2l6r5olnggx4pto2sii2s5jeeawgh4cbkhlbmp52gudbua",
"rank": 30,
"note": "REAL meta-hash fixture; meta-hash sets codec == mh"
},
{
"name": "sha3-256",
"cid": "baelbmidner7x3dd7t2kldh6wiq4w6lsye7qrwo6uoqot2t52zkhqk56fnq",
"rank": 30,
"note": "REAL meta-hash fixture"
},
{
"name": "sha3-384",
"cid": "baekrkmeglipkjf33exqexdvijl4twipruay4wsbvt44akce43lq5rgccxgr7v6dwksmj3sxedakevc53g6ga",
"rank": 30,
"note": "REAL meta-hash fixture"
},
{
"name": "nzb-posting",
"cid": "bagbsaeraaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 25,
"note": "codec 0x1003 + REAL sha2-256 multihash (0x12) over the normalised, sorted Message-ID set - NOT a locator, so it sits ABOVE tier 5, but it addresses no fetchable block (bitswap can never satisfy a want for it - the digest is over an id set, not over any block's bytes), so it stays BELOW a real content digest (30): once meta-share materialises the media and re-seeds it, that genuine sha2-256 cid must win the canonical election, not this one. Rank MUST match on codec BEFORE falling through to the mh switch, or this scores 30 (same trap as nzb-release/url, in reverse: those need a codec check because their mh is identity 0x00; this needs one because its mh is REAL sha2-256 and would otherwise tie with tier 30)."
},
{
"name": "btih-v2-codec-eq-mh",
"cid": "bag3sdnzbeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "meta-hash encoder convention"
},
{
"name": "btih-v2-raw-codec",
"cid": "bafk3oijaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "fullhash encoder convention - SAME tier, both conventions must match"
},
{
"name": "btih-v1-file",
"cid": "bagasbajacqaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "gateway per-file BitTorrent v1"
},
{
"name": "btih-v2-file",
"cid": "bagbcbaraeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 20,
"note": "gateway per-file BitTorrent v2"
},
{
"name": "midhash256-codec-eq-mh",
"cid": "bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 10,
"note": "the record address; internally meaningful, externally invisible"
},
{
"name": "midhash256-raw-codec",
"cid": "bafkyaibaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 10
},
{
"name": "nzb-release-locator",
"cid": "bagcsaaazcbqxa2jonz5gez3fmvvs42lomzx55ln657fp4aab",
"rank": 5,
"note": "codec 0x1005 + IDENTITY multihash 0x00. The locator code is in the CODEC slot, not the mh slot - rank MUST match on codec. Tier 5 = below midhash: a locator never outranks a real digest, it wins only as a record's sole cid."
},
{
"name": "url-locator",
"cid": "bagdcaaaznb2hi4dthixs6zlymfwxa3dffzxxezzpmexhazdg",
"rank": 5,
"note": "codec 0x1006 + IDENTITY multihash 0x00. Same tier/reasoning as nzb-release."
},
{
"name": "card-locator",
"cid": "bagdsaaanar2g2zdcor3duojvgq3ts",
"rank": 5,
"note": "codec 0x1007 + IDENTITY multihash 0x00, digest = varint(len(source)) || source || id — here (\"tmdb\",\"tv:95479\"). Same codec-slot rule as nzb-release/url: match on the CODEC, never the mh. The limit case of the family — the only CID in MetaMesh with NO bytes behind it anywhere, ever: it addresses a *work* (a series, a film), not a file, so the byte path 404s by design and bitswap never offers it (seeding is gated on sha2-256). Tier 5 is load-bearing for the planned cross-source merge: one card record may carry BOTH a TMDB and a MyAnimeList locator, and if a real card body is ever added (see card-tier-search.md §4.3) its sha2-256 must win the canonical election over both locators without any addressing change. Ranking it 0 instead would still lose to a digest, but would put the implementations back out of agreement — which is the exact failure this fixture exists to prevent."
},
{
"name": "sha1",
"cid": "baeircff3eqvxxaqqot2lt242stkdtk3gq4c2vra",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "md5",
"cid": "bahkqdvibcatpcfl7ine2n625eb5ml5mzkfaa",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "crc32",
"cid": "bagzafmqcathsbnrk",
"rank": 0,
"note": "REAL meta-hash fixture; weak digest"
},
{
"name": "bt-pieces-root",
"cid": "bafkyf3qceaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 0,
"note": "torrent-internal merkle root, NOT a file-identity cid"
},
{
"name": "cidv0-rejected",
"cid": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
"rank": 0,
"note": "TRAP: Go rejected this (rank 0) while Rust's MsCid::from_str ACCEPTED it and ranked it 40 (dag-pb). Strict parse = CIDv1 base32-lower only. This vector pins the fix."
},
{
"name": "token-form-rejected",
"cid": "midhash256:bagacbabaeaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
"rank": 0,
"note": "TRAP: the deprecated <algo>:<cid> token form. Rust used to strip the prefix and rank it 10. It must now be rejected."
},
{
"name": "garbage",
"cid": "not-a-cid",
"rank": 0
},
{
"name": "empty",
"cid": "",
"rank": 0
}
]
}
@@ -1,2 +1,16 @@
export type SimpleFileType = 'audio' | 'video' | 'document' | 'archive' | 'other' | 'undefined';
/**
* The `fileType` vocabulary — METADATA_KEYS.md §1. Kept in lockstep with
* `filename-tool`'s `SimpleFileType`, which is the classifier that produces
* these values; this copy is the wire/type-level mirror.
*/
export type SimpleFileType =
'audio'
| 'video'
| 'image'
| 'document'
| 'archive'
| 'subtitle'
| 'torrent'
| 'card'
| 'other'
| 'undefined';