From 5415eec81f633dc6d845558e1f5ac944c8ada57a Mon Sep 17 00:00:00 2001 From: Pierre Date: Sun, 31 May 2026 21:27:11 +0200 Subject: [PATCH] fullhash improvement --- src/api/manifest.rs | 9 ++++++- src/api/process.rs | 57 ++++++++++++++++++++++++++++++--------------- src/config.rs | 11 +++++++-- src/hasher/cid.rs | 8 +++++++ src/hasher/mod.rs | 2 +- 5 files changed, 64 insertions(+), 23 deletions(-) diff --git a/src/api/manifest.rs b/src/api/manifest.rs index 554bb34..8d398ee 100644 --- a/src/api/manifest.rs +++ b/src/api/manifest.rs @@ -84,13 +84,20 @@ pub async fn manifest() -> Json { "label": "BitTorrent v2 Info Hash", "type": "cid", "readonly": true, - "hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)" + "hint": "BitTorrent v2 info hash: SHA-256 of bencoded info dict (btih-v2 multicodec 0x10B7)" } }), config: json!({ "enabledHashes": { "type": "string", "label": "Enabled hash algorithms (comma-separated)", + // These keys select WHICH digests to compute — they are an + // internal algorithm-selection vocabulary, NOT record field + // names. Results are stored on the record as a bare-CID + // key-set (cids/ = "true"); the algorithm is recovered + // from each CID's multicodec, never a field name. The + // pieces-root is computed but not stored as a file-identity + // member (METADATA_KEYS.md §14.13). "default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash", "required": false } diff --git a/src/api/process.rs b/src/api/process.rs index ab49cea..058883a 100644 --- a/src/api/process.rs +++ b/src/api/process.rs @@ -8,7 +8,7 @@ use std::time::Instant; use crate::cache::CacheManager; use crate::client::MetaCoreClient; use crate::config::{HashAlgorithm, SharedConfig}; -use crate::hasher::{hash_file, hash_stream}; +use crate::hasher::{hash_file, hash_stream, multicodec_of}; use crate::webdav::{WebDavClient, WebDavReader}; #[derive(Deserialize)] @@ -185,40 +185,48 @@ async fn do_process_file( // Get config let config = state.config.read().await.clone(); - // Determine which hashes we need to compute - let required_keys: Vec<&str> = config + // Which digests does the record already carry? Sibling CIDs live as a + // bare-CID key-set (cids/); the algorithm is no longer in a field + // name, so we recover it from each member's multihash code. An algorithm + // is "already computed" iff its multicodec is present in that set. + let present_codecs: std::collections::HashSet = req + .existing_meta + .keys() + .filter_map(|k| k.strip_prefix("cids/")) + .filter_map(multicodec_of) + .collect(); + + let missing_algos: Vec = config .enabled_hashes .iter() - .map(|a| a.key()) - .collect(); - - // Check existing metadata - skip if all hashes already present - let missing_from_meta: Vec<&str> = required_keys - .iter() - .filter(|k| !req.existing_meta.contains_key(**k)) .copied() + .filter(|a| !present_codecs.contains(&a.multicodec())) .collect(); - if missing_from_meta.is_empty() { + if missing_algos.is_empty() { return Ok(ProcessResult::Skipped( "All hashes already computed".to_string(), )); } - // Check cache for remaining hashes + // Check cache for remaining hashes. The internal results/cache maps stay + // keyed by the algorithm key (cid_sha2-256, …) — only the meta-core write + // below is transformed into the cids/ key-set. let cached = state.cache.get(&filename, size, mtime).await; let mut results: HashMap = HashMap::new(); - let to_compute: Vec = missing_from_meta + let to_compute: Vec = missing_algos .iter() - .filter_map(|key| { + .copied() + .filter_map(|algo| { + let key = algo.key(); if let Some(ref cached_hashes) = cached { - if let Some(hash) = cached_hashes.get(*key) { + if let Some(hash) = cached_hashes.get(key) { results.insert(key.to_string(), hash.clone()); return None; } } - HashAlgorithm::from_key(key) + Some(algo) }) .collect(); @@ -270,11 +278,22 @@ async fn do_process_file( tracing::info!("Cache saved for {} ({} hashes)", filename, results.len()); } - // Store results to meta-core - if !results.is_empty() { + // Store results to meta-core as the bare-CID key-set: one + // cids/ = "true" entry per computed digest. The CID is + // self-describing (its algorithm is the multicodec), so there is no + // per-algorithm field name. The pieces-root is excluded — it's a + // torrent-internal merkle root, not a file-identity CID (§14.13). + let pieces_root_key = HashAlgorithm::BtPiecesRoot.key(); + let cids_payload: HashMap = results + .iter() + .filter(|(key, _)| key.as_str() != pieces_root_key) + .map(|(_, cid_value)| (format!("cids/{}", cid_value), "true".to_string())) + .collect(); + + if !cids_payload.is_empty() { match MetaCoreClient::new(&req.meta_core_url) { Ok(client) => { - if let Err(e) = client.merge_metadata(&req.cid, &results).await { + if let Err(e) = client.merge_metadata(&req.cid, &cids_payload).await { tracing::warn!("Failed to store metadata to meta-core: {}", e); } } diff --git a/src/config.rs b/src/config.rs index 1f34093..e49b03a 100644 --- a/src/config.rs +++ b/src/config.rs @@ -39,7 +39,14 @@ pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry // Official BitTorrent v2 code pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root -// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash + +// Custom BitTorrent v2 info-hash code (BEP 52), matching meta-hash's +// cid_btih_v2. The info hash is a SHA-256 of the bencoded info dict, but it +// MUST carry a distinct multihash code (not 0x12) so it can't be confused +// with the full-file sha2-256 in the bare-CID key-set — see METADATA_KEYS.md +// §2/§14.13. Consumers (meta-dup, meta-watch) disambiguate set members by +// this code. +pub const BTIH_V2_CODE: u64 = 0x10B7; impl HashAlgorithm { pub fn all() -> Vec { @@ -125,7 +132,7 @@ impl HashAlgorithm { HashAlgorithm::Sha3_384 => SHA3_384_CODE, HashAlgorithm::Midhash256 => MIDHASH256_CODE, HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE, - HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256 + HashAlgorithm::BtInfoHash => BTIH_V2_CODE, // distinct from sha2-256 (§14.13) } } } diff --git a/src/hasher/cid.rs b/src/hasher/cid.rs index 36927c6..eeca254 100644 --- a/src/hasher/cid.rs +++ b/src/hasher/cid.rs @@ -30,6 +30,14 @@ pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String { cid.to_string() } +/// Returns the multihash code (the algorithm) of a bare CID string, or None +/// if it doesn't parse. Used to tell which digests a record already carries +/// in its bare-CID key-set (cids/), since the algorithm is no longer in +/// a field name. +pub fn multicodec_of(cid_str: &str) -> Option { + Cid::try_from(cid_str).ok().map(|c| c.hash().code()) +} + #[cfg(test)] mod tests { use super::*; diff --git a/src/hasher/mod.rs b/src/hasher/mod.rs index 81ac21e..1a0469d 100644 --- a/src/hasher/mod.rs +++ b/src/hasher/mod.rs @@ -10,5 +10,5 @@ pub use btih_v2::{ compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link, format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE, }; -pub use cid::to_cid; +pub use cid::{multicodec_of, to_cid}; pub use midhash256::compute_midhash256;