fullhash improvement

This commit is contained in:
Pierre
2026-05-31 21:27:11 +02:00
parent f1107d129c
commit 5415eec81f
5 changed files with 64 additions and 23 deletions
+8 -1
View File
@@ -84,13 +84,20 @@ pub async fn manifest() -> Json<ManifestResponse> {
"label": "BitTorrent v2 Info Hash",
"type": "cid",
"readonly": true,
"hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)"
"hint": "BitTorrent v2 info hash: SHA-256 of bencoded info dict (btih-v2 multicodec 0x10B7)"
}
}),
config: json!({
"enabledHashes": {
"type": "string",
"label": "Enabled hash algorithms (comma-separated)",
// These keys select WHICH digests to compute — they are an
// internal algorithm-selection vocabulary, NOT record field
// names. Results are stored on the record as a bare-CID
// key-set (cids/<cid> = "true"); the algorithm is recovered
// from each CID's multicodec, never a field name. The
// pieces-root is computed but not stored as a file-identity
// member (METADATA_KEYS.md §14.13).
"default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash",
"required": false
}
+38 -19
View File
@@ -8,7 +8,7 @@ use std::time::Instant;
use crate::cache::CacheManager;
use crate::client::MetaCoreClient;
use crate::config::{HashAlgorithm, SharedConfig};
use crate::hasher::{hash_file, hash_stream};
use crate::hasher::{hash_file, hash_stream, multicodec_of};
use crate::webdav::{WebDavClient, WebDavReader};
#[derive(Deserialize)]
@@ -185,40 +185,48 @@ async fn do_process_file(
// Get config
let config = state.config.read().await.clone();
// Determine which hashes we need to compute
let required_keys: Vec<&str> = config
// Which digests does the record already carry? Sibling CIDs live as a
// bare-CID key-set (cids/<cid>); the algorithm is no longer in a field
// name, so we recover it from each member's multihash code. An algorithm
// is "already computed" iff its multicodec is present in that set.
let present_codecs: std::collections::HashSet<u64> = req
.existing_meta
.keys()
.filter_map(|k| k.strip_prefix("cids/"))
.filter_map(multicodec_of)
.collect();
let missing_algos: Vec<HashAlgorithm> = config
.enabled_hashes
.iter()
.map(|a| a.key())
.collect();
// Check existing metadata - skip if all hashes already present
let missing_from_meta: Vec<&str> = required_keys
.iter()
.filter(|k| !req.existing_meta.contains_key(**k))
.copied()
.filter(|a| !present_codecs.contains(&a.multicodec()))
.collect();
if missing_from_meta.is_empty() {
if missing_algos.is_empty() {
return Ok(ProcessResult::Skipped(
"All hashes already computed".to_string(),
));
}
// Check cache for remaining hashes
// Check cache for remaining hashes. The internal results/cache maps stay
// keyed by the algorithm key (cid_sha2-256, …) — only the meta-core write
// below is transformed into the cids/<cid> key-set.
let cached = state.cache.get(&filename, size, mtime).await;
let mut results: HashMap<String, String> = HashMap::new();
let to_compute: Vec<HashAlgorithm> = missing_from_meta
let to_compute: Vec<HashAlgorithm> = missing_algos
.iter()
.filter_map(|key| {
.copied()
.filter_map(|algo| {
let key = algo.key();
if let Some(ref cached_hashes) = cached {
if let Some(hash) = cached_hashes.get(*key) {
if let Some(hash) = cached_hashes.get(key) {
results.insert(key.to_string(), hash.clone());
return None;
}
}
HashAlgorithm::from_key(key)
Some(algo)
})
.collect();
@@ -270,11 +278,22 @@ async fn do_process_file(
tracing::info!("Cache saved for {} ({} hashes)", filename, results.len());
}
// Store results to meta-core
if !results.is_empty() {
// Store results to meta-core as the bare-CID key-set: one
// cids/<cid> = "true" entry per computed digest. The CID is
// self-describing (its algorithm is the multicodec), so there is no
// per-algorithm field name. The pieces-root is excluded — it's a
// torrent-internal merkle root, not a file-identity CID (§14.13).
let pieces_root_key = HashAlgorithm::BtPiecesRoot.key();
let cids_payload: HashMap<String, String> = results
.iter()
.filter(|(key, _)| key.as_str() != pieces_root_key)
.map(|(_, cid_value)| (format!("cids/{}", cid_value), "true".to_string()))
.collect();
if !cids_payload.is_empty() {
match MetaCoreClient::new(&req.meta_core_url) {
Ok(client) => {
if let Err(e) = client.merge_metadata(&req.cid, &results).await {
if let Err(e) = client.merge_metadata(&req.cid, &cids_payload).await {
tracing::warn!("Failed to store metadata to meta-core: {}", e);
}
}
+9 -2
View File
@@ -39,7 +39,14 @@ pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry
// Official BitTorrent v2 code
pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root
// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash
// Custom BitTorrent v2 info-hash code (BEP 52), matching meta-hash's
// cid_btih_v2. The info hash is a SHA-256 of the bencoded info dict, but it
// MUST carry a distinct multihash code (not 0x12) so it can't be confused
// with the full-file sha2-256 in the bare-CID key-set — see METADATA_KEYS.md
// §2/§14.13. Consumers (meta-dup, meta-watch) disambiguate set members by
// this code.
pub const BTIH_V2_CODE: u64 = 0x10B7;
impl HashAlgorithm {
pub fn all() -> Vec<HashAlgorithm> {
@@ -125,7 +132,7 @@ impl HashAlgorithm {
HashAlgorithm::Sha3_384 => SHA3_384_CODE,
HashAlgorithm::Midhash256 => MIDHASH256_CODE,
HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE,
HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256
HashAlgorithm::BtInfoHash => BTIH_V2_CODE, // distinct from sha2-256 (§14.13)
}
}
}
+8
View File
@@ -30,6 +30,14 @@ pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String {
cid.to_string()
}
/// Returns the multihash code (the algorithm) of a bare CID string, or None
/// if it doesn't parse. Used to tell which digests a record already carries
/// in its bare-CID key-set (cids/<cid>), since the algorithm is no longer in
/// a field name.
pub fn multicodec_of(cid_str: &str) -> Option<u64> {
Cid::try_from(cid_str).ok().map(|c| c.hash().code())
}
#[cfg(test)]
mod tests {
use super::*;
+1 -1
View File
@@ -10,5 +10,5 @@ pub use btih_v2::{
compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link,
format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE,
};
pub use cid::to_cid;
pub use cid::{multicodec_of, to_cid};
pub use midhash256::compute_midhash256;