fullhash improvement
This commit is contained in:
+8
-1
@@ -84,13 +84,20 @@ pub async fn manifest() -> Json<ManifestResponse> {
|
||||
"label": "BitTorrent v2 Info Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)"
|
||||
"hint": "BitTorrent v2 info hash: SHA-256 of bencoded info dict (btih-v2 multicodec 0x10B7)"
|
||||
}
|
||||
}),
|
||||
config: json!({
|
||||
"enabledHashes": {
|
||||
"type": "string",
|
||||
"label": "Enabled hash algorithms (comma-separated)",
|
||||
// These keys select WHICH digests to compute — they are an
|
||||
// internal algorithm-selection vocabulary, NOT record field
|
||||
// names. Results are stored on the record as a bare-CID
|
||||
// key-set (cids/<cid> = "true"); the algorithm is recovered
|
||||
// from each CID's multicodec, never a field name. The
|
||||
// pieces-root is computed but not stored as a file-identity
|
||||
// member (METADATA_KEYS.md §14.13).
|
||||
"default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash",
|
||||
"required": false
|
||||
}
|
||||
|
||||
+38
-19
@@ -8,7 +8,7 @@ use std::time::Instant;
|
||||
use crate::cache::CacheManager;
|
||||
use crate::client::MetaCoreClient;
|
||||
use crate::config::{HashAlgorithm, SharedConfig};
|
||||
use crate::hasher::{hash_file, hash_stream};
|
||||
use crate::hasher::{hash_file, hash_stream, multicodec_of};
|
||||
use crate::webdav::{WebDavClient, WebDavReader};
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@@ -185,40 +185,48 @@ async fn do_process_file(
|
||||
// Get config
|
||||
let config = state.config.read().await.clone();
|
||||
|
||||
// Determine which hashes we need to compute
|
||||
let required_keys: Vec<&str> = config
|
||||
// Which digests does the record already carry? Sibling CIDs live as a
|
||||
// bare-CID key-set (cids/<cid>); the algorithm is no longer in a field
|
||||
// name, so we recover it from each member's multihash code. An algorithm
|
||||
// is "already computed" iff its multicodec is present in that set.
|
||||
let present_codecs: std::collections::HashSet<u64> = req
|
||||
.existing_meta
|
||||
.keys()
|
||||
.filter_map(|k| k.strip_prefix("cids/"))
|
||||
.filter_map(multicodec_of)
|
||||
.collect();
|
||||
|
||||
let missing_algos: Vec<HashAlgorithm> = config
|
||||
.enabled_hashes
|
||||
.iter()
|
||||
.map(|a| a.key())
|
||||
.collect();
|
||||
|
||||
// Check existing metadata - skip if all hashes already present
|
||||
let missing_from_meta: Vec<&str> = required_keys
|
||||
.iter()
|
||||
.filter(|k| !req.existing_meta.contains_key(**k))
|
||||
.copied()
|
||||
.filter(|a| !present_codecs.contains(&a.multicodec()))
|
||||
.collect();
|
||||
|
||||
if missing_from_meta.is_empty() {
|
||||
if missing_algos.is_empty() {
|
||||
return Ok(ProcessResult::Skipped(
|
||||
"All hashes already computed".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Check cache for remaining hashes
|
||||
// Check cache for remaining hashes. The internal results/cache maps stay
|
||||
// keyed by the algorithm key (cid_sha2-256, …) — only the meta-core write
|
||||
// below is transformed into the cids/<cid> key-set.
|
||||
let cached = state.cache.get(&filename, size, mtime).await;
|
||||
let mut results: HashMap<String, String> = HashMap::new();
|
||||
|
||||
let to_compute: Vec<HashAlgorithm> = missing_from_meta
|
||||
let to_compute: Vec<HashAlgorithm> = missing_algos
|
||||
.iter()
|
||||
.filter_map(|key| {
|
||||
.copied()
|
||||
.filter_map(|algo| {
|
||||
let key = algo.key();
|
||||
if let Some(ref cached_hashes) = cached {
|
||||
if let Some(hash) = cached_hashes.get(*key) {
|
||||
if let Some(hash) = cached_hashes.get(key) {
|
||||
results.insert(key.to_string(), hash.clone());
|
||||
return None;
|
||||
}
|
||||
}
|
||||
HashAlgorithm::from_key(key)
|
||||
Some(algo)
|
||||
})
|
||||
.collect();
|
||||
|
||||
@@ -270,11 +278,22 @@ async fn do_process_file(
|
||||
tracing::info!("Cache saved for {} ({} hashes)", filename, results.len());
|
||||
}
|
||||
|
||||
// Store results to meta-core
|
||||
if !results.is_empty() {
|
||||
// Store results to meta-core as the bare-CID key-set: one
|
||||
// cids/<cid> = "true" entry per computed digest. The CID is
|
||||
// self-describing (its algorithm is the multicodec), so there is no
|
||||
// per-algorithm field name. The pieces-root is excluded — it's a
|
||||
// torrent-internal merkle root, not a file-identity CID (§14.13).
|
||||
let pieces_root_key = HashAlgorithm::BtPiecesRoot.key();
|
||||
let cids_payload: HashMap<String, String> = results
|
||||
.iter()
|
||||
.filter(|(key, _)| key.as_str() != pieces_root_key)
|
||||
.map(|(_, cid_value)| (format!("cids/{}", cid_value), "true".to_string()))
|
||||
.collect();
|
||||
|
||||
if !cids_payload.is_empty() {
|
||||
match MetaCoreClient::new(&req.meta_core_url) {
|
||||
Ok(client) => {
|
||||
if let Err(e) = client.merge_metadata(&req.cid, &results).await {
|
||||
if let Err(e) = client.merge_metadata(&req.cid, &cids_payload).await {
|
||||
tracing::warn!("Failed to store metadata to meta-core: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
+9
-2
@@ -39,7 +39,14 @@ pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry
|
||||
|
||||
// Official BitTorrent v2 code
|
||||
pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root
|
||||
// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash
|
||||
|
||||
// Custom BitTorrent v2 info-hash code (BEP 52), matching meta-hash's
|
||||
// cid_btih_v2. The info hash is a SHA-256 of the bencoded info dict, but it
|
||||
// MUST carry a distinct multihash code (not 0x12) so it can't be confused
|
||||
// with the full-file sha2-256 in the bare-CID key-set — see METADATA_KEYS.md
|
||||
// §2/§14.13. Consumers (meta-dup, meta-watch) disambiguate set members by
|
||||
// this code.
|
||||
pub const BTIH_V2_CODE: u64 = 0x10B7;
|
||||
|
||||
impl HashAlgorithm {
|
||||
pub fn all() -> Vec<HashAlgorithm> {
|
||||
@@ -125,7 +132,7 @@ impl HashAlgorithm {
|
||||
HashAlgorithm::Sha3_384 => SHA3_384_CODE,
|
||||
HashAlgorithm::Midhash256 => MIDHASH256_CODE,
|
||||
HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE,
|
||||
HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256
|
||||
HashAlgorithm::BtInfoHash => BTIH_V2_CODE, // distinct from sha2-256 (§14.13)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -30,6 +30,14 @@ pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String {
|
||||
cid.to_string()
|
||||
}
|
||||
|
||||
/// Returns the multihash code (the algorithm) of a bare CID string, or None
|
||||
/// if it doesn't parse. Used to tell which digests a record already carries
|
||||
/// in its bare-CID key-set (cids/<cid>), since the algorithm is no longer in
|
||||
/// a field name.
|
||||
pub fn multicodec_of(cid_str: &str) -> Option<u64> {
|
||||
Cid::try_from(cid_str).ok().map(|c| c.hash().code())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
+1
-1
@@ -10,5 +10,5 @@ pub use btih_v2::{
|
||||
compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link,
|
||||
format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE,
|
||||
};
|
||||
pub use cid::to_cid;
|
||||
pub use cid::{multicodec_of, to_cid};
|
||||
pub use midhash256::compute_midhash256;
|
||||
|
||||
Reference in New Issue
Block a user