fullhash improvement
This commit is contained in:
+8
-1
@@ -84,13 +84,20 @@ pub async fn manifest() -> Json<ManifestResponse> {
|
|||||||
"label": "BitTorrent v2 Info Hash",
|
"label": "BitTorrent v2 Info Hash",
|
||||||
"type": "cid",
|
"type": "cid",
|
||||||
"readonly": true,
|
"readonly": true,
|
||||||
"hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)"
|
"hint": "BitTorrent v2 info hash: SHA-256 of bencoded info dict (btih-v2 multicodec 0x10B7)"
|
||||||
}
|
}
|
||||||
}),
|
}),
|
||||||
config: json!({
|
config: json!({
|
||||||
"enabledHashes": {
|
"enabledHashes": {
|
||||||
"type": "string",
|
"type": "string",
|
||||||
"label": "Enabled hash algorithms (comma-separated)",
|
"label": "Enabled hash algorithms (comma-separated)",
|
||||||
|
// These keys select WHICH digests to compute — they are an
|
||||||
|
// internal algorithm-selection vocabulary, NOT record field
|
||||||
|
// names. Results are stored on the record as a bare-CID
|
||||||
|
// key-set (cids/<cid> = "true"); the algorithm is recovered
|
||||||
|
// from each CID's multicodec, never a field name. The
|
||||||
|
// pieces-root is computed but not stored as a file-identity
|
||||||
|
// member (METADATA_KEYS.md §14.13).
|
||||||
"default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash",
|
"default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash",
|
||||||
"required": false
|
"required": false
|
||||||
}
|
}
|
||||||
|
|||||||
+38
-19
@@ -8,7 +8,7 @@ use std::time::Instant;
|
|||||||
use crate::cache::CacheManager;
|
use crate::cache::CacheManager;
|
||||||
use crate::client::MetaCoreClient;
|
use crate::client::MetaCoreClient;
|
||||||
use crate::config::{HashAlgorithm, SharedConfig};
|
use crate::config::{HashAlgorithm, SharedConfig};
|
||||||
use crate::hasher::{hash_file, hash_stream};
|
use crate::hasher::{hash_file, hash_stream, multicodec_of};
|
||||||
use crate::webdav::{WebDavClient, WebDavReader};
|
use crate::webdav::{WebDavClient, WebDavReader};
|
||||||
|
|
||||||
#[derive(Deserialize)]
|
#[derive(Deserialize)]
|
||||||
@@ -185,40 +185,48 @@ async fn do_process_file(
|
|||||||
// Get config
|
// Get config
|
||||||
let config = state.config.read().await.clone();
|
let config = state.config.read().await.clone();
|
||||||
|
|
||||||
// Determine which hashes we need to compute
|
// Which digests does the record already carry? Sibling CIDs live as a
|
||||||
let required_keys: Vec<&str> = config
|
// bare-CID key-set (cids/<cid>); the algorithm is no longer in a field
|
||||||
|
// name, so we recover it from each member's multihash code. An algorithm
|
||||||
|
// is "already computed" iff its multicodec is present in that set.
|
||||||
|
let present_codecs: std::collections::HashSet<u64> = req
|
||||||
|
.existing_meta
|
||||||
|
.keys()
|
||||||
|
.filter_map(|k| k.strip_prefix("cids/"))
|
||||||
|
.filter_map(multicodec_of)
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
let missing_algos: Vec<HashAlgorithm> = config
|
||||||
.enabled_hashes
|
.enabled_hashes
|
||||||
.iter()
|
.iter()
|
||||||
.map(|a| a.key())
|
|
||||||
.collect();
|
|
||||||
|
|
||||||
// Check existing metadata - skip if all hashes already present
|
|
||||||
let missing_from_meta: Vec<&str> = required_keys
|
|
||||||
.iter()
|
|
||||||
.filter(|k| !req.existing_meta.contains_key(**k))
|
|
||||||
.copied()
|
.copied()
|
||||||
|
.filter(|a| !present_codecs.contains(&a.multicodec()))
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
if missing_from_meta.is_empty() {
|
if missing_algos.is_empty() {
|
||||||
return Ok(ProcessResult::Skipped(
|
return Ok(ProcessResult::Skipped(
|
||||||
"All hashes already computed".to_string(),
|
"All hashes already computed".to_string(),
|
||||||
));
|
));
|
||||||
}
|
}
|
||||||
|
|
||||||
// Check cache for remaining hashes
|
// Check cache for remaining hashes. The internal results/cache maps stay
|
||||||
|
// keyed by the algorithm key (cid_sha2-256, …) — only the meta-core write
|
||||||
|
// below is transformed into the cids/<cid> key-set.
|
||||||
let cached = state.cache.get(&filename, size, mtime).await;
|
let cached = state.cache.get(&filename, size, mtime).await;
|
||||||
let mut results: HashMap<String, String> = HashMap::new();
|
let mut results: HashMap<String, String> = HashMap::new();
|
||||||
|
|
||||||
let to_compute: Vec<HashAlgorithm> = missing_from_meta
|
let to_compute: Vec<HashAlgorithm> = missing_algos
|
||||||
.iter()
|
.iter()
|
||||||
.filter_map(|key| {
|
.copied()
|
||||||
|
.filter_map(|algo| {
|
||||||
|
let key = algo.key();
|
||||||
if let Some(ref cached_hashes) = cached {
|
if let Some(ref cached_hashes) = cached {
|
||||||
if let Some(hash) = cached_hashes.get(*key) {
|
if let Some(hash) = cached_hashes.get(key) {
|
||||||
results.insert(key.to_string(), hash.clone());
|
results.insert(key.to_string(), hash.clone());
|
||||||
return None;
|
return None;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
HashAlgorithm::from_key(key)
|
Some(algo)
|
||||||
})
|
})
|
||||||
.collect();
|
.collect();
|
||||||
|
|
||||||
@@ -270,11 +278,22 @@ async fn do_process_file(
|
|||||||
tracing::info!("Cache saved for {} ({} hashes)", filename, results.len());
|
tracing::info!("Cache saved for {} ({} hashes)", filename, results.len());
|
||||||
}
|
}
|
||||||
|
|
||||||
// Store results to meta-core
|
// Store results to meta-core as the bare-CID key-set: one
|
||||||
if !results.is_empty() {
|
// cids/<cid> = "true" entry per computed digest. The CID is
|
||||||
|
// self-describing (its algorithm is the multicodec), so there is no
|
||||||
|
// per-algorithm field name. The pieces-root is excluded — it's a
|
||||||
|
// torrent-internal merkle root, not a file-identity CID (§14.13).
|
||||||
|
let pieces_root_key = HashAlgorithm::BtPiecesRoot.key();
|
||||||
|
let cids_payload: HashMap<String, String> = results
|
||||||
|
.iter()
|
||||||
|
.filter(|(key, _)| key.as_str() != pieces_root_key)
|
||||||
|
.map(|(_, cid_value)| (format!("cids/{}", cid_value), "true".to_string()))
|
||||||
|
.collect();
|
||||||
|
|
||||||
|
if !cids_payload.is_empty() {
|
||||||
match MetaCoreClient::new(&req.meta_core_url) {
|
match MetaCoreClient::new(&req.meta_core_url) {
|
||||||
Ok(client) => {
|
Ok(client) => {
|
||||||
if let Err(e) = client.merge_metadata(&req.cid, &results).await {
|
if let Err(e) = client.merge_metadata(&req.cid, &cids_payload).await {
|
||||||
tracing::warn!("Failed to store metadata to meta-core: {}", e);
|
tracing::warn!("Failed to store metadata to meta-core: {}", e);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+9
-2
@@ -39,7 +39,14 @@ pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry
|
|||||||
|
|
||||||
// Official BitTorrent v2 code
|
// Official BitTorrent v2 code
|
||||||
pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root
|
pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root
|
||||||
// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash
|
|
||||||
|
// Custom BitTorrent v2 info-hash code (BEP 52), matching meta-hash's
|
||||||
|
// cid_btih_v2. The info hash is a SHA-256 of the bencoded info dict, but it
|
||||||
|
// MUST carry a distinct multihash code (not 0x12) so it can't be confused
|
||||||
|
// with the full-file sha2-256 in the bare-CID key-set — see METADATA_KEYS.md
|
||||||
|
// §2/§14.13. Consumers (meta-dup, meta-watch) disambiguate set members by
|
||||||
|
// this code.
|
||||||
|
pub const BTIH_V2_CODE: u64 = 0x10B7;
|
||||||
|
|
||||||
impl HashAlgorithm {
|
impl HashAlgorithm {
|
||||||
pub fn all() -> Vec<HashAlgorithm> {
|
pub fn all() -> Vec<HashAlgorithm> {
|
||||||
@@ -125,7 +132,7 @@ impl HashAlgorithm {
|
|||||||
HashAlgorithm::Sha3_384 => SHA3_384_CODE,
|
HashAlgorithm::Sha3_384 => SHA3_384_CODE,
|
||||||
HashAlgorithm::Midhash256 => MIDHASH256_CODE,
|
HashAlgorithm::Midhash256 => MIDHASH256_CODE,
|
||||||
HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE,
|
HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE,
|
||||||
HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256
|
HashAlgorithm::BtInfoHash => BTIH_V2_CODE, // distinct from sha2-256 (§14.13)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -30,6 +30,14 @@ pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String {
|
|||||||
cid.to_string()
|
cid.to_string()
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/// Returns the multihash code (the algorithm) of a bare CID string, or None
|
||||||
|
/// if it doesn't parse. Used to tell which digests a record already carries
|
||||||
|
/// in its bare-CID key-set (cids/<cid>), since the algorithm is no longer in
|
||||||
|
/// a field name.
|
||||||
|
pub fn multicodec_of(cid_str: &str) -> Option<u64> {
|
||||||
|
Cid::try_from(cid_str).ok().map(|c| c.hash().code())
|
||||||
|
}
|
||||||
|
|
||||||
#[cfg(test)]
|
#[cfg(test)]
|
||||||
mod tests {
|
mod tests {
|
||||||
use super::*;
|
use super::*;
|
||||||
|
|||||||
+1
-1
@@ -10,5 +10,5 @@ pub use btih_v2::{
|
|||||||
compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link,
|
compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link,
|
||||||
format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE,
|
format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE,
|
||||||
};
|
};
|
||||||
pub use cid::to_cid;
|
pub use cid::{multicodec_of, to_cid};
|
||||||
pub use midhash256::compute_midhash256;
|
pub use midhash256::compute_midhash256;
|
||||||
|
|||||||
Reference in New Issue
Block a user