diff --git a/.gitignore b/.gitignore index 397b0d9..5f420d8 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,5 @@ dist/ *.log .env .DS_Store +CLAUDE.md +.claude/settings.local.json diff --git a/Cargo.toml b/Cargo.toml new file mode 100644 index 0000000..d9df8d0 --- /dev/null +++ b/Cargo.toml @@ -0,0 +1,51 @@ +[package] +name = "metamesh-plugin-fast-full-hash" +version = "1.0.0" +edition = "2021" +description = "High-performance file hashing plugin for MetaMesh" +authors = ["MetaMesh"] + +[dependencies] +# HTTP server +axum = "0.7" +tokio = { version = "1", features = ["full"] } +tower-http = { version = "0.6", features = ["cors", "trace"] } + +# Hashing +sha2 = "0.10" +sha1 = "0.10" +md-5 = "0.10" +crc32fast = "1.4" +sha3 = "0.10" + +# CID/Multihash +cid = "0.11" +multihash = "0.19" +unsigned-varint = "0.8" + +# BitTorrent v2 bencoding +bendy = "0.3" + +# Serialization +serde = { version = "1", features = ["derive"] } +serde_json = "1" +bincode = "1.3" + +# HTTP client +reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] } + +# Utilities +tracing = "0.1" +tracing-subscriber = { version = "0.3", features = ["env-filter"] } +anyhow = "1" +thiserror = "2" + +[dev-dependencies] +tempfile = "3" +hex = "0.4" + +[profile.release] +opt-level = 3 +lto = true +codegen-units = 1 +strip = true diff --git a/docs/btih_v2.md b/docs/btih_v2.md new file mode 100644 index 0000000..15cca94 --- /dev/null +++ b/docs/btih_v2.md @@ -0,0 +1,118 @@ +# BitTorrent v2 Hashes (BEP 52) + +## Purpose + +Generate BitTorrent v2 compatible hashes for magnet links and peer verification. Enables creating magnet links for files without a full .torrent file. + +## Two Output Hashes + +### 1. Pieces Root (`cid_bt_pieces_root`) + +- **Multicodec**: `0xb702` (**official** `bittorrent-pieces-root`) +- **CID codec**: `0x55` (raw) +- **Description**: Merkle tree root of SHA-256 hashes of 16KB file blocks +- **Use case**: File verification, piece-level integrity checking + +### 2. Info Hash (`cid_bt_info_hash`) + +- **Multicodec**: `0x12` (standard SHA-256) +- **CID codec**: `0x55` (raw) +- **Description**: SHA-256 of bencoded info dictionary +- **Use case**: Magnet links, torrent identification + +## Algorithm + +``` +1. Read file in 16KB blocks +2. SHA-256 hash each block (pad last block with zeros) +3. Build merkle tree from leaf hashes: + - Pad to next power of 2 with zero hashes + - Each parent = SHA-256(left_child || right_child) + → Result: pieces_root (32 bytes) + +4. Create info dictionary: + { + "file tree": { filename: { "": { "length": N, "pieces root": } } }, + "meta version": 2, + "name": filename, + "piece length": 16384 + } + +5. Bencode the dictionary +6. SHA-256(bencoded_info) + → Result: info_hash (32 bytes) +``` + +### Merkle Tree Structure + +``` + [Root] + / \ + [Node] [Node] + / \ / \ + [Leaf0] [Leaf1] [Leaf2] [Zero] + +Leaf = SHA-256(16KB block) +Node = SHA-256(left || right) +``` + +## Magnet Link Generation + +``` +magnet:?xt=urn:btmh:1220&dn= +``` + +Where: +- `btmh` = BitTorrent Multihash +- `1220` = `0x12` (SHA-256 code) + `0x20` (32 bytes = 0x20 in hex) +- `info_hash_hex` = 64 hex characters +- `dn` = display name (URL-encoded filename) + +### Example + +``` +File: ubuntu-24.04-desktop-amd64.iso +Info Hash: a1b2c3d4... +Magnet: magnet:?xt=urn:btmh:1220a1b2c3d4...&dn=ubuntu-24.04-desktop-amd64.iso +``` + +## BEP 52 Compliance + +This implementation follows [BEP 52](https://www.bittorrent.org/beps/bep_0052.html): + +| Requirement | Implementation | +|-------------|----------------| +| Block size | 16 KiB (16,384 bytes) | +| Piece size | 16 KiB (minimum, power of 2) | +| Hash algorithm | SHA-256 | +| Merkle tree | Binary, power-of-2 padded | +| Info dict | v2 format with `file tree` | +| Meta version | 2 (v2-only torrent) | + +## Properties + +| Property | Pieces Root | Info Hash | +|----------|-------------|-----------| +| Hash size | 32 bytes | 32 bytes | +| Algorithm | Merkle SHA-256 | SHA-256 | +| Multicodec | 0xb702 (official) | 0x12 (SHA-256) | +| Time complexity | O(n) | O(n) | +| Space complexity | O(log n) | O(1) | + +## Use Cases + +1. **Magnet link creation** - Share files via magnet links without .torrent files +2. **Peer verification** - Verify file integrity in BitTorrent v2 swarms +3. **Deduplication** - Identify identical files across BitTorrent network +4. **Hybrid torrents** - Create v1+v2 hybrid torrents + +## Limitations + +- Pieces root depends on filename (included in info dict) +- Single-file torrents only (no multi-file directory support) +- Requires full file read for both hashes + +## References + +- [BEP 52: The BitTorrent Protocol Specification v2](https://www.bittorrent.org/beps/bep_0052.html) +- [Multicodec Table](https://github.com/multiformats/multicodec/blob/master/table.csv) - `bittorrent-pieces-root` = 0xb702 diff --git a/docs/midhash256.md b/docs/midhash256.md new file mode 100644 index 0000000..1832d2a --- /dev/null +++ b/docs/midhash256.md @@ -0,0 +1,80 @@ +# MidHash256 + +## Purpose + +O(1) file identification hash - constant time regardless of file size. Designed for quick duplicate detection and large file identification before expensive full-hash operations. + +## Multicodec + +- **Code**: `0x1000` (custom, not in official multicodec registry) +- **CID codec**: `0x55` (raw) +- **Output key**: `cid_midhash256` + +## Algorithm + +``` +1. Read file size (S) +2. If S <= 1MB: + sample = entire file content +3. If S > 1MB: + seek to (S - 1MB) / 2 + sample = read 1MB from middle +4. Hash = SHA-256(S as u64 big-endian || sample) +``` + +### Pseudocode + +```rust +fn midhash256(path: &Path) -> [u8; 32] { + let size = file_size(path); + let mut hasher = SHA256::new(); + + // Always include size as 8-byte big-endian prefix + hasher.update(size.to_be_bytes()); + + if size <= 1_048_576 { + // Small file: hash entire content + hasher.update(read_all(path)); + } else { + // Large file: hash middle 1MB + let middle_start = (size - 1_048_576) / 2; + hasher.update(read_at(path, middle_start, 1_048_576)); + } + + hasher.finalize() +} +``` + +## Properties + +| Property | Value | +|----------|-------| +| Time complexity | O(1) for files > 1MB | +| Space complexity | O(1) - fixed 1MB buffer | +| Hash size | 32 bytes (SHA-256) | +| Collision resistance | Same as SHA-256 for middle content | + +## Use Cases + +1. **Quick duplicate detection** - Identify potential duplicates before computing full hashes +2. **Large file identification** - Identify terabyte files in ~20ms regardless of size +3. **Pre-filtering** - Filter candidates before expensive operations (full hash, content analysis) +4. **Change detection** - Quick check if middle content changed (useful for video files) + +## Limitations + +- Files differing only at edges (first/last 512KB for 2MB+ files) will have same midhash +- Not suitable as a standalone deduplication hash - use as pre-filter only +- Custom multicodec code may conflict with future official allocations + +## CID Example + +``` +File: movie.mkv (4.7 GB) +MidHash256: 3a7b...c2d1 (32 bytes) +CID: bafkreibh4xvl5... (base32lower encoded CIDv1) +``` + +## Integration + +The hash is computed separately from the standard single-pass algorithms since it requires seeking to the middle of the file. For files <= 1MB, it reads the entire file like other algorithms. diff --git a/src/api/configure.rs b/src/api/configure.rs new file mode 100644 index 0000000..03c9b0a --- /dev/null +++ b/src/api/configure.rs @@ -0,0 +1,66 @@ +use axum::{extract::State, Json}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::sync::Arc; + +use crate::api::process::ProcessState; +use crate::config::PluginConfig; + +#[derive(Deserialize)] +pub struct ConfigureRequest { + pub config: HashMap, +} + +#[derive(Serialize)] +pub struct ConfigureResponse { + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +pub async fn configure( + State(state): State>, + Json(req): Json, +) -> Json { + let mut cfg = state.config.write().await; + + // Handle enabledHashes config + if let Some(hashes) = req.config.get("enabledHashes") { + let enabled = PluginConfig::parse_enabled_hashes(hashes); + if enabled.is_empty() { + return Json(ConfigureResponse { + status: "error", + error: Some("No valid hash algorithms specified".to_string()), + }); + } + cfg.enabled_hashes = enabled; + tracing::info!("Updated enabled hashes: {:?}", cfg.enabled_hashes); + } + + // Handle buffer size config + if let Some(size) = req.config.get("bufferSize") { + match size.parse::() { + Ok(s) if s >= 4096 => { + cfg.buffer_size = s; + tracing::info!("Updated buffer size: {}", s); + } + Ok(_) => { + return Json(ConfigureResponse { + status: "error", + error: Some("Buffer size must be at least 4096 bytes".to_string()), + }); + } + Err(_) => { + return Json(ConfigureResponse { + status: "error", + error: Some("Invalid buffer size".to_string()), + }); + } + } + } + + Json(ConfigureResponse { + status: "ok", + error: None, + }) +} diff --git a/src/api/health.rs b/src/api/health.rs new file mode 100644 index 0000000..f896594 --- /dev/null +++ b/src/api/health.rs @@ -0,0 +1,17 @@ +use axum::Json; +use serde::Serialize; + +#[derive(Serialize)] +pub struct HealthResponse { + pub status: &'static str, + pub ready: bool, + pub version: &'static str, +} + +pub async fn health() -> Json { + Json(HealthResponse { + status: "healthy", + ready: true, + version: env!("CARGO_PKG_VERSION"), + }) +} diff --git a/src/api/manifest.rs b/src/api/manifest.rs new file mode 100644 index 0000000..0e566ea --- /dev/null +++ b/src/api/manifest.rs @@ -0,0 +1,99 @@ +use axum::Json; +use serde::Serialize; +use serde_json::{json, Value}; + +#[derive(Serialize)] +pub struct ManifestResponse { + pub id: &'static str, + pub name: &'static str, + pub version: &'static str, + pub description: &'static str, + pub author: &'static str, + pub dependencies: Vec, + pub priority: u32, + pub color: &'static str, + #[serde(rename = "defaultQueue")] + pub default_queue: &'static str, + pub timeout: u64, + pub schema: Value, + pub config: Value, +} + +pub async fn manifest() -> Json { + Json(ManifestResponse { + id: "fast-full-hash", + name: "Fast Full File Hash (Rust)", + version: env!("CARGO_PKG_VERSION"), + description: "High-performance parallel hashing with optimized I/O", + author: "MetaMesh", + dependencies: vec![], + priority: 100, + color: "#FF5722", + default_queue: "background", + timeout: 300000, + schema: json!({ + "cid_sha2-256": { + "label": "SHA-256 Hash", + "type": "cid", + "readonly": true, + "hint": "SHA-256 hash in CID format (multicodec 0x12)" + }, + "cid_sha1": { + "label": "SHA-1 Hash", + "type": "cid", + "readonly": true, + "hint": "SHA-1 hash in CID format (multicodec 0x11)" + }, + "cid_md5": { + "label": "MD5 Hash", + "type": "cid", + "readonly": true, + "hint": "MD5 hash in CID format (multicodec 0xd5)" + }, + "cid_crc32": { + "label": "CRC32 Checksum", + "type": "cid", + "readonly": true, + "hint": "CRC32 checksum in CID format (multicodec 0x0132)" + }, + "cid_sha3-256": { + "label": "SHA3-256 Hash", + "type": "cid", + "readonly": true, + "hint": "SHA3-256 hash in CID format (multicodec 0x16)" + }, + "cid_sha3-384": { + "label": "SHA3-384 Hash", + "type": "cid", + "readonly": true, + "hint": "SHA3-384 hash in CID format (multicodec 0x15)" + }, + "cid_midhash256": { + "label": "MidHash256", + "type": "cid", + "readonly": true, + "hint": "O(1) file identification hash - SHA-256 of size + middle 1MB (custom multicodec 0x1000)" + }, + "cid_bt_pieces_root": { + "label": "BitTorrent v2 Pieces Root", + "type": "cid", + "readonly": true, + "hint": "Merkle tree root of 16KB blocks (official multicodec 0xb702)" + }, + "cid_bt_info_hash": { + "label": "BitTorrent v2 Info Hash", + "type": "cid", + "readonly": true, + "hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)" + } + }), + config: json!({ + "enabledHashes": { + "type": "string", + "label": "Enabled hash algorithms (comma-separated)", + "default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash", + "required": false + } + }), + }) +} diff --git a/src/api/mod.rs b/src/api/mod.rs new file mode 100644 index 0000000..0d7ca24 --- /dev/null +++ b/src/api/mod.rs @@ -0,0 +1,9 @@ +mod configure; +mod health; +mod manifest; +mod process; + +pub use configure::configure; +pub use health::health; +pub use manifest::manifest; +pub use process::{process, ProcessState}; diff --git a/src/api/process.rs b/src/api/process.rs new file mode 100644 index 0000000..b7427ea --- /dev/null +++ b/src/api/process.rs @@ -0,0 +1,249 @@ +use axum::{extract::State, Json}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::path::Path; +use std::sync::Arc; +use std::time::Instant; + +use crate::cache::CacheManager; +use crate::client::MetaCoreClient; +use crate::config::{HashAlgorithm, SharedConfig}; +use crate::hasher::hash_file; + +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +pub struct ProcessRequest { + pub task_id: String, + pub cid: String, + pub file_path: String, + pub callback_url: String, + pub meta_core_url: String, + #[serde(default)] + pub existing_meta: HashMap, +} + +#[derive(Serialize)] +pub struct ProcessResponse { + pub status: &'static str, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +pub struct CallbackPayload { + pub task_id: String, + pub status: &'static str, + pub duration: u64, + #[serde(skip_serializing_if = "Option::is_none")] + pub error: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub reason: Option, +} + +pub struct ProcessState { + pub config: SharedConfig, + pub cache: Arc, +} + +pub async fn process( + State(state): State>, + Json(req): Json, +) -> Json { + // Validate required fields + if req.task_id.is_empty() || req.file_path.is_empty() || req.callback_url.is_empty() { + return Json(ProcessResponse { + status: "rejected", + error: Some("Missing required fields".to_string()), + }); + } + + // Spawn async processing task + let state_clone = Arc::clone(&state); + tokio::spawn(async move { + process_file(state_clone, req).await; + }); + + Json(ProcessResponse { + status: "accepted", + error: None, + }) +} + +async fn process_file(state: Arc, req: ProcessRequest) { + let start = Instant::now(); + let task_id = req.task_id.clone(); + let callback_url = req.callback_url.clone(); + + let result = do_process_file(state, req).await; + + let duration = start.elapsed().as_millis() as u64; + + let payload = match result { + Ok(ProcessResult::Completed) => CallbackPayload { + task_id, + status: "completed", + duration, + error: None, + reason: None, + }, + Ok(ProcessResult::Skipped(reason)) => CallbackPayload { + task_id, + status: "skipped", + duration, + error: None, + reason: Some(reason), + }, + Err(e) => CallbackPayload { + task_id, + status: "failed", + duration, + error: Some(e.to_string()), + reason: None, + }, + }; + + // Send callback + if let Err(e) = send_callback(&callback_url, &payload).await { + tracing::error!("Failed to send callback: {}", e); + } +} + +enum ProcessResult { + Completed, + Skipped(String), +} + +async fn do_process_file( + state: Arc, + req: ProcessRequest, +) -> anyhow::Result { + let path = Path::new(&req.file_path); + + // Check if file exists + if !path.exists() { + anyhow::bail!("File not found: {}", req.file_path); + } + + // Get file metadata + let metadata = std::fs::metadata(path)?; + let size = metadata.len(); + let mtime = metadata + .modified() + .ok() + .and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok()) + .map(|d| d.as_secs() as i64) + .unwrap_or(0); + + let filename = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + + // Get config + let config = state.config.read().await.clone(); + + // Determine which hashes we need to compute + let required_keys: Vec<&str> = config + .enabled_hashes + .iter() + .map(|a| a.key()) + .collect(); + + // Check existing metadata - skip if all hashes already present + let missing_from_meta: Vec<&str> = required_keys + .iter() + .filter(|k| !req.existing_meta.contains_key(**k)) + .copied() + .collect(); + + if missing_from_meta.is_empty() { + return Ok(ProcessResult::Skipped( + "All hashes already computed".to_string(), + )); + } + + // Check cache for remaining hashes + let cached = state.cache.get(&filename, size, mtime).await; + let mut results: HashMap = HashMap::new(); + + let to_compute: Vec = missing_from_meta + .iter() + .filter_map(|key| { + if let Some(ref cached_hashes) = cached { + if let Some(hash) = cached_hashes.get(*key) { + results.insert(key.to_string(), hash.clone()); + return None; + } + } + HashAlgorithm::from_key(key) + }) + .collect(); + + // If all hashes were in cache, we're done + if to_compute.is_empty() { + tracing::info!("All hashes from cache for {}", filename); + } else { + // Compute missing hashes + tracing::info!( + "Computing {} hashes for {} ({} bytes)", + to_compute.len(), + filename, + size + ); + + let enabled_set = to_compute.iter().copied().collect(); + let computed = tokio::task::spawn_blocking({ + let path = path.to_path_buf(); + let buffer_size = config.buffer_size; + move || hash_file(&path, &enabled_set, buffer_size) + }) + .await??; + + // Merge computed hashes into results + for (key, value) in &computed { + results.insert(key.clone(), value.clone()); + } + + // Update cache with new hashes + let mut all_hashes = cached.unwrap_or_default(); + all_hashes.extend(computed); + state + .cache + .insert(filename.clone(), size, mtime, all_hashes) + .await; + + // Save cache periodically (could be optimized with debouncing) + if let Err(e) = state.cache.save().await { + tracing::warn!("Failed to save cache: {}", e); + } + } + + // Store results to meta-core + if !results.is_empty() { + match MetaCoreClient::new(&req.meta_core_url) { + Ok(client) => { + if let Err(e) = client.merge_metadata(&req.cid, &results).await { + tracing::warn!("Failed to store metadata to meta-core: {}", e); + } + } + Err(e) => { + tracing::warn!("Failed to create meta-core client: {}", e); + } + } + } + + Ok(ProcessResult::Completed) +} + +async fn send_callback(url: &str, payload: &CallbackPayload) -> anyhow::Result<()> { + let client = reqwest::Client::new(); + let resp = client.post(url).json(payload).send().await?; + + if !resp.status().is_success() { + anyhow::bail!("Callback failed with status {}", resp.status()); + } + + Ok(()) +} diff --git a/src/cache/mod.rs b/src/cache/mod.rs new file mode 100644 index 0000000..69d99bf --- /dev/null +++ b/src/cache/mod.rs @@ -0,0 +1,200 @@ +use anyhow::{Context, Result}; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; +use std::fs::{self, File}; +use std::io::{BufReader, BufWriter}; +use std::path::{Path, PathBuf}; +use std::sync::Arc; +use tokio::sync::RwLock; + +const CACHE_VERSION: u32 = 1; +const CACHE_FILENAME: &str = "hash_cache.bin"; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheEntry { + pub filename: String, + pub size: u64, + pub mtime: i64, + pub hashes: HashMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CacheIndex { + pub version: u32, + pub entries: HashMap, +} + +impl Default for CacheIndex { + fn default() -> Self { + Self { + version: CACHE_VERSION, + entries: HashMap::new(), + } + } +} + +impl CacheIndex { + /// Generate cache key from file metadata + pub fn make_key(filename: &str, size: u64, mtime: i64) -> String { + format!("{}-{}-{}", filename, size, mtime) + } + + /// Get cached entry if it exists and matches file metadata + pub fn get(&self, filename: &str, size: u64, mtime: i64) -> Option<&CacheEntry> { + let key = Self::make_key(filename, size, mtime); + self.entries.get(&key) + } + + /// Insert or update a cache entry + pub fn insert(&mut self, filename: String, size: u64, mtime: i64, hashes: HashMap) { + let key = Self::make_key(&filename, size, mtime); + self.entries.insert( + key, + CacheEntry { + filename, + size, + mtime, + hashes, + }, + ); + } +} + +pub struct CacheManager { + cache_path: PathBuf, + index: Arc>, +} + +impl CacheManager { + pub fn new(cache_dir: &str) -> Self { + let cache_path = PathBuf::from(cache_dir).join(CACHE_FILENAME); + Self { + cache_path, + index: Arc::new(RwLock::new(CacheIndex::default())), + } + } + + /// Load cache from disk + pub async fn load(&self) -> Result<()> { + if !self.cache_path.exists() { + tracing::info!("No existing cache found at {:?}", self.cache_path); + return Ok(()); + } + + let path = self.cache_path.clone(); + let index = tokio::task::spawn_blocking(move || -> Result { + let file = File::open(&path).context("Failed to open cache file")?; + let reader = BufReader::new(file); + let index: CacheIndex = + bincode::deserialize_from(reader).context("Failed to deserialize cache")?; + + // Version check + if index.version != CACHE_VERSION { + tracing::warn!( + "Cache version mismatch (got {}, expected {}), starting fresh", + index.version, + CACHE_VERSION + ); + return Ok(CacheIndex::default()); + } + + Ok(index) + }) + .await??; + + tracing::info!("Loaded {} cache entries", index.entries.len()); + *self.index.write().await = index; + Ok(()) + } + + /// Save cache to disk atomically + pub async fn save(&self) -> Result<()> { + let index = self.index.read().await.clone(); + let cache_path = self.cache_path.clone(); + + tokio::task::spawn_blocking(move || -> Result<()> { + // Ensure parent directory exists + if let Some(parent) = cache_path.parent() { + fs::create_dir_all(parent)?; + } + + // Write to temp file first + let temp_path = cache_path.with_extension("tmp"); + { + let file = File::create(&temp_path)?; + let writer = BufWriter::new(file); + bincode::serialize_into(writer, &index)?; + } + + // Atomic rename + fs::rename(&temp_path, &cache_path)?; + Ok(()) + }) + .await??; + + Ok(()) + } + + /// Get cached hashes for a file + pub async fn get(&self, filename: &str, size: u64, mtime: i64) -> Option> { + let index = self.index.read().await; + index.get(filename, size, mtime).map(|e| e.hashes.clone()) + } + + /// Store hashes in cache + pub async fn insert(&self, filename: String, size: u64, mtime: i64, hashes: HashMap) { + let mut index = self.index.write().await; + index.insert(filename, size, mtime, hashes); + } + + /// Get missing hash algorithms for a file (not in cache or not computed) + pub async fn get_missing_algos( + &self, + filename: &str, + size: u64, + mtime: i64, + required: &[&str], + ) -> Vec { + let index = self.index.read().await; + let cached = index.get(filename, size, mtime); + + required + .iter() + .filter(|algo| { + cached + .map(|e| !e.hashes.contains_key(**algo)) + .unwrap_or(true) + }) + .map(|s| s.to_string()) + .collect() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use tempfile::TempDir; + + #[tokio::test] + async fn test_cache_roundtrip() { + let temp_dir = TempDir::new().unwrap(); + let cache = CacheManager::new(temp_dir.path().to_str().unwrap()); + + // Insert some data + let mut hashes = HashMap::new(); + hashes.insert("cid_sha2-256".to_string(), "baejbeitest".to_string()); + cache + .insert("test.mkv".to_string(), 1000, 12345, hashes.clone()) + .await; + + // Save and reload + cache.save().await.unwrap(); + + let cache2 = CacheManager::new(temp_dir.path().to_str().unwrap()); + cache2.load().await.unwrap(); + + let result = cache2.get("test.mkv", 1000, 12345).await; + assert!(result.is_some()); + assert_eq!(result.unwrap().get("cid_sha2-256").unwrap(), "baejbeitest"); + } +} diff --git a/src/client/meta_core.rs b/src/client/meta_core.rs new file mode 100644 index 0000000..b99bc03 --- /dev/null +++ b/src/client/meta_core.rs @@ -0,0 +1,139 @@ +use anyhow::{Context, Result}; +use reqwest::Client; +use serde_json::json; +use std::collections::HashMap; +use std::time::Duration; + +const REQUEST_TIMEOUT: Duration = Duration::from_secs(30); +const MAX_RETRIES: u32 = 3; +const RETRY_DELAY: Duration = Duration::from_millis(500); + +pub struct MetaCoreClient { + client: Client, + base_url: String, +} + +impl MetaCoreClient { + pub fn new(base_url: &str) -> Result { + let client = Client::builder() + .timeout(REQUEST_TIMEOUT) + .build() + .context("Failed to create HTTP client")?; + + Ok(Self { + client, + base_url: base_url.trim_end_matches('/').to_string(), + }) + } + + /// Set a single property for a hash ID + pub async fn set_property(&self, hash_id: &str, key: &str, value: &str) -> Result<()> { + let url = format!("{}/api/meta/{}/{}", self.base_url, hash_id, key); + + for attempt in 1..=MAX_RETRIES { + match self + .client + .put(&url) + .json(&json!({ "value": value })) + .send() + .await + { + Ok(resp) if resp.status().is_success() => { + return Ok(()); + } + Ok(resp) => { + let status = resp.status(); + let body = resp.text().await.unwrap_or_default(); + if attempt == MAX_RETRIES { + anyhow::bail!( + "Failed to set property {} for {}: {} - {}", + key, + hash_id, + status, + body + ); + } + tracing::warn!( + "Retry {}/{} for set_property: {} - {}", + attempt, + MAX_RETRIES, + status, + body + ); + } + Err(e) => { + if attempt == MAX_RETRIES { + return Err(e).context(format!( + "Failed to set property {} for {}", + key, hash_id + )); + } + tracing::warn!("Retry {}/{} for set_property: {}", attempt, MAX_RETRIES, e); + } + } + tokio::time::sleep(RETRY_DELAY).await; + } + + Ok(()) + } + + /// Merge multiple properties at once + pub async fn merge_metadata( + &self, + hash_id: &str, + metadata: &HashMap, + ) -> Result<()> { + // Try batch endpoint first + let url = format!("{}/api/meta/{}", self.base_url, hash_id); + + match self.client.patch(&url).json(metadata).send().await { + Ok(resp) if resp.status().is_success() => { + return Ok(()); + } + Ok(resp) if resp.status().as_u16() == 404 => { + // Batch endpoint not available, fall back to individual sets + tracing::debug!("Batch endpoint not available, using individual sets"); + } + Ok(resp) => { + let status = resp.status(); + tracing::warn!("Batch merge failed with {}, falling back to individual sets", status); + } + Err(e) => { + tracing::warn!("Batch merge error: {}, falling back to individual sets", e); + } + } + + // Fall back to individual property sets + for (key, value) in metadata { + self.set_property(hash_id, key, value).await?; + } + + Ok(()) + } + + /// Health check + pub async fn health(&self) -> Result { + let url = format!("{}/health", self.base_url); + match self.client.get(&url).send().await { + Ok(resp) => Ok(resp.status().is_success()), + Err(_) => Ok(false), + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_client_creation() { + let client = MetaCoreClient::new("http://localhost:9000"); + assert!(client.is_ok()); + } + + #[test] + fn test_url_normalization() { + let client = MetaCoreClient::new("http://localhost:9000/").unwrap(); + assert_eq!(client.base_url, "http://localhost:9000"); + } +} diff --git a/src/client/mod.rs b/src/client/mod.rs new file mode 100644 index 0000000..be6d468 --- /dev/null +++ b/src/client/mod.rs @@ -0,0 +1,3 @@ +mod meta_core; + +pub use meta_core::MetaCoreClient; diff --git a/src/config.rs b/src/config.rs new file mode 100644 index 0000000..1f34093 --- /dev/null +++ b/src/config.rs @@ -0,0 +1,162 @@ +use serde::{Deserialize, Serialize}; +use std::collections::HashSet; +use std::sync::Arc; +use tokio::sync::RwLock; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +pub enum HashAlgorithm { + #[serde(rename = "cid_sha2-256")] + Sha256, + #[serde(rename = "cid_sha1")] + Sha1, + #[serde(rename = "cid_md5")] + Md5, + #[serde(rename = "cid_crc32")] + Crc32, + #[serde(rename = "cid_sha3-256")] + Sha3_256, + #[serde(rename = "cid_sha3-384")] + Sha3_384, + #[serde(rename = "cid_midhash256")] + Midhash256, + #[serde(rename = "cid_bt_pieces_root")] + BtPiecesRoot, + #[serde(rename = "cid_bt_info_hash")] + BtInfoHash, +} + +// Multicodec hash codes +// Standard codes from https://github.com/multiformats/multicodec +pub const SHA256_CODE: u64 = 0x12; +pub const SHA1_CODE: u64 = 0x11; +pub const MD5_CODE: u64 = 0xd5; +pub const CRC32_CODE: u64 = 0x0132; +pub const SHA3_256_CODE: u64 = 0x16; +pub const SHA3_384_CODE: u64 = 0x15; + +// Custom hash codes +pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry + +// Official BitTorrent v2 code +pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root +// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash + +impl HashAlgorithm { + pub fn all() -> Vec { + vec![ + HashAlgorithm::Sha256, + HashAlgorithm::Sha1, + HashAlgorithm::Md5, + HashAlgorithm::Crc32, + HashAlgorithm::Sha3_256, + HashAlgorithm::Sha3_384, + HashAlgorithm::Midhash256, + HashAlgorithm::BtPiecesRoot, + HashAlgorithm::BtInfoHash, + ] + } + + /// Standard algorithms that can be computed in a single I/O pass + pub fn standard_algorithms() -> Vec { + vec![ + HashAlgorithm::Sha256, + HashAlgorithm::Sha1, + HashAlgorithm::Md5, + HashAlgorithm::Crc32, + HashAlgorithm::Sha3_256, + HashAlgorithm::Sha3_384, + ] + } + + /// Special algorithms that require separate file reads + pub fn special_algorithms() -> Vec { + vec![ + HashAlgorithm::Midhash256, + HashAlgorithm::BtPiecesRoot, + HashAlgorithm::BtInfoHash, + ] + } + + /// Returns true if this algorithm requires a separate file read + pub fn is_special(&self) -> bool { + matches!( + self, + HashAlgorithm::Midhash256 | HashAlgorithm::BtPiecesRoot | HashAlgorithm::BtInfoHash + ) + } + + pub fn key(&self) -> &'static str { + match self { + HashAlgorithm::Sha256 => "cid_sha2-256", + HashAlgorithm::Sha1 => "cid_sha1", + HashAlgorithm::Md5 => "cid_md5", + HashAlgorithm::Crc32 => "cid_crc32", + HashAlgorithm::Sha3_256 => "cid_sha3-256", + HashAlgorithm::Sha3_384 => "cid_sha3-384", + HashAlgorithm::Midhash256 => "cid_midhash256", + HashAlgorithm::BtPiecesRoot => "cid_bt_pieces_root", + HashAlgorithm::BtInfoHash => "cid_bt_info_hash", + } + } + + pub fn from_key(key: &str) -> Option { + match key { + "cid_sha2-256" => Some(HashAlgorithm::Sha256), + "cid_sha1" => Some(HashAlgorithm::Sha1), + "cid_md5" => Some(HashAlgorithm::Md5), + "cid_crc32" => Some(HashAlgorithm::Crc32), + "cid_sha3-256" => Some(HashAlgorithm::Sha3_256), + "cid_sha3-384" => Some(HashAlgorithm::Sha3_384), + "cid_midhash256" => Some(HashAlgorithm::Midhash256), + "cid_bt_pieces_root" => Some(HashAlgorithm::BtPiecesRoot), + "cid_bt_info_hash" => Some(HashAlgorithm::BtInfoHash), + _ => None, + } + } + + /// Returns the multicodec code for this hash algorithm + pub fn multicodec(&self) -> u64 { + match self { + HashAlgorithm::Sha256 => SHA256_CODE, + HashAlgorithm::Sha1 => SHA1_CODE, + HashAlgorithm::Md5 => MD5_CODE, + HashAlgorithm::Crc32 => CRC32_CODE, + HashAlgorithm::Sha3_256 => SHA3_256_CODE, + HashAlgorithm::Sha3_384 => SHA3_384_CODE, + HashAlgorithm::Midhash256 => MIDHASH256_CODE, + HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE, + HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256 + } + } +} + +#[derive(Debug, Clone)] +pub struct PluginConfig { + pub enabled_hashes: HashSet, + pub cache_path: String, + pub buffer_size: usize, +} + +impl Default for PluginConfig { + fn default() -> Self { + Self { + enabled_hashes: HashAlgorithm::all().into_iter().collect(), + cache_path: std::env::var("CACHE_PATH").unwrap_or_else(|_| "/cache".to_string()), + buffer_size: 1024 * 1024, // 1MB + } + } +} + +impl PluginConfig { + pub fn parse_enabled_hashes(s: &str) -> HashSet { + s.split(',') + .filter_map(|k| HashAlgorithm::from_key(k.trim())) + .collect() + } +} + +pub type SharedConfig = Arc>; + +pub fn create_shared_config() -> SharedConfig { + Arc::new(RwLock::new(PluginConfig::default())) +} diff --git a/src/hasher/algorithms.rs b/src/hasher/algorithms.rs new file mode 100644 index 0000000..15313d8 --- /dev/null +++ b/src/hasher/algorithms.rs @@ -0,0 +1,383 @@ +use crate::config::HashAlgorithm; +use anyhow::Result; +use md5::Md5; +use sha1::Sha1; +use sha2::{Digest, Sha256}; +use sha3::{Sha3_256, Sha3_384}; +use std::collections::{HashMap, HashSet}; +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::Path; + +use super::btih_v2::{compute_info_hash, compute_merkle_root, BT_BLOCK_SIZE}; +use super::cid::to_cid; +use super::midhash256::compute_midhash256; + +/// Hash a file using all enabled algorithms in a single I/O pass +/// +/// All algorithms except Midhash256 are computed in the same pass: +/// - Standard algorithms update their hashers with each buffer chunk +/// - BitTorrent v2 computes 16KB block hashes from the same data +/// +/// Midhash256 requires a separate O(1) read (middle 1MB only). +pub fn hash_file( + path: &Path, + enabled: &HashSet, + buffer_size: usize, +) -> Result> { + let mut results = HashMap::new(); + + // Check what we need to compute + let needs_bt = enabled.contains(&HashAlgorithm::BtPiecesRoot) + || enabled.contains(&HashAlgorithm::BtInfoHash); + + // Get filename for BT v2 info hash (needed before we start) + let filename = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + + // Main single-pass computation (standard + BT v2) + let (standard_results, bt_leaf_hashes, file_size) = + hash_file_single_pass(path, enabled, buffer_size, needs_bt)?; + results.extend(standard_results); + + // Finalize BT v2 hashes from collected leaf hashes + if needs_bt { + let pieces_root = compute_merkle_root(&bt_leaf_hashes); + + if enabled.contains(&HashAlgorithm::BtPiecesRoot) { + results.insert( + HashAlgorithm::BtPiecesRoot.key().to_string(), + to_cid(&pieces_root, HashAlgorithm::BtPiecesRoot), + ); + } + + if enabled.contains(&HashAlgorithm::BtInfoHash) { + match compute_info_hash(&filename, file_size, &pieces_root) { + Ok(info_hash) => { + results.insert( + HashAlgorithm::BtInfoHash.key().to_string(), + to_cid(&info_hash, HashAlgorithm::BtInfoHash), + ); + } + Err(e) => { + tracing::warn!("Failed to compute BT info hash: {}", e); + } + } + } + } + + // Midhash256 requires separate O(1) read (only reads middle 1MB) + if enabled.contains(&HashAlgorithm::Midhash256) { + match compute_midhash256(path) { + Ok(hash) => { + results.insert( + HashAlgorithm::Midhash256.key().to_string(), + to_cid(&hash, HashAlgorithm::Midhash256), + ); + } + Err(e) => { + tracing::warn!("Failed to compute midhash256: {}", e); + } + } + } + + Ok(results) +} + +/// Single-pass file hashing for all standard algorithms + BT v2 leaf collection +/// +/// Returns: (standard hash results, BT v2 leaf hashes, file size) +fn hash_file_single_pass( + path: &Path, + enabled: &HashSet, + buffer_size: usize, + collect_bt_leaves: bool, +) -> Result<(HashMap, Vec<[u8; 32]>, u64)> { + let file = File::open(path)?; + let file_size = file.metadata()?.len(); + let mut reader = BufReader::with_capacity(buffer_size, file); + let mut buffer = vec![0u8; buffer_size]; + + // Initialize standard hashers (only if enabled) + let mut sha256 = enabled + .contains(&HashAlgorithm::Sha256) + .then(Sha256::new); + let mut sha1 = enabled.contains(&HashAlgorithm::Sha1).then(Sha1::new); + let mut md5 = enabled.contains(&HashAlgorithm::Md5).then(Md5::new); + let mut crc32 = enabled + .contains(&HashAlgorithm::Crc32) + .then(crc32fast::Hasher::new); + let mut sha3_256 = enabled + .contains(&HashAlgorithm::Sha3_256) + .then(Sha3_256::new); + let mut sha3_384 = enabled + .contains(&HashAlgorithm::Sha3_384) + .then(Sha3_384::new); + + // BT v2: collect leaf hashes (SHA-256 of each 16KB block) + let mut bt_leaf_hashes: Vec<[u8; 32]> = Vec::new(); + let mut bt_block_buffer = vec![0u8; BT_BLOCK_SIZE]; + let mut bt_block_offset = 0usize; + + let mut bytes_read: u64 = 0; + + // Single read pass + loop { + let n = reader.read(&mut buffer)?; + if n == 0 { + break; + } + + let chunk = &buffer[..n]; + bytes_read += n as u64; + + // Update all standard hashers + if let Some(ref mut h) = sha256 { + h.update(chunk); + } + if let Some(ref mut h) = sha1 { + h.update(chunk); + } + if let Some(ref mut h) = md5 { + h.update(chunk); + } + if let Some(ref mut h) = crc32 { + h.update(chunk); + } + if let Some(ref mut h) = sha3_256 { + h.update(chunk); + } + if let Some(ref mut h) = sha3_384 { + h.update(chunk); + } + + // BT v2: Process chunk into 16KB blocks + if collect_bt_leaves { + let mut chunk_offset = 0; + while chunk_offset < n { + let space_in_block = BT_BLOCK_SIZE - bt_block_offset; + let bytes_to_copy = space_in_block.min(n - chunk_offset); + + bt_block_buffer[bt_block_offset..bt_block_offset + bytes_to_copy] + .copy_from_slice(&chunk[chunk_offset..chunk_offset + bytes_to_copy]); + + bt_block_offset += bytes_to_copy; + chunk_offset += bytes_to_copy; + + // Block is full, hash it + if bt_block_offset == BT_BLOCK_SIZE { + let mut hasher = Sha256::new(); + hasher.update(&bt_block_buffer); + bt_leaf_hashes.push(hasher.finalize().into()); + bt_block_offset = 0; + } + } + } + + // Log progress for large files (every 100MB) + if bytes_read % (100 * 1024 * 1024) == 0 && file_size > 100 * 1024 * 1024 { + let pct = (bytes_read as f64 / file_size as f64 * 100.0) as u32; + tracing::debug!( + "Hashing progress: {}% ({}/{})", + pct, + bytes_read, + file_size + ); + } + } + + // BT v2: Handle final partial block (pad with zeros as per BEP 52) + if collect_bt_leaves { + if file_size == 0 { + // Empty file: single leaf of all zeros + bt_leaf_hashes.push([0u8; 32]); + } else if bt_block_offset > 0 { + // Pad remaining bytes with zeros + bt_block_buffer[bt_block_offset..].fill(0); + let mut hasher = Sha256::new(); + hasher.update(&bt_block_buffer); + bt_leaf_hashes.push(hasher.finalize().into()); + } + } + + // Finalize standard hashes and format as CIDs + let mut results = HashMap::new(); + + if let Some(h) = sha256 { + let hash = h.finalize(); + results.insert( + HashAlgorithm::Sha256.key().to_string(), + to_cid(&hash, HashAlgorithm::Sha256), + ); + } + + if let Some(h) = sha1 { + let hash = h.finalize(); + results.insert( + HashAlgorithm::Sha1.key().to_string(), + to_cid(&hash, HashAlgorithm::Sha1), + ); + } + + if let Some(h) = md5 { + let hash = h.finalize(); + results.insert( + HashAlgorithm::Md5.key().to_string(), + to_cid(&hash, HashAlgorithm::Md5), + ); + } + + if let Some(h) = crc32 { + let hash = h.finalize().to_be_bytes(); + results.insert( + HashAlgorithm::Crc32.key().to_string(), + to_cid(&hash, HashAlgorithm::Crc32), + ); + } + + if let Some(h) = sha3_256 { + let hash = h.finalize(); + results.insert( + HashAlgorithm::Sha3_256.key().to_string(), + to_cid(&hash, HashAlgorithm::Sha3_256), + ); + } + + if let Some(h) = sha3_384 { + let hash = h.finalize(); + results.insert( + HashAlgorithm::Sha3_384.key().to_string(), + to_cid(&hash, HashAlgorithm::Sha3_384), + ); + } + + Ok((results, bt_leaf_hashes, file_size)) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_hash_file_standard() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"hello world").unwrap(); + file.flush().unwrap(); + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha256, HashAlgorithm::Md5] + .into_iter() + .collect(); + + let results = hash_file(file.path(), &enabled, 1024).unwrap(); + + assert!(results.contains_key("cid_sha2-256")); + assert!(results.contains_key("cid_md5")); + assert!(!results.contains_key("cid_sha1")); + } + + #[test] + fn test_hash_file_with_midhash() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"test content for midhash").unwrap(); + file.flush().unwrap(); + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha256, HashAlgorithm::Midhash256] + .into_iter() + .collect(); + + let results = hash_file(file.path(), &enabled, 1024).unwrap(); + + assert!(results.contains_key("cid_sha2-256")); + assert!(results.contains_key("cid_midhash256")); + } + + #[test] + fn test_hash_file_with_bittorrent() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"test content for bittorrent").unwrap(); + file.flush().unwrap(); + + let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash] + .into_iter() + .collect(); + + let results = hash_file(file.path(), &enabled, 1024).unwrap(); + + assert!(results.contains_key("cid_bt_pieces_root")); + assert!(results.contains_key("cid_bt_info_hash")); + } + + #[test] + fn test_hash_file_all_algorithms() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"comprehensive test content").unwrap(); + file.flush().unwrap(); + + let enabled: HashSet<_> = HashAlgorithm::all().into_iter().collect(); + + let results = hash_file(file.path(), &enabled, 1024).unwrap(); + + // All 9 algorithms should be present + assert!(results.contains_key("cid_sha2-256")); + assert!(results.contains_key("cid_sha1")); + assert!(results.contains_key("cid_md5")); + assert!(results.contains_key("cid_crc32")); + assert!(results.contains_key("cid_sha3-256")); + assert!(results.contains_key("cid_sha3-384")); + assert!(results.contains_key("cid_midhash256")); + assert!(results.contains_key("cid_bt_pieces_root")); + assert!(results.contains_key("cid_bt_info_hash")); + } + + #[test] + fn test_bt_hashes_match_standalone() { + // Verify that single-pass BT hashes match the standalone implementation + let mut file = NamedTempFile::new().unwrap(); + // Use data larger than one BT block to test multi-block handling + let data = vec![0xab; BT_BLOCK_SIZE * 2 + 1000]; + file.write_all(&data).unwrap(); + file.flush().unwrap(); + + // Compute via single-pass + let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash] + .into_iter() + .collect(); + let single_pass_results = hash_file(file.path(), &enabled, 1024 * 1024).unwrap(); + + // Compute via standalone (which still exists for comparison) + use super::super::btih_v2::compute_btv2_hashes; + let standalone = compute_btv2_hashes(file.path()).unwrap(); + let standalone_pieces_root = to_cid(&standalone.pieces_root, HashAlgorithm::BtPiecesRoot); + let standalone_info_hash = to_cid(&standalone.info_hash, HashAlgorithm::BtInfoHash); + + // Results should match + assert_eq!( + single_pass_results.get("cid_bt_pieces_root").unwrap(), + &standalone_pieces_root, + "Pieces root mismatch" + ); + assert_eq!( + single_pass_results.get("cid_bt_info_hash").unwrap(), + &standalone_info_hash, + "Info hash mismatch" + ); + } + + #[test] + fn test_empty_file_bt() { + let file = NamedTempFile::new().unwrap(); + + let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash] + .into_iter() + .collect(); + let results = hash_file(file.path(), &enabled, 1024).unwrap(); + + assert!(results.contains_key("cid_bt_pieces_root")); + assert!(results.contains_key("cid_bt_info_hash")); + } +} diff --git a/src/hasher/btih_v2.rs b/src/hasher/btih_v2.rs new file mode 100644 index 0000000..a06841f --- /dev/null +++ b/src/hasher/btih_v2.rs @@ -0,0 +1,368 @@ +//! BitTorrent v2 Hash Computation (BEP 52) +//! +//! Computes two hashes for BitTorrent v2 compatibility: +//! - pieces_root: Merkle tree root of 16KB block hashes +//! - info_hash: SHA-256 of bencoded info dictionary +//! +//! Reference: https://www.bittorrent.org/beps/bep_0052.html + +use anyhow::Result; +use bendy::encoding::{AsString, SingleItemEncoder, ToBencode}; +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::io::{BufReader, Read}; +use std::path::Path; + +/// Block size for BitTorrent v2 (16 KiB) +pub const BT_BLOCK_SIZE: usize = 16 * 1024; + +/// Piece size (must be power of 2, minimum 16 KiB) +/// We use 16 KiB for maximum granularity +const PIECE_SIZE: usize = 16 * 1024; + +/// Result of BitTorrent v2 hash computation +pub struct BtV2Hashes { + /// Merkle tree root of file pieces (32 bytes) + pub pieces_root: [u8; 32], + /// SHA-256 of bencoded info dict (32 bytes) + pub info_hash: [u8; 32], +} + +/// Compute BitTorrent v2 hashes for a file +pub fn compute_btv2_hashes(path: &Path) -> Result { + let file = File::open(path)?; + let metadata = file.metadata()?; + let file_size = metadata.len(); + + // Get filename for info dict + let filename = path + .file_name() + .and_then(|n| n.to_str()) + .unwrap_or("unknown") + .to_string(); + + // Compute leaf hashes (hash of each 16KB block) + let leaf_hashes = compute_leaf_hashes(&file, file_size)?; + + // Build merkle tree and get root + let pieces_root = compute_merkle_root(&leaf_hashes); + + // Compute info hash + let info_hash = compute_info_hash(&filename, file_size, &pieces_root)?; + + Ok(BtV2Hashes { + pieces_root, + info_hash, + }) +} + +/// Compute SHA-256 hash of each 16KB block +fn compute_leaf_hashes(file: &File, file_size: u64) -> Result> { + if file_size == 0 { + // Empty file: single leaf of all zeros + return Ok(vec![[0u8; 32]]); + } + + let mut reader = BufReader::with_capacity(BT_BLOCK_SIZE * 4, file); + let mut buffer = vec![0u8; BT_BLOCK_SIZE]; + let mut leaf_hashes = Vec::new(); + + loop { + let bytes_read = reader.read(&mut buffer)?; + if bytes_read == 0 { + break; + } + + // Pad last block with zeros if needed (as per BEP 52) + if bytes_read < BT_BLOCK_SIZE { + buffer[bytes_read..].fill(0); + } + + let mut hasher = Sha256::new(); + hasher.update(&buffer[..BT_BLOCK_SIZE]); + let hash: [u8; 32] = hasher.finalize().into(); + leaf_hashes.push(hash); + + // If this was a partial read, we're done + if bytes_read < BT_BLOCK_SIZE { + break; + } + } + + Ok(leaf_hashes) +} + +/// Compute Merkle tree root from leaf hashes +/// +/// BEP 52 merkle tree: +/// - Leaves are SHA-256 hashes of 16KB blocks +/// - Internal nodes are SHA-256(left || right) +/// - Tree is padded to power of 2 with zero hashes +pub fn compute_merkle_root(leaf_hashes: &[[u8; 32]]) -> [u8; 32] { + if leaf_hashes.is_empty() { + return [0u8; 32]; + } + + if leaf_hashes.len() == 1 { + return leaf_hashes[0]; + } + + // Pad to next power of 2 + let target_size = leaf_hashes.len().next_power_of_two(); + let mut current_level: Vec<[u8; 32]> = leaf_hashes.to_vec(); + + // Pad with zero hashes + while current_level.len() < target_size { + current_level.push([0u8; 32]); + } + + // Build tree bottom-up + while current_level.len() > 1 { + let mut next_level = Vec::with_capacity(current_level.len() / 2); + + for chunk in current_level.chunks(2) { + let mut hasher = Sha256::new(); + hasher.update(&chunk[0]); + hasher.update(&chunk[1]); + let hash: [u8; 32] = hasher.finalize().into(); + next_level.push(hash); + } + + current_level = next_level; + } + + current_level[0] +} + +/// Info dictionary structure for bencoding +struct InfoDict<'a> { + file_tree: FileTree<'a>, + meta_version: i64, + name: &'a str, + piece_length: i64, +} + +/// File tree structure (single file case) +struct FileTree<'a> { + filename: &'a str, + length: i64, + pieces_root: &'a [u8; 32], +} + +impl ToBencode for InfoDict<'_> { + const MAX_DEPTH: usize = 5; + + fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> { + encoder.emit_dict(|mut e| { + // file tree (required for v2) + e.emit_pair(b"file tree", &self.file_tree)?; + // meta version (2 for v2-only torrents) + e.emit_pair(b"meta version", self.meta_version)?; + // name + e.emit_pair(b"name", self.name)?; + // piece length + e.emit_pair(b"piece length", self.piece_length)?; + Ok(()) + }) + } +} + +impl ToBencode for FileTree<'_> { + const MAX_DEPTH: usize = 4; + + fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> { + encoder.emit_dict(|mut e| { + // File tree is: { filename: { "": { length: N, pieces root: HASH } } } + e.emit_pair(self.filename.as_bytes(), FileEntry { + length: self.length, + pieces_root: self.pieces_root, + })?; + Ok(()) + }) + } +} + +struct FileEntry<'a> { + length: i64, + pieces_root: &'a [u8; 32], +} + +impl ToBencode for FileEntry<'_> { + const MAX_DEPTH: usize = 3; + + fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> { + encoder.emit_dict(|mut e| { + // Empty string key contains file attributes + e.emit_pair(b"", FileAttrs { + length: self.length, + pieces_root: self.pieces_root, + })?; + Ok(()) + }) + } +} + +struct FileAttrs<'a> { + length: i64, + pieces_root: &'a [u8; 32], +} + +impl ToBencode for FileAttrs<'_> { + const MAX_DEPTH: usize = 2; + + fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> { + encoder.emit_dict(|mut e| { + e.emit_pair(b"length", self.length)?; + e.emit_pair(b"pieces root", AsString(self.pieces_root))?; + Ok(()) + }) + } +} + +/// Compute info hash (SHA-256 of bencoded info dict) +pub fn compute_info_hash(filename: &str, file_size: u64, pieces_root: &[u8; 32]) -> Result<[u8; 32]> { + let info = InfoDict { + file_tree: FileTree { + filename, + length: file_size as i64, + pieces_root, + }, + meta_version: 2, + name: filename, + piece_length: PIECE_SIZE as i64, + }; + + let encoded = info + .to_bencode() + .map_err(|e| anyhow::anyhow!("Failed to bencode info dict: {:?}", e))?; + + let mut hasher = Sha256::new(); + hasher.update(&encoded); + Ok(hasher.finalize().into()) +} + +/// Format info hash as magnet link URN +/// +/// Format: urn:btmh:1220 +/// Where 1220 = 0x12 (SHA-256 code) + 0x20 (32 bytes length) +pub fn format_magnet_urn(info_hash: &[u8; 32]) -> String { + let hex = info_hash + .iter() + .map(|b| format!("{:02x}", b)) + .collect::(); + format!("urn:btmh:1220{}", hex) +} + +/// Format full magnet link +pub fn format_magnet_link(info_hash: &[u8; 32], filename: &str) -> String { + let urn = format_magnet_urn(info_hash); + let encoded_name = urlencoding_filename(filename); + format!("magnet:?xt={}&dn={}", urn, encoded_name) +} + +/// Simple URL encoding for filename (spaces and special chars) +fn urlencoding_filename(s: &str) -> String { + s.chars() + .map(|c| match c { + 'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(), + ' ' => "%20".to_string(), + _ => format!("%{:02X}", c as u8), + }) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_empty_file() { + let file = NamedTempFile::new().unwrap(); + let hashes = compute_btv2_hashes(file.path()).unwrap(); + + assert_eq!(hashes.pieces_root.len(), 32); + assert_eq!(hashes.info_hash.len(), 32); + } + + #[test] + fn test_small_file() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"hello world").unwrap(); + file.flush().unwrap(); + + let hashes = compute_btv2_hashes(file.path()).unwrap(); + + assert_eq!(hashes.pieces_root.len(), 32); + assert_eq!(hashes.info_hash.len(), 32); + + // Verify determinism + let hashes2 = compute_btv2_hashes(file.path()).unwrap(); + assert_eq!(hashes.pieces_root, hashes2.pieces_root); + assert_eq!(hashes.info_hash, hashes2.info_hash); + } + + #[test] + fn test_exactly_one_block() { + let mut file = NamedTempFile::new().unwrap(); + let data = vec![0xab; BT_BLOCK_SIZE]; + file.write_all(&data).unwrap(); + file.flush().unwrap(); + + let hashes = compute_btv2_hashes(file.path()).unwrap(); + assert_eq!(hashes.pieces_root.len(), 32); + } + + #[test] + fn test_multiple_blocks() { + let mut file = NamedTempFile::new().unwrap(); + // 3 full blocks + partial + let data = vec![0xcd; BT_BLOCK_SIZE * 3 + 100]; + file.write_all(&data).unwrap(); + file.flush().unwrap(); + + let hashes = compute_btv2_hashes(file.path()).unwrap(); + assert_eq!(hashes.pieces_root.len(), 32); + assert_eq!(hashes.info_hash.len(), 32); + } + + #[test] + fn test_merkle_root_single_leaf() { + let leaf = [0xab; 32]; + let root = compute_merkle_root(&[leaf]); + assert_eq!(root, leaf); + } + + #[test] + fn test_merkle_root_two_leaves() { + let leaf1 = [0x11; 32]; + let leaf2 = [0x22; 32]; + + let root = compute_merkle_root(&[leaf1, leaf2]); + + // Manually compute expected root + let mut hasher = Sha256::new(); + hasher.update(&leaf1); + hasher.update(&leaf2); + let expected: [u8; 32] = hasher.finalize().into(); + + assert_eq!(root, expected); + } + + #[test] + fn test_magnet_urn_format() { + let hash = [0u8; 32]; + let urn = format_magnet_urn(&hash); + assert!(urn.starts_with("urn:btmh:1220")); + assert_eq!(urn.len(), "urn:btmh:1220".len() + 64); // 64 hex chars + } + + #[test] + fn test_magnet_link_format() { + let hash = [0xab; 32]; + let link = format_magnet_link(&hash, "test file.mkv"); + assert!(link.starts_with("magnet:?xt=urn:btmh:1220")); + assert!(link.contains("&dn=test%20file.mkv")); + } +} diff --git a/src/hasher/cid.rs b/src/hasher/cid.rs new file mode 100644 index 0000000..36927c6 --- /dev/null +++ b/src/hasher/cid.rs @@ -0,0 +1,110 @@ +use crate::config::HashAlgorithm; +use cid::Cid; +use multihash::Multihash; + +/// CID codec for raw binary data +const RAW_CODEC: u64 = 0x55; + +/// Maximum multihash size (64 bytes for SHA3-512) +const MAX_HASH_SIZE: usize = 64; + +/// Convert raw hash bytes to proper CIDv1 format +/// +/// CID structure: +/// - Version: 1 +/// - Codec: 0x55 (raw binary data) +/// - Multihash: hash_code (varint) + hash_length (varint) + hash_bytes +/// +/// Returns base32lower encoded CID with 'b' multibase prefix +pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String { + let hash_code = algo.multicodec(); + + // Create multihash: wrap hash bytes with algorithm code + let multihash: Multihash = + Multihash::wrap(hash_code, hash).expect("Hash size exceeds maximum"); + + // Create CIDv1 with raw codec + let cid = Cid::new_v1(RAW_CODEC, multihash); + + // Return base32lower string (multibase 'b' prefix is default for CIDv1) + cid.to_string() +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_cid_format() { + // Test with a known SHA-256 hash + let hash = [0u8; 32]; // 32 zero bytes + let cid = to_cid(&hash, HashAlgorithm::Sha256); + + // CIDv1 base32lower starts with 'b' + assert!(cid.starts_with('b'), "CID should start with 'b' multibase prefix"); + + // Verify we can parse it back + let parsed = Cid::try_from(cid.as_str()).expect("Should parse valid CID"); + assert_eq!(parsed.version(), cid::Version::V1); + assert_eq!(parsed.codec(), RAW_CODEC); + assert_eq!(parsed.hash().code(), 0x12); // SHA-256 code + } + + #[test] + fn test_all_algorithms() { + let test_data = b"test data for hashing"; + + for algo in HashAlgorithm::all() { + // Create a fake hash of appropriate size + let hash_size = match algo { + HashAlgorithm::Sha256 | HashAlgorithm::Sha3_256 | HashAlgorithm::Midhash256 => 32, + HashAlgorithm::Sha1 => 20, + HashAlgorithm::Md5 => 16, + HashAlgorithm::Crc32 => 4, + HashAlgorithm::Sha3_384 => 48, + HashAlgorithm::BtPiecesRoot | HashAlgorithm::BtInfoHash => 32, + }; + + let hash = vec![0xab; hash_size]; + let cid = to_cid(&hash, algo); + + // All CIDs should start with 'b' (base32lower) + assert!( + cid.starts_with('b'), + "CID for {:?} should start with 'b'", + algo + ); + + // All CIDs should be parseable + let parsed = Cid::try_from(cid.as_str()) + .expect(&format!("Should parse valid CID for {:?}", algo)); + + assert_eq!(parsed.version(), cid::Version::V1); + assert_eq!(parsed.codec(), RAW_CODEC); + assert_eq!( + parsed.hash().code(), + algo.multicodec(), + "Multicodec mismatch for {:?}", + algo + ); + } + } + + #[test] + fn test_cid_determinism() { + // Same hash should always produce same CID + let hash = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16]; + let cid1 = to_cid(&hash, HashAlgorithm::Md5); + let cid2 = to_cid(&hash, HashAlgorithm::Md5); + assert_eq!(cid1, cid2); + } + + #[test] + fn test_different_algorithms_different_cids() { + // Same hash bytes with different algorithms should produce different CIDs + let hash = [0u8; 32]; + let sha256_cid = to_cid(&hash, HashAlgorithm::Sha256); + let sha3_256_cid = to_cid(&hash, HashAlgorithm::Sha3_256); + assert_ne!(sha256_cid, sha3_256_cid); + } +} diff --git a/src/hasher/midhash256.rs b/src/hasher/midhash256.rs new file mode 100644 index 0000000..a338d12 --- /dev/null +++ b/src/hasher/midhash256.rs @@ -0,0 +1,142 @@ +//! MidHash256 - O(1) file identification hash +//! +//! Computes a SHA-256 hash of the file size and middle portion, +//! providing constant-time file identification regardless of file size. +//! +//! Algorithm: +//! - Files ≤1MB: SHA-256(size_u64_be || entire_content) +//! - Files >1MB: SHA-256(size_u64_be || middle_1MB) + +use anyhow::Result; +use sha2::{Digest, Sha256}; +use std::fs::File; +use std::io::{Read, Seek, SeekFrom}; +use std::path::Path; + +/// Size threshold: files at or below this size are hashed entirely +const SIZE_THRESHOLD: u64 = 1024 * 1024; // 1MB + +/// Sample size for files larger than threshold +const SAMPLE_SIZE: usize = 1024 * 1024; // 1MB + +/// Compute midhash256 for a file +/// +/// Returns the raw 32-byte SHA-256 hash +pub fn compute_midhash256(path: &Path) -> Result<[u8; 32]> { + let mut file = File::open(path)?; + let metadata = file.metadata()?; + let file_size = metadata.len(); + + let mut hasher = Sha256::new(); + + // Always include file size as 8-byte big-endian prefix + hasher.update(file_size.to_be_bytes()); + + if file_size <= SIZE_THRESHOLD { + // Small file: hash entire content + let mut buffer = Vec::with_capacity(file_size as usize); + file.read_to_end(&mut buffer)?; + hasher.update(&buffer); + } else { + // Large file: hash middle 1MB + let middle_start = (file_size - SAMPLE_SIZE as u64) / 2; + file.seek(SeekFrom::Start(middle_start))?; + + let mut buffer = vec![0u8; SAMPLE_SIZE]; + file.read_exact(&mut buffer)?; + hasher.update(&buffer); + } + + let hash = hasher.finalize(); + Ok(hash.into()) +} + +#[cfg(test)] +mod tests { + use super::*; + use std::io::Write; + use tempfile::NamedTempFile; + + #[test] + fn test_small_file() { + let mut file = NamedTempFile::new().unwrap(); + file.write_all(b"hello world").unwrap(); + file.flush().unwrap(); + + let hash = compute_midhash256(file.path()).unwrap(); + assert_eq!(hash.len(), 32); + + // Same content should produce same hash + let hash2 = compute_midhash256(file.path()).unwrap(); + assert_eq!(hash, hash2); + } + + #[test] + fn test_different_sizes_different_hashes() { + // Two files with same content but different sizes should have different hashes + // because we include the size in the hash + let mut file1 = NamedTempFile::new().unwrap(); + file1.write_all(b"hello").unwrap(); + file1.flush().unwrap(); + + let mut file2 = NamedTempFile::new().unwrap(); + file2.write_all(b"hello world").unwrap(); + file2.flush().unwrap(); + + let hash1 = compute_midhash256(file1.path()).unwrap(); + let hash2 = compute_midhash256(file2.path()).unwrap(); + assert_ne!(hash1, hash2); + } + + #[test] + fn test_empty_file() { + let file = NamedTempFile::new().unwrap(); + let hash = compute_midhash256(file.path()).unwrap(); + assert_eq!(hash.len(), 32); + } + + #[test] + fn test_exactly_1mb_file() { + let mut file = NamedTempFile::new().unwrap(); + let data = vec![0xab; SIZE_THRESHOLD as usize]; + file.write_all(&data).unwrap(); + file.flush().unwrap(); + + let hash = compute_midhash256(file.path()).unwrap(); + assert_eq!(hash.len(), 32); + } + + #[test] + fn test_large_file_middle_sampling() { + // Create a file larger than 1MB with known content + let mut file = NamedTempFile::new().unwrap(); + + // Write 2MB: first 512KB of 'A', middle 1MB of 'B', last 512KB of 'C' + let first_half = vec![b'A'; 512 * 1024]; + let middle = vec![b'B'; 1024 * 1024]; + let last_half = vec![b'C'; 512 * 1024]; + + file.write_all(&first_half).unwrap(); + file.write_all(&middle).unwrap(); + file.write_all(&last_half).unwrap(); + file.flush().unwrap(); + + let hash = compute_midhash256(file.path()).unwrap(); + assert_eq!(hash.len(), 32); + + // Create another file with same middle but different edges + let mut file2 = NamedTempFile::new().unwrap(); + let first_half2 = vec![b'X'; 512 * 1024]; + let last_half2 = vec![b'Y'; 512 * 1024]; + + file2.write_all(&first_half2).unwrap(); + file2.write_all(&middle).unwrap(); + file2.write_all(&last_half2).unwrap(); + file2.flush().unwrap(); + + let hash2 = compute_midhash256(file2.path()).unwrap(); + + // Both files have same size and same middle 1MB, so hashes should match + assert_eq!(hash, hash2); + } +} diff --git a/src/hasher/mod.rs b/src/hasher/mod.rs new file mode 100644 index 0000000..be2747e --- /dev/null +++ b/src/hasher/mod.rs @@ -0,0 +1,12 @@ +mod algorithms; +pub mod btih_v2; +mod cid; +pub mod midhash256; + +pub use algorithms::hash_file; +pub use btih_v2::{ + compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link, + format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE, +}; +pub use cid::to_cid; +pub use midhash256::compute_midhash256; diff --git a/src/lib.rs b/src/lib.rs new file mode 100644 index 0000000..d6eb9db --- /dev/null +++ b/src/lib.rs @@ -0,0 +1,9 @@ +//! MetaMesh Fast Full Hash Plugin +//! +//! High-performance file hashing with multiple algorithms in a single I/O pass. + +pub mod api; +pub mod cache; +pub mod client; +pub mod config; +pub mod hasher; diff --git a/src/main.rs b/src/main.rs new file mode 100644 index 0000000..9dab8fa --- /dev/null +++ b/src/main.rs @@ -0,0 +1,65 @@ +use std::sync::Arc; + +use axum::{ + routing::{get, post}, + Router, +}; +use tower_http::trace::TraceLayer; +use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt}; + +use metamesh_plugin_fast_full_hash::api::{configure, health, manifest, process, ProcessState}; +use metamesh_plugin_fast_full_hash::cache::CacheManager; +use metamesh_plugin_fast_full_hash::config::create_shared_config; + +#[tokio::main] +async fn main() -> anyhow::Result<()> { + // Initialize logging + tracing_subscriber::registry() + .with( + tracing_subscriber::EnvFilter::try_from_default_env() + .unwrap_or_else(|_| "info,tower_http=debug".into()), + ) + .with(tracing_subscriber::fmt::layer()) + .init(); + + // Get configuration from environment + let port: u16 = std::env::var("PORT") + .ok() + .and_then(|p| p.parse().ok()) + .unwrap_or(8080); + + let cache_path = std::env::var("CACHE_PATH").unwrap_or_else(|_| "/cache".to_string()); + + // Initialize shared state + let config = create_shared_config(); + let cache = Arc::new(CacheManager::new(&cache_path)); + + // Load existing cache + if let Err(e) = cache.load().await { + tracing::warn!("Failed to load cache: {}", e); + } + + // Create shared app state + let app_state = Arc::new(ProcessState { + config, + cache, + }); + + // Build router + let app = Router::new() + .route("/health", get(health)) + .route("/manifest", get(manifest)) + .route("/configure", post(configure)) + .route("/process", post(process)) + .with_state(app_state) + .layer(TraceLayer::new_for_http()); + + // Start server + let addr = format!("0.0.0.0:{}", port); + tracing::info!("Starting fast-full-hash plugin on {}", addr); + + let listener = tokio::net::TcpListener::bind(&addr).await?; + axum::serve(listener, app).await?; + + Ok(()) +} diff --git a/testdata/compute_ground_truth.sh b/testdata/compute_ground_truth.sh new file mode 100644 index 0000000..ec58d3a --- /dev/null +++ b/testdata/compute_ground_truth.sh @@ -0,0 +1,237 @@ +#!/bin/bash +# Compute ground truth hash values for test reference files +# Uses standard Unix tools to verify our implementation + +set -e + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +# Reference file: Sintel trailer (Creative Commons licensed) +# Source: https://durian.blender.org/ +REFERENCE_URL="https://download.blender.org/durian/trailer/sintel_trailer-480p.mp4" +REFERENCE_FILE="sintel_trailer-480p.mp4" +EXPECTED_SIZE=23014356 # bytes + +echo "=== Ground Truth Hash Computation ===" +echo "" + +# Download reference file if not present +if [ ! -f "$REFERENCE_FILE" ]; then + echo "Downloading reference file: $REFERENCE_FILE" + curl -L -o "$REFERENCE_FILE" "$REFERENCE_URL" +fi + +# Verify file size +ACTUAL_SIZE=$(stat -f%z "$REFERENCE_FILE" 2>/dev/null || stat -c%s "$REFERENCE_FILE" 2>/dev/null) +if [ "$ACTUAL_SIZE" != "$EXPECTED_SIZE" ]; then + echo "WARNING: File size mismatch! Expected $EXPECTED_SIZE, got $ACTUAL_SIZE" + echo "The file may have changed. Please verify manually." +fi + +echo "Reference file: $REFERENCE_FILE" +echo "File size: $ACTUAL_SIZE bytes" +echo "" + +echo "=== Computing Standard Hashes ===" +echo "" + +# SHA-256 +echo -n "SHA-256: " +sha256sum "$REFERENCE_FILE" | cut -d' ' -f1 + +# SHA-1 +echo -n "SHA-1: " +sha1sum "$REFERENCE_FILE" | cut -d' ' -f1 + +# MD5 +echo -n "MD5: " +md5sum "$REFERENCE_FILE" | cut -d' ' -f1 + +# CRC32 (using Python as crc32 tool varies) +echo -n "CRC32: " +python3 -c " +import binascii +with open('$REFERENCE_FILE', 'rb') as f: + crc = 0 + while chunk := f.read(65536): + crc = binascii.crc32(chunk, crc) + print(format(crc & 0xffffffff, '08x')) +" + +# SHA3-256 (using openssl if available, otherwise Python) +echo -n "SHA3-256: " +if command -v openssl &> /dev/null && openssl dgst -sha3-256 /dev/null &> /dev/null 2>&1; then + openssl dgst -sha3-256 "$REFERENCE_FILE" | awk '{print $2}' +else + python3 -c " +import hashlib +with open('$REFERENCE_FILE', 'rb') as f: + h = hashlib.sha3_256() + while chunk := f.read(65536): + h.update(chunk) + print(h.hexdigest()) +" +fi + +# SHA3-384 +echo -n "SHA3-384: " +if command -v openssl &> /dev/null && openssl dgst -sha3-384 /dev/null &> /dev/null 2>&1; then + openssl dgst -sha3-384 "$REFERENCE_FILE" | awk '{print $2}' +else + python3 -c " +import hashlib +with open('$REFERENCE_FILE', 'rb') as f: + h = hashlib.sha3_384() + while chunk := f.read(65536): + h.update(chunk) + print(h.hexdigest()) +" +fi + +echo "" +echo "=== Computing MidHash256 ===" +echo "" + +# MidHash256: SHA-256(size_u64_be || middle_1MB) +python3 << 'PYTHON_EOF' +import hashlib +import struct +import os + +filename = 'sintel_trailer-480p.mp4' +size = os.path.getsize(filename) +THRESHOLD = 1024 * 1024 # 1MB + +with open(filename, 'rb') as f: + hasher = hashlib.sha256() + # Add size as 8-byte big-endian + hasher.update(struct.pack('>Q', size)) + + if size <= THRESHOLD: + # Small file: hash entire content + hasher.update(f.read()) + else: + # Large file: hash middle 1MB + middle_start = (size - THRESHOLD) // 2 + f.seek(middle_start) + hasher.update(f.read(THRESHOLD)) + + print(f"MidHash256: {hasher.hexdigest()}") +PYTHON_EOF + +echo "" +echo "=== Computing BitTorrent v2 Hashes ===" +echo "" + +# BitTorrent v2 pieces root and info hash +python3 << 'PYTHON_EOF' +import hashlib +import os + +filename = 'sintel_trailer-480p.mp4' +BLOCK_SIZE = 16 * 1024 # 16 KiB + +def compute_leaf_hashes(filepath): + """Compute SHA-256 hash of each 16KB block""" + leaves = [] + size = os.path.getsize(filepath) + + if size == 0: + return [bytes(32)] + + with open(filepath, 'rb') as f: + while True: + block = f.read(BLOCK_SIZE) + if not block: + break + # Pad last block with zeros + if len(block) < BLOCK_SIZE: + block = block + bytes(BLOCK_SIZE - len(block)) + leaves.append(hashlib.sha256(block).digest()) + + return leaves + +def compute_merkle_root(leaves): + """Compute merkle root from leaf hashes""" + if not leaves: + return bytes(32) + if len(leaves) == 1: + return leaves[0] + + # Pad to power of 2 + target = 1 + while target < len(leaves): + target *= 2 + + current = list(leaves) + while len(current) < target: + current.append(bytes(32)) + + # Build tree bottom-up + while len(current) > 1: + next_level = [] + for i in range(0, len(current), 2): + combined = current[i] + current[i+1] + next_level.append(hashlib.sha256(combined).digest()) + current = next_level + + return current[0] + +def bencode_int(i): + return f'i{i}e'.encode() + +def bencode_str(s): + if isinstance(s, str): + s = s.encode() + return f'{len(s)}:'.encode() + s + +def bencode_dict(d): + result = b'd' + for k in sorted(d.keys()): + result += bencode_str(k) + v = d[k] + if isinstance(v, int): + result += bencode_int(v) + elif isinstance(v, (str, bytes)): + result += bencode_str(v) + elif isinstance(v, dict): + result += bencode_dict(v) + result += b'e' + return result + +# Compute pieces root +leaves = compute_leaf_hashes(filename) +pieces_root = compute_merkle_root(leaves) +print(f"BT Pieces Root: {pieces_root.hex()}") + +# Compute info hash +size = os.path.getsize(filename) +basename = os.path.basename(filename) + +# Build info dict as per BEP 52 +info_dict = { + 'file tree': { + basename: { + '': { + 'length': size, + 'pieces root': pieces_root + } + } + }, + 'meta version': 2, + 'name': basename, + 'piece length': BLOCK_SIZE +} + +encoded = bencode_dict(info_dict) +info_hash = hashlib.sha256(encoded).digest() +print(f"BT Info Hash: {info_hash.hex()}") + +# Magnet link +print(f"Magnet: magnet:?xt=urn:btmh:1220{info_hash.hex()}&dn={basename}") +PYTHON_EOF + +echo "" +echo "=== Summary ===" +echo "Copy these values to tests/ground_truth.rs" diff --git a/testdata/sintel_trailer-480p.mp4 b/testdata/sintel_trailer-480p.mp4 new file mode 100644 index 0000000..c962013 Binary files /dev/null and b/testdata/sintel_trailer-480p.mp4 differ diff --git a/tests/ground_truth.rs b/tests/ground_truth.rs new file mode 100644 index 0000000..ba68d69 --- /dev/null +++ b/tests/ground_truth.rs @@ -0,0 +1,330 @@ +//! Ground Truth Integration Tests +//! +//! These tests verify our hash implementations against known-good values +//! computed using standard third-party tools (sha256sum, sha1sum, md5sum, Python hashlib). +//! +//! Reference file: Sintel trailer (480p) - Creative Commons licensed +//! Source: https://download.blender.org/durian/trailer/sintel_trailer-480p.mp4 +//! +//! To regenerate ground truth values, run: +//! cd testdata && ./compute_ground_truth.sh + +use metamesh_plugin_fast_full_hash::config::HashAlgorithm; +use metamesh_plugin_fast_full_hash::hasher::hash_file; +use std::collections::HashSet; +use std::path::Path; + +/// Reference file details +const REFERENCE_FILE: &str = "testdata/sintel_trailer-480p.mp4"; +const REFERENCE_SIZE: u64 = 4372373; + +/// Ground truth hash values (hex strings) +/// Computed using standard Unix tools and Python hashlib +mod expected { + pub const SHA256: &str = "b670602fa00934ca27c4351bb0efe7ea7a07fae57284e44226025eeed7c51254"; + pub const SHA1: &str = "9b678890fb8ca401c28e7ca09171ec008a154b97"; + pub const MD5: &str = "df6ed4bbc93613c68c8525e21bbddf98"; + pub const CRC32: &str = "61accd5e"; + pub const SHA3_256: &str = "fefc0c74ebd79a9872464ebbb35a0545db97431a6c9a96d36c061244883e9213"; + pub const SHA3_384: &str = "da29f6604c96d09332531a4a2f4881b7bd2401a995053db868d5aea4056ada0b45c0ba69016e931bc468c38788afad94"; + pub const MIDHASH256: &str = "297a6662a49c60da0cf2a75407a0eb0045a36f838d2921ab602483b708157773"; + pub const BT_PIECES_ROOT: &str = + "424febb4268900457464928a99547dacd07d2075cbbd7ae27fdf5c8d2d982817"; + pub const BT_INFO_HASH: &str = + "e2f048c0c8bb56c910c05b9efb4f39d2ff6fd0cf01e96e1dacecb56a067f8418"; +} + +/// Convert hex string to bytes +fn hex_to_bytes(hex: &str) -> Vec { + (0..hex.len()) + .step_by(2) + .map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap()) + .collect() +} + +/// Extract raw hash from CID (parse the CID and get the hash bytes) +fn extract_hash_from_cid(cid_str: &str) -> Vec { + use cid::Cid; + let cid = Cid::try_from(cid_str).expect("Invalid CID"); + cid.hash().digest().to_vec() +} + +/// Check if reference file exists, skip test if not +fn check_reference_file() -> Option<&'static Path> { + let path = Path::new(REFERENCE_FILE); + if !path.exists() { + eprintln!( + "Skipping ground truth test: reference file not found at {}", + REFERENCE_FILE + ); + eprintln!("To download, run: cd testdata && ./compute_ground_truth.sh"); + return None; + } + + // Verify file size + let metadata = std::fs::metadata(path).expect("Failed to read file metadata"); + if metadata.len() != REFERENCE_SIZE { + eprintln!( + "WARNING: Reference file size mismatch. Expected {}, got {}", + REFERENCE_SIZE, + metadata.len() + ); + eprintln!("Ground truth values may not match. Re-run compute_ground_truth.sh"); + } + + Some(path) +} + +#[test] +fn test_sha256_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha256].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_sha2-256").expect("SHA-256 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::SHA256); + + assert_eq!( + hash_bytes, expected_bytes, + "SHA-256 mismatch!\nExpected: {}\nGot: {}", + expected::SHA256, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_sha1_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha1].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_sha1").expect("SHA-1 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::SHA1); + + assert_eq!( + hash_bytes, expected_bytes, + "SHA-1 mismatch!\nExpected: {}\nGot: {}", + expected::SHA1, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_md5_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Md5].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_md5").expect("MD5 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::MD5); + + assert_eq!( + hash_bytes, expected_bytes, + "MD5 mismatch!\nExpected: {}\nGot: {}", + expected::MD5, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_crc32_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Crc32].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_crc32").expect("CRC32 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::CRC32); + + assert_eq!( + hash_bytes, expected_bytes, + "CRC32 mismatch!\nExpected: {}\nGot: {}", + expected::CRC32, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_sha3_256_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha3_256].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_sha3-256").expect("SHA3-256 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::SHA3_256); + + assert_eq!( + hash_bytes, expected_bytes, + "SHA3-256 mismatch!\nExpected: {}\nGot: {}", + expected::SHA3_256, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_sha3_384_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Sha3_384].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results.get("cid_sha3-384").expect("SHA3-384 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::SHA3_384); + + assert_eq!( + hash_bytes, expected_bytes, + "SHA3-384 mismatch!\nExpected: {}\nGot: {}", + expected::SHA3_384, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_midhash256_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::Midhash256].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results + .get("cid_midhash256") + .expect("MidHash256 not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::MIDHASH256); + + assert_eq!( + hash_bytes, expected_bytes, + "MidHash256 mismatch!\nExpected: {}\nGot: {}", + expected::MIDHASH256, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_bt_pieces_root_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results + .get("cid_bt_pieces_root") + .expect("BT Pieces Root not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::BT_PIECES_ROOT); + + assert_eq!( + hash_bytes, expected_bytes, + "BT Pieces Root mismatch!\nExpected: {}\nGot: {}", + expected::BT_PIECES_ROOT, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_bt_info_hash_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + let enabled: HashSet<_> = vec![HashAlgorithm::BtInfoHash].into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + let cid = results + .get("cid_bt_info_hash") + .expect("BT Info Hash not computed"); + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected::BT_INFO_HASH); + + assert_eq!( + hash_bytes, expected_bytes, + "BT Info Hash mismatch!\nExpected: {}\nGot: {}", + expected::BT_INFO_HASH, + hex::encode(&hash_bytes) + ); +} + +#[test] +fn test_all_algorithms_ground_truth() { + let Some(path) = check_reference_file() else { + return; + }; + + // Compute all algorithms in single pass + let enabled: HashSet<_> = HashAlgorithm::all().into_iter().collect(); + let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed"); + + // Verify all 9 algorithms + let test_cases = vec![ + ("cid_sha2-256", expected::SHA256, "SHA-256"), + ("cid_sha1", expected::SHA1, "SHA-1"), + ("cid_md5", expected::MD5, "MD5"), + ("cid_crc32", expected::CRC32, "CRC32"), + ("cid_sha3-256", expected::SHA3_256, "SHA3-256"), + ("cid_sha3-384", expected::SHA3_384, "SHA3-384"), + ("cid_midhash256", expected::MIDHASH256, "MidHash256"), + ( + "cid_bt_pieces_root", + expected::BT_PIECES_ROOT, + "BT Pieces Root", + ), + ("cid_bt_info_hash", expected::BT_INFO_HASH, "BT Info Hash"), + ]; + + let mut failures = Vec::new(); + + for (key, expected_hex, name) in test_cases { + let cid = match results.get(key) { + Some(c) => c, + None => { + failures.push(format!("{}: NOT COMPUTED", name)); + continue; + } + }; + + let hash_bytes = extract_hash_from_cid(cid); + let expected_bytes = hex_to_bytes(expected_hex); + + if hash_bytes != expected_bytes { + failures.push(format!( + "{}: MISMATCH\n Expected: {}\n Got: {}", + name, + expected_hex, + hex::encode(&hash_bytes) + )); + } + } + + if !failures.is_empty() { + panic!( + "Ground truth verification failed:\n{}", + failures.join("\n\n") + ); + } +}