perf improved hash plugin
This commit is contained in:
Pierre
@@ -3,3 +3,5 @@ dist/
|
||||
*.log
|
||||
.env
|
||||
.DS_Store
|
||||
CLAUDE.md
|
||||
.claude/settings.local.json
|
||||
|
||||
+51
@@ -0,0 +1,51 @@
|
||||
[package]
|
||||
name = "metamesh-plugin-fast-full-hash"
|
||||
version = "1.0.0"
|
||||
edition = "2021"
|
||||
description = "High-performance file hashing plugin for MetaMesh"
|
||||
authors = ["MetaMesh"]
|
||||
|
||||
[dependencies]
|
||||
# HTTP server
|
||||
axum = "0.7"
|
||||
tokio = { version = "1", features = ["full"] }
|
||||
tower-http = { version = "0.6", features = ["cors", "trace"] }
|
||||
|
||||
# Hashing
|
||||
sha2 = "0.10"
|
||||
sha1 = "0.10"
|
||||
md-5 = "0.10"
|
||||
crc32fast = "1.4"
|
||||
sha3 = "0.10"
|
||||
|
||||
# CID/Multihash
|
||||
cid = "0.11"
|
||||
multihash = "0.19"
|
||||
unsigned-varint = "0.8"
|
||||
|
||||
# BitTorrent v2 bencoding
|
||||
bendy = "0.3"
|
||||
|
||||
# Serialization
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
bincode = "1.3"
|
||||
|
||||
# HTTP client
|
||||
reqwest = { version = "0.12", default-features = false, features = ["json", "rustls-tls"] }
|
||||
|
||||
# Utilities
|
||||
tracing = "0.1"
|
||||
tracing-subscriber = { version = "0.3", features = ["env-filter"] }
|
||||
anyhow = "1"
|
||||
thiserror = "2"
|
||||
|
||||
[dev-dependencies]
|
||||
tempfile = "3"
|
||||
hex = "0.4"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
lto = true
|
||||
codegen-units = 1
|
||||
strip = true
|
||||
+118
@@ -0,0 +1,118 @@
|
||||
# BitTorrent v2 Hashes (BEP 52)
|
||||
|
||||
## Purpose
|
||||
|
||||
Generate BitTorrent v2 compatible hashes for magnet links and peer verification. Enables creating magnet links for files without a full .torrent file.
|
||||
|
||||
## Two Output Hashes
|
||||
|
||||
### 1. Pieces Root (`cid_bt_pieces_root`)
|
||||
|
||||
- **Multicodec**: `0xb702` (**official** `bittorrent-pieces-root`)
|
||||
- **CID codec**: `0x55` (raw)
|
||||
- **Description**: Merkle tree root of SHA-256 hashes of 16KB file blocks
|
||||
- **Use case**: File verification, piece-level integrity checking
|
||||
|
||||
### 2. Info Hash (`cid_bt_info_hash`)
|
||||
|
||||
- **Multicodec**: `0x12` (standard SHA-256)
|
||||
- **CID codec**: `0x55` (raw)
|
||||
- **Description**: SHA-256 of bencoded info dictionary
|
||||
- **Use case**: Magnet links, torrent identification
|
||||
|
||||
## Algorithm
|
||||
|
||||
```
|
||||
1. Read file in 16KB blocks
|
||||
2. SHA-256 hash each block (pad last block with zeros)
|
||||
3. Build merkle tree from leaf hashes:
|
||||
- Pad to next power of 2 with zero hashes
|
||||
- Each parent = SHA-256(left_child || right_child)
|
||||
→ Result: pieces_root (32 bytes)
|
||||
|
||||
4. Create info dictionary:
|
||||
{
|
||||
"file tree": { filename: { "": { "length": N, "pieces root": <pieces_root> } } },
|
||||
"meta version": 2,
|
||||
"name": filename,
|
||||
"piece length": 16384
|
||||
}
|
||||
|
||||
5. Bencode the dictionary
|
||||
6. SHA-256(bencoded_info)
|
||||
→ Result: info_hash (32 bytes)
|
||||
```
|
||||
|
||||
### Merkle Tree Structure
|
||||
|
||||
```
|
||||
[Root]
|
||||
/ \
|
||||
[Node] [Node]
|
||||
/ \ / \
|
||||
[Leaf0] [Leaf1] [Leaf2] [Zero]
|
||||
|
||||
Leaf = SHA-256(16KB block)
|
||||
Node = SHA-256(left || right)
|
||||
```
|
||||
|
||||
## Magnet Link Generation
|
||||
|
||||
```
|
||||
magnet:?xt=urn:btmh:1220<info_hash_hex>&dn=<filename>
|
||||
```
|
||||
|
||||
Where:
|
||||
- `btmh` = BitTorrent Multihash
|
||||
- `1220` = `0x12` (SHA-256 code) + `0x20` (32 bytes = 0x20 in hex)
|
||||
- `info_hash_hex` = 64 hex characters
|
||||
- `dn` = display name (URL-encoded filename)
|
||||
|
||||
### Example
|
||||
|
||||
```
|
||||
File: ubuntu-24.04-desktop-amd64.iso
|
||||
Info Hash: a1b2c3d4...
|
||||
Magnet: magnet:?xt=urn:btmh:1220a1b2c3d4...&dn=ubuntu-24.04-desktop-amd64.iso
|
||||
```
|
||||
|
||||
## BEP 52 Compliance
|
||||
|
||||
This implementation follows [BEP 52](https://www.bittorrent.org/beps/bep_0052.html):
|
||||
|
||||
| Requirement | Implementation |
|
||||
|-------------|----------------|
|
||||
| Block size | 16 KiB (16,384 bytes) |
|
||||
| Piece size | 16 KiB (minimum, power of 2) |
|
||||
| Hash algorithm | SHA-256 |
|
||||
| Merkle tree | Binary, power-of-2 padded |
|
||||
| Info dict | v2 format with `file tree` |
|
||||
| Meta version | 2 (v2-only torrent) |
|
||||
|
||||
## Properties
|
||||
|
||||
| Property | Pieces Root | Info Hash |
|
||||
|----------|-------------|-----------|
|
||||
| Hash size | 32 bytes | 32 bytes |
|
||||
| Algorithm | Merkle SHA-256 | SHA-256 |
|
||||
| Multicodec | 0xb702 (official) | 0x12 (SHA-256) |
|
||||
| Time complexity | O(n) | O(n) |
|
||||
| Space complexity | O(log n) | O(1) |
|
||||
|
||||
## Use Cases
|
||||
|
||||
1. **Magnet link creation** - Share files via magnet links without .torrent files
|
||||
2. **Peer verification** - Verify file integrity in BitTorrent v2 swarms
|
||||
3. **Deduplication** - Identify identical files across BitTorrent network
|
||||
4. **Hybrid torrents** - Create v1+v2 hybrid torrents
|
||||
|
||||
## Limitations
|
||||
|
||||
- Pieces root depends on filename (included in info dict)
|
||||
- Single-file torrents only (no multi-file directory support)
|
||||
- Requires full file read for both hashes
|
||||
|
||||
## References
|
||||
|
||||
- [BEP 52: The BitTorrent Protocol Specification v2](https://www.bittorrent.org/beps/bep_0052.html)
|
||||
- [Multicodec Table](https://github.com/multiformats/multicodec/blob/master/table.csv) - `bittorrent-pieces-root` = 0xb702
|
||||
@@ -0,0 +1,80 @@
|
||||
# MidHash256
|
||||
|
||||
## Purpose
|
||||
|
||||
O(1) file identification hash - constant time regardless of file size. Designed for quick duplicate detection and large file identification before expensive full-hash operations.
|
||||
|
||||
## Multicodec
|
||||
|
||||
- **Code**: `0x1000` (custom, not in official multicodec registry)
|
||||
- **CID codec**: `0x55` (raw)
|
||||
- **Output key**: `cid_midhash256`
|
||||
|
||||
## Algorithm
|
||||
|
||||
```
|
||||
1. Read file size (S)
|
||||
2. If S <= 1MB:
|
||||
sample = entire file content
|
||||
3. If S > 1MB:
|
||||
seek to (S - 1MB) / 2
|
||||
sample = read 1MB from middle
|
||||
4. Hash = SHA-256(S as u64 big-endian || sample)
|
||||
```
|
||||
|
||||
### Pseudocode
|
||||
|
||||
```rust
|
||||
fn midhash256(path: &Path) -> [u8; 32] {
|
||||
let size = file_size(path);
|
||||
let mut hasher = SHA256::new();
|
||||
|
||||
// Always include size as 8-byte big-endian prefix
|
||||
hasher.update(size.to_be_bytes());
|
||||
|
||||
if size <= 1_048_576 {
|
||||
// Small file: hash entire content
|
||||
hasher.update(read_all(path));
|
||||
} else {
|
||||
// Large file: hash middle 1MB
|
||||
let middle_start = (size - 1_048_576) / 2;
|
||||
hasher.update(read_at(path, middle_start, 1_048_576));
|
||||
}
|
||||
|
||||
hasher.finalize()
|
||||
}
|
||||
```
|
||||
|
||||
## Properties
|
||||
|
||||
| Property | Value |
|
||||
|----------|-------|
|
||||
| Time complexity | O(1) for files > 1MB |
|
||||
| Space complexity | O(1) - fixed 1MB buffer |
|
||||
| Hash size | 32 bytes (SHA-256) |
|
||||
| Collision resistance | Same as SHA-256 for middle content |
|
||||
|
||||
## Use Cases
|
||||
|
||||
1. **Quick duplicate detection** - Identify potential duplicates before computing full hashes
|
||||
2. **Large file identification** - Identify terabyte files in ~20ms regardless of size
|
||||
3. **Pre-filtering** - Filter candidates before expensive operations (full hash, content analysis)
|
||||
4. **Change detection** - Quick check if middle content changed (useful for video files)
|
||||
|
||||
## Limitations
|
||||
|
||||
- Files differing only at edges (first/last 512KB for 2MB+ files) will have same midhash
|
||||
- Not suitable as a standalone deduplication hash - use as pre-filter only
|
||||
- Custom multicodec code may conflict with future official allocations
|
||||
|
||||
## CID Example
|
||||
|
||||
```
|
||||
File: movie.mkv (4.7 GB)
|
||||
MidHash256: 3a7b...c2d1 (32 bytes)
|
||||
CID: bafkreibh4xvl5... (base32lower encoded CIDv1)
|
||||
```
|
||||
|
||||
## Integration
|
||||
|
||||
The hash is computed separately from the standard single-pass algorithms since it requires seeking to the middle of the file. For files <= 1MB, it reads the entire file like other algorithms.
|
||||
@@ -0,0 +1,66 @@
|
||||
use axum::{extract::State, Json};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::sync::Arc;
|
||||
|
||||
use crate::api::process::ProcessState;
|
||||
use crate::config::PluginConfig;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ConfigureRequest {
|
||||
pub config: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ConfigureResponse {
|
||||
pub status: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
pub async fn configure(
|
||||
State(state): State<Arc<ProcessState>>,
|
||||
Json(req): Json<ConfigureRequest>,
|
||||
) -> Json<ConfigureResponse> {
|
||||
let mut cfg = state.config.write().await;
|
||||
|
||||
// Handle enabledHashes config
|
||||
if let Some(hashes) = req.config.get("enabledHashes") {
|
||||
let enabled = PluginConfig::parse_enabled_hashes(hashes);
|
||||
if enabled.is_empty() {
|
||||
return Json(ConfigureResponse {
|
||||
status: "error",
|
||||
error: Some("No valid hash algorithms specified".to_string()),
|
||||
});
|
||||
}
|
||||
cfg.enabled_hashes = enabled;
|
||||
tracing::info!("Updated enabled hashes: {:?}", cfg.enabled_hashes);
|
||||
}
|
||||
|
||||
// Handle buffer size config
|
||||
if let Some(size) = req.config.get("bufferSize") {
|
||||
match size.parse::<usize>() {
|
||||
Ok(s) if s >= 4096 => {
|
||||
cfg.buffer_size = s;
|
||||
tracing::info!("Updated buffer size: {}", s);
|
||||
}
|
||||
Ok(_) => {
|
||||
return Json(ConfigureResponse {
|
||||
status: "error",
|
||||
error: Some("Buffer size must be at least 4096 bytes".to_string()),
|
||||
});
|
||||
}
|
||||
Err(_) => {
|
||||
return Json(ConfigureResponse {
|
||||
status: "error",
|
||||
error: Some("Invalid buffer size".to_string()),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Json(ConfigureResponse {
|
||||
status: "ok",
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,17 @@
|
||||
use axum::Json;
|
||||
use serde::Serialize;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HealthResponse {
|
||||
pub status: &'static str,
|
||||
pub ready: bool,
|
||||
pub version: &'static str,
|
||||
}
|
||||
|
||||
pub async fn health() -> Json<HealthResponse> {
|
||||
Json(HealthResponse {
|
||||
status: "healthy",
|
||||
ready: true,
|
||||
version: env!("CARGO_PKG_VERSION"),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,99 @@
|
||||
use axum::Json;
|
||||
use serde::Serialize;
|
||||
use serde_json::{json, Value};
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ManifestResponse {
|
||||
pub id: &'static str,
|
||||
pub name: &'static str,
|
||||
pub version: &'static str,
|
||||
pub description: &'static str,
|
||||
pub author: &'static str,
|
||||
pub dependencies: Vec<String>,
|
||||
pub priority: u32,
|
||||
pub color: &'static str,
|
||||
#[serde(rename = "defaultQueue")]
|
||||
pub default_queue: &'static str,
|
||||
pub timeout: u64,
|
||||
pub schema: Value,
|
||||
pub config: Value,
|
||||
}
|
||||
|
||||
pub async fn manifest() -> Json<ManifestResponse> {
|
||||
Json(ManifestResponse {
|
||||
id: "fast-full-hash",
|
||||
name: "Fast Full File Hash (Rust)",
|
||||
version: env!("CARGO_PKG_VERSION"),
|
||||
description: "High-performance parallel hashing with optimized I/O",
|
||||
author: "MetaMesh",
|
||||
dependencies: vec![],
|
||||
priority: 100,
|
||||
color: "#FF5722",
|
||||
default_queue: "background",
|
||||
timeout: 300000,
|
||||
schema: json!({
|
||||
"cid_sha2-256": {
|
||||
"label": "SHA-256 Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA-256 hash in CID format (multicodec 0x12)"
|
||||
},
|
||||
"cid_sha1": {
|
||||
"label": "SHA-1 Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA-1 hash in CID format (multicodec 0x11)"
|
||||
},
|
||||
"cid_md5": {
|
||||
"label": "MD5 Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "MD5 hash in CID format (multicodec 0xd5)"
|
||||
},
|
||||
"cid_crc32": {
|
||||
"label": "CRC32 Checksum",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "CRC32 checksum in CID format (multicodec 0x0132)"
|
||||
},
|
||||
"cid_sha3-256": {
|
||||
"label": "SHA3-256 Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA3-256 hash in CID format (multicodec 0x16)"
|
||||
},
|
||||
"cid_sha3-384": {
|
||||
"label": "SHA3-384 Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA3-384 hash in CID format (multicodec 0x15)"
|
||||
},
|
||||
"cid_midhash256": {
|
||||
"label": "MidHash256",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "O(1) file identification hash - SHA-256 of size + middle 1MB (custom multicodec 0x1000)"
|
||||
},
|
||||
"cid_bt_pieces_root": {
|
||||
"label": "BitTorrent v2 Pieces Root",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "Merkle tree root of 16KB blocks (official multicodec 0xb702)"
|
||||
},
|
||||
"cid_bt_info_hash": {
|
||||
"label": "BitTorrent v2 Info Hash",
|
||||
"type": "cid",
|
||||
"readonly": true,
|
||||
"hint": "SHA-256 of bencoded info dict for magnet links (multicodec 0x12)"
|
||||
}
|
||||
}),
|
||||
config: json!({
|
||||
"enabledHashes": {
|
||||
"type": "string",
|
||||
"label": "Enabled hash algorithms (comma-separated)",
|
||||
"default": "cid_sha2-256,cid_sha1,cid_md5,cid_crc32,cid_sha3-256,cid_sha3-384,cid_midhash256,cid_bt_pieces_root,cid_bt_info_hash",
|
||||
"required": false
|
||||
}
|
||||
}),
|
||||
})
|
||||
}
|
||||
@@ -0,0 +1,9 @@
|
||||
mod configure;
|
||||
mod health;
|
||||
mod manifest;
|
||||
mod process;
|
||||
|
||||
pub use configure::configure;
|
||||
pub use health::health;
|
||||
pub use manifest::manifest;
|
||||
pub use process::{process, ProcessState};
|
||||
@@ -0,0 +1,249 @@
|
||||
use axum::{extract::State, Json};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::path::Path;
|
||||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use crate::cache::CacheManager;
|
||||
use crate::client::MetaCoreClient;
|
||||
use crate::config::{HashAlgorithm, SharedConfig};
|
||||
use crate::hasher::hash_file;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct ProcessRequest {
|
||||
pub task_id: String,
|
||||
pub cid: String,
|
||||
pub file_path: String,
|
||||
pub callback_url: String,
|
||||
pub meta_core_url: String,
|
||||
#[serde(default)]
|
||||
pub existing_meta: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct ProcessResponse {
|
||||
pub status: &'static str,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub struct CallbackPayload {
|
||||
pub task_id: String,
|
||||
pub status: &'static str,
|
||||
pub duration: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub error: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub reason: Option<String>,
|
||||
}
|
||||
|
||||
pub struct ProcessState {
|
||||
pub config: SharedConfig,
|
||||
pub cache: Arc<CacheManager>,
|
||||
}
|
||||
|
||||
pub async fn process(
|
||||
State(state): State<Arc<ProcessState>>,
|
||||
Json(req): Json<ProcessRequest>,
|
||||
) -> Json<ProcessResponse> {
|
||||
// Validate required fields
|
||||
if req.task_id.is_empty() || req.file_path.is_empty() || req.callback_url.is_empty() {
|
||||
return Json(ProcessResponse {
|
||||
status: "rejected",
|
||||
error: Some("Missing required fields".to_string()),
|
||||
});
|
||||
}
|
||||
|
||||
// Spawn async processing task
|
||||
let state_clone = Arc::clone(&state);
|
||||
tokio::spawn(async move {
|
||||
process_file(state_clone, req).await;
|
||||
});
|
||||
|
||||
Json(ProcessResponse {
|
||||
status: "accepted",
|
||||
error: None,
|
||||
})
|
||||
}
|
||||
|
||||
async fn process_file(state: Arc<ProcessState>, req: ProcessRequest) {
|
||||
let start = Instant::now();
|
||||
let task_id = req.task_id.clone();
|
||||
let callback_url = req.callback_url.clone();
|
||||
|
||||
let result = do_process_file(state, req).await;
|
||||
|
||||
let duration = start.elapsed().as_millis() as u64;
|
||||
|
||||
let payload = match result {
|
||||
Ok(ProcessResult::Completed) => CallbackPayload {
|
||||
task_id,
|
||||
status: "completed",
|
||||
duration,
|
||||
error: None,
|
||||
reason: None,
|
||||
},
|
||||
Ok(ProcessResult::Skipped(reason)) => CallbackPayload {
|
||||
task_id,
|
||||
status: "skipped",
|
||||
duration,
|
||||
error: None,
|
||||
reason: Some(reason),
|
||||
},
|
||||
Err(e) => CallbackPayload {
|
||||
task_id,
|
||||
status: "failed",
|
||||
duration,
|
||||
error: Some(e.to_string()),
|
||||
reason: None,
|
||||
},
|
||||
};
|
||||
|
||||
// Send callback
|
||||
if let Err(e) = send_callback(&callback_url, &payload).await {
|
||||
tracing::error!("Failed to send callback: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
enum ProcessResult {
|
||||
Completed,
|
||||
Skipped(String),
|
||||
}
|
||||
|
||||
async fn do_process_file(
|
||||
state: Arc<ProcessState>,
|
||||
req: ProcessRequest,
|
||||
) -> anyhow::Result<ProcessResult> {
|
||||
let path = Path::new(&req.file_path);
|
||||
|
||||
// Check if file exists
|
||||
if !path.exists() {
|
||||
anyhow::bail!("File not found: {}", req.file_path);
|
||||
}
|
||||
|
||||
// Get file metadata
|
||||
let metadata = std::fs::metadata(path)?;
|
||||
let size = metadata.len();
|
||||
let mtime = metadata
|
||||
.modified()
|
||||
.ok()
|
||||
.and_then(|t| t.duration_since(std::time::UNIX_EPOCH).ok())
|
||||
.map(|d| d.as_secs() as i64)
|
||||
.unwrap_or(0);
|
||||
|
||||
let filename = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
// Get config
|
||||
let config = state.config.read().await.clone();
|
||||
|
||||
// Determine which hashes we need to compute
|
||||
let required_keys: Vec<&str> = config
|
||||
.enabled_hashes
|
||||
.iter()
|
||||
.map(|a| a.key())
|
||||
.collect();
|
||||
|
||||
// Check existing metadata - skip if all hashes already present
|
||||
let missing_from_meta: Vec<&str> = required_keys
|
||||
.iter()
|
||||
.filter(|k| !req.existing_meta.contains_key(**k))
|
||||
.copied()
|
||||
.collect();
|
||||
|
||||
if missing_from_meta.is_empty() {
|
||||
return Ok(ProcessResult::Skipped(
|
||||
"All hashes already computed".to_string(),
|
||||
));
|
||||
}
|
||||
|
||||
// Check cache for remaining hashes
|
||||
let cached = state.cache.get(&filename, size, mtime).await;
|
||||
let mut results: HashMap<String, String> = HashMap::new();
|
||||
|
||||
let to_compute: Vec<HashAlgorithm> = missing_from_meta
|
||||
.iter()
|
||||
.filter_map(|key| {
|
||||
if let Some(ref cached_hashes) = cached {
|
||||
if let Some(hash) = cached_hashes.get(*key) {
|
||||
results.insert(key.to_string(), hash.clone());
|
||||
return None;
|
||||
}
|
||||
}
|
||||
HashAlgorithm::from_key(key)
|
||||
})
|
||||
.collect();
|
||||
|
||||
// If all hashes were in cache, we're done
|
||||
if to_compute.is_empty() {
|
||||
tracing::info!("All hashes from cache for {}", filename);
|
||||
} else {
|
||||
// Compute missing hashes
|
||||
tracing::info!(
|
||||
"Computing {} hashes for {} ({} bytes)",
|
||||
to_compute.len(),
|
||||
filename,
|
||||
size
|
||||
);
|
||||
|
||||
let enabled_set = to_compute.iter().copied().collect();
|
||||
let computed = tokio::task::spawn_blocking({
|
||||
let path = path.to_path_buf();
|
||||
let buffer_size = config.buffer_size;
|
||||
move || hash_file(&path, &enabled_set, buffer_size)
|
||||
})
|
||||
.await??;
|
||||
|
||||
// Merge computed hashes into results
|
||||
for (key, value) in &computed {
|
||||
results.insert(key.clone(), value.clone());
|
||||
}
|
||||
|
||||
// Update cache with new hashes
|
||||
let mut all_hashes = cached.unwrap_or_default();
|
||||
all_hashes.extend(computed);
|
||||
state
|
||||
.cache
|
||||
.insert(filename.clone(), size, mtime, all_hashes)
|
||||
.await;
|
||||
|
||||
// Save cache periodically (could be optimized with debouncing)
|
||||
if let Err(e) = state.cache.save().await {
|
||||
tracing::warn!("Failed to save cache: {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Store results to meta-core
|
||||
if !results.is_empty() {
|
||||
match MetaCoreClient::new(&req.meta_core_url) {
|
||||
Ok(client) => {
|
||||
if let Err(e) = client.merge_metadata(&req.cid, &results).await {
|
||||
tracing::warn!("Failed to store metadata to meta-core: {}", e);
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to create meta-core client: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(ProcessResult::Completed)
|
||||
}
|
||||
|
||||
async fn send_callback(url: &str, payload: &CallbackPayload) -> anyhow::Result<()> {
|
||||
let client = reqwest::Client::new();
|
||||
let resp = client.post(url).json(payload).send().await?;
|
||||
|
||||
if !resp.status().is_success() {
|
||||
anyhow::bail!("Callback failed with status {}", resp.status());
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Vendored
+200
@@ -0,0 +1,200 @@
|
||||
use anyhow::{Context, Result};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashMap;
|
||||
use std::fs::{self, File};
|
||||
use std::io::{BufReader, BufWriter};
|
||||
use std::path::{Path, PathBuf};
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
const CACHE_VERSION: u32 = 1;
|
||||
const CACHE_FILENAME: &str = "hash_cache.bin";
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheEntry {
|
||||
pub filename: String,
|
||||
pub size: u64,
|
||||
pub mtime: i64,
|
||||
pub hashes: HashMap<String, String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct CacheIndex {
|
||||
pub version: u32,
|
||||
pub entries: HashMap<String, CacheEntry>,
|
||||
}
|
||||
|
||||
impl Default for CacheIndex {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
version: CACHE_VERSION,
|
||||
entries: HashMap::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl CacheIndex {
|
||||
/// Generate cache key from file metadata
|
||||
pub fn make_key(filename: &str, size: u64, mtime: i64) -> String {
|
||||
format!("{}-{}-{}", filename, size, mtime)
|
||||
}
|
||||
|
||||
/// Get cached entry if it exists and matches file metadata
|
||||
pub fn get(&self, filename: &str, size: u64, mtime: i64) -> Option<&CacheEntry> {
|
||||
let key = Self::make_key(filename, size, mtime);
|
||||
self.entries.get(&key)
|
||||
}
|
||||
|
||||
/// Insert or update a cache entry
|
||||
pub fn insert(&mut self, filename: String, size: u64, mtime: i64, hashes: HashMap<String, String>) {
|
||||
let key = Self::make_key(&filename, size, mtime);
|
||||
self.entries.insert(
|
||||
key,
|
||||
CacheEntry {
|
||||
filename,
|
||||
size,
|
||||
mtime,
|
||||
hashes,
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
pub struct CacheManager {
|
||||
cache_path: PathBuf,
|
||||
index: Arc<RwLock<CacheIndex>>,
|
||||
}
|
||||
|
||||
impl CacheManager {
|
||||
pub fn new(cache_dir: &str) -> Self {
|
||||
let cache_path = PathBuf::from(cache_dir).join(CACHE_FILENAME);
|
||||
Self {
|
||||
cache_path,
|
||||
index: Arc::new(RwLock::new(CacheIndex::default())),
|
||||
}
|
||||
}
|
||||
|
||||
/// Load cache from disk
|
||||
pub async fn load(&self) -> Result<()> {
|
||||
if !self.cache_path.exists() {
|
||||
tracing::info!("No existing cache found at {:?}", self.cache_path);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let path = self.cache_path.clone();
|
||||
let index = tokio::task::spawn_blocking(move || -> Result<CacheIndex> {
|
||||
let file = File::open(&path).context("Failed to open cache file")?;
|
||||
let reader = BufReader::new(file);
|
||||
let index: CacheIndex =
|
||||
bincode::deserialize_from(reader).context("Failed to deserialize cache")?;
|
||||
|
||||
// Version check
|
||||
if index.version != CACHE_VERSION {
|
||||
tracing::warn!(
|
||||
"Cache version mismatch (got {}, expected {}), starting fresh",
|
||||
index.version,
|
||||
CACHE_VERSION
|
||||
);
|
||||
return Ok(CacheIndex::default());
|
||||
}
|
||||
|
||||
Ok(index)
|
||||
})
|
||||
.await??;
|
||||
|
||||
tracing::info!("Loaded {} cache entries", index.entries.len());
|
||||
*self.index.write().await = index;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Save cache to disk atomically
|
||||
pub async fn save(&self) -> Result<()> {
|
||||
let index = self.index.read().await.clone();
|
||||
let cache_path = self.cache_path.clone();
|
||||
|
||||
tokio::task::spawn_blocking(move || -> Result<()> {
|
||||
// Ensure parent directory exists
|
||||
if let Some(parent) = cache_path.parent() {
|
||||
fs::create_dir_all(parent)?;
|
||||
}
|
||||
|
||||
// Write to temp file first
|
||||
let temp_path = cache_path.with_extension("tmp");
|
||||
{
|
||||
let file = File::create(&temp_path)?;
|
||||
let writer = BufWriter::new(file);
|
||||
bincode::serialize_into(writer, &index)?;
|
||||
}
|
||||
|
||||
// Atomic rename
|
||||
fs::rename(&temp_path, &cache_path)?;
|
||||
Ok(())
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get cached hashes for a file
|
||||
pub async fn get(&self, filename: &str, size: u64, mtime: i64) -> Option<HashMap<String, String>> {
|
||||
let index = self.index.read().await;
|
||||
index.get(filename, size, mtime).map(|e| e.hashes.clone())
|
||||
}
|
||||
|
||||
/// Store hashes in cache
|
||||
pub async fn insert(&self, filename: String, size: u64, mtime: i64, hashes: HashMap<String, String>) {
|
||||
let mut index = self.index.write().await;
|
||||
index.insert(filename, size, mtime, hashes);
|
||||
}
|
||||
|
||||
/// Get missing hash algorithms for a file (not in cache or not computed)
|
||||
pub async fn get_missing_algos(
|
||||
&self,
|
||||
filename: &str,
|
||||
size: u64,
|
||||
mtime: i64,
|
||||
required: &[&str],
|
||||
) -> Vec<String> {
|
||||
let index = self.index.read().await;
|
||||
let cached = index.get(filename, size, mtime);
|
||||
|
||||
required
|
||||
.iter()
|
||||
.filter(|algo| {
|
||||
cached
|
||||
.map(|e| !e.hashes.contains_key(**algo))
|
||||
.unwrap_or(true)
|
||||
})
|
||||
.map(|s| s.to_string())
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::TempDir;
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_cache_roundtrip() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let cache = CacheManager::new(temp_dir.path().to_str().unwrap());
|
||||
|
||||
// Insert some data
|
||||
let mut hashes = HashMap::new();
|
||||
hashes.insert("cid_sha2-256".to_string(), "baejbeitest".to_string());
|
||||
cache
|
||||
.insert("test.mkv".to_string(), 1000, 12345, hashes.clone())
|
||||
.await;
|
||||
|
||||
// Save and reload
|
||||
cache.save().await.unwrap();
|
||||
|
||||
let cache2 = CacheManager::new(temp_dir.path().to_str().unwrap());
|
||||
cache2.load().await.unwrap();
|
||||
|
||||
let result = cache2.get("test.mkv", 1000, 12345).await;
|
||||
assert!(result.is_some());
|
||||
assert_eq!(result.unwrap().get("cid_sha2-256").unwrap(), "baejbeitest");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,139 @@
|
||||
use anyhow::{Context, Result};
|
||||
use reqwest::Client;
|
||||
use serde_json::json;
|
||||
use std::collections::HashMap;
|
||||
use std::time::Duration;
|
||||
|
||||
const REQUEST_TIMEOUT: Duration = Duration::from_secs(30);
|
||||
const MAX_RETRIES: u32 = 3;
|
||||
const RETRY_DELAY: Duration = Duration::from_millis(500);
|
||||
|
||||
pub struct MetaCoreClient {
|
||||
client: Client,
|
||||
base_url: String,
|
||||
}
|
||||
|
||||
impl MetaCoreClient {
|
||||
pub fn new(base_url: &str) -> Result<Self> {
|
||||
let client = Client::builder()
|
||||
.timeout(REQUEST_TIMEOUT)
|
||||
.build()
|
||||
.context("Failed to create HTTP client")?;
|
||||
|
||||
Ok(Self {
|
||||
client,
|
||||
base_url: base_url.trim_end_matches('/').to_string(),
|
||||
})
|
||||
}
|
||||
|
||||
/// Set a single property for a hash ID
|
||||
pub async fn set_property(&self, hash_id: &str, key: &str, value: &str) -> Result<()> {
|
||||
let url = format!("{}/api/meta/{}/{}", self.base_url, hash_id, key);
|
||||
|
||||
for attempt in 1..=MAX_RETRIES {
|
||||
match self
|
||||
.client
|
||||
.put(&url)
|
||||
.json(&json!({ "value": value }))
|
||||
.send()
|
||||
.await
|
||||
{
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
return Ok(());
|
||||
}
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
let body = resp.text().await.unwrap_or_default();
|
||||
if attempt == MAX_RETRIES {
|
||||
anyhow::bail!(
|
||||
"Failed to set property {} for {}: {} - {}",
|
||||
key,
|
||||
hash_id,
|
||||
status,
|
||||
body
|
||||
);
|
||||
}
|
||||
tracing::warn!(
|
||||
"Retry {}/{} for set_property: {} - {}",
|
||||
attempt,
|
||||
MAX_RETRIES,
|
||||
status,
|
||||
body
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
if attempt == MAX_RETRIES {
|
||||
return Err(e).context(format!(
|
||||
"Failed to set property {} for {}",
|
||||
key, hash_id
|
||||
));
|
||||
}
|
||||
tracing::warn!("Retry {}/{} for set_property: {}", attempt, MAX_RETRIES, e);
|
||||
}
|
||||
}
|
||||
tokio::time::sleep(RETRY_DELAY).await;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Merge multiple properties at once
|
||||
pub async fn merge_metadata(
|
||||
&self,
|
||||
hash_id: &str,
|
||||
metadata: &HashMap<String, String>,
|
||||
) -> Result<()> {
|
||||
// Try batch endpoint first
|
||||
let url = format!("{}/api/meta/{}", self.base_url, hash_id);
|
||||
|
||||
match self.client.patch(&url).json(metadata).send().await {
|
||||
Ok(resp) if resp.status().is_success() => {
|
||||
return Ok(());
|
||||
}
|
||||
Ok(resp) if resp.status().as_u16() == 404 => {
|
||||
// Batch endpoint not available, fall back to individual sets
|
||||
tracing::debug!("Batch endpoint not available, using individual sets");
|
||||
}
|
||||
Ok(resp) => {
|
||||
let status = resp.status();
|
||||
tracing::warn!("Batch merge failed with {}, falling back to individual sets", status);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Batch merge error: {}, falling back to individual sets", e);
|
||||
}
|
||||
}
|
||||
|
||||
// Fall back to individual property sets
|
||||
for (key, value) in metadata {
|
||||
self.set_property(hash_id, key, value).await?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Health check
|
||||
pub async fn health(&self) -> Result<bool> {
|
||||
let url = format!("{}/health", self.base_url);
|
||||
match self.client.get(&url).send().await {
|
||||
Ok(resp) => Ok(resp.status().is_success()),
|
||||
Err(_) => Ok(false),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_client_creation() {
|
||||
let client = MetaCoreClient::new("http://localhost:9000");
|
||||
assert!(client.is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_url_normalization() {
|
||||
let client = MetaCoreClient::new("http://localhost:9000/").unwrap();
|
||||
assert_eq!(client.base_url, "http://localhost:9000");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,3 @@
|
||||
mod meta_core;
|
||||
|
||||
pub use meta_core::MetaCoreClient;
|
||||
+162
@@ -0,0 +1,162 @@
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::collections::HashSet;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::RwLock;
|
||||
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)]
|
||||
pub enum HashAlgorithm {
|
||||
#[serde(rename = "cid_sha2-256")]
|
||||
Sha256,
|
||||
#[serde(rename = "cid_sha1")]
|
||||
Sha1,
|
||||
#[serde(rename = "cid_md5")]
|
||||
Md5,
|
||||
#[serde(rename = "cid_crc32")]
|
||||
Crc32,
|
||||
#[serde(rename = "cid_sha3-256")]
|
||||
Sha3_256,
|
||||
#[serde(rename = "cid_sha3-384")]
|
||||
Sha3_384,
|
||||
#[serde(rename = "cid_midhash256")]
|
||||
Midhash256,
|
||||
#[serde(rename = "cid_bt_pieces_root")]
|
||||
BtPiecesRoot,
|
||||
#[serde(rename = "cid_bt_info_hash")]
|
||||
BtInfoHash,
|
||||
}
|
||||
|
||||
// Multicodec hash codes
|
||||
// Standard codes from https://github.com/multiformats/multicodec
|
||||
pub const SHA256_CODE: u64 = 0x12;
|
||||
pub const SHA1_CODE: u64 = 0x11;
|
||||
pub const MD5_CODE: u64 = 0xd5;
|
||||
pub const CRC32_CODE: u64 = 0x0132;
|
||||
pub const SHA3_256_CODE: u64 = 0x16;
|
||||
pub const SHA3_384_CODE: u64 = 0x15;
|
||||
|
||||
// Custom hash codes
|
||||
pub const MIDHASH256_CODE: u64 = 0x1000; // Custom, not in official registry
|
||||
|
||||
// Official BitTorrent v2 code
|
||||
pub const BT_PIECES_ROOT_CODE: u64 = 0xb702; // Official bittorrent-pieces-root
|
||||
// BT info hash uses SHA256_CODE (0x12) since it's a SHA-256 hash
|
||||
|
||||
impl HashAlgorithm {
|
||||
pub fn all() -> Vec<HashAlgorithm> {
|
||||
vec![
|
||||
HashAlgorithm::Sha256,
|
||||
HashAlgorithm::Sha1,
|
||||
HashAlgorithm::Md5,
|
||||
HashAlgorithm::Crc32,
|
||||
HashAlgorithm::Sha3_256,
|
||||
HashAlgorithm::Sha3_384,
|
||||
HashAlgorithm::Midhash256,
|
||||
HashAlgorithm::BtPiecesRoot,
|
||||
HashAlgorithm::BtInfoHash,
|
||||
]
|
||||
}
|
||||
|
||||
/// Standard algorithms that can be computed in a single I/O pass
|
||||
pub fn standard_algorithms() -> Vec<HashAlgorithm> {
|
||||
vec![
|
||||
HashAlgorithm::Sha256,
|
||||
HashAlgorithm::Sha1,
|
||||
HashAlgorithm::Md5,
|
||||
HashAlgorithm::Crc32,
|
||||
HashAlgorithm::Sha3_256,
|
||||
HashAlgorithm::Sha3_384,
|
||||
]
|
||||
}
|
||||
|
||||
/// Special algorithms that require separate file reads
|
||||
pub fn special_algorithms() -> Vec<HashAlgorithm> {
|
||||
vec![
|
||||
HashAlgorithm::Midhash256,
|
||||
HashAlgorithm::BtPiecesRoot,
|
||||
HashAlgorithm::BtInfoHash,
|
||||
]
|
||||
}
|
||||
|
||||
/// Returns true if this algorithm requires a separate file read
|
||||
pub fn is_special(&self) -> bool {
|
||||
matches!(
|
||||
self,
|
||||
HashAlgorithm::Midhash256 | HashAlgorithm::BtPiecesRoot | HashAlgorithm::BtInfoHash
|
||||
)
|
||||
}
|
||||
|
||||
pub fn key(&self) -> &'static str {
|
||||
match self {
|
||||
HashAlgorithm::Sha256 => "cid_sha2-256",
|
||||
HashAlgorithm::Sha1 => "cid_sha1",
|
||||
HashAlgorithm::Md5 => "cid_md5",
|
||||
HashAlgorithm::Crc32 => "cid_crc32",
|
||||
HashAlgorithm::Sha3_256 => "cid_sha3-256",
|
||||
HashAlgorithm::Sha3_384 => "cid_sha3-384",
|
||||
HashAlgorithm::Midhash256 => "cid_midhash256",
|
||||
HashAlgorithm::BtPiecesRoot => "cid_bt_pieces_root",
|
||||
HashAlgorithm::BtInfoHash => "cid_bt_info_hash",
|
||||
}
|
||||
}
|
||||
|
||||
pub fn from_key(key: &str) -> Option<HashAlgorithm> {
|
||||
match key {
|
||||
"cid_sha2-256" => Some(HashAlgorithm::Sha256),
|
||||
"cid_sha1" => Some(HashAlgorithm::Sha1),
|
||||
"cid_md5" => Some(HashAlgorithm::Md5),
|
||||
"cid_crc32" => Some(HashAlgorithm::Crc32),
|
||||
"cid_sha3-256" => Some(HashAlgorithm::Sha3_256),
|
||||
"cid_sha3-384" => Some(HashAlgorithm::Sha3_384),
|
||||
"cid_midhash256" => Some(HashAlgorithm::Midhash256),
|
||||
"cid_bt_pieces_root" => Some(HashAlgorithm::BtPiecesRoot),
|
||||
"cid_bt_info_hash" => Some(HashAlgorithm::BtInfoHash),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the multicodec code for this hash algorithm
|
||||
pub fn multicodec(&self) -> u64 {
|
||||
match self {
|
||||
HashAlgorithm::Sha256 => SHA256_CODE,
|
||||
HashAlgorithm::Sha1 => SHA1_CODE,
|
||||
HashAlgorithm::Md5 => MD5_CODE,
|
||||
HashAlgorithm::Crc32 => CRC32_CODE,
|
||||
HashAlgorithm::Sha3_256 => SHA3_256_CODE,
|
||||
HashAlgorithm::Sha3_384 => SHA3_384_CODE,
|
||||
HashAlgorithm::Midhash256 => MIDHASH256_CODE,
|
||||
HashAlgorithm::BtPiecesRoot => BT_PIECES_ROOT_CODE,
|
||||
HashAlgorithm::BtInfoHash => SHA256_CODE, // Info hash is SHA-256
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PluginConfig {
|
||||
pub enabled_hashes: HashSet<HashAlgorithm>,
|
||||
pub cache_path: String,
|
||||
pub buffer_size: usize,
|
||||
}
|
||||
|
||||
impl Default for PluginConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
enabled_hashes: HashAlgorithm::all().into_iter().collect(),
|
||||
cache_path: std::env::var("CACHE_PATH").unwrap_or_else(|_| "/cache".to_string()),
|
||||
buffer_size: 1024 * 1024, // 1MB
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl PluginConfig {
|
||||
pub fn parse_enabled_hashes(s: &str) -> HashSet<HashAlgorithm> {
|
||||
s.split(',')
|
||||
.filter_map(|k| HashAlgorithm::from_key(k.trim()))
|
||||
.collect()
|
||||
}
|
||||
}
|
||||
|
||||
pub type SharedConfig = Arc<RwLock<PluginConfig>>;
|
||||
|
||||
pub fn create_shared_config() -> SharedConfig {
|
||||
Arc::new(RwLock::new(PluginConfig::default()))
|
||||
}
|
||||
@@ -0,0 +1,383 @@
|
||||
use crate::config::HashAlgorithm;
|
||||
use anyhow::Result;
|
||||
use md5::Md5;
|
||||
use sha1::Sha1;
|
||||
use sha2::{Digest, Sha256};
|
||||
use sha3::{Sha3_256, Sha3_384};
|
||||
use std::collections::{HashMap, HashSet};
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read};
|
||||
use std::path::Path;
|
||||
|
||||
use super::btih_v2::{compute_info_hash, compute_merkle_root, BT_BLOCK_SIZE};
|
||||
use super::cid::to_cid;
|
||||
use super::midhash256::compute_midhash256;
|
||||
|
||||
/// Hash a file using all enabled algorithms in a single I/O pass
|
||||
///
|
||||
/// All algorithms except Midhash256 are computed in the same pass:
|
||||
/// - Standard algorithms update their hashers with each buffer chunk
|
||||
/// - BitTorrent v2 computes 16KB block hashes from the same data
|
||||
///
|
||||
/// Midhash256 requires a separate O(1) read (middle 1MB only).
|
||||
pub fn hash_file(
|
||||
path: &Path,
|
||||
enabled: &HashSet<HashAlgorithm>,
|
||||
buffer_size: usize,
|
||||
) -> Result<HashMap<String, String>> {
|
||||
let mut results = HashMap::new();
|
||||
|
||||
// Check what we need to compute
|
||||
let needs_bt = enabled.contains(&HashAlgorithm::BtPiecesRoot)
|
||||
|| enabled.contains(&HashAlgorithm::BtInfoHash);
|
||||
|
||||
// Get filename for BT v2 info hash (needed before we start)
|
||||
let filename = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
// Main single-pass computation (standard + BT v2)
|
||||
let (standard_results, bt_leaf_hashes, file_size) =
|
||||
hash_file_single_pass(path, enabled, buffer_size, needs_bt)?;
|
||||
results.extend(standard_results);
|
||||
|
||||
// Finalize BT v2 hashes from collected leaf hashes
|
||||
if needs_bt {
|
||||
let pieces_root = compute_merkle_root(&bt_leaf_hashes);
|
||||
|
||||
if enabled.contains(&HashAlgorithm::BtPiecesRoot) {
|
||||
results.insert(
|
||||
HashAlgorithm::BtPiecesRoot.key().to_string(),
|
||||
to_cid(&pieces_root, HashAlgorithm::BtPiecesRoot),
|
||||
);
|
||||
}
|
||||
|
||||
if enabled.contains(&HashAlgorithm::BtInfoHash) {
|
||||
match compute_info_hash(&filename, file_size, &pieces_root) {
|
||||
Ok(info_hash) => {
|
||||
results.insert(
|
||||
HashAlgorithm::BtInfoHash.key().to_string(),
|
||||
to_cid(&info_hash, HashAlgorithm::BtInfoHash),
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to compute BT info hash: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Midhash256 requires separate O(1) read (only reads middle 1MB)
|
||||
if enabled.contains(&HashAlgorithm::Midhash256) {
|
||||
match compute_midhash256(path) {
|
||||
Ok(hash) => {
|
||||
results.insert(
|
||||
HashAlgorithm::Midhash256.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Midhash256),
|
||||
);
|
||||
}
|
||||
Err(e) => {
|
||||
tracing::warn!("Failed to compute midhash256: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Ok(results)
|
||||
}
|
||||
|
||||
/// Single-pass file hashing for all standard algorithms + BT v2 leaf collection
|
||||
///
|
||||
/// Returns: (standard hash results, BT v2 leaf hashes, file size)
|
||||
fn hash_file_single_pass(
|
||||
path: &Path,
|
||||
enabled: &HashSet<HashAlgorithm>,
|
||||
buffer_size: usize,
|
||||
collect_bt_leaves: bool,
|
||||
) -> Result<(HashMap<String, String>, Vec<[u8; 32]>, u64)> {
|
||||
let file = File::open(path)?;
|
||||
let file_size = file.metadata()?.len();
|
||||
let mut reader = BufReader::with_capacity(buffer_size, file);
|
||||
let mut buffer = vec![0u8; buffer_size];
|
||||
|
||||
// Initialize standard hashers (only if enabled)
|
||||
let mut sha256 = enabled
|
||||
.contains(&HashAlgorithm::Sha256)
|
||||
.then(Sha256::new);
|
||||
let mut sha1 = enabled.contains(&HashAlgorithm::Sha1).then(Sha1::new);
|
||||
let mut md5 = enabled.contains(&HashAlgorithm::Md5).then(Md5::new);
|
||||
let mut crc32 = enabled
|
||||
.contains(&HashAlgorithm::Crc32)
|
||||
.then(crc32fast::Hasher::new);
|
||||
let mut sha3_256 = enabled
|
||||
.contains(&HashAlgorithm::Sha3_256)
|
||||
.then(Sha3_256::new);
|
||||
let mut sha3_384 = enabled
|
||||
.contains(&HashAlgorithm::Sha3_384)
|
||||
.then(Sha3_384::new);
|
||||
|
||||
// BT v2: collect leaf hashes (SHA-256 of each 16KB block)
|
||||
let mut bt_leaf_hashes: Vec<[u8; 32]> = Vec::new();
|
||||
let mut bt_block_buffer = vec![0u8; BT_BLOCK_SIZE];
|
||||
let mut bt_block_offset = 0usize;
|
||||
|
||||
let mut bytes_read: u64 = 0;
|
||||
|
||||
// Single read pass
|
||||
loop {
|
||||
let n = reader.read(&mut buffer)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
let chunk = &buffer[..n];
|
||||
bytes_read += n as u64;
|
||||
|
||||
// Update all standard hashers
|
||||
if let Some(ref mut h) = sha256 {
|
||||
h.update(chunk);
|
||||
}
|
||||
if let Some(ref mut h) = sha1 {
|
||||
h.update(chunk);
|
||||
}
|
||||
if let Some(ref mut h) = md5 {
|
||||
h.update(chunk);
|
||||
}
|
||||
if let Some(ref mut h) = crc32 {
|
||||
h.update(chunk);
|
||||
}
|
||||
if let Some(ref mut h) = sha3_256 {
|
||||
h.update(chunk);
|
||||
}
|
||||
if let Some(ref mut h) = sha3_384 {
|
||||
h.update(chunk);
|
||||
}
|
||||
|
||||
// BT v2: Process chunk into 16KB blocks
|
||||
if collect_bt_leaves {
|
||||
let mut chunk_offset = 0;
|
||||
while chunk_offset < n {
|
||||
let space_in_block = BT_BLOCK_SIZE - bt_block_offset;
|
||||
let bytes_to_copy = space_in_block.min(n - chunk_offset);
|
||||
|
||||
bt_block_buffer[bt_block_offset..bt_block_offset + bytes_to_copy]
|
||||
.copy_from_slice(&chunk[chunk_offset..chunk_offset + bytes_to_copy]);
|
||||
|
||||
bt_block_offset += bytes_to_copy;
|
||||
chunk_offset += bytes_to_copy;
|
||||
|
||||
// Block is full, hash it
|
||||
if bt_block_offset == BT_BLOCK_SIZE {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bt_block_buffer);
|
||||
bt_leaf_hashes.push(hasher.finalize().into());
|
||||
bt_block_offset = 0;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Log progress for large files (every 100MB)
|
||||
if bytes_read % (100 * 1024 * 1024) == 0 && file_size > 100 * 1024 * 1024 {
|
||||
let pct = (bytes_read as f64 / file_size as f64 * 100.0) as u32;
|
||||
tracing::debug!(
|
||||
"Hashing progress: {}% ({}/{})",
|
||||
pct,
|
||||
bytes_read,
|
||||
file_size
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// BT v2: Handle final partial block (pad with zeros as per BEP 52)
|
||||
if collect_bt_leaves {
|
||||
if file_size == 0 {
|
||||
// Empty file: single leaf of all zeros
|
||||
bt_leaf_hashes.push([0u8; 32]);
|
||||
} else if bt_block_offset > 0 {
|
||||
// Pad remaining bytes with zeros
|
||||
bt_block_buffer[bt_block_offset..].fill(0);
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&bt_block_buffer);
|
||||
bt_leaf_hashes.push(hasher.finalize().into());
|
||||
}
|
||||
}
|
||||
|
||||
// Finalize standard hashes and format as CIDs
|
||||
let mut results = HashMap::new();
|
||||
|
||||
if let Some(h) = sha256 {
|
||||
let hash = h.finalize();
|
||||
results.insert(
|
||||
HashAlgorithm::Sha256.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Sha256),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(h) = sha1 {
|
||||
let hash = h.finalize();
|
||||
results.insert(
|
||||
HashAlgorithm::Sha1.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Sha1),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(h) = md5 {
|
||||
let hash = h.finalize();
|
||||
results.insert(
|
||||
HashAlgorithm::Md5.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Md5),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(h) = crc32 {
|
||||
let hash = h.finalize().to_be_bytes();
|
||||
results.insert(
|
||||
HashAlgorithm::Crc32.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Crc32),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(h) = sha3_256 {
|
||||
let hash = h.finalize();
|
||||
results.insert(
|
||||
HashAlgorithm::Sha3_256.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Sha3_256),
|
||||
);
|
||||
}
|
||||
|
||||
if let Some(h) = sha3_384 {
|
||||
let hash = h.finalize();
|
||||
results.insert(
|
||||
HashAlgorithm::Sha3_384.key().to_string(),
|
||||
to_cid(&hash, HashAlgorithm::Sha3_384),
|
||||
);
|
||||
}
|
||||
|
||||
Ok((results, bt_leaf_hashes, file_size))
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
fn test_hash_file_standard() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"hello world").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha256, HashAlgorithm::Md5]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let results = hash_file(file.path(), &enabled, 1024).unwrap();
|
||||
|
||||
assert!(results.contains_key("cid_sha2-256"));
|
||||
assert!(results.contains_key("cid_md5"));
|
||||
assert!(!results.contains_key("cid_sha1"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hash_file_with_midhash() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"test content for midhash").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha256, HashAlgorithm::Midhash256]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let results = hash_file(file.path(), &enabled, 1024).unwrap();
|
||||
|
||||
assert!(results.contains_key("cid_sha2-256"));
|
||||
assert!(results.contains_key("cid_midhash256"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hash_file_with_bittorrent() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"test content for bittorrent").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let results = hash_file(file.path(), &enabled, 1024).unwrap();
|
||||
|
||||
assert!(results.contains_key("cid_bt_pieces_root"));
|
||||
assert!(results.contains_key("cid_bt_info_hash"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_hash_file_all_algorithms() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"comprehensive test content").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let enabled: HashSet<_> = HashAlgorithm::all().into_iter().collect();
|
||||
|
||||
let results = hash_file(file.path(), &enabled, 1024).unwrap();
|
||||
|
||||
// All 9 algorithms should be present
|
||||
assert!(results.contains_key("cid_sha2-256"));
|
||||
assert!(results.contains_key("cid_sha1"));
|
||||
assert!(results.contains_key("cid_md5"));
|
||||
assert!(results.contains_key("cid_crc32"));
|
||||
assert!(results.contains_key("cid_sha3-256"));
|
||||
assert!(results.contains_key("cid_sha3-384"));
|
||||
assert!(results.contains_key("cid_midhash256"));
|
||||
assert!(results.contains_key("cid_bt_pieces_root"));
|
||||
assert!(results.contains_key("cid_bt_info_hash"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bt_hashes_match_standalone() {
|
||||
// Verify that single-pass BT hashes match the standalone implementation
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
// Use data larger than one BT block to test multi-block handling
|
||||
let data = vec![0xab; BT_BLOCK_SIZE * 2 + 1000];
|
||||
file.write_all(&data).unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
// Compute via single-pass
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let single_pass_results = hash_file(file.path(), &enabled, 1024 * 1024).unwrap();
|
||||
|
||||
// Compute via standalone (which still exists for comparison)
|
||||
use super::super::btih_v2::compute_btv2_hashes;
|
||||
let standalone = compute_btv2_hashes(file.path()).unwrap();
|
||||
let standalone_pieces_root = to_cid(&standalone.pieces_root, HashAlgorithm::BtPiecesRoot);
|
||||
let standalone_info_hash = to_cid(&standalone.info_hash, HashAlgorithm::BtInfoHash);
|
||||
|
||||
// Results should match
|
||||
assert_eq!(
|
||||
single_pass_results.get("cid_bt_pieces_root").unwrap(),
|
||||
&standalone_pieces_root,
|
||||
"Pieces root mismatch"
|
||||
);
|
||||
assert_eq!(
|
||||
single_pass_results.get("cid_bt_info_hash").unwrap(),
|
||||
&standalone_info_hash,
|
||||
"Info hash mismatch"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_file_bt() {
|
||||
let file = NamedTempFile::new().unwrap();
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot, HashAlgorithm::BtInfoHash]
|
||||
.into_iter()
|
||||
.collect();
|
||||
let results = hash_file(file.path(), &enabled, 1024).unwrap();
|
||||
|
||||
assert!(results.contains_key("cid_bt_pieces_root"));
|
||||
assert!(results.contains_key("cid_bt_info_hash"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,368 @@
|
||||
//! BitTorrent v2 Hash Computation (BEP 52)
|
||||
//!
|
||||
//! Computes two hashes for BitTorrent v2 compatibility:
|
||||
//! - pieces_root: Merkle tree root of 16KB block hashes
|
||||
//! - info_hash: SHA-256 of bencoded info dictionary
|
||||
//!
|
||||
//! Reference: https://www.bittorrent.org/beps/bep_0052.html
|
||||
|
||||
use anyhow::Result;
|
||||
use bendy::encoding::{AsString, SingleItemEncoder, ToBencode};
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fs::File;
|
||||
use std::io::{BufReader, Read};
|
||||
use std::path::Path;
|
||||
|
||||
/// Block size for BitTorrent v2 (16 KiB)
|
||||
pub const BT_BLOCK_SIZE: usize = 16 * 1024;
|
||||
|
||||
/// Piece size (must be power of 2, minimum 16 KiB)
|
||||
/// We use 16 KiB for maximum granularity
|
||||
const PIECE_SIZE: usize = 16 * 1024;
|
||||
|
||||
/// Result of BitTorrent v2 hash computation
|
||||
pub struct BtV2Hashes {
|
||||
/// Merkle tree root of file pieces (32 bytes)
|
||||
pub pieces_root: [u8; 32],
|
||||
/// SHA-256 of bencoded info dict (32 bytes)
|
||||
pub info_hash: [u8; 32],
|
||||
}
|
||||
|
||||
/// Compute BitTorrent v2 hashes for a file
|
||||
pub fn compute_btv2_hashes(path: &Path) -> Result<BtV2Hashes> {
|
||||
let file = File::open(path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
// Get filename for info dict
|
||||
let filename = path
|
||||
.file_name()
|
||||
.and_then(|n| n.to_str())
|
||||
.unwrap_or("unknown")
|
||||
.to_string();
|
||||
|
||||
// Compute leaf hashes (hash of each 16KB block)
|
||||
let leaf_hashes = compute_leaf_hashes(&file, file_size)?;
|
||||
|
||||
// Build merkle tree and get root
|
||||
let pieces_root = compute_merkle_root(&leaf_hashes);
|
||||
|
||||
// Compute info hash
|
||||
let info_hash = compute_info_hash(&filename, file_size, &pieces_root)?;
|
||||
|
||||
Ok(BtV2Hashes {
|
||||
pieces_root,
|
||||
info_hash,
|
||||
})
|
||||
}
|
||||
|
||||
/// Compute SHA-256 hash of each 16KB block
|
||||
fn compute_leaf_hashes(file: &File, file_size: u64) -> Result<Vec<[u8; 32]>> {
|
||||
if file_size == 0 {
|
||||
// Empty file: single leaf of all zeros
|
||||
return Ok(vec![[0u8; 32]]);
|
||||
}
|
||||
|
||||
let mut reader = BufReader::with_capacity(BT_BLOCK_SIZE * 4, file);
|
||||
let mut buffer = vec![0u8; BT_BLOCK_SIZE];
|
||||
let mut leaf_hashes = Vec::new();
|
||||
|
||||
loop {
|
||||
let bytes_read = reader.read(&mut buffer)?;
|
||||
if bytes_read == 0 {
|
||||
break;
|
||||
}
|
||||
|
||||
// Pad last block with zeros if needed (as per BEP 52)
|
||||
if bytes_read < BT_BLOCK_SIZE {
|
||||
buffer[bytes_read..].fill(0);
|
||||
}
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&buffer[..BT_BLOCK_SIZE]);
|
||||
let hash: [u8; 32] = hasher.finalize().into();
|
||||
leaf_hashes.push(hash);
|
||||
|
||||
// If this was a partial read, we're done
|
||||
if bytes_read < BT_BLOCK_SIZE {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(leaf_hashes)
|
||||
}
|
||||
|
||||
/// Compute Merkle tree root from leaf hashes
|
||||
///
|
||||
/// BEP 52 merkle tree:
|
||||
/// - Leaves are SHA-256 hashes of 16KB blocks
|
||||
/// - Internal nodes are SHA-256(left || right)
|
||||
/// - Tree is padded to power of 2 with zero hashes
|
||||
pub fn compute_merkle_root(leaf_hashes: &[[u8; 32]]) -> [u8; 32] {
|
||||
if leaf_hashes.is_empty() {
|
||||
return [0u8; 32];
|
||||
}
|
||||
|
||||
if leaf_hashes.len() == 1 {
|
||||
return leaf_hashes[0];
|
||||
}
|
||||
|
||||
// Pad to next power of 2
|
||||
let target_size = leaf_hashes.len().next_power_of_two();
|
||||
let mut current_level: Vec<[u8; 32]> = leaf_hashes.to_vec();
|
||||
|
||||
// Pad with zero hashes
|
||||
while current_level.len() < target_size {
|
||||
current_level.push([0u8; 32]);
|
||||
}
|
||||
|
||||
// Build tree bottom-up
|
||||
while current_level.len() > 1 {
|
||||
let mut next_level = Vec::with_capacity(current_level.len() / 2);
|
||||
|
||||
for chunk in current_level.chunks(2) {
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&chunk[0]);
|
||||
hasher.update(&chunk[1]);
|
||||
let hash: [u8; 32] = hasher.finalize().into();
|
||||
next_level.push(hash);
|
||||
}
|
||||
|
||||
current_level = next_level;
|
||||
}
|
||||
|
||||
current_level[0]
|
||||
}
|
||||
|
||||
/// Info dictionary structure for bencoding
|
||||
struct InfoDict<'a> {
|
||||
file_tree: FileTree<'a>,
|
||||
meta_version: i64,
|
||||
name: &'a str,
|
||||
piece_length: i64,
|
||||
}
|
||||
|
||||
/// File tree structure (single file case)
|
||||
struct FileTree<'a> {
|
||||
filename: &'a str,
|
||||
length: i64,
|
||||
pieces_root: &'a [u8; 32],
|
||||
}
|
||||
|
||||
impl ToBencode for InfoDict<'_> {
|
||||
const MAX_DEPTH: usize = 5;
|
||||
|
||||
fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> {
|
||||
encoder.emit_dict(|mut e| {
|
||||
// file tree (required for v2)
|
||||
e.emit_pair(b"file tree", &self.file_tree)?;
|
||||
// meta version (2 for v2-only torrents)
|
||||
e.emit_pair(b"meta version", self.meta_version)?;
|
||||
// name
|
||||
e.emit_pair(b"name", self.name)?;
|
||||
// piece length
|
||||
e.emit_pair(b"piece length", self.piece_length)?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
impl ToBencode for FileTree<'_> {
|
||||
const MAX_DEPTH: usize = 4;
|
||||
|
||||
fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> {
|
||||
encoder.emit_dict(|mut e| {
|
||||
// File tree is: { filename: { "": { length: N, pieces root: HASH } } }
|
||||
e.emit_pair(self.filename.as_bytes(), FileEntry {
|
||||
length: self.length,
|
||||
pieces_root: self.pieces_root,
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FileEntry<'a> {
|
||||
length: i64,
|
||||
pieces_root: &'a [u8; 32],
|
||||
}
|
||||
|
||||
impl ToBencode for FileEntry<'_> {
|
||||
const MAX_DEPTH: usize = 3;
|
||||
|
||||
fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> {
|
||||
encoder.emit_dict(|mut e| {
|
||||
// Empty string key contains file attributes
|
||||
e.emit_pair(b"", FileAttrs {
|
||||
length: self.length,
|
||||
pieces_root: self.pieces_root,
|
||||
})?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
struct FileAttrs<'a> {
|
||||
length: i64,
|
||||
pieces_root: &'a [u8; 32],
|
||||
}
|
||||
|
||||
impl ToBencode for FileAttrs<'_> {
|
||||
const MAX_DEPTH: usize = 2;
|
||||
|
||||
fn encode(&self, encoder: SingleItemEncoder) -> Result<(), bendy::encoding::Error> {
|
||||
encoder.emit_dict(|mut e| {
|
||||
e.emit_pair(b"length", self.length)?;
|
||||
e.emit_pair(b"pieces root", AsString(self.pieces_root))?;
|
||||
Ok(())
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute info hash (SHA-256 of bencoded info dict)
|
||||
pub fn compute_info_hash(filename: &str, file_size: u64, pieces_root: &[u8; 32]) -> Result<[u8; 32]> {
|
||||
let info = InfoDict {
|
||||
file_tree: FileTree {
|
||||
filename,
|
||||
length: file_size as i64,
|
||||
pieces_root,
|
||||
},
|
||||
meta_version: 2,
|
||||
name: filename,
|
||||
piece_length: PIECE_SIZE as i64,
|
||||
};
|
||||
|
||||
let encoded = info
|
||||
.to_bencode()
|
||||
.map_err(|e| anyhow::anyhow!("Failed to bencode info dict: {:?}", e))?;
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&encoded);
|
||||
Ok(hasher.finalize().into())
|
||||
}
|
||||
|
||||
/// Format info hash as magnet link URN
|
||||
///
|
||||
/// Format: urn:btmh:1220<hex_hash>
|
||||
/// Where 1220 = 0x12 (SHA-256 code) + 0x20 (32 bytes length)
|
||||
pub fn format_magnet_urn(info_hash: &[u8; 32]) -> String {
|
||||
let hex = info_hash
|
||||
.iter()
|
||||
.map(|b| format!("{:02x}", b))
|
||||
.collect::<String>();
|
||||
format!("urn:btmh:1220{}", hex)
|
||||
}
|
||||
|
||||
/// Format full magnet link
|
||||
pub fn format_magnet_link(info_hash: &[u8; 32], filename: &str) -> String {
|
||||
let urn = format_magnet_urn(info_hash);
|
||||
let encoded_name = urlencoding_filename(filename);
|
||||
format!("magnet:?xt={}&dn={}", urn, encoded_name)
|
||||
}
|
||||
|
||||
/// Simple URL encoding for filename (spaces and special chars)
|
||||
fn urlencoding_filename(s: &str) -> String {
|
||||
s.chars()
|
||||
.map(|c| match c {
|
||||
'a'..='z' | 'A'..='Z' | '0'..='9' | '-' | '_' | '.' | '~' => c.to_string(),
|
||||
' ' => "%20".to_string(),
|
||||
_ => format!("%{:02X}", c as u8),
|
||||
})
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
fn test_empty_file() {
|
||||
let file = NamedTempFile::new().unwrap();
|
||||
let hashes = compute_btv2_hashes(file.path()).unwrap();
|
||||
|
||||
assert_eq!(hashes.pieces_root.len(), 32);
|
||||
assert_eq!(hashes.info_hash.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_small_file() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"hello world").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hashes = compute_btv2_hashes(file.path()).unwrap();
|
||||
|
||||
assert_eq!(hashes.pieces_root.len(), 32);
|
||||
assert_eq!(hashes.info_hash.len(), 32);
|
||||
|
||||
// Verify determinism
|
||||
let hashes2 = compute_btv2_hashes(file.path()).unwrap();
|
||||
assert_eq!(hashes.pieces_root, hashes2.pieces_root);
|
||||
assert_eq!(hashes.info_hash, hashes2.info_hash);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exactly_one_block() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
let data = vec![0xab; BT_BLOCK_SIZE];
|
||||
file.write_all(&data).unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hashes = compute_btv2_hashes(file.path()).unwrap();
|
||||
assert_eq!(hashes.pieces_root.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_multiple_blocks() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
// 3 full blocks + partial
|
||||
let data = vec![0xcd; BT_BLOCK_SIZE * 3 + 100];
|
||||
file.write_all(&data).unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hashes = compute_btv2_hashes(file.path()).unwrap();
|
||||
assert_eq!(hashes.pieces_root.len(), 32);
|
||||
assert_eq!(hashes.info_hash.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merkle_root_single_leaf() {
|
||||
let leaf = [0xab; 32];
|
||||
let root = compute_merkle_root(&[leaf]);
|
||||
assert_eq!(root, leaf);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_merkle_root_two_leaves() {
|
||||
let leaf1 = [0x11; 32];
|
||||
let leaf2 = [0x22; 32];
|
||||
|
||||
let root = compute_merkle_root(&[leaf1, leaf2]);
|
||||
|
||||
// Manually compute expected root
|
||||
let mut hasher = Sha256::new();
|
||||
hasher.update(&leaf1);
|
||||
hasher.update(&leaf2);
|
||||
let expected: [u8; 32] = hasher.finalize().into();
|
||||
|
||||
assert_eq!(root, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_magnet_urn_format() {
|
||||
let hash = [0u8; 32];
|
||||
let urn = format_magnet_urn(&hash);
|
||||
assert!(urn.starts_with("urn:btmh:1220"));
|
||||
assert_eq!(urn.len(), "urn:btmh:1220".len() + 64); // 64 hex chars
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_magnet_link_format() {
|
||||
let hash = [0xab; 32];
|
||||
let link = format_magnet_link(&hash, "test file.mkv");
|
||||
assert!(link.starts_with("magnet:?xt=urn:btmh:1220"));
|
||||
assert!(link.contains("&dn=test%20file.mkv"));
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
use crate::config::HashAlgorithm;
|
||||
use cid::Cid;
|
||||
use multihash::Multihash;
|
||||
|
||||
/// CID codec for raw binary data
|
||||
const RAW_CODEC: u64 = 0x55;
|
||||
|
||||
/// Maximum multihash size (64 bytes for SHA3-512)
|
||||
const MAX_HASH_SIZE: usize = 64;
|
||||
|
||||
/// Convert raw hash bytes to proper CIDv1 format
|
||||
///
|
||||
/// CID structure:
|
||||
/// - Version: 1
|
||||
/// - Codec: 0x55 (raw binary data)
|
||||
/// - Multihash: hash_code (varint) + hash_length (varint) + hash_bytes
|
||||
///
|
||||
/// Returns base32lower encoded CID with 'b' multibase prefix
|
||||
pub fn to_cid(hash: &[u8], algo: HashAlgorithm) -> String {
|
||||
let hash_code = algo.multicodec();
|
||||
|
||||
// Create multihash: wrap hash bytes with algorithm code
|
||||
let multihash: Multihash<MAX_HASH_SIZE> =
|
||||
Multihash::wrap(hash_code, hash).expect("Hash size exceeds maximum");
|
||||
|
||||
// Create CIDv1 with raw codec
|
||||
let cid = Cid::new_v1(RAW_CODEC, multihash);
|
||||
|
||||
// Return base32lower string (multibase 'b' prefix is default for CIDv1)
|
||||
cid.to_string()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_cid_format() {
|
||||
// Test with a known SHA-256 hash
|
||||
let hash = [0u8; 32]; // 32 zero bytes
|
||||
let cid = to_cid(&hash, HashAlgorithm::Sha256);
|
||||
|
||||
// CIDv1 base32lower starts with 'b'
|
||||
assert!(cid.starts_with('b'), "CID should start with 'b' multibase prefix");
|
||||
|
||||
// Verify we can parse it back
|
||||
let parsed = Cid::try_from(cid.as_str()).expect("Should parse valid CID");
|
||||
assert_eq!(parsed.version(), cid::Version::V1);
|
||||
assert_eq!(parsed.codec(), RAW_CODEC);
|
||||
assert_eq!(parsed.hash().code(), 0x12); // SHA-256 code
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_all_algorithms() {
|
||||
let test_data = b"test data for hashing";
|
||||
|
||||
for algo in HashAlgorithm::all() {
|
||||
// Create a fake hash of appropriate size
|
||||
let hash_size = match algo {
|
||||
HashAlgorithm::Sha256 | HashAlgorithm::Sha3_256 | HashAlgorithm::Midhash256 => 32,
|
||||
HashAlgorithm::Sha1 => 20,
|
||||
HashAlgorithm::Md5 => 16,
|
||||
HashAlgorithm::Crc32 => 4,
|
||||
HashAlgorithm::Sha3_384 => 48,
|
||||
HashAlgorithm::BtPiecesRoot | HashAlgorithm::BtInfoHash => 32,
|
||||
};
|
||||
|
||||
let hash = vec![0xab; hash_size];
|
||||
let cid = to_cid(&hash, algo);
|
||||
|
||||
// All CIDs should start with 'b' (base32lower)
|
||||
assert!(
|
||||
cid.starts_with('b'),
|
||||
"CID for {:?} should start with 'b'",
|
||||
algo
|
||||
);
|
||||
|
||||
// All CIDs should be parseable
|
||||
let parsed = Cid::try_from(cid.as_str())
|
||||
.expect(&format!("Should parse valid CID for {:?}", algo));
|
||||
|
||||
assert_eq!(parsed.version(), cid::Version::V1);
|
||||
assert_eq!(parsed.codec(), RAW_CODEC);
|
||||
assert_eq!(
|
||||
parsed.hash().code(),
|
||||
algo.multicodec(),
|
||||
"Multicodec mismatch for {:?}",
|
||||
algo
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_cid_determinism() {
|
||||
// Same hash should always produce same CID
|
||||
let hash = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16];
|
||||
let cid1 = to_cid(&hash, HashAlgorithm::Md5);
|
||||
let cid2 = to_cid(&hash, HashAlgorithm::Md5);
|
||||
assert_eq!(cid1, cid2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_different_algorithms_different_cids() {
|
||||
// Same hash bytes with different algorithms should produce different CIDs
|
||||
let hash = [0u8; 32];
|
||||
let sha256_cid = to_cid(&hash, HashAlgorithm::Sha256);
|
||||
let sha3_256_cid = to_cid(&hash, HashAlgorithm::Sha3_256);
|
||||
assert_ne!(sha256_cid, sha3_256_cid);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,142 @@
|
||||
//! MidHash256 - O(1) file identification hash
|
||||
//!
|
||||
//! Computes a SHA-256 hash of the file size and middle portion,
|
||||
//! providing constant-time file identification regardless of file size.
|
||||
//!
|
||||
//! Algorithm:
|
||||
//! - Files ≤1MB: SHA-256(size_u64_be || entire_content)
|
||||
//! - Files >1MB: SHA-256(size_u64_be || middle_1MB)
|
||||
|
||||
use anyhow::Result;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::fs::File;
|
||||
use std::io::{Read, Seek, SeekFrom};
|
||||
use std::path::Path;
|
||||
|
||||
/// Size threshold: files at or below this size are hashed entirely
|
||||
const SIZE_THRESHOLD: u64 = 1024 * 1024; // 1MB
|
||||
|
||||
/// Sample size for files larger than threshold
|
||||
const SAMPLE_SIZE: usize = 1024 * 1024; // 1MB
|
||||
|
||||
/// Compute midhash256 for a file
|
||||
///
|
||||
/// Returns the raw 32-byte SHA-256 hash
|
||||
pub fn compute_midhash256(path: &Path) -> Result<[u8; 32]> {
|
||||
let mut file = File::open(path)?;
|
||||
let metadata = file.metadata()?;
|
||||
let file_size = metadata.len();
|
||||
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
// Always include file size as 8-byte big-endian prefix
|
||||
hasher.update(file_size.to_be_bytes());
|
||||
|
||||
if file_size <= SIZE_THRESHOLD {
|
||||
// Small file: hash entire content
|
||||
let mut buffer = Vec::with_capacity(file_size as usize);
|
||||
file.read_to_end(&mut buffer)?;
|
||||
hasher.update(&buffer);
|
||||
} else {
|
||||
// Large file: hash middle 1MB
|
||||
let middle_start = (file_size - SAMPLE_SIZE as u64) / 2;
|
||||
file.seek(SeekFrom::Start(middle_start))?;
|
||||
|
||||
let mut buffer = vec![0u8; SAMPLE_SIZE];
|
||||
file.read_exact(&mut buffer)?;
|
||||
hasher.update(&buffer);
|
||||
}
|
||||
|
||||
let hash = hasher.finalize();
|
||||
Ok(hash.into())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
fn test_small_file() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
file.write_all(b"hello world").unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hash = compute_midhash256(file.path()).unwrap();
|
||||
assert_eq!(hash.len(), 32);
|
||||
|
||||
// Same content should produce same hash
|
||||
let hash2 = compute_midhash256(file.path()).unwrap();
|
||||
assert_eq!(hash, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_different_sizes_different_hashes() {
|
||||
// Two files with same content but different sizes should have different hashes
|
||||
// because we include the size in the hash
|
||||
let mut file1 = NamedTempFile::new().unwrap();
|
||||
file1.write_all(b"hello").unwrap();
|
||||
file1.flush().unwrap();
|
||||
|
||||
let mut file2 = NamedTempFile::new().unwrap();
|
||||
file2.write_all(b"hello world").unwrap();
|
||||
file2.flush().unwrap();
|
||||
|
||||
let hash1 = compute_midhash256(file1.path()).unwrap();
|
||||
let hash2 = compute_midhash256(file2.path()).unwrap();
|
||||
assert_ne!(hash1, hash2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_file() {
|
||||
let file = NamedTempFile::new().unwrap();
|
||||
let hash = compute_midhash256(file.path()).unwrap();
|
||||
assert_eq!(hash.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_exactly_1mb_file() {
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
let data = vec![0xab; SIZE_THRESHOLD as usize];
|
||||
file.write_all(&data).unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hash = compute_midhash256(file.path()).unwrap();
|
||||
assert_eq!(hash.len(), 32);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_large_file_middle_sampling() {
|
||||
// Create a file larger than 1MB with known content
|
||||
let mut file = NamedTempFile::new().unwrap();
|
||||
|
||||
// Write 2MB: first 512KB of 'A', middle 1MB of 'B', last 512KB of 'C'
|
||||
let first_half = vec![b'A'; 512 * 1024];
|
||||
let middle = vec![b'B'; 1024 * 1024];
|
||||
let last_half = vec![b'C'; 512 * 1024];
|
||||
|
||||
file.write_all(&first_half).unwrap();
|
||||
file.write_all(&middle).unwrap();
|
||||
file.write_all(&last_half).unwrap();
|
||||
file.flush().unwrap();
|
||||
|
||||
let hash = compute_midhash256(file.path()).unwrap();
|
||||
assert_eq!(hash.len(), 32);
|
||||
|
||||
// Create another file with same middle but different edges
|
||||
let mut file2 = NamedTempFile::new().unwrap();
|
||||
let first_half2 = vec![b'X'; 512 * 1024];
|
||||
let last_half2 = vec![b'Y'; 512 * 1024];
|
||||
|
||||
file2.write_all(&first_half2).unwrap();
|
||||
file2.write_all(&middle).unwrap();
|
||||
file2.write_all(&last_half2).unwrap();
|
||||
file2.flush().unwrap();
|
||||
|
||||
let hash2 = compute_midhash256(file2.path()).unwrap();
|
||||
|
||||
// Both files have same size and same middle 1MB, so hashes should match
|
||||
assert_eq!(hash, hash2);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
mod algorithms;
|
||||
pub mod btih_v2;
|
||||
mod cid;
|
||||
pub mod midhash256;
|
||||
|
||||
pub use algorithms::hash_file;
|
||||
pub use btih_v2::{
|
||||
compute_btv2_hashes, compute_info_hash, compute_merkle_root, format_magnet_link,
|
||||
format_magnet_urn, BtV2Hashes, BT_BLOCK_SIZE,
|
||||
};
|
||||
pub use cid::to_cid;
|
||||
pub use midhash256::compute_midhash256;
|
||||
@@ -0,0 +1,9 @@
|
||||
//! MetaMesh Fast Full Hash Plugin
|
||||
//!
|
||||
//! High-performance file hashing with multiple algorithms in a single I/O pass.
|
||||
|
||||
pub mod api;
|
||||
pub mod cache;
|
||||
pub mod client;
|
||||
pub mod config;
|
||||
pub mod hasher;
|
||||
+65
@@ -0,0 +1,65 @@
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::{
|
||||
routing::{get, post},
|
||||
Router,
|
||||
};
|
||||
use tower_http::trace::TraceLayer;
|
||||
use tracing_subscriber::{layer::SubscriberExt, util::SubscriberInitExt};
|
||||
|
||||
use metamesh_plugin_fast_full_hash::api::{configure, health, manifest, process, ProcessState};
|
||||
use metamesh_plugin_fast_full_hash::cache::CacheManager;
|
||||
use metamesh_plugin_fast_full_hash::config::create_shared_config;
|
||||
|
||||
#[tokio::main]
|
||||
async fn main() -> anyhow::Result<()> {
|
||||
// Initialize logging
|
||||
tracing_subscriber::registry()
|
||||
.with(
|
||||
tracing_subscriber::EnvFilter::try_from_default_env()
|
||||
.unwrap_or_else(|_| "info,tower_http=debug".into()),
|
||||
)
|
||||
.with(tracing_subscriber::fmt::layer())
|
||||
.init();
|
||||
|
||||
// Get configuration from environment
|
||||
let port: u16 = std::env::var("PORT")
|
||||
.ok()
|
||||
.and_then(|p| p.parse().ok())
|
||||
.unwrap_or(8080);
|
||||
|
||||
let cache_path = std::env::var("CACHE_PATH").unwrap_or_else(|_| "/cache".to_string());
|
||||
|
||||
// Initialize shared state
|
||||
let config = create_shared_config();
|
||||
let cache = Arc::new(CacheManager::new(&cache_path));
|
||||
|
||||
// Load existing cache
|
||||
if let Err(e) = cache.load().await {
|
||||
tracing::warn!("Failed to load cache: {}", e);
|
||||
}
|
||||
|
||||
// Create shared app state
|
||||
let app_state = Arc::new(ProcessState {
|
||||
config,
|
||||
cache,
|
||||
});
|
||||
|
||||
// Build router
|
||||
let app = Router::new()
|
||||
.route("/health", get(health))
|
||||
.route("/manifest", get(manifest))
|
||||
.route("/configure", post(configure))
|
||||
.route("/process", post(process))
|
||||
.with_state(app_state)
|
||||
.layer(TraceLayer::new_for_http());
|
||||
|
||||
// Start server
|
||||
let addr = format!("0.0.0.0:{}", port);
|
||||
tracing::info!("Starting fast-full-hash plugin on {}", addr);
|
||||
|
||||
let listener = tokio::net::TcpListener::bind(&addr).await?;
|
||||
axum::serve(listener, app).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Vendored
+237
@@ -0,0 +1,237 @@
|
||||
#!/bin/bash
|
||||
# Compute ground truth hash values for test reference files
|
||||
# Uses standard Unix tools to verify our implementation
|
||||
|
||||
set -e
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
cd "$SCRIPT_DIR"
|
||||
|
||||
# Reference file: Sintel trailer (Creative Commons licensed)
|
||||
# Source: https://durian.blender.org/
|
||||
REFERENCE_URL="https://download.blender.org/durian/trailer/sintel_trailer-480p.mp4"
|
||||
REFERENCE_FILE="sintel_trailer-480p.mp4"
|
||||
EXPECTED_SIZE=23014356 # bytes
|
||||
|
||||
echo "=== Ground Truth Hash Computation ==="
|
||||
echo ""
|
||||
|
||||
# Download reference file if not present
|
||||
if [ ! -f "$REFERENCE_FILE" ]; then
|
||||
echo "Downloading reference file: $REFERENCE_FILE"
|
||||
curl -L -o "$REFERENCE_FILE" "$REFERENCE_URL"
|
||||
fi
|
||||
|
||||
# Verify file size
|
||||
ACTUAL_SIZE=$(stat -f%z "$REFERENCE_FILE" 2>/dev/null || stat -c%s "$REFERENCE_FILE" 2>/dev/null)
|
||||
if [ "$ACTUAL_SIZE" != "$EXPECTED_SIZE" ]; then
|
||||
echo "WARNING: File size mismatch! Expected $EXPECTED_SIZE, got $ACTUAL_SIZE"
|
||||
echo "The file may have changed. Please verify manually."
|
||||
fi
|
||||
|
||||
echo "Reference file: $REFERENCE_FILE"
|
||||
echo "File size: $ACTUAL_SIZE bytes"
|
||||
echo ""
|
||||
|
||||
echo "=== Computing Standard Hashes ==="
|
||||
echo ""
|
||||
|
||||
# SHA-256
|
||||
echo -n "SHA-256: "
|
||||
sha256sum "$REFERENCE_FILE" | cut -d' ' -f1
|
||||
|
||||
# SHA-1
|
||||
echo -n "SHA-1: "
|
||||
sha1sum "$REFERENCE_FILE" | cut -d' ' -f1
|
||||
|
||||
# MD5
|
||||
echo -n "MD5: "
|
||||
md5sum "$REFERENCE_FILE" | cut -d' ' -f1
|
||||
|
||||
# CRC32 (using Python as crc32 tool varies)
|
||||
echo -n "CRC32: "
|
||||
python3 -c "
|
||||
import binascii
|
||||
with open('$REFERENCE_FILE', 'rb') as f:
|
||||
crc = 0
|
||||
while chunk := f.read(65536):
|
||||
crc = binascii.crc32(chunk, crc)
|
||||
print(format(crc & 0xffffffff, '08x'))
|
||||
"
|
||||
|
||||
# SHA3-256 (using openssl if available, otherwise Python)
|
||||
echo -n "SHA3-256: "
|
||||
if command -v openssl &> /dev/null && openssl dgst -sha3-256 /dev/null &> /dev/null 2>&1; then
|
||||
openssl dgst -sha3-256 "$REFERENCE_FILE" | awk '{print $2}'
|
||||
else
|
||||
python3 -c "
|
||||
import hashlib
|
||||
with open('$REFERENCE_FILE', 'rb') as f:
|
||||
h = hashlib.sha3_256()
|
||||
while chunk := f.read(65536):
|
||||
h.update(chunk)
|
||||
print(h.hexdigest())
|
||||
"
|
||||
fi
|
||||
|
||||
# SHA3-384
|
||||
echo -n "SHA3-384: "
|
||||
if command -v openssl &> /dev/null && openssl dgst -sha3-384 /dev/null &> /dev/null 2>&1; then
|
||||
openssl dgst -sha3-384 "$REFERENCE_FILE" | awk '{print $2}'
|
||||
else
|
||||
python3 -c "
|
||||
import hashlib
|
||||
with open('$REFERENCE_FILE', 'rb') as f:
|
||||
h = hashlib.sha3_384()
|
||||
while chunk := f.read(65536):
|
||||
h.update(chunk)
|
||||
print(h.hexdigest())
|
||||
"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
echo "=== Computing MidHash256 ==="
|
||||
echo ""
|
||||
|
||||
# MidHash256: SHA-256(size_u64_be || middle_1MB)
|
||||
python3 << 'PYTHON_EOF'
|
||||
import hashlib
|
||||
import struct
|
||||
import os
|
||||
|
||||
filename = 'sintel_trailer-480p.mp4'
|
||||
size = os.path.getsize(filename)
|
||||
THRESHOLD = 1024 * 1024 # 1MB
|
||||
|
||||
with open(filename, 'rb') as f:
|
||||
hasher = hashlib.sha256()
|
||||
# Add size as 8-byte big-endian
|
||||
hasher.update(struct.pack('>Q', size))
|
||||
|
||||
if size <= THRESHOLD:
|
||||
# Small file: hash entire content
|
||||
hasher.update(f.read())
|
||||
else:
|
||||
# Large file: hash middle 1MB
|
||||
middle_start = (size - THRESHOLD) // 2
|
||||
f.seek(middle_start)
|
||||
hasher.update(f.read(THRESHOLD))
|
||||
|
||||
print(f"MidHash256: {hasher.hexdigest()}")
|
||||
PYTHON_EOF
|
||||
|
||||
echo ""
|
||||
echo "=== Computing BitTorrent v2 Hashes ==="
|
||||
echo ""
|
||||
|
||||
# BitTorrent v2 pieces root and info hash
|
||||
python3 << 'PYTHON_EOF'
|
||||
import hashlib
|
||||
import os
|
||||
|
||||
filename = 'sintel_trailer-480p.mp4'
|
||||
BLOCK_SIZE = 16 * 1024 # 16 KiB
|
||||
|
||||
def compute_leaf_hashes(filepath):
|
||||
"""Compute SHA-256 hash of each 16KB block"""
|
||||
leaves = []
|
||||
size = os.path.getsize(filepath)
|
||||
|
||||
if size == 0:
|
||||
return [bytes(32)]
|
||||
|
||||
with open(filepath, 'rb') as f:
|
||||
while True:
|
||||
block = f.read(BLOCK_SIZE)
|
||||
if not block:
|
||||
break
|
||||
# Pad last block with zeros
|
||||
if len(block) < BLOCK_SIZE:
|
||||
block = block + bytes(BLOCK_SIZE - len(block))
|
||||
leaves.append(hashlib.sha256(block).digest())
|
||||
|
||||
return leaves
|
||||
|
||||
def compute_merkle_root(leaves):
|
||||
"""Compute merkle root from leaf hashes"""
|
||||
if not leaves:
|
||||
return bytes(32)
|
||||
if len(leaves) == 1:
|
||||
return leaves[0]
|
||||
|
||||
# Pad to power of 2
|
||||
target = 1
|
||||
while target < len(leaves):
|
||||
target *= 2
|
||||
|
||||
current = list(leaves)
|
||||
while len(current) < target:
|
||||
current.append(bytes(32))
|
||||
|
||||
# Build tree bottom-up
|
||||
while len(current) > 1:
|
||||
next_level = []
|
||||
for i in range(0, len(current), 2):
|
||||
combined = current[i] + current[i+1]
|
||||
next_level.append(hashlib.sha256(combined).digest())
|
||||
current = next_level
|
||||
|
||||
return current[0]
|
||||
|
||||
def bencode_int(i):
|
||||
return f'i{i}e'.encode()
|
||||
|
||||
def bencode_str(s):
|
||||
if isinstance(s, str):
|
||||
s = s.encode()
|
||||
return f'{len(s)}:'.encode() + s
|
||||
|
||||
def bencode_dict(d):
|
||||
result = b'd'
|
||||
for k in sorted(d.keys()):
|
||||
result += bencode_str(k)
|
||||
v = d[k]
|
||||
if isinstance(v, int):
|
||||
result += bencode_int(v)
|
||||
elif isinstance(v, (str, bytes)):
|
||||
result += bencode_str(v)
|
||||
elif isinstance(v, dict):
|
||||
result += bencode_dict(v)
|
||||
result += b'e'
|
||||
return result
|
||||
|
||||
# Compute pieces root
|
||||
leaves = compute_leaf_hashes(filename)
|
||||
pieces_root = compute_merkle_root(leaves)
|
||||
print(f"BT Pieces Root: {pieces_root.hex()}")
|
||||
|
||||
# Compute info hash
|
||||
size = os.path.getsize(filename)
|
||||
basename = os.path.basename(filename)
|
||||
|
||||
# Build info dict as per BEP 52
|
||||
info_dict = {
|
||||
'file tree': {
|
||||
basename: {
|
||||
'': {
|
||||
'length': size,
|
||||
'pieces root': pieces_root
|
||||
}
|
||||
}
|
||||
},
|
||||
'meta version': 2,
|
||||
'name': basename,
|
||||
'piece length': BLOCK_SIZE
|
||||
}
|
||||
|
||||
encoded = bencode_dict(info_dict)
|
||||
info_hash = hashlib.sha256(encoded).digest()
|
||||
print(f"BT Info Hash: {info_hash.hex()}")
|
||||
|
||||
# Magnet link
|
||||
print(f"Magnet: magnet:?xt=urn:btmh:1220{info_hash.hex()}&dn={basename}")
|
||||
PYTHON_EOF
|
||||
|
||||
echo ""
|
||||
echo "=== Summary ==="
|
||||
echo "Copy these values to tests/ground_truth.rs"
|
||||
Vendored
BIN
Binary file not shown.
@@ -0,0 +1,330 @@
|
||||
//! Ground Truth Integration Tests
|
||||
//!
|
||||
//! These tests verify our hash implementations against known-good values
|
||||
//! computed using standard third-party tools (sha256sum, sha1sum, md5sum, Python hashlib).
|
||||
//!
|
||||
//! Reference file: Sintel trailer (480p) - Creative Commons licensed
|
||||
//! Source: https://download.blender.org/durian/trailer/sintel_trailer-480p.mp4
|
||||
//!
|
||||
//! To regenerate ground truth values, run:
|
||||
//! cd testdata && ./compute_ground_truth.sh
|
||||
|
||||
use metamesh_plugin_fast_full_hash::config::HashAlgorithm;
|
||||
use metamesh_plugin_fast_full_hash::hasher::hash_file;
|
||||
use std::collections::HashSet;
|
||||
use std::path::Path;
|
||||
|
||||
/// Reference file details
|
||||
const REFERENCE_FILE: &str = "testdata/sintel_trailer-480p.mp4";
|
||||
const REFERENCE_SIZE: u64 = 4372373;
|
||||
|
||||
/// Ground truth hash values (hex strings)
|
||||
/// Computed using standard Unix tools and Python hashlib
|
||||
mod expected {
|
||||
pub const SHA256: &str = "b670602fa00934ca27c4351bb0efe7ea7a07fae57284e44226025eeed7c51254";
|
||||
pub const SHA1: &str = "9b678890fb8ca401c28e7ca09171ec008a154b97";
|
||||
pub const MD5: &str = "df6ed4bbc93613c68c8525e21bbddf98";
|
||||
pub const CRC32: &str = "61accd5e";
|
||||
pub const SHA3_256: &str = "fefc0c74ebd79a9872464ebbb35a0545db97431a6c9a96d36c061244883e9213";
|
||||
pub const SHA3_384: &str = "da29f6604c96d09332531a4a2f4881b7bd2401a995053db868d5aea4056ada0b45c0ba69016e931bc468c38788afad94";
|
||||
pub const MIDHASH256: &str = "297a6662a49c60da0cf2a75407a0eb0045a36f838d2921ab602483b708157773";
|
||||
pub const BT_PIECES_ROOT: &str =
|
||||
"424febb4268900457464928a99547dacd07d2075cbbd7ae27fdf5c8d2d982817";
|
||||
pub const BT_INFO_HASH: &str =
|
||||
"e2f048c0c8bb56c910c05b9efb4f39d2ff6fd0cf01e96e1dacecb56a067f8418";
|
||||
}
|
||||
|
||||
/// Convert hex string to bytes
|
||||
fn hex_to_bytes(hex: &str) -> Vec<u8> {
|
||||
(0..hex.len())
|
||||
.step_by(2)
|
||||
.map(|i| u8::from_str_radix(&hex[i..i + 2], 16).unwrap())
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Extract raw hash from CID (parse the CID and get the hash bytes)
|
||||
fn extract_hash_from_cid(cid_str: &str) -> Vec<u8> {
|
||||
use cid::Cid;
|
||||
let cid = Cid::try_from(cid_str).expect("Invalid CID");
|
||||
cid.hash().digest().to_vec()
|
||||
}
|
||||
|
||||
/// Check if reference file exists, skip test if not
|
||||
fn check_reference_file() -> Option<&'static Path> {
|
||||
let path = Path::new(REFERENCE_FILE);
|
||||
if !path.exists() {
|
||||
eprintln!(
|
||||
"Skipping ground truth test: reference file not found at {}",
|
||||
REFERENCE_FILE
|
||||
);
|
||||
eprintln!("To download, run: cd testdata && ./compute_ground_truth.sh");
|
||||
return None;
|
||||
}
|
||||
|
||||
// Verify file size
|
||||
let metadata = std::fs::metadata(path).expect("Failed to read file metadata");
|
||||
if metadata.len() != REFERENCE_SIZE {
|
||||
eprintln!(
|
||||
"WARNING: Reference file size mismatch. Expected {}, got {}",
|
||||
REFERENCE_SIZE,
|
||||
metadata.len()
|
||||
);
|
||||
eprintln!("Ground truth values may not match. Re-run compute_ground_truth.sh");
|
||||
}
|
||||
|
||||
Some(path)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sha256_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha256].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_sha2-256").expect("SHA-256 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::SHA256);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"SHA-256 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::SHA256,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sha1_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha1].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_sha1").expect("SHA-1 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::SHA1);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"SHA-1 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::SHA1,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_md5_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Md5].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_md5").expect("MD5 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::MD5);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"MD5 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::MD5,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_crc32_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Crc32].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_crc32").expect("CRC32 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::CRC32);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"CRC32 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::CRC32,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sha3_256_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha3_256].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_sha3-256").expect("SHA3-256 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::SHA3_256);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"SHA3-256 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::SHA3_256,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sha3_384_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Sha3_384].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results.get("cid_sha3-384").expect("SHA3-384 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::SHA3_384);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"SHA3-384 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::SHA3_384,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_midhash256_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::Midhash256].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results
|
||||
.get("cid_midhash256")
|
||||
.expect("MidHash256 not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::MIDHASH256);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"MidHash256 mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::MIDHASH256,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bt_pieces_root_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::BtPiecesRoot].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results
|
||||
.get("cid_bt_pieces_root")
|
||||
.expect("BT Pieces Root not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::BT_PIECES_ROOT);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"BT Pieces Root mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::BT_PIECES_ROOT,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bt_info_hash_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
let enabled: HashSet<_> = vec![HashAlgorithm::BtInfoHash].into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
let cid = results
|
||||
.get("cid_bt_info_hash")
|
||||
.expect("BT Info Hash not computed");
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected::BT_INFO_HASH);
|
||||
|
||||
assert_eq!(
|
||||
hash_bytes, expected_bytes,
|
||||
"BT Info Hash mismatch!\nExpected: {}\nGot: {}",
|
||||
expected::BT_INFO_HASH,
|
||||
hex::encode(&hash_bytes)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_all_algorithms_ground_truth() {
|
||||
let Some(path) = check_reference_file() else {
|
||||
return;
|
||||
};
|
||||
|
||||
// Compute all algorithms in single pass
|
||||
let enabled: HashSet<_> = HashAlgorithm::all().into_iter().collect();
|
||||
let results = hash_file(path, &enabled, 1024 * 1024).expect("Hashing failed");
|
||||
|
||||
// Verify all 9 algorithms
|
||||
let test_cases = vec![
|
||||
("cid_sha2-256", expected::SHA256, "SHA-256"),
|
||||
("cid_sha1", expected::SHA1, "SHA-1"),
|
||||
("cid_md5", expected::MD5, "MD5"),
|
||||
("cid_crc32", expected::CRC32, "CRC32"),
|
||||
("cid_sha3-256", expected::SHA3_256, "SHA3-256"),
|
||||
("cid_sha3-384", expected::SHA3_384, "SHA3-384"),
|
||||
("cid_midhash256", expected::MIDHASH256, "MidHash256"),
|
||||
(
|
||||
"cid_bt_pieces_root",
|
||||
expected::BT_PIECES_ROOT,
|
||||
"BT Pieces Root",
|
||||
),
|
||||
("cid_bt_info_hash", expected::BT_INFO_HASH, "BT Info Hash"),
|
||||
];
|
||||
|
||||
let mut failures = Vec::new();
|
||||
|
||||
for (key, expected_hex, name) in test_cases {
|
||||
let cid = match results.get(key) {
|
||||
Some(c) => c,
|
||||
None => {
|
||||
failures.push(format!("{}: NOT COMPUTED", name));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
let hash_bytes = extract_hash_from_cid(cid);
|
||||
let expected_bytes = hex_to_bytes(expected_hex);
|
||||
|
||||
if hash_bytes != expected_bytes {
|
||||
failures.push(format!(
|
||||
"{}: MISMATCH\n Expected: {}\n Got: {}",
|
||||
name,
|
||||
expected_hex,
|
||||
hex::encode(&hash_bytes)
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
if !failures.is_empty() {
|
||||
panic!(
|
||||
"Ground truth verification failed:\n{}",
|
||||
failures.join("\n\n")
|
||||
);
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user