file event improvement using midhash

internal/watcher/dispatcher.go

@@ -60,8 +60,8 @@ func (d *Dispatcher) dispatchToStream(event FileEvent) {
 	if event.Size > 0 {
 		fields["size"] = fmt.Sprintf("%d", event.Size)
 	}
-	if event.PartialHash != "" {
-		fields["partialHash"] = event.PartialHash
+	if event.MidHash256 != "" {
+		fields["midhash256"] = event.MidHash256
 	}
 	if event.OldPath != "" {
 		fields["oldPath"] = event.OldPath

internal/watcher/midhash.go

@@ -0,0 +1,107 @@
+package watcher
+
+import (
+	"crypto/sha256"
+	"encoding/base32"
+	"encoding/binary"
+	"io"
+	"os"
+	"strings"
+)
+
+const (
+	// MidHashSampleSize is the size of the sample to hash (1MB)
+	MidHashSampleSize = 1024 * 1024 // 1MB
+
+	// MidHash256Code is the custom multicodec for midhash256
+	MidHash256Code = 0x1000
+)
+
+// ComputeMidHash256 computes a midhash256 for a file.
+//
+// Algorithm:
+// - For files <= 1MB: Hashes entire file content + size prefix
+// - For files > 1MB: Hashes middle 1MB + size prefix
+//
+// The size prefix (8-byte big-endian) ensures that files with identical
+// middle content but different sizes produce different hashes.
+//
+// Returns: CID v1 string in base32lower format (prefixed with "b")
+func ComputeMidHash256(filePath string) (string, error) {
+	// Get file info
+	fileInfo, err := os.Stat(filePath)
+	if err != nil {
+		return "", err
+	}
+	fileSize := fileInfo.Size()
+
+	// Open file
+	file, err := os.Open(filePath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	// Create size buffer (8-byte big-endian)
+	sizeBuffer := make([]byte, 8)
+	binary.BigEndian.PutUint64(sizeBuffer, uint64(fileSize))
+
+	// Read sample data
+	var sampleData []byte
+	if fileSize <= MidHashSampleSize {
+		// Small file: read entire content
+		sampleData = make([]byte, fileSize)
+		_, err = io.ReadFull(file, sampleData)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			return "", err
+		}
+	} else {
+		// Large file: read middle 1MB
+		middleOffset := (fileSize - MidHashSampleSize) / 2
+		_, err = file.Seek(middleOffset, io.SeekStart)
+		if err != nil {
+			return "", err
+		}
+		sampleData = make([]byte, MidHashSampleSize)
+		_, err = io.ReadFull(file, sampleData)
+		if err != nil && err != io.EOF && err != io.ErrUnexpectedEOF {
+			return "", err
+		}
+	}
+
+	// Compute SHA-256 hash of [size + sample]
+	hasher := sha256.New()
+	hasher.Write(sizeBuffer)
+	hasher.Write(sampleData)
+	hashBytes := hasher.Sum(nil)
+
+	// Build CID v1 bytes:
+	// [version(0x01)] + [varint(codec 0x1000)] + [multihash]
+	// multihash = [varint(hashCode 0x1000)] + [varint(length 32)] + [hash bytes]
+	cidBytes := make([]byte, 0, 1+2+2+1+32)
+
+	// CID version 1
+	cidBytes = append(cidBytes, 0x01)
+
+	// Content codec as varint (0x1000 = 4096)
+	// varint encoding: 0x1000 = 0b0001_0000_0000_0000
+	// Split into 7-bit chunks: 0b00_0000_0000 (high) and 0b010_0000 (low with continuation)
+	// Result: [0x80, 0x20] for 0x1000
+	cidBytes = append(cidBytes, 0x80, 0x20)
+
+	// Multihash: hash function code as varint (0x1000)
+	cidBytes = append(cidBytes, 0x80, 0x20)
+
+	// Multihash: length as varint (32 = 0x20)
+	cidBytes = append(cidBytes, 0x20)
+
+	// Multihash: hash bytes
+	cidBytes = append(cidBytes, hashBytes...)
+
+	// Encode as base32lower (RFC 4648 without padding)
+	encoded := base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(cidBytes)
+	encoded = strings.ToLower(encoded)
+
+	// Add multibase prefix "b" for base32lower
+	return "b" + encoded, nil
+}

internal/watcher/types.go

@@ -14,12 +14,12 @@ const (
 
 // FileEvent represents a file system event
 type FileEvent struct {
-	Type        FileEventType `json:"type"`
-	Path        string        `json:"path"`        // Relative to FILES_PATH
-	Size        int64         `json:"size,omitempty"`
-	Timestamp   int64         `json:"timestamp"`
-	PartialHash string        `json:"partialHash,omitempty"` // Hash of first 64KB
-	OldPath     string        `json:"oldPath,omitempty"`     // For rename events
+	Type       FileEventType `json:"type"`
+	Path       string        `json:"path"`                 // Relative to FILES_PATH
+	Size       int64         `json:"size,omitempty"`
+	Timestamp  int64         `json:"timestamp"`
+	MidHash256 string        `json:"midhash256,omitempty"` // midhash256 CID (SHA-256 of middle 1MB + file size)
+	OldPath    string        `json:"oldPath,omitempty"`    // For rename events
 }
 
 // PendingEvent tracks a file event that's being debounced

internal/watcher/watcher.go

@@ -1,9 +1,6 @@
 package watcher
 
 import (
-	"crypto/sha256"
-	"encoding/hex"
-	"io"
 	"log"
 	"os"
 	"path/filepath"
@@ -13,10 +10,6 @@ import (
 	"github.com/metazla/meta-core/internal/config"
 )
 
-const (
-	// PartialHashSize is the number of bytes to hash for file identification
-	PartialHashSize = 64 * 1024 // 64KB
-)
 
 // Watcher provides file scanning and event dispatching
 type Watcher struct {
@@ -46,11 +39,11 @@ func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
 
 // handleDebouncedEvent processes a debounced event
 func (w *Watcher) handleDebouncedEvent(event FileEvent) {
-	// Compute partial hash for add/change events
+	// Compute midhash256 for add/change events
 	if event.Type == EventTypeAdd || event.Type == EventTypeChange {
 		fullPath := filepath.Join(w.filesPath, event.Path)
-		if hash, err := computePartialHash(fullPath); err == nil {
-			event.PartialHash = hash
+		if hash, err := ComputeMidHash256(fullPath); err == nil {
+			event.MidHash256 = hash
 		}
 	}
 
@@ -102,9 +95,9 @@ func (w *Watcher) ScanDirectory(root string) int {
 			Timestamp: NowMS(),
 		}
 
-		// Compute partial hash
-		if hash, err := computePartialHash(path); err == nil {
-			event.PartialHash = hash
+		// Compute midhash256
+		if hash, err := ComputeMidHash256(path); err == nil {
+			event.MidHash256 = hash
 		}
 
 		// Dispatch directly
@@ -178,21 +171,3 @@ func (w *Watcher) GetFilesPath() string {
 	return w.filesPath
 }
 
-// computePartialHash computes SHA-256 hash of first 64KB of a file
-func computePartialHash(path string) (string, error) {
-	file, err := os.Open(path)
-	if err != nil {
-		return "", err
-	}
-	defer file.Close()
-
-	hasher := sha256.New()
-	buffer := make([]byte, PartialHashSize)
-	n, err := file.Read(buffer)
-	if err != nil && err != io.EOF {
-		return "", err
-	}
-
-	hasher.Write(buffer[:n])
-	return hex.EncodeToString(hasher.Sum(nil)), nil
-}