improved kv metacore

internal/watcher/dispatcher.go

@@ -3,6 +3,8 @@ package watcher
 import (
 	"fmt"
 	"log"
+	"strconv"
+	"strings"
 	"sync"
 
 	"github.com/metazla/meta-core/internal/storage"
@@ -108,6 +110,85 @@ func (d *Dispatcher) EmitReset(watcherID string) error {
 	return nil
 }
 
+// GetRecentEvents reads recent events from the Redis stream, newest first.
+// If sinceMS > 0, only events strictly newer than that timestamp are returned.
+// limit caps the number of returned events (defaults to 100 when <= 0).
+func (d *Dispatcher) GetRecentEvents(sinceMS int64, limit int) ([]FileEvent, error) {
+	d.mu.RLock()
+	client := d.storageClient
+	d.mu.RUnlock()
+
+	if client == nil || !client.IsConnected() {
+		return []FileEvent{}, nil
+	}
+
+	if limit <= 0 {
+		limit = 100
+	}
+
+	start := "+"
+	stop := "-"
+	if sinceMS > 0 {
+		// Exclusive lower bound: skip entries at or before sinceMS.
+		stop = fmt.Sprintf("(%d-0", sinceMS)
+	}
+
+	entries, err := client.XRevRange(EventsStream, start, stop, int64(limit))
+	if err != nil {
+		return nil, err
+	}
+
+	events := make([]FileEvent, 0, len(entries))
+	for _, entry := range entries {
+		events = append(events, parseStreamEntry(entry))
+	}
+	return events, nil
+}
+
+// parseStreamEntry converts a Redis stream entry back into a FileEvent.
+// All fields are stored as strings by XADD, so numerics need parsing.
+func parseStreamEntry(entry storage.StreamEntry) FileEvent {
+	event := FileEvent{}
+
+	if v, ok := entry.Values["type"].(string); ok {
+		event.Type = FileEventType(v)
+	}
+	if v, ok := entry.Values["path"].(string); ok {
+		event.Path = v
+	}
+	if v, ok := entry.Values["timestamp"].(string); ok {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+			event.Timestamp = n
+		}
+	}
+	if v, ok := entry.Values["size"].(string); ok {
+		if n, err := strconv.ParseInt(v, 10, 64); err == nil {
+			event.Size = n
+		}
+	}
+	if v, ok := entry.Values["midhash256"].(string); ok {
+		event.MidHash256 = v
+	}
+	if v, ok := entry.Values["oldPath"].(string); ok {
+		event.OldPath = v
+	}
+	if v, ok := entry.Values["watcherId"].(string); ok {
+		event.WatcherID = v
+	}
+
+	// Fall back to the stream ID's millisecond prefix if the timestamp
+	// field is missing or unparseable — every Redis stream ID starts with ms.
+	if event.Timestamp == 0 && entry.ID != "" {
+		if dash := strings.IndexByte(entry.ID, '-'); dash > 0 {
+			if n, err := strconv.ParseInt(entry.ID[:dash], 10, 64); err == nil {
+				event.Timestamp = n
+			}
+		}
+	}
+
+	return event
+}
+
 // GetStreamLength returns the current stream length
 func (d *Dispatcher) GetStreamLength() (int64, error) {
 	d.mu.RLock()

internal/watcher/handlers.go

@@ -2,6 +2,7 @@ package watcher
 
 import (
 	"encoding/json"
+	"log"
 	"net/http"
 	"strconv"
 
@@ -52,7 +53,14 @@ func (h *Handlers) handlePoll(w http.ResponseWriter, r *http.Request) {
 		}
 	}
 
-	events := h.watcher.GetRecentEvents(sinceMS, limit)
+	// Read events from the Redis `file:events` stream rather than the
+	// in-memory buffer. The buffer is fed by the unused debouncer path,
+	// so it is always empty in practice — the stream is the source of truth.
+	events, err := h.dispatcher.GetRecentEvents(sinceMS, limit)
+	if err != nil {
+		log.Printf("[watcher.handlePoll] failed to read stream: %v", err)
+		events = []FileEvent{}
+	}
 
 	writeJSON(w, http.StatusOK, EventsListResponse{
 		Events: events,

internal/watcher/midhash.go

@@ -26,15 +26,13 @@ const (
 // The size prefix (8-byte big-endian) ensures that files with identical
 // middle content but different sizes produce different hashes.
 //
+// fileSize must be passed in by the caller (typically from a FileInfo it
+// already has from filepath.Walk or a previous os.Stat). This function
+// deliberately does NOT call os.Stat itself — re-statting every file during
+// a scan doubles the syscall count, which matters on slow / network drives.
+//
 // Returns: CID v1 string in base32lower format (prefixed with "b")
-func ComputeMidHash256(filePath string) (string, error) {
-	// Get file info
-	fileInfo, err := os.Stat(filePath)
-	if err != nil {
-		return "", err
-	}
-	fileSize := fileInfo.Size()
-
+func ComputeMidHash256(filePath string, fileSize int64) (string, error) {
 	// Open file
 	file, err := os.Open(filePath)
 	if err != nil {

internal/watcher/watcher.go

@@ -4,10 +4,12 @@ import (
 	"log"
 	"os"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 
 	"github.com/metazla/meta-core/internal/config"
+	"github.com/metazla/meta-core/internal/storage"
 )
 
 
@@ -18,11 +20,39 @@ type Watcher struct {
 	dispatcher    *Dispatcher
 	filesPath     string
 	stateRegistry *StateRegistry
+	storage       *storage.Client
 
 	mu          sync.RWMutex
 	eventBuffer []FileEvent
 }
 
+// SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
+// tuple to Redis whenever it computes a midhash. Without this, only the
+// in-memory state registry knows about the file — Redis would only learn
+// about it via downstream consumers of the event stream.
+func (w *Watcher) SetStorageClient(s *storage.Client) {
+	w.storage = s
+}
+
+// writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
+// Caller already has all four values from its existing os.Stat / FileInfo —
+// no extra IO is performed here beyond the Redis write itself.
+//
+// This is the inverse-index that lets a fresh meta-core boot skip
+// re-mid-hashing files whose (path, size, mtime) match what's in Redis.
+func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
+	if w.storage == nil || midhash == "" {
+		return
+	}
+	if _, err := w.storage.MergeMetadataFlat(midhash, map[string]string{
+		"filePath":  absPath,
+		"sizeByte":  strconv.FormatInt(size, 10),
+		"mtimeNano": strconv.FormatInt(mtimeNano, 10),
+	}); err != nil {
+		log.Printf("[Watcher] failed to write tuple for %s: %v", midhash, err)
+	}
+}
+
 // NewWatcher creates a new file watcher/scanner
 func NewWatcher(cfg *config.Config, dispatcher *Dispatcher) (*Watcher, error) {
 	debouncer := NewDebouncer(0) // No debouncing for polling-based scanning
@@ -44,7 +74,7 @@ func (w *Watcher) handleDebouncedEvent(event FileEvent) {
 	// Compute midhash256 for add/change events
 	if event.Type == EventTypeAdd || event.Type == EventTypeChange {
 		fullPath := filepath.Join(w.filesPath, event.Path)
-		if hash, err := ComputeMidHash256(fullPath); err == nil {
+		if hash, err := ComputeMidHash256(fullPath, event.Size); err == nil {
 			event.MidHash256 = hash
 		}
 	}
@@ -90,16 +120,20 @@ func (w *Watcher) ScanDirectory(root string) int {
 		relPath = filepath.ToSlash(relPath)
 
 		// Emit add event
+		curSize := info.Size()
+		curMtime := info.ModTime().UnixNano()
 		event := FileEvent{
 			Type:      EventTypeAdd,
 			Path:      relPath,
-			Size:      info.Size(),
+			Size:      curSize,
 			Timestamp: NowMS(),
 		}
 
-		// Compute midhash256
-		if hash, err := ComputeMidHash256(path); err == nil {
+		// Compute midhash256 — uses the size we already have from Walk's
+		// FileInfo, so no extra os.Stat is needed.
+		if hash, err := ComputeMidHash256(path, curSize); err == nil {
 			event.MidHash256 = hash
+			w.writeFileTuple(path, curSize, curMtime, hash)
 		}
 
 		// Dispatch directly
@@ -160,9 +194,11 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
 				Timestamp: NowMS(),
 			}
 
-			// Compute midhash256
-			if hash, err := ComputeMidHash256(path); err == nil {
+			// Compute midhash256 — size is already known from the walk's
+			// FileInfo, no extra stat needed.
+			if hash, err := ComputeMidHash256(path, currentSize); err == nil {
 				event.MidHash256 = hash
+				w.writeFileTuple(path, currentSize, currentMtime, hash)
 			}
 
 			// Dispatch event
@@ -185,9 +221,10 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
 				Timestamp: NowMS(),
 			}
 
-			// Compute midhash256
-			if hash, err := ComputeMidHash256(path); err == nil {
+			// Compute midhash256 — reuse the size we already have.
+			if hash, err := ComputeMidHash256(path, currentSize); err == nil {
 				event.MidHash256 = hash
+				w.writeFileTuple(path, currentSize, currentMtime, hash)
 			}
 
 			// Dispatch event
@@ -227,6 +264,75 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
 	return result
 }
 
+// WatcherRoot pairs a watcher's id with the absolute path it watches. The
+// hydrator uses Root to decide which Redis-tracked file belongs to which
+// in-memory WatcherState.
+type WatcherRoot struct {
+	ID   string
+	Root string
+}
+
+// HydrateStateFromStorage repopulates the per-watcher in-memory state from
+// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
+// next scan tick will see existing files as "known" and skip
+// ComputeMidHash256 for any whose size+mtime match — turning a cold-start
+// rescan from minutes to seconds on a populated library.
+//
+// Files whose filePath doesn't fall under any watcher's root are silently
+// skipped; files missing any tuple field are skipped too (a partial record
+// can't seed valid state).
+//
+// Returns counts so the caller can log / surface them.
+func (w *Watcher) HydrateStateFromStorage(roots []WatcherRoot) (loaded, skipped int, err error) {
+	if w.storage == nil || !w.storage.IsConnected() {
+		return 0, 0, nil
+	}
+
+	tuples, err := w.storage.GetTuplesForAllFiles()
+	if err != nil {
+		return 0, 0, err
+	}
+
+	// Sort roots by length descending so longest-prefix wins when paths
+	// nest (e.g. /files/watch/foo and /files/watch).
+	sortedRoots := make([]WatcherRoot, len(roots))
+	copy(sortedRoots, roots)
+	for i := 1; i < len(sortedRoots); i++ {
+		for j := i; j > 0 && len(sortedRoots[j].Root) > len(sortedRoots[j-1].Root); j-- {
+			sortedRoots[j], sortedRoots[j-1] = sortedRoots[j-1], sortedRoots[j]
+		}
+	}
+
+	// State keys are relative to w.filesPath (matching ScanDirectoryDiff's
+	// Rel(w.filesPath, path) convention). The watcher root only decides
+	// *which* per-watcher state map this file lives in.
+	for _, t := range tuples {
+		matched := false
+		for _, r := range sortedRoots {
+			if t.FilePath == r.Root || strings.HasPrefix(t.FilePath, r.Root+"/") {
+				rel, err := filepath.Rel(w.filesPath, t.FilePath)
+				if err != nil {
+					rel = t.FilePath
+				}
+				rel = filepath.ToSlash(rel)
+				ws := w.stateRegistry.GetOrCreate(r.ID)
+				ws.Set(rel, &FileState{
+					Size:      t.Size,
+					MtimeNano: t.MtimeNano,
+					MidHash:   t.HashID,
+				})
+				loaded++
+				matched = true
+				break
+			}
+		}
+		if !matched {
+			skipped++
+		}
+	}
+	return loaded, skipped, nil
+}
+
 // ClearState clears the tracked state for a watcher
 func (w *Watcher) ClearState(watcherID string) {
 	w.stateRegistry.Clear(watcherID)