replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s

This commit is contained in:
Worph
2026-10-19 20:51:00 +00:00
parent c53146e514
commit dc5cbdfb9b
80 changed files with 4601 additions and 6077 deletions
+6 -8
View File
@@ -11,7 +11,6 @@ import (
"github.com/metazla/meta-core/internal/storage"
)
// Watcher provides file scanning and event dispatching
type Watcher struct {
config *config.Config
@@ -26,8 +25,8 @@ type Watcher struct {
}
// SetStorageClient lets the watcher persist the (path, size, mtime, midhash)
// tuple to Redis whenever it computes a midhash. Without this, only the
// in-memory state registry knows about the file — Redis would only learn
// tuple to the store whenever it computes a midhash. Without this, only the
// in-memory state registry knows about the file — the store would only learn
// about it via downstream consumers of the event stream.
func (w *Watcher) SetStorageClient(s *storage.Client) {
w.storage = s
@@ -35,7 +34,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
// writeFileTuple persists path/size/mtime/midhash for a freshly-hashed file.
// Caller already has all four values from its existing os.Stat / FileInfo —
// no extra IO is performed here beyond the Redis writes themselves.
// no extra IO is performed here beyond the store writes themselves.
//
// Flow:
// - Look up the midhash CID in the reverse index (cid:<midhash> → uuid)
@@ -46,7 +45,7 @@ func (w *Watcher) SetStorageClient(s *storage.Client) {
// record the new path in the duplicates set and refresh size/mtime
//
// This is also the inverse-index that lets a fresh meta-core boot skip
// re-mid-hashing files whose (path, size, mtime) match what's in Redis,
// re-mid-hashing files whose (path, size, mtime) match what's stored,
// via HydrateStateFromStorage below.
func (w *Watcher) writeFileTuple(absPath string, size, mtimeNano int64, midhash string) {
if w.storage == nil || midhash == "" {
@@ -270,7 +269,7 @@ func (w *Watcher) ScanDirectoryDiff(root string, watcherID string) DiffScanResul
}
// WatcherRoot pairs a watcher's id with the absolute path it watches. The
// hydrator uses Root to decide which Redis-tracked file belongs to which
// hydrator uses Root to decide which stored file belongs to which
// in-memory WatcherState.
type WatcherRoot struct {
ID string
@@ -278,7 +277,7 @@ type WatcherRoot struct {
}
// HydrateStateFromStorage repopulates the per-watcher in-memory state from
// the (filePath, sizeByte, mtimeNano) tuples in Redis. After this runs, the
// the (filePath, sizeByte, mtimeNano) tuples in the store. After this runs, the
// next scan tick will see existing files as "known" and skip
// ComputeMidHash256 for any whose size+mtime match — turning a cold-start
// rescan from minutes to seconds on a populated library.
@@ -416,4 +415,3 @@ func (w *Watcher) GetRecentEvents(sinceMS int64, limit int) []FileEvent {
func (w *Watcher) GetFilesPath() string {
return w.filesPath
}