replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

internal/api/server.go

@@ -39,7 +39,6 @@ type Server struct {
 	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
-	search            *searchIndex
 	// challenges backs proof-of-possession on reveal/delete. In-process, so a
 	// restart invalidates the ones in flight — see identity/challenge.go.
 	challenges *identity.ChallengeStore
@@ -51,7 +50,7 @@ type Server struct {
 	router *mux.Router
 	server *http.Server
 	// stats caches the keyspace-walk half of GET /api/stats so a polling
-	// dashboard never puts a multi-second Redis walk on the request path.
+	// dashboard never puts a multi-second store walk on the request path.
 	stats *statsCache
 }
 
@@ -71,7 +70,6 @@ func NewServer(
 		config:         cfg,
 		leaderProvider: leaderProvider,
 		storage:        stor,
-		search:         &searchIndex{},
 		router:         mux.NewRouter(),
 		stats:          &statsCache{},
 	}
@@ -80,7 +78,7 @@ func NewServer(
 	var scanFunc mounts.ScanFunc
 	if cfg.EnableFileWatcher {
 		s.watcherDispatcher = watcher.NewDispatcher()
-		// Wire storage client for Redis Stream publishing
+		// Wire storage client for file:events publishing
 		if stor != nil {
 			s.watcherDispatcher.SetStorageClient(stor)
 		}
@@ -90,7 +88,7 @@ func NewServer(
 		} else {
 			s.fileWatcher = fileWatcher
 			// Wire storage so the watcher can persist (path,size,mtime,cid)
-			// tuples to Redis as it computes midhashes.
+			// tuples to the store as it computes midhashes.
 			if stor != nil {
 				fileWatcher.SetStorageClient(stor)
 			}
@@ -112,7 +110,7 @@ func NewServer(
 				// Pass republish callback to handlers (handles nil metaPublisher case)
 				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
 
-				// Hydrate the in-memory state registry from Redis tuples so
+				// Hydrate the in-memory state registry from stored tuples so
 				// the first scan tick after boot can skip ComputeMidHash256
 				// for files whose (size, mtime) match.
 				configs := watchersManager.List()
@@ -123,7 +121,7 @@ func NewServer(
 				if loaded, skipped, err := s.fileWatcher.HydrateStateFromStorage(roots); err != nil {
 					log.Printf("[API] Warning: failed to hydrate watcher state: %v", err)
 				} else {
-					log.Printf("[API] Hydrated watcher state from Redis: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
+					log.Printf("[API] Hydrated watcher state from the store: %d files loaded, %d skipped (no matching watcher root)", loaded, skipped)
 				}
 			}
 		}
@@ -272,20 +270,8 @@ func (s *Server) setupRoutes() {
 	// without going through Authelia.
 	s.router.HandleFunc("/api/meta/{cid}", s.handleGetMetaByCID).Methods("GET")
 
-	// Admin: reunify stranded midhash-rooted entries into their UUID roots
-	// (the dual-root pattern caused by historical writes that bypassed CID
-	// resolution).
-	s.router.HandleFunc("/api/admin/migrate-dual-roots", s.handleMigrateDualRoots).Methods("POST")
-	// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
-	// (METADATA_KEYS.md §14.17). Idempotent.
-	s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
-	s.router.HandleFunc("/api/admin/migrate-user-cids", s.handleMigrateUserCids).Methods("POST")
-	// One-shot sweep of query echoes on literature cards (meta-read
-	// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
-	s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
-
-	// SSE event streams. Mediated mirror of the Redis Streams; consumers
-	// reach them over HTTP instead of touching Redis directly. Auth-bypass
+	// SSE event streams. The only way consumers see meta-core's change
+	// streams — they never touch the database. Auth-bypass
 	// applies but exposure should be inside-only (do NOT proxy through
 	// Caddy publicly). See docs/api-mediated-access.md "Auth".
 	s.router.HandleFunc("/api/events/files", s.handleEventsFiles).Methods("GET")
@@ -363,44 +349,31 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
-	// Build + periodically refresh the in-memory search index so
-	// POST /api/metadata/search matches in memory instead of a Redis
-	// round-trip per record (see search_index.go).
-	if s.storage != nil {
-		go s.search.runRefreshLoop(s.storage)
-	}
-
-	// Start meta publisher (if storage is connected)
-	// Publishes keyspace notifications to meta:events stream
+	// Start meta publisher (if storage is connected). Live change events are
+	// written by the storage engine itself; the publisher owns the bulk
+	// republish below.
 	if s.storage != nil && s.storage.IsConnected() {
-		s.metaPublisher = events.NewMetaPublisher(s.storage.GetRedisClient())
+		s.metaPublisher = events.NewMetaPublisher(s.storage)
 		if err := s.metaPublisher.Start(); err != nil {
 			log.Printf("[API] Warning: failed to start meta publisher: %v", err)
 		} else {
-			// Schema indexer consumes the same stream to derive a live
-			// per-field schema (type/hint/breakdown). Start it before the
-			// republish flood so it captures the bootstrap events.
-			s.schemaIndexer = schema.NewIndexer(s.storage.GetRedisClient(), s.storage.GetPrefix())
+			// Schema indexer: per-field schema (type/hint/breakdown) from a
+			// periodic scan of the store.
+			s.schemaIndexer = schema.NewIndexer(s.storage.Store())
 			if err := s.schemaIndexer.Start(); err != nil {
 				log.Printf("[API] Warning: failed to start schema indexer: %v", err)
 				s.schemaIndexer = nil
 			}
 
-			// Republish events for all existing metadata
+			// Republish events for all existing metadata. Consumers that
+			// rebuild by replaying meta:events from the beginning (meta-fuse)
+			// depend on it.
 			go func() {
-				// Build any missing per-record field indexes first, in ONE
-				// keyspace pass. The republish below reads every record, and
-				// an un-indexed record read costs a full-keyspace SCAN — on a
-				// real store that walk warmed at ~1.3 records/s (a ~2.6h tail,
-				// paid again on every restart). Warming first turns the whole
-				// republish into index-served reads. Idempotent, so a
-				// fully-indexed store just pays the single scan.
-				// See storage/field_index.go.
 				started := time.Now()
-				if n, err := s.storage.WarmFieldIndexes(); err != nil {
-					log.Printf("[API] Warning: field index warm failed: %v (reads fall back to SCAN)", err)
+				if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
+					log.Printf("[API] Warning: search row rebuild failed: %v", err)
 				} else if n > 0 {
-					log.Printf("[API] Warmed %d field indexes in %s", n, time.Since(started).Round(time.Millisecond))
+					log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
 				}
 
 				count, err := s.RepublishMetadata()