v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/api/server.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/gorilla/mux"
 	"github.com/metazla/meta-core/internal/config"
-	"github.com/metazla/meta-core/internal/events"
 	"github.com/metazla/meta-core/internal/identity"
 	"github.com/metazla/meta-core/internal/leader"
 	"github.com/metazla/meta-core/internal/meshdisco"
@@ -36,7 +35,6 @@ type Server struct {
 	watchersManager   *watchers.Manager
 	watchersPoller    *watchers.Poller
 	watchersHandlers  *watchers.Handlers
-	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
 	// challenges backs proof-of-possession on reveal/delete. In-process, so a
@@ -107,8 +105,7 @@ func NewServer(
 			// Create watchers poller
 			if s.fileWatcher != nil {
 				s.watchersPoller = watchers.NewPoller(watchersManager, s.fileWatcher, s.watcherDispatcher)
-				// Pass republish callback to handlers (handles nil metaPublisher case)
-				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller, s.RepublishMetadata)
+				s.watchersHandlers = watchers.NewHandlers(watchersManager, s.watchersPoller)
 
 				// Hydrate the in-memory state registry from stored tuples so
 				// the first scan tick after boot can skip ComputeMidHash256
@@ -349,41 +346,22 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
-	// Start meta publisher (if storage is connected). Live change events are
-	// written by the storage engine itself; the publisher owns the bulk
-	// republish below.
+	// Schema: served from counters the write path keeps exact; nothing runs.
+	// Change events need no publisher either — the storage engine appends them
+	// in the same transaction as each change, and a consumer that wants the
+	// whole state asks /api/events/meta for "0" and gets a snapshot (see
+	// streamSnapshot). The boot republish of every field is gone with it.
 	if s.storage != nil && s.storage.IsConnected() {
-		s.metaPublisher = events.NewMetaPublisher(s.storage)
-		if err := s.metaPublisher.Start(); err != nil {
-			log.Printf("[API] Warning: failed to start meta publisher: %v", err)
-		} else {
-			// Schema indexer: per-field schema (type/hint/breakdown) from a
-			// periodic scan of the store.
-			s.schemaIndexer = schema.NewIndexer(s.storage.Store())
-			if err := s.schemaIndexer.Start(); err != nil {
-				log.Printf("[API] Warning: failed to start schema indexer: %v", err)
-				s.schemaIndexer = nil
+		s.schemaIndexer = schema.NewIndexer(s.storage.Store())
+		go func() {
+			started := time.Now()
+			if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
+				log.Printf("[API] Warning: search row rebuild failed: %v", err)
+			} else if n > 0 {
+				log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
 			}
-
-			// Republish events for all existing metadata. Consumers that
-			// rebuild by replaying meta:events from the beginning (meta-fuse)
-			// depend on it.
-			go func() {
-				started := time.Now()
-				if n, err := s.storage.RebuildSearchIfEmpty(); err != nil {
-					log.Printf("[API] Warning: search row rebuild failed: %v", err)
-				} else if n > 0 {
-					log.Printf("[API] Rebuilt search rows for %d records in %s", n, time.Since(started).Round(time.Millisecond))
-				}
-
-				count, err := s.RepublishMetadata()
-				if err != nil {
-					log.Printf("[API] Warning: failed to republish metadata on startup: %v", err)
-				} else {
-					log.Printf("[API] Republished %d metadata events on startup", count)
-				}
-			}()
-		}
+		}()
+		go s.logWriteStats()
 	}
 
 	return nil
@@ -410,20 +388,6 @@ func (s *Server) Stop() error {
 		s.mountStatsPoller.Stop()
 	}
 
-	// Stop schema indexer before the publisher so the consumer drains cleanly
-	if s.schemaIndexer != nil {
-		if err := s.schemaIndexer.Stop(); err != nil {
-			log.Printf("[API] Warning: failed to stop schema indexer: %v", err)
-		}
-	}
-
-	// Stop meta publisher
-	if s.metaPublisher != nil {
-		if err := s.metaPublisher.Stop(); err != nil {
-			log.Printf("[API] Warning: failed to stop meta publisher: %v", err)
-		}
-	}
-
 	if s.server == nil {
 		return nil
 	}
@@ -436,26 +400,22 @@ func (s *Server) Stop() error {
 	return s.server.Shutdown(ctx)
 }
 
-// RepublishMetadata republishes all existing metadata to the meta:events stream
-// This is called on startup and after reset-all
-func (s *Server) RepublishMetadata() (int, error) {
-	if s.metaPublisher == nil {
-		return 0, fmt.Errorf("meta publisher not initialized")
-	}
-	if s.storage == nil || !s.storage.IsConnected() {
-		return 0, fmt.Errorf("storage not connected")
-	}
-
-	// Create the republish function that wraps storage methods
-	republishFunc := func() ([]string, func(string) (map[string]string, error), error) {
-		hashIDs, err := s.storage.GetAllHashIDs()
-		if err != nil {
-			return nil, nil, err
+// logWriteStats reports, every 15 minutes, how many writes changed a value
+// and how many found it already in place (kv skips those). The ratio is what
+// the no-op skip buys on a permanent store that re-persists repeated hits.
+func (s *Server) logWriteStats() {
+	ticker := time.NewTicker(15 * time.Minute)
+	defer ticker.Stop()
+	for range ticker.C {
+		store := s.storage.Store()
+		if store == nil {
+			return
+		}
+		changed, unchanged := store.WriteStats()
+		if total := changed + unchanged; total > 0 {
+			log.Printf("[Storage] writes since start: %d changed, %d unchanged skipped (%.0f%% no-op)", changed, unchanged, 100*float64(unchanged)/float64(total))
 		}
-		return hashIDs, s.storage.GetMetadataFlat, nil
 	}
-
-	return s.metaPublisher.RepublishAllMetadata(republishFunc)
 }
 
 // statusRecorder captures the response status so loggingMiddleware can stay