search optimisation

internal/api/metadata.go

@@ -395,7 +395,18 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Get all hash IDs
+	// Free-text query: served from the in-memory search index (search_index.go)
+	// so we match in memory instead of doing a SCAN+MGET Redis round-trip per
+	// record (that was 40s+ over ~72k records). Full metadata is fetched only
+	// for the <=limit matches.
+	if req.Query != "" && req.Property == "" {
+		s.serveIndexedSearch(w, req)
+		return
+	}
+
+	// Property filter (rare) or empty query (returns the first `limit`): the
+	// legacy scan. Empty query only fetches `limit` records; property search is
+	// low-volume and not on the hot search path.
 	allHashIDs, err := s.storage.GetAllHashIDs()
 	if err != nil {
 		writeError(w, http.StatusInternalServerError, err.Error())
@@ -405,9 +416,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 	var results []MetadataSearchResult
 	total := len(allHashIDs)
 
-	// Search fields for query
-	searchFields := []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
-
 	for _, hashID := range allHashIDs {
 		if len(results) >= req.Limit {
 			break
@@ -428,28 +436,6 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) {
 			}
 		}
 
-		// General query search
-		if req.Query != "" && req.Property == "" {
-			queryLower := strings.ToLower(req.Query)
-
-			// Check if query matches hashId
-			if strings.Contains(strings.ToLower(hashID), queryLower) {
-				matches = true
-			}
-
-			// Check search fields
-			if !matches {
-				for _, field := range searchFields {
-					if value, ok := metadata[field]; ok {
-						if strings.Contains(strings.ToLower(value), queryLower) {
-							matches = true
-							break
-						}
-					}
-				}
-			}
-		}
-
 		// If no query or property filter, return all
 		if req.Query == "" && req.Property == "" {
 			matches = true

internal/api/search_index.go

@@ -0,0 +1,110 @@
+package api
+
+import (
+	"log"
+	"net/http"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"github.com/metazla/meta-core/internal/storage"
+)
+
+// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
+// from Redis in the background. Records ingest continuously, so a newly stored
+// record becomes searchable within one interval; the rebuild is a single
+// batched bulk read (storage.GetSearchIndexTuples), off the request path.
+const searchIndexRefreshInterval = 60 * time.Second
+
+// searchIndex is an in-memory snapshot of every record's search haystack
+// (hashID + lowercased searchable fields), rebuilt periodically from Redis. It
+// exists so POST /api/metadata/search matches in memory (string ops over a
+// slice) instead of doing a SCAN+MGET Redis round-trip per record — the latter
+// took 40s+ over ~72k records and made a live parallel search path unusable.
+type searchIndex struct {
+	mu     sync.RWMutex
+	tuples []storage.SearchTuple
+	ready  atomic.Bool
+}
+
+// snapshot returns the current tuples slice and whether the index has built at
+// least once. The refresh swaps in a fresh slice under the write lock (never
+// mutates in place), so the returned slice is safe to read lock-free.
+func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
+	idx.mu.RLock()
+	defer idx.mu.RUnlock()
+	return idx.tuples, idx.ready.Load()
+}
+
+// refresh rebuilds the index from Redis once (a batched bulk read).
+func (idx *searchIndex) refresh(stor *storage.Client) {
+	if stor == nil || !stor.IsConnected() {
+		return
+	}
+	tuples, err := stor.GetSearchIndexTuples()
+	if err != nil {
+		log.Printf("[API] search index refresh failed: %v", err)
+		return
+	}
+	idx.mu.Lock()
+	idx.tuples = tuples
+	idx.mu.Unlock()
+	idx.ready.Store(true)
+}
+
+// runRefreshLoop builds the index immediately, then rebuilds on a ticker for the
+// process lifetime.
+func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
+	idx.refresh(stor)
+	ticker := time.NewTicker(searchIndexRefreshInterval)
+	defer ticker.Stop()
+	for range ticker.C {
+		idx.refresh(stor)
+	}
+}
+
+// serveIndexedSearch answers a free-text (non-property) query from the in-memory
+// index: token-AND match against each record's haystack (or a whole-query hashID
+// substring, preserving the legacy behaviour), then fetch full metadata for only
+// the <=limit matches. No Redis round-trip per scanned record.
+func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
+	tuples, ready := s.search.snapshot()
+	if !ready {
+		// First call before the background build finished: warm inline with the
+		// efficient batched read (seconds), never the per-record SCAN fallback.
+		s.search.refresh(s.storage)
+		tuples, _ = s.search.snapshot()
+	}
+	tokens := strings.Fields(strings.ToLower(req.Query))
+	queryLower := strings.ToLower(req.Query)
+	results := make([]MetadataSearchResult, 0, req.Limit)
+	for _, t := range tuples {
+		if len(results) >= req.Limit {
+			break
+		}
+		if !matchAllTokens(t.Haystack, tokens) &&
+			!strings.Contains(strings.ToLower(t.HashID), queryLower) {
+			continue
+		}
+		// Full metadata is cached in the index — return it straight from memory,
+		// no Redis round-trip per result.
+		results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
+	}
+	writeJSON(w, http.StatusOK, MetadataSearchResponse{
+		Results: results,
+		Count:   len(results),
+		Total:   len(tuples),
+	})
+}
+
+// matchAllTokens reports whether haystack contains every token (AND semantics).
+// An empty token list matches everything.
+func matchAllTokens(haystack string, tokens []string) bool {
+	for _, t := range tokens {
+		if !strings.Contains(haystack, t) {
+			return false
+		}
+	}
+	return true
+}

internal/api/server.go

@@ -41,6 +41,7 @@ type Server struct {
 	metaPublisher     *events.MetaPublisher
 	schemaIndexer     *schema.Indexer
 	webdavHandler     *webdav.Handler
+	search            *searchIndex
 	router            *mux.Router
 	server            *http.Server
 }
@@ -59,6 +60,7 @@ func NewServer(
 		discovery:      disc,
 		cleaner:        cleaner,
 		storage:        stor,
+		search:         &searchIndex{},
 		router:         mux.NewRouter(),
 	}
 
@@ -250,29 +252,22 @@ func (s *Server) setupRoutes() {
 	// Mount management routes (if manager initialized)
 	if s.mountsHandlers != nil {
 		s.mountsHandlers.RegisterRoutes(s.router)
-		log.Println("[API] Mount management routes registered at /api/mounts/*")
 	}
 
-	log.Println("[API] Metadata Editor routes registered at /api/metadata/*")
-	log.Println("[API] KV Browser routes registered at /api/kv/*")
-
 	// File watcher routes (if watcher initialized)
 	if s.watcherHandlers != nil {
 		s.watcherHandlers.RegisterRoutes(s.router)
-		log.Println("[API] File watcher routes registered at /api/events/*, /api/scan/*")
 	}
 
 	// Watchers management routes (if watchers initialized)
 	if s.watchersHandlers != nil {
 		s.watchersHandlers.RegisterRoutes(s.router)
-		log.Println("[API] Watchers management routes registered at /api/watchers/*")
 	}
 
 	// WebDAV handler (caching is handled by nginx proxy_cache)
 	if s.webdavHandler != nil {
 		s.router.PathPrefix("/webdav/").Handler(s.webdavHandler)
 		s.router.PathPrefix("/webdav").Handler(s.webdavHandler)
-		log.Println("[API] WebDAV routes registered at /webdav/*")
 	}
 
 	// Add middleware
@@ -321,6 +316,13 @@ func (s *Server) Start() error {
 		s.mountStatsPoller.Start()
 	}
 
+	// Build + periodically refresh the in-memory search index so
+	// POST /api/metadata/search matches in memory instead of a Redis
+	// round-trip per record (see search_index.go).
+	if s.storage != nil {
+		go s.search.runRefreshLoop(s.storage)
+	}
+
 	// Start meta publisher (if storage is connected)
 	// Publishes keyspace notifications to meta:events stream
 	if s.storage != nil && s.storage.IsConnected() {
@@ -421,12 +423,49 @@ func (s *Server) RepublishMetadata() (int, error) {
 	return s.metaPublisher.RepublishAllMetadata(republishFunc)
 }
 
-// loggingMiddleware logs all requests
+// statusRecorder captures the response status so loggingMiddleware can stay
+// silent on success and only report failures. It forwards Flush/Unwrap so SSE
+// and other streaming handlers keep working through the wrapper.
+type statusRecorder struct {
+	http.ResponseWriter
+	status  int
+	written bool
+}
+
+func (r *statusRecorder) WriteHeader(code int) {
+	if !r.written {
+		r.status = code
+		r.written = true
+	}
+	r.ResponseWriter.WriteHeader(code)
+}
+
+func (r *statusRecorder) Write(b []byte) (int, error) {
+	if !r.written {
+		r.status = http.StatusOK
+		r.written = true
+	}
+	return r.ResponseWriter.Write(b)
+}
+
+func (r *statusRecorder) Flush() {
+	if f, ok := r.ResponseWriter.(http.Flusher); ok {
+		f.Flush()
+	}
+}
+
+func (r *statusRecorder) Unwrap() http.ResponseWriter { return r.ResponseWriter }
+
+// loggingMiddleware logs a request only when it fails (status >= 400), keeping
+// the nominal case quiet. Successful requests are not logged.
 func loggingMiddleware(next http.Handler) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		start := time.Now()
-		next.ServeHTTP(w, r)
-		log.Printf("[API] %s %s %s", r.Method, r.URL.Path, time.Since(start))
+		rec := &statusRecorder{ResponseWriter: w, status: http.StatusOK}
+		next.ServeHTTP(rec, r)
+		if rec.status >= 400 {
+			log.Printf("[API] %s %s -> %d (%s)", r.Method, r.URL.Path, rec.status, time.Since(start))
+		}
 	})
 }
 

internal/mounts/poller.go

@@ -44,6 +44,8 @@ type PollingStatus struct {
 	CurrentScanStartedAt int64 `json:"currentScanStartedAt,omitempty"` // ms; start of in-flight scan
 	LastScanDurationMs   int64 `json:"lastScanDurationMs,omitempty"`   // duration of the previous scan
 	NextScanAt           int64 `json:"nextScanAt,omitempty"`           // ms; planned start of next adaptive scan
+	LastFileCount        int   `json:"-"`                              // last observed file count; used to suppress no-change scan logs
+	scanned              bool  // true once at least one scan has completed for this mount
 }
 
 // Poller manages per-mount polling goroutines
@@ -309,16 +311,22 @@ func (p *Poller) executeScan(mountID, mountPath string) (time.Duration, error) {
 	end := time.Now()
 	duration := end.Sub(start)
 
+	// Only log a completed scan when the file count changed (or on the very
+	// first scan) so a steady mount doesn't emit one line per poll tick.
+	changed := true
 	p.mu.Lock()
 	if status, exists := p.status[mountID]; exists {
 		status.Scanning = false
 		status.CurrentScanStartedAt = 0
 		status.LastScan = end.UnixMilli()
 		status.LastScanDurationMs = duration.Milliseconds()
+		changed = !status.scanned || status.LastFileCount != fileCount
+		status.LastFileCount = fileCount
+		status.scanned = true
 	}
 	p.mu.Unlock()
 
-	if err == nil {
+	if err == nil && changed {
 		log.Printf("[Poller] Poll scan complete for mount %s: %d files in %s", mountID, fileCount, duration.Round(time.Millisecond))
 	}
 

internal/storage/client.go

@@ -683,6 +683,115 @@ func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
 	return out, nil
 }
 
+// SearchTuple is one record's search state: its hashID, a lowercased haystack of
+// its searchable text fields (for token/substring match), and the full flat
+// metadata map (so a search hit is returned straight from memory — no per-result
+// Redis fetch).
+type SearchTuple struct {
+	HashID   string
+	Haystack string
+	Fields   map[string]string
+}
+
+// searchIndexFields are the metadata fields folded into a record's search
+// haystack — must stay in sync with the fields the search handler documents.
+var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
+
+// GetSearchIndexTuples reads EVERY record's full metadata in a single keyspace
+// SCAN pass over the flat `file:{hashID}/{field}` keys plus batched MGETs,
+// grouped by record — one bulk pass instead of a SCAN+MGET per record (which is
+// O(records) Redis round-trips, ~40s over 72k field keys). It is the bulk source
+// the API folds into its in-memory search index, so per-request search does zero
+// Redis round-trips (match AND result metadata both come from memory).
+func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
+	defer cancel()
+
+	filePrefix := c.buildKey("file:") // key layout: {filePrefix}{hashID}/{field}
+	indexKey := c.buildIndexKey()     // {filePrefix}__index__ (a SET, skip it)
+	pattern := filePrefix + "*"
+
+	records := make(map[string]map[string]string)
+	keyBatch := make([]string, 0, 1000)
+
+	flush := func() error {
+		if len(keyBatch) == 0 {
+			return nil
+		}
+		values, err := c.client.MGet(ctx, keyBatch...).Result()
+		if err != nil {
+			return fmt.Errorf("mget: %w", err)
+		}
+		for i, k := range keyBatch {
+			v, ok := values[i].(string)
+			if !ok {
+				continue
+			}
+			// {filePrefix}{hashID}/{field}; hashID has no '/', field may.
+			rest := strings.TrimPrefix(k, filePrefix)
+			slash := strings.IndexByte(rest, '/')
+			if slash < 0 {
+				continue
+			}
+			hashID, field := rest[:slash], rest[slash+1:]
+			m := records[hashID]
+			if m == nil {
+				m = make(map[string]string)
+				records[hashID] = m
+			}
+			m[field] = v
+		}
+		keyBatch = keyBatch[:0]
+		return nil
+	}
+
+	var cursor uint64
+	for {
+		keys, next, err := c.client.Scan(ctx, cursor, pattern, 1000).Result()
+		if err != nil {
+			return nil, fmt.Errorf("scan: %w", err)
+		}
+		for _, k := range keys {
+			if k == indexKey {
+				continue
+			}
+			keyBatch = append(keyBatch, k)
+			if len(keyBatch) >= 1000 {
+				if err := flush(); err != nil {
+					return nil, err
+				}
+			}
+		}
+		cursor = next
+		if cursor == 0 {
+			break
+		}
+	}
+	if err := flush(); err != nil {
+		return nil, err
+	}
+
+	out := make([]SearchTuple, 0, len(records))
+	for hashID, fields := range records {
+		var sb strings.Builder
+		for _, f := range searchIndexFields {
+			if v, ok := fields[f]; ok && v != "" {
+				sb.WriteString(strings.ToLower(v))
+				sb.WriteByte('\n')
+			}
+		}
+		out = append(out, SearchTuple{HashID: hashID, Haystack: sb.String(), Fields: fields})
+	}
+	return out, nil
+}
+
 // GetMemoryInfo returns Redis memory usage information
 func (c *Client) GetMemoryInfo() (string, error) {
 	c.mu.RLock()

internal/watcher/watcher.go

@@ -422,10 +422,9 @@ func (w *Watcher) ScanMountPath(mountPath string) (int, error) {
 		return 0, nil
 	}
 
-	// Scan the directory
-	log.Printf("[Watcher] Scanning mount path: %s", mountPath)
+	// Scan the directory. The mount poller reports completed scans (only when
+	// the file count changes), so nothing is logged per scan here.
 	count := w.ScanDirectory(mountPath)
-	log.Printf("[Watcher] Mount scan complete: %s (%d files)", mountPath, count)
 
 	return count, nil
 }

internal/webdav/handler.go

@@ -50,8 +50,6 @@ func (h *Handler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		path = "/"
 	}
 
-	log.Printf("[WebDAV] %s %s", r.Method, path)
-
 	switch r.Method {
 	case "GET", "HEAD":
 		h.handleRead(w, r, path)