111 lines
3.6 KiB
Go
111 lines
3.6 KiB
Go
package api
|
|
|
|
import (
|
|
"log"
|
|
"net/http"
|
|
"strings"
|
|
"sync"
|
|
"sync/atomic"
|
|
"time"
|
|
|
|
"github.com/metazla/meta-core/internal/storage"
|
|
)
|
|
|
|
// searchIndexRefreshInterval is how often the in-memory search index is rebuilt
|
|
// from Redis in the background. Records ingest continuously, so a newly stored
|
|
// record becomes searchable within one interval; the rebuild is a single
|
|
// batched bulk read (storage.GetSearchIndexTuples), off the request path.
|
|
const searchIndexRefreshInterval = 60 * time.Second
|
|
|
|
// searchIndex is an in-memory snapshot of every record's search haystack
|
|
// (hashID + lowercased searchable fields), rebuilt periodically from Redis. It
|
|
// exists so POST /api/metadata/search matches in memory (string ops over a
|
|
// slice) instead of doing a SCAN+MGET Redis round-trip per record — the latter
|
|
// took 40s+ over ~72k records and made a live parallel search path unusable.
|
|
type searchIndex struct {
|
|
mu sync.RWMutex
|
|
tuples []storage.SearchTuple
|
|
ready atomic.Bool
|
|
}
|
|
|
|
// snapshot returns the current tuples slice and whether the index has built at
|
|
// least once. The refresh swaps in a fresh slice under the write lock (never
|
|
// mutates in place), so the returned slice is safe to read lock-free.
|
|
func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) {
|
|
idx.mu.RLock()
|
|
defer idx.mu.RUnlock()
|
|
return idx.tuples, idx.ready.Load()
|
|
}
|
|
|
|
// refresh rebuilds the index from Redis once (a batched bulk read).
|
|
func (idx *searchIndex) refresh(stor *storage.Client) {
|
|
if stor == nil || !stor.IsConnected() {
|
|
return
|
|
}
|
|
tuples, err := stor.GetSearchIndexTuples()
|
|
if err != nil {
|
|
log.Printf("[API] search index refresh failed: %v", err)
|
|
return
|
|
}
|
|
idx.mu.Lock()
|
|
idx.tuples = tuples
|
|
idx.mu.Unlock()
|
|
idx.ready.Store(true)
|
|
}
|
|
|
|
// runRefreshLoop builds the index immediately, then rebuilds on a ticker for the
|
|
// process lifetime.
|
|
func (idx *searchIndex) runRefreshLoop(stor *storage.Client) {
|
|
idx.refresh(stor)
|
|
ticker := time.NewTicker(searchIndexRefreshInterval)
|
|
defer ticker.Stop()
|
|
for range ticker.C {
|
|
idx.refresh(stor)
|
|
}
|
|
}
|
|
|
|
// serveIndexedSearch answers a free-text (non-property) query from the in-memory
|
|
// index: token-AND match against each record's haystack (or a whole-query hashID
|
|
// substring, preserving the legacy behaviour), then fetch full metadata for only
|
|
// the <=limit matches. No Redis round-trip per scanned record.
|
|
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
|
|
tuples, ready := s.search.snapshot()
|
|
if !ready {
|
|
// First call before the background build finished: warm inline with the
|
|
// efficient batched read (seconds), never the per-record SCAN fallback.
|
|
s.search.refresh(s.storage)
|
|
tuples, _ = s.search.snapshot()
|
|
}
|
|
tokens := strings.Fields(strings.ToLower(req.Query))
|
|
queryLower := strings.ToLower(req.Query)
|
|
results := make([]MetadataSearchResult, 0, req.Limit)
|
|
for _, t := range tuples {
|
|
if len(results) >= req.Limit {
|
|
break
|
|
}
|
|
if !matchAllTokens(t.Haystack, tokens) &&
|
|
!strings.Contains(strings.ToLower(t.HashID), queryLower) {
|
|
continue
|
|
}
|
|
// Full metadata is cached in the index — return it straight from memory,
|
|
// no Redis round-trip per result.
|
|
results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields})
|
|
}
|
|
writeJSON(w, http.StatusOK, MetadataSearchResponse{
|
|
Results: results,
|
|
Count: len(results),
|
|
Total: len(tuples),
|
|
})
|
|
}
|
|
|
|
// matchAllTokens reports whether haystack contains every token (AND semantics).
|
|
// An empty token list matches everything.
|
|
func matchAllTokens(haystack string, tokens []string) bool {
|
|
for _, t := range tokens {
|
|
if !strings.Contains(haystack, t) {
|
|
return false
|
|
}
|
|
}
|
|
return true
|
|
}
|