align to metadata keys doc

internal/storage/cid_resolution.go

@@ -309,6 +309,10 @@ func (c *Client) DeleteRoot(uuid string) error {
 	if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil {
 		return fmt.Errorf("srem index: %w", err)
 	}
+	// Marked so the incremental search-index refresh reads it, finds it gone,
+	// and evicts it — otherwise a deleted root keeps answering searches until
+	// the next full rebuild.
+	c.markDirtyLocked(ctx, uuid)
 	return nil
 }
 

internal/storage/client.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"fmt"
 	"log"
+	"os"
 	"sort"
 	"strconv"
 	"strings"
@@ -173,6 +174,7 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
 	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
@@ -257,6 +259,7 @@ func (c *Client) SetProperty(hashID, property, value string) error {
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	// Auto-register reverse-index alias if this is a CID field.
 	_ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value)
@@ -325,6 +328,10 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) {
 	if err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return deletedCount, fmt.Errorf("srem index failed: %w", err)
 	}
+	// Marked, not unmarked: the incremental refresh reads the record, finds it
+	// gone, and evicts it from the search index. Skipping this would leave a
+	// deleted record answering searches until the next full rebuild.
+	c.markDirtyLocked(ctx, hashID)
 
 	return deletedCount, nil
 }
@@ -441,6 +448,7 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return len(metadata), fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	// Auto-register reverse-index aliases for any cids/<cid> key-set members in
 	// the payload. That is the ONLY shape recognised — the legacy flat cid_* /
@@ -468,6 +476,7 @@ func (c *Client) DeleteProperty(hashID, property string) error {
 		return err
 	}
 	c.removeFieldLocked(ctx, hashID, property)
+	c.markDirtyLocked(ctx, hashID)
 	return nil
 }
 
@@ -521,6 +530,7 @@ func (c *Client) AddToSet(hashID, property, value string) (bool, error) {
 	if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil {
 		return true, fmt.Errorf("sadd index failed: %w", err)
 	}
+	c.markDirtyLocked(ctx, hashID)
 
 	return true, nil
 }
@@ -574,11 +584,16 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) {
 		}
 		// The key is gone, so the field name must leave the index too.
 		c.removeFieldLocked(ctx, hashID, property)
+		c.markDirtyLocked(ctx, hashID)
 		return true, nil
 	}
 
 	newValue := strings.Join(newValues, "|")
-	return true, c.client.Set(ctx, key, newValue, 0).Err()
+	err = c.client.Set(ctx, key, newValue, 0).Err()
+	// The field survives with fewer members, so no field-index change — but the
+	// VALUE changed, which is exactly what the search index caches.
+	c.markDirtyLocked(ctx, hashID)
+	return true, err
 }
 
 // FileTuple holds the (uuid, path, size, mtime, midhash) record for one
@@ -691,6 +706,44 @@ type SearchTuple struct {
 // haystack — must stay in sync with the fields the search handler documents.
 var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
 
+// defaultIndexExcludePrefixes are field-name prefixes kept OUT of the in-memory
+// search index. Redis still stores them and every per-record read still returns
+// them — this narrows the *index*, never the store.
+//
+// `categories/newznab/<code>` is the raw Newznab numeric category code, which
+// METADATA_KEYS documents as a mirror of the canonical `categories/<label>`
+// vocabulary that rides alongside it. Nothing in meta-gateway, meta-search,
+// meta-watch or meta-core reads it, and it is the single largest contributor to
+// the index: measured on a live 19.6k-record corpus, `categories/*` was 40% of
+// the whole keyspace and the `newznab` sub-vocabulary was 59% of that — ~24% of
+// all keys, for zero readers.
+//
+// ⚠ Excluding the WHOLE `categories/` family instead would be a security bug,
+// not an optimisation. `categories/XXX` (and `categories/XXX/<leaf>`) is the
+// server-side adult-content enforcement point — meta-search's
+// `query_eval::fields_are_adult` and meta-gateway's `cards::fields_are_adult`
+// both match on it, and the `exclude_adult` preference defaults to ON. A record
+// served from the index without its categories would read as non-adult and the
+// filter would fail OPEN. Only prefixes with no semantic readers belong here.
+var defaultIndexExcludePrefixes = []string{"categories/newznab/"}
+
+// indexExcludePrefixes resolves the exclusion list, allowing an operator to
+// override it with a comma-separated META_CORE_INDEX_EXCLUDE_PREFIXES. An
+// explicitly empty value disables exclusion entirely (index everything).
+func indexExcludePrefixes() []string {
+	raw, set := os.LookupEnv("META_CORE_INDEX_EXCLUDE_PREFIXES")
+	if !set {
+		return defaultIndexExcludePrefixes
+	}
+	out := make([]string, 0, 4)
+	for _, p := range strings.Split(raw, ",") {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
+
 // GetSearchIndexTuples reads EVERY record's full metadata in a single keyspace
 // SCAN pass over the flat `file:{hashID}/{field}` keys plus batched MGETs,
 // grouped by record — one bulk pass instead of a SCAN+MGET per record (which is
@@ -712,6 +765,7 @@ func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
 	indexKey := c.buildIndexKey()     // {filePrefix}__index__ (a SET, skip it)
 	pattern := filePrefix + "*"
 
+	excluded := indexExcludePrefixes()
 	records := make(map[string]map[string]string)
 	keyBatch := make([]string, 0, 1000)
 
@@ -762,6 +816,11 @@ func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
 			if strings.HasSuffix(k, "/"+FieldsField) {
 				continue
 			}
+			// Index-excluded fields are skipped here rather than after the
+			// MGET, so an excluded key costs neither the fetch nor the memory.
+			if _, field, ok := c.splitRecordKey(k); ok && hasAnyPrefix(field, excluded) {
+				continue
+			}
 			keyBatch = append(keyBatch, k)
 			if len(keyBatch) >= 1000 {
 				if err := flush(); err != nil {
@@ -780,18 +839,23 @@ func (c *Client) GetSearchIndexTuples() ([]SearchTuple, error) {
 
 	out := make([]SearchTuple, 0, len(records))
 	for hashID, fields := range records {
-		var sb strings.Builder
-		for _, f := range searchIndexFields {
-			if v, ok := fields[f]; ok && v != "" {
-				sb.WriteString(strings.ToLower(v))
-				sb.WriteByte('\n')
-			}
-		}
-		out = append(out, SearchTuple{HashID: hashID, Haystack: sb.String(), Fields: fields})
+		// Shared with the incremental path (search_dirty.go) so the two can
+		// never disagree about what is searchable.
+		out = append(out, SearchTuple{HashID: hashID, Haystack: buildHaystack(fields), Fields: fields})
 	}
 	return out, nil
 }
 
+// hasAnyPrefix reports whether s starts with any of the given prefixes.
+func hasAnyPrefix(s string, prefixes []string) bool {
+	for _, p := range prefixes {
+		if strings.HasPrefix(s, p) {
+			return true
+		}
+	}
+	return false
+}
+
 // GetMemoryInfo returns Redis memory usage information
 func (c *Client) GetMemoryInfo() (string, error) {
 	c.mu.RLock()

internal/storage/search_dirty.go

@@ -0,0 +1,214 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"strings"
+	"time"
+)
+
+// Incremental search-index maintenance.
+//
+// # Why this exists
+//
+// The API's in-memory search index used to be rebuilt in full every 60 s:
+// GetSearchIndexTuples does a whole-keyspace SCAN plus batched MGETs, builds an
+// intermediate map of every record, then allocates a fresh []SearchTuple — all
+// while the previous index is still live serving reads. That cost is O(corpus),
+// not O(change): adding 200 records in a minute re-read every one of the other
+// ~620 000 field keys to notice them.
+//
+// Measured on a 19 622-record staging corpus (2026-08-30): SCAN was 57.8 % and
+// MGET 19.1 % of ALL Redis CPU, and ~90 % of the MGET calls plus ~48 % of the
+// SCAN calls were this loop — roughly 45 % of Redis's total work, spent
+// continuously whether or not anything changed. The Go side held 783 MB for
+// data Redis stores in 135 MB, largely because of the double copy.
+//
+// That was tolerable while the corpus was small and mostly static. It stops
+// being tolerable now that meta-core is the platform's permanent, additive
+// store: every search hit on every gateway is retained forever, so the rebuild
+// cost is a ratchet.
+//
+// # The shape
+//
+// Every write path marks its record in a dirty set; the refresh drains that set
+// and re-reads only those records. Cost becomes O(change).
+//
+// Marking rides along with the field-index maintenance every write path already
+// performs, so there is no new bookkeeping discipline to remember — if you are
+// adding a write path, the same line that keeps `__fields__` correct keeps this
+// correct.
+//
+// # Correctness
+//
+// The drain is a destructive SPOP, so a write that lands *during* a refresh is
+// either already in the drained batch or still in the set for the next tick —
+// never dropped. The cost of the race is one tick of staleness, which is well
+// inside the interval the index was already stale by.
+//
+// A refresh that fails after draining WOULD lose its batch, so drainDirty's
+// caller re-marks on error (see MarkDirty / the API's refreshIncremental).
+//
+// ⚠ **The destructive drain assumes ONE index consumer per Redis.** Two
+// meta-core processes sharing a keyspace would steal each other's marks, and
+// each would end up with an index missing whatever the other drained. That is
+// fine today — multi-instance leader election was deprecated and the dev/prod
+// stacks run a single meta-core — but it is the assumption to revisit first if
+// a second instance is ever pointed at the same Redis. A per-consumer dirty key
+// (`file:__dirty__:<consumer>`) is the shape that would fix it.
+//
+// The API's periodic full reconcile (searchIndexReconcileEvery) is the backstop
+// for both this and for a write path that forgets to mark: a missed mark costs
+// staleness until the next reconcile, never permanent divergence.
+
+// dirtyField is the reserved key holding the set of record ids whose metadata
+// changed since the last search-index refresh. It sits under the same `file:`
+// namespace as `__index__` and is likewise not a record.
+const dirtyField = "file:__dirty__"
+
+// dirtyDrainCap bounds one incremental refresh. A burst larger than this is
+// drained over several ticks rather than in one long-held pass — the point of
+// the change is to keep any single refresh cheap. Sized well above a normal
+// search-persist flush (which writes at most a few hundred records).
+const dirtyDrainCap = 5000
+
+// buildDirtyKey constructs the key for the dirty-record set.
+func (c *Client) buildDirtyKey() string {
+	return c.buildKey(dirtyField)
+}
+
+// markDirtyLocked records that hashID's metadata changed. Caller must hold c.mu.
+//
+// Best-effort by design: a failure here costs one record its incremental
+// refresh, and the periodic full rebuild still repairs it. Failing the caller's
+// write instead would trade a stale index entry for a lost write, which is
+// strictly worse.
+func (c *Client) markDirtyLocked(ctx context.Context, hashID string) {
+	if hashID == "" {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildDirtyKey(), hashID).Err(); err != nil {
+		log.Printf("[STORAGE] dirty-mark failed for %s: %v (index repairs on the next full rebuild)", hashID, err)
+	}
+}
+
+// MarkDirty records that hashID needs re-indexing. Exported for the API layer,
+// which re-marks a batch whose refresh failed after the drain.
+func (c *Client) MarkDirty(hashIDs ...string) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+	if c.client == nil || len(hashIDs) == 0 {
+		return
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second)
+	defer cancel()
+	members := make([]interface{}, 0, len(hashIDs))
+	for _, h := range hashIDs {
+		if h != "" {
+			members = append(members, h)
+		}
+	}
+	if len(members) == 0 {
+		return
+	}
+	if err := c.client.SAdd(ctx, c.buildDirtyKey(), members...).Err(); err != nil {
+		log.Printf("[STORAGE] dirty re-mark failed for %d records: %v", len(members), err)
+	}
+}
+
+// DrainDirty removes and returns up to dirtyDrainCap record ids from the dirty
+// set. `more` reports whether entries remain, so the caller can schedule
+// another pass instead of waiting a full interval to catch up on a burst.
+//
+// Destructive on purpose: SPOP makes concurrent refreshes disjoint and stops a
+// record being re-read every tick forever. The caller owns the drained ids
+// until it has indexed them — on failure it must MarkDirty them again.
+func (c *Client) DrainDirty() (ids []string, more bool, err error) {
+	c.mu.RLock()
+	defer c.mu.RUnlock()
+
+	if c.client == nil {
+		return nil, false, fmt.Errorf("not connected")
+	}
+
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	key := c.buildDirtyKey()
+	ids, err = c.client.SPopN(ctx, key, dirtyDrainCap).Result()
+	if err != nil {
+		return nil, false, fmt.Errorf("spop dirty: %w", err)
+	}
+	if len(ids) < dirtyDrainCap {
+		return ids, false, nil
+	}
+	remaining, err := c.client.SCard(ctx, key).Result()
+	if err != nil {
+		// We have the batch; not knowing the remainder just means the next
+		// tick picks it up on schedule.
+		return ids, false, nil
+	}
+	return ids, remaining > 0, nil
+}
+
+// GetSearchIndexTuplesFor reads the search tuples for a specific set of records
+// — the incremental counterpart to GetSearchIndexTuples.
+//
+// Reads go through the per-record field index (GetMetadataFlat), so this is two
+// round-trips per record rather than a keyspace scan. That trade is what makes
+// it O(change): it beats the full pass whenever the changed set is a small
+// fraction of the corpus, and the caller bounds the batch (dirtyDrainCap).
+//
+// A record present in the dirty set but absent from storage was deleted; it is
+// reported in `gone` so the caller can evict it from the index rather than
+// leaving a tombstone that still answers searches.
+func (c *Client) GetSearchIndexTuplesFor(hashIDs []string) (tuples []SearchTuple, gone []string, err error) {
+	if len(hashIDs) == 0 {
+		return nil, nil, nil
+	}
+
+	excluded := indexExcludePrefixes()
+	tuples = make([]SearchTuple, 0, len(hashIDs))
+
+	for _, hashID := range hashIDs {
+		fields, ferr := c.GetMetadataFlat(hashID)
+		if ferr != nil {
+			// Re-mark so the record is retried rather than silently missing
+			// from the index until the next full rebuild.
+			c.MarkDirty(hashID)
+			log.Printf("[STORAGE] incremental index read failed for %s: %v", hashID, ferr)
+			continue
+		}
+		if len(fields) == 0 {
+			gone = append(gone, hashID)
+			continue
+		}
+		for field := range fields {
+			if hasAnyPrefix(field, excluded) {
+				delete(fields, field)
+			}
+		}
+		tuples = append(tuples, SearchTuple{
+			HashID:   hashID,
+			Haystack: buildHaystack(fields),
+			Fields:   fields,
+		})
+	}
+	return tuples, gone, nil
+}
+
+// buildHaystack folds a record's searchable text fields into one lowercased
+// blob. Shared by the full and incremental index reads so the two can never
+// disagree about what is searchable — a drift there would make a record's
+// matchability depend on which path happened to index it.
+func buildHaystack(fields map[string]string) string {
+	var sb strings.Builder
+	for _, f := range searchIndexFields {
+		if v, ok := fields[f]; ok && v != "" {
+			sb.WriteString(strings.ToLower(v))
+			sb.WriteByte('\n')
+		}
+	}
+	return sb.String()
+}

internal/storage/search_dirty_test.go

@@ -0,0 +1,211 @@
+package storage
+
+import (
+	"context"
+	"sort"
+	"testing"
+)
+
+// Tests for incremental search-index maintenance (search_dirty.go).
+//
+// The dirty set is derived state on the hot write path, which makes its failure
+// mode the dangerous kind: a MISSING mark doesn't error, it just leaves the
+// search index quietly serving a stale copy of that record until someone
+// triggers a full rebuild. So the property worth pinning is coverage — every
+// path that changes what a search would match must mark — and that is what the
+// bulk of these tests assert, one write path at a time.
+//
+// The second property is that a delete is a mark too, not an unmark: the
+// refresh has to LEARN the record is gone in order to evict it. Marking on
+// delete is counter-intuitive enough to be worth a test of its own.
+
+func dirtyMembers(t *testing.T, c *Client) []string {
+	t.Helper()
+	got, err := c.client.SMembers(context.Background(), c.buildDirtyKey()).Result()
+	if err != nil {
+		t.Fatalf("smembers dirty: %v", err)
+	}
+	sort.Strings(got)
+	return got
+}
+
+func clearDirty(t *testing.T, c *Client) {
+	t.Helper()
+	if err := c.client.Del(context.Background(), c.buildDirtyKey()).Err(); err != nil {
+		t.Fatalf("del dirty: %v", err)
+	}
+}
+
+// Every write path must mark. A path that doesn't leaves the index serving a
+// stale record with no error anywhere.
+func TestDirty_EveryWritePathMarks(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		cases := []struct {
+			name  string
+			id    string
+			write func(id string) error
+		}{
+			{"SetMetadataFlat", "r-set", func(id string) error {
+				return c.SetMetadataFlat(id, map[string]string{"title": "A"})
+			}},
+			{"SetProperty", "r-prop", func(id string) error {
+				return c.SetProperty(id, "title", "B")
+			}},
+			{"MergeMetadataFlat", "r-merge", func(id string) error {
+				_, err := c.MergeMetadataFlat(id, map[string]string{"title": "C"})
+				return err
+			}},
+			{"AddToSet", "r-add", func(id string) error {
+				_, err := c.AddToSet(id, "tags", "x")
+				return err
+			}},
+		}
+		for _, tc := range cases {
+			t.Run(tc.name, func(t *testing.T) {
+				clearDirty(t, c)
+				if err := tc.write(tc.id); err != nil {
+					t.Fatalf("%s: %v", tc.name, err)
+				}
+				got := dirtyMembers(t, c)
+				if len(got) != 1 || got[0] != tc.id {
+					t.Fatalf("%s must mark %q dirty, got %v", tc.name, tc.id, got)
+				}
+			})
+		}
+	})
+}
+
+// RemoveFromSet's non-empty branch changes a field's VALUE without changing the
+// field-name index, so it is the one path that a mark riding on field-index
+// maintenance would have missed. Pinned explicitly for that reason.
+func TestDirty_RemoveFromSetMarksEvenWhenTheFieldSurvives(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-rem"
+		for _, v := range []string{"x", "y"} {
+			if _, err := c.AddToSet(id, "tags", v); err != nil {
+				t.Fatalf("AddToSet: %v", err)
+			}
+		}
+		clearDirty(t, c)
+
+		if _, err := c.RemoveFromSet(id, "tags", "x"); err != nil {
+			t.Fatalf("RemoveFromSet: %v", err)
+		}
+		// `tags` still exists (y remains) — no field-index change, but the
+		// searchable value changed.
+		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
+			t.Fatalf("a value-only change must still mark dirty, got %v", got)
+		}
+	})
+}
+
+// A delete MARKS rather than unmarks: the refresh must read the record, find it
+// gone, and evict it. Unmarking would leave a deleted record answering searches
+// until the next full rebuild.
+func TestDirty_DeletesMarkSoTheRefreshCanEvict(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-del"
+		if err := c.SetMetadataFlat(id, map[string]string{"title": "Doomed"}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+		clearDirty(t, c)
+
+		if _, err := c.DeleteMetadata(id); err != nil {
+			t.Fatalf("DeleteMetadata: %v", err)
+		}
+		if got := dirtyMembers(t, c); len(got) != 1 || got[0] != id {
+			t.Fatalf("a delete must mark dirty so the index evicts it, got %v", got)
+		}
+
+		// …and the incremental read reports it as gone rather than as an error
+		// or an empty-but-present tuple.
+		tuples, gone, err := c.GetSearchIndexTuplesFor([]string{id})
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
+		}
+		if len(tuples) != 0 {
+			t.Fatalf("a deleted record must produce no tuple, got %d", len(tuples))
+		}
+		if len(gone) != 1 || gone[0] != id {
+			t.Fatalf("a deleted record must be reported gone, got %v", gone)
+		}
+	})
+}
+
+// The drain is destructive so concurrent refreshes stay disjoint and a record
+// isn't re-read every tick forever. The caller owns what it drained.
+func TestDirty_DrainIsDestructive(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		clearDirty(t, c)
+		c.MarkDirty("a", "b", "c")
+
+		ids, more, err := c.DrainDirty()
+		if err != nil {
+			t.Fatalf("DrainDirty: %v", err)
+		}
+		sort.Strings(ids)
+		if len(ids) != 3 {
+			t.Fatalf("expected 3 drained ids, got %v", ids)
+		}
+		if more {
+			t.Fatal("a fully-drained set must not report more")
+		}
+		if got := dirtyMembers(t, c); len(got) != 0 {
+			t.Fatalf("drain must empty the set, got %v", got)
+		}
+
+		// A second drain is empty, not an error — the steady state when nothing
+		// has been written since the last tick.
+		ids, _, err = c.DrainDirty()
+		if err != nil || len(ids) != 0 {
+			t.Fatalf("draining an empty set must be a clean no-op, got %v / %v", ids, err)
+		}
+	})
+}
+
+// The incremental read must produce tuples indistinguishable from the full
+// rebuild's — same haystack, same fields, same exclusions. A drift here would
+// make a record's matchability depend on which path happened to index it, which
+// is invisible until someone searches for the one term that differs.
+func TestDirty_IncrementalTuplesMatchTheFullRebuild(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "r-parity"
+		if err := c.SetMetadataFlat(id, map[string]string{
+			"title":                   "Parity Release",
+			"fileName":                "parity.s01e01.mkv",
+			"fileType":                "video",
+			"categories/XXX":          "true",
+			"categories/newznab/5070": "true",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		full := tuplesByHashID(t, c)[id]
+		incr, gone, err := c.GetSearchIndexTuplesFor([]string{id})
+		if err != nil {
+			t.Fatalf("GetSearchIndexTuplesFor: %v", err)
+		}
+		if len(gone) != 0 || len(incr) != 1 {
+			t.Fatalf("expected exactly one live tuple, got %d live / %d gone", len(incr), len(gone))
+		}
+
+		if incr[0].Haystack != full.Haystack {
+			t.Errorf("haystack drift:\n incremental %q\n full        %q", incr[0].Haystack, full.Haystack)
+		}
+		if len(incr[0].Fields) != len(full.Fields) {
+			t.Errorf("field-count drift: incremental %d, full %d", len(incr[0].Fields), len(full.Fields))
+		}
+		for k, v := range full.Fields {
+			if incr[0].Fields[k] != v {
+				t.Errorf("field %q drift: incremental %q, full %q", k, incr[0].Fields[k], v)
+			}
+		}
+		// The index projection must apply on both paths.
+		if _, ok := incr[0].Fields["categories/newznab/5070"]; ok {
+			t.Error("the incremental path must honour the index exclusion list too")
+		}
+		if _, ok := incr[0].Fields["categories/XXX"]; !ok {
+			t.Error("the incremental path must keep the adult-filter category")
+		}
+	})
+}

internal/storage/search_index_exclude_test.go

@@ -0,0 +1,116 @@
+package storage
+
+import (
+	"testing"
+)
+
+// Tests for the search-index field projection (GetSearchIndexTuples +
+// indexExcludePrefixes).
+//
+// The projection narrows the in-memory index only — Redis keeps every field and
+// every per-record read still returns it. What has to be pinned is the boundary
+// between the two, because getting it wrong in either direction is silent:
+// exclude too little and the index keeps paying for fields nothing reads;
+// exclude too much and a record served *from the index* comes back missing a
+// field a downstream consumer depends on.
+//
+// The specific hazard the default guards is adult-content filtering:
+// `categories/XXX` is the server-side enforcement point (meta-search's
+// `query_eval::fields_are_adult`, meta-gateway's `cards::fields_are_adult`),
+// and `exclude_adult` defaults to ON. Excluding the whole `categories/` family
+// would make every index-served record read as non-adult and the filter would
+// fail OPEN — so the default must keep `categories/XXX` while dropping the
+// reader-less `categories/newznab/` mirror.
+
+func tuplesByHashID(t *testing.T, c *Client) map[string]SearchTuple {
+	t.Helper()
+	got, err := c.GetSearchIndexTuples()
+	if err != nil {
+		t.Fatalf("GetSearchIndexTuples: %v", err)
+	}
+	out := make(map[string]SearchTuple, len(got))
+	for _, tu := range got {
+		out[tu.HashID] = tu
+	}
+	return out
+}
+
+func TestSearchIndex_ExcludesNewznabCategoriesButKeepsTheRest(t *testing.T) {
+	forEachPrefix(t, func(t *testing.T, c *Client) {
+		const id = "root-1"
+		if err := c.SetMetadataFlat(id, map[string]string{
+			"title":                   "Some Release",
+			"fileType":                "video",
+			"categories/TV/Anime":     "true",
+			"categories/XXX":          "true",
+			"categories/newznab/5070": "true",
+			"categories/newznab/2040": "true",
+		}); err != nil {
+			t.Fatalf("SetMetadataFlat: %v", err)
+		}
+
+		fields := tuplesByHashID(t, c)[id].Fields
+		if fields == nil {
+			t.Fatal("record missing from the search index")
+		}
+
+		for _, excluded := range []string{"categories/newznab/5070", "categories/newznab/2040"} {
+			if _, ok := fields[excluded]; ok {
+				t.Errorf("%s must not be indexed — it is a reader-less mirror vocabulary", excluded)
+			}
+		}
+		// The load-bearing ones must survive. XXX especially: without it the
+		// adult filter fails open on every index-served record.
+		for _, kept := range []string{"categories/XXX", "categories/TV/Anime", "title", "fileType"} {
+			if _, ok := fields[kept]; !ok {
+				t.Errorf("%s must stay in the index", kept)
+			}
+		}
+
+		// The store itself is untouched — the projection is index-only.
+		stored, err := c.GetMetadataFlat(id)
+		if err != nil {
+			t.Fatalf("GetMetadataFlat: %v", err)
+		}
+		if _, ok := stored["categories/newznab/5070"]; !ok {
+			t.Error("an index-excluded field must still be stored and readable per-record")
+		}
+	})
+}
+
+func TestIndexExcludePrefixes_EnvOverride(t *testing.T) {
+	if got := indexExcludePrefixes(); len(got) != 1 || got[0] != "categories/newznab/" {
+		t.Fatalf("unset env must yield the default, got %v", got)
+	}
+
+	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "foo/, bar/ ,")
+	got := indexExcludePrefixes()
+	if len(got) != 2 || got[0] != "foo/" || got[1] != "bar/" {
+		t.Fatalf("override must trim and drop blanks, got %v", got)
+	}
+
+	// An explicitly empty value is the escape hatch: index everything.
+	t.Setenv("META_CORE_INDEX_EXCLUDE_PREFIXES", "")
+	if got := indexExcludePrefixes(); len(got) != 0 {
+		t.Fatalf("an empty override must disable exclusion, got %v", got)
+	}
+}
+
+func TestHasAnyPrefix(t *testing.T) {
+	prefixes := []string{"categories/newznab/", "tmp/"}
+	cases := map[string]bool{
+		"categories/newznab/5070": true,
+		"tmp/scratch":             true,
+		"categories/XXX":          false,
+		"categories/newznab":      false, // the family root is not the mirror
+		"title":                   false,
+	}
+	for field, want := range cases {
+		if got := hasAnyPrefix(field, prefixes); got != want {
+			t.Errorf("hasAnyPrefix(%q) = %v, want %v", field, got, want)
+		}
+	}
+	if hasAnyPrefix("anything", nil) {
+		t.Error("an empty prefix list must exclude nothing")
+	}
+}