From 0c4e3a222794dd1fd0ad20571f77274825bb663b Mon Sep 17 00:00:00 2001 From: Worph Date: Sun, 30 Aug 2026 19:52:36 +0200 Subject: [PATCH] align to metadata keys doc --- internal/api/metadata.go | 102 +++---- internal/api/search_filter_test.go | 211 ++++++++++++++ internal/api/search_index.go | 264 ++++++++++++++++-- internal/storage/cid_resolution.go | 4 + internal/storage/client.go | 82 +++++- internal/storage/search_dirty.go | 214 ++++++++++++++ internal/storage/search_dirty_test.go | 211 ++++++++++++++ internal/storage/search_index_exclude_test.go | 116 ++++++++ 8 files changed, 1119 insertions(+), 85 deletions(-) create mode 100644 internal/api/search_filter_test.go create mode 100644 internal/storage/search_dirty.go create mode 100644 internal/storage/search_dirty_test.go create mode 100644 internal/storage/search_index_exclude_test.go diff --git a/internal/api/metadata.go b/internal/api/metadata.go index 6f03650..2db7e2d 100644 --- a/internal/api/metadata.go +++ b/internal/api/metadata.go @@ -4,7 +4,6 @@ import ( "encoding/json" "net/http" "strconv" - "strings" "github.com/gorilla/mux" ) @@ -39,6 +38,36 @@ type MetadataSearchRequest struct { Property string `json:"property,omitempty"` PropertyValue string `json:"propertyValue,omitempty"` Limit int `json:"limit,omitempty"` + + // Filters is the structured form: AND across entries, OR within an + // entry's Values. Evaluated in memory against the search index alongside + // the free-text Query, so an id-anchored lookup costs a map probe per + // record rather than a Redis round-trip per record. + // + // Property/PropertyValue is the single-filter legacy shape and folds into + // this list (see normalizedFilters) with Exact=false, preserving its + // substring semantics byte-for-byte. + Filters []MetadataFilter `json:"filters,omitempty"` +} + +// MetadataFilter is one structured constraint on a record. +// +// A record satisfies it when ANY of Values matches, and a search result must +// satisfy EVERY filter (OR within, AND across). An entry with no usable value +// constrains nothing — same rule the gateway's `genres_filter_matches` uses, so +// a caller that lowers an empty filter doesn't accidentally empty the result. +type MetadataFilter struct { + // Property is the field name, WITHOUT a key-set suffix: `genres`, not + // `genres/Action`. Both storage shapes are probed — see recordMatchesFilter. + Property string `json:"property"` + // Values are OR'd. Compared case-insensitively. + Values []string `json:"values,omitempty"` + // Exact selects CSV-exact comparison (the semantics meta-gateway's + // `any_csv_value_matches` applies at its own Layer-B re-filter) over + // substring containment. Callers doing identity lookups want this: with a + // result limit, a substring match on `tmdbid=1405` also admits `21405` and + // can crowd the real matches out of the page. + Exact bool `json:"exact,omitempty"` } // MetadataSearchResult is a single search result @@ -415,65 +444,18 @@ func (s *Server) handleSearchMetadata(w http.ResponseWriter, r *http.Request) { return } - // Free-text query: served from the in-memory search index (search_index.go) - // so we match in memory instead of doing a SCAN+MGET Redis round-trip per - // record (that was 40s+ over ~72k records). Full metadata is fetched only - // for the <=limit matches. - if req.Query != "" && req.Property == "" { - s.serveIndexedSearch(w, req) - return - } - - // Property filter (rare) or empty query (returns the first `limit`): the - // legacy scan. Empty query only fetches `limit` records; property search is - // low-volume and not on the hot search path. - allHashIDs, err := s.storage.GetAllHashIDs() - if err != nil { - writeError(w, http.StatusInternalServerError, err.Error()) - return - } - - var results []MetadataSearchResult - total := len(allHashIDs) - - for _, hashID := range allHashIDs { - if len(results) >= req.Limit { - break - } - - metadata, err := s.storage.GetMetadataFlat(hashID) - if err != nil || metadata == nil { - continue - } - - matches := false - - // Property search - if req.Property != "" && req.PropertyValue != "" { - value, ok := metadata[req.Property] - if ok && strings.Contains(strings.ToLower(value), strings.ToLower(req.PropertyValue)) { - matches = true - } - } - - // If no query or property filter, return all - if req.Query == "" && req.Property == "" { - matches = true - } - - if matches { - results = append(results, MetadataSearchResult{ - HashID: hashID, - Metadata: metadata, - }) - } - } - - writeJSON(w, http.StatusOK, MetadataSearchResponse{ - Results: results, - Count: len(results), - Total: total, - }) + // Everything else — free text, structured filters, both, or neither + // (which returns the first `limit`) — is served from the in-memory search + // index (search_index.go). Matching and the returned metadata both come + // from memory, so a query costs zero Redis round-trips. + // + // There is deliberately no second path. The property branch used to fall + // through to a `GetAllHashIDs` + per-record `GetMetadataFlat` scan, which + // is what made an id-anchored lookup 3.63s over ~20k records (vs 10ms for + // the free-text branch on the same corpus) and kept meta-core out of the + // gateway's anchored search path entirely. The index already holds every + // field of every record, so that scan was re-reading data it had in hand. + s.serveIndexedSearch(w, req) } // handleBatchUpdate handles POST /api/metadata/batch diff --git a/internal/api/search_filter_test.go b/internal/api/search_filter_test.go new file mode 100644 index 0000000..4845deb --- /dev/null +++ b/internal/api/search_filter_test.go @@ -0,0 +1,211 @@ +package api + +import "testing" + +// Tests for the structured-filter matcher that lets meta-core answer an +// id-anchored query from the in-memory search index (search_index.go). +// +// Why this matters beyond "filters work": meta-core is a *coarse pre-filter* +// for meta-gateway, which re-evaluates every record it emits through its own +// `query_eval::record_matches`. So the property that has to hold is that this +// matcher stays a SUPERSET of that one — it may admit records the gateway will +// drop, but it must never drop a record the gateway would have kept. Every +// case below is written from that angle: +// +// - key-set fields (`genres/Action`) must match a `genres:action` filter, or +// card-tier records vanish from a genre-scoped query; +// - a missing field fails CLOSED, matching record_matches; +// - an empty/unusable filter constrains NOTHING, so a caller that lowers a +// blank filter gets its results rather than an empty page. + +func f(property string, exact bool, values ...string) MetadataFilter { + return MetadataFilter{Property: property, Values: values, Exact: exact} +} + +func TestRecordMatchesFilter(t *testing.T) { + fields := map[string]string{ + "fileType": "video", + "videoType": "tv,anime", + "tmdbid": "1405", + "genres/Action": "true", + "genres/Science Fiction": "true", + "source/gateway:nyaa.si": "true", + "categories/XXX": "true", + "retracted": "false", + } + + cases := []struct { + name string + filter MetadataFilter + want bool + }{ + // Flat scalar. + {"flat exact hit", f("fileType", true, "video"), true}, + {"flat exact miss", f("fileType", true, "audio"), false}, + {"flat exact is case-insensitive", f("fileType", true, "VIDEO"), true}, + + // csv-set field: each comma piece is compared independently. + {"csv member hit", f("videoType", true, "anime"), true}, + {"csv member miss", f("videoType", true, "movie"), false}, + + // Exact vs contains — the reason Exact exists. A substring match on an + // identity field admits neighbours and, under a result limit, crowds + // the real matches off the page. + {"contains admits a superstring", f("tmdbid", false, "140"), true}, + {"exact rejects a superstring", f("tmdbid", true, "140"), false}, + {"exact accepts the whole value", f("tmdbid", true, "1405"), true}, + + // Key-set shape. The stored member is verbatim (`genres/Action`) while + // query values arrive lowercased, so the case-insensitive sweep is what + // makes a genre-scoped row work at all. + {"key-set exact casing", f("genres", true, "Action"), true}, + {"key-set lowercased query", f("genres", true, "action"), true}, + {"key-set with a space", f("genres", true, "science fiction"), true}, + {"key-set miss", f("genres", true, "horror"), false}, + {"key-set label containing a colon", f("source", true, "gateway:nyaa.si"), true}, + + // OR within one filter's values. + {"or hits on the second value", f("fileType", true, "audio", "video"), true}, + {"or misses on all values", f("fileType", true, "audio", "image"), false}, + + // Fail-closed on an absent field — mirrors record_matches, and is what + // keeps a filtered query from returning unrelated records. + {"absent field fails closed", f("imdbid", true, "tt0903747"), false}, + + // A filter that constrains nothing must not empty the result. + {"no values constrains nothing", f("fileType", true), true}, + {"blank value constrains nothing", f("fileType", true, " "), true}, + {"no property constrains nothing", f("", true, "video"), true}, + + // A FLAT field whose value happens to be "false" is still a value, and + // an exact filter for it matches. Only the key-set arm treats "false" + // as non-membership (pinned in the test below). + {"flat field valued false still matches", f("retracted", true, "false"), true}, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := recordMatchesFilter(fields, &tc.filter); got != tc.want { + t.Fatalf("recordMatchesFilter(%+v) = %v, want %v", tc.filter, got, tc.want) + } + }) + } +} + +// `retracted = "false"` is a flat field, not a key-set member, so the flat arm +// legitimately matches the literal string "false". The key-set arm is what must +// treat it as non-membership — pinned separately so the two arms don't get +// conflated if either is refactored. +func TestFalseyKeySetMemberIsNotMembership(t *testing.T) { + fields := map[string]string{"genres/Action": "false", "genres/Drama": "0"} + for _, v := range []string{"Action", "Drama"} { + filter := f("genres", true, v) + if recordMatchesFilter(fields, &filter) { + t.Fatalf("genres/%s with a falsey value must not assert membership", v) + } + } +} + +func TestRecordMatchesFiltersIsAndAcrossEntries(t *testing.T) { + fields := map[string]string{"fileType": "video", "season": "2"} + + all := []MetadataFilter{f("fileType", true, "video"), f("season", true, "2")} + if !recordMatchesFilters(fields, all) { + t.Fatal("a record satisfying every filter must match") + } + + one := []MetadataFilter{f("fileType", true, "video"), f("season", true, "3")} + if recordMatchesFilters(fields, one) { + t.Fatal("filters are AND across entries — one miss must reject the record") + } + + if !recordMatchesFilters(fields, nil) { + t.Fatal("no filters must match everything") + } +} + +// The legacy Property/PropertyValue pair keeps its substring semantics: it is +// the shape existing callers already use, and tightening it silently would +// change results for them. New callers pass Filters and opt into Exact. +func TestNormalizedFiltersFoldsTheLegacyPair(t *testing.T) { + got := normalizedFilters(MetadataSearchRequest{Property: "tmdbid", PropertyValue: "1405"}) + if len(got) != 1 { + t.Fatalf("legacy pair must fold into exactly one filter, got %d", len(got)) + } + if got[0].Property != "tmdbid" || len(got[0].Values) != 1 || got[0].Values[0] != "1405" { + t.Fatalf("legacy pair folded incorrectly: %+v", got[0]) + } + if got[0].Exact { + t.Fatal("the legacy pair must stay substring (Exact=false) for compatibility") + } + + // A half-specified pair is not a filter — it must not become one that + // matches nothing and empties the response. + if n := len(normalizedFilters(MetadataSearchRequest{Property: "tmdbid"})); n != 0 { + t.Fatalf("a property with no value must not fold into a filter, got %d", n) + } + if n := len(normalizedFilters(MetadataSearchRequest{PropertyValue: "1405"})); n != 0 { + t.Fatalf("a value with no property must not fold into a filter, got %d", n) + } + + both := normalizedFilters(MetadataSearchRequest{ + Property: "tmdbid", + PropertyValue: "1405", + Filters: []MetadataFilter{f("fileType", true, "video")}, + }) + if len(both) != 2 { + t.Fatalf("the legacy pair must be appended to explicit filters, got %d", len(both)) + } +} + +// The acceptance bar for routing the anchored path through the index: a +// full-miss filtered sweep over a realistic corpus must stay in the same class +// as the free-text branch (measured 10 ms over 19 622 live records), not the +// legacy per-record Redis scan it replaces (measured 3.63 s over the same +// corpus). Full-miss is the worst case — the loop cannot short-circuit on the +// result limit, so every record is evaluated. +// +// Run: go test ./internal/api/ -bench BenchmarkFilterSweep -benchtime 10x +func BenchmarkFilterSweep(b *testing.B) { + const records = 20000 + corpus := make([]map[string]string, records) + for i := range corpus { + corpus[i] = map[string]string{ + "title": "Some Release", + "fileType": "video", + "videoType": "tv", + "tmdbid": "1405", + "season": "2", + "genres/Action": "true", + "genres/Drama": "true", + "source/gateway:nyaa.si": "true", + "categories/TV/Anime": "true", + "indexer": "nyaa.si", + "fileName": "some.release.s02e05.mkv", + "description/eng": "text", + "poster": "bafy", + "sizeByte": "123456789", + "contentKind": "episode", + "anchoredBy/tmdb:1405": "true", + } + } + // A miss on the identity field: the shape an id-anchored gateway query + // takes, and the one that must not degrade. + filters := []MetadataFilter{ + f("tmdbid", true, "999999"), + f("fileType", true, "video"), + f("genres", true, "action"), + } + b.ResetTimer() + for n := 0; n < b.N; n++ { + hits := 0 + for _, fields := range corpus { + if recordMatchesFilters(fields, filters) { + hits++ + } + } + if hits != 0 { + b.Fatalf("expected a full miss, got %d hits", hits) + } + } +} diff --git a/internal/api/search_index.go b/internal/api/search_index.go index 8b26a4b..e29f63c 100644 --- a/internal/api/search_index.go +++ b/internal/api/search_index.go @@ -3,6 +3,7 @@ package api import ( "log" "net/http" + "sort" "strings" "sync" "sync/atomic" @@ -17,19 +18,46 @@ import ( // batched bulk read (storage.GetSearchIndexTuples), off the request path. const searchIndexRefreshInterval = 60 * time.Second +// searchIndexReconcileEvery is how many refresh ticks pass between full +// rebuilds. Every other tick is incremental. 30 ticks × 60 s = a full +// reconcile every 30 minutes — see runRefreshLoop for why one is kept at all. +const searchIndexReconcileEvery = 30 + // searchIndex is an in-memory snapshot of every record's search haystack -// (hashID + lowercased searchable fields), rebuilt periodically from Redis. It -// exists so POST /api/metadata/search matches in memory (string ops over a -// slice) instead of doing a SCAN+MGET Redis round-trip per record — the latter -// took 40s+ over ~72k records and made a live parallel search path unusable. +// (hashID + lowercased searchable fields), kept current from Redis. It exists +// so POST /api/metadata/search matches in memory (string ops over a slice) +// instead of doing a SCAN+MGET Redis round-trip per record — the latter took +// 40s+ over ~72k records and made a live parallel search path unusable. +// +// # Two update paths, and why +// +// `refresh` rebuilds everything: a whole-keyspace SCAN, batched MGETs, and a +// complete fresh copy. That is O(corpus) regardless of how much changed, and it +// used to run every 60 s forever. Measured on a 19 622-record staging corpus +// (2026-08-30): the loop accounted for ~45 % of ALL Redis CPU, and the Go side +// held 783 MB for data Redis stores in 135 MB — largely the double copy. +// +// `refreshIncremental` re-reads only the records that changed, tracked by +// storage's dirty set (search_dirty.go). Cost is O(change). It is the steady +// state; the full rebuild runs once at startup (there is nothing to be +// incremental against) and then only as an explicit repair. +// +// The distinction matters because meta-core is the platform's permanent, +// additive store — every search hit on every gateway is retained — so a rebuild +// priced against the whole corpus is a cost that only ever grows. type searchIndex struct { - mu sync.RWMutex + mu sync.RWMutex + // byID is the authoritative content; tuples is a stable-order view of it + // rebuilt on each update. Map iteration order in Go is randomised, so + // serving a limited query straight off the map would return a different + // subset each call — the slice is what keeps results stable. + byID map[string]storage.SearchTuple tuples []storage.SearchTuple ready atomic.Bool } // snapshot returns the current tuples slice and whether the index has built at -// least once. The refresh swaps in a fresh slice under the write lock (never +// least once. Every update swaps in a fresh slice under the write lock (never // mutates in place), so the returned slice is safe to read lock-free. func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) { idx.mu.RLock() @@ -37,7 +65,21 @@ func (idx *searchIndex) snapshot() ([]storage.SearchTuple, bool) { return idx.tuples, idx.ready.Load() } -// refresh rebuilds the index from Redis once (a batched bulk read). +// rebuildViewLocked regenerates the stable-order slice from byID. Caller holds +// the write lock. +func (idx *searchIndex) rebuildViewLocked() { + view := make([]storage.SearchTuple, 0, len(idx.byID)) + for _, t := range idx.byID { + view = append(view, t) + } + // Stable across rebuilds so a limited query returns a consistent page + // rather than a fresh random subset every interval. + sort.Slice(view, func(i, j int) bool { return view[i].HashID < view[j].HashID }) + idx.tuples = view +} + +// refresh rebuilds the whole index from Redis (a batched bulk read). Used for +// the initial build and as an explicit repair — not the steady state. func (idx *searchIndex) refresh(stor *storage.Client) { if stor == nil || !stor.IsConnected() { return @@ -47,27 +89,101 @@ func (idx *searchIndex) refresh(stor *storage.Client) { log.Printf("[API] search index refresh failed: %v", err) return } + byID := make(map[string]storage.SearchTuple, len(tuples)) + for _, t := range tuples { + byID[t.HashID] = t + } idx.mu.Lock() - idx.tuples = tuples + idx.byID = byID + idx.rebuildViewLocked() idx.mu.Unlock() idx.ready.Store(true) } -// runRefreshLoop builds the index immediately, then rebuilds on a ticker for the -// process lifetime. +// refreshIncremental drains the dirty set and patches only those records. +// Returns whether more dirty entries remain, so a burst is caught up over +// consecutive passes rather than one per interval. +// +// A record in the dirty set but absent from storage was deleted and is evicted +// here — otherwise it would keep answering searches until the next full +// rebuild. A batch that fails after the (destructive) drain is re-marked by +// storage so it is retried rather than silently lost. +func (idx *searchIndex) refreshIncremental(stor *storage.Client) (more bool) { + if stor == nil || !stor.IsConnected() { + return false + } + ids, more, err := stor.DrainDirty() + if err != nil { + log.Printf("[API] search index drain failed: %v", err) + return false + } + if len(ids) == 0 { + return false + } + tuples, gone, err := stor.GetSearchIndexTuplesFor(ids) + if err != nil { + stor.MarkDirty(ids...) + log.Printf("[API] incremental search index refresh failed: %v", err) + return false + } + + idx.mu.Lock() + if idx.byID == nil { + idx.byID = make(map[string]storage.SearchTuple, len(tuples)) + } + for _, t := range tuples { + idx.byID[t.HashID] = t + } + for _, id := range gone { + delete(idx.byID, id) + } + idx.rebuildViewLocked() + idx.mu.Unlock() + + log.Printf("[API] search index: %d updated, %d evicted (%d records)", len(tuples), len(gone), len(idx.tuples)) + return more +} + +// runRefreshLoop builds the index once in full, then keeps it current from the +// dirty set, with a periodic full reconcile as a safety net. +// +// The reconcile exists because a *missed* dirty mark is silent: no error, just +// a record whose indexed copy quietly diverges from storage until something +// forces a rebuild. Marking is spread across eight write paths plus anything +// added later, and the cost of getting it wrong is invisible — so the index +// should not depend on that coverage being perfect forever. Every +// searchIndexReconcileEvery-th tick pays the old O(corpus) pass and repairs +// whatever drifted. +// +// That still drops the full rebuild from every 60 s to every 30 min — a ~30× +// reduction in the steady-state cost that measured at ~45 % of all Redis CPU — +// while keeping the self-healing property that made the naive loop safe. func (idx *searchIndex) runRefreshLoop(stor *storage.Client) { idx.refresh(stor) ticker := time.NewTicker(searchIndexRefreshInterval) defer ticker.Stop() + ticks := 0 for range ticker.C { - idx.refresh(stor) + ticks++ + if ticks%searchIndexReconcileEvery == 0 { + idx.refresh(stor) + continue + } + // Drain until caught up, so a burst larger than one batch does not take + // one interval per batch to land. + for idx.refreshIncremental(stor) { + } } } -// serveIndexedSearch answers a free-text (non-property) query from the in-memory -// index: token-AND match against each record's haystack (or a whole-query hashID -// substring, preserving the legacy behaviour), then fetch full metadata for only -// the <=limit matches. No Redis round-trip per scanned record. +// serveIndexedSearch answers a search from the in-memory index: a token-AND +// match against each record's haystack (or a whole-query hashID substring, +// preserving the legacy behaviour) AND every structured filter, then returns +// the full metadata for the <=limit survivors. No Redis round-trip per scanned +// record — matching and results both come from memory. +// +// This is the ONLY search path. Free text, filters, both, or neither all land +// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it. func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) { tuples, ready := s.search.snapshot() if !ready { @@ -78,15 +194,20 @@ func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchReq } tokens := strings.Fields(strings.ToLower(req.Query)) queryLower := strings.ToLower(req.Query) + filters := normalizedFilters(req) results := make([]MetadataSearchResult, 0, req.Limit) for _, t := range tuples { if len(results) >= req.Limit { break } - if !matchAllTokens(t.Haystack, tokens) && + if req.Query != "" && + !matchAllTokens(t.Haystack, tokens) && !strings.Contains(strings.ToLower(t.HashID), queryLower) { continue } + if !recordMatchesFilters(t.Fields, filters) { + continue + } // Full metadata is cached in the index — return it straight from memory, // no Redis round-trip per result. results = append(results, MetadataSearchResult{HashID: t.HashID, Metadata: t.Fields}) @@ -108,3 +229,114 @@ func matchAllTokens(haystack string, tokens []string) bool { } return true } + +// normalizedFilters returns the request's structured filters with the legacy +// Property/PropertyValue pair folded in as one more entry. +// +// The legacy pair keeps Exact=false so its long-standing substring semantics +// are unchanged for existing callers; new callers pass Filters and choose. +func normalizedFilters(req MetadataSearchRequest) []MetadataFilter { + if req.Property == "" || req.PropertyValue == "" { + return req.Filters + } + out := make([]MetadataFilter, 0, len(req.Filters)+1) + out = append(out, req.Filters...) + return append(out, MetadataFilter{ + Property: req.Property, + Values: []string{req.PropertyValue}, + }) +} + +// recordMatchesFilters reports whether fields satisfies every filter (AND). +func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool { + for i := range filters { + if !recordMatchesFilter(fields, &filters[i]) { + return false + } + } + return true +} + +// recordMatchesFilter reports whether fields satisfies one filter (OR within +// its values), probing BOTH storage shapes a field can take: +// +// - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`; +// - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"` +// (METADATA_KEYS "How the store is shaped"). +// +// Probing both is what lets one filter shape serve the whole vocabulary. The +// gateway needs a `genres:action` filter to find a record whose genre lives in +// the key `genres/Action`, and a `fileType:video` filter to find one whose value +// lives in the field `fileType` — without meta-core knowing which fields are +// key-sets. It also means this stays a SUPERSET of the gateway's own +// `record_matches`, which is the contract: meta-core pre-filters, the gateway +// re-filters authoritatively. +// +// A filter with no usable value constrains nothing (matches). A filter whose +// field is absent fails closed, matching `record_matches`. +func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool { + values := make([]string, 0, len(f.Values)) + for _, v := range f.Values { + if v = strings.TrimSpace(v); v != "" { + values = append(values, v) + } + } + if f.Property == "" || len(values) == 0 { + return true + } + + // Key-set shape: `/` present and truthy. Try the exact + // key first — that hits whenever the caller's casing matches what was + // written, which is the common case and costs one map probe. + for _, v := range values { + if isTruthyMember(fields[f.Property+"/"+v]) { + return true + } + } + // Members are written verbatim (`genres/Action`) while query values arrive + // lowercased, so fall back to one case-insensitive sweep. Single pass over + // the record's fields for ALL values, not one pass per value. + wanted := make(map[string]struct{}, len(values)) + for _, v := range values { + wanted[strings.ToLower(v)] = struct{}{} + } + prefix := strings.ToLower(f.Property) + "/" + for k, val := range fields { + if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) { + continue + } + if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) { + return true + } + } + + // Flat shape: the field's own value, split on `,` for csv-set fields. + fieldValue, ok := fields[f.Property] + if !ok { + return false + } + for _, piece := range strings.Split(fieldValue, ",") { + piece = strings.TrimSpace(piece) + if piece == "" { + continue + } + for _, v := range values { + if f.Exact { + if strings.EqualFold(piece, v) { + return true + } + } else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) { + return true + } + } + } + return false +} + +// isTruthyMember reports whether a key-set member's value asserts membership. +// Mirrors the gateway's `is_truthy_member`: presence with a non-empty, +// non-"false"/"0" value. +func isTruthyMember(v string) bool { + v = strings.TrimSpace(strings.ToLower(v)) + return v != "" && v != "false" && v != "0" +} diff --git a/internal/storage/cid_resolution.go b/internal/storage/cid_resolution.go index bc7c034..eab6ec4 100644 --- a/internal/storage/cid_resolution.go +++ b/internal/storage/cid_resolution.go @@ -309,6 +309,10 @@ func (c *Client) DeleteRoot(uuid string) error { if err := c.client.SRem(ctx, c.buildIndexKey(), uuid).Err(); err != nil { return fmt.Errorf("srem index: %w", err) } + // Marked so the incremental search-index refresh reads it, finds it gone, + // and evicts it — otherwise a deleted root keeps answering searches until + // the next full rebuild. + c.markDirtyLocked(ctx, uuid) return nil } diff --git a/internal/storage/client.go b/internal/storage/client.go index d1799cc..10b7b61 100644 --- a/internal/storage/client.go +++ b/internal/storage/client.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "log" + "os" "sort" "strconv" "strings" @@ -173,6 +174,7 @@ func (c *Client) SetMetadataFlat(hashID string, metadata map[string]string) erro if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { return fmt.Errorf("sadd index failed: %w", err) } + c.markDirtyLocked(ctx, hashID) // Auto-register reverse-index aliases for any cids/ key-set members in // the payload. That is the ONLY shape recognised — the legacy flat cid_* / @@ -257,6 +259,7 @@ func (c *Client) SetProperty(hashID, property, value string) error { if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { return fmt.Errorf("sadd index failed: %w", err) } + c.markDirtyLocked(ctx, hashID) // Auto-register reverse-index alias if this is a CID field. _ = c.maybeAddAliasFromFieldLocked(ctx, hashID, property, value) @@ -325,6 +328,10 @@ func (c *Client) DeleteMetadata(hashID string) (int64, error) { if err := c.client.SRem(ctx, c.buildIndexKey(), hashID).Err(); err != nil { return deletedCount, fmt.Errorf("srem index failed: %w", err) } + // Marked, not unmarked: the incremental refresh reads the record, finds it + // gone, and evicts it from the search index. Skipping this would leave a + // deleted record answering searches until the next full rebuild. + c.markDirtyLocked(ctx, hashID) return deletedCount, nil } @@ -441,6 +448,7 @@ func (c *Client) MergeMetadataFlat(hashID string, metadata map[string]string) (i if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { return len(metadata), fmt.Errorf("sadd index failed: %w", err) } + c.markDirtyLocked(ctx, hashID) // Auto-register reverse-index aliases for any cids/ key-set members in // the payload. That is the ONLY shape recognised — the legacy flat cid_* / @@ -468,6 +476,7 @@ func (c *Client) DeleteProperty(hashID, property string) error { return err } c.removeFieldLocked(ctx, hashID, property) + c.markDirtyLocked(ctx, hashID) return nil } @@ -521,6 +530,7 @@ func (c *Client) AddToSet(hashID, property, value string) (bool, error) { if err := c.client.SAdd(ctx, c.buildIndexKey(), hashID).Err(); err != nil { return true, fmt.Errorf("sadd index failed: %w", err) } + c.markDirtyLocked(ctx, hashID) return true, nil } @@ -574,11 +584,16 @@ func (c *Client) RemoveFromSet(hashID, property, value string) (bool, error) { } // The key is gone, so the field name must leave the index too. c.removeFieldLocked(ctx, hashID, property) + c.markDirtyLocked(ctx, hashID) return true, nil } newValue := strings.Join(newValues, "|") - return true, c.client.Set(ctx, key, newValue, 0).Err() + err = c.client.Set(ctx, key, newValue, 0).Err() + // The field survives with fewer members, so no field-index change — but the + // VALUE changed, which is exactly what the search index caches. + c.markDirtyLocked(ctx, hashID) + return true, err } // FileTuple holds the (uuid, path, size, mtime, midhash) record for one @@ -691,6 +706,44 @@ type SearchTuple struct { // haystack — must stay in sync with the fields the search handler documents. var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"} +// defaultIndexExcludePrefixes are field-name prefixes kept OUT of the in-memory +// search index. Redis still stores them and every per-record read still returns +// them — this narrows the *index*, never the store. +// +// `categories/newznab/` is the raw Newznab numeric category code, which +// METADATA_KEYS documents as a mirror of the canonical `categories/