package api import ( "net/http" "strings" "github.com/metazla/meta-core/internal/storage" ) // serveIndexedSearch answers a search from the database's derived indexes // (storage.Search): SQL narrows to a superset of candidates through the field // expression index and the FTS5 trigram table, then every candidate is checked // here with the exact matchers below — a token-AND match against the record's // haystack (or a whole-query hashID substring, preserving the legacy // behaviour) AND every structured filter. // // Until 2026-10 this ran against an in-memory copy of every record, rebuilt // from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh) // and is gone. The matchers did not change, so neither did the semantics. // // This is the ONLY search path. Free text, filters, both, or neither all land // here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it. func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) { tokens := strings.Fields(strings.ToLower(req.Query)) queryLower := strings.ToLower(req.Query) filters := normalizedFilters(req) narrow := make([]storage.SearchFilter, 0, len(filters)) for _, f := range filters { narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values}) } hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool { if req.Query != "" && !matchAllTokens(storage.Haystack(fields), tokens) && !strings.Contains(strings.ToLower(hashID), queryLower) { return false } return recordMatchesFilters(fields, filters) }) if err != nil { writeError(w, http.StatusInternalServerError, "search failed: "+err.Error()) return } total, err := s.storage.CountRoots() if err != nil { total = 0 } results := make([]MetadataSearchResult, 0, len(hits)) for _, h := range hits { results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields}) } writeJSON(w, http.StatusOK, MetadataSearchResponse{ Results: results, Count: len(results), Total: int(total), }) } // matchAllTokens reports whether haystack contains every token (AND semantics). // An empty token list matches everything. func matchAllTokens(haystack string, tokens []string) bool { for _, t := range tokens { if !strings.Contains(haystack, t) { return false } } return true } // normalizedFilters returns the request's structured filters with the legacy // Property/PropertyValue pair folded in as one more entry. // // The legacy pair keeps Exact=false so its long-standing substring semantics // are unchanged for existing callers; new callers pass Filters and choose. func normalizedFilters(req MetadataSearchRequest) []MetadataFilter { if req.Property == "" || req.PropertyValue == "" { return req.Filters } out := make([]MetadataFilter, 0, len(req.Filters)+1) out = append(out, req.Filters...) return append(out, MetadataFilter{ Property: req.Property, Values: []string{req.PropertyValue}, }) } // recordMatchesFilters reports whether fields satisfies every filter (AND). func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool { for i := range filters { if !recordMatchesFilter(fields, &filters[i]) { return false } } return true } // recordMatchesFilter reports whether fields satisfies one filter (OR within // its values), probing BOTH storage shapes a field can take: // // - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`; // - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"` // (METADATA_KEYS "How the store is shaped"). // // Probing both is what lets one filter shape serve the whole vocabulary. The // gateway needs a `genres:action` filter to find a record whose genre lives in // the key `genres/Action`, and a `fileType:video` filter to find one whose value // lives in the field `fileType` — without meta-core knowing which fields are // key-sets. It also means this stays a SUPERSET of the gateway's own // `record_matches`, which is the contract: meta-core pre-filters, the gateway // re-filters authoritatively. // // A filter with no usable value constrains nothing (matches). A filter whose // field is absent fails closed, matching `record_matches`. func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool { values := make([]string, 0, len(f.Values)) for _, v := range f.Values { if v = strings.TrimSpace(v); v != "" { values = append(values, v) } } if f.Property == "" || len(values) == 0 { return true } // Key-set shape: `/` present and truthy. Try the exact // key first — that hits whenever the caller's casing matches what was // written, which is the common case and costs one map probe. for _, v := range values { if isTruthyMember(fields[f.Property+"/"+v]) { return true } } // Members are written verbatim (`genres/Action`) while query values arrive // lowercased, so fall back to one case-insensitive sweep. Single pass over // the record's fields for ALL values, not one pass per value. wanted := make(map[string]struct{}, len(values)) for _, v := range values { wanted[strings.ToLower(v)] = struct{}{} } prefix := strings.ToLower(f.Property) + "/" for k, val := range fields { if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) { continue } if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) { return true } } // Flat shape: the field's own value, split on `,` for csv-set fields. fieldValue, ok := fields[f.Property] if !ok { return false } for _, piece := range strings.Split(fieldValue, ",") { piece = strings.TrimSpace(piece) if piece == "" { continue } for _, v := range values { if f.Exact { if strings.EqualFold(piece, v) { return true } } else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) { return true } } } return false } // isTruthyMember reports whether a key-set member's value asserts membership. // Mirrors the gateway's `is_truthy_member`: presence with a non-empty, // non-"false"/"0" value. func isTruthyMember(v string) bool { v = strings.TrimSpace(strings.ToLower(v)) return v != "" && v != "false" && v != "0" }