meta-core/internal/api/search_index.go

package api

import (
	"net/http"
	"strings"

	"github.com/metazla/meta-core/internal/storage"
)

// serveIndexedSearch answers a search from the database's derived indexes
// (storage.Search): SQL narrows to a superset of candidates through the field
// expression index and the FTS5 trigram table, then every candidate is checked
// here with the exact matchers below — a token-AND match against the record's
// haystack (or a whole-query hashID substring, preserving the legacy
// behaviour) AND every structured filter.
//
// Until 2026-10 this ran against an in-memory copy of every record, rebuilt
// from Redis; that copy cost ~30 KB of RAM per record (5.1 GB on watch.nsl.sh)
// and is gone. The matchers did not change, so neither did the semantics.
//
// This is the ONLY search path. Free text, filters, both, or neither all land
// here; only the `hashId` exact-match branch in handleSearchMetadata bypasses it.
func (s *Server) serveIndexedSearch(w http.ResponseWriter, req MetadataSearchRequest) {
	tokens := strings.Fields(strings.ToLower(req.Query))
	queryLower := strings.ToLower(req.Query)
	filters := normalizedFilters(req)

	narrow := make([]storage.SearchFilter, 0, len(filters))
	for _, f := range filters {
		narrow = append(narrow, storage.SearchFilter{Property: f.Property, Values: f.Values})
	}
	hits, err := s.storage.SearchAs(req.Query, narrow, req.Limit, req.As, func(hashID string, fields map[string]string) bool {
		if req.Query != "" &&
			!matchAllTokens(storage.Haystack(fields), tokens) &&
			!strings.Contains(strings.ToLower(hashID), queryLower) {
			return false
		}
		return recordMatchesFilters(fields, filters)
	})
	if err != nil {
		writeError(w, http.StatusInternalServerError, "search failed: "+err.Error())
		return
	}
	total, err := s.storage.CountRoots()
	if err != nil {
		total = 0
	}

	results := make([]MetadataSearchResult, 0, len(hits))
	for _, h := range hits {
		results = append(results, MetadataSearchResult{HashID: h.HashID, Metadata: h.Fields})
	}
	writeJSON(w, http.StatusOK, MetadataSearchResponse{
		Results: results,
		Count:   len(results),
		Total:   int(total),
	})
}

// matchAllTokens reports whether haystack contains every token (AND semantics).
// An empty token list matches everything.
func matchAllTokens(haystack string, tokens []string) bool {
	for _, t := range tokens {
		if !strings.Contains(haystack, t) {
			return false
		}
	}
	return true
}

// normalizedFilters returns the request's structured filters with the legacy
// Property/PropertyValue pair folded in as one more entry.
//
// The legacy pair keeps Exact=false so its long-standing substring semantics
// are unchanged for existing callers; new callers pass Filters and choose.
func normalizedFilters(req MetadataSearchRequest) []MetadataFilter {
	if req.Property == "" || req.PropertyValue == "" {
		return req.Filters
	}
	out := make([]MetadataFilter, 0, len(req.Filters)+1)
	out = append(out, req.Filters...)
	return append(out, MetadataFilter{
		Property: req.Property,
		Values:   []string{req.PropertyValue},
	})
}

// recordMatchesFilters reports whether fields satisfies every filter (AND).
func recordMatchesFilters(fields map[string]string, filters []MetadataFilter) bool {
	for i := range filters {
		if !recordMatchesFilter(fields, &filters[i]) {
			return false
		}
	}
	return true
}

// recordMatchesFilter reports whether fields satisfies one filter (OR within
// its values), probing BOTH storage shapes a field can take:
//
//   - a flat scalar/csv-set field — `fileType = "video"`, `videoType = "tv,anime"`;
//   - a key-set member — `genres/Action = "true"`, `source/gateway:nyaa.si = "true"`
//     (METADATA_KEYS "How the store is shaped").
//
// Probing both is what lets one filter shape serve the whole vocabulary. The
// gateway needs a `genres:action` filter to find a record whose genre lives in
// the key `genres/Action`, and a `fileType:video` filter to find one whose value
// lives in the field `fileType` — without meta-core knowing which fields are
// key-sets. It also means this stays a SUPERSET of the gateway's own
// `record_matches`, which is the contract: meta-core pre-filters, the gateway
// re-filters authoritatively.
//
// A filter with no usable value constrains nothing (matches). A filter whose
// field is absent fails closed, matching `record_matches`.
func recordMatchesFilter(fields map[string]string, f *MetadataFilter) bool {
	values := make([]string, 0, len(f.Values))
	for _, v := range f.Values {
		if v = strings.TrimSpace(v); v != "" {
			values = append(values, v)
		}
	}
	if f.Property == "" || len(values) == 0 {
		return true
	}

	// Key-set shape: `<property>/<value>` present and truthy. Try the exact
	// key first — that hits whenever the caller's casing matches what was
	// written, which is the common case and costs one map probe.
	for _, v := range values {
		if isTruthyMember(fields[f.Property+"/"+v]) {
			return true
		}
	}
	// Members are written verbatim (`genres/Action`) while query values arrive
	// lowercased, so fall back to one case-insensitive sweep. Single pass over
	// the record's fields for ALL values, not one pass per value.
	wanted := make(map[string]struct{}, len(values))
	for _, v := range values {
		wanted[strings.ToLower(v)] = struct{}{}
	}
	prefix := strings.ToLower(f.Property) + "/"
	for k, val := range fields {
		if len(k) <= len(prefix) || !strings.EqualFold(k[:len(prefix)], prefix) {
			continue
		}
		if _, ok := wanted[strings.ToLower(k[len(prefix):])]; ok && isTruthyMember(val) {
			return true
		}
	}

	// Flat shape: the field's own value, split on `,` for csv-set fields.
	fieldValue, ok := fields[f.Property]
	if !ok {
		return false
	}
	for _, piece := range strings.Split(fieldValue, ",") {
		piece = strings.TrimSpace(piece)
		if piece == "" {
			continue
		}
		for _, v := range values {
			if f.Exact {
				if strings.EqualFold(piece, v) {
					return true
				}
			} else if strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
				return true
			}
		}
	}
	return false
}

// isTruthyMember reports whether a key-set member's value asserts membership.
// Mirrors the gateway's `is_truthy_member`: presence with a non-empty,
// non-"false"/"0" value.
func isTruthyMember(v string) bool {
	v = strings.TrimSpace(strings.ToLower(v))
	return v != "" && v != "false" && v != "0"
}