meta-core/internal/storage/search.go

package storage

// Search over the KV store.
//
// The Redis era answered POST /api/metadata/search from an in-memory copy of
// every record (the API's searchIndex), because Redis could not filter
// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
// gone: search now runs against two derived structures inside the database.
//
//   - kv_field_v — an expression index over (lowercased field name parsed out
//     of the key, lowercased value, value), covering only record keys. It answers "which
//     records have field F with a value like V" for ANY field, including ones
//     invented tomorrow: the data stays schema-free, the index is over the key
//     itself.
//   - fts — an FTS5 trigram table holding one row per record: the record's
//     haystack (buildHaystack) plus its rid. Trigrams make every substring of
//     3+ characters indexable, which is the matching rule search always had.
//
// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
// semantics are bit-identical to the in-memory matcher and nothing depends on
// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.

import (
	"context"
	"encoding/json"
	"fmt"
	"os"
	"sort"
	"strconv"
	"strings"
	"unicode/utf8"

	"github.com/metazla/meta-core/internal/kv"
)

// The field expression and the partial-index predicate MUST appear verbatim
// (as literals, never bound parameters) in every query meant to use kv_field_v —
// SQLite only matches a partial index whose WHERE clause it can see implied by
// the query text. TestSearchFilterUsesFieldIndex pins this.
const (
	fieldExpr  = `lower(substr(k, instr(k,'/')+1))`
	recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
	ridExpr    = `substr(k, 6, instr(k,'/')-6)`
)

// searchSchema is the derived DDL applied when the database is opened.
//
// kv_field_v carries the raw value as a third column so the filters' substring
// test (instr(lower(v), …)) is answered from the index alone. Without it,
// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
// lower(v)), which is dropped here; the rebuild costs ~80 s once.
const searchSchema = `
DROP INDEX IF EXISTS kv_field;
CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
`

// searchIndexFields are the metadata fields folded into a record's search
// haystack — must stay in sync with the fields the search handler documents.
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}

// affectsHaystack reports whether writing field can change the haystack.
func affectsHaystack(field string) bool {
	if strings.HasPrefix(field, "titles/") {
		return true
	}
	for _, f := range searchIndexFields {
		if field == f {
			return true
		}
	}
	return false
}

// Haystack folds a record's searchable text fields into one lowercased blob —
// what free-text tokens must each be a substring of.
//
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
// record whose `title` is `3%`.
func Haystack(fields map[string]string) string {
	var sb strings.Builder
	for _, f := range searchIndexFields {
		if v, ok := fields[f]; ok && v != "" {
			sb.WriteString(strings.ToLower(v))
			sb.WriteByte('\n')
		}
	}
	for _, name := range titleMemberNames(fields) {
		sb.WriteString(strings.ToLower(name))
		sb.WriteByte('\n')
	}
	return sb.String()
}

// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
// back. Sorted so the haystack is deterministic.
func titleMemberNames(fields map[string]string) []string {
	var names []string
	for k, v := range fields {
		rest, ok := strings.CutPrefix(k, "titles/")
		if !ok || v == "" || v == "false" {
			continue
		}
		_, name, ok := strings.Cut(rest, "/")
		if !ok || name == "" {
			continue
		}
		names = append(names, strings.ReplaceAll(name, "∕", "/"))
	}
	sort.Strings(names)
	return names
}

// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
// "query is a substring of the hashID" rule also has an indexed candidate path.
// Verification checks the two separately (a rid token plus a title token must
// not match together).
func ftsHay(hashID string, fields map[string]string) string {
	return Haystack(fields) + strings.ToLower(hashID) + "\n"
}

// refreshSearchTx rewrites a record's FTS row from its current fields, or
// drops it when the record is gone.
func refreshSearchTx(tx *kv.Tx, hashID string) error {
	fields, err := readRecord(tx.Context(), tx, hashID)
	if err != nil {
		return err
	}
	if len(fields) == 0 {
		return deleteSearchRowTx(tx, hashID)
	}
	return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
}

// ensureSearchRowTx creates the record's FTS row if it has none (a record
// whose writes never touched a haystack field still needs one for the rid
// rule).
func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
	_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
	if err != nil || ok {
		return err
	}
	return refreshSearchTx(tx, hashID)
}

func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
	rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
	if err != nil {
		return err
	}
	if ok {
		if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
			return fmt.Errorf("fts update %s: %w", hashID, err)
		}
		return nil
	}
	res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
	if err != nil {
		return fmt.Errorf("fts insert %s: %w", hashID, err)
	}
	rowid, err := res.LastInsertId()
	if err != nil {
		return err
	}
	return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
}

func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
	rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
	if err != nil || !ok {
		return err
	}
	if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
		return fmt.Errorf("fts delete %s: %w", hashID, err)
	}
	_, err = tx.Delete(ftsNS + hashID)
	return err
}

// SearchFilter is one structured filter: AND across filters, OR across Values.
type SearchFilter struct {
	Property string
	Values   []string
}

// SearchHit is one verified result.
type SearchHit struct {
	HashID string
	Fields map[string]string
}

// Search returns up to limit records, in hashID order, that satisfy match.
//
// query and filters only NARROW the candidates (SQL, superset semantics);
// match is the authoritative test, run on each hydrated record. Pass the same
// query/filters match evaluates, or results will be missing.
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
	s, err := c.kv()
	if err != nil {
		return nil, err
	}
	if limit <= 0 {
		limit = 100
	}
	ctx, cancel := opCtx()
	defer cancel()

	plan, err := c.planCandidates(ctx, s, query, filters)
	if err != nil {
		return nil, err
	}
	page := max(limit*2, 500)
	hits := make([]SearchHit, 0, limit)
	after := ""
	for pos := 0; len(hits) < limit; {
		var ids []string
		if plan.walk {
			rows, err := s.Range(ctx, rootsNS, after, page)
			if err != nil {
				return nil, err
			}
			for _, row := range rows {
				ids = append(ids, row.K[len(rootsNS):])
			}
			if len(rows) > 0 {
				after = rows[len(rows)-1].K
			}
		} else {
			ids = plan.ordered[pos:min(pos+page, len(plan.ordered))]
			pos += len(ids)
		}
		if len(ids) == 0 {
			break
		}
		survivors := ids
		if len(plan.probes) > 0 {
			if survivors, err = probeIDs(ctx, s, ids, plan.probes); err != nil {
				return nil, err
			}
		}
		for _, id := range survivors {
			fields, err := readRecord(ctx, s, id)
			if err != nil {
				return nil, err
			}
			if fields == nil || !match(id, fields) {
				continue
			}
			hits = append(hits, SearchHit{HashID: id, Fields: fields})
			if len(hits) >= limit {
				break
			}
		}
		if len(ids) < page {
			break
		}
	}
	return hits, nil
}

// probeSQL applies the non-driving filters to one page of candidate rids
// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
func probeSQL(probes []candidatePart) (string, []any) {
	where := make([]string, len(probes))
	var args []any
	for i, p := range probes {
		where[i] = p.exists
		args = append(args, p.existsArgs...)
	}
	return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
}

// probeIDs keeps the ids (in their order) that pass every probe.
func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
	q, args := probeSQL(probes)
	list, err := json.Marshal(ids)
	if err != nil {
		return nil, err
	}
	rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
	if err != nil {
		return nil, fmt.Errorf("search probes: %w", err)
	}
	defer rows.Close()
	pass := make(map[string]bool, len(ids))
	for rows.Next() {
		var id string
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		pass[id] = true
	}
	if err := rows.Err(); err != nil {
		return nil, err
	}
	out := make([]string, 0, len(ids))
	for _, id := range ids {
		if pass[id] {
			out = append(out, id)
		}
	}
	return out, nil
}

// candidatePart is one narrowing condition: free text, or one filter.
type candidatePart struct {
	sql  string // SELECT … AS rid
	args []any
	// exists is the same filter as a predicate on candidate row c.rid (empty
	// for free text, which has no per-record probe: the FTS rid column is not
	// indexed).
	exists     string
	existsArgs []any
}

// estimateCap bounds the count used to rank parts by selectivity: past it, a
// part is "broad" and only ties with other broad parts are left to resolve.
var estimateCap = 50_000

// estimateCapForTest sets estimateCap and returns the previous value.
func estimateCapForTest(n int) int { old := estimateCap; estimateCap = n; return old }

// searchPlan is how one search walks its candidates.
type searchPlan struct {
	ordered []string        // the driver's rids, sorted (when !walk)
	walk    bool            // no part narrows: walk idx:roots in order
	probes  []candidatePart // filters checked per page as EXISTS
}

// planCandidates picks how to narrow a search.
//
// A capped count ranks the parts; the SMALLEST is collected once, sorted in Go
// and paged from memory, and every other filter becomes a probe — an EXISTS
// over each candidate's own key range, run one page at a time, so a search
// stops as soon as `limit` records verify. The driver is collected in full even
// when every part is broad: that costs O(part) — at most every root, ~0.3 s at
// 169k — whereas walking idx:roots in hashID order (tried) costs O(corpus) when
// matches are sparse early in that order, and on watch.nsl.sh they are: the
// first 500 roots held zero episodes, so `contentKind:episode` walked ~3 s.
// Only a search with nothing to narrow walks the roots, where every root
// matches.
//
// Also rejected, measured on watch.nsl.sh: INTERSECTing fully materialised
// sets (5.7 s for `naruto` + `fileType:video`); probing inside a driver query
// re-sorted per page (SQLite evaluates the probe on every driver row before
// ORDER BY … LIMIT can stop: 10 s for `1080p` + `fileType:video`). Free text
// that does not drive is dropped from SQL — the verifier still applies it, and
// dropping a condition only widens the superset.
func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (searchPlan, error) {
	var parts []candidatePart
	if p, ok := textCandidates(query); ok {
		parts = append(parts, p)
	}
	for _, f := range filters {
		if p, ok := filterCandidates(f); ok {
			parts = append(parts, p)
		}
	}
	if len(parts) == 0 {
		return searchPlan{walk: true}, nil
	}

	driver := 0
	if len(parts) > 1 {
		best := int64(-1)
		for i, p := range parts {
			var n int64
			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap+1) + `)`
			if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
				return searchPlan{}, fmt.Errorf("estimate candidates: %w", err)
			}
			if best < 0 || n < best {
				best, driver = n, i
			}
			if n == 0 {
				break // nothing can match
			}
		}
	}
	ids, err := collectIDs(ctx, s, parts[driver], 0)
	if err != nil {
		return searchPlan{}, err
	}
	sort.Strings(ids)
	plan := searchPlan{ordered: ids}
	for i, p := range parts {
		if i != driver && p.exists != "" {
			plan.probes = append(plan.probes, p)
		}
	}
	return plan, nil
}

// collectIDs reads the distinct rids of one part (all of them when limit <= 0).
func collectIDs(ctx context.Context, s *kv.Store, p candidatePart, limit int) ([]string, error) {
	q := `SELECT DISTINCT rid FROM (` + p.sql + `)`
	args := append([]any{}, p.args...)
	if limit > 0 {
		q += ` LIMIT ?`
		args = append(args, limit)
	}
	rows, err := s.DB().QueryContext(ctx, q, args...)
	if err != nil {
		return nil, fmt.Errorf("search candidates: %w", err)
	}
	defer rows.Close()
	var out []string
	for rows.Next() {
		var id string
		if err := rows.Scan(&id); err != nil {
			return nil, err
		}
		out = append(out, id)
	}
	return out, rows.Err()
}

// candidateSQL is the first part's SQL, for tests that inspect the plan.
func candidateSQL(query string, filters []SearchFilter) (string, []any) {
	if p, ok := textCandidates(query); ok {
		return p.sql, p.args
	}
	for _, f := range filters {
		if p, ok := filterCandidates(f); ok {
			return p.sql, p.args
		}
	}
	return "", nil
}

// textCandidates narrows by free text: every whitespace token must be a
// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
// same rows.
func textCandidates(query string) (candidatePart, bool) {
	tokens := strings.Fields(strings.ToLower(query))
	if len(tokens) == 0 {
		return candidatePart{}, false
	}
	var match []string
	var likes []string
	var likeArgs []any
	for _, t := range tokens {
		if utf8.RuneCountInString(t) >= 3 {
			match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
		} else {
			likes = append(likes, `hay LIKE ? ESCAPE '\'`)
			likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
		}
	}
	var where []string
	var args []any
	if len(match) > 0 {
		where = append(where, `fts MATCH ?`)
		args = append(args, strings.Join(match, " "))
	}
	where = append(where, likes...)
	args = append(args, likeArgs...)
	return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
}

func escapeLike(s string) string {
	r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
	return r.Replace(s)
}

// filterCandidates narrows by one filter — the UNION, over its values, of the
// two storage shapes recordMatchesFilter probes:
//
//   - key-set member `<prop>/<value>` (any case): the lowercased field equals
//     lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
//     lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
//     comparison could miss);
//   - flat field `<prop>` whose lowercased value contains the lowercased value
//     — covers exact and contains alike, and comma-split csv-set values.
//
// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
// name): it constrains nothing in SQL and is left to the verifier.
func filterCandidates(f SearchFilter) (candidatePart, bool) {
	var values []string
	for _, v := range f.Values {
		if v = strings.TrimSpace(v); v != "" {
			values = append(values, v)
		}
	}
	if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
		return candidatePart{}, false
	}
	prop := asciiLower(f.Property)
	var conds []string
	var args []any
	memberRange := false
	for _, v := range values {
		if isASCII(v) {
			lv := asciiLower(v)
			conds = append(conds, fieldExpr+` = ?`)
			args = append(args, prop+"/"+lv)
			conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
			args = append(args, prop, lv)
		} else {
			memberRange = true
			conds = append(conds, fieldExpr+` = ?`)
			args = append(args, prop)
		}
	}
	if memberRange {
		conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
		args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
	}

	// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
	// OR inside one SELECT would defeat the partial index; plain UNION would
	// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
	// count over fileType:video on watch.nsl.sh). collectIDs applies DISTINCT.
	base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
	selects := make([]string, len(conds))
	for i, cond := range conds {
		selects[i] = base + cond
	}
	// As a probe: the same conditions over one candidate's key range.
	exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
		strings.Join(conds, ") OR (") + `)))`
	return candidatePart{
		sql: strings.Join(selects, " UNION ALL "), args: args,
		exists: exists, existsArgs: args,
	}, true
}

func isASCII(s string) bool {
	for i := 0; i < len(s); i++ {
		if s[i] >= utf8.RuneSelf {
			return false
		}
	}
	return true
}

// asciiLower lowercases A–Z only, exactly like SQLite's lower().
func asciiLower(s string) string {
	b := []byte(s)
	for i, c := range b {
		if 'A' <= c && c <= 'Z' {
			b[i] = c + ('a' - 'A')
		}
	}
	return string(b)
}

func envInt(name string, def int) int {
	if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
		return v
	}
	return def
}

// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
// empty but records exist — a store restored from a snapshot, a copy made
// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
// change its tokenizer. The rows are derived, so this is a repair, not a
// migration. Runs in bounded batches so writes keep flowing; until it
// finishes, free-text search under-matches (filters are unaffected).
func (c *Client) RebuildSearchIfEmpty() (int, error) {
	s, err := c.kv()
	if err != nil {
		return 0, err
	}
	ctx, cancel := opCtx()
	var rows int64
	err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
	cancel()
	if err != nil {
		return 0, fmt.Errorf("count fts rows: %w", err)
	}
	if rows > 0 {
		return 0, nil
	}
	ids, err := c.GetAllHashIDs()
	if err != nil {
		return 0, err
	}
	const batch = 500
	for start := 0; start < len(ids); start += batch {
		chunk := ids[start:min(start+batch, len(ids))]
		if err := c.update(func(tx *kv.Tx) error {
			for _, id := range chunk {
				if err := refreshSearchTx(tx, id); err != nil {
					return err
				}
			}
			return nil
		}); err != nil {
			return start, fmt.Errorf("rebuild search rows: %w", err)
		}
	}
	return len(ids), nil
}