566 lines
18 KiB
Go
566 lines
18 KiB
Go
package storage
|
||
|
||
// Search over the KV store.
|
||
//
|
||
// The Redis era answered POST /api/metadata/search from an in-memory copy of
|
||
// every record (the API's searchIndex), because Redis could not filter
|
||
// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
|
||
// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
|
||
// gone: search now runs against two derived structures inside the database.
|
||
//
|
||
// - kv_field_v — an expression index over (lowercased field name parsed out
|
||
// of the key, lowercased value, value), covering only record keys. It answers "which
|
||
// records have field F with a value like V" for ANY field, including ones
|
||
// invented tomorrow: the data stays schema-free, the index is over the key
|
||
// itself.
|
||
// - fts — an FTS5 trigram table holding one row per record: the record's
|
||
// haystack (buildHaystack) plus its rid. Trigrams make every substring of
|
||
// 3+ characters indexable, which is the matching rule search always had.
|
||
//
|
||
// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
|
||
// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
|
||
// semantics are bit-identical to the in-memory matcher and nothing depends on
|
||
// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.
|
||
|
||
import (
|
||
"context"
|
||
"encoding/json"
|
||
"fmt"
|
||
"os"
|
||
"sort"
|
||
"strconv"
|
||
"strings"
|
||
"unicode/utf8"
|
||
|
||
"github.com/metazla/meta-core/internal/kv"
|
||
)
|
||
|
||
// The field expression and the partial-index predicate MUST appear verbatim
|
||
// (as literals, never bound parameters) in every query meant to use kv_field_v —
|
||
// SQLite only matches a partial index whose WHERE clause it can see implied by
|
||
// the query text. TestSearchFilterUsesFieldIndex pins this.
|
||
const (
|
||
fieldExpr = `lower(substr(k, instr(k,'/')+1))`
|
||
recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
|
||
ridExpr = `substr(k, 6, instr(k,'/')-6)`
|
||
)
|
||
|
||
// searchSchema is the derived DDL applied when the database is opened.
|
||
//
|
||
// kv_field_v carries the raw value as a third column so the filters' substring
|
||
// test (instr(lower(v), …)) is answered from the index alone. Without it,
|
||
// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
|
||
// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
|
||
// lower(v)), which is dropped here; the rebuild costs ~80 s once.
|
||
const searchSchema = `
|
||
DROP INDEX IF EXISTS kv_field;
|
||
CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
|
||
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
|
||
`
|
||
|
||
// searchIndexFields are the metadata fields folded into a record's search
|
||
// haystack — must stay in sync with the fields the search handler documents.
|
||
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
|
||
|
||
// affectsHaystack reports whether writing field can change the haystack.
|
||
func affectsHaystack(field string) bool {
|
||
if strings.HasPrefix(field, "titles/") {
|
||
return true
|
||
}
|
||
for _, f := range searchIndexFields {
|
||
if field == f {
|
||
return true
|
||
}
|
||
}
|
||
return false
|
||
}
|
||
|
||
// Haystack folds a record's searchable text fields into one lowercased blob —
|
||
// what free-text tokens must each be a substring of.
|
||
//
|
||
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
|
||
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
|
||
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
|
||
// record whose `title` is `3%`.
|
||
func Haystack(fields map[string]string) string {
|
||
var sb strings.Builder
|
||
for _, f := range searchIndexFields {
|
||
if v, ok := fields[f]; ok && v != "" {
|
||
sb.WriteString(strings.ToLower(v))
|
||
sb.WriteByte('\n')
|
||
}
|
||
}
|
||
for _, name := range titleMemberNames(fields) {
|
||
sb.WriteString(strings.ToLower(name))
|
||
sb.WriteByte('\n')
|
||
}
|
||
return sb.String()
|
||
}
|
||
|
||
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
|
||
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
|
||
// back. Sorted so the haystack is deterministic.
|
||
func titleMemberNames(fields map[string]string) []string {
|
||
var names []string
|
||
for k, v := range fields {
|
||
rest, ok := strings.CutPrefix(k, "titles/")
|
||
if !ok || v == "" || v == "false" {
|
||
continue
|
||
}
|
||
_, name, ok := strings.Cut(rest, "/")
|
||
if !ok || name == "" {
|
||
continue
|
||
}
|
||
names = append(names, strings.ReplaceAll(name, "∕", "/"))
|
||
}
|
||
sort.Strings(names)
|
||
return names
|
||
}
|
||
|
||
// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
|
||
// "query is a substring of the hashID" rule also has an indexed candidate path.
|
||
// Verification checks the two separately (a rid token plus a title token must
|
||
// not match together).
|
||
func ftsHay(hashID string, fields map[string]string) string {
|
||
return Haystack(fields) + strings.ToLower(hashID) + "\n"
|
||
}
|
||
|
||
// refreshSearchTx rewrites a record's FTS row from its current fields, or
|
||
// drops it when the record is gone.
|
||
func refreshSearchTx(tx *kv.Tx, hashID string) error {
|
||
fields, err := readRecord(tx.Context(), tx, hashID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if len(fields) == 0 {
|
||
return deleteSearchRowTx(tx, hashID)
|
||
}
|
||
return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
|
||
}
|
||
|
||
// ensureSearchRowTx creates the record's FTS row if it has none (a record
|
||
// whose writes never touched a haystack field still needs one for the rid
|
||
// rule).
|
||
func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
|
||
_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||
if err != nil || ok {
|
||
return err
|
||
}
|
||
return refreshSearchTx(tx, hashID)
|
||
}
|
||
|
||
func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
|
||
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||
if err != nil {
|
||
return err
|
||
}
|
||
if ok {
|
||
if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
|
||
return fmt.Errorf("fts update %s: %w", hashID, err)
|
||
}
|
||
return nil
|
||
}
|
||
res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
|
||
if err != nil {
|
||
return fmt.Errorf("fts insert %s: %w", hashID, err)
|
||
}
|
||
rowid, err := res.LastInsertId()
|
||
if err != nil {
|
||
return err
|
||
}
|
||
return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
|
||
}
|
||
|
||
func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
|
||
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||
if err != nil || !ok {
|
||
return err
|
||
}
|
||
if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
|
||
return fmt.Errorf("fts delete %s: %w", hashID, err)
|
||
}
|
||
_, err = tx.Delete(ftsNS + hashID)
|
||
return err
|
||
}
|
||
|
||
// SearchFilter is one structured filter: AND across filters, OR across Values.
|
||
type SearchFilter struct {
|
||
Property string
|
||
Values []string
|
||
}
|
||
|
||
// SearchHit is one verified result.
|
||
type SearchHit struct {
|
||
HashID string
|
||
Fields map[string]string
|
||
}
|
||
|
||
// Search returns up to limit records, in hashID order, that satisfy match.
|
||
//
|
||
// query and filters only NARROW the candidates (SQL, superset semantics);
|
||
// match is the authoritative test, run on each hydrated record. Pass the same
|
||
// query/filters match evaluates, or results will be missing.
|
||
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if limit <= 0 {
|
||
limit = 100
|
||
}
|
||
ctx, cancel := opCtx()
|
||
defer cancel()
|
||
|
||
driver, probes, err := c.planCandidates(ctx, s, query, filters)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
page := max(limit*2, 500)
|
||
hits := make([]SearchHit, 0, limit)
|
||
after := ""
|
||
for len(hits) < limit {
|
||
ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
survivors := ids
|
||
if len(probes) > 0 && len(ids) > 0 {
|
||
if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
|
||
return nil, err
|
||
}
|
||
}
|
||
for _, id := range survivors {
|
||
fields, err := readRecord(ctx, s, id)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
if fields == nil || !match(id, fields) {
|
||
continue
|
||
}
|
||
hits = append(hits, SearchHit{HashID: id, Fields: fields})
|
||
if len(hits) >= limit {
|
||
break
|
||
}
|
||
}
|
||
if len(ids) < page {
|
||
break
|
||
}
|
||
after = ids[len(ids)-1]
|
||
}
|
||
return hits, nil
|
||
}
|
||
|
||
// queryIDs returns one page of the driver's candidates, in rid order.
|
||
func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
|
||
q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
|
||
rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("search candidates: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
var out []string
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, err
|
||
}
|
||
out = append(out, id)
|
||
}
|
||
return out, rows.Err()
|
||
}
|
||
|
||
// probeSQL applies the non-driving filters to one page of candidate rids
|
||
// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
|
||
func probeSQL(probes []candidatePart) (string, []any) {
|
||
where := make([]string, len(probes))
|
||
var args []any
|
||
for i, p := range probes {
|
||
where[i] = p.exists
|
||
args = append(args, p.existsArgs...)
|
||
}
|
||
return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
|
||
}
|
||
|
||
// probeIDs keeps the ids (in their order) that pass every probe.
|
||
func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
|
||
q, args := probeSQL(probes)
|
||
list, err := json.Marshal(ids)
|
||
if err != nil {
|
||
return nil, err
|
||
}
|
||
rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
|
||
if err != nil {
|
||
return nil, fmt.Errorf("search probes: %w", err)
|
||
}
|
||
defer rows.Close()
|
||
pass := make(map[string]bool, len(ids))
|
||
for rows.Next() {
|
||
var id string
|
||
if err := rows.Scan(&id); err != nil {
|
||
return nil, err
|
||
}
|
||
pass[id] = true
|
||
}
|
||
if err := rows.Err(); err != nil {
|
||
return nil, err
|
||
}
|
||
out := ids[:0:0]
|
||
for _, id := range ids {
|
||
if pass[id] {
|
||
out = append(out, id)
|
||
}
|
||
}
|
||
return out, nil
|
||
}
|
||
|
||
// candidatePart is one narrowing condition: free text, or one filter.
|
||
type candidatePart struct {
|
||
sql string // SELECT … AS rid
|
||
args []any
|
||
// exists is the same filter as a predicate on candidate row c.rid (empty
|
||
// for free text, which has no per-record probe: the FTS rid column is not
|
||
// indexed).
|
||
exists string
|
||
existsArgs []any
|
||
}
|
||
|
||
// estimateCap bounds the count used to pick the driving part: it only has to
|
||
// tell a 50-row part from a 100k-row one.
|
||
const estimateCap = 1000
|
||
|
||
// planCandidates picks how to narrow a search: one DRIVING part (the most
|
||
// selective, by a capped count) that lists candidate rids in order, plus PROBES
|
||
// — every other filter, checked as an EXISTS over each candidate's own key
|
||
// range (a primary-key range read of ~40 rows).
|
||
//
|
||
// Search pages the driver in Go and probes one page at a time, so a broad
|
||
// query stops as soon as `limit` records verify. Two earlier shapes were
|
||
// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
|
||
// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
|
||
// and probing inside the driver query (SQLite evaluates the probe on every
|
||
// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
|
||
// both ~100k, 10 s). Free text that does not drive is dropped from SQL
|
||
// entirely — the verifier still applies it, and dropping a condition only
|
||
// widens the superset.
|
||
func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
|
||
var parts []candidatePart
|
||
if p, ok := textCandidates(query); ok {
|
||
parts = append(parts, p)
|
||
}
|
||
for _, f := range filters {
|
||
if p, ok := filterCandidates(f); ok {
|
||
parts = append(parts, p)
|
||
}
|
||
}
|
||
if len(parts) == 0 {
|
||
return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
|
||
}
|
||
|
||
driver := 0
|
||
if len(parts) > 1 {
|
||
best := int64(-1)
|
||
for i, p := range parts {
|
||
var n int64
|
||
q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
|
||
if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
|
||
return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
|
||
}
|
||
if best < 0 || n < best {
|
||
best, driver = n, i
|
||
}
|
||
}
|
||
}
|
||
var probes []candidatePart
|
||
for i, p := range parts {
|
||
if i != driver && p.exists != "" {
|
||
probes = append(probes, p)
|
||
}
|
||
}
|
||
return parts[driver], probes, nil
|
||
}
|
||
|
||
// candidateSQL is planCandidates without the store: text plus filters, the
|
||
// first part driving. Used by tests that inspect the plan.
|
||
func candidateSQL(query string, filters []SearchFilter) (string, []any) {
|
||
var parts []candidatePart
|
||
if p, ok := textCandidates(query); ok {
|
||
parts = append(parts, p)
|
||
}
|
||
for _, f := range filters {
|
||
if p, ok := filterCandidates(f); ok {
|
||
parts = append(parts, p)
|
||
}
|
||
}
|
||
if len(parts) == 0 {
|
||
return "", nil
|
||
}
|
||
return parts[0].sql, parts[0].args
|
||
}
|
||
|
||
// textCandidates narrows by free text: every whitespace token must be a
|
||
// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
|
||
// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
|
||
// same rows.
|
||
func textCandidates(query string) (candidatePart, bool) {
|
||
tokens := strings.Fields(strings.ToLower(query))
|
||
if len(tokens) == 0 {
|
||
return candidatePart{}, false
|
||
}
|
||
var match []string
|
||
var likes []string
|
||
var likeArgs []any
|
||
for _, t := range tokens {
|
||
if utf8.RuneCountInString(t) >= 3 {
|
||
match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
|
||
} else {
|
||
likes = append(likes, `hay LIKE ? ESCAPE '\'`)
|
||
likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
|
||
}
|
||
}
|
||
var where []string
|
||
var args []any
|
||
if len(match) > 0 {
|
||
where = append(where, `fts MATCH ?`)
|
||
args = append(args, strings.Join(match, " "))
|
||
}
|
||
where = append(where, likes...)
|
||
args = append(args, likeArgs...)
|
||
return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
|
||
}
|
||
|
||
func escapeLike(s string) string {
|
||
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
||
return r.Replace(s)
|
||
}
|
||
|
||
// filterCandidates narrows by one filter — the UNION, over its values, of the
|
||
// two storage shapes recordMatchesFilter probes:
|
||
//
|
||
// - key-set member `<prop>/<value>` (any case): the lowercased field equals
|
||
// lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
|
||
// lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
|
||
// comparison could miss);
|
||
// - flat field `<prop>` whose lowercased value contains the lowercased value
|
||
// — covers exact and contains alike, and comma-split csv-set values.
|
||
//
|
||
// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
|
||
// name): it constrains nothing in SQL and is left to the verifier.
|
||
func filterCandidates(f SearchFilter) (candidatePart, bool) {
|
||
var values []string
|
||
for _, v := range f.Values {
|
||
if v = strings.TrimSpace(v); v != "" {
|
||
values = append(values, v)
|
||
}
|
||
}
|
||
if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
|
||
return candidatePart{}, false
|
||
}
|
||
prop := asciiLower(f.Property)
|
||
var conds []string
|
||
var args []any
|
||
memberRange := false
|
||
for _, v := range values {
|
||
if isASCII(v) {
|
||
lv := asciiLower(v)
|
||
conds = append(conds, fieldExpr+` = ?`)
|
||
args = append(args, prop+"/"+lv)
|
||
conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
|
||
args = append(args, prop, lv)
|
||
} else {
|
||
memberRange = true
|
||
conds = append(conds, fieldExpr+` = ?`)
|
||
args = append(args, prop)
|
||
}
|
||
}
|
||
if memberRange {
|
||
conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
|
||
args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
|
||
}
|
||
|
||
// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
|
||
// OR inside one SELECT would defeat the partial index; plain UNION would
|
||
// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
|
||
// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
|
||
base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
|
||
selects := make([]string, len(conds))
|
||
for i, cond := range conds {
|
||
selects[i] = base + cond
|
||
}
|
||
// As a probe: the same conditions over one candidate's key range.
|
||
exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
|
||
strings.Join(conds, ") OR (") + `)))`
|
||
return candidatePart{
|
||
sql: strings.Join(selects, " UNION ALL "), args: args,
|
||
exists: exists, existsArgs: args,
|
||
}, true
|
||
}
|
||
|
||
func isASCII(s string) bool {
|
||
for i := 0; i < len(s); i++ {
|
||
if s[i] >= utf8.RuneSelf {
|
||
return false
|
||
}
|
||
}
|
||
return true
|
||
}
|
||
|
||
// asciiLower lowercases A–Z only, exactly like SQLite's lower().
|
||
func asciiLower(s string) string {
|
||
b := []byte(s)
|
||
for i, c := range b {
|
||
if 'A' <= c && c <= 'Z' {
|
||
b[i] = c + ('a' - 'A')
|
||
}
|
||
}
|
||
return string(b)
|
||
}
|
||
|
||
func envInt(name string, def int) int {
|
||
if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
|
||
return v
|
||
}
|
||
return def
|
||
}
|
||
|
||
// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
|
||
// empty but records exist — a store restored from a snapshot, a copy made
|
||
// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
|
||
// change its tokenizer. The rows are derived, so this is a repair, not a
|
||
// migration. Runs in bounded batches so writes keep flowing; until it
|
||
// finishes, free-text search under-matches (filters are unaffected).
|
||
func (c *Client) RebuildSearchIfEmpty() (int, error) {
|
||
s, err := c.kv()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
ctx, cancel := opCtx()
|
||
var rows int64
|
||
err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
|
||
cancel()
|
||
if err != nil {
|
||
return 0, fmt.Errorf("count fts rows: %w", err)
|
||
}
|
||
if rows > 0 {
|
||
return 0, nil
|
||
}
|
||
ids, err := c.GetAllHashIDs()
|
||
if err != nil {
|
||
return 0, err
|
||
}
|
||
const batch = 500
|
||
for start := 0; start < len(ids); start += batch {
|
||
chunk := ids[start:min(start+batch, len(ids))]
|
||
if err := c.update(func(tx *kv.Tx) error {
|
||
for _, id := range chunk {
|
||
if err := refreshSearchTx(tx, id); err != nil {
|
||
return err
|
||
}
|
||
}
|
||
return nil
|
||
}); err != nil {
|
||
return start, fmt.Errorf("rebuild search rows: %w", err)
|
||
}
|
||
}
|
||
return len(ids), nil
|
||
}
|