replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)

2026-10-19 20:51:00 +00:00
parent c53146e514
commit dc5cbdfb9b
80 changed files with 4601 additions and 6077 deletions
@@ -0,0 +1,327 @@
+package storage
+
+import (
+	"context"
+	"fmt"
+	"sort"
+	"strings"
+	"testing"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+// These tests pin the one property storage.Search must have: its SQL
+// candidates are a SUPERSET of the records the exact matcher accepts. A
+// candidate miss is a silently dropped search result, so each case below is a
+// shape where SQLite and Go disagree about text (case folding, csv values,
+// FTS syntax, trigram length) and the candidate query has to cover for it.
+//
+// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
+// every candidate path; the API tests own the matcher itself.
+
+func seed(t *testing.T, c *Client, records map[string]map[string]string) {
+	t.Helper()
+	for id, fields := range records {
+		if err := c.SetMetadataFlat(id, fields); err != nil {
+			t.Fatalf("seed %s: %v", id, err)
+		}
+	}
+}
+
+// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
+func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
+	for _, v := range values {
+		if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" {
+			return true
+		}
+	}
+	prefix := strings.ToLower(prop) + "/"
+	for k, val := range fields {
+		if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) {
+			for _, v := range values {
+				if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" {
+					return true
+				}
+			}
+		}
+	}
+	fv, ok := fields[prop]
+	if !ok {
+		return false
+	}
+	for _, piece := range strings.Split(fv, ",") {
+		piece = strings.TrimSpace(piece)
+		for _, v := range values {
+			if exact && strings.EqualFold(piece, v) {
+				return true
+			}
+			if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
+				return true
+			}
+		}
+	}
+	return false
+}
+
+func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string {
+	t.Helper()
+	hits, err := c.Search(query, filters, 100, match)
+	if err != nil {
+		t.Fatalf("search: %v", err)
+	}
+	ids := make([]string, 0, len(hits))
+	for _, h := range hits {
+		ids = append(ids, h.HashID)
+	}
+	return ids
+}
+
+func textMatch(query string) func(string, map[string]string) bool {
+	tokens := strings.Fields(strings.ToLower(query))
+	return func(id string, fields map[string]string) bool {
+		hay := Haystack(fields)
+		for _, tok := range tokens {
+			if !strings.Contains(hay, tok) {
+				return strings.Contains(strings.ToLower(id), strings.ToLower(query))
+			}
+		}
+		return true
+	}
+}
+
+func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool {
+	return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) }
+}
+
+func equalIDs(t *testing.T, got []string, want ...string) {
+	t.Helper()
+	sort.Strings(want)
+	if strings.Join(got, ",") != strings.Join(want, ",") {
+		t.Fatalf("got %v, want %v", got, want)
+	}
+}
+
+func TestSearchFilterCandidates(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"},
+		"R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"},
+		"R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"},
+		"R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"},
+	})
+
+	cases := []struct {
+		name   string
+		prop   string
+		exact  bool
+		values []string
+		want   []string
+	}{
+		{"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}},
+		{"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}},
+		{"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}},
+		{"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}},
+		{"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}},
+		{"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}},
+		{"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}},
+		{"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}},
+		{"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}},
+	}
+	for _, tc := range cases {
+		t.Run(tc.name, func(t *testing.T) {
+			got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...))
+			equalIDs(t, got, tc.want...)
+		})
+	}
+}
+
+func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"R1": {"genres/Action": "true", "fileType": "video"},
+		"R2": {"genres/Action": "true", "fileType": "audio"},
+	})
+	match := func(_ string, f map[string]string) bool {
+		return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video")
+	}
+	got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match)
+	equalIDs(t, got, "R1")
+}
+
+func TestSearchFreeTextCandidates(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{
+		"01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"},
+		"01KBBB": {"title": "Dune: Part Two"},
+		"01KCCC": {"title": "進撃の巨人"},
+		"01KDDD": {"title": "3%", "titles/eng/3 percent": "true"},
+		"01KEEE": {"title": "AND -x a:b * \"quoted\""},
+		"01KFFF": {"title": "Unrelated"},
+	})
+	cases := []struct {
+		query string
+		want  []string
+	}{
+		{"naruto", []string{"01KAAA"}},
+		{"NARUTO s01e02", []string{"01KAAA"}},
+		{"dune:", []string{"01KBBB"}},
+		{"巨人", []string{"01KCCC"}},        // 2 runes: below trigram length
+		{"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set
+		{"3%", []string{"01KDDD"}},        // LIKE metacharacter in a short token
+		{"AND", []string{"01KEEE"}},       // FTS operator as a literal
+		{"-x", []string{"01KEEE"}},
+		{"a:b", []string{"01KEEE"}},
+		{"*", []string{"01KEEE"}},
+		{`"quoted"`, []string{"01KEEE"}},
+		{"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring
+	}
+	for _, tc := range cases {
+		t.Run(tc.query, func(t *testing.T) {
+			got := search(t, c, tc.query, nil, textMatch(tc.query))
+			equalIDs(t, got, tc.want...)
+		})
+	}
+}
+
+// A rid token and a title token must not match together: the rid is in the
+// FTS row only as a candidate path for the "whole query is a hashID substring"
+// rule, and verification checks the two separately.
+func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}})
+	if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 {
+		t.Fatalf("rid + title token matched: %v", got)
+	}
+}
+
+func TestSearchFollowsWritesAndDeletes(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
+	if err := c.SetProperty("R1", "title", "After"); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
+	equalIDs(t, search(t, c, "before", nil, textMatch("before")))
+	if _, err := c.DeleteMetadata("R1"); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "after", nil, textMatch("after")))
+	equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
+}
+
+func TestSearchPagesPastRejectedCandidates(t *testing.T) {
+	c := newTestClient(t)
+	records := map[string]map[string]string{}
+	for i := 0; i < 450; i++ {
+		id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3)
+		records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"}
+	}
+	records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"}
+	seed(t, c, records)
+	got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}},
+		func(_ string, f map[string]string) bool { return f["keep"] == "yes" })
+	equalIDs(t, got, "RZZZZ")
+}
+
+// The filter candidates only scale if SQLite actually uses the partial
+// expression index — which it does only when the query repeats the index's
+// expression and WHERE clause as literals.
+func TestSearchFilterUsesFieldIndex(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}})
+	q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}})
+	rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	var plan []string
+	for rows.Next() {
+		var id, parent, notused int
+		var detail string
+		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
+			t.Fatal(err)
+		}
+		plan = append(plan, detail)
+	}
+	joined := strings.Join(plan, " | ")
+	if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") {
+		t.Fatalf("filter candidates do not use kv_field_v: %s", joined)
+	}
+}
+
+func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) {
+	if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" {
+		t.Fatalf("got %q %q %v", h, f, ok)
+	}
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}})
+	got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video"))
+	equalIDs(t, got, "bafkreiabcdefghijklmnop")
+}
+
+func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) {
+	c := newTestClient(t)
+	seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}})
+	if err := c.update(func(tx *kv.Tx) error {
+		if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
+			return err
+		}
+		_, err := tx.DeletePrefix(ftsNS, 0)
+		return err
+	}); err != nil {
+		t.Fatal(err)
+	}
+	equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")))
+	n, err := c.RebuildSearchIfEmpty()
+	if err != nil || n != 2 {
+		t.Fatalf("rebuilt %d (err %v), want 2", n, err)
+	}
+	equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1")
+	if n, _ := c.RebuildSearchIfEmpty(); n != 0 {
+		t.Fatalf("second rebuild ran again (%d)", n)
+	}
+}
+
+// Text plus a broad filter: the selective part drives and the filter becomes a
+// per-candidate probe over the record's own key range — never a full
+// materialisation of the filter's result set.
+func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
+	c := newTestClient(t)
+	recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}}
+	for i := 1; i < 300; i++ {
+		recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"}
+	}
+	seed(t, c, recs)
+	filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
+	driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 {
+		t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes))
+	}
+	q, pargs := probeSQL(probes)
+	args := append([]any{`["R0000"]`}, pargs...)
+	rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer rows.Close()
+	var plan []string
+	for rows.Next() {
+		var id, parent, notused int
+		var detail string
+		if err := rows.Scan(&id, &parent, &notused, &detail); err != nil {
+			t.Fatal(err)
+		}
+		plan = append(plan, detail)
+	}
+	joined := strings.Join(plan, " | ")
+	if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
+		t.Fatalf("filter probe is not a primary-key range read: %s", joined)
+	}
+	equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool {
+		return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
+	}), "R0000")
+}