343 lines
13 KiB
Go
343 lines
13 KiB
Go
package storage
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"sort"
|
|
"strings"
|
|
"testing"
|
|
|
|
"github.com/metazla/meta-core/internal/kv"
|
|
)
|
|
|
|
// These tests pin the one property storage.Search must have: its SQL
|
|
// candidates are a SUPERSET of the records the exact matcher accepts. A
|
|
// candidate miss is a silently dropped search result, so each case below is a
|
|
// shape where SQLite and Go disagree about text (case folding, csv values,
|
|
// FTS syntax, trigram length) and the candidate query has to cover for it.
|
|
//
|
|
// The matcher here mirrors api.recordMatchesFilter closely enough to exercise
|
|
// every candidate path; the API tests own the matcher itself.
|
|
|
|
// matchFilter is a faithful copy of api.recordMatchesFilter's semantics.
|
|
func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool {
|
|
for _, v := range values {
|
|
if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" {
|
|
return true
|
|
}
|
|
}
|
|
prefix := strings.ToLower(prop) + "/"
|
|
for k, val := range fields {
|
|
if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) {
|
|
for _, v := range values {
|
|
if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
}
|
|
fv, ok := fields[prop]
|
|
if !ok {
|
|
return false
|
|
}
|
|
for _, piece := range strings.Split(fv, ",") {
|
|
piece = strings.TrimSpace(piece)
|
|
for _, v := range values {
|
|
if exact && strings.EqualFold(piece, v) {
|
|
return true
|
|
}
|
|
if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) {
|
|
return true
|
|
}
|
|
}
|
|
}
|
|
return false
|
|
}
|
|
|
|
func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string {
|
|
t.Helper()
|
|
hits, err := c.Search(query, filters, 100, match)
|
|
if err != nil {
|
|
t.Fatalf("search: %v", err)
|
|
}
|
|
ids := make([]string, 0, len(hits))
|
|
for _, h := range hits {
|
|
ids = append(ids, h.HashID)
|
|
}
|
|
return ids
|
|
}
|
|
|
|
func textMatch(query string) func(string, map[string]string) bool {
|
|
tokens := strings.Fields(strings.ToLower(query))
|
|
return func(id string, fields map[string]string) bool {
|
|
hay := Haystack(fields)
|
|
for _, tok := range tokens {
|
|
if !strings.Contains(hay, tok) {
|
|
return strings.Contains(strings.ToLower(id), strings.ToLower(query))
|
|
}
|
|
}
|
|
return true
|
|
}
|
|
}
|
|
|
|
func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool {
|
|
return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) }
|
|
}
|
|
|
|
func equalIDs(t *testing.T, got []string, want ...string) {
|
|
t.Helper()
|
|
sort.Strings(want)
|
|
if strings.Join(got, ",") != strings.Join(want, ",") {
|
|
t.Fatalf("got %v, want %v", got, want)
|
|
}
|
|
}
|
|
|
|
func TestSearchFilterCandidates(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{
|
|
"R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"},
|
|
"R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"},
|
|
"R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"},
|
|
"R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"},
|
|
})
|
|
|
|
cases := []struct {
|
|
name string
|
|
prop string
|
|
exact bool
|
|
values []string
|
|
want []string
|
|
}{
|
|
{"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}},
|
|
{"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}},
|
|
{"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}},
|
|
{"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}},
|
|
{"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}},
|
|
{"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}},
|
|
{"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}},
|
|
{"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}},
|
|
{"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}},
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.name, func(t *testing.T) {
|
|
got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...))
|
|
equalIDs(t, got, tc.want...)
|
|
})
|
|
}
|
|
}
|
|
|
|
func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{
|
|
"R1": {"genres/Action": "true", "fileType": "video"},
|
|
"R2": {"genres/Action": "true", "fileType": "audio"},
|
|
})
|
|
match := func(_ string, f map[string]string) bool {
|
|
return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video")
|
|
}
|
|
got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match)
|
|
equalIDs(t, got, "R1")
|
|
}
|
|
|
|
func TestSearchFreeTextCandidates(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{
|
|
"01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"},
|
|
"01KBBB": {"title": "Dune: Part Two"},
|
|
"01KCCC": {"title": "進撃の巨人"},
|
|
"01KDDD": {"title": "3%", "titles/eng/3 percent": "true"},
|
|
"01KEEE": {"title": "AND -x a:b * \"quoted\""},
|
|
"01KFFF": {"title": "Unrelated"},
|
|
})
|
|
cases := []struct {
|
|
query string
|
|
want []string
|
|
}{
|
|
{"naruto", []string{"01KAAA"}},
|
|
{"NARUTO s01e02", []string{"01KAAA"}},
|
|
{"dune:", []string{"01KBBB"}},
|
|
{"巨人", []string{"01KCCC"}}, // 2 runes: below trigram length
|
|
{"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set
|
|
{"3%", []string{"01KDDD"}}, // LIKE metacharacter in a short token
|
|
{"AND", []string{"01KEEE"}}, // FTS operator as a literal
|
|
{"-x", []string{"01KEEE"}},
|
|
{"a:b", []string{"01KEEE"}},
|
|
{"*", []string{"01KEEE"}},
|
|
{`"quoted"`, []string{"01KEEE"}},
|
|
{"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring
|
|
}
|
|
for _, tc := range cases {
|
|
t.Run(tc.query, func(t *testing.T) {
|
|
got := search(t, c, tc.query, nil, textMatch(tc.query))
|
|
equalIDs(t, got, tc.want...)
|
|
})
|
|
}
|
|
}
|
|
|
|
// A rid token and a title token must not match together: the rid is in the
|
|
// FTS row only as a candidate path for the "whole query is a hashID substring"
|
|
// rule, and verification checks the two separately.
|
|
func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}})
|
|
if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 {
|
|
t.Fatalf("rid + title token matched: %v", got)
|
|
}
|
|
}
|
|
|
|
func TestSearchFollowsWritesAndDeletes(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}})
|
|
setMerged(t, c, "R1", map[string]string{"title": "After"})
|
|
equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1")
|
|
equalIDs(t, search(t, c, "before", nil, textMatch("before")))
|
|
deleteMerged(t, c, "R1")
|
|
equalIDs(t, search(t, c, "after", nil, textMatch("after")))
|
|
equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true }))
|
|
}
|
|
|
|
func TestSearchPagesPastRejectedCandidates(t *testing.T) {
|
|
c := newTestClient(t)
|
|
records := map[string]map[string]string{}
|
|
for i := 0; i < 450; i++ {
|
|
id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3)
|
|
records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"}
|
|
}
|
|
records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"}
|
|
seed(t, c, records)
|
|
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}},
|
|
func(_ string, f map[string]string) bool { return f["keep"] == "yes" })
|
|
equalIDs(t, got, "RZZZZ")
|
|
}
|
|
|
|
// The filter candidates only scale if SQLite actually uses the partial
|
|
// expression index — which it does only when the query repeats the index's
|
|
// expression and WHERE clause as literals.
|
|
func TestSearchFilterUsesFieldIndex(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}})
|
|
q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}})
|
|
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var plan []string
|
|
for rows.Next() {
|
|
var id, parent, notused int
|
|
var detail string
|
|
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
plan = append(plan, detail)
|
|
}
|
|
joined := strings.Join(plan, " | ")
|
|
if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") {
|
|
t.Fatalf("filter candidates do not use kv_field_v: %s", joined)
|
|
}
|
|
}
|
|
|
|
func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) {
|
|
if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" {
|
|
t.Fatalf("got %q %q %v", h, f, ok)
|
|
}
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}})
|
|
got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video"))
|
|
equalIDs(t, got, "bafkreiabcdefghijklmnop")
|
|
}
|
|
|
|
func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) {
|
|
c := newTestClient(t)
|
|
seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}})
|
|
if err := c.update(func(tx *kv.Tx) error {
|
|
if _, err := tx.Exec(`DELETE FROM fts`); err != nil {
|
|
return err
|
|
}
|
|
_, err := tx.DeletePrefix(ftsNS, 0)
|
|
return err
|
|
}); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")))
|
|
n, err := c.RebuildSearchIfEmpty()
|
|
if err != nil || n != 2 {
|
|
t.Fatalf("rebuilt %d (err %v), want 2", n, err)
|
|
}
|
|
equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1")
|
|
if n, _ := c.RebuildSearchIfEmpty(); n != 0 {
|
|
t.Fatalf("second rebuild ran again (%d)", n)
|
|
}
|
|
}
|
|
|
|
// Text plus a broad filter: the selective part drives and the filter becomes a
|
|
// per-candidate probe over the record's own key range — never a full
|
|
// materialisation of the filter's result set.
|
|
func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) {
|
|
c := newTestClient(t)
|
|
recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}}
|
|
for i := 1; i < 300; i++ {
|
|
recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"}
|
|
}
|
|
seed(t, c, recs)
|
|
filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}}
|
|
plan, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if plan.walk || len(plan.ordered) != 1 || len(plan.probes) != 1 {
|
|
t.Fatalf("the rare text should drive (1 rid) with the filter as a probe, got %+v", plan)
|
|
}
|
|
probes := plan.probes
|
|
q, pargs := probeSQL(probes)
|
|
args := append([]any{`["R0000"]`}, pargs...)
|
|
rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer rows.Close()
|
|
var steps []string
|
|
for rows.Next() {
|
|
var id, parent, notused int
|
|
var detail string
|
|
if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
steps = append(steps, detail)
|
|
}
|
|
joined := strings.Join(steps, " | ")
|
|
if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") {
|
|
t.Fatalf("filter probe is not a primary-key range read: %s", joined)
|
|
}
|
|
equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool {
|
|
return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video")
|
|
}), "R0000")
|
|
}
|
|
|
|
// When every part is broad (above estimateCap), the smallest is still
|
|
// collected and drives — no walk over the roots — and the result is exact.
|
|
func TestSearchBroadPartsStillDriveFromTheSmallest(t *testing.T) {
|
|
defer func(old int) { estimateCapForTest(old) }(estimateCapForTest(3))
|
|
c := newTestClient(t)
|
|
recs := map[string]map[string]string{}
|
|
for i := 0; i < 20; i++ {
|
|
ft := "video"
|
|
if i%2 == 1 {
|
|
ft = "audio"
|
|
}
|
|
recs[fmt.Sprintf("R%02d", i)] = map[string]string{"title": "Common words", "fileType": ft}
|
|
}
|
|
seed(t, c, recs)
|
|
filters := []SearchFilter{{Property: "fileType", Values: []string{"audio"}}}
|
|
plan, err := c.planCandidates(context.Background(), c.Store(), "common", filters)
|
|
if err != nil || plan.walk || len(plan.ordered) != 20 || len(plan.probes) != 1 {
|
|
t.Fatalf("want the text (20 rids) driving with one probe, got walk=%v ordered=%d probes=%d err %v", plan.walk, len(plan.ordered), len(plan.probes), err)
|
|
}
|
|
got := search(t, c, "common", filters, func(id string, f map[string]string) bool {
|
|
return textMatch("common")(id, f) && matchFilter(f, "fileType", false, "audio")
|
|
})
|
|
if len(got) != 10 || got[0] != "R01" {
|
|
t.Fatalf("got %v", got)
|
|
}
|
|
}
|