package storage import ( "context" "fmt" "sort" "strings" "testing" "github.com/metazla/meta-core/internal/kv" ) // These tests pin the one property storage.Search must have: its SQL // candidates are a SUPERSET of the records the exact matcher accepts. A // candidate miss is a silently dropped search result, so each case below is a // shape where SQLite and Go disagree about text (case folding, csv values, // FTS syntax, trigram length) and the candidate query has to cover for it. // // The matcher here mirrors api.recordMatchesFilter closely enough to exercise // every candidate path; the API tests own the matcher itself. func seed(t *testing.T, c *Client, records map[string]map[string]string) { t.Helper() for id, fields := range records { if err := c.SetMetadataFlat(id, fields); err != nil { t.Fatalf("seed %s: %v", id, err) } } } // matchFilter is a faithful copy of api.recordMatchesFilter's semantics. func matchFilter(fields map[string]string, prop string, exact bool, values ...string) bool { for _, v := range values { if fields[prop+"/"+v] != "" && fields[prop+"/"+v] != "false" { return true } } prefix := strings.ToLower(prop) + "/" for k, val := range fields { if len(k) > len(prefix) && strings.EqualFold(k[:len(prefix)], prefix) { for _, v := range values { if strings.EqualFold(k[len(prefix):], v) && val != "" && val != "false" { return true } } } } fv, ok := fields[prop] if !ok { return false } for _, piece := range strings.Split(fv, ",") { piece = strings.TrimSpace(piece) for _, v := range values { if exact && strings.EqualFold(piece, v) { return true } if !exact && strings.Contains(strings.ToLower(piece), strings.ToLower(v)) { return true } } } return false } func search(t *testing.T, c *Client, query string, filters []SearchFilter, match func(string, map[string]string) bool) []string { t.Helper() hits, err := c.Search(query, filters, 100, match) if err != nil { t.Fatalf("search: %v", err) } ids := make([]string, 0, len(hits)) for _, h := range hits { ids = append(ids, h.HashID) } return ids } func textMatch(query string) func(string, map[string]string) bool { tokens := strings.Fields(strings.ToLower(query)) return func(id string, fields map[string]string) bool { hay := Haystack(fields) for _, tok := range tokens { if !strings.Contains(hay, tok) { return strings.Contains(strings.ToLower(id), strings.ToLower(query)) } } return true } } func filterMatch(prop string, exact bool, values ...string) func(string, map[string]string) bool { return func(_ string, fields map[string]string) bool { return matchFilter(fields, prop, exact, values...) } } func equalIDs(t *testing.T, got []string, want ...string) { t.Helper() sort.Strings(want) if strings.Join(got, ",") != strings.Join(want, ",") { t.Fatalf("got %v, want %v", got, want) } } func TestSearchFilterCandidates(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{ "R1": {"title": "One", "genres/Action": "true", "videoType": "tv, anime", "fileType": "video"}, "R2": {"title": "Two", "genres/Comedy": "true", "videoType": "movie", "fileType": "VIDEO"}, "R3": {"title": "Three", "Genres/Drama": "true", "fileType": "audio", "tmdbid": "21405"}, "R4": {"title": "Four", "genres/Ärger": "true", "tmdbid": "1405", "genres/Action": "false"}, }) cases := []struct { name string prop string exact bool values []string want []string }{ {"key-set member, query lowercased", "genres", false, []string{"action"}, []string{"R1"}}, {"key-set member under a capitalised property", "genres", false, []string{"drama"}, []string{"R3"}}, {"key-set member, non-ASCII value", "genres", false, []string{"ärger"}, []string{"R4"}}, {"csv value, exact", "videoType", true, []string{"anime"}, []string{"R1"}}, {"flat value, case differs", "fileType", true, []string{"video"}, []string{"R1", "R2"}}, {"exact rejects a superstring", "tmdbid", true, []string{"1405"}, []string{"R4"}}, {"contains admits a superstring", "tmdbid", false, []string{"1405"}, []string{"R3", "R4"}}, {"OR across values", "genres", false, []string{"action", "comedy"}, []string{"R1", "R2"}}, {"falsey member is not membership", "genres", false, []string{"action"}, []string{"R1"}}, } for _, tc := range cases { t.Run(tc.name, func(t *testing.T) { got := search(t, c, "", []SearchFilter{{Property: tc.prop, Values: tc.values}}, filterMatch(tc.prop, tc.exact, tc.values...)) equalIDs(t, got, tc.want...) }) } } func TestSearchFiltersAreAndedAcrossEntries(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{ "R1": {"genres/Action": "true", "fileType": "video"}, "R2": {"genres/Action": "true", "fileType": "audio"}, }) match := func(_ string, f map[string]string) bool { return matchFilter(f, "genres", false, "action") && matchFilter(f, "fileType", true, "video") } got := search(t, c, "", []SearchFilter{{Property: "genres", Values: []string{"action"}}, {Property: "fileType", Values: []string{"video"}}}, match) equalIDs(t, got, "R1") } func TestSearchFreeTextCandidates(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{ "01KAAA": {"title": "Naruto Shippuden", "fileName": "naruto.s01e02.mkv"}, "01KBBB": {"title": "Dune: Part Two"}, "01KCCC": {"title": "進撃の巨人"}, "01KDDD": {"title": "3%", "titles/eng/3 percent": "true"}, "01KEEE": {"title": "AND -x a:b * \"quoted\""}, "01KFFF": {"title": "Unrelated"}, }) cases := []struct { query string want []string }{ {"naruto", []string{"01KAAA"}}, {"NARUTO s01e02", []string{"01KAAA"}}, {"dune:", []string{"01KBBB"}}, {"巨人", []string{"01KCCC"}}, // 2 runes: below trigram length {"3 percent", []string{"01KDDD"}}, // AKA via titles/ key-set {"3%", []string{"01KDDD"}}, // LIKE metacharacter in a short token {"AND", []string{"01KEEE"}}, // FTS operator as a literal {"-x", []string{"01KEEE"}}, {"a:b", []string{"01KEEE"}}, {"*", []string{"01KEEE"}}, {`"quoted"`, []string{"01KEEE"}}, {"01kbb", []string{"01KBBB"}}, // whole query is a hashID substring } for _, tc := range cases { t.Run(tc.query, func(t *testing.T) { got := search(t, c, tc.query, nil, textMatch(tc.query)) equalIDs(t, got, tc.want...) }) } } // A rid token and a title token must not match together: the rid is in the // FTS row only as a candidate path for the "whole query is a hashID substring" // rule, and verification checks the two separately. func TestSearchRidAndTitleTokensDoNotCombine(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{"01KZZZ": {"title": "Naruto"}}) if got := search(t, c, "01kzz naruto", nil, textMatch("01kzz naruto")); len(got) != 0 { t.Fatalf("rid + title token matched: %v", got) } } func TestSearchFollowsWritesAndDeletes(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{"R1": {"title": "Before"}}) if err := c.SetProperty("R1", "title", "After"); err != nil { t.Fatal(err) } equalIDs(t, search(t, c, "after", nil, textMatch("after")), "R1") equalIDs(t, search(t, c, "before", nil, textMatch("before"))) if _, err := c.DeleteMetadata("R1"); err != nil { t.Fatal(err) } equalIDs(t, search(t, c, "after", nil, textMatch("after"))) equalIDs(t, search(t, c, "", nil, func(string, map[string]string) bool { return true })) } func TestSearchPagesPastRejectedCandidates(t *testing.T) { c := newTestClient(t) records := map[string]map[string]string{} for i := 0; i < 450; i++ { id := "R" + string(rune('A'+i/26%26)) + string(rune('A'+i%26)) + strings.Repeat("0", 3) records[id+string(rune('a'+i/676))] = map[string]string{"fileType": "video"} } records["RZZZZ"] = map[string]string{"fileType": "video", "keep": "yes"} seed(t, c, records) got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, func(_ string, f map[string]string) bool { return f["keep"] == "yes" }) equalIDs(t, got, "RZZZZ") } // The filter candidates only scale if SQLite actually uses the partial // expression index — which it does only when the query repeats the index's // expression and WHERE clause as literals. func TestSearchFilterUsesFieldIndex(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{"R1": {"fileType": "video"}}) q, args := candidateSQL("", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}) rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...) if err != nil { t.Fatal(err) } defer rows.Close() var plan []string for rows.Next() { var id, parent, notused int var detail string if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { t.Fatal(err) } plan = append(plan, detail) } joined := strings.Join(plan, " | ") if !strings.Contains(joined, "COVERING INDEX kv_field_v") || strings.Contains(joined, "SCAN kv") { t.Fatalf("filter candidates do not use kv_field_v: %s", joined) } } func TestCIDRootedRecordsParseAtTheFirstSlash(t *testing.T) { if h, f, ok := splitRecordKey("file:bafkreiabc/titles/eng/x"); !ok || h != "bafkreiabc" || f != "titles/eng/x" { t.Fatalf("got %q %q %v", h, f, ok) } c := newTestClient(t) seed(t, c, map[string]map[string]string{"bafkreiabcdefghijklmnop": {"fileType": "video"}}) got := search(t, c, "", []SearchFilter{{Property: "fileType", Values: []string{"video"}}}, filterMatch("fileType", true, "video")) equalIDs(t, got, "bafkreiabcdefghijklmnop") } func TestRebuildSearchIfEmptyRestoresFreeText(t *testing.T) { c := newTestClient(t) seed(t, c, map[string]map[string]string{"R1": {"title": "Naruto"}, "R2": {"title": "Dune"}}) if err := c.update(func(tx *kv.Tx) error { if _, err := tx.Exec(`DELETE FROM fts`); err != nil { return err } _, err := tx.DeletePrefix(ftsNS, 0) return err }); err != nil { t.Fatal(err) } equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto"))) n, err := c.RebuildSearchIfEmpty() if err != nil || n != 2 { t.Fatalf("rebuilt %d (err %v), want 2", n, err) } equalIDs(t, search(t, c, "naruto", nil, textMatch("naruto")), "R1") if n, _ := c.RebuildSearchIfEmpty(); n != 0 { t.Fatalf("second rebuild ran again (%d)", n) } } // Text plus a broad filter: the selective part drives and the filter becomes a // per-candidate probe over the record's own key range — never a full // materialisation of the filter's result set. func TestSearchCombinedPlanProbesByPrimaryKey(t *testing.T) { c := newTestClient(t) recs := map[string]map[string]string{"R0000": {"title": "Naruto", "fileType": "video"}} for i := 1; i < 300; i++ { recs[fmt.Sprintf("R%04d", i)] = map[string]string{"title": "Other", "fileType": "video"} } seed(t, c, recs) filters := []SearchFilter{{Property: "fileType", Values: []string{"video"}}} driver, probes, err := c.planCandidates(context.Background(), c.Store(), "naruto", filters) if err != nil { t.Fatal(err) } if !strings.HasPrefix(driver.sql, "SELECT rid FROM fts") || len(probes) != 1 { t.Fatalf("free text should drive with the filter as a probe, got driver %q, %d probes", driver.sql, len(probes)) } q, pargs := probeSQL(probes) args := append([]any{`["R0000"]`}, pargs...) rows, err := c.Store().DB().QueryContext(context.Background(), "EXPLAIN QUERY PLAN "+q, args...) if err != nil { t.Fatal(err) } defer rows.Close() var plan []string for rows.Next() { var id, parent, notused int var detail string if err := rows.Scan(&id, &parent, ¬used, &detail); err != nil { t.Fatal(err) } plan = append(plan, detail) } joined := strings.Join(plan, " | ") if !strings.Contains(joined, "PRIMARY KEY") || strings.Contains(joined, "SCAN kv") { t.Fatalf("filter probe is not a primary-key range read: %s", joined) } equalIDs(t, search(t, c, "naruto", filters, func(id string, f map[string]string) bool { return textMatch("naruto")(id, f) && matchFilter(f, "fileType", false, "video") }), "R0000") }