replace Redis with an embedded SQLite string->string KV store (group commit, in-tx events, SQL search)
Build and Push Docker Image / build (push) Successful in 2m44s
Build and Push Docker Image / build (push) Successful in 2m44s
This commit is contained in:
Worph
@@ -0,0 +1,565 @@
|
||||
package storage
|
||||
|
||||
// Search over the KV store.
|
||||
//
|
||||
// The Redis era answered POST /api/metadata/search from an in-memory copy of
|
||||
// every record (the API's searchIndex), because Redis could not filter
|
||||
// server-side. That copy cost ~30 KB of RAM per record (5.1 GB at 169k records
|
||||
// on watch.nsl.sh) and was the reason meta-core kept getting OOM-killed. It is
|
||||
// gone: search now runs against two derived structures inside the database.
|
||||
//
|
||||
// - kv_field_v — an expression index over (lowercased field name parsed out
|
||||
// of the key, lowercased value, value), covering only record keys. It answers "which
|
||||
// records have field F with a value like V" for ANY field, including ones
|
||||
// invented tomorrow: the data stays schema-free, the index is over the key
|
||||
// itself.
|
||||
// - fts — an FTS5 trigram table holding one row per record: the record's
|
||||
// haystack (buildHaystack) plus its rid. Trigrams make every substring of
|
||||
// 3+ characters indexable, which is the matching rule search always had.
|
||||
//
|
||||
// SQL produces a SUPERSET of candidates; the caller verifies each hydrated
|
||||
// record with the exact Go matchers (api.recordMatchesFilters et al.), so the
|
||||
// semantics are bit-identical to the in-memory matcher and nothing depends on
|
||||
// SQLite's ASCII-only lower() agreeing with Go's Unicode folding.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
|
||||
"github.com/metazla/meta-core/internal/kv"
|
||||
)
|
||||
|
||||
// The field expression and the partial-index predicate MUST appear verbatim
|
||||
// (as literals, never bound parameters) in every query meant to use kv_field_v —
|
||||
// SQLite only matches a partial index whose WHERE clause it can see implied by
|
||||
// the query text. TestSearchFilterUsesFieldIndex pins this.
|
||||
const (
|
||||
fieldExpr = `lower(substr(k, instr(k,'/')+1))`
|
||||
recordPred = `k >= 'file:' AND k < 'file;' AND instr(k,'/') > 0`
|
||||
ridExpr = `substr(k, 6, instr(k,'/')-6)`
|
||||
)
|
||||
|
||||
// searchSchema is the derived DDL applied when the database is opened.
|
||||
//
|
||||
// kv_field_v carries the raw value as a third column so the filters' substring
|
||||
// test (instr(lower(v), …)) is answered from the index alone. Without it,
|
||||
// SQLite fetched the table row for every entry of the field: 1.48 s vs 13 ms
|
||||
// for tmdbid (110k entries) on watch.nsl.sh. It replaced kv_field (field,
|
||||
// lower(v)), which is dropped here; the rebuild costs ~80 s once.
|
||||
const searchSchema = `
|
||||
DROP INDEX IF EXISTS kv_field;
|
||||
CREATE INDEX IF NOT EXISTS kv_field_v ON kv(` + fieldExpr + `, lower(v), v) WHERE ` + recordPred + `;
|
||||
CREATE VIRTUAL TABLE IF NOT EXISTS fts USING fts5(rid UNINDEXED, hay, tokenize='trigram');
|
||||
`
|
||||
|
||||
// searchIndexFields are the metadata fields folded into a record's search
|
||||
// haystack — must stay in sync with the fields the search handler documents.
|
||||
var searchIndexFields = []string{"title", "fileName", "originaltitle", "showtitle", "filePath"}
|
||||
|
||||
// affectsHaystack reports whether writing field can change the haystack.
|
||||
func affectsHaystack(field string) bool {
|
||||
if strings.HasPrefix(field, "titles/") {
|
||||
return true
|
||||
}
|
||||
for _, f := range searchIndexFields {
|
||||
if field == f {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// Haystack folds a record's searchable text fields into one lowercased blob —
|
||||
// what free-text tokens must each be a substring of.
|
||||
//
|
||||
// Beyond the flat fields it folds in every `titles/<lang3>/<name>` member's
|
||||
// name (METADATA_KEYS.md §3): the name lives in the key and the value is the
|
||||
// sentinel "true", so a search for an alternative title (`3 percent`) finds the
|
||||
// record whose `title` is `3%`.
|
||||
func Haystack(fields map[string]string) string {
|
||||
var sb strings.Builder
|
||||
for _, f := range searchIndexFields {
|
||||
if v, ok := fields[f]; ok && v != "" {
|
||||
sb.WriteString(strings.ToLower(v))
|
||||
sb.WriteByte('\n')
|
||||
}
|
||||
}
|
||||
for _, name := range titleMemberNames(fields) {
|
||||
sb.WriteString(strings.ToLower(name))
|
||||
sb.WriteByte('\n')
|
||||
}
|
||||
return sb.String()
|
||||
}
|
||||
|
||||
// titleMemberNames returns the names of a record's `titles/<lang3>/<name>`
|
||||
// key-set members, with the `∕` (U+2215) a writer substitutes for `/` mapped
|
||||
// back. Sorted so the haystack is deterministic.
|
||||
func titleMemberNames(fields map[string]string) []string {
|
||||
var names []string
|
||||
for k, v := range fields {
|
||||
rest, ok := strings.CutPrefix(k, "titles/")
|
||||
if !ok || v == "" || v == "false" {
|
||||
continue
|
||||
}
|
||||
_, name, ok := strings.Cut(rest, "/")
|
||||
if !ok || name == "" {
|
||||
continue
|
||||
}
|
||||
names = append(names, strings.ReplaceAll(name, "∕", "/"))
|
||||
}
|
||||
sort.Strings(names)
|
||||
return names
|
||||
}
|
||||
|
||||
// ftsHay is what the FTS row indexes: the haystack plus the rid, so the
|
||||
// "query is a substring of the hashID" rule also has an indexed candidate path.
|
||||
// Verification checks the two separately (a rid token plus a title token must
|
||||
// not match together).
|
||||
func ftsHay(hashID string, fields map[string]string) string {
|
||||
return Haystack(fields) + strings.ToLower(hashID) + "\n"
|
||||
}
|
||||
|
||||
// refreshSearchTx rewrites a record's FTS row from its current fields, or
|
||||
// drops it when the record is gone.
|
||||
func refreshSearchTx(tx *kv.Tx, hashID string) error {
|
||||
fields, err := readRecord(tx.Context(), tx, hashID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(fields) == 0 {
|
||||
return deleteSearchRowTx(tx, hashID)
|
||||
}
|
||||
return upsertSearchRowTx(tx, hashID, ftsHay(hashID, fields))
|
||||
}
|
||||
|
||||
// ensureSearchRowTx creates the record's FTS row if it has none (a record
|
||||
// whose writes never touched a haystack field still needs one for the rid
|
||||
// rule).
|
||||
func ensureSearchRowTx(tx *kv.Tx, hashID string) error {
|
||||
_, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||||
if err != nil || ok {
|
||||
return err
|
||||
}
|
||||
return refreshSearchTx(tx, hashID)
|
||||
}
|
||||
|
||||
func upsertSearchRowTx(tx *kv.Tx, hashID, hay string) error {
|
||||
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if ok {
|
||||
if _, err := tx.Exec(`UPDATE fts SET hay = ? WHERE rowid = ?`, hay, rowidStr); err != nil {
|
||||
return fmt.Errorf("fts update %s: %w", hashID, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
res, err := tx.Exec(`INSERT INTO fts(rid, hay) VALUES(?, ?)`, hashID, hay)
|
||||
if err != nil {
|
||||
return fmt.Errorf("fts insert %s: %w", hashID, err)
|
||||
}
|
||||
rowid, err := res.LastInsertId()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return tx.Put(ftsNS+hashID, strconv.FormatInt(rowid, 10))
|
||||
}
|
||||
|
||||
func deleteSearchRowTx(tx *kv.Tx, hashID string) error {
|
||||
rowidStr, ok, err := tx.Get(tx.Context(), ftsNS+hashID)
|
||||
if err != nil || !ok {
|
||||
return err
|
||||
}
|
||||
if _, err := tx.Exec(`DELETE FROM fts WHERE rowid = ?`, rowidStr); err != nil {
|
||||
return fmt.Errorf("fts delete %s: %w", hashID, err)
|
||||
}
|
||||
_, err = tx.Delete(ftsNS + hashID)
|
||||
return err
|
||||
}
|
||||
|
||||
// SearchFilter is one structured filter: AND across filters, OR across Values.
|
||||
type SearchFilter struct {
|
||||
Property string
|
||||
Values []string
|
||||
}
|
||||
|
||||
// SearchHit is one verified result.
|
||||
type SearchHit struct {
|
||||
HashID string
|
||||
Fields map[string]string
|
||||
}
|
||||
|
||||
// Search returns up to limit records, in hashID order, that satisfy match.
|
||||
//
|
||||
// query and filters only NARROW the candidates (SQL, superset semantics);
|
||||
// match is the authoritative test, run on each hydrated record. Pass the same
|
||||
// query/filters match evaluates, or results will be missing.
|
||||
func (c *Client) Search(query string, filters []SearchFilter, limit int, match func(hashID string, fields map[string]string) bool) ([]SearchHit, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if limit <= 0 {
|
||||
limit = 100
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
defer cancel()
|
||||
|
||||
driver, probes, err := c.planCandidates(ctx, s, query, filters)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
page := max(limit*2, 500)
|
||||
hits := make([]SearchHit, 0, limit)
|
||||
after := ""
|
||||
for len(hits) < limit {
|
||||
ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
survivors := ids
|
||||
if len(probes) > 0 && len(ids) > 0 {
|
||||
if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
for _, id := range survivors {
|
||||
fields, err := readRecord(ctx, s, id)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if fields == nil || !match(id, fields) {
|
||||
continue
|
||||
}
|
||||
hits = append(hits, SearchHit{HashID: id, Fields: fields})
|
||||
if len(hits) >= limit {
|
||||
break
|
||||
}
|
||||
}
|
||||
if len(ids) < page {
|
||||
break
|
||||
}
|
||||
after = ids[len(ids)-1]
|
||||
}
|
||||
return hits, nil
|
||||
}
|
||||
|
||||
// queryIDs returns one page of the driver's candidates, in rid order.
|
||||
func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
|
||||
q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
|
||||
rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search candidates: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
var out []string
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out = append(out, id)
|
||||
}
|
||||
return out, rows.Err()
|
||||
}
|
||||
|
||||
// probeSQL applies the non-driving filters to one page of candidate rids
|
||||
// (passed as a JSON array) — each an EXISTS over the candidate's own key range.
|
||||
func probeSQL(probes []candidatePart) (string, []any) {
|
||||
where := make([]string, len(probes))
|
||||
var args []any
|
||||
for i, p := range probes {
|
||||
where[i] = p.exists
|
||||
args = append(args, p.existsArgs...)
|
||||
}
|
||||
return `WITH c(rid) AS (SELECT value FROM json_each(?)) SELECT rid FROM c WHERE ` + strings.Join(where, " AND "), args
|
||||
}
|
||||
|
||||
// probeIDs keeps the ids (in their order) that pass every probe.
|
||||
func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidatePart) ([]string, error) {
|
||||
q, args := probeSQL(probes)
|
||||
list, err := json.Marshal(ids)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
rows, err := s.DB().QueryContext(ctx, q, append([]any{string(list)}, args...)...)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("search probes: %w", err)
|
||||
}
|
||||
defer rows.Close()
|
||||
pass := make(map[string]bool, len(ids))
|
||||
for rows.Next() {
|
||||
var id string
|
||||
if err := rows.Scan(&id); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
pass[id] = true
|
||||
}
|
||||
if err := rows.Err(); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
out := ids[:0:0]
|
||||
for _, id := range ids {
|
||||
if pass[id] {
|
||||
out = append(out, id)
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// candidatePart is one narrowing condition: free text, or one filter.
|
||||
type candidatePart struct {
|
||||
sql string // SELECT … AS rid
|
||||
args []any
|
||||
// exists is the same filter as a predicate on candidate row c.rid (empty
|
||||
// for free text, which has no per-record probe: the FTS rid column is not
|
||||
// indexed).
|
||||
exists string
|
||||
existsArgs []any
|
||||
}
|
||||
|
||||
// estimateCap bounds the count used to pick the driving part: it only has to
|
||||
// tell a 50-row part from a 100k-row one.
|
||||
const estimateCap = 1000
|
||||
|
||||
// planCandidates picks how to narrow a search: one DRIVING part (the most
|
||||
// selective, by a capped count) that lists candidate rids in order, plus PROBES
|
||||
// — every other filter, checked as an EXISTS over each candidate's own key
|
||||
// range (a primary-key range read of ~40 rows).
|
||||
//
|
||||
// Search pages the driver in Go and probes one page at a time, so a broad
|
||||
// query stops as soon as `limit` records verify. Two earlier shapes were
|
||||
// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
|
||||
// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
|
||||
// and probing inside the driver query (SQLite evaluates the probe on every
|
||||
// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
|
||||
// both ~100k, 10 s). Free text that does not drive is dropped from SQL
|
||||
// entirely — the verifier still applies it, and dropping a condition only
|
||||
// widens the superset.
|
||||
func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
|
||||
var parts []candidatePart
|
||||
if p, ok := textCandidates(query); ok {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
for _, f := range filters {
|
||||
if p, ok := filterCandidates(f); ok {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
|
||||
}
|
||||
|
||||
driver := 0
|
||||
if len(parts) > 1 {
|
||||
best := int64(-1)
|
||||
for i, p := range parts {
|
||||
var n int64
|
||||
q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
|
||||
if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
|
||||
return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
|
||||
}
|
||||
if best < 0 || n < best {
|
||||
best, driver = n, i
|
||||
}
|
||||
}
|
||||
}
|
||||
var probes []candidatePart
|
||||
for i, p := range parts {
|
||||
if i != driver && p.exists != "" {
|
||||
probes = append(probes, p)
|
||||
}
|
||||
}
|
||||
return parts[driver], probes, nil
|
||||
}
|
||||
|
||||
// candidateSQL is planCandidates without the store: text plus filters, the
|
||||
// first part driving. Used by tests that inspect the plan.
|
||||
func candidateSQL(query string, filters []SearchFilter) (string, []any) {
|
||||
var parts []candidatePart
|
||||
if p, ok := textCandidates(query); ok {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
for _, f := range filters {
|
||||
if p, ok := filterCandidates(f); ok {
|
||||
parts = append(parts, p)
|
||||
}
|
||||
}
|
||||
if len(parts) == 0 {
|
||||
return "", nil
|
||||
}
|
||||
return parts[0].sql, parts[0].args
|
||||
}
|
||||
|
||||
// textCandidates narrows by free text: every whitespace token must be a
|
||||
// substring of the FTS hay (haystack + rid). Tokens of 3+ runes go through the
|
||||
// trigram index; shorter ones (trigrams cannot index them) are a LIKE on the
|
||||
// same rows.
|
||||
func textCandidates(query string) (candidatePart, bool) {
|
||||
tokens := strings.Fields(strings.ToLower(query))
|
||||
if len(tokens) == 0 {
|
||||
return candidatePart{}, false
|
||||
}
|
||||
var match []string
|
||||
var likes []string
|
||||
var likeArgs []any
|
||||
for _, t := range tokens {
|
||||
if utf8.RuneCountInString(t) >= 3 {
|
||||
match = append(match, `"`+strings.ReplaceAll(t, `"`, `""`)+`"`)
|
||||
} else {
|
||||
likes = append(likes, `hay LIKE ? ESCAPE '\'`)
|
||||
likeArgs = append(likeArgs, "%"+escapeLike(t)+"%")
|
||||
}
|
||||
}
|
||||
var where []string
|
||||
var args []any
|
||||
if len(match) > 0 {
|
||||
where = append(where, `fts MATCH ?`)
|
||||
args = append(args, strings.Join(match, " "))
|
||||
}
|
||||
where = append(where, likes...)
|
||||
args = append(args, likeArgs...)
|
||||
return candidatePart{sql: `SELECT rid FROM fts WHERE ` + strings.Join(where, " AND "), args: args}, true
|
||||
}
|
||||
|
||||
func escapeLike(s string) string {
|
||||
r := strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`)
|
||||
return r.Replace(s)
|
||||
}
|
||||
|
||||
// filterCandidates narrows by one filter — the UNION, over its values, of the
|
||||
// two storage shapes recordMatchesFilter probes:
|
||||
//
|
||||
// - key-set member `<prop>/<value>` (any case): the lowercased field equals
|
||||
// lower(prop/value); for a non-ASCII value, any member of prop (SQLite's
|
||||
// lower() only folds ASCII, Go's folds Unicode, so an exact lowercase
|
||||
// comparison could miss);
|
||||
// - flat field `<prop>` whose lowercased value contains the lowercased value
|
||||
// — covers exact and contains alike, and comma-split csv-set values.
|
||||
//
|
||||
// ok=false means "this filter cannot narrow" (blank, or a non-ASCII property
|
||||
// name): it constrains nothing in SQL and is left to the verifier.
|
||||
func filterCandidates(f SearchFilter) (candidatePart, bool) {
|
||||
var values []string
|
||||
for _, v := range f.Values {
|
||||
if v = strings.TrimSpace(v); v != "" {
|
||||
values = append(values, v)
|
||||
}
|
||||
}
|
||||
if f.Property == "" || len(values) == 0 || !isASCII(f.Property) {
|
||||
return candidatePart{}, false
|
||||
}
|
||||
prop := asciiLower(f.Property)
|
||||
var conds []string
|
||||
var args []any
|
||||
memberRange := false
|
||||
for _, v := range values {
|
||||
if isASCII(v) {
|
||||
lv := asciiLower(v)
|
||||
conds = append(conds, fieldExpr+` = ?`)
|
||||
args = append(args, prop+"/"+lv)
|
||||
conds = append(conds, fieldExpr+` = ? AND instr(lower(v), ?) > 0`)
|
||||
args = append(args, prop, lv)
|
||||
} else {
|
||||
memberRange = true
|
||||
conds = append(conds, fieldExpr+` = ?`)
|
||||
args = append(args, prop)
|
||||
}
|
||||
}
|
||||
if memberRange {
|
||||
conds = append(conds, fieldExpr+` >= ? AND `+fieldExpr+` < ?`)
|
||||
args = append(args, prop+"/", kv.PrefixEnd(prop+"/"))
|
||||
}
|
||||
|
||||
// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
|
||||
// OR inside one SELECT would defeat the partial index; plain UNION would
|
||||
// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
|
||||
// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
|
||||
base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
|
||||
selects := make([]string, len(conds))
|
||||
for i, cond := range conds {
|
||||
selects[i] = base + cond
|
||||
}
|
||||
// As a probe: the same conditions over one candidate's key range.
|
||||
exists := `EXISTS (SELECT 1 FROM kv WHERE k >= 'file:' || c.rid || '/' AND k < 'file:' || c.rid || '0' AND ((` +
|
||||
strings.Join(conds, ") OR (") + `)))`
|
||||
return candidatePart{
|
||||
sql: strings.Join(selects, " UNION ALL "), args: args,
|
||||
exists: exists, existsArgs: args,
|
||||
}, true
|
||||
}
|
||||
|
||||
func isASCII(s string) bool {
|
||||
for i := 0; i < len(s); i++ {
|
||||
if s[i] >= utf8.RuneSelf {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
// asciiLower lowercases A–Z only, exactly like SQLite's lower().
|
||||
func asciiLower(s string) string {
|
||||
b := []byte(s)
|
||||
for i, c := range b {
|
||||
if 'A' <= c && c <= 'Z' {
|
||||
b[i] = c + ('a' - 'A')
|
||||
}
|
||||
}
|
||||
return string(b)
|
||||
}
|
||||
|
||||
func envInt(name string, def int) int {
|
||||
if v, err := strconv.Atoi(os.Getenv(name)); err == nil && v > 0 {
|
||||
return v
|
||||
}
|
||||
return def
|
||||
}
|
||||
|
||||
// RebuildSearchIfEmpty rebuilds every record's FTS row when the FTS table is
|
||||
// empty but records exist — a store restored from a snapshot, a copy made
|
||||
// outside meta-core (the 2026-10 Redis export), or an FTS table dropped to
|
||||
// change its tokenizer. The rows are derived, so this is a repair, not a
|
||||
// migration. Runs in bounded batches so writes keep flowing; until it
|
||||
// finishes, free-text search under-matches (filters are unaffected).
|
||||
func (c *Client) RebuildSearchIfEmpty() (int, error) {
|
||||
s, err := c.kv()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
ctx, cancel := opCtx()
|
||||
var rows int64
|
||||
err = s.DB().QueryRowContext(ctx, `SELECT count(*) FROM fts`).Scan(&rows)
|
||||
cancel()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("count fts rows: %w", err)
|
||||
}
|
||||
if rows > 0 {
|
||||
return 0, nil
|
||||
}
|
||||
ids, err := c.GetAllHashIDs()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
const batch = 500
|
||||
for start := 0; start < len(ids); start += batch {
|
||||
chunk := ids[start:min(start+batch, len(ids))]
|
||||
if err := c.update(func(tx *kv.Tx) error {
|
||||
for _, id := range chunk {
|
||||
if err := refreshSearchTx(tx, id); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}); err != nil {
|
||||
return start, fmt.Errorf("rebuild search rows: %w", err)
|
||||
}
|
||||
}
|
||||
return len(ids), nil
|
||||
}
|
||||
Reference in New Issue
Block a user