v1.2.0: skip unchanged writes, transactional schema/root counters, snapshot replay for meta:events 0-0 (drops boot republish), collect-once search driver

internal/storage/search.go

@@ -211,21 +211,36 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
 	ctx, cancel := opCtx()
 	defer cancel()
 
-	driver, probes, err := c.planCandidates(ctx, s, query, filters)
+	plan, err := c.planCandidates(ctx, s, query, filters)
 	if err != nil {
 		return nil, err
 	}
 	page := max(limit*2, 500)
 	hits := make([]SearchHit, 0, limit)
 	after := ""
-	for len(hits) < limit {
-		ids, err := queryIDs(ctx, s, driver.sql, driver.args, after, page)
-		if err != nil {
-			return nil, err
+	for pos := 0; len(hits) < limit; {
+		var ids []string
+		if plan.walk {
+			rows, err := s.Range(ctx, rootsNS, after, page)
+			if err != nil {
+				return nil, err
+			}
+			for _, row := range rows {
+				ids = append(ids, row.K[len(rootsNS):])
+			}
+			if len(rows) > 0 {
+				after = rows[len(rows)-1].K
+			}
+		} else {
+			ids = plan.ordered[pos:min(pos+page, len(plan.ordered))]
+			pos += len(ids)
+		}
+		if len(ids) == 0 {
+			break
 		}
 		survivors := ids
-		if len(probes) > 0 && len(ids) > 0 {
-			if survivors, err = probeIDs(ctx, s, ids, probes); err != nil {
+		if len(plan.probes) > 0 {
+			if survivors, err = probeIDs(ctx, s, ids, plan.probes); err != nil {
 				return nil, err
 			}
 		}
@@ -245,30 +260,10 @@ func (c *Client) Search(query string, filters []SearchFilter, limit int, match f
 		if len(ids) < page {
 			break
 		}
-		after = ids[len(ids)-1]
 	}
 	return hits, nil
 }
 
-// queryIDs returns one page of the driver's candidates, in rid order.
-func queryIDs(ctx context.Context, s *kv.Store, candSQL string, args []any, after string, limit int) ([]string, error) {
-	q := `SELECT DISTINCT rid FROM (` + candSQL + `) WHERE rid > ? ORDER BY rid LIMIT ?`
-	rows, err := s.DB().QueryContext(ctx, q, append(append([]any{}, args...), after, limit)...)
-	if err != nil {
-		return nil, fmt.Errorf("search candidates: %w", err)
-	}
-	defer rows.Close()
-	var out []string
-	for rows.Next() {
-		var id string
-		if err := rows.Scan(&id); err != nil {
-			return nil, err
-		}
-		out = append(out, id)
-	}
-	return out, rows.Err()
-}
-
 // probeSQL applies the non-driving filters to one page of candidate rids
 // (passed as a JSON array) — each an EXISTS over the candidate's own key range.
 func probeSQL(probes []candidatePart) (string, []any) {
@@ -304,7 +299,7 @@ func probeIDs(ctx context.Context, s *kv.Store, ids []string, probes []candidate
 	if err := rows.Err(); err != nil {
 		return nil, err
 	}
-	out := ids[:0:0]
+	out := make([]string, 0, len(ids))
 	for _, id := range ids {
 		if pass[id] {
 			out = append(out, id)
@@ -324,25 +319,40 @@ type candidatePart struct {
 	existsArgs []any
 }
 
-// estimateCap bounds the count used to pick the driving part: it only has to
-// tell a 50-row part from a 100k-row one.
-const estimateCap = 1000
+// estimateCap bounds the count used to rank parts by selectivity: past it, a
+// part is "broad" and only ties with other broad parts are left to resolve.
+var estimateCap = 50_000
 
-// planCandidates picks how to narrow a search: one DRIVING part (the most
-// selective, by a capped count) that lists candidate rids in order, plus PROBES
-// — every other filter, checked as an EXISTS over each candidate's own key
-// range (a primary-key range read of ~40 rows).
+// estimateCapForTest sets estimateCap and returns the previous value.
+func estimateCapForTest(n int) int { old := estimateCap; estimateCap = n; return old }
+
+// searchPlan is how one search walks its candidates.
+type searchPlan struct {
+	ordered []string        // the driver's rids, sorted (when !walk)
+	walk    bool            // no part narrows: walk idx:roots in order
+	probes  []candidatePart // filters checked per page as EXISTS
+}
+
+// planCandidates picks how to narrow a search.
 //
-// Search pages the driver in Go and probes one page at a time, so a broad
-// query stops as soon as `limit` records verify. Two earlier shapes were
-// measured on watch.nsl.sh and rejected: INTERSECTing fully materialised sets
-// (`naruto` + `fileType:video` built all ~120k video rows on every page, 5.7 s)
-// and probing inside the driver query (SQLite evaluates the probe on every
-// driver row before ORDER BY … LIMIT can stop: `1080p` + `fileType:video`,
-// both ~100k, 10 s). Free text that does not drive is dropped from SQL
-// entirely — the verifier still applies it, and dropping a condition only
-// widens the superset.
-func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (candidatePart, []candidatePart, error) {
+// A capped count ranks the parts; the SMALLEST is collected once, sorted in Go
+// and paged from memory, and every other filter becomes a probe — an EXISTS
+// over each candidate's own key range, run one page at a time, so a search
+// stops as soon as `limit` records verify. The driver is collected in full even
+// when every part is broad: that costs O(part) — at most every root, ~0.3 s at
+// 169k — whereas walking idx:roots in hashID order (tried) costs O(corpus) when
+// matches are sparse early in that order, and on watch.nsl.sh they are: the
+// first 500 roots held zero episodes, so `contentKind:episode` walked ~3 s.
+// Only a search with nothing to narrow walks the roots, where every root
+// matches.
+//
+// Also rejected, measured on watch.nsl.sh: INTERSECTing fully materialised
+// sets (5.7 s for `naruto` + `fileType:video`); probing inside a driver query
+// re-sorted per page (SQLite evaluates the probe on every driver row before
+// ORDER BY … LIMIT can stop: 10 s for `1080p` + `fileType:video`). Free text
+// that does not drive is dropped from SQL — the verifier still applies it, and
+// dropping a condition only widens the superset.
+func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string, filters []SearchFilter) (searchPlan, error) {
 	var parts []candidatePart
 	if p, ok := textCandidates(query); ok {
 		parts = append(parts, p)
@@ -353,7 +363,7 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string,
 		}
 	}
 	if len(parts) == 0 {
-		return candidatePart{sql: `SELECT substr(k, ` + strconv.Itoa(len(rootsNS)+1) + `) AS rid FROM kv WHERE k >= '` + rootsNS + `' AND k < '` + kv.PrefixEnd(rootsNS) + `'`}, nil, nil
+		return searchPlan{walk: true}, nil
 	}
 
 	driver := 0
@@ -361,40 +371,67 @@ func (c *Client) planCandidates(ctx context.Context, s *kv.Store, query string,
 		best := int64(-1)
 		for i, p := range parts {
 			var n int64
-			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap) + `)`
+			q := `SELECT count(*) FROM (SELECT rid FROM (` + p.sql + `) LIMIT ` + strconv.Itoa(estimateCap+1) + `)`
 			if err := s.DB().QueryRowContext(ctx, q, p.args...).Scan(&n); err != nil {
-				return candidatePart{}, nil, fmt.Errorf("estimate candidates: %w", err)
+				return searchPlan{}, fmt.Errorf("estimate candidates: %w", err)
 			}
 			if best < 0 || n < best {
 				best, driver = n, i
 			}
+			if n == 0 {
+				break // nothing can match
+			}
 		}
 	}
-	var probes []candidatePart
+	ids, err := collectIDs(ctx, s, parts[driver], 0)
+	if err != nil {
+		return searchPlan{}, err
+	}
+	sort.Strings(ids)
+	plan := searchPlan{ordered: ids}
 	for i, p := range parts {
 		if i != driver && p.exists != "" {
-			probes = append(probes, p)
+			plan.probes = append(plan.probes, p)
 		}
 	}
-	return parts[driver], probes, nil
+	return plan, nil
 }
 
-// candidateSQL is planCandidates without the store: text plus filters, the
-// first part driving. Used by tests that inspect the plan.
+// collectIDs reads the distinct rids of one part (all of them when limit <= 0).
+func collectIDs(ctx context.Context, s *kv.Store, p candidatePart, limit int) ([]string, error) {
+	q := `SELECT DISTINCT rid FROM (` + p.sql + `)`
+	args := append([]any{}, p.args...)
+	if limit > 0 {
+		q += ` LIMIT ?`
+		args = append(args, limit)
+	}
+	rows, err := s.DB().QueryContext(ctx, q, args...)
+	if err != nil {
+		return nil, fmt.Errorf("search candidates: %w", err)
+	}
+	defer rows.Close()
+	var out []string
+	for rows.Next() {
+		var id string
+		if err := rows.Scan(&id); err != nil {
+			return nil, err
+		}
+		out = append(out, id)
+	}
+	return out, rows.Err()
+}
+
+// candidateSQL is the first part's SQL, for tests that inspect the plan.
 func candidateSQL(query string, filters []SearchFilter) (string, []any) {
-	var parts []candidatePart
 	if p, ok := textCandidates(query); ok {
-		parts = append(parts, p)
+		return p.sql, p.args
 	}
 	for _, f := range filters {
 		if p, ok := filterCandidates(f); ok {
-			parts = append(parts, p)
+			return p.sql, p.args
 		}
 	}
-	if len(parts) == 0 {
-		return "", nil
-	}
-	return parts[0].sql, parts[0].args
+	return "", nil
 }
 
 // textCandidates narrows by free text: every whitespace token must be a
@@ -480,7 +517,7 @@ func filterCandidates(f SearchFilter) (candidatePart, bool) {
 	// As a driver: one indexed SELECT per condition, joined with UNION ALL (an
 	// OR inside one SELECT would defeat the partial index; plain UNION would
 	// dedupe by materialising every row first — 1.8 s vs 0.25 s for a capped
-	// count over fileType:video on watch.nsl.sh). queryIDs applies DISTINCT.
+	// count over fileType:video on watch.nsl.sh). collectIDs applies DISTINCT.
 	base := `SELECT ` + ridExpr + ` AS rid FROM kv WHERE ` + recordPred + ` AND `
 	selects := make([]string, len(conds))
 	for i, cond := range conds {