improved search
This commit is contained in:
Worph
@@ -2,6 +2,7 @@ package api
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
// handleMigrateDualRoots triggers the one-shot sweep that re-unifies
|
||||
@@ -54,3 +55,47 @@ func (s *Server) handleMigrateDomainScreen(w http.ResponseWriter, r *http.Reques
|
||||
"fixed": fixed,
|
||||
})
|
||||
}
|
||||
|
||||
// handleSweepLiteratureEchoes removes query echoes stored on literature cards:
|
||||
// `anilistid` on a site card, and `workcid` / position keys on any card
|
||||
// (meta-read docs/reading-model.md §7). See storage.SweepLiteratureEchoes for
|
||||
// why chapters and anchor cards' ids are deliberately left alone.
|
||||
//
|
||||
// Dry-run unless `?apply=true`: unlike the migrations above this one deletes
|
||||
// keys, so a bare call only reports what it would delete. An unparseable
|
||||
// `apply` is a 400 rather than a silent dry run, so a typo cannot pass for a
|
||||
// sweep that ran.
|
||||
//
|
||||
// Returns {success, dryRun, planned, fixed, byKey} — planned/fixed count
|
||||
// records, byKey counts fields per key. Safe to call repeatedly: a swept
|
||||
// record plans nothing on the next pass.
|
||||
func (s *Server) handleSweepLiteratureEchoes(w http.ResponseWriter, r *http.Request) {
|
||||
if !s.storage.IsConnected() {
|
||||
writeError(w, http.StatusServiceUnavailable, "storage not connected")
|
||||
return
|
||||
}
|
||||
|
||||
apply := false
|
||||
if v := r.URL.Query().Get("apply"); v != "" {
|
||||
parsed, err := strconv.ParseBool(v)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusBadRequest, "apply must be true or false")
|
||||
return
|
||||
}
|
||||
apply = parsed
|
||||
}
|
||||
|
||||
res, err := s.storage.SweepLiteratureEchoesWithTimeout(apply)
|
||||
if err != nil {
|
||||
writeError(w, http.StatusInternalServerError, err.Error())
|
||||
return
|
||||
}
|
||||
|
||||
writeJSON(w, http.StatusOK, map[string]interface{}{
|
||||
"success": true,
|
||||
"dryRun": !apply,
|
||||
"planned": res.Planned,
|
||||
"fixed": res.Fixed,
|
||||
"byKey": res.ByKey,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -0,0 +1,93 @@
|
||||
package api
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
|
||||
"github.com/alicebob/miniredis/v2"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// newSweepTestServer is a Server over miniredis with one literature card
|
||||
// carrying a copied `workcid`.
|
||||
func newSweepTestServer(t *testing.T) *Server {
|
||||
t.Helper()
|
||||
mr := miniredis.RunT(t)
|
||||
c := storage.NewClient("")
|
||||
if err := c.Connect("redis://" + mr.Addr()); err != nil {
|
||||
t.Fatalf("connect: %v", err)
|
||||
}
|
||||
t.Cleanup(func() { _ = c.Close() })
|
||||
for k, v := range map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"workcid": "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy",
|
||||
} {
|
||||
if err := c.SetProperty("card-1", k, v); err != nil {
|
||||
t.Fatalf("seed %s: %v", k, err)
|
||||
}
|
||||
}
|
||||
return &Server{storage: c}
|
||||
}
|
||||
|
||||
type sweepResponse struct {
|
||||
Success bool `json:"success"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
Planned int `json:"planned"`
|
||||
Fixed int `json:"fixed"`
|
||||
ByKey map[string]int `json:"byKey"`
|
||||
}
|
||||
|
||||
func postSweep(t *testing.T, s *Server, query string) (*httptest.ResponseRecorder, sweepResponse) {
|
||||
t.Helper()
|
||||
rr := httptest.NewRecorder()
|
||||
req := httptest.NewRequest(http.MethodPost, "/api/admin/sweep-literature-echoes"+query, nil)
|
||||
s.handleSweepLiteratureEchoes(rr, req)
|
||||
var body sweepResponse
|
||||
if rr.Code == http.StatusOK {
|
||||
if err := json.Unmarshal(rr.Body.Bytes(), &body); err != nil {
|
||||
t.Fatalf("decode %s: %v", rr.Body.String(), err)
|
||||
}
|
||||
}
|
||||
return rr, body
|
||||
}
|
||||
|
||||
// A bare call must never delete: the default is the dry run.
|
||||
func TestSweepLiteratureEchoes_DefaultsToDryRun(t *testing.T) {
|
||||
s := newSweepTestServer(t)
|
||||
|
||||
rr, body := postSweep(t, s, "")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if !body.Success || !body.DryRun || body.Planned != 1 || body.Fixed != 0 || body.ByKey["workcid"] != 1 {
|
||||
t.Errorf("dry run = %+v", body)
|
||||
}
|
||||
if v, _ := s.storage.GetProperty("card-1", "workcid"); v == "" {
|
||||
t.Errorf("dry run deleted workcid")
|
||||
}
|
||||
|
||||
rr, body = postSweep(t, s, "?apply=true")
|
||||
if rr.Code != http.StatusOK {
|
||||
t.Fatalf("status = %d, body %s", rr.Code, rr.Body.String())
|
||||
}
|
||||
if body.DryRun || body.Planned != 1 || body.Fixed != 1 || body.ByKey["workcid"] != 1 {
|
||||
t.Errorf("apply = %+v", body)
|
||||
}
|
||||
if v, _ := s.storage.GetProperty("card-1", "workcid"); v != "" {
|
||||
t.Errorf("apply left workcid = %q", v)
|
||||
}
|
||||
}
|
||||
|
||||
// A typo must not pass for a sweep that ran.
|
||||
func TestSweepLiteratureEchoes_RejectsUnparseableApply(t *testing.T) {
|
||||
s := newSweepTestServer(t)
|
||||
|
||||
rr, _ := postSweep(t, s, "?apply=yes-please")
|
||||
if rr.Code != http.StatusBadRequest {
|
||||
t.Errorf("status = %d, want 400", rr.Code)
|
||||
}
|
||||
}
|
||||
@@ -256,6 +256,9 @@ func (s *Server) setupRoutes() {
|
||||
// One-shot vocabulary sweep: domain film|tv → screen, workForm backfill
|
||||
// (METADATA_KEYS.md §14.17). Idempotent.
|
||||
s.router.HandleFunc("/api/admin/migrate-domain-screen", s.handleMigrateDomainScreen).Methods("POST")
|
||||
// One-shot sweep of query echoes on literature cards (meta-read
|
||||
// reading-model.md §7). Dry-run unless ?apply=true. Idempotent.
|
||||
s.router.HandleFunc("/api/admin/sweep-literature-echoes", s.handleSweepLiteratureEchoes).Methods("POST")
|
||||
|
||||
// SSE event streams. Mediated mirror of the Redis Streams; consumers
|
||||
// reach them over HTTP instead of touching Redis directly. Auth-bypass
|
||||
|
||||
@@ -0,0 +1,92 @@
|
||||
package cid
|
||||
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"fmt"
|
||||
"strings"
|
||||
"unicode/utf8"
|
||||
)
|
||||
|
||||
// DecodeCard recovers the (source, id) pair a card locator (CodeCard, 0x1007)
|
||||
// wraps. Wire form:
|
||||
//
|
||||
// 0x01 ‖ varint(0x1007) ‖ 0x00 (identity mh) ‖ varint(len(digest)) ‖ digest
|
||||
// digest = varint(len(source)) ‖ source ‖ id (both UTF-8)
|
||||
//
|
||||
// Decode stops at the multihash code because ranking needs nothing past it;
|
||||
// reading *which work* a card addresses needs the digest too, so this is the
|
||||
// one place in Go that walks the whole thing.
|
||||
//
|
||||
// ⚠ Mirror of meta-read `card_cid::decode` and of meta-feeder-sdk
|
||||
// `compute_card_cid`. The vectors in card_test.go are pinned literals copied
|
||||
// from meta-read, not round-trips, so a drift on either side fails a test
|
||||
// instead of silently reading a different work out of the same CID.
|
||||
//
|
||||
// Identity multihash only: a 0x1007 CID carrying any other multihash is not
|
||||
// something anyone mints, and reading its digest as (source, id) would be
|
||||
// reading hash bytes as text. Bytes past the declared digest length are
|
||||
// ignored, as meta-read does.
|
||||
func DecodeCard(cidStr string) (source, id string, err error) {
|
||||
if len(cidStr) < 2 || cidStr[0] != 'b' {
|
||||
return "", "", fmt.Errorf("cid %q: not a multibase base32 CIDv1", cidStr)
|
||||
}
|
||||
raw, err := base32.StdEncoding.WithPadding(base32.NoPadding).DecodeString(strings.ToUpper(cidStr[1:]))
|
||||
if err != nil {
|
||||
return "", "", fmt.Errorf("cid %q: base32 decode: %w", cidStr, err)
|
||||
}
|
||||
|
||||
next := func(buf []byte, what string) (uint64, []byte, error) {
|
||||
v, n := binary.Uvarint(buf)
|
||||
if n <= 0 {
|
||||
return 0, nil, fmt.Errorf("cid %q: bad %s varint", cidStr, what)
|
||||
}
|
||||
return v, buf[n:], nil
|
||||
}
|
||||
|
||||
version, raw, err := next(raw, "version")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if version != 1 {
|
||||
return "", "", fmt.Errorf("cid %q: unsupported CID version %d", cidStr, version)
|
||||
}
|
||||
codec, raw, err := next(raw, "codec")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if codec != CodeCard {
|
||||
return "", "", fmt.Errorf("cid %q: codec 0x%x is not a card locator", cidStr, codec)
|
||||
}
|
||||
mh, raw, err := next(raw, "multihash-code")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if mh != 0 {
|
||||
return "", "", fmt.Errorf("cid %q: card locator with non-identity multihash 0x%x", cidStr, mh)
|
||||
}
|
||||
digestLen, raw, err := next(raw, "digest-length")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if digestLen > uint64(len(raw)) {
|
||||
return "", "", fmt.Errorf("cid %q: digest truncated", cidStr)
|
||||
}
|
||||
digest := raw[:digestLen]
|
||||
|
||||
sourceLen, rest, err := next(digest, "source-length")
|
||||
if err != nil {
|
||||
return "", "", err
|
||||
}
|
||||
if sourceLen > uint64(len(rest)) {
|
||||
return "", "", fmt.Errorf("cid %q: source truncated", cidStr)
|
||||
}
|
||||
source, id = string(rest[:sourceLen]), string(rest[sourceLen:])
|
||||
if source == "" || id == "" {
|
||||
return "", "", fmt.Errorf("cid %q: empty source or id", cidStr)
|
||||
}
|
||||
if !utf8.ValidString(source) || !utf8.ValidString(id) {
|
||||
return "", "", fmt.Errorf("cid %q: source or id is not UTF-8", cidStr)
|
||||
}
|
||||
return source, id, nil
|
||||
}
|
||||
@@ -0,0 +1,71 @@
|
||||
package cid
|
||||
|
||||
import (
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Pinned from meta-read `src/card_cid.rs` / meta-feeder-sdk `compute_card_cid`
|
||||
// — literals, not round-trips (see DecodeCard).
|
||||
const (
|
||||
goldenAnilistCard = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy"
|
||||
goldenMangadexCard = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
|
||||
)
|
||||
|
||||
// wireCID builds a CIDv1 from raw parts so the negative cases below can vary
|
||||
// one field at a time.
|
||||
func wireCID(codec, mh uint64, digest []byte) string {
|
||||
buf := []byte{0x01}
|
||||
buf = binary.AppendUvarint(buf, codec)
|
||||
buf = binary.AppendUvarint(buf, mh)
|
||||
buf = binary.AppendUvarint(buf, uint64(len(digest)))
|
||||
buf = append(buf, digest...)
|
||||
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(buf))
|
||||
}
|
||||
|
||||
func cardDigest(source, id string) []byte {
|
||||
d := binary.AppendUvarint(nil, uint64(len(source)))
|
||||
d = append(d, source...)
|
||||
return append(d, id...)
|
||||
}
|
||||
|
||||
func TestDecodeCard_GoldenVectors(t *testing.T) {
|
||||
for _, tc := range []struct {
|
||||
cid, source, id string
|
||||
}{
|
||||
{goldenAnilistCard, "anilist", "manga:105778"},
|
||||
{goldenMangadexCard, "mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"},
|
||||
} {
|
||||
source, id, err := DecodeCard(tc.cid)
|
||||
if err != nil {
|
||||
t.Fatalf("DecodeCard(%s): %v", tc.cid, err)
|
||||
}
|
||||
if source != tc.source || id != tc.id {
|
||||
t.Errorf("DecodeCard(%s) = (%q, %q), want (%q, %q)", tc.cid, source, id, tc.source, tc.id)
|
||||
}
|
||||
// The test encoder must agree with the pinned literal, or the negative
|
||||
// cases below are testing a different wire format.
|
||||
if got := wireCID(CodeCard, 0, cardDigest(tc.source, tc.id)); got != tc.cid {
|
||||
t.Errorf("wireCID(%q, %q) = %s, want %s", tc.source, tc.id, got, tc.cid)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestDecodeCard_RejectsEverythingElse(t *testing.T) {
|
||||
for name, s := range map[string]string{
|
||||
"url locator codec": wireCID(CodeURL, 0, cardDigest("anilist", "1")),
|
||||
"non-identity multihash": wireCID(CodeCard, CodeSha256, cardDigest("anilist", "1")),
|
||||
"empty id": wireCID(CodeCard, 0, cardDigest("anilist", "")),
|
||||
"source past digest": wireCID(CodeCard, 0, []byte{0x20, 'a'}),
|
||||
"not a CID": "title:berserk",
|
||||
"CIDv0": "QmYwAPJzv5CZsnA625s3Xf2nemtYgPpHdWEz79ojWnPbdG",
|
||||
"truncated": goldenAnilistCard[:12],
|
||||
"empty": "",
|
||||
} {
|
||||
if source, id, err := DecodeCard(s); err == nil {
|
||||
t.Errorf("%s: DecodeCard(%q) = (%q, %q), want error", name, s, source, id)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,326 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/redis/go-redis/v9"
|
||||
|
||||
"github.com/metazla/meta-core/internal/cid"
|
||||
)
|
||||
|
||||
// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
|
||||
// identity and position keys are resolved, never copied from a query, and a
|
||||
// card carries no position keys and no `workcid`.
|
||||
//
|
||||
// # Why a sweep
|
||||
//
|
||||
// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
|
||||
// own filter values onto what it returned. The gateway persists whatever passes
|
||||
// the filter, so the copy became a permanent binding — and stored records never
|
||||
// pick up the writer fix: a covered `(upstream, query)` never reaches the
|
||||
// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
|
||||
// one. So the copies already at rest are removed here, once.
|
||||
//
|
||||
// What the copies cost: an `anilistid` on a MangaDex site card claims an
|
||||
// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
|
||||
// anchor — the site card then competes with (and can hide) the real AniList
|
||||
// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
|
||||
// join and sort like a position inside one.
|
||||
//
|
||||
// # What is swept, and what deliberately is not
|
||||
//
|
||||
// - Only `domain=literature` + `fileType=card` records. Nothing else is read
|
||||
// for writing, whatever keys it carries.
|
||||
// - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
|
||||
// go from **every** such card: a card is a work, never a position in one,
|
||||
// so no legitimate writer puts them there.
|
||||
// - `anilistid` goes only from a **site card** — a card whose own `cids/`
|
||||
// key-set holds a site-form card locator and no locator of any other form.
|
||||
// On an anchor card the id *is* the card's identity (its locator is
|
||||
// `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
|
||||
// it: which half the id belongs to is not recoverable at rest, and leaving
|
||||
// a stamp is the safe direction.
|
||||
// - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
|
||||
// Matched by rule, and a query-copied stamp is indistinguishable at rest
|
||||
// from a title-matched one — deleting would drop real bindings with the
|
||||
// fake ones.
|
||||
//
|
||||
// # Contract
|
||||
//
|
||||
// Dry-run unless `apply`: the sweep deletes keys, so the default call only
|
||||
// reports. Idempotent: a swept record plans nothing on the next pass, so the
|
||||
// endpoint is safe to call repeatedly and on a fresh box.
|
||||
|
||||
// literatureEchoAnchorSources are the card-locator sources that name an id-bag
|
||||
// work (an anchor), never a site.
|
||||
//
|
||||
// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
|
||||
// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
|
||||
// excludes them and the two rules agree on every locator minted today. They are
|
||||
// listed so the rule reads on its own here.
|
||||
var literatureEchoAnchorSources = map[string]bool{
|
||||
"anilist": true,
|
||||
"openlibrary": true,
|
||||
"tmdb": true,
|
||||
"imdb": true,
|
||||
"musicbrainz": true,
|
||||
"mal": true,
|
||||
}
|
||||
|
||||
// literatureCardEchoFields are removed from every literature card carrying
|
||||
// them — position keys and the work reference, none of which a card may hold.
|
||||
var literatureCardEchoFields = []string{
|
||||
"workcid",
|
||||
"chapterNumber",
|
||||
"volumeNumber",
|
||||
"chapterStart",
|
||||
"chapterEnd",
|
||||
}
|
||||
|
||||
// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
|
||||
// then `anilistid`, then the echo fields in literatureCardEchoFields order.
|
||||
var literatureEchoReadFields = append(
|
||||
[]string{"domain", "fileType", "anilistid"},
|
||||
literatureCardEchoFields...,
|
||||
)
|
||||
|
||||
// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
|
||||
// records; ByKey counts fields per key — removed on an apply pass, that would
|
||||
// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
|
||||
type LiteratureEchoSweep struct {
|
||||
Planned int
|
||||
Fixed int
|
||||
ByKey map[string]int
|
||||
}
|
||||
|
||||
// literatureEchoPlan is one record's pending deletes, in a fixed order
|
||||
// (`anilistid` first, then literatureCardEchoFields order).
|
||||
type literatureEchoPlan struct {
|
||||
hashID string
|
||||
fields []string
|
||||
}
|
||||
|
||||
// isSiteFormCardSource is the site-form rule on an already-decoded source: not
|
||||
// an anchor source, and shaped like a host.
|
||||
func isSiteFormCardSource(source string) bool {
|
||||
return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
|
||||
}
|
||||
|
||||
// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
|
||||
// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
|
||||
func isSiteFormCardLocator(cidStr string) bool {
|
||||
source, _, err := cid.DecodeCard(cidStr)
|
||||
return err == nil && isSiteFormCardSource(source)
|
||||
}
|
||||
|
||||
// isSiteCard reports whether a card's field names prove it a site card: at
|
||||
// least one site-form `cids/` card locator and no card locator of another form.
|
||||
// Non-card members (a content CID) say nothing about which work the card is and
|
||||
// are ignored. An empty field list — a record with no field-name index — is
|
||||
// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
|
||||
func isSiteCard(fields []string) bool {
|
||||
site := false
|
||||
for _, f := range fields {
|
||||
member := cidFromKeysetField(f)
|
||||
if member == "" {
|
||||
continue
|
||||
}
|
||||
source, _, err := cid.DecodeCard(member)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
if !isSiteFormCardSource(source) {
|
||||
return false
|
||||
}
|
||||
site = true
|
||||
}
|
||||
return site
|
||||
}
|
||||
|
||||
// SweepLiteratureEchoes removes query echoes stored on literature cards (see
|
||||
// the file comment). With apply=false it only plans.
|
||||
//
|
||||
// Reads in batched passes (never a per-record SCAN — a single record read
|
||||
// costs a full keyspace sweep on this store), then deletes only the planned
|
||||
// fields, through DeleteProperty so the field-name index and the search-dirty
|
||||
// set stay consistent.
|
||||
func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
|
||||
res := LiteratureEchoSweep{ByKey: map[string]int{}}
|
||||
|
||||
plans, err := c.planLiteratureEchoes(ctx)
|
||||
if err != nil {
|
||||
return res, err
|
||||
}
|
||||
res.Planned = len(plans)
|
||||
|
||||
if !apply {
|
||||
for _, p := range plans {
|
||||
for _, f := range p.fields {
|
||||
res.ByKey[f]++
|
||||
}
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// Writes take the client lock themselves, so they happen after the read
|
||||
// phase has released it. ByKey counts what was actually deleted, so a
|
||||
// failure midway reports the true partial state.
|
||||
for _, p := range plans {
|
||||
// DeleteProperty carries its own per-call timeout; checking ctx here is
|
||||
// what makes the wrapper's deadline bound the whole write phase.
|
||||
if err := ctx.Err(); err != nil {
|
||||
return res, err
|
||||
}
|
||||
for _, f := range p.fields {
|
||||
if err := c.DeleteProperty(p.hashID, f); err != nil {
|
||||
return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
|
||||
}
|
||||
res.ByKey[f]++
|
||||
}
|
||||
res.Fixed++
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
|
||||
// planLiteratureEchoes is the read half: what would be deleted, without
|
||||
// deleting it.
|
||||
func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
|
||||
c.mu.RLock()
|
||||
defer c.mu.RUnlock()
|
||||
|
||||
if c.client == nil {
|
||||
return nil, fmt.Errorf("not connected")
|
||||
}
|
||||
|
||||
hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("smembers index: %w", err)
|
||||
}
|
||||
if len(hashIDs) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
// Pass 1: the gate and echo fields for every record, in a fixed order so
|
||||
// the MGET response decodes by position: keys[i*per+n] belongs to
|
||||
// hashIDs[i].
|
||||
per := len(literatureEchoReadFields)
|
||||
keys := make([]string, 0, len(hashIDs)*per)
|
||||
for _, h := range hashIDs {
|
||||
prefix := c.buildKeyPrefix(h)
|
||||
for _, f := range literatureEchoReadFields {
|
||||
keys = append(keys, prefix+f)
|
||||
}
|
||||
}
|
||||
|
||||
// Chunked so one MGET never carries the whole corpus.
|
||||
const chunk = 3000
|
||||
values := make([]interface{}, 0, len(keys))
|
||||
for start := 0; start < len(keys); start += chunk {
|
||||
end := start + chunk
|
||||
if end > len(keys) {
|
||||
end = len(keys)
|
||||
}
|
||||
got, err := c.client.MGet(ctx, keys[start:end]...).Result()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("mget: %w", err)
|
||||
}
|
||||
values = append(values, got...)
|
||||
}
|
||||
|
||||
// A nil slot is an absent key. A present key holding "" is still an echo
|
||||
// and still goes, so presence — not the value — decides.
|
||||
get := func(i int) (string, bool) {
|
||||
if i >= len(values) || values[i] == nil {
|
||||
return "", false
|
||||
}
|
||||
s, ok := values[i].(string)
|
||||
return s, ok
|
||||
}
|
||||
|
||||
type candidate struct {
|
||||
hashID string
|
||||
hasAnilist bool
|
||||
echoes []string
|
||||
}
|
||||
var cands []candidate
|
||||
var probe []string // candidates whose anilistid needs their cids/ key-set
|
||||
for i, h := range hashIDs {
|
||||
base := i * per
|
||||
domain, _ := get(base)
|
||||
fileType, _ := get(base + 1)
|
||||
if domain != "literature" || fileType != "card" {
|
||||
continue
|
||||
}
|
||||
cand := candidate{hashID: h}
|
||||
_, cand.hasAnilist = get(base + 2)
|
||||
for j, f := range literatureCardEchoFields {
|
||||
if _, ok := get(base + 3 + j); ok {
|
||||
cand.echoes = append(cand.echoes, f)
|
||||
}
|
||||
}
|
||||
if !cand.hasAnilist && len(cand.echoes) == 0 {
|
||||
continue
|
||||
}
|
||||
if cand.hasAnilist {
|
||||
probe = append(probe, h)
|
||||
}
|
||||
cands = append(cands, cand)
|
||||
}
|
||||
|
||||
// Pass 2: the field-name index of only the cards carrying an `anilistid`
|
||||
// — the `cids/` members are field NAMES, so the index is where they are
|
||||
// listed. Pipelined per chunk; still no SCAN.
|
||||
//
|
||||
// A card with no index (never touched since before the index existed, and
|
||||
// missed by the startup WarmFieldIndexes pass) reads as an empty set and
|
||||
// keeps its `anilistid`: proving it a site card would need exactly the
|
||||
// per-record SCAN this sweep refuses, and a stamp left in place is the
|
||||
// safe direction.
|
||||
siteCard := make(map[string]bool, len(probe))
|
||||
for start := 0; start < len(probe); start += mgetBatch {
|
||||
end := start + mgetBatch
|
||||
if end > len(probe) {
|
||||
end = len(probe)
|
||||
}
|
||||
pipe := c.client.Pipeline()
|
||||
cmds := make([]*redis.StringSliceCmd, 0, end-start)
|
||||
for _, h := range probe[start:end] {
|
||||
cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
|
||||
}
|
||||
if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
|
||||
return nil, fmt.Errorf("smembers fields: %w", err)
|
||||
}
|
||||
for k, cmd := range cmds {
|
||||
fields, err := cmd.Result()
|
||||
if err != nil && err != redis.Nil {
|
||||
return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
|
||||
}
|
||||
siteCard[probe[start+k]] = isSiteCard(fields)
|
||||
}
|
||||
}
|
||||
|
||||
plans := make([]literatureEchoPlan, 0, len(cands))
|
||||
for _, cand := range cands {
|
||||
var fields []string
|
||||
if cand.hasAnilist && siteCard[cand.hashID] {
|
||||
fields = append(fields, "anilistid")
|
||||
}
|
||||
fields = append(fields, cand.echoes...)
|
||||
if len(fields) > 0 {
|
||||
plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
|
||||
}
|
||||
}
|
||||
return plans, nil
|
||||
}
|
||||
|
||||
// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
|
||||
// handler. Generous timeout: the read phase is batched, but the write phase is
|
||||
// one round-trip per deleted field.
|
||||
func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
|
||||
defer cancel()
|
||||
return c.SweepLiteratureEchoes(ctx, apply)
|
||||
}
|
||||
@@ -0,0 +1,249 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/base32"
|
||||
"encoding/binary"
|
||||
"reflect"
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
// Pinned from meta-read `src/card_cid.rs` — literals, not round-trips.
|
||||
const (
|
||||
literatureAnchorCardCID = "bagdsaaaua5qw42lmnfzxi3lbnztwcorrga2tonzy" // ("anilist", "manga:105778")
|
||||
literatureSiteCardCID = "bagdsaabybrwwc3thmfsgk6bon5zgol3nmfxgoyjpmiygenzsgftgmlldgm4dqljugq4dmllbmeygmlldgjrdaytcgmzdcnjrgi"
|
||||
// A non-card CID a card may also carry.
|
||||
literatureContentCID = "bagacbabaec7v3fu2ygzh3e2sybg3fbzmisry2hbtpmck6vx3yftea6vzq35r4"
|
||||
)
|
||||
|
||||
// mintCardCID encodes a card locator the way meta-feeder-sdk
|
||||
// `compute_card_cid` does. Asserted against the pinned literals before use.
|
||||
func mintCardCID(source, id string) string {
|
||||
digest := binary.AppendUvarint(nil, uint64(len(source)))
|
||||
digest = append(digest, source...)
|
||||
digest = append(digest, id...)
|
||||
wire := []byte{0x01}
|
||||
wire = binary.AppendUvarint(wire, 0x1007)
|
||||
wire = append(wire, 0x00)
|
||||
wire = binary.AppendUvarint(wire, uint64(len(digest)))
|
||||
wire = append(wire, digest...)
|
||||
return "b" + strings.ToLower(base32.StdEncoding.WithPadding(base32.NoPadding).EncodeToString(wire))
|
||||
}
|
||||
|
||||
func TestIsSiteFormCardLocator(t *testing.T) {
|
||||
if got := mintCardCID("anilist", "manga:105778"); got != literatureAnchorCardCID {
|
||||
t.Fatalf("test encoder drifted: %s != %s", got, literatureAnchorCardCID)
|
||||
}
|
||||
if got := mintCardCID("mangadex.org", "/manga/b0b721ff-c388-4486-aa0f-c2b0bb321512"); got != literatureSiteCardCID {
|
||||
t.Fatalf("test encoder drifted: %s != %s", got, literatureSiteCardCID)
|
||||
}
|
||||
|
||||
for _, tc := range []struct {
|
||||
name string
|
||||
cid string
|
||||
want bool
|
||||
}{
|
||||
{"anilist anchor (golden)", literatureAnchorCardCID, false},
|
||||
{"mangadex site (golden)", literatureSiteCardCID, true},
|
||||
{"openlibrary anchor", mintCardCID("openlibrary", "OL45804W"), false},
|
||||
{"another site", mintCardCID("weebcentral.com", "/series/01J76XY/chainsaw-man"), true},
|
||||
// Not an anchor source, but not host-shaped either: meta-read reads it
|
||||
// as an id-bag key, so it must not be treated as a site.
|
||||
{"dotless unknown source", mintCardCID("suwayomi", "42"), false},
|
||||
{"content CID", literatureContentCID, false},
|
||||
{"title key", "title:chainsaw man", false},
|
||||
} {
|
||||
if got := isSiteFormCardLocator(tc.cid); got != tc.want {
|
||||
t.Errorf("%s: isSiteFormCardLocator = %v, want %v", tc.name, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// seedLiteratureCorpus writes one record per rule the sweep has to honour.
|
||||
func seedLiteratureCorpus(t *testing.T, c *Client) {
|
||||
t.Helper()
|
||||
|
||||
// A site card carrying every echo: all of it goes.
|
||||
seedRecord(t, c, "site-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"title": "Chainsaw Man",
|
||||
"cids/" + literatureSiteCardCID: "true",
|
||||
"anilistid": "105778",
|
||||
"workcid": literatureSiteCardCID,
|
||||
"chapterNumber": "12",
|
||||
"volumeNumber": "2",
|
||||
"chapterStart": "1",
|
||||
"chapterEnd": "20",
|
||||
})
|
||||
// An anchor card: its anilistid is its identity and stays; the position
|
||||
// echo and the workcid still go.
|
||||
seedRecord(t, c, "anchor-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"title": "Chainsaw Man",
|
||||
"cids/" + literatureAnchorCardCID: "true",
|
||||
"anilistid": "105778",
|
||||
"workcid": literatureAnchorCardCID,
|
||||
"chapterNumber": "3",
|
||||
})
|
||||
// A site card that is already clean: nothing planned.
|
||||
seedRecord(t, c, "clean-site-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"cids/" + literatureSiteCardCID: "true",
|
||||
})
|
||||
// Both locator forms: which one the id belongs to is unknowable, so it
|
||||
// stays.
|
||||
seedRecord(t, c, "merged-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"cids/" + literatureSiteCardCID: "true",
|
||||
"cids/" + literatureAnchorCardCID: "true",
|
||||
"anilistid": "105778",
|
||||
})
|
||||
// No card locator at all: not provably a site card.
|
||||
seedRecord(t, c, "unlocated-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"cids/" + literatureContentCID: "true",
|
||||
"anilistid": "105778",
|
||||
})
|
||||
// A stored empty value is still an echo.
|
||||
seedRecord(t, c, "blank-echo-card", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "card",
|
||||
"volumeNumber": "",
|
||||
})
|
||||
// A chapter is never swept, whatever it carries.
|
||||
seedRecord(t, c, "chapter", map[string]string{
|
||||
"domain": "literature",
|
||||
"fileType": "archive",
|
||||
"cids/" + literatureContentCID: "true",
|
||||
"anilistid": "105778",
|
||||
"workcid": literatureSiteCardCID,
|
||||
"chapterNumber": "12",
|
||||
})
|
||||
// Not literature: out of scope even though it is a card.
|
||||
seedRecord(t, c, "screen-card", map[string]string{
|
||||
"domain": "screen",
|
||||
"fileType": "card",
|
||||
"workcid": literatureSiteCardCID,
|
||||
"chapterNumber": "1",
|
||||
})
|
||||
}
|
||||
|
||||
func TestPlanLiteratureEchoes_ScopesToLiteratureCards(t *testing.T) {
|
||||
c, _ := newTestClient(t, "")
|
||||
seedLiteratureCorpus(t, c)
|
||||
|
||||
plans, err := c.planLiteratureEchoes(context.Background())
|
||||
if err != nil {
|
||||
t.Fatalf("plan: %v", err)
|
||||
}
|
||||
got := map[string][]string{}
|
||||
for _, p := range plans {
|
||||
got[p.hashID] = p.fields
|
||||
}
|
||||
want := map[string][]string{
|
||||
"site-card": {"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"},
|
||||
"anchor-card": {"workcid", "chapterNumber"},
|
||||
"blank-echo-card": {"volumeNumber"},
|
||||
}
|
||||
if !reflect.DeepEqual(got, want) {
|
||||
t.Errorf("plan =\n %v\nwant\n %v", got, want)
|
||||
}
|
||||
}
|
||||
|
||||
func TestSweepLiteratureEchoes_DryRunThenApply(t *testing.T) {
|
||||
c, s := newTestClient(t, "mm:")
|
||||
seedLiteratureCorpus(t, c)
|
||||
|
||||
wantByKey := map[string]int{
|
||||
"anilistid": 1,
|
||||
"workcid": 2,
|
||||
"chapterNumber": 2,
|
||||
"volumeNumber": 2,
|
||||
"chapterStart": 1,
|
||||
"chapterEnd": 1,
|
||||
}
|
||||
|
||||
// Dry run: counts, and nothing written.
|
||||
dry, err := c.SweepLiteratureEchoes(context.Background(), false)
|
||||
if err != nil {
|
||||
t.Fatalf("dry run: %v", err)
|
||||
}
|
||||
if dry.Planned != 3 || dry.Fixed != 0 {
|
||||
t.Errorf("dry run planned/fixed = %d/%d, want 3/0", dry.Planned, dry.Fixed)
|
||||
}
|
||||
if !reflect.DeepEqual(dry.ByKey, wantByKey) {
|
||||
t.Errorf("dry run byKey = %v, want %v", dry.ByKey, wantByKey)
|
||||
}
|
||||
if m, _ := c.GetMetadataFlat("site-card"); m["anilistid"] != "105778" {
|
||||
t.Fatalf("dry run deleted anilistid: %v", m)
|
||||
}
|
||||
|
||||
applied, err := c.SweepLiteratureEchoes(context.Background(), true)
|
||||
if err != nil {
|
||||
t.Fatalf("apply: %v", err)
|
||||
}
|
||||
if applied.Planned != 3 || applied.Fixed != 3 {
|
||||
t.Errorf("apply planned/fixed = %d/%d, want 3/3", applied.Planned, applied.Fixed)
|
||||
}
|
||||
if !reflect.DeepEqual(applied.ByKey, wantByKey) {
|
||||
t.Errorf("apply byKey = %v, want %v", applied.ByKey, wantByKey)
|
||||
}
|
||||
|
||||
site, err := c.GetMetadataFlat("site-card")
|
||||
if err != nil {
|
||||
t.Fatalf("read site-card: %v", err)
|
||||
}
|
||||
for _, k := range []string{"anilistid", "workcid", "chapterNumber", "volumeNumber", "chapterStart", "chapterEnd"} {
|
||||
if _, ok := site[k]; ok {
|
||||
t.Errorf("site-card still carries %s", k)
|
||||
}
|
||||
// The flat key and its field-name index entry must both be gone, or the
|
||||
// next read would MGET a ghost field.
|
||||
if s.Exists("mm:file:site-card/" + k) {
|
||||
t.Errorf("site-card flat key %s survived", k)
|
||||
}
|
||||
if member, _ := s.IsMember("mm:file:site-card/"+FieldsField, k); member {
|
||||
t.Errorf("site-card field index still lists %s", k)
|
||||
}
|
||||
}
|
||||
if site["title"] != "Chainsaw Man" || site["cids/"+literatureSiteCardCID] != "true" {
|
||||
t.Errorf("site-card lost fields it should keep: %v", site)
|
||||
}
|
||||
|
||||
anchor, _ := c.GetMetadataFlat("anchor-card")
|
||||
if anchor["anilistid"] != "105778" {
|
||||
t.Errorf("anchor-card anilistid = %q, want kept", anchor["anilistid"])
|
||||
}
|
||||
if _, ok := anchor["workcid"]; ok {
|
||||
t.Errorf("anchor-card still carries workcid")
|
||||
}
|
||||
|
||||
chapter, _ := c.GetMetadataFlat("chapter")
|
||||
if chapter["anilistid"] != "105778" || chapter["workcid"] != literatureSiteCardCID || chapter["chapterNumber"] != "12" {
|
||||
t.Errorf("chapter was touched: %v", chapter)
|
||||
}
|
||||
screen, _ := c.GetMetadataFlat("screen-card")
|
||||
if screen["workcid"] != literatureSiteCardCID {
|
||||
t.Errorf("screen-card was touched: %v", screen)
|
||||
}
|
||||
merged, _ := c.GetMetadataFlat("merged-card")
|
||||
if merged["anilistid"] != "105778" {
|
||||
t.Errorf("merged-card anilistid = %q, want kept", merged["anilistid"])
|
||||
}
|
||||
|
||||
// Idempotent: the endpoint is called by hand, and a second call must find
|
||||
// nothing — with byKey still an empty map, not nil.
|
||||
again, err := c.SweepLiteratureEchoes(context.Background(), true)
|
||||
if err != nil {
|
||||
t.Fatalf("second pass: %v", err)
|
||||
}
|
||||
if again.Planned != 0 || again.Fixed != 0 || again.ByKey == nil || len(again.ByKey) != 0 {
|
||||
t.Errorf("second pass = %+v, want nothing planned and an empty byKey", again)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user