meta-core/internal/storage/literature_echo_sweep.go

package storage

import (
	"context"
	"fmt"
	"strings"
	"time"

	"github.com/redis/go-redis/v9"

	"github.com/metazla/meta-core/internal/cid"
)

// The one-shot corpus sweep for meta-read `docs/reading-model.md` §7:
// identity and position keys are resolved, never copied from a query, and a
// card carries no position keys and no `workcid`.
//
// # Why a sweep
//
// A feeder answering a `workcid:` / `anilistid:` query could stamp the query's
// own filter values onto what it returned. The gateway persists whatever passes
// the filter, so the copy became a permanent binding — and stored records never
// pick up the writer fix: a covered `(upstream, query)` never reaches the
// feeder again, and the streaming dedup keeps meta-core's copy over a fresh
// one. So the copies already at rest are removed here, once.
//
// What the copies cost: an `anilistid` on a MangaDex site card claims an
// AniList binding nobody resolved, and meta-read reads a bare `anilistid` as an
// anchor — the site card then competes with (and can hide) the real AniList
// card for the same work. A `workcid` or `chapterNumber` on a card makes a work
// join and sort like a position inside one.
//
// # What is swept, and what deliberately is not
//
//   - Only `domain=literature` + `fileType=card` records. Nothing else is read
//     for writing, whatever keys it carries.
//   - `workcid`, `chapterNumber`, `volumeNumber`, `chapterStart`, `chapterEnd`
//     go from **every** such card: a card is a work, never a position in one,
//     so no legitimate writer puts them there.
//   - `anilistid` goes only from a **site card** — a card whose own `cids/`
//     key-set holds a site-form card locator and no locator of any other form.
//     On an anchor card the id *is* the card's identity (its locator is
//     `("anilist", …)`). A card holding both forms (a cross-source merge) keeps
//     it: which half the id belongs to is not recoverable at rest, and leaving
//     a stamp is the safe direction.
//   - **Chapters are never swept.** A bare `anilistid` on a chapter is graded
//     Matched by rule, and a query-copied stamp is indistinguishable at rest
//     from a title-matched one — deleting would drop real bindings with the
//     fake ones.
//
// # Contract
//
// Dry-run unless `apply`: the sweep deletes keys, so the default call only
// reports. Idempotent: a swept record plans nothing on the next pass, so the
// endpoint is safe to call repeatedly and on a fresh box.

// literatureEchoAnchorSources are the card-locator sources that name an id-bag
// work (an anchor), never a site.
//
// ⚠ Mirror of meta-read `card_cid::is_host_form`. meta-read spells out only
// `anilist` and `openlibrary`; the others carry no '.', so the '.' test already
// excludes them and the two rules agree on every locator minted today. They are
// listed so the rule reads on its own here.
var literatureEchoAnchorSources = map[string]bool{
	"anilist":     true,
	"openlibrary": true,
	"tmdb":        true,
	"imdb":        true,
	"musicbrainz": true,
	"mal":         true,
}

// literatureCardEchoFields are removed from every literature card carrying
// them — position keys and the work reference, none of which a card may hold.
var literatureCardEchoFields = []string{
	"workcid",
	"chapterNumber",
	"volumeNumber",
	"chapterStart",
	"chapterEnd",
}

// literatureEchoReadFields is the per-record MGET layout: the two gate fields,
// then `anilistid`, then the echo fields in literatureCardEchoFields order.
var literatureEchoReadFields = append(
	[]string{"domain", "fileType", "anilistid"},
	literatureCardEchoFields...,
)

// LiteratureEchoSweep is the result of one sweep pass. Planned and Fixed count
// records; ByKey counts fields per key — removed on an apply pass, that would
// be removed on a dry run. ByKey is never nil so it serialises as `{}`.
type LiteratureEchoSweep struct {
	Planned int
	Fixed   int
	ByKey   map[string]int
}

// literatureEchoPlan is one record's pending deletes, in a fixed order
// (`anilistid` first, then literatureCardEchoFields order).
type literatureEchoPlan struct {
	hashID string
	fields []string
}

// isSiteFormCardSource is the site-form rule on an already-decoded source: not
// an anchor source, and shaped like a host.
func isSiteFormCardSource(source string) bool {
	return !literatureEchoAnchorSources[source] && strings.Contains(source, ".")
}

// isSiteFormCardLocator reports whether a bare CID is a card locator (0x1007)
// whose source is a site (`mangadex.org`), not an anchor (`anilist`).
func isSiteFormCardLocator(cidStr string) bool {
	source, _, err := cid.DecodeCard(cidStr)
	return err == nil && isSiteFormCardSource(source)
}

// isSiteCard reports whether a card's field names prove it a site card: at
// least one site-form `cids/` card locator and no card locator of another form.
// Non-card members (a content CID) say nothing about which work the card is and
// are ignored. An empty field list — a record with no field-name index — is
// never a site card, which keeps its `anilistid`; see planLiteratureEchoes.
func isSiteCard(fields []string) bool {
	site := false
	for _, f := range fields {
		member := cidFromKeysetField(f)
		if member == "" {
			continue
		}
		source, _, err := cid.DecodeCard(member)
		if err != nil {
			continue
		}
		if !isSiteFormCardSource(source) {
			return false
		}
		site = true
	}
	return site
}

// SweepLiteratureEchoes removes query echoes stored on literature cards (see
// the file comment). With apply=false it only plans.
//
// Reads in batched passes (never a per-record SCAN — a single record read
// costs a full keyspace sweep on this store), then deletes only the planned
// fields, through DeleteProperty so the field-name index and the search-dirty
// set stay consistent.
func (c *Client) SweepLiteratureEchoes(ctx context.Context, apply bool) (LiteratureEchoSweep, error) {
	res := LiteratureEchoSweep{ByKey: map[string]int{}}

	plans, err := c.planLiteratureEchoes(ctx)
	if err != nil {
		return res, err
	}
	res.Planned = len(plans)

	if !apply {
		for _, p := range plans {
			for _, f := range p.fields {
				res.ByKey[f]++
			}
		}
		return res, nil
	}

	// Writes take the client lock themselves, so they happen after the read
	// phase has released it. ByKey counts what was actually deleted, so a
	// failure midway reports the true partial state.
	for _, p := range plans {
		// DeleteProperty carries its own per-call timeout; checking ctx here is
		// what makes the wrapper's deadline bound the whole write phase.
		if err := ctx.Err(); err != nil {
			return res, err
		}
		for _, f := range p.fields {
			if err := c.DeleteProperty(p.hashID, f); err != nil {
				return res, fmt.Errorf("delete %s on %s: %w", f, p.hashID, err)
			}
			res.ByKey[f]++
		}
		res.Fixed++
	}
	return res, nil
}

// planLiteratureEchoes is the read half: what would be deleted, without
// deleting it.
func (c *Client) planLiteratureEchoes(ctx context.Context) ([]literatureEchoPlan, error) {
	c.mu.RLock()
	defer c.mu.RUnlock()

	if c.client == nil {
		return nil, fmt.Errorf("not connected")
	}

	hashIDs, err := c.client.SMembers(ctx, c.buildIndexKey()).Result()
	if err != nil {
		return nil, fmt.Errorf("smembers index: %w", err)
	}
	if len(hashIDs) == 0 {
		return nil, nil
	}

	// Pass 1: the gate and echo fields for every record, in a fixed order so
	// the MGET response decodes by position: keys[i*per+n] belongs to
	// hashIDs[i].
	per := len(literatureEchoReadFields)
	keys := make([]string, 0, len(hashIDs)*per)
	for _, h := range hashIDs {
		prefix := c.buildKeyPrefix(h)
		for _, f := range literatureEchoReadFields {
			keys = append(keys, prefix+f)
		}
	}

	// Chunked so one MGET never carries the whole corpus.
	const chunk = 3000
	values := make([]interface{}, 0, len(keys))
	for start := 0; start < len(keys); start += chunk {
		end := start + chunk
		if end > len(keys) {
			end = len(keys)
		}
		got, err := c.client.MGet(ctx, keys[start:end]...).Result()
		if err != nil {
			return nil, fmt.Errorf("mget: %w", err)
		}
		values = append(values, got...)
	}

	// A nil slot is an absent key. A present key holding "" is still an echo
	// and still goes, so presence — not the value — decides.
	get := func(i int) (string, bool) {
		if i >= len(values) || values[i] == nil {
			return "", false
		}
		s, ok := values[i].(string)
		return s, ok
	}

	type candidate struct {
		hashID     string
		hasAnilist bool
		echoes     []string
	}
	var cands []candidate
	var probe []string // candidates whose anilistid needs their cids/ key-set
	for i, h := range hashIDs {
		base := i * per
		domain, _ := get(base)
		fileType, _ := get(base + 1)
		if domain != "literature" || fileType != "card" {
			continue
		}
		cand := candidate{hashID: h}
		_, cand.hasAnilist = get(base + 2)
		for j, f := range literatureCardEchoFields {
			if _, ok := get(base + 3 + j); ok {
				cand.echoes = append(cand.echoes, f)
			}
		}
		if !cand.hasAnilist && len(cand.echoes) == 0 {
			continue
		}
		if cand.hasAnilist {
			probe = append(probe, h)
		}
		cands = append(cands, cand)
	}

	// Pass 2: the field-name index of only the cards carrying an `anilistid`
	// — the `cids/` members are field NAMES, so the index is where they are
	// listed. Pipelined per chunk; still no SCAN.
	//
	// A card with no index (never touched since before the index existed, and
	// missed by the startup WarmFieldIndexes pass) reads as an empty set and
	// keeps its `anilistid`: proving it a site card would need exactly the
	// per-record SCAN this sweep refuses, and a stamp left in place is the
	// safe direction.
	siteCard := make(map[string]bool, len(probe))
	for start := 0; start < len(probe); start += mgetBatch {
		end := start + mgetBatch
		if end > len(probe) {
			end = len(probe)
		}
		pipe := c.client.Pipeline()
		cmds := make([]*redis.StringSliceCmd, 0, end-start)
		for _, h := range probe[start:end] {
			cmds = append(cmds, pipe.SMembers(ctx, c.buildFieldsKey(h)))
		}
		if _, err := pipe.Exec(ctx); err != nil && err != redis.Nil {
			return nil, fmt.Errorf("smembers fields: %w", err)
		}
		for k, cmd := range cmds {
			fields, err := cmd.Result()
			if err != nil && err != redis.Nil {
				return nil, fmt.Errorf("smembers fields %s: %w", probe[start+k], err)
			}
			siteCard[probe[start+k]] = isSiteCard(fields)
		}
	}

	plans := make([]literatureEchoPlan, 0, len(cands))
	for _, cand := range cands {
		var fields []string
		if cand.hasAnilist && siteCard[cand.hashID] {
			fields = append(fields, "anilistid")
		}
		fields = append(fields, cand.echoes...)
		if len(fields) > 0 {
			plans = append(plans, literatureEchoPlan{hashID: cand.hashID, fields: fields})
		}
	}
	return plans, nil
}

// SweepLiteratureEchoesWithTimeout is the convenience wrapper used by the API
// handler. Generous timeout: the read phase is batched, but the write phase is
// one round-trip per deleted field.
func (c *Client) SweepLiteratureEchoesWithTimeout(apply bool) (LiteratureEchoSweep, error) {
	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
	defer cancel()
	return c.SweepLiteratureEchoes(ctx, apply)
}