v2.0.0: claims model - signed MMUR2 votes in a truth DB, merged view in a rebuildable cache DB; POST /api/claims, claims and merge routes, unsigned writes answer 410

2026-10-25 17:36:00 +00:00
parent b0adefb97b
commit 84af298792
78 changed files with 7203 additions and 5617 deletions
@@ -0,0 +1,196 @@
+package storage
+
+// Box-local file facts: where the watcher saw each file on this disk.
+//
+//	local_file(path) → midhash, size, mtime     (cache table, unsigned)
+//
+// These are facts about this disk, not about the content, so they are never
+// claims and never travel. meta-core cannot sign anyway, and a rescan of
+// /files rebuilds them. The midhash gets a cid_group entry in truth (local
+// naming, not a vote), and the record view joins filePath / sizeByte /
+// mtimeNano in as `file:<rid>/<field>` rows — which publishes the meta events
+// meta-fuse builds its tree from.
+
+import (
+	"context"
+	"database/sql"
+	"errors"
+	"fmt"
+
+	"github.com/metazla/meta-core/internal/kv"
+)
+
+const cacheTablesSchema = `
+CREATE TABLE IF NOT EXISTS claim_by_pid(
+  pid INTEGER NOT NULL,
+  cid TEXT    NOT NULL,
+  key TEXT    NOT NULL,
+  PRIMARY KEY (pid, cid, key)
+) WITHOUT ROWID;
+CREATE TABLE IF NOT EXISTS local_file(
+  path    TEXT    PRIMARY KEY,
+  midhash TEXT    NOT NULL,
+  size    INTEGER NOT NULL,
+  mtime   INTEGER NOT NULL
+) WITHOUT ROWID;
+CREATE INDEX IF NOT EXISTS local_file_midhash ON local_file(midhash, path);
+`
+
+// RegisterFileTuple records one physical file under its midhash256 CID, the
+// way the watcher does for every file it scans (and POST /api/files/register
+// for a box with the watcher off). Returns the record's rid and whether its
+// group was newly created.
+func (c *Client) RegisterFileTuple(absPath string, size, mtimeNano int64, midhash string) (string, bool, error) {
+	if midhash == "" {
+		return "", false, fmt.Errorf("midhash is required")
+	}
+	truth, err := c.truthStore()
+	if err != nil {
+		return "", false, err
+	}
+	var rid string
+	var created bool
+	ctx, cancel := opCtx()
+	defer cancel()
+	if err := truth.Update(ctx, func(tx *kv.Tx) error {
+		var err error
+		rid, created, err = ensureGroup(tx, midhash)
+		return err
+	}); err != nil {
+		return "", false, err
+	}
+	err = c.update(func(tx *kv.Tx) error {
+		var prev string
+		err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&prev)
+		if err != nil && !errors.Is(err, sql.ErrNoRows) {
+			return err
+		}
+		if _, err := tx.Exec(`INSERT INTO local_file(path, midhash, size, mtime) VALUES(?, ?, ?, ?)
+			ON CONFLICT(path) DO UPDATE SET midhash = excluded.midhash, size = excluded.size, mtime = excluded.mtime`,
+			absPath, midhash, size, mtimeNano); err != nil {
+			return err
+		}
+		if _, err := syncMember(tx, midhash, rid); err != nil {
+			return err
+		}
+		if err := c.remergeLocalTx(tx, rid); err != nil {
+			return err
+		}
+		if prev != "" && prev != midhash {
+			// The file at this path changed content: its old record loses it.
+			if old, _, err := tx.Get(tx.Context(), CIDIndexPrefix+prev); err == nil && old != "" && old != rid {
+				return c.remergeLocalTx(tx, old)
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return "", false, err
+	}
+	return rid, created, nil
+}
+
+// RemoveFileTuple forgets a path the watcher saw disappear. Only the local
+// facts go — the record's claims stay (facts about content stay true).
+func (c *Client) RemoveFileTuple(absPath string) error {
+	return c.update(func(tx *kv.Tx) error {
+		var midhash string
+		err := tx.QueryRow(`SELECT midhash FROM local_file WHERE path = ?`, absPath).Scan(&midhash)
+		if errors.Is(err, sql.ErrNoRows) {
+			return nil
+		}
+		if err != nil {
+			return err
+		}
+		if _, err := tx.Exec(`DELETE FROM local_file WHERE path = ?`, absPath); err != nil {
+			return err
+		}
+		rid, _, err := tx.Get(tx.Context(), CIDIndexPrefix+midhash)
+		if err != nil || rid == "" {
+			return err
+		}
+		return c.remergeLocalTx(tx, rid)
+	})
+}
+
+// remergeLocalTx recomputes the box-local fields of rid (and its search row
+// when filePath changed).
+func (c *Client) remergeLocalTx(tx *kv.Tx, rid string) error {
+	cids, err := cidsForRoot(tx.Context(), tx, rid)
+	if err != nil {
+		return err
+	}
+	searchChanged := false
+	for _, f := range localFields {
+		changed, err := c.remergeTx(tx, rid, cids, f)
+		if err != nil {
+			return err
+		}
+		searchChanged = searchChanged || (changed && affectsHaystack(f))
+	}
+	if searchChanged {
+		return refreshSearchTx(tx, rid)
+	}
+	return nil
+}
+
+// FileTuple holds the (rid, path, size, mtime) record for one file. Used by
+// the watcher hydrator at startup to avoid re-mid-hashing files whose tuple
+// matches what's already stored.
+type FileTuple struct {
+	HashID    string
+	FilePath  string
+	Size      int64
+	MtimeNano int64
+}
+
+// GetTuplesForAllFiles returns one FileTuple per local file.
+func (c *Client) GetTuplesForAllFiles() ([]FileTuple, error) {
+	s, err := c.kv()
+	if err != nil {
+		return nil, err
+	}
+	ctx, cancel := context.WithTimeout(context.Background(), 2*opTimeout)
+	defer cancel()
+	rows, err := s.DB().QueryContext(ctx,
+		`SELECT l.path, l.size, l.mtime, COALESCE(k.v, '') FROM local_file l
+		 LEFT JOIN kv k ON k.k = 'cid:' || l.midhash ORDER BY l.path`)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	var out []FileTuple
+	for rows.Next() {
+		var t FileTuple
+		if err := rows.Scan(&t.FilePath, &t.Size, &t.MtimeNano, &t.HashID); err != nil {
+			return nil, err
+		}
+		out = append(out, t)
+	}
+	return out, rows.Err()
+}
+
+// localPaths lists every local path whose midhash is one of cids.
+func localPaths(ctx context.Context, q interface {
+	QueryContext(context.Context, string, ...any) (*sql.Rows, error)
+}, cids []string) ([]string, error) {
+	var out []string
+	for _, cid := range cids {
+		rows, err := q.QueryContext(ctx, `SELECT path FROM local_file WHERE midhash = ? ORDER BY path`, cid)
+		if err != nil {
+			return nil, err
+		}
+		for rows.Next() {
+			var p string
+			if err := rows.Scan(&p); err != nil {
+				rows.Close()
+				return nil, err
+			}
+			out = append(out, p)
+		}
+		if err := rows.Close(); err != nil {
+			return nil, err
+		}
+	}
+	return out, nil
+}