meta-core/internal/reports/reports.go

// Package reports is meta-core's bug-report / feedback inbox.
//
// Client apps (meta-watch, meta-read, …) build a report in the browser — the
// line the person typed, the page they were on, the build, the device, the
// tail of the client log, and whatever app-specific state explains the moment
// (a player pipeline, a reader position) — and their server forwards it here,
// server-to-server, to the box's own core. meta-core stores it, lists it and
// shows it on the dashboard's Reports page, so triage reads one inbox instead
// of one per app.
//
// # Why a third SQLite file
//
// Reports are diagnostics, not metadata. They do not belong in the truth
// database (signed claims only) nor in the cache (unsynchronised, never backed
// up, deleted as the recovery procedure). They live in their own file,
// reports.db, so neither a cache rebuild nor POST /api/metadata/clear touches
// them, and a full report table can never slow a record write.
//
// # What meta-core knows about a report
//
// Almost nothing, on purpose. A handful of top-level fields are lifted into
// columns so the inbox can filter and headline them (app, build, uid, page,
// route, cid, title, note); everything else is kept verbatim in body as JSON
// and rendered generically by the dashboard. An app can add a new block of
// state without a meta-core release.
//
// ⚠ A report is a piece of viewing/reading history (cids, titles, a uid, a
// timestamp). It stays on the box, behind the same perimeter as the rest of
// meta-core's API; nothing here is federated.
package reports

import (
	"context"
	"database/sql"
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"sort"
	"strings"
	"time"
	"unicode/utf8"

	"github.com/metazla/meta-core/internal/kv"
)

const schema = `
CREATE TABLE IF NOT EXISTS report(
  id          INTEGER PRIMARY KEY AUTOINCREMENT,
  ts          INTEGER NOT NULL,           -- when the report was filed (ms)
  received    INTEGER NOT NULL,           -- when meta-core stored it (ms)
  app         TEXT    NOT NULL,
  build       TEXT    NOT NULL DEFAULT '',
  uid         TEXT    NOT NULL DEFAULT '',
  page        TEXT    NOT NULL DEFAULT '',
  route       TEXT    NOT NULL DEFAULT '',
  cid         TEXT    NOT NULL DEFAULT '',
  title       TEXT    NOT NULL DEFAULT '',
  note        TEXT    NOT NULL DEFAULT '',
  status      TEXT    NOT NULL DEFAULT 'open',
  triage_note TEXT    NOT NULL DEFAULT '',
  updated     INTEGER NOT NULL DEFAULT 0,
  log_lines   INTEGER NOT NULL DEFAULT 0,
  bytes       INTEGER NOT NULL,
  body        TEXT    NOT NULL
) STRICT;
CREATE INDEX IF NOT EXISTS report_app_status ON report(app, status, id DESC);
CREATE INDEX IF NOT EXISTS report_status ON report(status, id DESC);
`

// Statuses a report moves through. "open" is where every report starts.
const (
	StatusOpen    = "open"
	StatusTriaged = "triaged"
	StatusClosed  = "closed"
)

// ValidStatus reports whether s is a status a report can be set to.
func ValidStatus(s string) bool {
	return s == StatusOpen || s == StatusTriaged || s == StatusClosed
}

// Bounds on what one report may carry. The string caps match the ones
// meta-watch applied when it kept reports itself, so moving the inbox loses
// nothing a report used to hold.
const (
	maxApp        = 32
	maxBuild      = 4000
	maxUID        = 128
	maxPage       = 32
	maxRoute      = 512
	maxCID        = 128
	maxTitle      = 200
	maxNote       = 1000
	maxTriageNote = 2000
	// MaxLogLines / MaxLogLineChars bound the client log tail.
	MaxLogLines     = 200
	MaxLogLineChars = 600
	// MaxBlockBytes bounds every other top-level block (context, device,
	// app-specific state). A larger block is replaced by a marker naming its
	// size, never dropped silently.
	MaxBlockBytes = 32 * 1024
	// maxBlocks bounds the number of top-level keys.
	maxBlocks = 64
	// filed-at plausibility window: a report spooled by an app while the core
	// was down arrives late but keeps the time it was filed.
	maxFiledAge    = 30 * 24 * time.Hour
	maxFiledFuture = 5 * time.Minute
)

// ErrNotFound is returned for an id that does not exist.
var ErrNotFound = errors.New("report not found")

// Summary is a report without its body: what the inbox list shows.
type Summary struct {
	ID         int64  `json:"id"`
	TS         int64  `json:"ts"`
	Received   int64  `json:"received"`
	App        string `json:"app"`
	Build      string `json:"build,omitempty"`
	UID        string `json:"uid,omitempty"`
	Page       string `json:"page,omitempty"`
	Route      string `json:"route,omitempty"`
	CID        string `json:"cid,omitempty"`
	Title      string `json:"title,omitempty"`
	Note       string `json:"note,omitempty"`
	Status     string `json:"status"`
	TriageNote string `json:"triage_note,omitempty"`
	Updated    int64  `json:"updated,omitempty"`
	LogLines   int    `json:"log_lines"`
	Bytes      int64  `json:"bytes"`
}

// Report is a stored report with its full body.
type Report struct {
	Summary
	Body json.RawMessage `json:"body"`
}

// Filter selects reports for List.
type Filter struct {
	App    string
	Status string
	Q      string // substring of note, title or cid
	Before int64  // only ids below this (paging); 0 = from the newest
	Limit  int
}

// Page is one page of List.
type Page struct {
	Reports []Summary        `json:"reports"`
	Apps    map[string]int64 `json:"apps"`   // every app with a report, and how many
	Counts  map[string]int64 `json:"counts"` // per status, within the App filter
	More    bool             `json:"more"`
}

// Store is the open reports database.
type Store struct {
	db       *kv.Store
	maxRows  int
	maxBytes int64
	now      func() time.Time
}

// Open opens (creating if needed) the reports database at path. maxRows and
// maxBytes bound what Prune keeps; <= 0 disables that bound.
func Open(path string, maxRows int, maxBytes int64) (*Store, error) {
	if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
		return nil, fmt.Errorf("reports dir: %w", err)
	}
	db, err := kv.Open(path, kv.Options{
		NoKVTable:     true,
		Schema:        schema,
		WriterCacheMB: 4,
		ReaderCacheMB: 2,
		Readers:       2,
	})
	if err != nil {
		return nil, fmt.Errorf("open reports %s: %w", path, err)
	}
	return &Store{db: db, maxRows: maxRows, maxBytes: maxBytes, now: time.Now}, nil
}

// Close closes the database.
func (s *Store) Close() error { return s.db.Close() }

// Path is the database file.
func (s *Store) Path() string { return s.db.Path() }

func opCtx() (context.Context, context.CancelFunc) {
	return context.WithTimeout(context.Background(), 30*time.Second)
}

// ── Normalisation ─────────────────────────────────────────────────────────

// Normalize turns a posted envelope into a storable report: the indexed
// fields lifted and clamped, the log tail bounded, every other block size-
// checked. It never fails on content an app might plausibly send; the only
// error is a report that names no app.
func Normalize(in map[string]json.RawMessage, now time.Time) (Report, error) {
	var r Report
	str := func(key string, n int) string {
		raw, ok := in[key]
		if !ok {
			return ""
		}
		var v string
		if json.Unmarshal(raw, &v) != nil {
			return ""
		}
		return clamp(strings.TrimSpace(v), n)
	}

	r.App = str("app", maxApp)
	if r.App == "" {
		return r, errors.New("app is required")
	}
	r.Build = str("build", maxBuild)
	r.UID = str("uid", maxUID)
	r.Page = str("page", maxPage)
	r.Route = str("route", maxRoute)
	r.CID = str("cid", maxCID)
	r.Title = str("title", maxTitle)
	r.Note = str("note", maxNote)
	r.Status = StatusOpen
	r.Received = now.UnixMilli()
	r.TS = r.Received
	if raw, ok := in["filedAt"]; ok {
		var f float64
		if json.Unmarshal(raw, &f) == nil {
			t := time.UnixMilli(int64(f))
			if t.After(now.Add(-maxFiledAge)) && t.Before(now.Add(maxFiledFuture)) {
				r.TS = t.UnixMilli()
			}
		}
	}

	// The body repeats the lifted fields (clamped), so a report exported as
	// JSON is complete on its own.
	body := make(map[string]any, len(in))
	for k, v := range map[string]string{
		"app": r.App, "build": r.Build, "uid": r.UID, "page": r.Page,
		"route": r.Route, "cid": r.CID, "title": r.Title, "note": r.Note,
	} {
		if v != "" {
			body[k] = v
		}
	}
	if raw, ok := in["log"]; ok {
		var lines []string
		if json.Unmarshal(raw, &lines) == nil {
			if len(lines) > MaxLogLines {
				lines = lines[len(lines)-MaxLogLines:]
			}
			for i, l := range lines {
				lines[i] = clamp(l, MaxLogLineChars)
			}
			if len(lines) > 0 {
				body["log"] = lines
				r.LogLines = len(lines)
			}
		}
	}
	// Remaining blocks in key order, so which ones survive maxBlocks does not
	// depend on map iteration.
	keys := make([]string, 0, len(in))
	for k := range in {
		if k != "log" && k != "filedAt" && !isLifted(k) {
			keys = append(keys, k)
		}
	}
	sort.Strings(keys)
	blocks := 0
	for _, k := range keys {
		raw := in[k]
		if string(raw) == "null" {
			continue
		}
		if blocks >= maxBlocks {
			break
		}
		blocks++
		key := clamp(k, 64)
		if len(raw) > MaxBlockBytes {
			body[key] = map[string]any{"truncated": true, "bytes": len(raw)}
			continue
		}
		body[key] = raw
	}

	b, err := json.Marshal(body)
	if err != nil {
		return r, fmt.Errorf("encode body: %w", err)
	}
	r.Body = b
	r.Bytes = int64(len(b))
	return r, nil
}

func isLifted(k string) bool {
	switch k {
	case "app", "build", "uid", "page", "route", "cid", "title", "note":
		return true
	}
	return false
}

// clamp keeps at most n characters (runes) of s, never splitting one.
func clamp(s string, n int) string {
	if utf8.RuneCountInString(s) <= n {
		return s
	}
	i := 0
	for j := range s {
		if i == n {
			return s[:j]
		}
		i++
	}
	return s
}

// ── Writes ────────────────────────────────────────────────────────────────

// Insert stores a normalised report and returns its id. The store is pruned
// to its bounds in the same transaction, so it never sits over them.
func (s *Store) Insert(r Report) (int64, error) {
	ctx, cancel := opCtx()
	defer cancel()
	var id int64
	err := s.db.Update(ctx, func(tx *kv.Tx) error {
		res, err := tx.Exec(`INSERT INTO report(ts, received, app, build, uid, page, route, cid, title, note,
			status, log_lines, bytes, body) VALUES(?,?,?,?,?,?,?,?,?,?,?,?,?,?)`,
			r.TS, r.Received, r.App, r.Build, r.UID, r.Page, r.Route, r.CID, r.Title, r.Note,
			StatusOpen, r.LogLines, r.Bytes, string(r.Body))
		if err != nil {
			return err
		}
		if id, err = res.LastInsertId(); err != nil {
			return err
		}
		_, err = s.prune(tx)
		return err
	})
	return id, err
}

// Update sets a report's status and/or triage note. A nil argument leaves
// that field as it is.
func (s *Store) Update(id int64, status, triageNote *string) (Report, error) {
	if status != nil && !ValidStatus(*status) {
		return Report{}, fmt.Errorf("invalid status %q", *status)
	}
	ctx, cancel := opCtx()
	defer cancel()
	err := s.db.Update(ctx, func(tx *kv.Tx) error {
		sets, args := []string{"updated = ?"}, []any{s.now().UnixMilli()}
		if status != nil {
			sets, args = append(sets, "status = ?"), append(args, *status)
		}
		if triageNote != nil {
			sets, args = append(sets, "triage_note = ?"), append(args, clamp(strings.TrimSpace(*triageNote), maxTriageNote))
		}
		res, err := tx.Exec(`UPDATE report SET `+strings.Join(sets, ", ")+` WHERE id = ?`, append(args, id)...)
		if err != nil {
			return err
		}
		if n, _ := res.RowsAffected(); n == 0 {
			return ErrNotFound
		}
		return nil
	})
	if err != nil {
		return Report{}, err
	}
	return s.Get(id)
}

// Delete removes a report.
func (s *Store) Delete(id int64) error {
	ctx, cancel := opCtx()
	defer cancel()
	return s.db.Update(ctx, func(tx *kv.Tx) error {
		res, err := tx.Exec(`DELETE FROM report WHERE id = ?`, id)
		if err != nil {
			return err
		}
		if n, _ := res.RowsAffected(); n == 0 {
			return ErrNotFound
		}
		return nil
	})
}

// Prune brings the store back within its bounds and returns how many reports
// it removed. Insert already prunes; this is for a bound lowered by config.
func (s *Store) Prune() (int, error) {
	ctx, cancel := opCtx()
	defer cancel()
	var n int
	err := s.db.Update(ctx, func(tx *kv.Tx) error {
		var err error
		n, err = s.prune(tx)
		return err
	})
	return n, err
}

// prune evicts closed reports first, oldest first, then any report, oldest
// first, until both the row and the byte bound hold. An open report is the
// last thing to go: it is the one nobody has read yet.
func (s *Store) prune(tx *kv.Tx) (int, error) {
	if s.maxRows <= 0 && s.maxBytes <= 0 {
		return 0, nil
	}
	var rows, bytes int64
	if err := tx.QueryRow(`SELECT count(*), coalesce(sum(bytes), 0) FROM report`).Scan(&rows, &bytes); err != nil {
		return 0, err
	}
	over := func() bool {
		return (s.maxRows > 0 && rows > int64(s.maxRows)) || (s.maxBytes > 0 && bytes > s.maxBytes)
	}
	if !over() {
		return 0, nil
	}
	q, err := tx.Query(`SELECT id, bytes FROM report ORDER BY (status = 'closed') DESC, id ASC`)
	if err != nil {
		return 0, err
	}
	var victims []int64
	for q.Next() && over() {
		var id, b int64
		if err := q.Scan(&id, &b); err != nil {
			q.Close()
			return 0, err
		}
		victims = append(victims, id)
		rows--
		bytes -= b
	}
	q.Close()
	for _, id := range victims {
		if _, err := tx.Exec(`DELETE FROM report WHERE id = ?`, id); err != nil {
			return 0, err
		}
	}
	return len(victims), nil
}

// ── Reads ─────────────────────────────────────────────────────────────────

const summaryCols = `id, ts, received, app, build, uid, page, route, cid, title, note, status, triage_note, updated, log_lines, bytes`

func scanSummary(sc interface{ Scan(...any) error }, extra ...any) (Summary, error) {
	var r Summary
	dst := append([]any{&r.ID, &r.TS, &r.Received, &r.App, &r.Build, &r.UID, &r.Page, &r.Route,
		&r.CID, &r.Title, &r.Note, &r.Status, &r.TriageNote, &r.Updated, &r.LogLines, &r.Bytes}, extra...)
	return r, sc.Scan(dst...)
}

// Get returns one report with its body.
func (s *Store) Get(id int64) (Report, error) {
	ctx, cancel := opCtx()
	defer cancel()
	var body string
	row := s.db.DB().QueryRowContext(ctx, `SELECT `+summaryCols+`, body FROM report WHERE id = ?`, id)
	sum, err := scanSummary(row, &body)
	if errors.Is(err, sql.ErrNoRows) {
		return Report{}, ErrNotFound
	}
	if err != nil {
		return Report{}, err
	}
	return Report{Summary: sum, Body: json.RawMessage(body)}, nil
}

// List returns one page of summaries, newest first, with the counts the
// inbox's filter bar needs.
func (s *Store) List(f Filter) (Page, error) {
	if f.Limit <= 0 || f.Limit > 200 {
		f.Limit = 50
	}
	ctx, cancel := opCtx()
	defer cancel()
	db := s.db.DB()

	var where []string
	var args []any
	if f.App != "" {
		where, args = append(where, "app = ?"), append(args, f.App)
	}
	appWhere, appArgs := append([]string(nil), where...), append([]any(nil), args...)
	if f.Status != "" {
		where, args = append(where, "status = ?"), append(args, f.Status)
	}
	if q := strings.TrimSpace(f.Q); q != "" {
		like := "%" + escapeLike(q) + "%"
		where = append(where, `(note LIKE ? ESCAPE '\' OR title LIKE ? ESCAPE '\' OR cid LIKE ? ESCAPE '\' OR uid LIKE ? ESCAPE '\')`)
		args = append(args, like, like, like, like)
	}
	if f.Before > 0 {
		where, args = append(where, "id < ?"), append(args, f.Before)
	}
	clause := ""
	if len(where) > 0 {
		clause = " WHERE " + strings.Join(where, " AND ")
	}

	page := Page{Reports: []Summary{}, Apps: map[string]int64{}, Counts: map[string]int64{}}
	rows, err := db.QueryContext(ctx, `SELECT `+summaryCols+` FROM report`+clause+` ORDER BY id DESC LIMIT ?`,
		append(args, f.Limit+1)...)
	if err != nil {
		return page, err
	}
	for rows.Next() {
		sum, err := scanSummary(rows)
		if err != nil {
			rows.Close()
			return page, err
		}
		page.Reports = append(page.Reports, sum)
	}
	rows.Close()
	if len(page.Reports) > f.Limit {
		page.Reports, page.More = page.Reports[:f.Limit], true
	}

	if err := groupCount(ctx, db, `SELECT app, count(*) FROM report GROUP BY app`, nil, page.Apps); err != nil {
		return page, err
	}
	statusQ := `SELECT status, count(*) FROM report`
	if len(appWhere) > 0 {
		statusQ += " WHERE " + strings.Join(appWhere, " AND ")
	}
	if err := groupCount(ctx, db, statusQ+` GROUP BY status`, appArgs, page.Counts); err != nil {
		return page, err
	}
	return page, nil
}

func groupCount(ctx context.Context, db *sql.DB, q string, args []any, into map[string]int64) error {
	rows, err := db.QueryContext(ctx, q, args...)
	if err != nil {
		return err
	}
	defer rows.Close()
	for rows.Next() {
		var k string
		var n int64
		if err := rows.Scan(&k, &n); err != nil {
			return err
		}
		into[k] = n
	}
	return rows.Err()
}

func escapeLike(s string) string {
	return strings.NewReplacer(`\`, `\\`, `%`, `\%`, `_`, `\_`).Replace(s)
}