improved kv metacore
This commit is contained in:
Pierre
@@ -0,0 +1,143 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// PluginOutputArchivePrefix is where plugin-output files live inside the ZIP.
|
||||
// Trailing slash because ZIP entry names are forward-slash relative paths.
|
||||
const PluginOutputArchivePrefix = "cache/plugin-output/"
|
||||
|
||||
// addPluginOutput walks srcDir (typically $FILES_PATH/plugin) and copies every
|
||||
// file under it into zw at PluginOutputArchivePrefix + relPath.
|
||||
//
|
||||
// Returns the number of files added. A missing srcDir is not an error: it
|
||||
// just means no plugin has produced output yet, so the section stays empty.
|
||||
func addPluginOutput(zw *zip.Writer, srcDir string) (int, error) {
|
||||
info, err := os.Stat(srcDir)
|
||||
if errors.Is(err, os.ErrNotExist) {
|
||||
return 0, nil
|
||||
}
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("stat %s: %w", srcDir, err)
|
||||
}
|
||||
if !info.IsDir() {
|
||||
return 0, fmt.Errorf("%s is not a directory", srcDir)
|
||||
}
|
||||
|
||||
count := 0
|
||||
err = filepath.WalkDir(srcDir, func(path string, d os.DirEntry, walkErr error) error {
|
||||
if walkErr != nil {
|
||||
return walkErr
|
||||
}
|
||||
if d.IsDir() {
|
||||
return nil
|
||||
}
|
||||
rel, err := filepath.Rel(srcDir, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
entry := PluginOutputArchivePrefix + filepath.ToSlash(rel)
|
||||
|
||||
w, err := zw.Create(entry)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
if _, err := io.Copy(w, f); err != nil {
|
||||
return err
|
||||
}
|
||||
count++
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return count, fmt.Errorf("walk %s: %w", srcDir, err)
|
||||
}
|
||||
return count, nil
|
||||
}
|
||||
|
||||
// extractPluginOutput pulls every entry under PluginOutputArchivePrefix from
|
||||
// the ZIP and writes it under dstDir. Existing files are overwritten when
|
||||
// overwrite=true; otherwise they're left alone (treats existing files as the
|
||||
// "mine wins" choice — same toggle the metadata import uses).
|
||||
//
|
||||
// Path traversal is rejected: any entry whose relative path resolves outside
|
||||
// dstDir is skipped with a warning.
|
||||
func extractPluginOutput(zr *zip.Reader, dstDir string, overwrite bool) (written int, skipped int, err error) {
|
||||
if dstDir == "" {
|
||||
return 0, 0, fmt.Errorf("plugin-output destination dir is empty")
|
||||
}
|
||||
if err := os.MkdirAll(dstDir, 0o755); err != nil {
|
||||
return 0, 0, fmt.Errorf("mkdir %s: %w", dstDir, err)
|
||||
}
|
||||
|
||||
cleanRoot, err := filepath.Abs(dstDir)
|
||||
if err != nil {
|
||||
return 0, 0, fmt.Errorf("abs %s: %w", dstDir, err)
|
||||
}
|
||||
|
||||
for _, f := range zr.File {
|
||||
if !strings.HasPrefix(f.Name, PluginOutputArchivePrefix) {
|
||||
continue
|
||||
}
|
||||
if strings.HasSuffix(f.Name, "/") {
|
||||
continue
|
||||
}
|
||||
rel := strings.TrimPrefix(f.Name, PluginOutputArchivePrefix)
|
||||
dst := filepath.Join(cleanRoot, filepath.FromSlash(rel))
|
||||
|
||||
// Reject anything that escapes the dst root.
|
||||
if !strings.HasPrefix(dst, cleanRoot+string(os.PathSeparator)) && dst != cleanRoot {
|
||||
log.Printf("[Snapshot] import: skipping path-escaping entry %q", f.Name)
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
|
||||
if !overwrite {
|
||||
if _, err := os.Stat(dst); err == nil {
|
||||
skipped++
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
if err := os.MkdirAll(filepath.Dir(dst), 0o755); err != nil {
|
||||
return written, skipped, fmt.Errorf("mkdir for %s: %w", dst, err)
|
||||
}
|
||||
|
||||
if err := writeZipFile(f, dst); err != nil {
|
||||
return written, skipped, err
|
||||
}
|
||||
written++
|
||||
}
|
||||
return written, skipped, nil
|
||||
}
|
||||
|
||||
func writeZipFile(f *zip.File, dst string) error {
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return fmt.Errorf("open zip entry %s: %w", f.Name, err)
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
out, err := os.Create(dst)
|
||||
if err != nil {
|
||||
return fmt.Errorf("create %s: %w", dst, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
if _, err := io.Copy(out, rc); err != nil {
|
||||
return fmt.Errorf("write %s: %w", dst, err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,132 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// ExportOptions controls what an export includes.
|
||||
type ExportOptions struct {
|
||||
IncludePluginOutput bool
|
||||
// PluginOutputDir is the absolute path on disk that holds plugin-produced
|
||||
// artifacts (typically $FILES_PATH/plugin). Required when
|
||||
// IncludePluginOutput is true.
|
||||
PluginOutputDir string
|
||||
// SourceHost is recorded in the manifest so imports can attribute
|
||||
// where the snapshot came from (defaults to OS hostname).
|
||||
SourceHost string
|
||||
}
|
||||
|
||||
// Exporter writes snapshot ZIPs from a live Redis store.
|
||||
type Exporter struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewExporter(stor *storage.Client) *Exporter {
|
||||
return &Exporter{storage: stor}
|
||||
}
|
||||
|
||||
// Export streams a ZIP archive to w. The archive layout is:
|
||||
//
|
||||
// manifest.json
|
||||
// index.json
|
||||
// metadata/{cid}.json (one per known CID)
|
||||
//
|
||||
// Optional sections (hash-index, plugin-output) are gated by ExportOptions
|
||||
// but are not yet implemented — they live on filesystems meta-core does not
|
||||
// own, and need cross-service collection. The flag is plumbed for forward
|
||||
// compatibility but does nothing today.
|
||||
func (e *Exporter) Export(w io.Writer, opts ExportOptions) (Manifest, error) {
|
||||
if e.storage == nil || !e.storage.IsConnected() {
|
||||
return Manifest{}, fmt.Errorf("storage not connected")
|
||||
}
|
||||
|
||||
hashIDs, err := e.storage.GetAllHashIDs()
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("list hashIDs: %w", err)
|
||||
}
|
||||
|
||||
zw := zip.NewWriter(w)
|
||||
defer zw.Close()
|
||||
|
||||
// 1. manifest.json — written last after we know FileCount, but we need
|
||||
// the entry order to put it first in the archive for readers. Solve
|
||||
// by buffering in memory: it's tiny.
|
||||
host := opts.SourceHost
|
||||
if host == "" {
|
||||
if h, err := os.Hostname(); err == nil {
|
||||
host = h
|
||||
}
|
||||
}
|
||||
manifest := Manifest{
|
||||
SchemaVersion: ManifestSchemaVersion,
|
||||
GeneratedAt: time.Now().UTC(),
|
||||
SourceHost: host,
|
||||
FileCount: len(hashIDs),
|
||||
Includes: Includes{
|
||||
Metadata: true,
|
||||
PluginOutput: opts.IncludePluginOutput,
|
||||
},
|
||||
}
|
||||
if err := writeJSON(zw, ManifestFile, manifest); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write manifest: %w", err)
|
||||
}
|
||||
|
||||
// 2. index.json — flat list of CIDs. Redundant with metadata/ but
|
||||
// cheap and useful for readers that only need the keyset.
|
||||
if err := writeJSON(zw, IndexFile, IndexFileBody{HashIDs: hashIDs}); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write index: %w", err)
|
||||
}
|
||||
|
||||
// 3. metadata/{cid}.json — one nested document per CID.
|
||||
for _, hashID := range hashIDs {
|
||||
flat, err := e.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
log.Printf("[Snapshot] export: skipping %s: %v", hashID, err)
|
||||
continue
|
||||
}
|
||||
if len(flat) == 0 {
|
||||
continue
|
||||
}
|
||||
nested := Reconstruct(flat)
|
||||
entry := MetadataDir + hashID + ".json"
|
||||
if err := writeJSON(zw, entry, nested); err != nil {
|
||||
return Manifest{}, fmt.Errorf("write %s: %w", entry, err)
|
||||
}
|
||||
}
|
||||
|
||||
// 4. cache/plugin-output/* — optional. Walks the on-disk plugin dir and
|
||||
// streams each file into the ZIP. Heavy (~120 MB on a populated dev
|
||||
// stack), so it's gated by the ExportOptions toggle.
|
||||
if opts.IncludePluginOutput {
|
||||
if opts.PluginOutputDir == "" {
|
||||
return Manifest{}, fmt.Errorf("PluginOutputDir is required when IncludePluginOutput is set")
|
||||
}
|
||||
n, err := addPluginOutput(zw, opts.PluginOutputDir)
|
||||
if err != nil {
|
||||
return Manifest{}, fmt.Errorf("add plugin-output: %w", err)
|
||||
}
|
||||
log.Printf("[Snapshot] exported %d plugin-output files from %s", n, opts.PluginOutputDir)
|
||||
}
|
||||
|
||||
log.Printf("[Snapshot] exported %d files (host=%s, pluginOutput=%v)",
|
||||
len(hashIDs), host, opts.IncludePluginOutput)
|
||||
return manifest, nil
|
||||
}
|
||||
|
||||
func writeJSON(zw *zip.Writer, name string, body interface{}) error {
|
||||
w, err := zw.Create(name)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
enc := json.NewEncoder(w)
|
||||
enc.SetIndent("", " ")
|
||||
return enc.Encode(body)
|
||||
}
|
||||
@@ -0,0 +1,195 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// Flat keys use "/" as the separator. Property paths inside Redis are
|
||||
// stored as e.g. "fileinfo/streamdetails/video/0/codec".
|
||||
const flatSep = "/"
|
||||
|
||||
// numericKey matches a path segment that is a non-negative decimal integer.
|
||||
// Used to detect array slots (".../video/0/codec").
|
||||
var numericKey = regexp.MustCompile(`^\d+$`)
|
||||
|
||||
// Reconstruct rebuilds a nested JSON-shaped value from a flat property map.
|
||||
// Keys are property paths (no leading slash, no file:{cid}/ prefix).
|
||||
//
|
||||
// Sibling keys whose path segments are all sequential integers starting at 0
|
||||
// are emitted as JSON arrays. Other groups become JSON objects. Leaf values
|
||||
// are coerced from string to int/float/bool/null when they parse cleanly so
|
||||
// the exported JSON is human-readable; on import we round-trip back to strings.
|
||||
func Reconstruct(flat map[string]string) interface{} {
|
||||
root := map[string]interface{}{}
|
||||
for path, value := range flat {
|
||||
parts := strings.Split(path, flatSep)
|
||||
setPath(root, parts, parseValue(value))
|
||||
}
|
||||
return convertNumericMapsToArrays(root)
|
||||
}
|
||||
|
||||
// Flatten is the inverse of Reconstruct: walks a nested value and produces a
|
||||
// flat property map whose values are all strings (Redis-ready).
|
||||
//
|
||||
// Arrays become numeric path segments ("genre/0", "genre/1"). Nil/missing
|
||||
// values are stored as "" to preserve the field's presence — this matches
|
||||
// what flattenMetadata in the TS implementation does.
|
||||
func Flatten(value interface{}) map[string]string {
|
||||
out := map[string]string{}
|
||||
flattenInto(value, "", out)
|
||||
return out
|
||||
}
|
||||
|
||||
func setPath(root map[string]interface{}, parts []string, leaf interface{}) {
|
||||
if len(parts) == 0 {
|
||||
return
|
||||
}
|
||||
cur := root
|
||||
for i, part := range parts {
|
||||
if i == len(parts)-1 {
|
||||
cur[part] = leaf
|
||||
return
|
||||
}
|
||||
next, ok := cur[part].(map[string]interface{})
|
||||
if !ok {
|
||||
next = map[string]interface{}{}
|
||||
cur[part] = next
|
||||
}
|
||||
cur = next
|
||||
}
|
||||
}
|
||||
|
||||
func convertNumericMapsToArrays(v interface{}) interface{} {
|
||||
switch t := v.(type) {
|
||||
case map[string]interface{}:
|
||||
// Recurse first so child arrays are detected before parent.
|
||||
for k, child := range t {
|
||||
t[k] = convertNumericMapsToArrays(child)
|
||||
}
|
||||
if !looksLikeArray(t) {
|
||||
return t
|
||||
}
|
||||
// Sort numerically and emit as []interface{}.
|
||||
idx := make([]int, 0, len(t))
|
||||
for k := range t {
|
||||
n, _ := strconv.Atoi(k)
|
||||
idx = append(idx, n)
|
||||
}
|
||||
sort.Ints(idx)
|
||||
arr := make([]interface{}, len(idx))
|
||||
for i, n := range idx {
|
||||
arr[i] = t[strconv.Itoa(n)]
|
||||
}
|
||||
return arr
|
||||
case []interface{}:
|
||||
for i := range t {
|
||||
t[i] = convertNumericMapsToArrays(t[i])
|
||||
}
|
||||
return t
|
||||
default:
|
||||
return v
|
||||
}
|
||||
}
|
||||
|
||||
// looksLikeArray returns true when every key in m is a non-negative integer
|
||||
// and the keys form a contiguous range starting at 0. Empty maps return false
|
||||
// — we can't tell whether the original was {} or [], so prefer {}.
|
||||
func looksLikeArray(m map[string]interface{}) bool {
|
||||
if len(m) == 0 {
|
||||
return false
|
||||
}
|
||||
seen := make(map[int]bool, len(m))
|
||||
for k := range m {
|
||||
if !numericKey.MatchString(k) {
|
||||
return false
|
||||
}
|
||||
n, err := strconv.Atoi(k)
|
||||
if err != nil || n < 0 {
|
||||
return false
|
||||
}
|
||||
seen[n] = true
|
||||
}
|
||||
for i := 0; i < len(m); i++ {
|
||||
if !seen[i] {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return true
|
||||
}
|
||||
|
||||
func flattenInto(v interface{}, path string, out map[string]string) {
|
||||
switch t := v.(type) {
|
||||
case nil:
|
||||
out[path] = ""
|
||||
case map[string]interface{}:
|
||||
if len(t) == 0 && path != "" {
|
||||
// Empty objects don't have a Redis representation; skip.
|
||||
return
|
||||
}
|
||||
for k, child := range t {
|
||||
next := k
|
||||
if path != "" {
|
||||
next = path + flatSep + k
|
||||
}
|
||||
flattenInto(child, next, out)
|
||||
}
|
||||
case []interface{}:
|
||||
for i, child := range t {
|
||||
next := strconv.Itoa(i)
|
||||
if path != "" {
|
||||
next = path + flatSep + next
|
||||
}
|
||||
flattenInto(child, next, out)
|
||||
}
|
||||
case string:
|
||||
out[path] = t
|
||||
case bool:
|
||||
if t {
|
||||
out[path] = "true"
|
||||
} else {
|
||||
out[path] = "false"
|
||||
}
|
||||
case float64:
|
||||
// JSON numbers always decode to float64; emit ints without decimals.
|
||||
if t == float64(int64(t)) {
|
||||
out[path] = strconv.FormatInt(int64(t), 10)
|
||||
} else {
|
||||
out[path] = strconv.FormatFloat(t, 'f', -1, 64)
|
||||
}
|
||||
default:
|
||||
out[path] = fmt.Sprintf("%v", t)
|
||||
}
|
||||
}
|
||||
|
||||
// parseValue mirrors the TS reconstruct logic: convert obvious primitives
|
||||
// back to their typed form for nicer exported JSON, leave anything ambiguous
|
||||
// as a string so we don't lose data (e.g. "5.1" channel layouts).
|
||||
func parseValue(s string) interface{} {
|
||||
if s == "" {
|
||||
return nil
|
||||
}
|
||||
switch s {
|
||||
case "true":
|
||||
return true
|
||||
case "false":
|
||||
return false
|
||||
case "null":
|
||||
return nil
|
||||
}
|
||||
// Only treat as number when the string is a clean integer or decimal,
|
||||
// no leading zeros / trailing junk. Matches the TS regex.
|
||||
if isCleanNumber(s) {
|
||||
if n, err := strconv.ParseFloat(s, 64); err == nil {
|
||||
return n
|
||||
}
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
var cleanNumber = regexp.MustCompile(`^-?\d+(\.\d+)?$`)
|
||||
|
||||
func isCleanNumber(s string) bool { return cleanNumber.MatchString(s) }
|
||||
@@ -0,0 +1,258 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"archive/zip"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// Mode controls how an import reconciles existing CIDs.
|
||||
type Mode string
|
||||
|
||||
const (
|
||||
// ModeReplace deletes every existing property of an imported CID and
|
||||
// writes the snapshot's values verbatim. Untouched CIDs are left alone.
|
||||
ModeReplace Mode = "replace"
|
||||
// ModeMerge keeps unmentioned existing properties and only touches the
|
||||
// fields present in the snapshot. Conflict resolution within merge is
|
||||
// controlled by Conflict.
|
||||
ModeMerge Mode = "merge"
|
||||
)
|
||||
|
||||
// Conflict picks a winner for fields present on both sides during merge.
|
||||
type Conflict string
|
||||
|
||||
const (
|
||||
// ConflictMine keeps the existing (target) value when both sides have one.
|
||||
ConflictMine Conflict = "mine"
|
||||
// ConflictSource overwrites the existing value with the snapshot's value.
|
||||
ConflictSource Conflict = "source"
|
||||
)
|
||||
|
||||
// ImportOptions controls how Import applies a snapshot.
|
||||
type ImportOptions struct {
|
||||
Mode Mode
|
||||
Conflict Conflict
|
||||
DryRun bool
|
||||
// PluginOutputDir is the on-disk destination for cache/plugin-output/*
|
||||
// entries. When empty, plugin-output is silently skipped on import.
|
||||
PluginOutputDir string
|
||||
}
|
||||
|
||||
// FileResult summarizes what happened (or would happen, in dry-run) for one CID.
|
||||
type FileResult struct {
|
||||
HashID string `json:"hashId"`
|
||||
Added int `json:"added"` // new property keys written
|
||||
Updated int `json:"updated"` // existing property keys whose value changed
|
||||
Unchanged int `json:"unchanged"` // existing property keys whose value matched
|
||||
Kept int `json:"kept"` // existing property keys preserved by ConflictMine
|
||||
Deleted int `json:"deleted"` // existing property keys removed (replace mode)
|
||||
Error string `json:"error,omitempty"`
|
||||
}
|
||||
|
||||
// PluginOutputResult summarizes what happened to cache/plugin-output entries.
|
||||
type PluginOutputResult struct {
|
||||
Written int `json:"written"`
|
||||
Skipped int `json:"skipped"`
|
||||
}
|
||||
|
||||
// ImportResult is the response body for the import endpoint.
|
||||
type ImportResult struct {
|
||||
Mode Mode `json:"mode"`
|
||||
Conflict Conflict `json:"conflict,omitempty"`
|
||||
DryRun bool `json:"dryRun"`
|
||||
TotalFiles int `json:"totalFiles"`
|
||||
FilesOK int `json:"filesOk"`
|
||||
FilesFailed int `json:"filesFailed"`
|
||||
Files []FileResult `json:"files,omitempty"`
|
||||
PluginOutput *PluginOutputResult `json:"pluginOutput,omitempty"`
|
||||
}
|
||||
|
||||
// Importer applies snapshot ZIPs to a live Redis store.
|
||||
type Importer struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewImporter(stor *storage.Client) *Importer {
|
||||
return &Importer{storage: stor}
|
||||
}
|
||||
|
||||
// Import reads a ZIP from r (which must support io.ReaderAt; callers pass
|
||||
// *bytes.Reader after buffering the upload) and applies it to Redis.
|
||||
func (i *Importer) Import(r io.ReaderAt, size int64, opts ImportOptions) (*ImportResult, error) {
|
||||
if i.storage == nil || !i.storage.IsConnected() {
|
||||
return nil, fmt.Errorf("storage not connected")
|
||||
}
|
||||
if opts.Mode != ModeReplace && opts.Mode != ModeMerge {
|
||||
return nil, fmt.Errorf("invalid mode %q (want replace|merge)", opts.Mode)
|
||||
}
|
||||
if opts.Mode == ModeMerge && opts.Conflict != ConflictMine && opts.Conflict != ConflictSource {
|
||||
return nil, fmt.Errorf("invalid conflict %q (want mine|source)", opts.Conflict)
|
||||
}
|
||||
|
||||
zr, err := zip.NewReader(r, size)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("open zip: %w", err)
|
||||
}
|
||||
|
||||
var manifest Manifest
|
||||
if err := readJSON(zr, ManifestFile, &manifest); err != nil {
|
||||
return nil, fmt.Errorf("read manifest: %w", err)
|
||||
}
|
||||
if manifest.SchemaVersion > ManifestSchemaVersion {
|
||||
return nil, fmt.Errorf("snapshot schemaVersion %d exceeds supported %d",
|
||||
manifest.SchemaVersion, ManifestSchemaVersion)
|
||||
}
|
||||
|
||||
result := &ImportResult{
|
||||
Mode: opts.Mode,
|
||||
Conflict: opts.Conflict,
|
||||
DryRun: opts.DryRun,
|
||||
}
|
||||
|
||||
for _, f := range zr.File {
|
||||
if !strings.HasPrefix(f.Name, MetadataDir) || !strings.HasSuffix(f.Name, ".json") {
|
||||
continue
|
||||
}
|
||||
hashID := strings.TrimSuffix(strings.TrimPrefix(f.Name, MetadataDir), ".json")
|
||||
if hashID == "" {
|
||||
continue
|
||||
}
|
||||
fr := i.applyOne(f, hashID, opts)
|
||||
result.TotalFiles++
|
||||
if fr.Error != "" {
|
||||
result.FilesFailed++
|
||||
} else {
|
||||
result.FilesOK++
|
||||
}
|
||||
result.Files = append(result.Files, fr)
|
||||
}
|
||||
|
||||
// Plugin-output extraction. Only applied on real imports (not dry-run).
|
||||
// If the snapshot doesn't contain a cache/plugin-output/ section, this is
|
||||
// a no-op. If the destination is unset, we skip silently — the manifest's
|
||||
// pluginOutput flag tells the caller it was on the source side either way.
|
||||
if !opts.DryRun && opts.PluginOutputDir != "" && manifest.Includes.PluginOutput {
|
||||
// Existing files are kept under ConflictMine / replace-doesn't-apply,
|
||||
// overwritten under ConflictSource. Replace mode behaves like
|
||||
// "overwrite" for cache files (it's the destructive choice for
|
||||
// metadata; mirror that intent here).
|
||||
overwrite := opts.Mode == ModeReplace || opts.Conflict == ConflictSource
|
||||
written, skipped, err := extractPluginOutput(zr, opts.PluginOutputDir, overwrite)
|
||||
if err != nil {
|
||||
return result, fmt.Errorf("extract plugin-output: %w", err)
|
||||
}
|
||||
result.PluginOutput = &PluginOutputResult{Written: written, Skipped: skipped}
|
||||
log.Printf("[Snapshot] import: plugin-output written=%d skipped=%d", written, skipped)
|
||||
}
|
||||
|
||||
log.Printf("[Snapshot] import: mode=%s conflict=%s dryRun=%v files=%d ok=%d failed=%d",
|
||||
opts.Mode, opts.Conflict, opts.DryRun, result.TotalFiles, result.FilesOK, result.FilesFailed)
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func (i *Importer) applyOne(f *zip.File, hashID string, opts ImportOptions) FileResult {
|
||||
fr := FileResult{HashID: hashID}
|
||||
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
fr.Error = fmt.Sprintf("open entry: %v", err)
|
||||
return fr
|
||||
}
|
||||
defer rc.Close()
|
||||
|
||||
var nested interface{}
|
||||
if err := json.NewDecoder(rc).Decode(&nested); err != nil {
|
||||
fr.Error = fmt.Sprintf("decode json: %v", err)
|
||||
return fr
|
||||
}
|
||||
incoming := Flatten(nested)
|
||||
|
||||
existing, err := i.storage.GetMetadataFlat(hashID)
|
||||
if err != nil {
|
||||
fr.Error = fmt.Sprintf("read existing: %v", err)
|
||||
return fr
|
||||
}
|
||||
if existing == nil {
|
||||
existing = map[string]string{}
|
||||
}
|
||||
|
||||
switch opts.Mode {
|
||||
case ModeReplace:
|
||||
// Diff for reporting.
|
||||
toWrite := map[string]string{}
|
||||
for k, v := range incoming {
|
||||
toWrite[k] = v
|
||||
if cur, ok := existing[k]; ok {
|
||||
if cur == v {
|
||||
fr.Unchanged++
|
||||
} else {
|
||||
fr.Updated++
|
||||
}
|
||||
} else {
|
||||
fr.Added++
|
||||
}
|
||||
}
|
||||
for k := range existing {
|
||||
if _, ok := incoming[k]; !ok {
|
||||
fr.Deleted++
|
||||
}
|
||||
}
|
||||
if !opts.DryRun {
|
||||
if _, err := i.storage.DeleteMetadata(hashID); err != nil {
|
||||
fr.Error = fmt.Sprintf("delete existing: %v", err)
|
||||
return fr
|
||||
}
|
||||
if err := i.storage.SetMetadataFlat(hashID, toWrite); err != nil {
|
||||
fr.Error = fmt.Sprintf("write: %v", err)
|
||||
return fr
|
||||
}
|
||||
}
|
||||
|
||||
case ModeMerge:
|
||||
toWrite := map[string]string{}
|
||||
for k, v := range incoming {
|
||||
cur, present := existing[k]
|
||||
switch {
|
||||
case !present:
|
||||
toWrite[k] = v
|
||||
fr.Added++
|
||||
case cur == v:
|
||||
fr.Unchanged++
|
||||
case opts.Conflict == ConflictSource:
|
||||
toWrite[k] = v
|
||||
fr.Updated++
|
||||
case opts.Conflict == ConflictMine:
|
||||
fr.Kept++
|
||||
}
|
||||
}
|
||||
if !opts.DryRun && len(toWrite) > 0 {
|
||||
if _, err := i.storage.MergeMetadataFlat(hashID, toWrite); err != nil {
|
||||
fr.Error = fmt.Sprintf("merge: %v", err)
|
||||
return fr
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return fr
|
||||
}
|
||||
|
||||
func readJSON(zr *zip.Reader, name string, out interface{}) error {
|
||||
for _, f := range zr.File {
|
||||
if f.Name != name {
|
||||
continue
|
||||
}
|
||||
rc, err := f.Open()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer rc.Close()
|
||||
return json.NewDecoder(rc).Decode(out)
|
||||
}
|
||||
return fmt.Errorf("entry %q not found", name)
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
// Package snapshot implements export/import/wipe of MetaMesh metadata.
|
||||
//
|
||||
// The on-disk format is a ZIP archive containing one nested-JSON file per
|
||||
// known CID (cid_midhash256). Schema is defined by ManifestSchemaVersion;
|
||||
// readers must reject archives whose schemaVersion exceeds what they know.
|
||||
package snapshot
|
||||
|
||||
import "time"
|
||||
|
||||
// ManifestSchemaVersion is the on-disk format version. Bump when the layout
|
||||
// of metadata/{cid}.json or the manifest changes in a non-additive way.
|
||||
const ManifestSchemaVersion = 1
|
||||
|
||||
// MetadataDir is the path inside the ZIP that holds per-CID JSON files.
|
||||
const MetadataDir = "metadata/"
|
||||
|
||||
// ManifestFile is the path inside the ZIP that holds the manifest JSON.
|
||||
const ManifestFile = "manifest.json"
|
||||
|
||||
// IndexFile is the path inside the ZIP that holds the file index (CID list).
|
||||
const IndexFile = "index.json"
|
||||
|
||||
// Manifest is the metadata header stored at the root of every snapshot ZIP.
|
||||
type Manifest struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
GeneratedAt time.Time `json:"generatedAt"`
|
||||
SourceHost string `json:"sourceHost,omitempty"`
|
||||
MetaCoreVer string `json:"metaCoreVersion,omitempty"`
|
||||
FileCount int `json:"fileCount"`
|
||||
Includes Includes `json:"includes"`
|
||||
}
|
||||
|
||||
// Includes records which optional sections are present in a snapshot.
|
||||
type Includes struct {
|
||||
Metadata bool `json:"metadata"`
|
||||
PluginOutput bool `json:"pluginOutput,omitempty"`
|
||||
}
|
||||
|
||||
// IndexFileBody is the structure of index.json inside the ZIP.
|
||||
type IndexFileBody struct {
|
||||
HashIDs []string `json:"hashIds"`
|
||||
}
|
||||
@@ -0,0 +1,42 @@
|
||||
package snapshot
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
|
||||
"github.com/metazla/meta-core/internal/storage"
|
||||
)
|
||||
|
||||
// WipeScope picks which classes of data to clear.
|
||||
type WipeScope struct {
|
||||
Metadata bool // file:* keys + file:__index__
|
||||
}
|
||||
|
||||
// WipeResult reports what was cleared.
|
||||
type WipeResult struct {
|
||||
MetadataDeleted int64 `json:"metadataDeleted"`
|
||||
}
|
||||
|
||||
// Wiper clears metadata from Redis.
|
||||
type Wiper struct {
|
||||
storage *storage.Client
|
||||
}
|
||||
|
||||
func NewWiper(stor *storage.Client) *Wiper {
|
||||
return &Wiper{storage: stor}
|
||||
}
|
||||
|
||||
// Wipe applies the requested scope. Returns the count of files removed.
|
||||
func (w *Wiper) Wipe(scope WipeScope) (*WipeResult, error) {
|
||||
if w.storage == nil || !w.storage.IsConnected() {
|
||||
return nil, fmt.Errorf("storage not connected")
|
||||
}
|
||||
res := &WipeResult{}
|
||||
if scope.Metadata {
|
||||
n, err := w.storage.ClearAllMetadata()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("clear metadata: %w", err)
|
||||
}
|
||||
res.MetadataDeleted = n
|
||||
}
|
||||
return res, nil
|
||||
}
|
||||
Reference in New Issue
Block a user