From 84e70bda76a589fc29fea748524a51e5f13a9e41 Mon Sep 17 00:00:00 2001 From: Ein Anderssono Date: Mon, 15 Jun 2026 02:32:20 +0200 Subject: [PATCH] v0.9.2: add manifest repair --- CHANGELOG.md | 8 +++ Makefile | 2 +- README.md | 1 + RELEASE_NOTES.md | 14 ++--- USERGUIDE.md | 9 +++ cmd/photoscli/main.go | 91 ++++++++++++++++++++++++++++ cmd/photoscli/main_test.go | 118 +++++++++++++++++++++++++++++++++++++ 7 files changed, 235 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ea551a..32de9b8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page. +## v0.9.2 + +Manifest repair release. + +- Add `manifest repair --out ` to fill missing manifest size metadata from files on disk. +- Add `manifest repair --checksum sha256` to fill missing manifest checksums. +- Add `manifest repair --dry-run` to preview repairs without writing manifest updates. + ## v0.9.1 Deep checksum verification release. diff --git a/Makefile b/Makefile index 4ee5ce2..e330b1e 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ BINARY := ./bin/photoscli MODULE := gitea.k3s.k0.nu/tools/photocli -VERSION := 0.9.1 +VERSION := 0.9.2 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip RELEASE_NOTES := RELEASE_NOTES.md BRIDGE_DIR := bridge diff --git a/README.md b/README.md index 694fe03..76e64ce 100644 --- a/README.md +++ b/README.md @@ -118,6 +118,7 @@ Verify a backup later: ```bash ./bin/photoscli verify --out ./photos-backup --manifest sqlite ./bin/photoscli verify --out ./photos-backup --deep +./bin/photoscli manifest repair --out ./photos-backup --checksum sha256 --dry-run ``` ## Commands diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 864e1b4..dbd9a69 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,18 +1,18 @@ -# v0.9.1 +# v0.9.2 -This release adds deep checksum verification for manifest-backed backups. +This release adds manifest repair for existing backups. ## Highlights -- Add `verify --deep` to recompute SHA-256 checksums when manifest entries include checksums. -- Report `checksum-mismatch` when file contents differ from the manifest checksum. -- Keep normal `verify` unchanged unless `--deep` is selected. -- `--deep` works with JSONL and SQLite manifests. +- Add `manifest repair --out ` to fill missing size metadata from existing files. +- Add `manifest repair --checksum sha256` to fill missing checksums. +- Add `manifest repair --dry-run` to preview repairs without writing updates. +- Repair works with JSONL and SQLite manifests. ## Assets - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`). -- `photoscli-0.9.1-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. +- `photoscli-0.9.2-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. - `USERGUIDE.md`: standalone user guide. Intel Macs are not currently a supported release target. diff --git a/USERGUIDE.md b/USERGUIDE.md index 7db3819..7e45131 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -838,6 +838,15 @@ Deep-verify checksum-backed manifests: `--deep` recomputes SHA-256 only for manifest entries that already have checksum metadata. +Repair missing manifest metadata from files already present on disk: + +```bash +./bin/photoscli manifest repair --out ./PhotosBackup --checksum sha256 --dry-run +./bin/photoscli manifest repair --out ./PhotosBackup --checksum sha256 +``` + +Repair fills missing size/checksum metadata and skips missing, zero-byte, or unreadable files. + ## Safe Operating Practices - Run `--dry-run` before the first large backup. diff --git a/cmd/photoscli/main.go b/cmd/photoscli/main.go index b37695e..9c1495b 100644 --- a/cmd/photoscli/main.go +++ b/cmd/photoscli/main.go @@ -92,6 +92,8 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int { return cmdDiff(args[1:], stdout, stderr, bridge) case "verify": return cmdVerify(args[1:], stdout, stderr) + case "manifest": + return cmdManifest(args[1:], stdout, stderr) case "retry-failed": return cmdRetryFailed(args[1:], stdout, stderr, bridge) case "failures": @@ -137,6 +139,7 @@ USAGE photoscli report --out [--manifest jsonl|sqlite] photoscli diff --album-id --out [--manifest jsonl|sqlite] photoscli verify --out [--manifest jsonl|sqlite] + photoscli manifest repair --out [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run] photoscli retry-failed --out photoscli retry-failed --out --clear-on-success photoscli failures list --out @@ -182,6 +185,9 @@ COMMANDS --sidecar to require photoscli schema/generator and exported filename metadata. Add --deep to verify manifest SHA-256 checksums when present. + manifest repair --out [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run] + Fill missing manifest size/checksum metadata from files that exist on disk. + retry-failed --out Retry assets previously written to failures.jsonl. @@ -2383,6 +2389,91 @@ func verifySidecar(stdout io.Writer, outDir, id, checkPath string, strict bool) return 0 } +func cmdManifest(args []string, stdout, stderr io.Writer) int { + if len(args) < 1 || args[0] != "repair" { + fmt.Fprintln(stderr, "error: expected manifest repair") + return exitErr + } + outDir := flagVal(args, "--out") + if outDir == "" { + fmt.Fprintln(stderr, "error: --out is required") + return exitErr + } + checksumMode := flagValWithDefault(args, "--checksum", "none") + if checksumMode != "none" && checksumMode != "sha256" { + fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", checksumMode) + return exitErr + } + mf, err := manifest.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl")) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + m, err := manifest.Open(outDir, mf) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + defer m.Close() + reader := m.(manifest.EntryReader) + if !hasFlag(args, "--dry-run") { + if err := m.OpenAppend(); err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + } + repaired := 0 + skipped := 0 + for id, entry := range reader.Entries() { + checkPath := entry.Path + if checkPath == "" { + checkPath = entry.Filename + } + if checkPath == "" { + skipped++ + continue + } + fullPath := filepath.Join(outDir, checkPath) + info, err := statFunc(fullPath) + if err != nil || info.IsDir() || info.Size() == 0 { + skipped++ + continue + } + updated := entry + updated.ID = id + changed := false + if updated.Size != info.Size() { + updated.Size = info.Size() + changed = true + } + if checksumMode == "sha256" && updated.Checksum == "" { + checksum, err := fileSHA256(fullPath) + if err != nil { + skipped++ + continue + } + updated.Checksum = checksum + changed = true + } + if !changed { + continue + } + repaired++ + fmt.Fprintf(stdout, "%s\t%s\trepaired\n", id, checkPath) + if !hasFlag(args, "--dry-run") { + m.AddEntry(updated) + } + } + if !hasFlag(args, "--dry-run") { + if err := m.Save(); err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + } + fmt.Fprintf(stdout, "repaired\t%d\nskipped\t%d\n", repaired, skipped) + return exitOK +} + func cmdSidecar(args []string, stdout, stderr io.Writer) int { if len(args) < 1 || args[0] != "inspect" { fmt.Fprintln(stderr, "error: expected sidecar inspect ") diff --git a/cmd/photoscli/main_test.go b/cmd/photoscli/main_test.go index 59877c2..44e1af6 100644 --- a/cmd/photoscli/main_test.go +++ b/cmd/photoscli/main_test.go @@ -39,6 +39,15 @@ type errWriter struct{} func (errWriter) Write([]byte) (int, error) { return 0, fmt.Errorf("write") } +type testFileInfo struct{ size int64 } + +func (t testFileInfo) Name() string { return "test" } +func (t testFileInfo) Size() int64 { return t.size } +func (t testFileInfo) Mode() os.FileMode { return 0644 } +func (t testFileInfo) ModTime() time.Time { return time.Time{} } +func (t testFileInfo) IsDir() bool { return false } +func (t testFileInfo) Sys() any { return nil } + type noEntryManifest struct{} func (noEntryManifest) Has(string) bool { return false } @@ -4306,6 +4315,115 @@ func TestVerifyDeepChecksums(t *testing.T) { } } +func TestManifestRepair(t *testing.T) { + dir := t.TempDir() + if err := os.WriteFile(filepath.Join(dir, "photo.jpg"), []byte("abc"), 0644); err != nil { + t.Fatal(err) + } + m := manifest.LoadJSONL(dir) + if err := m.OpenAppend(); err != nil { + t.Fatal(err) + } + m.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Size: 0, Cloud: "local", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "missing", Filename: "missing.jpg", Path: "missing.jpg", Size: 0, Cloud: "local", Exported: time.Now().Unix()}) + m.Close() + out, stderr, rc := runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256", "--dry-run"}, &mockBridge{}) + if rc != exitOK || stderr != "" || !strings.Contains(out, "x1\tphoto.jpg\trepaired") || !strings.Contains(out, "skipped\t1") { + t.Fatalf("dry repair rc=%d out=%q stderr=%q", rc, out, stderr) + } + if got := manifest.LoadJSONL(dir).Entries()["x1"].Checksum; got != "" { + t.Fatalf("dry run wrote checksum %q", got) + } + out, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256"}, &mockBridge{}) + if rc != exitOK || stderr != "" || !strings.Contains(out, "repaired\t1") { + t.Fatalf("repair rc=%d out=%q stderr=%q", rc, out, stderr) + } + entry := manifest.LoadJSONL(dir).Entries()["x1"] + if entry.Size != 3 || entry.Checksum != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" { + t.Fatalf("entry not repaired: %+v", entry) + } + out, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256"}, &mockBridge{}) + if rc != exitOK || stderr != "" || !strings.Contains(out, "repaired\t0") { + t.Fatalf("second repair rc=%d out=%q stderr=%q", rc, out, stderr) + } +} + +func TestManifestRepairErrors(t *testing.T) { + dir := t.TempDir() + _, stderr, rc := runWith([]string{"manifest"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "expected manifest repair") { + t.Fatalf("manifest missing subcommand rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"manifest", "repair"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "--out") { + t.Fatalf("manifest repair missing out rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "bad"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "--checksum") { + t.Fatalf("manifest repair bad checksum rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--manifest", "bad"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "manifest") { + t.Fatalf("manifest repair bad manifest rc=%d stderr=%q", rc, stderr) + } + fileOut := filepath.Join(dir, "file-out") + if err := os.WriteFile(fileOut, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + _, stderr, rc = runWith([]string{"manifest", "repair", "--out", fileOut}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "error:") { + t.Fatalf("manifest repair open append rc=%d stderr=%q", rc, stderr) + } + badDBDir := t.TempDir() + if err := os.WriteFile(manifest.SQLitePath(badDBDir), []byte("not sqlite"), 0644); err != nil { + t.Fatal(err) + } + _, stderr, rc = runWith([]string{"manifest", "repair", "--out", badDBDir}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "error:") { + t.Fatalf("manifest repair open rc=%d stderr=%q", rc, stderr) + } + saveDir := t.TempDir() + sm := manifest.LoadJSONL(saveDir) + if err := sm.OpenAppend(); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(saveDir, "photo.jpg"), []byte("abc"), 0644); err != nil { + t.Fatal(err) + } + sm.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Exported: time.Now().Unix()}) + sm.Close() + oldHook := manifest.SetJSONLSaveHook(func() error { return fmt.Errorf("save") }) + _, stderr, rc = runWith([]string{"manifest", "repair", "--out", saveDir}, &mockBridge{}) + manifest.SetJSONLSaveHook(oldHook) + if rc != exitErr || !strings.Contains(stderr, "save") { + t.Fatalf("manifest repair save rc=%d stderr=%q", rc, stderr) + } +} + +func TestManifestRepairBranches(t *testing.T) { + dir := t.TempDir() + m := manifest.LoadJSONL(dir) + if err := m.OpenAppend(); err != nil { + t.Fatal(err) + } + m.AddEntry(manifest.Entry{ID: "empty"}) + m.AddEntry(manifest.Entry{ID: "stat-only", Filename: "missing.jpg", Path: "missing.jpg", Checksum: "", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "hash-fail", Filename: "hash.jpg", Path: "hash.jpg", Checksum: "", Exported: time.Now().Unix()}) + m.Close() + oldStat := statFunc + statFunc = func(path string) (os.FileInfo, error) { + if strings.HasSuffix(path, "hash.jpg") { + return testFileInfo{size: 3}, nil + } + return oldStat(path) + } + out, stderr, rc := runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256", "--dry-run"}, &mockBridge{}) + statFunc = oldStat + if rc != exitOK || stderr != "" || !strings.Contains(out, "skipped\t3") { + t.Fatalf("manifest repair branches rc=%d out=%q stderr=%q", rc, out, stderr) + } +} + func TestVerifySidecarBranches(t *testing.T) { dir := t.TempDir() subdir := filepath.Join(dir, "sub")