v0.9.2: add manifest repair

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
 
+## v0.9.2
+
+Manifest repair release.
+
+- Add `manifest repair --out <dir>` to fill missing manifest size metadata from files on disk.
+- Add `manifest repair --checksum sha256` to fill missing manifest checksums.
+- Add `manifest repair --dry-run` to preview repairs without writing manifest updates.
+
+## v0.9.1
+
+Deep checksum verification release.
+
+- Add `verify --deep` to recompute SHA-256 checksums for manifest entries that have checksum metadata.
+- Report `checksum-mismatch` when disk contents differ from the manifest checksum.
+- Keep normal `verify` unchanged unless `--deep` is selected.
+
 ## v0.9.0
 
 Manifest checksum release.

Makefile

@@ -1,6 +1,6 @@
 BINARY     := ./bin/photoscli
 MODULE     := gitea.k3s.k0.nu/tools/photocli
-VERSION    := 0.9.0
+VERSION    := 0.9.2
 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip
 RELEASE_NOTES := RELEASE_NOTES.md
 BRIDGE_DIR := bridge

README.md

@@ -117,6 +117,8 @@ Verify a backup later:
 
 ```bash
 ./bin/photoscli verify --out ./photos-backup --manifest sqlite
+./bin/photoscli verify --out ./photos-backup --deep
+./bin/photoscli manifest repair --out ./photos-backup --checksum sha256 --dry-run
 ```
 
 ## Commands

RELEASE_NOTES.md

@@ -1,18 +1,18 @@
-# v0.9.0
+# v0.9.2
 
-This release starts the backup-integrity series with manifest checksums.
+This release adds manifest repair for existing backups.
 
 ## Highlights
 
-- Add opt-in `--checksum sha256` to store SHA-256 checksums in manifests.
-- Support checksums in both JSONL and SQLite manifests.
-- Add SQLite migration support for the new checksum column.
-- Keep checksum collection disabled by default with `--checksum none`.
+- Add `manifest repair --out <dir>` to fill missing size metadata from existing files.
+- Add `manifest repair --checksum sha256` to fill missing checksums.
+- Add `manifest repair --dry-run` to preview repairs without writing updates.
+- Repair works with JSONL and SQLite manifests.
 
 ## Assets
 
 - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`).
-- `photoscli-0.9.0-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
+- `photoscli-0.9.2-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
 - `USERGUIDE.md`: standalone user guide.
 
 Intel Macs are not currently a supported release target.

USERGUIDE.md

@@ -830,6 +830,23 @@ Store SHA-256 checksums in the manifest when exporting:
 
 Checksums are opt-in and stored in JSONL or SQLite manifests. The default is `--checksum none`.
 
+Deep-verify checksum-backed manifests:
+
+```bash
+./bin/photoscli verify --out ./PhotosBackup --deep
+```
+
+`--deep` recomputes SHA-256 only for manifest entries that already have checksum metadata.
+
+Repair missing manifest metadata from files already present on disk:
+
+```bash
+./bin/photoscli manifest repair --out ./PhotosBackup --checksum sha256 --dry-run
+./bin/photoscli manifest repair --out ./PhotosBackup --checksum sha256
+```
+
+Repair fills missing size/checksum metadata and skips missing, zero-byte, or unreadable files.
+
 ## Safe Operating Practices
 
 - Run `--dry-run` before the first large backup.

cmd/photoscli/main.go

@@ -92,6 +92,8 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
 		return cmdDiff(args[1:], stdout, stderr, bridge)
 	case "verify":
 		return cmdVerify(args[1:], stdout, stderr)
+	case "manifest":
+		return cmdManifest(args[1:], stdout, stderr)
 	case "retry-failed":
 		return cmdRetryFailed(args[1:], stdout, stderr, bridge)
 	case "failures":
@@ -137,6 +139,7 @@ USAGE
   photoscli report --out <dir> [--manifest jsonl|sqlite]
   photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
   photoscli verify --out <dir> [--manifest jsonl|sqlite]
+  photoscli manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
   photoscli retry-failed --out <dir>
   photoscli retry-failed --out <dir> --clear-on-success
   photoscli failures list --out <dir>
@@ -180,6 +183,10 @@ COMMANDS
       files are printed as <asset-id><TAB><filename>. Exits 2 on missing files.
       Add --sidecar to verify expected XMP sidecars too. Add --strict with
       --sidecar to require photoscli schema/generator and exported filename metadata.
+      Add --deep to verify manifest SHA-256 checksums when present.
+
+  manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
+      Fill missing manifest size/checksum metadata from files that exist on disk.
 
   retry-failed --out <dir>
       Retry assets previously written to failures.jsonl.
@@ -2279,6 +2286,7 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
 	outDir := flagVal(args, "--out")
 	checkSidecar := hasFlag(args, "--sidecar")
 	strictSidecar := hasFlag(args, "--strict")
+	deep := hasFlag(args, "--deep")
 	if outDir == "" {
 		fmt.Fprintln(stderr, "error: --out is required")
 		return exitErr
@@ -2317,6 +2325,16 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
 			bad++
 			fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
 		}
+		if deep && e.Checksum != "" {
+			checksum, err := fileSHA256(filepath.Join(outDir, checkPath))
+			if err != nil {
+				bad++
+				fmt.Fprintf(stdout, "%s\t%s\tchecksum-unreadable\n", id, checkPath)
+			} else if checksum != e.Checksum {
+				bad++
+				fmt.Fprintf(stdout, "%s\t%s\tchecksum-mismatch\tmanifest=%s\tdisk=%s\n", id, checkPath, e.Checksum, checksum)
+			}
+		}
 		if checkSidecar {
 			bad += verifySidecar(stdout, outDir, id, checkPath, strictSidecar)
 		}
@@ -2371,6 +2389,91 @@ func verifySidecar(stdout io.Writer, outDir, id, checkPath string, strict bool)
 	return 0
 }
 
+func cmdManifest(args []string, stdout, stderr io.Writer) int {
+	if len(args) < 1 || args[0] != "repair" {
+		fmt.Fprintln(stderr, "error: expected manifest repair")
+		return exitErr
+	}
+	outDir := flagVal(args, "--out")
+	if outDir == "" {
+		fmt.Fprintln(stderr, "error: --out is required")
+		return exitErr
+	}
+	checksumMode := flagValWithDefault(args, "--checksum", "none")
+	if checksumMode != "none" && checksumMode != "sha256" {
+		fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", checksumMode)
+		return exitErr
+	}
+	mf, err := manifest.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	m, err := manifest.Open(outDir, mf)
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	defer m.Close()
+	reader := m.(manifest.EntryReader)
+	if !hasFlag(args, "--dry-run") {
+		if err := m.OpenAppend(); err != nil {
+			fmt.Fprintf(stderr, "error: %v\n", err)
+			return exitErr
+		}
+	}
+	repaired := 0
+	skipped := 0
+	for id, entry := range reader.Entries() {
+		checkPath := entry.Path
+		if checkPath == "" {
+			checkPath = entry.Filename
+		}
+		if checkPath == "" {
+			skipped++
+			continue
+		}
+		fullPath := filepath.Join(outDir, checkPath)
+		info, err := statFunc(fullPath)
+		if err != nil || info.IsDir() || info.Size() == 0 {
+			skipped++
+			continue
+		}
+		updated := entry
+		updated.ID = id
+		changed := false
+		if updated.Size != info.Size() {
+			updated.Size = info.Size()
+			changed = true
+		}
+		if checksumMode == "sha256" && updated.Checksum == "" {
+			checksum, err := fileSHA256(fullPath)
+			if err != nil {
+				skipped++
+				continue
+			}
+			updated.Checksum = checksum
+			changed = true
+		}
+		if !changed {
+			continue
+		}
+		repaired++
+		fmt.Fprintf(stdout, "%s\t%s\trepaired\n", id, checkPath)
+		if !hasFlag(args, "--dry-run") {
+			m.AddEntry(updated)
+		}
+	}
+	if !hasFlag(args, "--dry-run") {
+		if err := m.Save(); err != nil {
+			fmt.Fprintf(stderr, "error: %v\n", err)
+			return exitErr
+		}
+	}
+	fmt.Fprintf(stdout, "repaired\t%d\nskipped\t%d\n", repaired, skipped)
+	return exitOK
+}
+
 func cmdSidecar(args []string, stdout, stderr io.Writer) int {
 	if len(args) < 1 || args[0] != "inspect" {
 		fmt.Fprintln(stderr, "error: expected sidecar inspect <file.xmp>")

cmd/photoscli/main_test.go

@@ -39,6 +39,15 @@ type errWriter struct{}
 
 func (errWriter) Write([]byte) (int, error) { return 0, fmt.Errorf("write") }
 
+type testFileInfo struct{ size int64 }
+
+func (t testFileInfo) Name() string       { return "test" }
+func (t testFileInfo) Size() int64        { return t.size }
+func (t testFileInfo) Mode() os.FileMode  { return 0644 }
+func (t testFileInfo) ModTime() time.Time { return time.Time{} }
+func (t testFileInfo) IsDir() bool        { return false }
+func (t testFileInfo) Sys() any           { return nil }
+
 type noEntryManifest struct{}
 
 func (noEntryManifest) Has(string) bool                   { return false }
@@ -4273,6 +4282,148 @@ func TestMoreIntegrityBranches(t *testing.T) {
 	}
 }
 
+func TestVerifyDeepChecksums(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "good.jpg"), []byte("abc"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "bad.jpg"), []byte("bad"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(dir, "nocheck.jpg"), []byte("data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(filepath.Join(dir, "adir"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	m := manifest.LoadJSONL(dir)
+	if err := m.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	m.AddEntry(manifest.Entry{ID: "good", Filename: "good.jpg", Path: "good.jpg", Size: 3, Cloud: "local", Checksum: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "bad", Filename: "bad.jpg", Path: "bad.jpg", Size: 3, Cloud: "local", Checksum: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "nocheck", Filename: "nocheck.jpg", Path: "nocheck.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "unreadable", Filename: "adir", Path: "adir", Cloud: "local", Checksum: "abc", Exported: time.Now().Unix()})
+	m.Close()
+	out, stderr, rc := runWith([]string{"verify", "--out", dir, "--deep"}, &mockBridge{})
+	if rc != exitPartial || stderr != "" || !strings.Contains(out, "checksum-mismatch") || !strings.Contains(out, "checksum-unreadable") || strings.Contains(out, "good.jpg") || strings.Contains(out, "nocheck") {
+		t.Fatalf("deep verify rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	out, stderr, rc = runWith([]string{"verify", "--out", dir}, &mockBridge{})
+	if rc != exitOK || stderr != "" || strings.Contains(out, "checksum") {
+		t.Fatalf("plain verify rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+}
+
+func TestManifestRepair(t *testing.T) {
+	dir := t.TempDir()
+	if err := os.WriteFile(filepath.Join(dir, "photo.jpg"), []byte("abc"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	m := manifest.LoadJSONL(dir)
+	if err := m.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	m.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Size: 0, Cloud: "local", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "missing", Filename: "missing.jpg", Path: "missing.jpg", Size: 0, Cloud: "local", Exported: time.Now().Unix()})
+	m.Close()
+	out, stderr, rc := runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256", "--dry-run"}, &mockBridge{})
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "x1\tphoto.jpg\trepaired") || !strings.Contains(out, "skipped\t1") {
+		t.Fatalf("dry repair rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	if got := manifest.LoadJSONL(dir).Entries()["x1"].Checksum; got != "" {
+		t.Fatalf("dry run wrote checksum %q", got)
+	}
+	out, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256"}, &mockBridge{})
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "repaired\t1") {
+		t.Fatalf("repair rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	entry := manifest.LoadJSONL(dir).Entries()["x1"]
+	if entry.Size != 3 || entry.Checksum != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" {
+		t.Fatalf("entry not repaired: %+v", entry)
+	}
+	out, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256"}, &mockBridge{})
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "repaired\t0") {
+		t.Fatalf("second repair rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+}
+
+func TestManifestRepairErrors(t *testing.T) {
+	dir := t.TempDir()
+	_, stderr, rc := runWith([]string{"manifest"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "expected manifest repair") {
+		t.Fatalf("manifest missing subcommand rc=%d stderr=%q", rc, stderr)
+	}
+	_, stderr, rc = runWith([]string{"manifest", "repair"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "--out") {
+		t.Fatalf("manifest repair missing out rc=%d stderr=%q", rc, stderr)
+	}
+	_, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "bad"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "--checksum") {
+		t.Fatalf("manifest repair bad checksum rc=%d stderr=%q", rc, stderr)
+	}
+	_, stderr, rc = runWith([]string{"manifest", "repair", "--out", dir, "--manifest", "bad"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "manifest") {
+		t.Fatalf("manifest repair bad manifest rc=%d stderr=%q", rc, stderr)
+	}
+	fileOut := filepath.Join(dir, "file-out")
+	if err := os.WriteFile(fileOut, []byte("x"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	_, stderr, rc = runWith([]string{"manifest", "repair", "--out", fileOut}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "error:") {
+		t.Fatalf("manifest repair open append rc=%d stderr=%q", rc, stderr)
+	}
+	badDBDir := t.TempDir()
+	if err := os.WriteFile(manifest.SQLitePath(badDBDir), []byte("not sqlite"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	_, stderr, rc = runWith([]string{"manifest", "repair", "--out", badDBDir}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "error:") {
+		t.Fatalf("manifest repair open rc=%d stderr=%q", rc, stderr)
+	}
+	saveDir := t.TempDir()
+	sm := manifest.LoadJSONL(saveDir)
+	if err := sm.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.WriteFile(filepath.Join(saveDir, "photo.jpg"), []byte("abc"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	sm.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Exported: time.Now().Unix()})
+	sm.Close()
+	oldHook := manifest.SetJSONLSaveHook(func() error { return fmt.Errorf("save") })
+	_, stderr, rc = runWith([]string{"manifest", "repair", "--out", saveDir}, &mockBridge{})
+	manifest.SetJSONLSaveHook(oldHook)
+	if rc != exitErr || !strings.Contains(stderr, "save") {
+		t.Fatalf("manifest repair save rc=%d stderr=%q", rc, stderr)
+	}
+}
+
+func TestManifestRepairBranches(t *testing.T) {
+	dir := t.TempDir()
+	m := manifest.LoadJSONL(dir)
+	if err := m.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	m.AddEntry(manifest.Entry{ID: "empty"})
+	m.AddEntry(manifest.Entry{ID: "stat-only", Filename: "missing.jpg", Path: "missing.jpg", Checksum: "", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "hash-fail", Filename: "hash.jpg", Path: "hash.jpg", Checksum: "", Exported: time.Now().Unix()})
+	m.Close()
+	oldStat := statFunc
+	statFunc = func(path string) (os.FileInfo, error) {
+		if strings.HasSuffix(path, "hash.jpg") {
+			return testFileInfo{size: 3}, nil
+		}
+		return oldStat(path)
+	}
+	out, stderr, rc := runWith([]string{"manifest", "repair", "--out", dir, "--checksum", "sha256", "--dry-run"}, &mockBridge{})
+	statFunc = oldStat
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "skipped\t3") {
+		t.Fatalf("manifest repair branches rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+}
+
 func TestVerifySidecarBranches(t *testing.T) {
 	dir := t.TempDir()
 	subdir := filepath.Join(dir, "sub")