v0.9.0: add manifest checksums

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
 
+## v0.9.0
+
+Manifest checksum release.
+
+- Add opt-in `--checksum sha256` to store SHA-256 file checksums in JSONL and SQLite manifests.
+- Add SQLite migration support for the manifest `checksum` column.
+- Keep checksum collection disabled by default with `--checksum none`.
+
 ## v0.8.7
 
 JSON sidecar release.

Makefile

@@ -1,6 +1,6 @@
 BINARY     := ./bin/photoscli
 MODULE     := gitea.k3s.k0.nu/tools/photocli
-VERSION    := 0.8.7
+VERSION    := 0.9.0
 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip
 RELEASE_NOTES := RELEASE_NOTES.md
 BRIDGE_DIR := bridge

README.md

@@ -21,6 +21,7 @@ For a practical step-by-step manual with recommended backup workflows, recovery
 - Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`.
 - Verification, reporting, and diff commands for backup integrity.
 - Optional XMP sidecar verification with `verify --sidecar`.
+- Optional SHA-256 manifest checksums with `--checksum sha256`.
 - Metadata-only XMP refresh for manifest-backed exports with `--metadata-only`.
 - Status command for quick backup summaries.
 - Opt-in rich XMP sidecar metadata with `--sidecar xmp`.
@@ -186,6 +187,7 @@ Useful examples:
 
 ```bash
 photoscli backup-all --out ./backup --manifest sqlite --log
+photoscli backup-all --out ./backup --checksum sha256
 photoscli backup-all --out ./backup --exclude-album "Recently Deleted" --exclude-album "Temp*"
 photoscli backup-all --out ./backup --since 2024-01-01 --sort newest
 photoscli backup-all --out ./backup --concurrency 8 --retry 3

RELEASE_NOTES.md

@@ -1,18 +1,18 @@
-# v0.8.7
+# v0.9.0
 
-This release adds JSON sidecars alongside XMP sidecars.
+This release starts the backup-integrity series with manifest checksums.
 
 ## Highlights
 
-- Add `--sidecar json` for structured JSON metadata sidecars.
+- Add opt-in `--checksum sha256` to store SHA-256 checksums in manifests.
-- Add `--sidecar xmp,json` to write both XMP and JSON sidecars.
+- Support checksums in both JSONL and SQLite manifests.
-- Keep `--sidecar none` as the default.
+- Add SQLite migration support for the new checksum column.
-- JSON sidecars use the same metadata as XMP sidecars and the exported file basename.
+- Keep checksum collection disabled by default with `--checksum none`.
 
 ## Assets
 
 - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`).
-- `photoscli-0.8.7-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
+- `photoscli-0.9.0-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
 - `USERGUIDE.md`: standalone user guide.
 
 Intel Macs are not currently a supported release target.

USERGUIDE.md

@@ -21,6 +21,7 @@ It is especially useful when you want to:
 - Retry transient iCloud failures.
 - Verify that files referenced by a manifest exist on disk.
 - Detect missing, zero-byte, and size-mismatched manifest files.
+- Store optional SHA-256 checksums in manifests.
 - Inspect or clear deduplicated failure records.
 - Write optional XMP sidecar metadata for archival workflows.
 - Add optional reverse-geocoded address metadata to XMP sidecars for GPS assets.
@@ -821,6 +822,14 @@ Use structured logs:
 ./bin/photoscli backup-all --out ./PhotosBackup --manifest sqlite --log
 ```
 
+Store SHA-256 checksums in the manifest when exporting:
+
+```bash
+./bin/photoscli backup-all --out ./PhotosBackup --checksum sha256
+```
+
+Checksums are opt-in and stored in JSONL or SQLite manifests. The default is `--checksum none`.
+
 ## Safe Operating Practices
 
 - Run `--dry-run` before the first large backup.

cmd/photoscli/main.go

@@ -2,6 +2,8 @@ package main
 
 import (
 	"bytes"
+	"crypto/sha256"
+	"encoding/hex"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
@@ -42,6 +44,7 @@ type exportOptions struct {
 	verify         bool
 	format         string
 	sidecar        string
+	checksum       string
 	xmpPrivacy     string
 	xmpKeywords    string
 	xmpRating      string
@@ -228,6 +231,9 @@ COMMON EXPORT FLAGS
   --verify
       Run manifest/file verification after export or backup-all.
 
+  --checksum none|sha256
+      Store optional file checksum metadata in the manifest. Default: none.
+
   --sidecar none|xmp|json|xmp,json
       Write opt-in metadata sidecars next to each exported file. Default: none.
       If sidecar writing fails, the asset is counted as failed.
@@ -789,7 +795,7 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64,
 	}
 }
 
-func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) {
+func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, checksum string) {
 	if m == nil {
 		return
 	}
@@ -802,7 +808,33 @@ func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.Export
 	if err != nil || strings.HasPrefix(relPath, "..") {
 		relPath = result.Filename
 	}
-	m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud))
+	m.AddEntry(manifest.NewEntryWithChecksum(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud, checksum))
+}
+
+func addManifestEntryForResult(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, opts exportOptions) error {
+	checksum := ""
+	if opts.checksum == "sha256" && !result.Skipped {
+		var err error
+		checksum, err = fileSHA256(filepath.Join(pa.path, result.Filename))
+		if err != nil {
+			return err
+		}
+	}
+	addManifestEntry(m, pa, result, checksum)
+	return nil
+}
+
+func fileSHA256(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	h := sha256.New()
+	if _, err := io.Copy(h, f); err != nil {
+		return "", err
+	}
+	return hex.EncodeToString(h.Sum(nil)), nil
 }
 
 type xmpSidecarData struct {
@@ -1468,16 +1500,20 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
 			failed++
 			appendFailure(pa.path, pa, exportErr)
 		} else if isSkipped {
-			addManifestEntry(m, pa, result)
+			_ = addManifestEntryForResult(m, pa, result, opts)
 		} else {
 			if sidecarErr := writeSidecarIfNeeded(pa, result, originals, opts, geo, bridge); sidecarErr != nil {
 				failed++
 				exportErr = sidecarErr
 				isErr = true
 				appendFailure(pa.path, pa, sidecarErr)
+			} else if checksumErr := addManifestEntryForResult(m, pa, result, opts); checksumErr != nil {
+				failed++
+				exportErr = checksumErr
+				isErr = true
+				appendFailure(pa.path, pa, checksumErr)
 			} else {
 				done++
-				addManifestEntry(m, pa, result)
 			}
 		}
 		avgSpeed := float64(0)
@@ -1608,16 +1644,20 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
 			failed++
 			appendFailure(entry.pa.path, entry.pa, entry.err)
 		} else if isSkipped {
-			addManifestEntry(m, entry.pa, entry.result)
+			_ = addManifestEntryForResult(m, entry.pa, entry.result, opts)
 		} else {
 			if sidecarErr := writeSidecarIfNeeded(entry.pa, entry.result, originals, opts, geo, bridge); sidecarErr != nil {
 				failed++
 				entry.err = sidecarErr
 				isErr = true
 				appendFailure(entry.pa.path, entry.pa, sidecarErr)
+			} else if checksumErr := addManifestEntryForResult(m, entry.pa, entry.result, opts); checksumErr != nil {
+				failed++
+				entry.err = checksumErr
+				isErr = true
+				appendFailure(entry.pa.path, entry.pa, checksumErr)
 			} else {
 				done++
-				addManifestEntry(m, entry.pa, entry.result)
 			}
 		}
 		avgSpeed := float64(0)
@@ -1966,6 +2006,7 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) {
 		verify:         hasFlag(args, "--verify"),
 		format:         flagValWithDefault(args, "--format", "jpeg"),
 		sidecar:        flagValWithDefault(args, "--sidecar", "none"),
+		checksum:       flagValWithDefault(args, "--checksum", "none"),
 		xmpPrivacy:     flagValWithDefault(args, "--xmp-privacy", "keep"),
 		xmpKeywords:    flagValWithDefault(args, "--xmp-keywords", "album-path"),
 		xmpRating:      flagValWithDefault(args, "--xmp-rating", "favorite"),
@@ -1994,6 +2035,10 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) {
 			}
 		}
 	}
+	if opts.checksum != "none" && opts.checksum != "sha256" {
+		fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", opts.checksum)
+		return opts, false
+	}
 	if opts.xmpPrivacy != "keep" && opts.xmpPrivacy != "strip-location" && opts.xmpPrivacy != "strip-address" {
 		fmt.Fprintf(stderr, "error: --xmp-privacy must be keep, strip-location, or strip-address, got %q\n", opts.xmpPrivacy)
 		return opts, false

cmd/photoscli/main_test.go

@@ -2662,6 +2662,23 @@ func TestExportPendingParallelManifestAdd(t *testing.T) {
 	}
 }
 
+func TestExportPendingParallelChecksumError(t *testing.T) {
+	dir := t.TempDir()
+	mf := manifest.LoadJSONL(dir)
+	if err := mf.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}}
+	b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
+		return photos.ExportResult{Filename: "missing.jpg", Size: 3}, nil
+	}
+	bar := newProgressBar(io.Discard, 1)
+	done, failed := exportPendingParallel([]pendingAsset{{asset: b.assets[0], path: dir}}, 1024, 85, false, 1, bar, b, 1, mf, manifest.NoopLogWriter, exportOptions{checksum: "sha256"})
+	if done != 0 || failed != 1 {
+		t.Fatalf("done=%d failed=%d", done, failed)
+	}
+}
+
 func TestExportPendingParallelCancel(t *testing.T) {
 	b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}}
 	b.Cancel()
@@ -2804,6 +2821,58 @@ func TestExportAssetsManifestWrite(t *testing.T) {
 	}
 }
 
+func TestExportAssetsChecksum(t *testing.T) {
+	b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}}
+	dir := t.TempDir()
+	b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
+		if err := os.WriteFile(filepath.Join(out, "img.jpg"), []byte("abc"), 0644); err != nil {
+			return photos.ExportResult{}, err
+		}
+		return photos.ExportResult{Filename: "img.jpg", Size: 3, Cloud: "local"}, nil
+	}
+	done, failed := exportAssets(b.assets, dir, 1024, 85, 1, false, 1, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"})
+	if done != 1 || failed != 0 {
+		t.Fatalf("done=%d failed=%d", done, failed)
+	}
+	entry := manifest.LoadJSONL(dir).Entries()["x1"]
+	if entry.Checksum != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" {
+		t.Fatalf("checksum=%q", entry.Checksum)
+	}
+	got, err := fileSHA256(filepath.Join(dir, "img.jpg"))
+	if err != nil || got != entry.Checksum {
+		t.Fatalf("fileSHA256 got=%q err=%v", got, err)
+	}
+	if _, err := fileSHA256(filepath.Join(dir, "missing.jpg")); err == nil {
+		t.Fatal("expected missing checksum error")
+	}
+	if _, err := fileSHA256(dir); err == nil {
+		t.Fatal("expected directory checksum error")
+	}
+	if err := addManifestEntryForResult(nil, pendingAsset{}, photos.ExportResult{}, exportOptions{}); err != nil {
+		t.Fatalf("nil manifest add error: %v", err)
+	}
+	if err := addManifestEntryForResult(manifest.LoadJSONL(dir), pendingAsset{asset: photos.Asset{ID: "skip"}, path: dir}, photos.ExportResult{Filename: "missing.jpg", Skipped: true}, exportOptions{checksum: "sha256"}); err != nil {
+		t.Fatalf("skipped checksum add error: %v", err)
+	}
+}
+
+func TestExportAssetsChecksumErrors(t *testing.T) {
+	dir := t.TempDir()
+	b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}}
+	b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
+		return photos.ExportResult{Filename: "missing.jpg", Size: 3}, nil
+	}
+	done, failed := exportAssets(b.assets, dir, 1024, 85, 1, false, 1, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"})
+	if done != 0 || failed != 1 {
+		t.Fatalf("serial checksum error done=%d failed=%d", done, failed)
+	}
+	dir = t.TempDir()
+	done, failed = exportAssets(b.assets, dir, 1024, 85, 2, false, 2, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"})
+	if done != 0 || failed != 1 {
+		t.Fatalf("parallel checksum error done=%d failed=%d", done, failed)
+	}
+}
+
 func TestBackupTreeManifestOpenErr(t *testing.T) {
 	b := &mockBridge{
 		tree: []photos.CollectionNode{
@@ -4657,6 +4726,10 @@ func TestSidecarConfigAndErrors(t *testing.T) {
 	if _, ok := parseExportOptions([]string{"--xmp-rating", "bad"}, &stderr); ok || !strings.Contains(stderr.String(), "--xmp-rating") {
 		t.Fatalf("expected xmp rating validation error, stderr=%q", stderr.String())
 	}
+	stderr.Reset()
+	if _, ok := parseExportOptions([]string{"--checksum", "bad"}, &stderr); ok || !strings.Contains(stderr.String(), "--checksum") {
+		t.Fatalf("expected checksum validation error, stderr=%q", stderr.String())
+	}
 
 	b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "photo.jpg"}}}
 	b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
@@ -5018,7 +5091,7 @@ func TestInjectedErrorBranchesForCoverage(t *testing.T) {
 	removeFunc = oldRemove
 
 	mf := &mockManifest{}
-	addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1})
+	addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1}, "")
 	if mf.last.Path != "file.jpg" {
 		t.Fatalf("expected fallback rel path, got %+v", mf.last)
 	}

internal/manifest/entry_test.go

@@ -11,6 +11,10 @@ func TestNewEntryPath(t *testing.T) {
 	if e.Path != "Album/file2.jpg" || e.Filename != "file2.jpg" || e.Size != 456 || e.Cloud != "cloud" {
 		t.Fatalf("unexpected entry with path: %+v", e)
 	}
+	e = NewEntryWithChecksum("id3", "file3.jpg", "Album/file3.jpg", 789, "local", "sha256:abc")
+	if e.Checksum != "sha256:abc" || e.Path != "Album/file3.jpg" {
+		t.Fatalf("unexpected checksum entry: %+v", e)
+	}
 }
 
 func TestAddEntryDefaultsPath(t *testing.T) {
@@ -19,11 +23,15 @@ func TestAddEntryDefaultsPath(t *testing.T) {
 	if err := jm.OpenAppend(); err != nil {
 		t.Fatal(err)
 	}
-	jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
+	jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local", Checksum: "sha256:abc"})
 	jm.Close()
-	if got := LoadJSONL(dir).Entries()["x1"].Path; got != "file.jpg" {
+	loaded := LoadJSONL(dir).Entries()["x1"]
+	if got := loaded.Path; got != "file.jpg" {
 		t.Fatalf("jsonl path = %q", got)
 	}
+	if loaded.Checksum != "sha256:abc" {
+		t.Fatalf("jsonl checksum = %q", loaded.Checksum)
+	}
 
 	sdir := t.TempDir()
 	sm, err := LoadSQLite(sdir)
@@ -33,12 +41,16 @@ func TestAddEntryDefaultsPath(t *testing.T) {
 	if err := sm.OpenAppend(); err != nil {
 		t.Fatal(err)
 	}
-	sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
+	sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local", Checksum: "sha256:def"})
 	if _, err := sm.db.Exec(`UPDATE downloads SET path = '' WHERE id = 'x1'`); err != nil {
 		t.Fatal(err)
 	}
-	if got := sm.Entries()["x1"].Path; got != "file.jpg" {
+	sloaded := sm.Entries()["x1"]
+	if got := sloaded.Path; got != "file.jpg" {
 		t.Fatalf("sqlite path = %q", got)
 	}
+	if sloaded.Checksum != "sha256:def" {
+		t.Fatalf("sqlite checksum = %q", sloaded.Checksum)
+	}
 	sm.Close()
 }

internal/manifest/jsonl.go

@@ -43,6 +43,7 @@ func LoadJSONL(dir string) *jsonlManifest {
 		Path     string `json:"path,omitempty"`
 		Size     int64  `json:"size"`
 		Cloud    string `json:"cloud"`
+		Checksum string `json:"checksum,omitempty"`
 		Exported int64  `json:"exported"`
 	}
 	for _, line := range strings.Split(string(data), "\n") {
@@ -61,6 +62,7 @@ func LoadJSONL(dir string) *jsonlManifest {
 				Path:     raw.Path,
 				Size:     raw.Size,
 				Cloud:    raw.Cloud,
+				Checksum: raw.Checksum,
 				Exported: raw.Exported,
 			}
 		}
@@ -93,8 +95,9 @@ func (m *jsonlManifest) AddEntry(entry Entry) {
 			Path     string `json:"path,omitempty"`
 			Size     int64  `json:"size"`
 			Cloud    string `json:"cloud"`
+			Checksum string `json:"checksum,omitempty"`
 			Exported int64  `json:"exported"`
-		}{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported})
+		}{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Checksum: entry.Checksum, Exported: entry.Exported})
 		m.file.Write(data)
 		m.file.Write([]byte("\n"))
 	}

internal/manifest/manifest.go

@@ -8,6 +8,7 @@ type Entry struct {
 	Path     string
 	Size     int64
 	Cloud    string
+	Checksum string
 	Exported int64
 }
 
@@ -42,3 +43,9 @@ func NewEntry(id, filename, path string, size int64, cloud string) Entry {
 	}
 	return e
 }
+
+func NewEntryWithChecksum(id, filename, path string, size int64, cloud, checksum string) Entry {
+	e := NewEntry(id, filename, path, size, cloud)
+	e.Checksum = checksum
+	return e
+}

internal/manifest/sqlite.go

@@ -64,6 +64,7 @@ func (m *sqliteManifest) OpenAppend() error {
 		path TEXT NOT NULL DEFAULT '',
 		size INTEGER NOT NULL DEFAULT 0,
 		cloud TEXT NOT NULL DEFAULT '',
+		checksum TEXT NOT NULL DEFAULT '',
 		exported INTEGER NOT NULL DEFAULT 0
 	)`)
 	if err != nil {
@@ -71,6 +72,7 @@ func (m *sqliteManifest) OpenAppend() error {
 		return fmt.Errorf("create table: %w", err)
 	}
 	_, _ = execFn(`ALTER TABLE downloads ADD COLUMN path TEXT NOT NULL DEFAULT ''`)
+	_, _ = execFn(`ALTER TABLE downloads ADD COLUMN checksum TEXT NOT NULL DEFAULT ''`)
 	_, err = execFn(`CREATE INDEX IF NOT EXISTS idx_downloads_id ON downloads(id)`)
 	if err != nil {
 		db.Close()
@@ -103,8 +105,8 @@ func (m *sqliteManifest) AddEntry(entry Entry) {
 	if entry.Path == "" {
 		entry.Path = entry.Filename
 	}
-	m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, exported) VALUES (?, ?, ?, ?, ?, ?)`,
+	m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, checksum, exported) VALUES (?, ?, ?, ?, ?, ?, ?)`,
-		entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Exported)
+		entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Checksum, entry.Exported)
 }
 
 func (m *sqliteManifest) Save() error {
@@ -127,14 +129,14 @@ func (m *sqliteManifest) Entries() map[string]Entry {
 		return nil
 	}
 	out := make(map[string]Entry)
-	rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, exported FROM downloads`)
+	rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, checksum, exported FROM downloads`)
 	if err != nil {
 		return out
 	}
 	defer rows.Close()
 	for rows.Next() {
 		var e Entry
-		if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Exported); err == nil {
+		if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Checksum, &e.Exported); err == nil {
 			if e.Path == "" {
 				e.Path = e.Filename
 			}