From 7555b561bdecd43b09283b256f4679c4477196b8 Mon Sep 17 00:00:00 2001 From: Ein Anderssono Date: Mon, 15 Jun 2026 02:19:47 +0200 Subject: [PATCH] v0.9.0: add manifest checksums --- CHANGELOG.md | 8 ++++ Makefile | 2 +- README.md | 2 + RELEASE_NOTES.md | 14 +++--- USERGUIDE.md | 9 ++++ cmd/photoscli/main.go | 57 ++++++++++++++++++++++--- cmd/photoscli/main_test.go | 75 ++++++++++++++++++++++++++++++++- internal/manifest/entry_test.go | 20 +++++++-- internal/manifest/jsonl.go | 5 ++- internal/manifest/manifest.go | 7 +++ internal/manifest/sqlite.go | 10 +++-- 11 files changed, 185 insertions(+), 24 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dd2e784..38c09ca 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page. +## v0.9.0 + +Manifest checksum release. + +- Add opt-in `--checksum sha256` to store SHA-256 file checksums in JSONL and SQLite manifests. +- Add SQLite migration support for the manifest `checksum` column. +- Keep checksum collection disabled by default with `--checksum none`. + ## v0.8.7 JSON sidecar release. diff --git a/Makefile b/Makefile index 49dda8e..ebe7d77 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ BINARY := ./bin/photoscli MODULE := gitea.k3s.k0.nu/tools/photocli -VERSION := 0.8.7 +VERSION := 0.9.0 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip RELEASE_NOTES := RELEASE_NOTES.md BRIDGE_DIR := bridge diff --git a/README.md b/README.md index 7e1f2d4..1cc4dbd 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,7 @@ For a practical step-by-step manual with recommended backup workflows, recovery - Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`. - Verification, reporting, and diff commands for backup integrity. - Optional XMP sidecar verification with `verify --sidecar`. +- Optional SHA-256 manifest checksums with `--checksum sha256`. - Metadata-only XMP refresh for manifest-backed exports with `--metadata-only`. - Status command for quick backup summaries. - Opt-in rich XMP sidecar metadata with `--sidecar xmp`. @@ -186,6 +187,7 @@ Useful examples: ```bash photoscli backup-all --out ./backup --manifest sqlite --log +photoscli backup-all --out ./backup --checksum sha256 photoscli backup-all --out ./backup --exclude-album "Recently Deleted" --exclude-album "Temp*" photoscli backup-all --out ./backup --since 2024-01-01 --sort newest photoscli backup-all --out ./backup --concurrency 8 --retry 3 diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 97497f5..7629c62 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,18 +1,18 @@ -# v0.8.7 +# v0.9.0 -This release adds JSON sidecars alongside XMP sidecars. +This release starts the backup-integrity series with manifest checksums. ## Highlights -- Add `--sidecar json` for structured JSON metadata sidecars. -- Add `--sidecar xmp,json` to write both XMP and JSON sidecars. -- Keep `--sidecar none` as the default. -- JSON sidecars use the same metadata as XMP sidecars and the exported file basename. +- Add opt-in `--checksum sha256` to store SHA-256 checksums in manifests. +- Support checksums in both JSONL and SQLite manifests. +- Add SQLite migration support for the new checksum column. +- Keep checksum collection disabled by default with `--checksum none`. ## Assets - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`). -- `photoscli-0.8.7-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. +- `photoscli-0.9.0-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. - `USERGUIDE.md`: standalone user guide. Intel Macs are not currently a supported release target. diff --git a/USERGUIDE.md b/USERGUIDE.md index 334d038..88c360e 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -21,6 +21,7 @@ It is especially useful when you want to: - Retry transient iCloud failures. - Verify that files referenced by a manifest exist on disk. - Detect missing, zero-byte, and size-mismatched manifest files. +- Store optional SHA-256 checksums in manifests. - Inspect or clear deduplicated failure records. - Write optional XMP sidecar metadata for archival workflows. - Add optional reverse-geocoded address metadata to XMP sidecars for GPS assets. @@ -821,6 +822,14 @@ Use structured logs: ./bin/photoscli backup-all --out ./PhotosBackup --manifest sqlite --log ``` +Store SHA-256 checksums in the manifest when exporting: + +```bash +./bin/photoscli backup-all --out ./PhotosBackup --checksum sha256 +``` + +Checksums are opt-in and stored in JSONL or SQLite manifests. The default is `--checksum none`. + ## Safe Operating Practices - Run `--dry-run` before the first large backup. diff --git a/cmd/photoscli/main.go b/cmd/photoscli/main.go index b5b9daf..ac749ff 100644 --- a/cmd/photoscli/main.go +++ b/cmd/photoscli/main.go @@ -2,6 +2,8 @@ package main import ( "bytes" + "crypto/sha256" + "encoding/hex" "encoding/json" "encoding/xml" "fmt" @@ -42,6 +44,7 @@ type exportOptions struct { verify bool format string sidecar string + checksum string xmpPrivacy string xmpKeywords string xmpRating string @@ -228,6 +231,9 @@ COMMON EXPORT FLAGS --verify Run manifest/file verification after export or backup-all. + --checksum none|sha256 + Store optional file checksum metadata in the manifest. Default: none. + --sidecar none|xmp|json|xmp,json Write opt-in metadata sidecars next to each exported file. Default: none. If sidecar writing fails, the asset is counted as failed. @@ -789,7 +795,7 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64, } } -func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) { +func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, checksum string) { if m == nil { return } @@ -802,7 +808,33 @@ func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.Export if err != nil || strings.HasPrefix(relPath, "..") { relPath = result.Filename } - m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud)) + m.AddEntry(manifest.NewEntryWithChecksum(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud, checksum)) +} + +func addManifestEntryForResult(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, opts exportOptions) error { + checksum := "" + if opts.checksum == "sha256" && !result.Skipped { + var err error + checksum, err = fileSHA256(filepath.Join(pa.path, result.Filename)) + if err != nil { + return err + } + } + addManifestEntry(m, pa, result, checksum) + return nil +} + +func fileSHA256(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + h := sha256.New() + if _, err := io.Copy(h, f); err != nil { + return "", err + } + return hex.EncodeToString(h.Sum(nil)), nil } type xmpSidecarData struct { @@ -1468,16 +1500,20 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin failed++ appendFailure(pa.path, pa, exportErr) } else if isSkipped { - addManifestEntry(m, pa, result) + _ = addManifestEntryForResult(m, pa, result, opts) } else { if sidecarErr := writeSidecarIfNeeded(pa, result, originals, opts, geo, bridge); sidecarErr != nil { failed++ exportErr = sidecarErr isErr = true appendFailure(pa.path, pa, sidecarErr) + } else if checksumErr := addManifestEntryForResult(m, pa, result, opts); checksumErr != nil { + failed++ + exportErr = checksumErr + isErr = true + appendFailure(pa.path, pa, checksumErr) } else { done++ - addManifestEntry(m, pa, result) } } avgSpeed := float64(0) @@ -1608,16 +1644,20 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig failed++ appendFailure(entry.pa.path, entry.pa, entry.err) } else if isSkipped { - addManifestEntry(m, entry.pa, entry.result) + _ = addManifestEntryForResult(m, entry.pa, entry.result, opts) } else { if sidecarErr := writeSidecarIfNeeded(entry.pa, entry.result, originals, opts, geo, bridge); sidecarErr != nil { failed++ entry.err = sidecarErr isErr = true appendFailure(entry.pa.path, entry.pa, sidecarErr) + } else if checksumErr := addManifestEntryForResult(m, entry.pa, entry.result, opts); checksumErr != nil { + failed++ + entry.err = checksumErr + isErr = true + appendFailure(entry.pa.path, entry.pa, checksumErr) } else { done++ - addManifestEntry(m, entry.pa, entry.result) } } avgSpeed := float64(0) @@ -1966,6 +2006,7 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) { verify: hasFlag(args, "--verify"), format: flagValWithDefault(args, "--format", "jpeg"), sidecar: flagValWithDefault(args, "--sidecar", "none"), + checksum: flagValWithDefault(args, "--checksum", "none"), xmpPrivacy: flagValWithDefault(args, "--xmp-privacy", "keep"), xmpKeywords: flagValWithDefault(args, "--xmp-keywords", "album-path"), xmpRating: flagValWithDefault(args, "--xmp-rating", "favorite"), @@ -1994,6 +2035,10 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) { } } } + if opts.checksum != "none" && opts.checksum != "sha256" { + fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", opts.checksum) + return opts, false + } if opts.xmpPrivacy != "keep" && opts.xmpPrivacy != "strip-location" && opts.xmpPrivacy != "strip-address" { fmt.Fprintf(stderr, "error: --xmp-privacy must be keep, strip-location, or strip-address, got %q\n", opts.xmpPrivacy) return opts, false diff --git a/cmd/photoscli/main_test.go b/cmd/photoscli/main_test.go index b441673..e3a3826 100644 --- a/cmd/photoscli/main_test.go +++ b/cmd/photoscli/main_test.go @@ -2662,6 +2662,23 @@ func TestExportPendingParallelManifestAdd(t *testing.T) { } } +func TestExportPendingParallelChecksumError(t *testing.T) { + dir := t.TempDir() + mf := manifest.LoadJSONL(dir) + if err := mf.OpenAppend(); err != nil { + t.Fatal(err) + } + b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + return photos.ExportResult{Filename: "missing.jpg", Size: 3}, nil + } + bar := newProgressBar(io.Discard, 1) + done, failed := exportPendingParallel([]pendingAsset{{asset: b.assets[0], path: dir}}, 1024, 85, false, 1, bar, b, 1, mf, manifest.NoopLogWriter, exportOptions{checksum: "sha256"}) + if done != 0 || failed != 1 { + t.Fatalf("done=%d failed=%d", done, failed) + } +} + func TestExportPendingParallelCancel(t *testing.T) { b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}} b.Cancel() @@ -2804,6 +2821,58 @@ func TestExportAssetsManifestWrite(t *testing.T) { } } +func TestExportAssetsChecksum(t *testing.T) { + b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}} + dir := t.TempDir() + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "img.jpg"), []byte("abc"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "img.jpg", Size: 3, Cloud: "local"}, nil + } + done, failed := exportAssets(b.assets, dir, 1024, 85, 1, false, 1, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"}) + if done != 1 || failed != 0 { + t.Fatalf("done=%d failed=%d", done, failed) + } + entry := manifest.LoadJSONL(dir).Entries()["x1"] + if entry.Checksum != "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad" { + t.Fatalf("checksum=%q", entry.Checksum) + } + got, err := fileSHA256(filepath.Join(dir, "img.jpg")) + if err != nil || got != entry.Checksum { + t.Fatalf("fileSHA256 got=%q err=%v", got, err) + } + if _, err := fileSHA256(filepath.Join(dir, "missing.jpg")); err == nil { + t.Fatal("expected missing checksum error") + } + if _, err := fileSHA256(dir); err == nil { + t.Fatal("expected directory checksum error") + } + if err := addManifestEntryForResult(nil, pendingAsset{}, photos.ExportResult{}, exportOptions{}); err != nil { + t.Fatalf("nil manifest add error: %v", err) + } + if err := addManifestEntryForResult(manifest.LoadJSONL(dir), pendingAsset{asset: photos.Asset{ID: "skip"}, path: dir}, photos.ExportResult{Filename: "missing.jpg", Skipped: true}, exportOptions{checksum: "sha256"}); err != nil { + t.Fatalf("skipped checksum add error: %v", err) + } +} + +func TestExportAssetsChecksumErrors(t *testing.T) { + dir := t.TempDir() + b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "img.jpg"}}} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + return photos.ExportResult{Filename: "missing.jpg", Size: 3}, nil + } + done, failed := exportAssets(b.assets, dir, 1024, 85, 1, false, 1, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"}) + if done != 0 || failed != 1 { + t.Fatalf("serial checksum error done=%d failed=%d", done, failed) + } + dir = t.TempDir() + done, failed = exportAssets(b.assets, dir, 1024, 85, 2, false, 2, io.Discard, b, "", false, manifest.FormatJSONL, false, exportOptions{checksum: "sha256"}) + if done != 0 || failed != 1 { + t.Fatalf("parallel checksum error done=%d failed=%d", done, failed) + } +} + func TestBackupTreeManifestOpenErr(t *testing.T) { b := &mockBridge{ tree: []photos.CollectionNode{ @@ -4657,6 +4726,10 @@ func TestSidecarConfigAndErrors(t *testing.T) { if _, ok := parseExportOptions([]string{"--xmp-rating", "bad"}, &stderr); ok || !strings.Contains(stderr.String(), "--xmp-rating") { t.Fatalf("expected xmp rating validation error, stderr=%q", stderr.String()) } + stderr.Reset() + if _, ok := parseExportOptions([]string{"--checksum", "bad"}, &stderr); ok || !strings.Contains(stderr.String(), "--checksum") { + t.Fatalf("expected checksum validation error, stderr=%q", stderr.String()) + } b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "photo.jpg"}}} b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { @@ -5018,7 +5091,7 @@ func TestInjectedErrorBranchesForCoverage(t *testing.T) { removeFunc = oldRemove mf := &mockManifest{} - addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1}) + addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1}, "") if mf.last.Path != "file.jpg" { t.Fatalf("expected fallback rel path, got %+v", mf.last) } diff --git a/internal/manifest/entry_test.go b/internal/manifest/entry_test.go index cba8e8f..507d4f6 100644 --- a/internal/manifest/entry_test.go +++ b/internal/manifest/entry_test.go @@ -11,6 +11,10 @@ func TestNewEntryPath(t *testing.T) { if e.Path != "Album/file2.jpg" || e.Filename != "file2.jpg" || e.Size != 456 || e.Cloud != "cloud" { t.Fatalf("unexpected entry with path: %+v", e) } + e = NewEntryWithChecksum("id3", "file3.jpg", "Album/file3.jpg", 789, "local", "sha256:abc") + if e.Checksum != "sha256:abc" || e.Path != "Album/file3.jpg" { + t.Fatalf("unexpected checksum entry: %+v", e) + } } func TestAddEntryDefaultsPath(t *testing.T) { @@ -19,11 +23,15 @@ func TestAddEntryDefaultsPath(t *testing.T) { if err := jm.OpenAppend(); err != nil { t.Fatal(err) } - jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"}) + jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local", Checksum: "sha256:abc"}) jm.Close() - if got := LoadJSONL(dir).Entries()["x1"].Path; got != "file.jpg" { + loaded := LoadJSONL(dir).Entries()["x1"] + if got := loaded.Path; got != "file.jpg" { t.Fatalf("jsonl path = %q", got) } + if loaded.Checksum != "sha256:abc" { + t.Fatalf("jsonl checksum = %q", loaded.Checksum) + } sdir := t.TempDir() sm, err := LoadSQLite(sdir) @@ -33,12 +41,16 @@ func TestAddEntryDefaultsPath(t *testing.T) { if err := sm.OpenAppend(); err != nil { t.Fatal(err) } - sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"}) + sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local", Checksum: "sha256:def"}) if _, err := sm.db.Exec(`UPDATE downloads SET path = '' WHERE id = 'x1'`); err != nil { t.Fatal(err) } - if got := sm.Entries()["x1"].Path; got != "file.jpg" { + sloaded := sm.Entries()["x1"] + if got := sloaded.Path; got != "file.jpg" { t.Fatalf("sqlite path = %q", got) } + if sloaded.Checksum != "sha256:def" { + t.Fatalf("sqlite checksum = %q", sloaded.Checksum) + } sm.Close() } diff --git a/internal/manifest/jsonl.go b/internal/manifest/jsonl.go index 8934af8..13125ea 100644 --- a/internal/manifest/jsonl.go +++ b/internal/manifest/jsonl.go @@ -43,6 +43,7 @@ func LoadJSONL(dir string) *jsonlManifest { Path string `json:"path,omitempty"` Size int64 `json:"size"` Cloud string `json:"cloud"` + Checksum string `json:"checksum,omitempty"` Exported int64 `json:"exported"` } for _, line := range strings.Split(string(data), "\n") { @@ -61,6 +62,7 @@ func LoadJSONL(dir string) *jsonlManifest { Path: raw.Path, Size: raw.Size, Cloud: raw.Cloud, + Checksum: raw.Checksum, Exported: raw.Exported, } } @@ -93,8 +95,9 @@ func (m *jsonlManifest) AddEntry(entry Entry) { Path string `json:"path,omitempty"` Size int64 `json:"size"` Cloud string `json:"cloud"` + Checksum string `json:"checksum,omitempty"` Exported int64 `json:"exported"` - }{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported}) + }{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Checksum: entry.Checksum, Exported: entry.Exported}) m.file.Write(data) m.file.Write([]byte("\n")) } diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index 01912c0..067ad4e 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -8,6 +8,7 @@ type Entry struct { Path string Size int64 Cloud string + Checksum string Exported int64 } @@ -42,3 +43,9 @@ func NewEntry(id, filename, path string, size int64, cloud string) Entry { } return e } + +func NewEntryWithChecksum(id, filename, path string, size int64, cloud, checksum string) Entry { + e := NewEntry(id, filename, path, size, cloud) + e.Checksum = checksum + return e +} diff --git a/internal/manifest/sqlite.go b/internal/manifest/sqlite.go index 84aea66..8957f19 100644 --- a/internal/manifest/sqlite.go +++ b/internal/manifest/sqlite.go @@ -64,6 +64,7 @@ func (m *sqliteManifest) OpenAppend() error { path TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, cloud TEXT NOT NULL DEFAULT '', + checksum TEXT NOT NULL DEFAULT '', exported INTEGER NOT NULL DEFAULT 0 )`) if err != nil { @@ -71,6 +72,7 @@ func (m *sqliteManifest) OpenAppend() error { return fmt.Errorf("create table: %w", err) } _, _ = execFn(`ALTER TABLE downloads ADD COLUMN path TEXT NOT NULL DEFAULT ''`) + _, _ = execFn(`ALTER TABLE downloads ADD COLUMN checksum TEXT NOT NULL DEFAULT ''`) _, err = execFn(`CREATE INDEX IF NOT EXISTS idx_downloads_id ON downloads(id)`) if err != nil { db.Close() @@ -103,8 +105,8 @@ func (m *sqliteManifest) AddEntry(entry Entry) { if entry.Path == "" { entry.Path = entry.Filename } - m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, exported) VALUES (?, ?, ?, ?, ?, ?)`, - entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Exported) + m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, checksum, exported) VALUES (?, ?, ?, ?, ?, ?, ?)`, + entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Checksum, entry.Exported) } func (m *sqliteManifest) Save() error { @@ -127,14 +129,14 @@ func (m *sqliteManifest) Entries() map[string]Entry { return nil } out := make(map[string]Entry) - rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, exported FROM downloads`) + rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, checksum, exported FROM downloads`) if err != nil { return out } defer rows.Close() for rows.Next() { var e Entry - if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Exported); err == nil { + if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Checksum, &e.Exported); err == nil { if e.Path == "" { e.Path = e.Filename }