v0.9.1: add deep checksum verification
This commit is contained in:
@@ -2,6 +2,14 @@
|
||||
|
||||
This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
|
||||
|
||||
## v0.9.1
|
||||
|
||||
Deep checksum verification release.
|
||||
|
||||
- Add `verify --deep` to recompute SHA-256 checksums for manifest entries that have checksum metadata.
|
||||
- Report `checksum-mismatch` when disk contents differ from the manifest checksum.
|
||||
- Keep normal `verify` unchanged unless `--deep` is selected.
|
||||
|
||||
## v0.9.0
|
||||
|
||||
Manifest checksum release.
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
BINARY := ./bin/photoscli
|
||||
MODULE := gitea.k3s.k0.nu/tools/photocli
|
||||
VERSION := 0.9.0
|
||||
VERSION := 0.9.1
|
||||
RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip
|
||||
RELEASE_NOTES := RELEASE_NOTES.md
|
||||
BRIDGE_DIR := bridge
|
||||
|
||||
@@ -117,6 +117,7 @@ Verify a backup later:
|
||||
|
||||
```bash
|
||||
./bin/photoscli verify --out ./photos-backup --manifest sqlite
|
||||
./bin/photoscli verify --out ./photos-backup --deep
|
||||
```
|
||||
|
||||
## Commands
|
||||
|
||||
+7
-7
@@ -1,18 +1,18 @@
|
||||
# v0.9.0
|
||||
# v0.9.1
|
||||
|
||||
This release starts the backup-integrity series with manifest checksums.
|
||||
This release adds deep checksum verification for manifest-backed backups.
|
||||
|
||||
## Highlights
|
||||
|
||||
- Add opt-in `--checksum sha256` to store SHA-256 checksums in manifests.
|
||||
- Support checksums in both JSONL and SQLite manifests.
|
||||
- Add SQLite migration support for the new checksum column.
|
||||
- Keep checksum collection disabled by default with `--checksum none`.
|
||||
- Add `verify --deep` to recompute SHA-256 checksums when manifest entries include checksums.
|
||||
- Report `checksum-mismatch` when file contents differ from the manifest checksum.
|
||||
- Keep normal `verify` unchanged unless `--deep` is selected.
|
||||
- `--deep` works with JSONL and SQLite manifests.
|
||||
|
||||
## Assets
|
||||
|
||||
- `photoscli`: Apple Silicon macOS binary (`darwin/arm64`).
|
||||
- `photoscli-0.9.0-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
|
||||
- `photoscli-0.9.1-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
|
||||
- `USERGUIDE.md`: standalone user guide.
|
||||
|
||||
Intel Macs are not currently a supported release target.
|
||||
|
||||
@@ -830,6 +830,14 @@ Store SHA-256 checksums in the manifest when exporting:
|
||||
|
||||
Checksums are opt-in and stored in JSONL or SQLite manifests. The default is `--checksum none`.
|
||||
|
||||
Deep-verify checksum-backed manifests:
|
||||
|
||||
```bash
|
||||
./bin/photoscli verify --out ./PhotosBackup --deep
|
||||
```
|
||||
|
||||
`--deep` recomputes SHA-256 only for manifest entries that already have checksum metadata.
|
||||
|
||||
## Safe Operating Practices
|
||||
|
||||
- Run `--dry-run` before the first large backup.
|
||||
|
||||
@@ -180,6 +180,7 @@ COMMANDS
|
||||
files are printed as <asset-id><TAB><filename>. Exits 2 on missing files.
|
||||
Add --sidecar to verify expected XMP sidecars too. Add --strict with
|
||||
--sidecar to require photoscli schema/generator and exported filename metadata.
|
||||
Add --deep to verify manifest SHA-256 checksums when present.
|
||||
|
||||
retry-failed --out <dir>
|
||||
Retry assets previously written to failures.jsonl.
|
||||
@@ -2279,6 +2280,7 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
checkSidecar := hasFlag(args, "--sidecar")
|
||||
strictSidecar := hasFlag(args, "--strict")
|
||||
deep := hasFlag(args, "--deep")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
@@ -2317,6 +2319,16 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
|
||||
}
|
||||
if deep && e.Checksum != "" {
|
||||
checksum, err := fileSHA256(filepath.Join(outDir, checkPath))
|
||||
if err != nil {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tchecksum-unreadable\n", id, checkPath)
|
||||
} else if checksum != e.Checksum {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tchecksum-mismatch\tmanifest=%s\tdisk=%s\n", id, checkPath, e.Checksum, checksum)
|
||||
}
|
||||
}
|
||||
if checkSidecar {
|
||||
bad += verifySidecar(stdout, outDir, id, checkPath, strictSidecar)
|
||||
}
|
||||
|
||||
@@ -4273,6 +4273,39 @@ func TestMoreIntegrityBranches(t *testing.T) {
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifyDeepChecksums(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
if err := os.WriteFile(filepath.Join(dir, "good.jpg"), []byte("abc"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "bad.jpg"), []byte("bad"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "nocheck.jpg"), []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.Mkdir(filepath.Join(dir, "adir"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.Entry{ID: "good", Filename: "good.jpg", Path: "good.jpg", Size: 3, Cloud: "local", Checksum: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "bad", Filename: "bad.jpg", Path: "bad.jpg", Size: 3, Cloud: "local", Checksum: "ba7816bf8f01cfea414140de5dae2223b00361a396177a9cb410ff61f20015ad", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "nocheck", Filename: "nocheck.jpg", Path: "nocheck.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "unreadable", Filename: "adir", Path: "adir", Cloud: "local", Checksum: "abc", Exported: time.Now().Unix()})
|
||||
m.Close()
|
||||
out, stderr, rc := runWith([]string{"verify", "--out", dir, "--deep"}, &mockBridge{})
|
||||
if rc != exitPartial || stderr != "" || !strings.Contains(out, "checksum-mismatch") || !strings.Contains(out, "checksum-unreadable") || strings.Contains(out, "good.jpg") || strings.Contains(out, "nocheck") {
|
||||
t.Fatalf("deep verify rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"verify", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || stderr != "" || strings.Contains(out, "checksum") {
|
||||
t.Fatalf("plain verify rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestVerifySidecarBranches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
subdir := filepath.Join(dir, "sub")
|
||||
|
||||
Reference in New Issue
Block a user