diff --git a/CHANGELOG.md b/CHANGELOG.md index 32de9b8..87c1217 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,14 @@ This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page. +## v0.9.3 + +Cleanup release. + +- Add `cleanup --out ` to remove files not referenced by the manifest. +- Add `cleanup --dry-run` to preview orphaned files before deletion. +- Preserve manifest/log/failure files, `.photoscli`, and sidecars for manifest-referenced media. + ## v0.9.2 Manifest repair release. diff --git a/Makefile b/Makefile index e330b1e..9bf4c13 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ BINARY := ./bin/photoscli MODULE := gitea.k3s.k0.nu/tools/photocli -VERSION := 0.9.2 +VERSION := 0.9.3 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip RELEASE_NOTES := RELEASE_NOTES.md BRIDGE_DIR := bridge diff --git a/README.md b/README.md index 76e64ce..5f95381 100644 --- a/README.md +++ b/README.md @@ -119,6 +119,7 @@ Verify a backup later: ./bin/photoscli verify --out ./photos-backup --manifest sqlite ./bin/photoscli verify --out ./photos-backup --deep ./bin/photoscli manifest repair --out ./photos-backup --checksum sha256 --dry-run +./bin/photoscli cleanup --out ./photos-backup --dry-run ``` ## Commands diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index dbd9a69..99b9e2c 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,18 +1,18 @@ -# v0.9.2 +# v0.9.3 -This release adds manifest repair for existing backups. +This release adds safe cleanup for manifest-backed backups. ## Highlights -- Add `manifest repair --out ` to fill missing size metadata from existing files. -- Add `manifest repair --checksum sha256` to fill missing checksums. -- Add `manifest repair --dry-run` to preview repairs without writing updates. -- Repair works with JSONL and SQLite manifests. +- Add `cleanup --out ` to remove files not referenced by the manifest. +- Add `cleanup --dry-run` to preview orphaned files before deletion. +- Preserve manifest/log/failure files, `.photoscli`, and sidecars for manifest-referenced media. +- Cleanup works with JSONL and SQLite manifests. ## Assets - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`). -- `photoscli-0.9.2-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. +- `photoscli-0.9.3-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG. - `USERGUIDE.md`: standalone user guide. Intel Macs are not currently a supported release target. diff --git a/USERGUIDE.md b/USERGUIDE.md index 7e45131..ec7f12d 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -847,6 +847,15 @@ Repair missing manifest metadata from files already present on disk: Repair fills missing size/checksum metadata and skips missing, zero-byte, or unreadable files. +Preview and remove files not referenced by the manifest: + +```bash +./bin/photoscli cleanup --out ./PhotosBackup --dry-run +./bin/photoscli cleanup --out ./PhotosBackup +``` + +Cleanup preserves manifest/log/failure files, `.photoscli`, and sidecars next to manifest-referenced media. + ## Safe Operating Practices - Run `--dry-run` before the first large backup. diff --git a/cmd/photoscli/main.go b/cmd/photoscli/main.go index 9c1495b..f5ac17f 100644 --- a/cmd/photoscli/main.go +++ b/cmd/photoscli/main.go @@ -8,6 +8,7 @@ import ( "encoding/xml" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" @@ -94,6 +95,8 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int { return cmdVerify(args[1:], stdout, stderr) case "manifest": return cmdManifest(args[1:], stdout, stderr) + case "cleanup": + return cmdCleanup(args[1:], stdout, stderr) case "retry-failed": return cmdRetryFailed(args[1:], stdout, stderr, bridge) case "failures": @@ -140,6 +143,7 @@ USAGE photoscli diff --album-id --out [--manifest jsonl|sqlite] photoscli verify --out [--manifest jsonl|sqlite] photoscli manifest repair --out [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run] + photoscli cleanup --out [--manifest jsonl|sqlite] [--dry-run] photoscli retry-failed --out photoscli retry-failed --out --clear-on-success photoscli failures list --out @@ -188,6 +192,9 @@ COMMANDS manifest repair --out [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run] Fill missing manifest size/checksum metadata from files that exist on disk. + cleanup --out [--manifest jsonl|sqlite] [--dry-run] + Remove files not referenced by the manifest. Use --dry-run first. + retry-failed --out Retry assets previously written to failures.jsonl. @@ -2389,6 +2396,75 @@ func verifySidecar(stdout io.Writer, outDir, id, checkPath string, strict bool) return 0 } +func cmdCleanup(args []string, stdout, stderr io.Writer) int { + outDir := flagVal(args, "--out") + if outDir == "" { + fmt.Fprintln(stderr, "error: --out is required") + return exitErr + } + mf, err := manifest.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl")) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + entries, err := loadManifestEntries(outDir, mf) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + keep := map[string]bool{ + "downloads.jsonl": true, + "downloads.db": true, + "export.log": true, + "failures.jsonl": true, + } + for _, e := range entries { + checkPath := e.Path + if checkPath == "" { + checkPath = e.Filename + } + if checkPath == "" { + continue + } + clean := filepath.Clean(checkPath) + if clean == "." || strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) { + continue + } + keep[clean] = true + keep[filepath.Clean(sidecarPath(clean))] = true + keep[filepath.Clean(jsonSidecarPath(clean))] = true + } + dryRun := hasFlag(args, "--dry-run") + removed := 0 + skipped := 0 + _ = filepath.WalkDir(outDir, func(path string, d fs.DirEntry, _ error) error { + if path == outDir { + return nil + } + rel, _ := filepath.Rel(outDir, path) + rel = filepath.Clean(rel) + if d.IsDir() { + if rel == ".photoscli" || strings.HasPrefix(rel, ".photoscli"+string(os.PathSeparator)) { + return filepath.SkipDir + } + return nil + } + if keep[rel] { + return nil + } + fmt.Fprintf(stdout, "%s\torphan\n", rel) + removed++ + if !dryRun { + if err := removeFunc(path); err != nil { + skipped++ + } + } + return nil + }) + fmt.Fprintf(stdout, "removed\t%d\nskipped\t%d\n", removed, skipped) + return exitOK +} + func cmdManifest(args []string, stdout, stderr io.Writer) int { if len(args) < 1 || args[0] != "repair" { fmt.Fprintln(stderr, "error: expected manifest repair") diff --git a/cmd/photoscli/main_test.go b/cmd/photoscli/main_test.go index 44e1af6..bf77ef4 100644 --- a/cmd/photoscli/main_test.go +++ b/cmd/photoscli/main_test.go @@ -4424,6 +4424,88 @@ func TestManifestRepairBranches(t *testing.T) { } } +func TestCleanup(t *testing.T) { + dir := t.TempDir() + for _, path := range []string{"keep.jpg", "keep.xmp", "keep.json", "orphan.jpg", "export.log", "failures.jsonl", filepath.Join(".photoscli", "geocode-cache.jsonl")} { + full := filepath.Join(dir, path) + if err := os.MkdirAll(filepath.Dir(full), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(full, []byte("data"), 0644); err != nil { + t.Fatal(err) + } + } + m := manifest.LoadJSONL(dir) + if err := m.OpenAppend(); err != nil { + t.Fatal(err) + } + m.AddEntry(manifest.Entry{ID: "x1", Filename: "keep.jpg", Path: "keep.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "fallback", Filename: "fallback.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "empty", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "badrel", Filename: "../bad.jpg", Path: "../bad.jpg", Exported: time.Now().Unix()}) + m.Close() + if err := os.WriteFile(filepath.Join(dir, "fallback.jpg"), []byte("data"), 0644); err != nil { + t.Fatal(err) + } + if err := os.Mkdir(filepath.Join(dir, "subdir"), 0755); err != nil { + t.Fatal(err) + } + out, stderr, rc := runWith([]string{"cleanup", "--out", dir, "--dry-run"}, &mockBridge{}) + if rc != exitOK || stderr != "" || !strings.Contains(out, "orphan.jpg\torphan") || !strings.Contains(out, "removed\t1") { + t.Fatalf("cleanup dry rc=%d out=%q stderr=%q", rc, out, stderr) + } + if _, err := os.Stat(filepath.Join(dir, "orphan.jpg")); err != nil { + t.Fatalf("dry-run removed orphan: %v", err) + } + out, stderr, rc = runWith([]string{"cleanup", "--out", dir}, &mockBridge{}) + if rc != exitOK || stderr != "" || !strings.Contains(out, "removed\t1") { + t.Fatalf("cleanup rc=%d out=%q stderr=%q", rc, out, stderr) + } + if _, err := os.Stat(filepath.Join(dir, "orphan.jpg")); !os.IsNotExist(err) { + t.Fatalf("orphan still exists or bad error: %v", err) + } + for _, path := range []string{"keep.jpg", "keep.xmp", "keep.json", "fallback.jpg", "downloads.jsonl", "export.log", "failures.jsonl", filepath.Join(".photoscli", "geocode-cache.jsonl")} { + if _, err := os.Stat(filepath.Join(dir, path)); err != nil { + t.Fatalf("kept file missing %s: %v", path, err) + } + } +} + +func TestCleanupErrors(t *testing.T) { + dir := t.TempDir() + _, stderr, rc := runWith([]string{"cleanup"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "--out") { + t.Fatalf("cleanup missing out rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"cleanup", "--out", dir, "--manifest", "bad"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "manifest") { + t.Fatalf("cleanup bad manifest rc=%d stderr=%q", rc, stderr) + } + badDBDir := t.TempDir() + if err := os.WriteFile(manifest.SQLitePath(badDBDir), []byte("not sqlite"), 0644); err != nil { + t.Fatal(err) + } + _, stderr, rc = runWith([]string{"cleanup", "--out", badDBDir, "--manifest", "sqlite"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "error:") { + t.Fatalf("cleanup load error rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"cleanup", "--out", filepath.Join(dir, "missing")}, &mockBridge{}) + if rc != exitOK || stderr != "" { + t.Fatalf("cleanup missing root rc=%d stderr=%q", rc, stderr) + } + dir = t.TempDir() + oldRemove := removeFunc + removeFunc = func(string) error { return fmt.Errorf("remove") } + if err := os.WriteFile(filepath.Join(dir, "orphan.jpg"), []byte("data"), 0644); err != nil { + t.Fatal(err) + } + out, stderr, rc := runWith([]string{"cleanup", "--out", dir}, &mockBridge{}) + removeFunc = oldRemove + if rc != exitOK || stderr != "" || !strings.Contains(out, "skipped\t1") { + t.Fatalf("cleanup remove error rc=%d out=%q stderr=%q", rc, out, stderr) + } +} + func TestVerifySidecarBranches(t *testing.T) { dir := t.TempDir() subdir := filepath.Join(dir, "sub")