v0.9.4: add doctor diagnostics

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
 
+## v0.9.4
+
+Doctor release.
+
+- Add `doctor` to check Photos access and optional backup health.
+- Add `doctor --out <dir>` to report backup directory, manifest entries, and failure count.
+- Add `doctor --json` for scriptable diagnostics.
+
+## v0.9.3
+
+Cleanup release.
+
+- Add `cleanup --out <dir>` to remove files not referenced by the manifest.
+- Add `cleanup --dry-run` to preview orphaned files before deletion.
+- Preserve manifest/log/failure files, `.photoscli`, and sidecars for manifest-referenced media.
+
 ## v0.9.2
 
 Manifest repair release.

Makefile

@@ -1,6 +1,6 @@
 BINARY     := ./bin/photoscli
 MODULE     := gitea.k3s.k0.nu/tools/photocli
-VERSION    := 0.9.2
+VERSION    := 0.9.4
 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip
 RELEASE_NOTES := RELEASE_NOTES.md
 BRIDGE_DIR := bridge

README.md

@@ -119,6 +119,8 @@ Verify a backup later:
 ./bin/photoscli verify --out ./photos-backup --manifest sqlite
 ./bin/photoscli verify --out ./photos-backup --deep
 ./bin/photoscli manifest repair --out ./photos-backup --checksum sha256 --dry-run
+./bin/photoscli cleanup --out ./photos-backup --dry-run
+./bin/photoscli doctor --out ./photos-backup
 ```
 
 ## Commands

RELEASE_NOTES.md

@@ -1,18 +1,18 @@
-# v0.9.2
+# v0.9.4
 
-This release adds manifest repair for existing backups.
+This release adds `doctor` diagnostics for Photos access and backups.
 
 ## Highlights
 
-- Add `manifest repair --out <dir>` to fill missing size metadata from existing files.
+- Add `doctor` to check Photos access.
-- Add `manifest repair --checksum sha256` to fill missing checksums.
+- Add `doctor --out <dir>` to report backup directory, manifest entries, and failure count.
-- Add `manifest repair --dry-run` to preview repairs without writing updates.
+- Add `doctor --json` for scriptable diagnostics.
-- Repair works with JSONL and SQLite manifests.
+- `doctor` is read-only and does not modify backup data.
 
 ## Assets
 
 - `photoscli`: Apple Silicon macOS binary (`darwin/arm64`).
-- `photoscli-0.9.2-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
+- `photoscli-0.9.4-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
 - `USERGUIDE.md`: standalone user guide.
 
 Intel Macs are not currently a supported release target.

USERGUIDE.md

@@ -847,6 +847,25 @@ Repair missing manifest metadata from files already present on disk:
 
 Repair fills missing size/checksum metadata and skips missing, zero-byte, or unreadable files.
 
+Preview and remove files not referenced by the manifest:
+
+```bash
+./bin/photoscli cleanup --out ./PhotosBackup --dry-run
+./bin/photoscli cleanup --out ./PhotosBackup
+```
+
+Cleanup preserves manifest/log/failure files, `.photoscli`, and sidecars next to manifest-referenced media.
+
+Run read-only diagnostics:
+
+```bash
+./bin/photoscli doctor
+./bin/photoscli doctor --out ./PhotosBackup
+./bin/photoscli doctor --out ./PhotosBackup --json
+```
+
+Doctor checks Photos access and, when `--out` is provided, backup directory status, manifest entries, and failure count.
+
 ## Safe Operating Practices
 
 - Run `--dry-run` before the first large backup.

cmd/photoscli/main.go

@@ -8,6 +8,7 @@ import (
 	"encoding/xml"
 	"fmt"
 	"io"
+	"io/fs"
 	"os"
 	"path/filepath"
 	"sort"
@@ -94,6 +95,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
 		return cmdVerify(args[1:], stdout, stderr)
 	case "manifest":
 		return cmdManifest(args[1:], stdout, stderr)
+	case "cleanup":
+		return cmdCleanup(args[1:], stdout, stderr)
+	case "doctor":
+		return cmdDoctor(args[1:], stdout, stderr, bridge)
 	case "retry-failed":
 		return cmdRetryFailed(args[1:], stdout, stderr, bridge)
 	case "failures":
@@ -140,6 +145,8 @@ USAGE
   photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
   photoscli verify --out <dir> [--manifest jsonl|sqlite]
   photoscli manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
+  photoscli cleanup --out <dir> [--manifest jsonl|sqlite] [--dry-run]
+  photoscli doctor [--out <dir>] [--manifest jsonl|sqlite] [--json]
   photoscli retry-failed --out <dir>
   photoscli retry-failed --out <dir> --clear-on-success
   photoscli failures list --out <dir>
@@ -188,6 +195,12 @@ COMMANDS
   manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
       Fill missing manifest size/checksum metadata from files that exist on disk.
 
+  cleanup --out <dir> [--manifest jsonl|sqlite] [--dry-run]
+      Remove files not referenced by the manifest. Use --dry-run first.
+
+  doctor [--out <dir>] [--manifest jsonl|sqlite] [--json]
+      Check Photos access and optional backup/manifest health without changing data.
+
   retry-failed --out <dir>
       Retry assets previously written to failures.jsonl.
 
@@ -2389,6 +2402,133 @@ func verifySidecar(stdout io.Writer, outDir, id, checkPath string, strict bool)
 	return 0
 }
 
+func cmdDoctor(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
+	result := map[string]any{"photos_access": "ok"}
+	problems := 0
+	if rc := mustAuth(stderr, bridge); rc != exitOK {
+		result["photos_access"] = "denied"
+		problems++
+	}
+	outDir := flagVal(args, "--out")
+	if outDir != "" {
+		result["out"] = outDir
+		if info, err := statFunc(outDir); err != nil || !info.IsDir() {
+			result["backup_dir"] = "missing"
+			problems++
+		} else {
+			result["backup_dir"] = "ok"
+			mf, err := manifest.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
+			if err != nil {
+				fmt.Fprintf(stderr, "error: %v\n", err)
+				return exitErr
+			}
+			entries, err := loadManifestEntries(outDir, mf)
+			if err != nil {
+				result["manifest"] = "error"
+				problems++
+			} else {
+				result["manifest"] = string(mf)
+				result["entries"] = len(entries)
+			}
+			failures := loadFailures(outDir)
+			result["failures"] = len(failures)
+			if len(failures) > 0 {
+				problems++
+			}
+		}
+	}
+	if hasFlag(args, "--json") {
+		result["problems"] = problems
+		if err := json.NewEncoder(stdout).Encode(result); err != nil {
+			fmt.Fprintf(stderr, "error: %v\n", err)
+			return exitErr
+		}
+	} else {
+		keys := make([]string, 0, len(result))
+		for k := range result {
+			keys = append(keys, k)
+		}
+		sort.Strings(keys)
+		for _, k := range keys {
+			fmt.Fprintf(stdout, "%s\t%v\n", k, result[k])
+		}
+		fmt.Fprintf(stdout, "problems\t%d\n", problems)
+	}
+	if problems > 0 {
+		return exitPartial
+	}
+	return exitOK
+}
+
+func cmdCleanup(args []string, stdout, stderr io.Writer) int {
+	outDir := flagVal(args, "--out")
+	if outDir == "" {
+		fmt.Fprintln(stderr, "error: --out is required")
+		return exitErr
+	}
+	mf, err := manifest.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	entries, err := loadManifestEntries(outDir, mf)
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	keep := map[string]bool{
+		"downloads.jsonl": true,
+		"downloads.db":    true,
+		"export.log":      true,
+		"failures.jsonl":  true,
+	}
+	for _, e := range entries {
+		checkPath := e.Path
+		if checkPath == "" {
+			checkPath = e.Filename
+		}
+		if checkPath == "" {
+			continue
+		}
+		clean := filepath.Clean(checkPath)
+		if clean == "." || strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) {
+			continue
+		}
+		keep[clean] = true
+		keep[filepath.Clean(sidecarPath(clean))] = true
+		keep[filepath.Clean(jsonSidecarPath(clean))] = true
+	}
+	dryRun := hasFlag(args, "--dry-run")
+	removed := 0
+	skipped := 0
+	_ = filepath.WalkDir(outDir, func(path string, d fs.DirEntry, _ error) error {
+		if path == outDir {
+			return nil
+		}
+		rel, _ := filepath.Rel(outDir, path)
+		rel = filepath.Clean(rel)
+		if d.IsDir() {
+			if rel == ".photoscli" || strings.HasPrefix(rel, ".photoscli"+string(os.PathSeparator)) {
+				return filepath.SkipDir
+			}
+			return nil
+		}
+		if keep[rel] {
+			return nil
+		}
+		fmt.Fprintf(stdout, "%s\torphan\n", rel)
+		removed++
+		if !dryRun {
+			if err := removeFunc(path); err != nil {
+				skipped++
+			}
+		}
+		return nil
+	})
+	fmt.Fprintf(stdout, "removed\t%d\nskipped\t%d\n", removed, skipped)
+	return exitOK
+}
+
 func cmdManifest(args []string, stdout, stderr io.Writer) int {
 	if len(args) < 1 || args[0] != "repair" {
 		fmt.Fprintln(stderr, "error: expected manifest repair")

cmd/photoscli/main_test.go

@@ -4424,6 +4424,135 @@ func TestManifestRepairBranches(t *testing.T) {
 	}
 }
 
+func TestCleanup(t *testing.T) {
+	dir := t.TempDir()
+	for _, path := range []string{"keep.jpg", "keep.xmp", "keep.json", "orphan.jpg", "export.log", "failures.jsonl", filepath.Join(".photoscli", "geocode-cache.jsonl")} {
+		full := filepath.Join(dir, path)
+		if err := os.MkdirAll(filepath.Dir(full), 0755); err != nil {
+			t.Fatal(err)
+		}
+		if err := os.WriteFile(full, []byte("data"), 0644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	m := manifest.LoadJSONL(dir)
+	if err := m.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	m.AddEntry(manifest.Entry{ID: "x1", Filename: "keep.jpg", Path: "keep.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "fallback", Filename: "fallback.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "empty", Exported: time.Now().Unix()})
+	m.AddEntry(manifest.Entry{ID: "badrel", Filename: "../bad.jpg", Path: "../bad.jpg", Exported: time.Now().Unix()})
+	m.Close()
+	if err := os.WriteFile(filepath.Join(dir, "fallback.jpg"), []byte("data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	if err := os.Mkdir(filepath.Join(dir, "subdir"), 0755); err != nil {
+		t.Fatal(err)
+	}
+	out, stderr, rc := runWith([]string{"cleanup", "--out", dir, "--dry-run"}, &mockBridge{})
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "orphan.jpg\torphan") || !strings.Contains(out, "removed\t1") {
+		t.Fatalf("cleanup dry rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "orphan.jpg")); err != nil {
+		t.Fatalf("dry-run removed orphan: %v", err)
+	}
+	out, stderr, rc = runWith([]string{"cleanup", "--out", dir}, &mockBridge{})
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "removed\t1") {
+		t.Fatalf("cleanup rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	if _, err := os.Stat(filepath.Join(dir, "orphan.jpg")); !os.IsNotExist(err) {
+		t.Fatalf("orphan still exists or bad error: %v", err)
+	}
+	for _, path := range []string{"keep.jpg", "keep.xmp", "keep.json", "fallback.jpg", "downloads.jsonl", "export.log", "failures.jsonl", filepath.Join(".photoscli", "geocode-cache.jsonl")} {
+		if _, err := os.Stat(filepath.Join(dir, path)); err != nil {
+			t.Fatalf("kept file missing %s: %v", path, err)
+		}
+	}
+}
+
+func TestCleanupErrors(t *testing.T) {
+	dir := t.TempDir()
+	_, stderr, rc := runWith([]string{"cleanup"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "--out") {
+		t.Fatalf("cleanup missing out rc=%d stderr=%q", rc, stderr)
+	}
+	_, stderr, rc = runWith([]string{"cleanup", "--out", dir, "--manifest", "bad"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "manifest") {
+		t.Fatalf("cleanup bad manifest rc=%d stderr=%q", rc, stderr)
+	}
+	badDBDir := t.TempDir()
+	if err := os.WriteFile(manifest.SQLitePath(badDBDir), []byte("not sqlite"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	_, stderr, rc = runWith([]string{"cleanup", "--out", badDBDir, "--manifest", "sqlite"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "error:") {
+		t.Fatalf("cleanup load error rc=%d stderr=%q", rc, stderr)
+	}
+	_, stderr, rc = runWith([]string{"cleanup", "--out", filepath.Join(dir, "missing")}, &mockBridge{})
+	if rc != exitOK || stderr != "" {
+		t.Fatalf("cleanup missing root rc=%d stderr=%q", rc, stderr)
+	}
+	dir = t.TempDir()
+	oldRemove := removeFunc
+	removeFunc = func(string) error { return fmt.Errorf("remove") }
+	if err := os.WriteFile(filepath.Join(dir, "orphan.jpg"), []byte("data"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	out, stderr, rc := runWith([]string{"cleanup", "--out", dir}, &mockBridge{})
+	removeFunc = oldRemove
+	if rc != exitOK || stderr != "" || !strings.Contains(out, "skipped\t1") {
+		t.Fatalf("cleanup remove error rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+}
+
+func TestDoctor(t *testing.T) {
+	dir := t.TempDir()
+	m := manifest.LoadJSONL(dir)
+	if err := m.OpenAppend(); err != nil {
+		t.Fatal(err)
+	}
+	m.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
+	m.Close()
+	out, stderr, rc := runWith([]string{"doctor", "--out", dir}, &mockBridge{})
+	if rc != exitOK || !strings.Contains(stderr, "access granted") || !strings.Contains(out, "backup_dir\tok") || !strings.Contains(out, "entries\t1") || !strings.Contains(out, "problems\t0") {
+		t.Fatalf("doctor rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	out, stderr, rc = runWith([]string{"doctor", "--out", dir, "--json"}, &mockBridge{})
+	if rc != exitOK || !strings.Contains(out, `"entries":1`) || !strings.Contains(out, `"problems":0`) {
+		t.Fatalf("doctor json rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+}
+
+func TestDoctorProblems(t *testing.T) {
+	dir := t.TempDir()
+	appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}}, fmt.Errorf("failed"))
+	out, stderr, rc := runWith([]string{"doctor", "--out", dir}, &mockBridge{accessErr: fmt.Errorf("denied")})
+	if rc != exitPartial || !strings.Contains(stderr, "denied") || !strings.Contains(out, "photos_access\tdenied") || !strings.Contains(out, "failures\t1") {
+		t.Fatalf("doctor problems rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	_, stderr, rc = runWith([]string{"doctor", "--out", dir, "--manifest", "bad"}, &mockBridge{})
+	if rc != exitErr || !strings.Contains(stderr, "manifest") {
+		t.Fatalf("doctor bad manifest rc=%d stderr=%q", rc, stderr)
+	}
+	out, stderr, rc = runWith([]string{"doctor", "--out", filepath.Join(dir, "missing")}, &mockBridge{})
+	if rc != exitPartial || !strings.Contains(out, "backup_dir\tmissing") {
+		t.Fatalf("doctor missing dir rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	badDBDir := t.TempDir()
+	if err := os.WriteFile(manifest.SQLitePath(badDBDir), []byte("not sqlite"), 0644); err != nil {
+		t.Fatal(err)
+	}
+	out, stderr, rc = runWith([]string{"doctor", "--out", badDBDir, "--manifest", "sqlite"}, &mockBridge{})
+	if rc != exitPartial || !strings.Contains(out, "manifest\terror") {
+		t.Fatalf("doctor manifest error rc=%d out=%q stderr=%q", rc, out, stderr)
+	}
+	stderrBuf := &bytes.Buffer{}
+	if rc := cmdDoctor([]string{"--json"}, errWriter{}, stderrBuf, &mockBridge{}); rc != exitErr || !strings.Contains(stderrBuf.String(), "error:") {
+		t.Fatalf("doctor json writer error rc=%d stderr=%q", rc, stderrBuf.String())
+	}
+}
+
 func TestVerifySidecarBranches(t *testing.T) {
 	dir := t.TempDir()
 	subdir := filepath.Join(dir, "sub")