v0.6.0: strengthen backup integrity

cmd/photoscli/main.go

@@ -21,6 +21,10 @@ var (
 	exportTimeout        = 2 * time.Second
 	configValues         map[string]string
 	configLoaded         bool
+	mkdirTempFunc        = os.MkdirTemp
+	renameFunc           = os.Rename
+	openFileFunc         = os.OpenFile
+	removeFunc           = os.Remove
 )
 
 type exportOptions struct {
@@ -75,6 +79,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
 		return cmdVerify(args[1:], stdout, stderr)
 	case "retry-failed":
 		return cmdRetryFailed(args[1:], stdout, stderr, bridge)
+	case "failures":
+		return cmdFailures(args[1:], stdout, stderr)
+	case "status":
+		return cmdStatus(args[1:], stdout, stderr)
 	case "version", "--version", "-v":
 		fmt.Fprintln(stdout, version)
 		return exitOK
@@ -110,6 +118,10 @@ USAGE
   photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
   photoscli verify --out <dir> [--manifest jsonl|sqlite]
   photoscli retry-failed --out <dir>
+  photoscli retry-failed --out <dir> --clear-on-success
+  photoscli failures list --out <dir>
+  photoscli failures clear --out <dir>
+  photoscli status --out <dir> [--json]
   photoscli version
   photoscli help
 
@@ -149,6 +161,12 @@ COMMANDS
   retry-failed --out <dir>
       Retry assets previously written to failures.jsonl.
 
+  failures list|clear --out <dir>
+      List or clear deduplicated failure records.
+
+  status --out <dir> [--manifest jsonl|sqlite] [--json]
+      Show manifest type, entry count, and failure count for a backup.
+
 COMMON EXPORT FLAGS
   --out <dir>
       Destination directory. Required for export, backup-all, report, diff,
@@ -240,7 +258,9 @@ LOGGING AND FAILURES
       table in downloads.db.
 
   failures.jsonl
-      Failed exports are appended here and can be retried with retry-failed.
+      Failed exports are deduplicated by asset ID and can be retried with
+      retry-failed. Use retry-failed --clear-on-success to remove successful
+      retries from the failure list.
 
 CONFIGURATION
   Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG.
@@ -665,6 +685,7 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
 
 type pendingAsset struct {
 	asset photos.Asset
+	root  string
 	path  string
 	album string
 }
@@ -691,6 +712,22 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64,
 	}
 }
 
+func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) {
+	if m == nil {
+		return
+	}
+	root := pa.root
+	if root == "" {
+		root = pa.path
+	}
+	fullPath := filepath.Join(pa.path, result.Filename)
+	relPath, err := filepath.Rel(root, fullPath)
+	if err != nil || strings.HasPrefix(relPath, "..") {
+		relPath = result.Filename
+	}
+	m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud))
+}
+
 func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
 	var items []pendingAsset
 	var skipped int
@@ -769,7 +806,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br
 					*skipped++
 					continue
 				}
-				*items = append(*items, pendingAsset{asset: a, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
+				*items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
 			}
 			if onProgress != nil {
 				onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
@@ -850,7 +887,7 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
 		bar.setAlbum(pa.album, 0, 0)
 		bar.draw()
 		start := time.Now()
-		result, exportErr := exportOneWithRetry(bridge, pa.asset, pa.path, targetSize, quality, originals, i, opts.retry)
+		result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry)
 		dur := time.Since(start)
 		isErr := exportErr != nil
 		isSkipped := result.Skipped
@@ -862,14 +899,10 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
 			failed++
 			appendFailure(pa.path, pa, exportErr)
 		} else if isSkipped {
-			if m != nil {
-				m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
-			}
+			addManifestEntry(m, pa, result)
 		} else {
 			done++
-			if m != nil {
-				m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
-			}
+			addManifestEntry(m, pa, result)
 		}
 		avgSpeed := float64(0)
 		if totalDur > 0 {
@@ -922,7 +955,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
 				start := time.Now()
 				var result photos.ExportResult
 				var exportErr error
-				result, exportErr = exportOneWithSlotRetry(bridge, pending[i].asset, pending[i].path, targetSize, quality, originals, i, workerID, opts.retry)
+				result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry)
 				dur := time.Since(start)
 				bar.setWorker(workerID, "", 0, "", "")
 				completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
@@ -995,14 +1028,10 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
 			failed++
 			appendFailure(entry.pa.path, entry.pa, entry.err)
 		} else if isSkipped {
-			if m != nil {
-				m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
-			}
+			addManifestEntry(m, entry.pa, entry.result)
 		} else {
 			done++
-			if m != nil {
-				m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
-			}
+			addManifestEntry(m, entry.pa, entry.result)
 		}
 		avgSpeed := float64(0)
 		if totalDur > 0 {
@@ -1036,7 +1065,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
 func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) {
 	pending := make([]pendingAsset, len(assets))
 	for i, a := range assets {
-		pending[i] = pendingAsset{asset: a, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
+		pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
 	}
 	var m manifest.Manifest
 	if !noManifest {
@@ -1080,11 +1109,93 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize,
 	return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index)
 }
 
-func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
+func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
+	root := pa.root
+	if root == "" {
+		root = pa.path
+	}
+	stagingRoot := filepath.Join(root, ".photoscli-tmp")
+	if err := os.MkdirAll(stagingRoot, 0755); err != nil {
+		return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index)
+	}
+	stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
+	if err != nil {
+		return photos.ExportResult{}, err
+	}
+	defer os.RemoveAll(stagingDir)
+
+	result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index)
+	if err != nil || result.Skipped {
+		return result, err
+	}
+	src := filepath.Join(stagingDir, result.Filename)
+	info, statErr := os.Stat(src)
+	if statErr != nil {
+		return result, nil
+	}
+	if info.Size() == 0 {
+		return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
+	}
+	if err := os.MkdirAll(pa.path, 0755); err != nil {
+		return result, err
+	}
+	dst := filepath.Join(pa.path, result.Filename)
+	if err := renameFunc(src, dst); err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) {
+	root := pa.root
+	if root == "" {
+		root = pa.path
+	}
+	stagingRoot := filepath.Join(root, ".photoscli-tmp")
+	if err := os.MkdirAll(stagingRoot, 0755); err != nil {
+		if originals {
+			return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex)
+		}
+		return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex)
+	}
+	stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
+	if err != nil {
+		return photos.ExportResult{}, err
+	}
+	defer os.RemoveAll(stagingDir)
+
+	var result photos.ExportResult
+	if originals {
+		result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex)
+	} else {
+		result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex)
+	}
+	if err != nil || result.Skipped {
+		return result, err
+	}
+	src := filepath.Join(stagingDir, result.Filename)
+	info, statErr := os.Stat(src)
+	if statErr != nil {
+		return result, nil
+	}
+	if info.Size() == 0 {
+		return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
+	}
+	if err := os.MkdirAll(pa.path, 0755); err != nil {
+		return result, err
+	}
+	dst := filepath.Join(pa.path, result.Filename)
+	if err := renameFunc(src, dst); err != nil {
+		return result, err
+	}
+	return result, nil
+}
+
+func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
 	var result photos.ExportResult
 	var err error
 	for attempt := 0; attempt <= retry; attempt++ {
-		result, err = exportOne(bridge, a, outDir, targetSize, quality, originals, index)
+		result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index)
 		if err == nil {
 			return result, nil
 		}
@@ -1093,15 +1204,11 @@ func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, tar
 	return result, err
 }
 
-func exportOneWithSlotRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
+func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
 	var result photos.ExportResult
 	var err error
 	for attempt := 0; attempt <= retry; attempt++ {
-		if originals {
-			result, err = bridge.ExportOriginalWithSlot(a.ID, outDir, index, slotIndex)
-		} else {
-			result, err = bridge.ExportPreviewWithSlot(a.ID, outDir, targetSize, quality, index, slotIndex)
-		}
+		result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex)
 		if err == nil {
 			return result, nil
 		}
@@ -1356,22 +1463,73 @@ func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest
 
 func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") }
 
-func appendFailure(dir string, pa pendingAsset, err error) {
-	_ = os.MkdirAll(dir, 0755)
-	f, openErr := os.OpenFile(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
-	if openErr != nil {
-		return
+type failureEntry struct {
+	ID       string `json:"id"`
+	Filename string `json:"filename"`
+	Album    string `json:"album"`
+	Path     string `json:"path"`
+	Error    string `json:"error"`
+	FailedAt int64  `json:"failed_at"`
+	Attempts int    `json:"attempts"`
+}
+
+func loadFailures(dir string) map[string]failureEntry {
+	out := map[string]failureEntry{}
+	data, err := os.ReadFile(failuresPath(dir))
+	if err != nil {
+		return out
+	}
+	for _, line := range strings.Split(string(data), "\n") {
+		line = strings.TrimSpace(line)
+		if line == "" {
+			continue
+		}
+		var f failureEntry
+		if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
+			out[f.ID] = f
+		}
+	}
+	return out
+}
+
+func saveFailures(dir string, failures map[string]failureEntry) error {
+	if err := os.MkdirAll(dir, 0755); err != nil {
+		return err
+	}
+	f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
+	if err != nil {
+		return err
 	}
 	defer f.Close()
-	data, _ := json.Marshal(struct {
-		ID       string `json:"id"`
-		Filename string `json:"filename"`
-		Album    string `json:"album"`
-		Path     string `json:"path"`
-		Error    string `json:"error"`
-	}{pa.asset.ID, pa.asset.Filename, pa.album, pa.path, err.Error()})
-	f.Write(data)
-	f.Write([]byte("\n"))
+	ids := make([]string, 0, len(failures))
+	for id := range failures {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		data, _ := json.Marshal(failures[id])
+		f.Write(data)
+		f.Write([]byte("\n"))
+	}
+	return nil
+}
+
+func appendFailure(dir string, pa pendingAsset, err error) {
+	root := pa.root
+	if root == "" {
+		root = dir
+	}
+	failures := loadFailures(root)
+	f := failures[pa.asset.ID]
+	f.ID = pa.asset.ID
+	f.Filename = pa.asset.Filename
+	f.Album = pa.album
+	f.Path = pa.path
+	f.Error = err.Error()
+	f.FailedAt = time.Now().Unix()
+	f.Attempts++
+	failures[pa.asset.ID] = f
+	_ = saveFailures(root, failures)
 }
 
 func writeJSONSummary(stdout io.Writer, s commandSummary) {
@@ -1466,17 +1624,32 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
 		fmt.Fprintf(stderr, "error: %v\n", err)
 		return exitErr
 	}
-	missing := 0
+	bad := 0
 	for id, e := range entries {
-		if e.Filename == "" {
+		checkPath := e.Path
+		if checkPath == "" {
+			checkPath = e.Filename
+		}
+		if checkPath == "" {
 			continue
 		}
-		if _, err := os.Stat(filepath.Join(outDir, e.Filename)); err != nil {
-			missing++
-			fmt.Fprintf(stdout, "%s\t%s\n", id, e.Filename)
+		info, err := os.Stat(filepath.Join(outDir, checkPath))
+		if err != nil {
+			bad++
+			fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath)
+			continue
+		}
+		if info.Size() == 0 {
+			bad++
+			fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath)
+			continue
+		}
+		if e.Size > 0 && info.Size() != e.Size {
+			bad++
+			fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
 		}
 	}
-	if missing > 0 {
+	if bad > 0 {
 		return exitPartial
 	}
 	fmt.Fprintf(stdout, "verified\t%d\n", len(entries))
@@ -1485,31 +1658,103 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
 
 func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
 	outDir := flagVal(args, "--out")
+	clearOnSuccess := hasFlag(args, "--clear-on-success")
 	if outDir == "" {
 		fmt.Fprintln(stderr, "error: --out is required")
 		return exitErr
 	}
-	data, err := os.ReadFile(failuresPath(outDir))
-	if err != nil {
-		fmt.Fprintf(stderr, "error: %v\n", err)
+	failures := loadFailures(outDir)
+	if len(failures) == 0 {
+		fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir))
 		return exitErr
 	}
 	var pending []pendingAsset
-	for _, line := range strings.Split(string(data), "\n") {
-		line = strings.TrimSpace(line)
-		if line == "" {
-			continue
-		}
-		var f struct{ ID, Filename, Album, Path string }
-		if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
-			pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, path: f.Path, album: f.Album})
-		}
+	ids := make([]string, 0, len(failures))
+	for id := range failures {
+		ids = append(ids, id)
+	}
+	sort.Strings(ids)
+	for _, id := range ids {
+		f := failures[id]
+		pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album})
 	}
 	bar := newProgressBar(stderr, 1)
 	done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{})
+	if clearOnSuccess && done > 0 {
+		for i := 0; i < done && i < len(pending); i++ {
+			delete(failures, pending[i].asset.ID)
+		}
+		_ = saveFailures(outDir, failures)
+	}
 	writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)})
 	if failed > 0 {
 		return exitPartial
 	}
 	return exitOK
 }
+
+func cmdFailures(args []string, stdout, stderr io.Writer) int {
+	if len(args) < 1 {
+		fmt.Fprintln(stderr, "error: failures requires list or clear")
+		return exitErr
+	}
+	outDir := flagVal(args, "--out")
+	if outDir == "" {
+		fmt.Fprintln(stderr, "error: --out is required")
+		return exitErr
+	}
+	switch args[0] {
+	case "list":
+		failures := loadFailures(outDir)
+		ids := make([]string, 0, len(failures))
+		for id := range failures {
+			ids = append(ids, id)
+		}
+		sort.Strings(ids)
+		for _, id := range ids {
+			f := failures[id]
+			fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts)
+		}
+		return exitOK
+	case "clear":
+		if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) {
+			fmt.Fprintf(stderr, "error: %v\n", err)
+			return exitErr
+		}
+		return exitOK
+	default:
+		fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0])
+		return exitErr
+	}
+}
+
+func cmdStatus(args []string, stdout, stderr io.Writer) int {
+	outDir := flagVal(args, "--out")
+	if outDir == "" {
+		fmt.Fprintln(stderr, "error: --out is required")
+		return exitErr
+	}
+	manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
+	mf, err := manifest.ParseFormat(manifestFmt)
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	entries, err := loadManifestEntries(outDir, mf)
+	if err != nil {
+		fmt.Fprintf(stderr, "error: %v\n", err)
+		return exitErr
+	}
+	failures := loadFailures(outDir)
+	if hasFlag(args, "--json") {
+		data, _ := json.Marshal(struct {
+			Manifest string `json:"manifest"`
+			Entries  int    `json:"entries"`
+			Failures int    `json:"failures"`
+		}{manifestFmt, len(entries), len(failures)})
+		fmt.Fprintln(stdout, string(data))
+		return exitOK
+	}
+	fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures))
+	return exitOK
+}