v0.6.0: strengthen backup integrity
This commit is contained in:
+302
-57
@@ -21,6 +21,10 @@ var (
|
||||
exportTimeout = 2 * time.Second
|
||||
configValues map[string]string
|
||||
configLoaded bool
|
||||
mkdirTempFunc = os.MkdirTemp
|
||||
renameFunc = os.Rename
|
||||
openFileFunc = os.OpenFile
|
||||
removeFunc = os.Remove
|
||||
)
|
||||
|
||||
type exportOptions struct {
|
||||
@@ -75,6 +79,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
return cmdVerify(args[1:], stdout, stderr)
|
||||
case "retry-failed":
|
||||
return cmdRetryFailed(args[1:], stdout, stderr, bridge)
|
||||
case "failures":
|
||||
return cmdFailures(args[1:], stdout, stderr)
|
||||
case "status":
|
||||
return cmdStatus(args[1:], stdout, stderr)
|
||||
case "version", "--version", "-v":
|
||||
fmt.Fprintln(stdout, version)
|
||||
return exitOK
|
||||
@@ -110,6 +118,10 @@ USAGE
|
||||
photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli verify --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli retry-failed --out <dir>
|
||||
photoscli retry-failed --out <dir> --clear-on-success
|
||||
photoscli failures list --out <dir>
|
||||
photoscli failures clear --out <dir>
|
||||
photoscli status --out <dir> [--json]
|
||||
photoscli version
|
||||
photoscli help
|
||||
|
||||
@@ -149,6 +161,12 @@ COMMANDS
|
||||
retry-failed --out <dir>
|
||||
Retry assets previously written to failures.jsonl.
|
||||
|
||||
failures list|clear --out <dir>
|
||||
List or clear deduplicated failure records.
|
||||
|
||||
status --out <dir> [--manifest jsonl|sqlite] [--json]
|
||||
Show manifest type, entry count, and failure count for a backup.
|
||||
|
||||
COMMON EXPORT FLAGS
|
||||
--out <dir>
|
||||
Destination directory. Required for export, backup-all, report, diff,
|
||||
@@ -240,7 +258,9 @@ LOGGING AND FAILURES
|
||||
table in downloads.db.
|
||||
|
||||
failures.jsonl
|
||||
Failed exports are appended here and can be retried with retry-failed.
|
||||
Failed exports are deduplicated by asset ID and can be retried with
|
||||
retry-failed. Use retry-failed --clear-on-success to remove successful
|
||||
retries from the failure list.
|
||||
|
||||
CONFIGURATION
|
||||
Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG.
|
||||
@@ -665,6 +685,7 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
|
||||
|
||||
type pendingAsset struct {
|
||||
asset photos.Asset
|
||||
root string
|
||||
path string
|
||||
album string
|
||||
}
|
||||
@@ -691,6 +712,22 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64,
|
||||
}
|
||||
}
|
||||
|
||||
func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
fullPath := filepath.Join(pa.path, result.Filename)
|
||||
relPath, err := filepath.Rel(root, fullPath)
|
||||
if err != nil || strings.HasPrefix(relPath, "..") {
|
||||
relPath = result.Filename
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud))
|
||||
}
|
||||
|
||||
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
|
||||
var items []pendingAsset
|
||||
var skipped int
|
||||
@@ -769,7 +806,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br
|
||||
*skipped++
|
||||
continue
|
||||
}
|
||||
*items = append(*items, pendingAsset{asset: a, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
*items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
}
|
||||
if onProgress != nil {
|
||||
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
|
||||
@@ -850,7 +887,7 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
bar.setAlbum(pa.album, 0, 0)
|
||||
bar.draw()
|
||||
start := time.Now()
|
||||
result, exportErr := exportOneWithRetry(bridge, pa.asset, pa.path, targetSize, quality, originals, i, opts.retry)
|
||||
result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry)
|
||||
dur := time.Since(start)
|
||||
isErr := exportErr != nil
|
||||
isSkipped := result.Skipped
|
||||
@@ -862,14 +899,10 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
failed++
|
||||
appendFailure(pa.path, pa, exportErr)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -922,7 +955,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
start := time.Now()
|
||||
var result photos.ExportResult
|
||||
var exportErr error
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i].asset, pending[i].path, targetSize, quality, originals, i, workerID, opts.retry)
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry)
|
||||
dur := time.Since(start)
|
||||
bar.setWorker(workerID, "", 0, "", "")
|
||||
completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
|
||||
@@ -995,14 +1028,10 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
failed++
|
||||
appendFailure(entry.pa.path, entry.pa, entry.err)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -1036,7 +1065,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) {
|
||||
pending := make([]pendingAsset, len(assets))
|
||||
for i, a := range assets {
|
||||
pending[i] = pendingAsset{asset: a, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
}
|
||||
var m manifest.Manifest
|
||||
if !noManifest {
|
||||
@@ -1080,11 +1109,93 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize,
|
||||
return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index)
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index)
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
if originals {
|
||||
return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex)
|
||||
}
|
||||
return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
var result photos.ExportResult
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
result, err = exportOne(bridge, a, outDir, targetSize, quality, originals, index)
|
||||
result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1093,15 +1204,11 @@ func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, tar
|
||||
return result, err
|
||||
}
|
||||
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(a.ID, outDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(a.ID, outDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1356,22 +1463,73 @@ func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest
|
||||
|
||||
func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") }
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
_ = os.MkdirAll(dir, 0755)
|
||||
f, openErr := os.OpenFile(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if openErr != nil {
|
||||
return
|
||||
type failureEntry struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
FailedAt int64 `json:"failed_at"`
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
func loadFailures(dir string) map[string]failureEntry {
|
||||
out := map[string]failureEntry{}
|
||||
data, err := os.ReadFile(failuresPath(dir))
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f failureEntry
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
out[f.ID] = f
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func saveFailures(dir string, failures map[string]failureEntry) error {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
data, _ := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
}{pa.asset.ID, pa.asset.Filename, pa.album, pa.path, err.Error()})
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
data, _ := json.Marshal(failures[id])
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = dir
|
||||
}
|
||||
failures := loadFailures(root)
|
||||
f := failures[pa.asset.ID]
|
||||
f.ID = pa.asset.ID
|
||||
f.Filename = pa.asset.Filename
|
||||
f.Album = pa.album
|
||||
f.Path = pa.path
|
||||
f.Error = err.Error()
|
||||
f.FailedAt = time.Now().Unix()
|
||||
f.Attempts++
|
||||
failures[pa.asset.ID] = f
|
||||
_ = saveFailures(root, failures)
|
||||
}
|
||||
|
||||
func writeJSONSummary(stdout io.Writer, s commandSummary) {
|
||||
@@ -1466,17 +1624,32 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
missing := 0
|
||||
bad := 0
|
||||
for id, e := range entries {
|
||||
if e.Filename == "" {
|
||||
checkPath := e.Path
|
||||
if checkPath == "" {
|
||||
checkPath = e.Filename
|
||||
}
|
||||
if checkPath == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outDir, e.Filename)); err != nil {
|
||||
missing++
|
||||
fmt.Fprintf(stdout, "%s\t%s\n", id, e.Filename)
|
||||
info, err := os.Stat(filepath.Join(outDir, checkPath))
|
||||
if err != nil {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if e.Size > 0 && info.Size() != e.Size {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
|
||||
}
|
||||
}
|
||||
if missing > 0 {
|
||||
if bad > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
fmt.Fprintf(stdout, "verified\t%d\n", len(entries))
|
||||
@@ -1485,31 +1658,103 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
clearOnSuccess := hasFlag(args, "--clear-on-success")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
data, err := os.ReadFile(failuresPath(outDir))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
failures := loadFailures(outDir)
|
||||
if len(failures) == 0 {
|
||||
fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir))
|
||||
return exitErr
|
||||
}
|
||||
var pending []pendingAsset
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f struct{ ID, Filename, Album, Path string }
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, path: f.Path, album: f.Album})
|
||||
}
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album})
|
||||
}
|
||||
bar := newProgressBar(stderr, 1)
|
||||
done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{})
|
||||
if clearOnSuccess && done > 0 {
|
||||
for i := 0; i < done && i < len(pending); i++ {
|
||||
delete(failures, pending[i].asset.ID)
|
||||
}
|
||||
_ = saveFailures(outDir, failures)
|
||||
}
|
||||
writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)})
|
||||
if failed > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func cmdFailures(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) < 1 {
|
||||
fmt.Fprintln(stderr, "error: failures requires list or clear")
|
||||
return exitErr
|
||||
}
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
failures := loadFailures(outDir)
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts)
|
||||
}
|
||||
return exitOK
|
||||
case "clear":
|
||||
if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
return exitOK
|
||||
default:
|
||||
fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0])
|
||||
return exitErr
|
||||
}
|
||||
}
|
||||
|
||||
func cmdStatus(args []string, stdout, stderr io.Writer) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
|
||||
mf, err := manifest.ParseFormat(manifestFmt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
entries, err := loadManifestEntries(outDir, mf)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
failures := loadFailures(outDir)
|
||||
if hasFlag(args, "--json") {
|
||||
data, _ := json.Marshal(struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Entries int `json:"entries"`
|
||||
Failures int `json:"failures"`
|
||||
}{manifestFmt, len(entries), len(failures)})
|
||||
fmt.Fprintln(stdout, string(data))
|
||||
return exitOK
|
||||
}
|
||||
fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures))
|
||||
return exitOK
|
||||
}
|
||||
|
||||
+270
-1
@@ -3815,7 +3815,7 @@ func TestReportDiffVerifyRetryFailedAndConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("x"), 0644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("0123456789"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
@@ -4043,3 +4043,272 @@ func TestNewFeatureRemainingBranches(t *testing.T) {
|
||||
t.Fatalf("expected retry partial, got %d", rc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailuresAndStatusCommands(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry("x1", "file.jpg", "Album/file.jpg", 4, "local"))
|
||||
m.Close()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "Album"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "Album", "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom"))
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom2"))
|
||||
|
||||
out, stderr, rc := runWith([]string{"status", "--out", dir, "--json"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "failures") || stderr != "" {
|
||||
t.Fatalf("status json rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"status", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "entries\t1") || !strings.Contains(out, "failures\t1") || stderr != "" {
|
||||
t.Fatalf("status rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"failures", "list", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "bad") || !strings.Contains(out, "2") || stderr != "" {
|
||||
t.Fatalf("failures list rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "requires") {
|
||||
t.Fatalf("failures missing subcommand rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "bogus", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "unknown") {
|
||||
t.Fatalf("failures bogus rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || stderr != "" {
|
||||
t.Fatalf("failures clear rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected failures cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicExportHelpers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "photo.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "photo.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneAtomic(b, pa, 1024, 85, false, 0)
|
||||
if err != nil || result.Filename != "photo.jpg" {
|
||||
t.Fatalf("atomic export result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "Album", "photo.jpg")); err != nil {
|
||||
t.Fatalf("expected final file: %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 1); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected zero-byte error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoreIntegrityBranches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.Entry{ID: "zero", Filename: "zero.jpg", Path: "zero.jpg", Size: 1, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "mismatch", Filename: "mismatch.jpg", Path: "mismatch.jpg", Size: 10, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "zero.jpg"), nil, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "mismatch.jpg"), []byte("bad"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _, rc := runWith([]string{"verify", "--out", dir}, &mockBridge{})
|
||||
if rc != exitPartial || !strings.Contains(out, "zero-byte") || !strings.Contains(out, "size-mismatch") {
|
||||
t.Fatalf("verify rc=%d out=%q", rc, out)
|
||||
}
|
||||
_, stderr, rc := runWith([]string{"status"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("status missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", dir, "--manifest", "bad"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "manifest") {
|
||||
t.Fatalf("status bad manifest rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", "/proc/cannot-create"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "error:") {
|
||||
t.Fatalf("status load error rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "list"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("failures missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryFailedClearOnSuccess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
out, _, rc := runWith([]string{"retry-failed", "--out", dir, "--clear-on-success"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "exported") {
|
||||
t.Fatalf("retry clear rc=%d out=%q", rc, out)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected successful retry to clear failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicSlotExportHelper(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "slot.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "slot.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "slot", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0)
|
||||
if err != nil || result.Filename != "slot.jpg" {
|
||||
t.Fatalf("slot atomic result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedErrorBranchesForCoverage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x", Filename: "x.jpg"}, root: dir, path: filepath.Join(dir, "Album")}
|
||||
|
||||
oldMkdirTemp := mkdirTempFunc
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("mkdirtemp") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "mkdirtemp") {
|
||||
t.Fatalf("expected mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
oldRename := renameFunc
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("rename") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "rename") {
|
||||
t.Fatalf("expected rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
oldOpen := openFileFunc
|
||||
openFileFunc = func(string, int, os.FileMode) (*os.File, error) { return nil, fmt.Errorf("open") }
|
||||
if err := saveFailures(dir, map[string]failureEntry{"x": {ID: "x"}}); err == nil || !strings.Contains(err.Error(), "open") {
|
||||
t.Fatalf("expected open error, got %v", err)
|
||||
}
|
||||
openFileFunc = oldOpen
|
||||
|
||||
oldRemove := removeFunc
|
||||
removeFunc = func(string) error { return fmt.Errorf("remove") }
|
||||
_, stderr, rc := runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "remove") {
|
||||
t.Fatalf("expected remove error, rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
removeFunc = oldRemove
|
||||
|
||||
mf := &mockManifest{}
|
||||
addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1})
|
||||
if mf.last.Path != "file.jpg" {
|
||||
t.Fatalf("expected fallback rel path, got %+v", mf.last)
|
||||
}
|
||||
|
||||
badPathRoot := t.TempDir()
|
||||
badPath := filepath.Join(badPathRoot, "notdir")
|
||||
if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "x2", Filename: "x2.jpg"}, root: badPathRoot, path: badPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil {
|
||||
t.Fatal("expected final mkdir error")
|
||||
}
|
||||
|
||||
slotRootFile := filepath.Join(t.TempDir(), "rootfile")
|
||||
if err := os.WriteFile(slotRootFile, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotfallback", Filename: "slot.jpg"}, root: slotRootFile, path: t.TempDir()}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, true, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected original fallback error: %v", err)
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected preview fallback error: %v", err)
|
||||
}
|
||||
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotzero", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Slot")}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected slot zero-byte error, got %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("slot rename") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot rename") {
|
||||
t.Fatalf("expected slot rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("slot mkdirtemp") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot mkdirtemp") {
|
||||
t.Fatalf("expected slot mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
badSlotRoot := t.TempDir()
|
||||
badSlotPath := filepath.Join(badSlotRoot, "notdir")
|
||||
if err := os.WriteFile(badSlotPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotbadpath", Filename: "slot.jpg"}, root: badSlotRoot, path: badSlotPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil {
|
||||
t.Fatal("expected slot final mkdir error")
|
||||
}
|
||||
}
|
||||
|
||||
type mockManifest struct{ last manifest.Entry }
|
||||
|
||||
func (m *mockManifest) Has(string) bool { return false }
|
||||
func (m *mockManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.last = manifest.NewEntry(id, filename, filename, size, cloud)
|
||||
}
|
||||
func (m *mockManifest) AddEntry(e manifest.Entry) { m.last = e }
|
||||
func (m *mockManifest) Save() error { return nil }
|
||||
func (m *mockManifest) Close() {}
|
||||
func (m *mockManifest) OpenAppend() error { return nil }
|
||||
|
||||
Reference in New Issue
Block a user