v0.6.0: strengthen backup integrity
This commit is contained in:
+98
-23
@@ -1,48 +1,123 @@
|
||||
# Changelog
|
||||
|
||||
This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
|
||||
|
||||
## v0.6.0
|
||||
|
||||
Backup integrity and recovery release.
|
||||
|
||||
- Add manifest-relative paths for exported files so tree backups can be verified accurately.
|
||||
- Add SQLite manifest migration for the new `path` column.
|
||||
- Upgrade `verify` to report missing files, zero-byte files, and size mismatches by default.
|
||||
- Add atomic staging export writes with per-asset temporary directories before final rename.
|
||||
- Deduplicate `failures.jsonl` entries by asset ID and track attempts plus latest failure time.
|
||||
- Add `failures list` and `failures clear` commands.
|
||||
- Add `retry-failed --clear-on-success`.
|
||||
- Add `status` command with text and JSON output.
|
||||
- Update release workflow to publish release notes from `RELEASE_NOTES.md`.
|
||||
- Keep `CHANGELOG.md` as the canonical release history.
|
||||
|
||||
## v0.5.0
|
||||
|
||||
- Add JSONL and SQLite manifests with resumable skip tracking
|
||||
- Add structured export logging to export.log or SQLite logs table
|
||||
- Add --dry-run, --retry, --only-favorites, --media, --json, --verify, --date-template, and size filters
|
||||
- Add report, diff, verify, and retry-failed commands
|
||||
- Add failure tracking in failures.jsonl
|
||||
- Add configurable preview quality and export concurrency
|
||||
- Add album exclusion, since-date filtering, album collision handling, and config-file defaults
|
||||
- Expand README and CLI help output
|
||||
- Maintain 100% test coverage in the release pipeline
|
||||
Manifest, filtering, logging, and documentation release.
|
||||
|
||||
- Add JSONL and SQLite manifests with resumable skip tracking.
|
||||
- Add automatic conversion between JSONL and SQLite manifests.
|
||||
- Add structured export logging to `export.log` or SQLite `logs` table.
|
||||
- Add explicit exit codes: success, error, partial failure, and access denied.
|
||||
- Add `--quality` for JPEG preview compression.
|
||||
- Add `--concurrency` with progress slot cap.
|
||||
- Add `--exclude-album` with exact/glob matching.
|
||||
- Add `--since` date filtering.
|
||||
- Add `--dry-run`.
|
||||
- Add `--retry`.
|
||||
- Add `--only-favorites`.
|
||||
- Add `--media photos|videos|all`.
|
||||
- Add `--json` command summaries.
|
||||
- Add `--verify` integration.
|
||||
- Add `--format jpeg|heic|png` validation/hint.
|
||||
- Add `--min-size` and `--max-size` estimated pixel-count filters.
|
||||
- Add `--date-template` folder layout support.
|
||||
- Add `report`, `diff`, `verify`, and `retry-failed` commands.
|
||||
- Add failure tracking in `failures.jsonl`.
|
||||
- Add config-file defaults via `~/.photoscli.toml` or `PHOTOSCLI_CONFIG`.
|
||||
- Add album name collision handling for duplicate sibling album names.
|
||||
- Expand `README.md` and verbose CLI help.
|
||||
- Add 100% test coverage for the CLI and manifest/photos packages.
|
||||
|
||||
## v0.4.1
|
||||
|
||||
Documentation and release asset follow-up after v0.5.0.
|
||||
|
||||
- Add `USERGUIDE.md` as a practical end-user manual.
|
||||
- Link the user guide from `README.md`.
|
||||
- Add release zip packaging with binary, README, USERGUIDE, and CHANGELOG.
|
||||
- Attach `USERGUIDE.md` and the macOS zip package to the `v0.5.0` release.
|
||||
|
||||
## v0.4.0
|
||||
|
||||
- Scroll log for completed exports (copied, downloaded, skipped, failed)
|
||||
- Worker status lines with live download progress bars for cloud files
|
||||
- Color gradient progress bar (red to yellow to green)
|
||||
- Disk skip: files already on disk are skipped during backup-all
|
||||
- Parallel export with 3 workers for backup-all
|
||||
Progress and backup-all usability release.
|
||||
|
||||
- Add scroll log for completed exports: copied, downloaded, skipped, and failed.
|
||||
- Add worker status lines with live download progress bars for cloud files.
|
||||
- Add color-gradient progress bars from red to yellow to green.
|
||||
- Add disk skip for files already present during `backup-all`.
|
||||
- Add parallel export with multiple workers for `backup-all`.
|
||||
- Increase progress slot support in the bridge.
|
||||
- Improve progress rendering for large export sessions.
|
||||
|
||||
## v0.3.x
|
||||
|
||||
Published on Gitea as `v0.3.0` through `v0.3.5`. These releases are not represented by local git tags in the current clone, so the exact per-release commit mapping is unavailable here.
|
||||
|
||||
Known purpose of this series:
|
||||
|
||||
- Iterative release packaging and Gitea release workflow improvements.
|
||||
- Continued progress display and backup/export polish between `v0.2.5` and `v0.4.0`.
|
||||
- Preparation for the larger `v0.4.0` progress/backup release.
|
||||
|
||||
## v0.2.6
|
||||
|
||||
Published on Gitea but not represented by a local git tag in the current clone.
|
||||
|
||||
- Post-`v0.2.5` maintenance release in the progress/export series.
|
||||
|
||||
## v0.2.5
|
||||
|
||||
- Unicode progress bar with cloud download speed display
|
||||
- Add Unicode progress bar rendering.
|
||||
- Add cloud download speed display.
|
||||
|
||||
## v0.2.4
|
||||
|
||||
- Stop export loop on Ctrl+C instead of flooding failures
|
||||
- Stop export loop on Ctrl+C instead of flooding failures.
|
||||
- Improve graceful cancellation behavior after interrupt.
|
||||
|
||||
## v0.2.3
|
||||
|
||||
- Fix export write failures and Ctrl+C cancellation
|
||||
- Fix export write failures.
|
||||
- Fix Ctrl+C cancellation behavior.
|
||||
|
||||
## v0.2.2
|
||||
|
||||
No local tag is present for this version in the current clone.
|
||||
|
||||
## v0.2.1
|
||||
|
||||
- Add status messages during export
|
||||
- Fix parallel export progress display
|
||||
- Add status messages during export.
|
||||
- Fix parallel export progress display.
|
||||
|
||||
## v0.2.0
|
||||
|
||||
- Semaphore timeouts, error logging, dead code removal
|
||||
- Parallel exports
|
||||
- Add semaphore timeouts.
|
||||
- Add export error logging.
|
||||
- Remove dead code.
|
||||
- Add parallel exports.
|
||||
|
||||
## Pre-release
|
||||
|
||||
- Initial applephotos CLI with progress, cloud status, per-asset export
|
||||
- Renamed from applephotos to photoscli
|
||||
- Initial `applephotos` CLI with progress, cloud status, and per-asset export.
|
||||
- Add `.gitignore` and remove build artifacts from tracking.
|
||||
- Rename `applephotos` to `photoscli`.
|
||||
- Update module path to `gitea.k3s.k0.nu/tools/photocli`.
|
||||
- Add version flag.
|
||||
- Add Gitea release targets and release pipeline.
|
||||
|
||||
@@ -1,7 +1,8 @@
|
||||
BINARY := ./bin/photoscli
|
||||
MODULE := gitea.k3s.k0.nu/tools/photocli
|
||||
VERSION := 0.5.0
|
||||
VERSION := 0.6.0
|
||||
RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos.zip
|
||||
RELEASE_NOTES := RELEASE_NOTES.md
|
||||
BRIDGE_DIR := bridge
|
||||
LDFLAGS := -X main.version=$(VERSION)
|
||||
OBJ := $(BRIDGE_DIR)/photokit_bridge.o
|
||||
@@ -54,7 +55,7 @@ tag:
|
||||
git push origin v$(VERSION)
|
||||
|
||||
release: package
|
||||
tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --asset $(BINARY) --asset USERGUIDE.md --asset $(RELEASE_ZIP)
|
||||
tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --note-file $(RELEASE_NOTES) --asset $(BINARY) --asset USERGUIDE.md --asset $(RELEASE_ZIP)
|
||||
|
||||
pipeline: clean test build
|
||||
@echo "--- verifying version ---"
|
||||
|
||||
@@ -18,7 +18,9 @@ For a practical step-by-step manual with recommended backup workflows, recovery
|
||||
- Dry-run mode for planning large backups.
|
||||
- Filters for favorites, media type, date, estimated size, and excluded albums.
|
||||
- Failure tracking with `failures.jsonl` and `retry-failed`.
|
||||
- Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`.
|
||||
- Verification, reporting, and diff commands for backup integrity.
|
||||
- Status command for quick backup summaries.
|
||||
- Script-friendly exit codes and optional JSON summaries.
|
||||
- 100% test coverage for the Go CLI and parsing layers.
|
||||
|
||||
@@ -219,6 +221,26 @@ photoscli retry-failed --out ./backup
|
||||
|
||||
This is useful after transient iCloud or network failures.
|
||||
|
||||
Use `--clear-on-success` to remove successfully retried assets from `failures.jsonl`.
|
||||
|
||||
### `failures`
|
||||
|
||||
Lists or clears deduplicated failure records.
|
||||
|
||||
```bash
|
||||
photoscli failures list --out ./backup
|
||||
photoscli failures clear --out ./backup
|
||||
```
|
||||
|
||||
### `status`
|
||||
|
||||
Shows a quick backup summary.
|
||||
|
||||
```bash
|
||||
photoscli status --out ./backup --manifest sqlite
|
||||
photoscli status --out ./backup --manifest sqlite --json
|
||||
```
|
||||
|
||||
## Export Flags
|
||||
|
||||
Common flags for `export` and `backup-all`:
|
||||
@@ -297,7 +319,7 @@ Logged events include session start/end, completed exports, skipped assets, and
|
||||
|
||||
## Failure Tracking
|
||||
|
||||
Failed exports are appended to:
|
||||
Failed exports are deduplicated by asset ID and stored in:
|
||||
|
||||
```text
|
||||
failures.jsonl
|
||||
@@ -309,6 +331,12 @@ Retry them with:
|
||||
photoscli retry-failed --out ./backup
|
||||
```
|
||||
|
||||
Clear successful retries automatically:
|
||||
|
||||
```bash
|
||||
photoscli retry-failed --out ./backup --clear-on-success
|
||||
```
|
||||
|
||||
Use `--retry N` during normal exports to handle transient iCloud or network failures automatically.
|
||||
|
||||
## Configuration File
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# v0.6.0
|
||||
|
||||
This release focuses on backup integrity, recovery workflows, and clearer operational status.
|
||||
|
||||
## Highlights
|
||||
|
||||
- Manifest entries now include relative paths, so tree backups can be verified accurately.
|
||||
- SQLite manifests migrate automatically to include the new path column.
|
||||
- `verify` now detects missing files, zero-byte files, and size mismatches by default.
|
||||
- Exports use per-asset staging directories and rename completed files into place when possible.
|
||||
- Failure tracking is deduplicated by asset ID and records attempts and latest error details.
|
||||
- New `failures list` and `failures clear` commands.
|
||||
- `retry-failed --clear-on-success` removes successfully retried failures.
|
||||
- New `status` command with text and JSON output.
|
||||
- Release workflow now publishes these release notes to the release page.
|
||||
|
||||
## Recommended Upgrade Notes
|
||||
|
||||
- Existing JSONL manifests continue to load; entries without `path` fall back to `filename`.
|
||||
- Existing SQLite manifests are migrated with `ALTER TABLE downloads ADD COLUMN path ...` during open.
|
||||
- `verify` is stricter now and may report problems that older versions ignored.
|
||||
|
||||
## Assets
|
||||
|
||||
- `photoscli`: macOS binary.
|
||||
- `photoscli-0.6.0-macos.zip`: binary plus README, USERGUIDE, and CHANGELOG.
|
||||
- `USERGUIDE.md`: standalone user guide.
|
||||
+18
-1
@@ -20,6 +20,8 @@ It is especially useful when you want to:
|
||||
- Keep a manifest of already-exported assets.
|
||||
- Retry transient iCloud failures.
|
||||
- Verify that files referenced by a manifest exist on disk.
|
||||
- Detect missing, zero-byte, and size-mismatched manifest files.
|
||||
- Inspect or clear deduplicated failure records.
|
||||
- Script Photos exports with stable exit codes.
|
||||
|
||||
It is not intended to replace Apple Photos, iCloud Photos, or Time Machine. Think of it as an additional file-based export and backup tool.
|
||||
@@ -339,6 +341,8 @@ Failed exports are written to:
|
||||
failures.jsonl
|
||||
```
|
||||
|
||||
Failure records are deduplicated by asset ID. Repeated failures update the existing record and increment the attempt count.
|
||||
|
||||
For transient failures, first use retries during normal export:
|
||||
|
||||
```bash
|
||||
@@ -351,6 +355,19 @@ If failures remain, retry later:
|
||||
./bin/photoscli retry-failed --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Clear successful retry records automatically:
|
||||
|
||||
```bash
|
||||
./bin/photoscli retry-failed --out ./PhotosBackup --clear-on-success
|
||||
```
|
||||
|
||||
List or clear failure records:
|
||||
|
||||
```bash
|
||||
./bin/photoscli failures list --out ./PhotosBackup
|
||||
./bin/photoscli failures clear --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Typical reasons for failures:
|
||||
|
||||
- iCloud asset not currently downloadable.
|
||||
@@ -374,7 +391,7 @@ Run verification after large backups:
|
||||
./bin/photoscli verify --out ./PhotosBackup --manifest sqlite
|
||||
```
|
||||
|
||||
Verification checks that manifest entries point to files on disk.
|
||||
Verification checks that manifest entries point to files on disk. It also reports zero-byte files and size mismatches when the manifest has a recorded size.
|
||||
|
||||
It exits with:
|
||||
|
||||
|
||||
+302
-57
@@ -21,6 +21,10 @@ var (
|
||||
exportTimeout = 2 * time.Second
|
||||
configValues map[string]string
|
||||
configLoaded bool
|
||||
mkdirTempFunc = os.MkdirTemp
|
||||
renameFunc = os.Rename
|
||||
openFileFunc = os.OpenFile
|
||||
removeFunc = os.Remove
|
||||
)
|
||||
|
||||
type exportOptions struct {
|
||||
@@ -75,6 +79,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
return cmdVerify(args[1:], stdout, stderr)
|
||||
case "retry-failed":
|
||||
return cmdRetryFailed(args[1:], stdout, stderr, bridge)
|
||||
case "failures":
|
||||
return cmdFailures(args[1:], stdout, stderr)
|
||||
case "status":
|
||||
return cmdStatus(args[1:], stdout, stderr)
|
||||
case "version", "--version", "-v":
|
||||
fmt.Fprintln(stdout, version)
|
||||
return exitOK
|
||||
@@ -110,6 +118,10 @@ USAGE
|
||||
photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli verify --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli retry-failed --out <dir>
|
||||
photoscli retry-failed --out <dir> --clear-on-success
|
||||
photoscli failures list --out <dir>
|
||||
photoscli failures clear --out <dir>
|
||||
photoscli status --out <dir> [--json]
|
||||
photoscli version
|
||||
photoscli help
|
||||
|
||||
@@ -149,6 +161,12 @@ COMMANDS
|
||||
retry-failed --out <dir>
|
||||
Retry assets previously written to failures.jsonl.
|
||||
|
||||
failures list|clear --out <dir>
|
||||
List or clear deduplicated failure records.
|
||||
|
||||
status --out <dir> [--manifest jsonl|sqlite] [--json]
|
||||
Show manifest type, entry count, and failure count for a backup.
|
||||
|
||||
COMMON EXPORT FLAGS
|
||||
--out <dir>
|
||||
Destination directory. Required for export, backup-all, report, diff,
|
||||
@@ -240,7 +258,9 @@ LOGGING AND FAILURES
|
||||
table in downloads.db.
|
||||
|
||||
failures.jsonl
|
||||
Failed exports are appended here and can be retried with retry-failed.
|
||||
Failed exports are deduplicated by asset ID and can be retried with
|
||||
retry-failed. Use retry-failed --clear-on-success to remove successful
|
||||
retries from the failure list.
|
||||
|
||||
CONFIGURATION
|
||||
Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG.
|
||||
@@ -665,6 +685,7 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
|
||||
|
||||
type pendingAsset struct {
|
||||
asset photos.Asset
|
||||
root string
|
||||
path string
|
||||
album string
|
||||
}
|
||||
@@ -691,6 +712,22 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64,
|
||||
}
|
||||
}
|
||||
|
||||
func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
fullPath := filepath.Join(pa.path, result.Filename)
|
||||
relPath, err := filepath.Rel(root, fullPath)
|
||||
if err != nil || strings.HasPrefix(relPath, "..") {
|
||||
relPath = result.Filename
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud))
|
||||
}
|
||||
|
||||
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
|
||||
var items []pendingAsset
|
||||
var skipped int
|
||||
@@ -769,7 +806,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br
|
||||
*skipped++
|
||||
continue
|
||||
}
|
||||
*items = append(*items, pendingAsset{asset: a, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
*items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
}
|
||||
if onProgress != nil {
|
||||
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
|
||||
@@ -850,7 +887,7 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
bar.setAlbum(pa.album, 0, 0)
|
||||
bar.draw()
|
||||
start := time.Now()
|
||||
result, exportErr := exportOneWithRetry(bridge, pa.asset, pa.path, targetSize, quality, originals, i, opts.retry)
|
||||
result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry)
|
||||
dur := time.Since(start)
|
||||
isErr := exportErr != nil
|
||||
isSkipped := result.Skipped
|
||||
@@ -862,14 +899,10 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
failed++
|
||||
appendFailure(pa.path, pa, exportErr)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -922,7 +955,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
start := time.Now()
|
||||
var result photos.ExportResult
|
||||
var exportErr error
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i].asset, pending[i].path, targetSize, quality, originals, i, workerID, opts.retry)
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry)
|
||||
dur := time.Since(start)
|
||||
bar.setWorker(workerID, "", 0, "", "")
|
||||
completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
|
||||
@@ -995,14 +1028,10 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
failed++
|
||||
appendFailure(entry.pa.path, entry.pa, entry.err)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -1036,7 +1065,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) {
|
||||
pending := make([]pendingAsset, len(assets))
|
||||
for i, a := range assets {
|
||||
pending[i] = pendingAsset{asset: a, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
}
|
||||
var m manifest.Manifest
|
||||
if !noManifest {
|
||||
@@ -1080,11 +1109,93 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize,
|
||||
return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index)
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index)
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
if originals {
|
||||
return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex)
|
||||
}
|
||||
return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
var result photos.ExportResult
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
result, err = exportOne(bridge, a, outDir, targetSize, quality, originals, index)
|
||||
result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1093,15 +1204,11 @@ func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, tar
|
||||
return result, err
|
||||
}
|
||||
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(a.ID, outDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(a.ID, outDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1356,22 +1463,73 @@ func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest
|
||||
|
||||
func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") }
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
_ = os.MkdirAll(dir, 0755)
|
||||
f, openErr := os.OpenFile(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if openErr != nil {
|
||||
return
|
||||
type failureEntry struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
FailedAt int64 `json:"failed_at"`
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
func loadFailures(dir string) map[string]failureEntry {
|
||||
out := map[string]failureEntry{}
|
||||
data, err := os.ReadFile(failuresPath(dir))
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f failureEntry
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
out[f.ID] = f
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func saveFailures(dir string, failures map[string]failureEntry) error {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
data, _ := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
}{pa.asset.ID, pa.asset.Filename, pa.album, pa.path, err.Error()})
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
data, _ := json.Marshal(failures[id])
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = dir
|
||||
}
|
||||
failures := loadFailures(root)
|
||||
f := failures[pa.asset.ID]
|
||||
f.ID = pa.asset.ID
|
||||
f.Filename = pa.asset.Filename
|
||||
f.Album = pa.album
|
||||
f.Path = pa.path
|
||||
f.Error = err.Error()
|
||||
f.FailedAt = time.Now().Unix()
|
||||
f.Attempts++
|
||||
failures[pa.asset.ID] = f
|
||||
_ = saveFailures(root, failures)
|
||||
}
|
||||
|
||||
func writeJSONSummary(stdout io.Writer, s commandSummary) {
|
||||
@@ -1466,17 +1624,32 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
missing := 0
|
||||
bad := 0
|
||||
for id, e := range entries {
|
||||
if e.Filename == "" {
|
||||
checkPath := e.Path
|
||||
if checkPath == "" {
|
||||
checkPath = e.Filename
|
||||
}
|
||||
if checkPath == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outDir, e.Filename)); err != nil {
|
||||
missing++
|
||||
fmt.Fprintf(stdout, "%s\t%s\n", id, e.Filename)
|
||||
info, err := os.Stat(filepath.Join(outDir, checkPath))
|
||||
if err != nil {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if e.Size > 0 && info.Size() != e.Size {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
|
||||
}
|
||||
}
|
||||
if missing > 0 {
|
||||
if bad > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
fmt.Fprintf(stdout, "verified\t%d\n", len(entries))
|
||||
@@ -1485,31 +1658,103 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
clearOnSuccess := hasFlag(args, "--clear-on-success")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
data, err := os.ReadFile(failuresPath(outDir))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
failures := loadFailures(outDir)
|
||||
if len(failures) == 0 {
|
||||
fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir))
|
||||
return exitErr
|
||||
}
|
||||
var pending []pendingAsset
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f struct{ ID, Filename, Album, Path string }
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, path: f.Path, album: f.Album})
|
||||
}
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album})
|
||||
}
|
||||
bar := newProgressBar(stderr, 1)
|
||||
done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{})
|
||||
if clearOnSuccess && done > 0 {
|
||||
for i := 0; i < done && i < len(pending); i++ {
|
||||
delete(failures, pending[i].asset.ID)
|
||||
}
|
||||
_ = saveFailures(outDir, failures)
|
||||
}
|
||||
writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)})
|
||||
if failed > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func cmdFailures(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) < 1 {
|
||||
fmt.Fprintln(stderr, "error: failures requires list or clear")
|
||||
return exitErr
|
||||
}
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
failures := loadFailures(outDir)
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts)
|
||||
}
|
||||
return exitOK
|
||||
case "clear":
|
||||
if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
return exitOK
|
||||
default:
|
||||
fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0])
|
||||
return exitErr
|
||||
}
|
||||
}
|
||||
|
||||
func cmdStatus(args []string, stdout, stderr io.Writer) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
|
||||
mf, err := manifest.ParseFormat(manifestFmt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
entries, err := loadManifestEntries(outDir, mf)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
failures := loadFailures(outDir)
|
||||
if hasFlag(args, "--json") {
|
||||
data, _ := json.Marshal(struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Entries int `json:"entries"`
|
||||
Failures int `json:"failures"`
|
||||
}{manifestFmt, len(entries), len(failures)})
|
||||
fmt.Fprintln(stdout, string(data))
|
||||
return exitOK
|
||||
}
|
||||
fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures))
|
||||
return exitOK
|
||||
}
|
||||
|
||||
+270
-1
@@ -3815,7 +3815,7 @@ func TestReportDiffVerifyRetryFailedAndConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("x"), 0644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("0123456789"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
@@ -4043,3 +4043,272 @@ func TestNewFeatureRemainingBranches(t *testing.T) {
|
||||
t.Fatalf("expected retry partial, got %d", rc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailuresAndStatusCommands(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry("x1", "file.jpg", "Album/file.jpg", 4, "local"))
|
||||
m.Close()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "Album"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "Album", "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom"))
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom2"))
|
||||
|
||||
out, stderr, rc := runWith([]string{"status", "--out", dir, "--json"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "failures") || stderr != "" {
|
||||
t.Fatalf("status json rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"status", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "entries\t1") || !strings.Contains(out, "failures\t1") || stderr != "" {
|
||||
t.Fatalf("status rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"failures", "list", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "bad") || !strings.Contains(out, "2") || stderr != "" {
|
||||
t.Fatalf("failures list rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "requires") {
|
||||
t.Fatalf("failures missing subcommand rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "bogus", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "unknown") {
|
||||
t.Fatalf("failures bogus rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || stderr != "" {
|
||||
t.Fatalf("failures clear rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected failures cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicExportHelpers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "photo.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "photo.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneAtomic(b, pa, 1024, 85, false, 0)
|
||||
if err != nil || result.Filename != "photo.jpg" {
|
||||
t.Fatalf("atomic export result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "Album", "photo.jpg")); err != nil {
|
||||
t.Fatalf("expected final file: %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 1); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected zero-byte error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoreIntegrityBranches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.Entry{ID: "zero", Filename: "zero.jpg", Path: "zero.jpg", Size: 1, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "mismatch", Filename: "mismatch.jpg", Path: "mismatch.jpg", Size: 10, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "zero.jpg"), nil, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "mismatch.jpg"), []byte("bad"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _, rc := runWith([]string{"verify", "--out", dir}, &mockBridge{})
|
||||
if rc != exitPartial || !strings.Contains(out, "zero-byte") || !strings.Contains(out, "size-mismatch") {
|
||||
t.Fatalf("verify rc=%d out=%q", rc, out)
|
||||
}
|
||||
_, stderr, rc := runWith([]string{"status"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("status missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", dir, "--manifest", "bad"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "manifest") {
|
||||
t.Fatalf("status bad manifest rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", "/proc/cannot-create"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "error:") {
|
||||
t.Fatalf("status load error rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "list"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("failures missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryFailedClearOnSuccess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
out, _, rc := runWith([]string{"retry-failed", "--out", dir, "--clear-on-success"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "exported") {
|
||||
t.Fatalf("retry clear rc=%d out=%q", rc, out)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected successful retry to clear failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicSlotExportHelper(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "slot.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "slot.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "slot", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0)
|
||||
if err != nil || result.Filename != "slot.jpg" {
|
||||
t.Fatalf("slot atomic result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedErrorBranchesForCoverage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x", Filename: "x.jpg"}, root: dir, path: filepath.Join(dir, "Album")}
|
||||
|
||||
oldMkdirTemp := mkdirTempFunc
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("mkdirtemp") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "mkdirtemp") {
|
||||
t.Fatalf("expected mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
oldRename := renameFunc
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("rename") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "rename") {
|
||||
t.Fatalf("expected rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
oldOpen := openFileFunc
|
||||
openFileFunc = func(string, int, os.FileMode) (*os.File, error) { return nil, fmt.Errorf("open") }
|
||||
if err := saveFailures(dir, map[string]failureEntry{"x": {ID: "x"}}); err == nil || !strings.Contains(err.Error(), "open") {
|
||||
t.Fatalf("expected open error, got %v", err)
|
||||
}
|
||||
openFileFunc = oldOpen
|
||||
|
||||
oldRemove := removeFunc
|
||||
removeFunc = func(string) error { return fmt.Errorf("remove") }
|
||||
_, stderr, rc := runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "remove") {
|
||||
t.Fatalf("expected remove error, rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
removeFunc = oldRemove
|
||||
|
||||
mf := &mockManifest{}
|
||||
addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1})
|
||||
if mf.last.Path != "file.jpg" {
|
||||
t.Fatalf("expected fallback rel path, got %+v", mf.last)
|
||||
}
|
||||
|
||||
badPathRoot := t.TempDir()
|
||||
badPath := filepath.Join(badPathRoot, "notdir")
|
||||
if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "x2", Filename: "x2.jpg"}, root: badPathRoot, path: badPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil {
|
||||
t.Fatal("expected final mkdir error")
|
||||
}
|
||||
|
||||
slotRootFile := filepath.Join(t.TempDir(), "rootfile")
|
||||
if err := os.WriteFile(slotRootFile, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotfallback", Filename: "slot.jpg"}, root: slotRootFile, path: t.TempDir()}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, true, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected original fallback error: %v", err)
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected preview fallback error: %v", err)
|
||||
}
|
||||
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotzero", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Slot")}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected slot zero-byte error, got %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("slot rename") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot rename") {
|
||||
t.Fatalf("expected slot rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("slot mkdirtemp") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot mkdirtemp") {
|
||||
t.Fatalf("expected slot mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
badSlotRoot := t.TempDir()
|
||||
badSlotPath := filepath.Join(badSlotRoot, "notdir")
|
||||
if err := os.WriteFile(badSlotPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotbadpath", Filename: "slot.jpg"}, root: badSlotRoot, path: badSlotPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil {
|
||||
t.Fatal("expected slot final mkdir error")
|
||||
}
|
||||
}
|
||||
|
||||
type mockManifest struct{ last manifest.Entry }
|
||||
|
||||
func (m *mockManifest) Has(string) bool { return false }
|
||||
func (m *mockManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.last = manifest.NewEntry(id, filename, filename, size, cloud)
|
||||
}
|
||||
func (m *mockManifest) AddEntry(e manifest.Entry) { m.last = e }
|
||||
func (m *mockManifest) Save() error { return nil }
|
||||
func (m *mockManifest) Close() {}
|
||||
func (m *mockManifest) OpenAppend() error { return nil }
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package manifest
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewEntryPath(t *testing.T) {
|
||||
e := newEntry("id1", "file.jpg", 123, "local")
|
||||
if e.ID != "id1" || e.Filename != "file.jpg" || e.Path != "file.jpg" || e.Size != 123 || e.Cloud != "local" || e.Exported == 0 {
|
||||
t.Fatalf("unexpected entry: %+v", e)
|
||||
}
|
||||
e = NewEntry("id2", "file2.jpg", "Album/file2.jpg", 456, "cloud")
|
||||
if e.Path != "Album/file2.jpg" || e.Filename != "file2.jpg" || e.Size != 456 || e.Cloud != "cloud" {
|
||||
t.Fatalf("unexpected entry with path: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddEntryDefaultsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
jm := LoadJSONL(dir)
|
||||
if err := jm.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
|
||||
jm.Close()
|
||||
if got := LoadJSONL(dir).Entries()["x1"].Path; got != "file.jpg" {
|
||||
t.Fatalf("jsonl path = %q", got)
|
||||
}
|
||||
|
||||
sdir := t.TempDir()
|
||||
sm, err := LoadSQLite(sdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sm.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
|
||||
if _, err := sm.db.Exec(`UPDATE downloads SET path = '' WHERE id = 'x1'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := sm.Entries()["x1"].Path; got != "file.jpg" {
|
||||
t.Fatalf("sqlite path = %q", got)
|
||||
}
|
||||
sm.Close()
|
||||
}
|
||||
@@ -40,6 +40,7 @@ func LoadJSONL(dir string) *jsonlManifest {
|
||||
type entryWithID struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Size int64 `json:"size"`
|
||||
Cloud string `json:"cloud"`
|
||||
Exported int64 `json:"exported"`
|
||||
@@ -51,8 +52,13 @@ func LoadJSONL(dir string) *jsonlManifest {
|
||||
}
|
||||
var raw entryWithID
|
||||
if json.Unmarshal([]byte(line), &raw) == nil && raw.ID != "" {
|
||||
if raw.Path == "" {
|
||||
raw.Path = raw.Filename
|
||||
}
|
||||
m.entries[raw.ID] = Entry{
|
||||
ID: raw.ID,
|
||||
Filename: raw.Filename,
|
||||
Path: raw.Path,
|
||||
Size: raw.Size,
|
||||
Cloud: raw.Cloud,
|
||||
Exported: raw.Exported,
|
||||
@@ -70,18 +76,25 @@ func (m *jsonlManifest) Has(id string) bool {
|
||||
}
|
||||
|
||||
func (m *jsonlManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.AddEntry(newEntry(id, filename, size, cloud))
|
||||
}
|
||||
|
||||
func (m *jsonlManifest) AddEntry(entry Entry) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
entry := newEntry(id, filename, size, cloud)
|
||||
m.entries[id] = entry
|
||||
if entry.Path == "" {
|
||||
entry.Path = entry.Filename
|
||||
}
|
||||
m.entries[entry.ID] = entry
|
||||
if m.file != nil {
|
||||
data, _ := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Size int64 `json:"size"`
|
||||
Cloud string `json:"cloud"`
|
||||
Exported int64 `json:"exported"`
|
||||
}{ID: id, Filename: entry.Filename, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported})
|
||||
}{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported})
|
||||
m.file.Write(data)
|
||||
m.file.Write([]byte("\n"))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import "time"
|
||||
type Entry struct {
|
||||
ID string
|
||||
Filename string
|
||||
Path string
|
||||
Size int64
|
||||
Cloud string
|
||||
Exported int64
|
||||
@@ -13,6 +14,7 @@ type Entry struct {
|
||||
type Manifest interface {
|
||||
Has(id string) bool
|
||||
Add(id string, filename string, size int64, cloud string)
|
||||
AddEntry(entry Entry)
|
||||
Save() error
|
||||
Close()
|
||||
OpenAppend() error
|
||||
@@ -26,8 +28,17 @@ func newEntry(id, filename string, size int64, cloud string) Entry {
|
||||
return Entry{
|
||||
ID: id,
|
||||
Filename: filename,
|
||||
Path: filename,
|
||||
Size: size,
|
||||
Cloud: cloud,
|
||||
Exported: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func NewEntry(id, filename, path string, size int64, cloud string) Entry {
|
||||
e := newEntry(id, filename, size, cloud)
|
||||
if path != "" {
|
||||
e.Path = path
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -49,7 +49,8 @@ func ConvertFromJSONL(dir string) (Manifest, error) {
|
||||
}
|
||||
|
||||
for id, e := range src.Entries() {
|
||||
dst.Add(id, e.Filename, e.Size, e.Cloud)
|
||||
e.ID = id
|
||||
dst.AddEntry(e)
|
||||
}
|
||||
|
||||
os.Remove(JSONLPath(dir))
|
||||
@@ -69,7 +70,8 @@ func ConvertFromSQLite(dir string) (Manifest, error) {
|
||||
}
|
||||
|
||||
for id, e := range src.Entries() {
|
||||
dst.Add(id, e.Filename, e.Size, e.Cloud)
|
||||
e.ID = id
|
||||
dst.AddEntry(e)
|
||||
}
|
||||
if err := dst.Save(); err != nil {
|
||||
return nil, fmt.Errorf("save jsonl: %w", err)
|
||||
|
||||
@@ -61,6 +61,7 @@ func (m *sqliteManifest) OpenAppend() error {
|
||||
_, err = execFn(`CREATE TABLE IF NOT EXISTS downloads (
|
||||
id TEXT PRIMARY KEY,
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
path TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
cloud TEXT NOT NULL DEFAULT '',
|
||||
exported INTEGER NOT NULL DEFAULT 0
|
||||
@@ -69,6 +70,7 @@ func (m *sqliteManifest) OpenAppend() error {
|
||||
db.Close()
|
||||
return fmt.Errorf("create table: %w", err)
|
||||
}
|
||||
_, _ = execFn(`ALTER TABLE downloads ADD COLUMN path TEXT NOT NULL DEFAULT ''`)
|
||||
_, err = execFn(`CREATE INDEX IF NOT EXISTS idx_downloads_id ON downloads(id)`)
|
||||
if err != nil {
|
||||
db.Close()
|
||||
@@ -91,12 +93,18 @@ func (m *sqliteManifest) Has(id string) bool {
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.AddEntry(newEntry(id, filename, size, cloud))
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) AddEntry(entry Entry) {
|
||||
if m.db == nil {
|
||||
return
|
||||
}
|
||||
entry := newEntry(id, filename, size, cloud)
|
||||
m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, size, cloud, exported) VALUES (?, ?, ?, ?, ?)`,
|
||||
id, entry.Filename, entry.Size, entry.Cloud, entry.Exported)
|
||||
if entry.Path == "" {
|
||||
entry.Path = entry.Filename
|
||||
}
|
||||
m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, exported) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Exported)
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) Save() error {
|
||||
@@ -119,14 +127,17 @@ func (m *sqliteManifest) Entries() map[string]Entry {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]Entry)
|
||||
rows, err := m.db.Query(`SELECT id, filename, size, cloud, exported FROM downloads`)
|
||||
rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, exported FROM downloads`)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var e Entry
|
||||
if err := rows.Scan(&e.ID, &e.Filename, &e.Size, &e.Cloud, &e.Exported); err == nil {
|
||||
if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Exported); err == nil {
|
||||
if e.Path == "" {
|
||||
e.Path = e.Filename
|
||||
}
|
||||
out[e.ID] = e
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,10 +249,10 @@ func TestSQLiteCreateIndexError(t *testing.T) {
|
||||
return nil, err
|
||||
}
|
||||
m.execFunc = func(query string, args ...any) (sql.Result, error) {
|
||||
callCount++
|
||||
if callCount == 2 {
|
||||
if strings.Contains(query, "CREATE INDEX") {
|
||||
return nil, fmt.Errorf("injected CREATE INDEX error")
|
||||
}
|
||||
callCount++
|
||||
return db.Exec(query, args...)
|
||||
}
|
||||
return db, nil
|
||||
|
||||
Reference in New Issue
Block a user