Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 Ein Anderssono
|
05188e5451 | ||
|
Ein Anderssono
|
0a905758cc |
+98
-23
@@ -1,48 +1,123 @@
|
||||
# Changelog
|
||||
|
||||
This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
|
||||
|
||||
## v0.6.0
|
||||
|
||||
Backup integrity and recovery release.
|
||||
|
||||
- Add manifest-relative paths for exported files so tree backups can be verified accurately.
|
||||
- Add SQLite manifest migration for the new `path` column.
|
||||
- Upgrade `verify` to report missing files, zero-byte files, and size mismatches by default.
|
||||
- Add atomic staging export writes with per-asset temporary directories before final rename.
|
||||
- Deduplicate `failures.jsonl` entries by asset ID and track attempts plus latest failure time.
|
||||
- Add `failures list` and `failures clear` commands.
|
||||
- Add `retry-failed --clear-on-success`.
|
||||
- Add `status` command with text and JSON output.
|
||||
- Update release workflow to publish release notes from `RELEASE_NOTES.md`.
|
||||
- Keep `CHANGELOG.md` as the canonical release history.
|
||||
|
||||
## v0.5.0
|
||||
|
||||
- Add JSONL and SQLite manifests with resumable skip tracking
|
||||
- Add structured export logging to export.log or SQLite logs table
|
||||
- Add --dry-run, --retry, --only-favorites, --media, --json, --verify, --date-template, and size filters
|
||||
- Add report, diff, verify, and retry-failed commands
|
||||
- Add failure tracking in failures.jsonl
|
||||
- Add configurable preview quality and export concurrency
|
||||
- Add album exclusion, since-date filtering, album collision handling, and config-file defaults
|
||||
- Expand README and CLI help output
|
||||
- Maintain 100% test coverage in the release pipeline
|
||||
Manifest, filtering, logging, and documentation release.
|
||||
|
||||
- Add JSONL and SQLite manifests with resumable skip tracking.
|
||||
- Add automatic conversion between JSONL and SQLite manifests.
|
||||
- Add structured export logging to `export.log` or SQLite `logs` table.
|
||||
- Add explicit exit codes: success, error, partial failure, and access denied.
|
||||
- Add `--quality` for JPEG preview compression.
|
||||
- Add `--concurrency` with progress slot cap.
|
||||
- Add `--exclude-album` with exact/glob matching.
|
||||
- Add `--since` date filtering.
|
||||
- Add `--dry-run`.
|
||||
- Add `--retry`.
|
||||
- Add `--only-favorites`.
|
||||
- Add `--media photos|videos|all`.
|
||||
- Add `--json` command summaries.
|
||||
- Add `--verify` integration.
|
||||
- Add `--format jpeg|heic|png` validation/hint.
|
||||
- Add `--min-size` and `--max-size` estimated pixel-count filters.
|
||||
- Add `--date-template` folder layout support.
|
||||
- Add `report`, `diff`, `verify`, and `retry-failed` commands.
|
||||
- Add failure tracking in `failures.jsonl`.
|
||||
- Add config-file defaults via `~/.photoscli.toml` or `PHOTOSCLI_CONFIG`.
|
||||
- Add album name collision handling for duplicate sibling album names.
|
||||
- Expand `README.md` and verbose CLI help.
|
||||
- Add 100% test coverage for the CLI and manifest/photos packages.
|
||||
|
||||
## v0.4.1
|
||||
|
||||
Documentation and release asset follow-up after v0.5.0.
|
||||
|
||||
- Add `USERGUIDE.md` as a practical end-user manual.
|
||||
- Link the user guide from `README.md`.
|
||||
- Add release zip packaging with binary, README, USERGUIDE, and CHANGELOG.
|
||||
- Attach `USERGUIDE.md` and the macOS zip package to the `v0.5.0` release.
|
||||
|
||||
## v0.4.0
|
||||
|
||||
- Scroll log for completed exports (copied, downloaded, skipped, failed)
|
||||
- Worker status lines with live download progress bars for cloud files
|
||||
- Color gradient progress bar (red to yellow to green)
|
||||
- Disk skip: files already on disk are skipped during backup-all
|
||||
- Parallel export with 3 workers for backup-all
|
||||
Progress and backup-all usability release.
|
||||
|
||||
- Add scroll log for completed exports: copied, downloaded, skipped, and failed.
|
||||
- Add worker status lines with live download progress bars for cloud files.
|
||||
- Add color-gradient progress bars from red to yellow to green.
|
||||
- Add disk skip for files already present during `backup-all`.
|
||||
- Add parallel export with multiple workers for `backup-all`.
|
||||
- Increase progress slot support in the bridge.
|
||||
- Improve progress rendering for large export sessions.
|
||||
|
||||
## v0.3.x
|
||||
|
||||
Published on Gitea as `v0.3.0` through `v0.3.5`. These releases are not represented by local git tags in the current clone, so the exact per-release commit mapping is unavailable here.
|
||||
|
||||
Known purpose of this series:
|
||||
|
||||
- Iterative release packaging and Gitea release workflow improvements.
|
||||
- Continued progress display and backup/export polish between `v0.2.5` and `v0.4.0`.
|
||||
- Preparation for the larger `v0.4.0` progress/backup release.
|
||||
|
||||
## v0.2.6
|
||||
|
||||
Published on Gitea but not represented by a local git tag in the current clone.
|
||||
|
||||
- Post-`v0.2.5` maintenance release in the progress/export series.
|
||||
|
||||
## v0.2.5
|
||||
|
||||
- Unicode progress bar with cloud download speed display
|
||||
- Add Unicode progress bar rendering.
|
||||
- Add cloud download speed display.
|
||||
|
||||
## v0.2.4
|
||||
|
||||
- Stop export loop on Ctrl+C instead of flooding failures
|
||||
- Stop export loop on Ctrl+C instead of flooding failures.
|
||||
- Improve graceful cancellation behavior after interrupt.
|
||||
|
||||
## v0.2.3
|
||||
|
||||
- Fix export write failures and Ctrl+C cancellation
|
||||
- Fix export write failures.
|
||||
- Fix Ctrl+C cancellation behavior.
|
||||
|
||||
## v0.2.2
|
||||
|
||||
No local tag is present for this version in the current clone.
|
||||
|
||||
## v0.2.1
|
||||
|
||||
- Add status messages during export
|
||||
- Fix parallel export progress display
|
||||
- Add status messages during export.
|
||||
- Fix parallel export progress display.
|
||||
|
||||
## v0.2.0
|
||||
|
||||
- Semaphore timeouts, error logging, dead code removal
|
||||
- Parallel exports
|
||||
- Add semaphore timeouts.
|
||||
- Add export error logging.
|
||||
- Remove dead code.
|
||||
- Add parallel exports.
|
||||
|
||||
## Pre-release
|
||||
|
||||
- Initial applephotos CLI with progress, cloud status, per-asset export
|
||||
- Renamed from applephotos to photoscli
|
||||
- Initial `applephotos` CLI with progress, cloud status, and per-asset export.
|
||||
- Add `.gitignore` and remove build artifacts from tracking.
|
||||
- Rename `applephotos` to `photoscli`.
|
||||
- Update module path to `gitea.k3s.k0.nu/tools/photocli`.
|
||||
- Add version flag.
|
||||
- Add Gitea release targets and release pipeline.
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
BINARY := ./bin/photoscli
|
||||
MODULE := gitea.k3s.k0.nu/tools/photocli
|
||||
VERSION := 0.5.0
|
||||
VERSION := 0.6.0
|
||||
RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos.zip
|
||||
RELEASE_NOTES := RELEASE_NOTES.md
|
||||
BRIDGE_DIR := bridge
|
||||
LDFLAGS := -X main.version=$(VERSION)
|
||||
OBJ := $(BRIDGE_DIR)/photokit_bridge.o
|
||||
@@ -10,7 +12,7 @@ STUB_LIB := $(BRIDGE_DIR)/libphotokit_bridge_stub.a
|
||||
GITEA_HOST := gitea-1.tail82444.ts.net
|
||||
GITEA_REPO := tools/photocli
|
||||
|
||||
.PHONY: all build clean test coverage tag release pipeline
|
||||
.PHONY: all build clean test coverage tag package release pipeline
|
||||
|
||||
all: build
|
||||
|
||||
@@ -43,14 +45,17 @@ coverage: $(STUB_LIB)
|
||||
go tool cover -func=coverage.out
|
||||
|
||||
clean:
|
||||
rm -f $(BINARY) $(OBJ) $(LIB) $(STUB_OBJ) $(STUB_LIB) coverage.out
|
||||
rm -f $(BINARY) $(RELEASE_ZIP) $(OBJ) $(LIB) $(STUB_OBJ) $(STUB_LIB) coverage.out
|
||||
|
||||
package: build
|
||||
zip -j $(RELEASE_ZIP) $(BINARY) README.md USERGUIDE.md CHANGELOG.md
|
||||
|
||||
tag:
|
||||
git tag v$(VERSION)
|
||||
git push origin v$(VERSION)
|
||||
|
||||
release:
|
||||
tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --asset $(BINARY)
|
||||
release: package
|
||||
tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --note-file $(RELEASE_NOTES) --asset $(BINARY) --asset USERGUIDE.md --asset $(RELEASE_ZIP)
|
||||
|
||||
pipeline: clean test build
|
||||
@echo "--- verifying version ---"
|
||||
|
||||
@@ -4,6 +4,8 @@
|
||||
|
||||
The tool is designed for repeatable, resumable Photos backups rather than one-off drag-and-drop exports.
|
||||
|
||||
For a practical step-by-step manual with recommended backup workflows, recovery steps, automation examples, and troubleshooting, see `USERGUIDE.md`.
|
||||
|
||||
## Highlights
|
||||
|
||||
- List albums, photos, and the Photos folder/album tree.
|
||||
@@ -16,7 +18,9 @@ The tool is designed for repeatable, resumable Photos backups rather than one-of
|
||||
- Dry-run mode for planning large backups.
|
||||
- Filters for favorites, media type, date, estimated size, and excluded albums.
|
||||
- Failure tracking with `failures.jsonl` and `retry-failed`.
|
||||
- Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`.
|
||||
- Verification, reporting, and diff commands for backup integrity.
|
||||
- Status command for quick backup summaries.
|
||||
- Script-friendly exit codes and optional JSON summaries.
|
||||
- 100% test coverage for the Go CLI and parsing layers.
|
||||
|
||||
@@ -217,6 +221,26 @@ photoscli retry-failed --out ./backup
|
||||
|
||||
This is useful after transient iCloud or network failures.
|
||||
|
||||
Use `--clear-on-success` to remove successfully retried assets from `failures.jsonl`.
|
||||
|
||||
### `failures`
|
||||
|
||||
Lists or clears deduplicated failure records.
|
||||
|
||||
```bash
|
||||
photoscli failures list --out ./backup
|
||||
photoscli failures clear --out ./backup
|
||||
```
|
||||
|
||||
### `status`
|
||||
|
||||
Shows a quick backup summary.
|
||||
|
||||
```bash
|
||||
photoscli status --out ./backup --manifest sqlite
|
||||
photoscli status --out ./backup --manifest sqlite --json
|
||||
```
|
||||
|
||||
## Export Flags
|
||||
|
||||
Common flags for `export` and `backup-all`:
|
||||
@@ -295,7 +319,7 @@ Logged events include session start/end, completed exports, skipped assets, and
|
||||
|
||||
## Failure Tracking
|
||||
|
||||
Failed exports are appended to:
|
||||
Failed exports are deduplicated by asset ID and stored in:
|
||||
|
||||
```text
|
||||
failures.jsonl
|
||||
@@ -307,6 +331,12 @@ Retry them with:
|
||||
photoscli retry-failed --out ./backup
|
||||
```
|
||||
|
||||
Clear successful retries automatically:
|
||||
|
||||
```bash
|
||||
photoscli retry-failed --out ./backup --clear-on-success
|
||||
```
|
||||
|
||||
Use `--retry N` during normal exports to handle transient iCloud or network failures automatically.
|
||||
|
||||
## Configuration File
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
# v0.6.0
|
||||
|
||||
This release focuses on backup integrity, recovery workflows, and clearer operational status.
|
||||
|
||||
## Highlights
|
||||
|
||||
- Manifest entries now include relative paths, so tree backups can be verified accurately.
|
||||
- SQLite manifests migrate automatically to include the new path column.
|
||||
- `verify` now detects missing files, zero-byte files, and size mismatches by default.
|
||||
- Exports use per-asset staging directories and rename completed files into place when possible.
|
||||
- Failure tracking is deduplicated by asset ID and records attempts and latest error details.
|
||||
- New `failures list` and `failures clear` commands.
|
||||
- `retry-failed --clear-on-success` removes successfully retried failures.
|
||||
- New `status` command with text and JSON output.
|
||||
- Release workflow now publishes these release notes to the release page.
|
||||
|
||||
## Recommended Upgrade Notes
|
||||
|
||||
- Existing JSONL manifests continue to load; entries without `path` fall back to `filename`.
|
||||
- Existing SQLite manifests are migrated with `ALTER TABLE downloads ADD COLUMN path ...` during open.
|
||||
- `verify` is stricter now and may report problems that older versions ignored.
|
||||
|
||||
## Assets
|
||||
|
||||
- `photoscli`: macOS binary.
|
||||
- `photoscli-0.6.0-macos.zip`: binary plus README, USERGUIDE, and CHANGELOG.
|
||||
- `USERGUIDE.md`: standalone user guide.
|
||||
+755
@@ -0,0 +1,755 @@
|
||||
# photoscli User Guide
|
||||
|
||||
This guide explains how to use `photoscli` safely for Apple Photos exports and backups. It is written for day-to-day users who want reliable commands, clear recovery steps, and predictable backup behavior.
|
||||
|
||||
For a compact command reference, see `README.md` or run:
|
||||
|
||||
```bash
|
||||
photoscli help
|
||||
```
|
||||
|
||||
## What photoscli Is For
|
||||
|
||||
`photoscli` exports assets from Apple Photos on macOS.
|
||||
|
||||
It is especially useful when you want to:
|
||||
|
||||
- Back up the whole Photos album tree to normal folders.
|
||||
- Export one album for sharing or archiving.
|
||||
- Resume a large export without starting over.
|
||||
- Keep a manifest of already-exported assets.
|
||||
- Retry transient iCloud failures.
|
||||
- Verify that files referenced by a manifest exist on disk.
|
||||
- Detect missing, zero-byte, and size-mismatched manifest files.
|
||||
- Inspect or clear deduplicated failure records.
|
||||
- Script Photos exports with stable exit codes.
|
||||
|
||||
It is not intended to replace Apple Photos, iCloud Photos, or Time Machine. Think of it as an additional file-based export and backup tool.
|
||||
|
||||
## Before You Start
|
||||
|
||||
You need:
|
||||
|
||||
- macOS.
|
||||
- Apple Photos library available on the machine.
|
||||
- Photos permission granted to the terminal app or binary.
|
||||
- Enough disk space for the export.
|
||||
- A built `photoscli` binary.
|
||||
|
||||
Build it from the repo:
|
||||
|
||||
```bash
|
||||
make build
|
||||
```
|
||||
|
||||
Run the binary directly:
|
||||
|
||||
```bash
|
||||
./bin/photoscli version
|
||||
```
|
||||
|
||||
If you install or copy it somewhere else, replace `./bin/photoscli` in examples with `photoscli`.
|
||||
|
||||
## First Run And Permissions
|
||||
|
||||
Start with a harmless command:
|
||||
|
||||
```bash
|
||||
./bin/photoscli albums
|
||||
```
|
||||
|
||||
macOS may ask for Photos access. Allow it.
|
||||
|
||||
If access is denied, open:
|
||||
|
||||
```text
|
||||
System Settings > Privacy & Security > Photos
|
||||
```
|
||||
|
||||
Then enable access for the terminal application you use, such as Terminal, iTerm, or another launcher. Depending on how you start the binary, macOS may associate the permission with the terminal app rather than the binary itself.
|
||||
|
||||
Verify access:
|
||||
|
||||
```bash
|
||||
./bin/photoscli tree
|
||||
```
|
||||
|
||||
## Recommended Backup Strategy
|
||||
|
||||
For most users, the safest full-library backup command is:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out /Volumes/BackupDrive/PhotosExport \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3 \
|
||||
--concurrency 4
|
||||
```
|
||||
|
||||
This gives you:
|
||||
|
||||
- Full album-tree export.
|
||||
- SQLite manifest for fast resumable backups.
|
||||
- Structured logs in the same database.
|
||||
- Automatic retries for transient failures.
|
||||
- Moderate concurrency that is usually safe for large libraries.
|
||||
|
||||
Before running it for real, run a dry-run:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out /Volumes/BackupDrive/PhotosExport \
|
||||
--manifest sqlite \
|
||||
--dry-run \
|
||||
--json
|
||||
```
|
||||
|
||||
## Preview Export vs Original Export
|
||||
|
||||
By default, `photoscli` exports optimized preview images.
|
||||
|
||||
Preview export:
|
||||
|
||||
- Produces smaller files.
|
||||
- Uses `--size` and `--quality`.
|
||||
- Is good for browsing, sharing, and lightweight archives.
|
||||
- May not preserve original file bytes or all original metadata.
|
||||
|
||||
Original export:
|
||||
|
||||
- Uses `--originals`.
|
||||
- Exports original asset resources where PhotoKit exposes them.
|
||||
- Ignores `--size` and `--quality`.
|
||||
- Uses more disk space.
|
||||
- Is usually the better choice for archival backups.
|
||||
|
||||
Preview example:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Vacation" --out ./VacationPreview --size 2048 --quality 90
|
||||
```
|
||||
|
||||
Original example:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Vacation" --out ./VacationOriginals --originals
|
||||
```
|
||||
|
||||
## Export One Album
|
||||
|
||||
List albums first:
|
||||
|
||||
```bash
|
||||
./bin/photoscli albums
|
||||
```
|
||||
|
||||
Export by title:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Vacation" --out ./Vacation
|
||||
```
|
||||
|
||||
Export by PhotoKit local identifier:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "5E9F.../L0/001" --out ./Vacation
|
||||
```
|
||||
|
||||
Use the local identifier when album names are duplicated or ambiguous.
|
||||
|
||||
Recommended one-album archival export:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export \
|
||||
--album-id "Vacation" \
|
||||
--out ./VacationOriginals \
|
||||
--originals \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3
|
||||
```
|
||||
|
||||
## Back Up The Whole Library
|
||||
|
||||
Use `backup-all` to export every album into a folder tree matching Photos folders and albums.
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Recommended full backup:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./PhotosBackup \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3 \
|
||||
--concurrency 4
|
||||
```
|
||||
|
||||
Recommended full original backup:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./PhotosOriginals \
|
||||
--originals \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3 \
|
||||
--concurrency 4
|
||||
```
|
||||
|
||||
If two sibling albums have the same name, `photoscli` adds the album ID to the generated folder name so the folders do not collide.
|
||||
|
||||
## Dry Runs
|
||||
|
||||
Dry-runs are strongly recommended before large exports.
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --dry-run
|
||||
```
|
||||
|
||||
With JSON output:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --dry-run --json
|
||||
```
|
||||
|
||||
Dry-run mode does not write exported files, manifests, logs, or failure files. It prints what would be exported.
|
||||
|
||||
Use dry-run when:
|
||||
|
||||
- You are testing filters.
|
||||
- You are checking output paths.
|
||||
- You are preparing a large first-time backup.
|
||||
- You are validating an exclude list.
|
||||
|
||||
## Incremental Backups
|
||||
|
||||
Incremental backups are enabled by manifests.
|
||||
|
||||
Run the same backup command repeatedly:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./PhotosBackup \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3
|
||||
```
|
||||
|
||||
Already-exported assets are skipped based on the manifest.
|
||||
|
||||
For new assets since a date:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./PhotosBackup \
|
||||
--manifest sqlite \
|
||||
--since 2024-01-01
|
||||
```
|
||||
|
||||
The `--since` flag accepts:
|
||||
|
||||
- `YYYY-MM-DD`
|
||||
- RFC3339, for example `2024-01-01T00:00:00Z`
|
||||
|
||||
## Manifests
|
||||
|
||||
Manifests track exported asset IDs and allow safe resume behavior.
|
||||
|
||||
JSONL manifest:
|
||||
|
||||
```text
|
||||
downloads.jsonl
|
||||
```
|
||||
|
||||
SQLite manifest:
|
||||
|
||||
```text
|
||||
downloads.db
|
||||
```
|
||||
|
||||
Use JSONL when:
|
||||
|
||||
- You want a simple append-friendly text file.
|
||||
- Your library is small or medium-sized.
|
||||
- You want easy inspection with normal text tools.
|
||||
|
||||
Use SQLite when:
|
||||
|
||||
- Your library is large.
|
||||
- You want faster lookup behavior.
|
||||
- You want logs stored in the same database.
|
||||
- You plan to run repeated backups over time.
|
||||
|
||||
Recommended for serious backups:
|
||||
|
||||
```bash
|
||||
--manifest sqlite
|
||||
```
|
||||
|
||||
Disable manifests only for temporary exports:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Scratch" --out ./Scratch --no-manifest
|
||||
```
|
||||
|
||||
Without a manifest, `photoscli` cannot use manifest-based resume behavior.
|
||||
|
||||
## Logging
|
||||
|
||||
Enable logs with:
|
||||
|
||||
```bash
|
||||
--log
|
||||
```
|
||||
|
||||
For JSONL or no-manifest exports, logs go to:
|
||||
|
||||
```text
|
||||
export.log
|
||||
```
|
||||
|
||||
For SQLite manifests, logs go to the `logs` table in:
|
||||
|
||||
```text
|
||||
downloads.db
|
||||
```
|
||||
|
||||
Logged events include:
|
||||
|
||||
- Session start.
|
||||
- Session end.
|
||||
- Export completed.
|
||||
- Export skipped.
|
||||
- Export failed.
|
||||
|
||||
Use logs when:
|
||||
|
||||
- You are running unattended backups.
|
||||
- You need auditability.
|
||||
- You want details about failed or skipped assets.
|
||||
|
||||
## Failed Exports And Retries
|
||||
|
||||
Failed exports are written to:
|
||||
|
||||
```text
|
||||
failures.jsonl
|
||||
```
|
||||
|
||||
Failure records are deduplicated by asset ID. Repeated failures update the existing record and increment the attempt count.
|
||||
|
||||
For transient failures, first use retries during normal export:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --retry 3
|
||||
```
|
||||
|
||||
If failures remain, retry later:
|
||||
|
||||
```bash
|
||||
./bin/photoscli retry-failed --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Clear successful retry records automatically:
|
||||
|
||||
```bash
|
||||
./bin/photoscli retry-failed --out ./PhotosBackup --clear-on-success
|
||||
```
|
||||
|
||||
List or clear failure records:
|
||||
|
||||
```bash
|
||||
./bin/photoscli failures list --out ./PhotosBackup
|
||||
./bin/photoscli failures clear --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Typical reasons for failures:
|
||||
|
||||
- iCloud asset not currently downloadable.
|
||||
- Network interruption.
|
||||
- Photos permission issue.
|
||||
- Destination disk unavailable.
|
||||
- Destination disk full.
|
||||
- Invalid or changing Photos library state.
|
||||
|
||||
After retrying, run verification:
|
||||
|
||||
```bash
|
||||
./bin/photoscli verify --out ./PhotosBackup --manifest sqlite
|
||||
```
|
||||
|
||||
## Verification
|
||||
|
||||
Run verification after large backups:
|
||||
|
||||
```bash
|
||||
./bin/photoscli verify --out ./PhotosBackup --manifest sqlite
|
||||
```
|
||||
|
||||
Verification checks that manifest entries point to files on disk. It also reports zero-byte files and size mismatches when the manifest has a recorded size.
|
||||
|
||||
It exits with:
|
||||
|
||||
- `0` when all checked files exist.
|
||||
- `2` when files are missing.
|
||||
- `1` for argument or runtime errors.
|
||||
|
||||
You can also verify immediately after export:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --manifest sqlite --verify
|
||||
```
|
||||
|
||||
## Reporting
|
||||
|
||||
Show manifest and failure counts:
|
||||
|
||||
```bash
|
||||
./bin/photoscli report --out ./PhotosBackup --manifest sqlite
|
||||
```
|
||||
|
||||
Example output:
|
||||
|
||||
```text
|
||||
entries 12345
|
||||
failures 2
|
||||
```
|
||||
|
||||
Use this after long runs to quickly check whether work was recorded and whether failures occurred.
|
||||
|
||||
## Diffing An Album Against A Backup
|
||||
|
||||
Use `diff` to find album assets that are not in the manifest:
|
||||
|
||||
```bash
|
||||
./bin/photoscli diff --album-id "Vacation" --out ./PhotosBackup --manifest sqlite
|
||||
```
|
||||
|
||||
Missing assets are printed as:
|
||||
|
||||
```text
|
||||
<asset-id> <filename>
|
||||
```
|
||||
|
||||
Exit code is `2` if missing assets are found.
|
||||
|
||||
## Filtering
|
||||
|
||||
### Favorites Only
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Favorites" --out ./Favorites --only-favorites
|
||||
```
|
||||
|
||||
### Media Type
|
||||
|
||||
Photos only, the default:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --media photos
|
||||
```
|
||||
|
||||
Videos only:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./VideoBackup --media videos --include-videos
|
||||
```
|
||||
|
||||
Everything:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./FullBackup --media all --include-videos
|
||||
```
|
||||
|
||||
### Exclude Albums
|
||||
|
||||
Exclude exact album names:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --exclude-album "Recently Deleted"
|
||||
```
|
||||
|
||||
Exclude by glob:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --exclude-album "Temp*"
|
||||
```
|
||||
|
||||
Use multiple exclusions:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./PhotosBackup \
|
||||
--exclude-album "Recently Deleted" \
|
||||
--exclude-album "Temp*" \
|
||||
--exclude-album "Screenshots"
|
||||
```
|
||||
|
||||
### Date Filter
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --since 2024-01-01
|
||||
```
|
||||
|
||||
### Estimated Size Filter
|
||||
|
||||
`--min-size` and `--max-size` currently filter by estimated pixel count from dimensions, not encoded file size.
|
||||
|
||||
```bash
|
||||
./bin/photoscli export --album-id "Large" --out ./LargeOnly --min-size 12000000
|
||||
```
|
||||
|
||||
## Date-Based Folder Layouts
|
||||
|
||||
Use `--date-template` to append date folders based on asset creation date.
|
||||
|
||||
Example:
|
||||
|
||||
```bash
|
||||
./bin/photoscli export \
|
||||
--album-id "Vacation" \
|
||||
--out ./Vacation \
|
||||
--date-template YYYY/MM/DD
|
||||
```
|
||||
|
||||
Possible output:
|
||||
|
||||
```text
|
||||
Vacation/
|
||||
2024/
|
||||
07/
|
||||
15/
|
||||
0000_....jpg
|
||||
```
|
||||
|
||||
Supported tokens:
|
||||
|
||||
- `YYYY`
|
||||
- `MM`
|
||||
- `DD`
|
||||
|
||||
Assets without parseable creation dates stay in the base output path.
|
||||
|
||||
## Configuration File
|
||||
|
||||
You can store default values in:
|
||||
|
||||
```text
|
||||
~/.photoscli.toml
|
||||
```
|
||||
|
||||
Example:
|
||||
|
||||
```toml
|
||||
size = 2048
|
||||
quality = 90
|
||||
concurrency = 4
|
||||
manifest = "sqlite"
|
||||
sort = "newest"
|
||||
media = "photos"
|
||||
retry = 3
|
||||
log = true
|
||||
```
|
||||
|
||||
Use a custom config path:
|
||||
|
||||
```bash
|
||||
PHOTOSCLI_CONFIG=./photoscli.toml ./bin/photoscli backup-all --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Command-line flags override config-file defaults.
|
||||
|
||||
Recommended config for recurring backups:
|
||||
|
||||
```toml
|
||||
manifest = "sqlite"
|
||||
log = true
|
||||
retry = 3
|
||||
concurrency = 4
|
||||
sort = "oldest"
|
||||
media = "photos"
|
||||
quality = 90
|
||||
size = 2048
|
||||
```
|
||||
|
||||
Then run:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup
|
||||
```
|
||||
|
||||
## Automation Examples
|
||||
|
||||
### Weekly Incremental Backup
|
||||
|
||||
```bash
|
||||
#!/bin/sh
|
||||
set -eu
|
||||
|
||||
OUT="/Volumes/BackupDrive/PhotosExport"
|
||||
BIN="$HOME/bin/photoscli"
|
||||
|
||||
"$BIN" backup-all \
|
||||
--out "$OUT" \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3 \
|
||||
--concurrency 4 \
|
||||
--json
|
||||
|
||||
"$BIN" verify --out "$OUT" --manifest sqlite
|
||||
```
|
||||
|
||||
### Export Favorites For Sharing
|
||||
|
||||
```bash
|
||||
./bin/photoscli export \
|
||||
--album-id "Favorites" \
|
||||
--out ./FavoritesShare \
|
||||
--only-favorites \
|
||||
--size 2048 \
|
||||
--quality 88 \
|
||||
--no-manifest
|
||||
```
|
||||
|
||||
### Archive Originals From One Album
|
||||
|
||||
```bash
|
||||
./bin/photoscli export \
|
||||
--album-id "Family Archive" \
|
||||
--out ./FamilyArchiveOriginals \
|
||||
--originals \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3
|
||||
```
|
||||
|
||||
### Backup Everything Including Videos
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all \
|
||||
--out ./FullPhotosBackup \
|
||||
--media all \
|
||||
--include-videos \
|
||||
--manifest sqlite \
|
||||
--log \
|
||||
--retry 3
|
||||
```
|
||||
|
||||
## Interpreting Exit Codes
|
||||
|
||||
`photoscli` uses stable exit codes for scripts:
|
||||
|
||||
- `0`: success.
|
||||
- `1`: invalid arguments, runtime error, or all exports failed.
|
||||
- `2`: partial failure, diff found missing manifest entries, or verify found missing files.
|
||||
- `3`: Photos access denied.
|
||||
|
||||
Example shell handling:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --manifest sqlite --log
|
||||
rc=$?
|
||||
|
||||
case "$rc" in
|
||||
0) echo "backup succeeded" ;;
|
||||
2) echo "backup partially succeeded; inspect failures.jsonl" ;;
|
||||
3) echo "Photos access denied; check macOS privacy settings" ;;
|
||||
*) echo "backup failed with exit code $rc" ;;
|
||||
esac
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
### Access Denied
|
||||
|
||||
Symptom:
|
||||
|
||||
```text
|
||||
error: access denied
|
||||
```
|
||||
|
||||
Fix:
|
||||
|
||||
- Open System Settings.
|
||||
- Go to Privacy & Security > Photos.
|
||||
- Enable access for your terminal app or launcher.
|
||||
- Run `photoscli albums` again.
|
||||
|
||||
### iCloud Assets Fail To Export
|
||||
|
||||
Try:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --retry 3
|
||||
```
|
||||
|
||||
If failures remain:
|
||||
|
||||
```bash
|
||||
./bin/photoscli retry-failed --out ./PhotosBackup
|
||||
```
|
||||
|
||||
Also check:
|
||||
|
||||
- Network connectivity.
|
||||
- iCloud Photos status.
|
||||
- Whether Photos.app can open/download the asset.
|
||||
|
||||
### Destination Disk Is Full
|
||||
|
||||
Free space, then rerun the same command. The manifest should skip already-exported assets and continue with remaining work.
|
||||
|
||||
### Duplicate Album Names
|
||||
|
||||
`backup-all` automatically appends the album ID to duplicate sibling album names. If you want exact control, export ambiguous albums individually using their PhotoKit local identifiers.
|
||||
|
||||
### Too Slow
|
||||
|
||||
Try a moderate concurrency increase:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --concurrency 8
|
||||
```
|
||||
|
||||
If your library is heavily iCloud-backed, too much concurrency may increase transient failures. Use `--retry 3` and reduce concurrency if needed.
|
||||
|
||||
### Too Much Output Or Hard To Parse
|
||||
|
||||
Use JSON summary output:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --json
|
||||
```
|
||||
|
||||
Use structured logs:
|
||||
|
||||
```bash
|
||||
./bin/photoscli backup-all --out ./PhotosBackup --manifest sqlite --log
|
||||
```
|
||||
|
||||
## Safe Operating Practices
|
||||
|
||||
- Run `--dry-run` before the first large backup.
|
||||
- Use `--manifest sqlite` for long-term backups.
|
||||
- Use `--log` for unattended runs.
|
||||
- Use `--retry 3` for iCloud-heavy libraries.
|
||||
- Run `verify` after large runs.
|
||||
- Keep the destination on a reliable disk.
|
||||
- Do not use `--no-manifest` for backups you expect to resume.
|
||||
- Prefer album local identifiers when names are duplicated.
|
||||
|
||||
## Limitations
|
||||
|
||||
- This is macOS-only.
|
||||
- Real Photos access requires macOS privacy permission.
|
||||
- `--format heic|png` is currently a validated CLI hint; true non-JPEG preview output still needs bridge support.
|
||||
- `--min-size` and `--max-size` use estimated pixel count from dimensions, not encoded file size.
|
||||
- Original export depends on asset resources exposed by PhotoKit.
|
||||
- iCloud-backed assets may require network downloads and can fail for reasons outside the CLI's control.
|
||||
- The manifest records exported asset IDs; it is not a cryptographic integrity database.
|
||||
+302
-57
@@ -21,6 +21,10 @@ var (
|
||||
exportTimeout = 2 * time.Second
|
||||
configValues map[string]string
|
||||
configLoaded bool
|
||||
mkdirTempFunc = os.MkdirTemp
|
||||
renameFunc = os.Rename
|
||||
openFileFunc = os.OpenFile
|
||||
removeFunc = os.Remove
|
||||
)
|
||||
|
||||
type exportOptions struct {
|
||||
@@ -75,6 +79,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
return cmdVerify(args[1:], stdout, stderr)
|
||||
case "retry-failed":
|
||||
return cmdRetryFailed(args[1:], stdout, stderr, bridge)
|
||||
case "failures":
|
||||
return cmdFailures(args[1:], stdout, stderr)
|
||||
case "status":
|
||||
return cmdStatus(args[1:], stdout, stderr)
|
||||
case "version", "--version", "-v":
|
||||
fmt.Fprintln(stdout, version)
|
||||
return exitOK
|
||||
@@ -110,6 +118,10 @@ USAGE
|
||||
photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli verify --out <dir> [--manifest jsonl|sqlite]
|
||||
photoscli retry-failed --out <dir>
|
||||
photoscli retry-failed --out <dir> --clear-on-success
|
||||
photoscli failures list --out <dir>
|
||||
photoscli failures clear --out <dir>
|
||||
photoscli status --out <dir> [--json]
|
||||
photoscli version
|
||||
photoscli help
|
||||
|
||||
@@ -149,6 +161,12 @@ COMMANDS
|
||||
retry-failed --out <dir>
|
||||
Retry assets previously written to failures.jsonl.
|
||||
|
||||
failures list|clear --out <dir>
|
||||
List or clear deduplicated failure records.
|
||||
|
||||
status --out <dir> [--manifest jsonl|sqlite] [--json]
|
||||
Show manifest type, entry count, and failure count for a backup.
|
||||
|
||||
COMMON EXPORT FLAGS
|
||||
--out <dir>
|
||||
Destination directory. Required for export, backup-all, report, diff,
|
||||
@@ -240,7 +258,9 @@ LOGGING AND FAILURES
|
||||
table in downloads.db.
|
||||
|
||||
failures.jsonl
|
||||
Failed exports are appended here and can be retried with retry-failed.
|
||||
Failed exports are deduplicated by asset ID and can be retried with
|
||||
retry-failed. Use retry-failed --clear-on-success to remove successful
|
||||
retries from the failure list.
|
||||
|
||||
CONFIGURATION
|
||||
Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG.
|
||||
@@ -665,6 +685,7 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
|
||||
|
||||
type pendingAsset struct {
|
||||
asset photos.Asset
|
||||
root string
|
||||
path string
|
||||
album string
|
||||
}
|
||||
@@ -691,6 +712,22 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64,
|
||||
}
|
||||
}
|
||||
|
||||
func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) {
|
||||
if m == nil {
|
||||
return
|
||||
}
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
fullPath := filepath.Join(pa.path, result.Filename)
|
||||
relPath, err := filepath.Rel(root, fullPath)
|
||||
if err != nil || strings.HasPrefix(relPath, "..") {
|
||||
relPath = result.Filename
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud))
|
||||
}
|
||||
|
||||
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
|
||||
var items []pendingAsset
|
||||
var skipped int
|
||||
@@ -769,7 +806,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br
|
||||
*skipped++
|
||||
continue
|
||||
}
|
||||
*items = append(*items, pendingAsset{asset: a, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
*items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
|
||||
}
|
||||
if onProgress != nil {
|
||||
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
|
||||
@@ -850,7 +887,7 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
bar.setAlbum(pa.album, 0, 0)
|
||||
bar.draw()
|
||||
start := time.Now()
|
||||
result, exportErr := exportOneWithRetry(bridge, pa.asset, pa.path, targetSize, quality, originals, i, opts.retry)
|
||||
result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry)
|
||||
dur := time.Since(start)
|
||||
isErr := exportErr != nil
|
||||
isSkipped := result.Skipped
|
||||
@@ -862,14 +899,10 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin
|
||||
failed++
|
||||
appendFailure(pa.path, pa, exportErr)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, pa, result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -922,7 +955,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
start := time.Now()
|
||||
var result photos.ExportResult
|
||||
var exportErr error
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i].asset, pending[i].path, targetSize, quality, originals, i, workerID, opts.retry)
|
||||
result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry)
|
||||
dur := time.Since(start)
|
||||
bar.setWorker(workerID, "", 0, "", "")
|
||||
completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
|
||||
@@ -995,14 +1028,10 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
failed++
|
||||
appendFailure(entry.pa.path, entry.pa, entry.err)
|
||||
} else if isSkipped {
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
} else {
|
||||
done++
|
||||
if m != nil {
|
||||
m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud)
|
||||
}
|
||||
addManifestEntry(m, entry.pa, entry.result)
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
@@ -1036,7 +1065,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig
|
||||
func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) {
|
||||
pending := make([]pendingAsset, len(assets))
|
||||
for i, a := range assets {
|
||||
pending[i] = pendingAsset{asset: a, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
|
||||
}
|
||||
var m manifest.Manifest
|
||||
if !noManifest {
|
||||
@@ -1080,11 +1109,93 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize,
|
||||
return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index)
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index)
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = pa.path
|
||||
}
|
||||
stagingRoot := filepath.Join(root, ".photoscli-tmp")
|
||||
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
|
||||
if originals {
|
||||
return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex)
|
||||
}
|
||||
return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
|
||||
if err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
defer os.RemoveAll(stagingDir)
|
||||
|
||||
var result photos.ExportResult
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
if err != nil || result.Skipped {
|
||||
return result, err
|
||||
}
|
||||
src := filepath.Join(stagingDir, result.Filename)
|
||||
info, statErr := os.Stat(src)
|
||||
if statErr != nil {
|
||||
return result, nil
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
|
||||
}
|
||||
if err := os.MkdirAll(pa.path, 0755); err != nil {
|
||||
return result, err
|
||||
}
|
||||
dst := filepath.Join(pa.path, result.Filename)
|
||||
if err := renameFunc(src, dst); err != nil {
|
||||
return result, err
|
||||
}
|
||||
return result, nil
|
||||
}
|
||||
|
||||
func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
result, err = exportOne(bridge, a, outDir, targetSize, quality, originals, index)
|
||||
result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1093,15 +1204,11 @@ func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, tar
|
||||
return result, err
|
||||
}
|
||||
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
|
||||
var result photos.ExportResult
|
||||
var err error
|
||||
for attempt := 0; attempt <= retry; attempt++ {
|
||||
if originals {
|
||||
result, err = bridge.ExportOriginalWithSlot(a.ID, outDir, index, slotIndex)
|
||||
} else {
|
||||
result, err = bridge.ExportPreviewWithSlot(a.ID, outDir, targetSize, quality, index, slotIndex)
|
||||
}
|
||||
result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex)
|
||||
if err == nil {
|
||||
return result, nil
|
||||
}
|
||||
@@ -1356,22 +1463,73 @@ func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest
|
||||
|
||||
func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") }
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
_ = os.MkdirAll(dir, 0755)
|
||||
f, openErr := os.OpenFile(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644)
|
||||
if openErr != nil {
|
||||
return
|
||||
type failureEntry struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
FailedAt int64 `json:"failed_at"`
|
||||
Attempts int `json:"attempts"`
|
||||
}
|
||||
|
||||
func loadFailures(dir string) map[string]failureEntry {
|
||||
out := map[string]failureEntry{}
|
||||
data, err := os.ReadFile(failuresPath(dir))
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f failureEntry
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
out[f.ID] = f
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func saveFailures(dir string, failures map[string]failureEntry) error {
|
||||
if err := os.MkdirAll(dir, 0755); err != nil {
|
||||
return err
|
||||
}
|
||||
f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer f.Close()
|
||||
data, _ := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Album string `json:"album"`
|
||||
Path string `json:"path"`
|
||||
Error string `json:"error"`
|
||||
}{pa.asset.ID, pa.asset.Filename, pa.album, pa.path, err.Error()})
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
data, _ := json.Marshal(failures[id])
|
||||
f.Write(data)
|
||||
f.Write([]byte("\n"))
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func appendFailure(dir string, pa pendingAsset, err error) {
|
||||
root := pa.root
|
||||
if root == "" {
|
||||
root = dir
|
||||
}
|
||||
failures := loadFailures(root)
|
||||
f := failures[pa.asset.ID]
|
||||
f.ID = pa.asset.ID
|
||||
f.Filename = pa.asset.Filename
|
||||
f.Album = pa.album
|
||||
f.Path = pa.path
|
||||
f.Error = err.Error()
|
||||
f.FailedAt = time.Now().Unix()
|
||||
f.Attempts++
|
||||
failures[pa.asset.ID] = f
|
||||
_ = saveFailures(root, failures)
|
||||
}
|
||||
|
||||
func writeJSONSummary(stdout io.Writer, s commandSummary) {
|
||||
@@ -1466,17 +1624,32 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
missing := 0
|
||||
bad := 0
|
||||
for id, e := range entries {
|
||||
if e.Filename == "" {
|
||||
checkPath := e.Path
|
||||
if checkPath == "" {
|
||||
checkPath = e.Filename
|
||||
}
|
||||
if checkPath == "" {
|
||||
continue
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(outDir, e.Filename)); err != nil {
|
||||
missing++
|
||||
fmt.Fprintf(stdout, "%s\t%s\n", id, e.Filename)
|
||||
info, err := os.Stat(filepath.Join(outDir, checkPath))
|
||||
if err != nil {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if info.Size() == 0 {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath)
|
||||
continue
|
||||
}
|
||||
if e.Size > 0 && info.Size() != e.Size {
|
||||
bad++
|
||||
fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
|
||||
}
|
||||
}
|
||||
if missing > 0 {
|
||||
if bad > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
fmt.Fprintf(stdout, "verified\t%d\n", len(entries))
|
||||
@@ -1485,31 +1658,103 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int {
|
||||
|
||||
func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
clearOnSuccess := hasFlag(args, "--clear-on-success")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
data, err := os.ReadFile(failuresPath(outDir))
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
failures := loadFailures(outDir)
|
||||
if len(failures) == 0 {
|
||||
fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir))
|
||||
return exitErr
|
||||
}
|
||||
var pending []pendingAsset
|
||||
for _, line := range strings.Split(string(data), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" {
|
||||
continue
|
||||
}
|
||||
var f struct{ ID, Filename, Album, Path string }
|
||||
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, path: f.Path, album: f.Album})
|
||||
}
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album})
|
||||
}
|
||||
bar := newProgressBar(stderr, 1)
|
||||
done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{})
|
||||
if clearOnSuccess && done > 0 {
|
||||
for i := 0; i < done && i < len(pending); i++ {
|
||||
delete(failures, pending[i].asset.ID)
|
||||
}
|
||||
_ = saveFailures(outDir, failures)
|
||||
}
|
||||
writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)})
|
||||
if failed > 0 {
|
||||
return exitPartial
|
||||
}
|
||||
return exitOK
|
||||
}
|
||||
|
||||
func cmdFailures(args []string, stdout, stderr io.Writer) int {
|
||||
if len(args) < 1 {
|
||||
fmt.Fprintln(stderr, "error: failures requires list or clear")
|
||||
return exitErr
|
||||
}
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
switch args[0] {
|
||||
case "list":
|
||||
failures := loadFailures(outDir)
|
||||
ids := make([]string, 0, len(failures))
|
||||
for id := range failures {
|
||||
ids = append(ids, id)
|
||||
}
|
||||
sort.Strings(ids)
|
||||
for _, id := range ids {
|
||||
f := failures[id]
|
||||
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts)
|
||||
}
|
||||
return exitOK
|
||||
case "clear":
|
||||
if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
return exitOK
|
||||
default:
|
||||
fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0])
|
||||
return exitErr
|
||||
}
|
||||
}
|
||||
|
||||
func cmdStatus(args []string, stdout, stderr io.Writer) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
if outDir == "" {
|
||||
fmt.Fprintln(stderr, "error: --out is required")
|
||||
return exitErr
|
||||
}
|
||||
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
|
||||
mf, err := manifest.ParseFormat(manifestFmt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
entries, err := loadManifestEntries(outDir, mf)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return exitErr
|
||||
}
|
||||
failures := loadFailures(outDir)
|
||||
if hasFlag(args, "--json") {
|
||||
data, _ := json.Marshal(struct {
|
||||
Manifest string `json:"manifest"`
|
||||
Entries int `json:"entries"`
|
||||
Failures int `json:"failures"`
|
||||
}{manifestFmt, len(entries), len(failures)})
|
||||
fmt.Fprintln(stdout, string(data))
|
||||
return exitOK
|
||||
}
|
||||
fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures))
|
||||
return exitOK
|
||||
}
|
||||
|
||||
+270
-1
@@ -3815,7 +3815,7 @@ func TestReportDiffVerifyRetryFailedAndConfig(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("x"), 0644); err != nil {
|
||||
if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("0123456789"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
@@ -4043,3 +4043,272 @@ func TestNewFeatureRemainingBranches(t *testing.T) {
|
||||
t.Fatalf("expected retry partial, got %d", rc)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailuresAndStatusCommands(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.NewEntry("x1", "file.jpg", "Album/file.jpg", 4, "local"))
|
||||
m.Close()
|
||||
if err := os.MkdirAll(filepath.Join(dir, "Album"), 0755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "Album", "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom"))
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom2"))
|
||||
|
||||
out, stderr, rc := runWith([]string{"status", "--out", dir, "--json"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "failures") || stderr != "" {
|
||||
t.Fatalf("status json rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"status", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "entries\t1") || !strings.Contains(out, "failures\t1") || stderr != "" {
|
||||
t.Fatalf("status rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
out, stderr, rc = runWith([]string{"failures", "list", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "bad") || !strings.Contains(out, "2") || stderr != "" {
|
||||
t.Fatalf("failures list rc=%d out=%q stderr=%q", rc, out, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "requires") {
|
||||
t.Fatalf("failures missing subcommand rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "bogus", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "unknown") {
|
||||
t.Fatalf("failures bogus rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitOK || stderr != "" {
|
||||
t.Fatalf("failures clear rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected failures cleared")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicExportHelpers(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "photo.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "photo.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneAtomic(b, pa, 1024, 85, false, 0)
|
||||
if err != nil || result.Filename != "photo.jpg" {
|
||||
t.Fatalf("atomic export result=%+v err=%v", result, err)
|
||||
}
|
||||
if _, err := os.Stat(filepath.Join(dir, "Album", "photo.jpg")); err != nil {
|
||||
t.Fatalf("expected final file: %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 1); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected zero-byte error, got %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestMoreIntegrityBranches(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
m := manifest.LoadJSONL(dir)
|
||||
if err := m.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
m.AddEntry(manifest.Entry{ID: "zero", Filename: "zero.jpg", Path: "zero.jpg", Size: 1, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.AddEntry(manifest.Entry{ID: "mismatch", Filename: "mismatch.jpg", Path: "mismatch.jpg", Size: 10, Cloud: "local", Exported: time.Now().Unix()})
|
||||
m.Close()
|
||||
if err := os.WriteFile(filepath.Join(dir, "zero.jpg"), nil, 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(filepath.Join(dir, "mismatch.jpg"), []byte("bad"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
out, _, rc := runWith([]string{"verify", "--out", dir}, &mockBridge{})
|
||||
if rc != exitPartial || !strings.Contains(out, "zero-byte") || !strings.Contains(out, "size-mismatch") {
|
||||
t.Fatalf("verify rc=%d out=%q", rc, out)
|
||||
}
|
||||
_, stderr, rc := runWith([]string{"status"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("status missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", dir, "--manifest", "bad"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "manifest") {
|
||||
t.Fatalf("status bad manifest rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"status", "--out", "/proc/cannot-create"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "error:") {
|
||||
t.Fatalf("status load error rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
_, stderr, rc = runWith([]string{"failures", "list"}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "--out") {
|
||||
t.Fatalf("failures missing out rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRetryFailedClearOnSuccess(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: dir, album: "Album"}, fmt.Errorf("boom"))
|
||||
out, _, rc := runWith([]string{"retry-failed", "--out", dir, "--clear-on-success"}, &mockBridge{})
|
||||
if rc != exitOK || !strings.Contains(out, "exported") {
|
||||
t.Fatalf("retry clear rc=%d out=%q", rc, out)
|
||||
}
|
||||
if len(loadFailures(dir)) != 0 {
|
||||
t.Fatal("expected successful retry to clear failure")
|
||||
}
|
||||
}
|
||||
|
||||
func TestAtomicSlotExportHelper(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "slot.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "slot.jpg", Size: 4, Cloud: "local"}, nil
|
||||
}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "slot", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}
|
||||
result, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0)
|
||||
if err != nil || result.Filename != "slot.jpg" {
|
||||
t.Fatalf("slot atomic result=%+v err=%v", result, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestInjectedErrorBranchesForCoverage(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
b := &mockBridge{}
|
||||
pa := pendingAsset{asset: photos.Asset{ID: "x", Filename: "x.jpg"}, root: dir, path: filepath.Join(dir, "Album")}
|
||||
|
||||
oldMkdirTemp := mkdirTempFunc
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("mkdirtemp") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "mkdirtemp") {
|
||||
t.Fatalf("expected mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
oldRename := renameFunc
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("rename") }
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "rename") {
|
||||
t.Fatalf("expected rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
oldOpen := openFileFunc
|
||||
openFileFunc = func(string, int, os.FileMode) (*os.File, error) { return nil, fmt.Errorf("open") }
|
||||
if err := saveFailures(dir, map[string]failureEntry{"x": {ID: "x"}}); err == nil || !strings.Contains(err.Error(), "open") {
|
||||
t.Fatalf("expected open error, got %v", err)
|
||||
}
|
||||
openFileFunc = oldOpen
|
||||
|
||||
oldRemove := removeFunc
|
||||
removeFunc = func(string) error { return fmt.Errorf("remove") }
|
||||
_, stderr, rc := runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{})
|
||||
if rc != exitErr || !strings.Contains(stderr, "remove") {
|
||||
t.Fatalf("expected remove error, rc=%d stderr=%q", rc, stderr)
|
||||
}
|
||||
removeFunc = oldRemove
|
||||
|
||||
mf := &mockManifest{}
|
||||
addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1})
|
||||
if mf.last.Path != "file.jpg" {
|
||||
t.Fatalf("expected fallback rel path, got %+v", mf.last)
|
||||
}
|
||||
|
||||
badPathRoot := t.TempDir()
|
||||
badPath := filepath.Join(badPathRoot, "notdir")
|
||||
if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "x2", Filename: "x2.jpg"}, root: badPathRoot, path: badPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil {
|
||||
t.Fatal("expected final mkdir error")
|
||||
}
|
||||
|
||||
slotRootFile := filepath.Join(t.TempDir(), "rootfile")
|
||||
if err := os.WriteFile(slotRootFile, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotfallback", Filename: "slot.jpg"}, root: slotRootFile, path: t.TempDir()}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, true, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected original fallback error: %v", err)
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err != nil {
|
||||
t.Fatalf("unexpected preview fallback error: %v", err)
|
||||
}
|
||||
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotzero", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Slot")}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "empty.jpg"}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "zero-byte") {
|
||||
t.Fatalf("expected slot zero-byte error, got %v", err)
|
||||
}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
renameFunc = func(string, string) error { return fmt.Errorf("slot rename") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot rename") {
|
||||
t.Fatalf("expected slot rename error, got %v", err)
|
||||
}
|
||||
renameFunc = oldRename
|
||||
|
||||
mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("slot mkdirtemp") }
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot mkdirtemp") {
|
||||
t.Fatalf("expected slot mkdirtemp error, got %v", err)
|
||||
}
|
||||
mkdirTempFunc = oldMkdirTemp
|
||||
|
||||
badSlotRoot := t.TempDir()
|
||||
badSlotPath := filepath.Join(badSlotRoot, "notdir")
|
||||
if err := os.WriteFile(badSlotPath, []byte("x"), 0644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
pa = pendingAsset{asset: photos.Asset{ID: "slotbadpath", Filename: "slot.jpg"}, root: badSlotRoot, path: badSlotPath}
|
||||
b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) {
|
||||
if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil {
|
||||
return photos.ExportResult{}, err
|
||||
}
|
||||
return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil
|
||||
}
|
||||
if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil {
|
||||
t.Fatal("expected slot final mkdir error")
|
||||
}
|
||||
}
|
||||
|
||||
type mockManifest struct{ last manifest.Entry }
|
||||
|
||||
func (m *mockManifest) Has(string) bool { return false }
|
||||
func (m *mockManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.last = manifest.NewEntry(id, filename, filename, size, cloud)
|
||||
}
|
||||
func (m *mockManifest) AddEntry(e manifest.Entry) { m.last = e }
|
||||
func (m *mockManifest) Save() error { return nil }
|
||||
func (m *mockManifest) Close() {}
|
||||
func (m *mockManifest) OpenAppend() error { return nil }
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
package manifest
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestNewEntryPath(t *testing.T) {
|
||||
e := newEntry("id1", "file.jpg", 123, "local")
|
||||
if e.ID != "id1" || e.Filename != "file.jpg" || e.Path != "file.jpg" || e.Size != 123 || e.Cloud != "local" || e.Exported == 0 {
|
||||
t.Fatalf("unexpected entry: %+v", e)
|
||||
}
|
||||
e = NewEntry("id2", "file2.jpg", "Album/file2.jpg", 456, "cloud")
|
||||
if e.Path != "Album/file2.jpg" || e.Filename != "file2.jpg" || e.Size != 456 || e.Cloud != "cloud" {
|
||||
t.Fatalf("unexpected entry with path: %+v", e)
|
||||
}
|
||||
}
|
||||
|
||||
func TestAddEntryDefaultsPath(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
jm := LoadJSONL(dir)
|
||||
if err := jm.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
|
||||
jm.Close()
|
||||
if got := LoadJSONL(dir).Entries()["x1"].Path; got != "file.jpg" {
|
||||
t.Fatalf("jsonl path = %q", got)
|
||||
}
|
||||
|
||||
sdir := t.TempDir()
|
||||
sm, err := LoadSQLite(sdir)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := sm.OpenAppend(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"})
|
||||
if _, err := sm.db.Exec(`UPDATE downloads SET path = '' WHERE id = 'x1'`); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := sm.Entries()["x1"].Path; got != "file.jpg" {
|
||||
t.Fatalf("sqlite path = %q", got)
|
||||
}
|
||||
sm.Close()
|
||||
}
|
||||
@@ -40,6 +40,7 @@ func LoadJSONL(dir string) *jsonlManifest {
|
||||
type entryWithID struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Size int64 `json:"size"`
|
||||
Cloud string `json:"cloud"`
|
||||
Exported int64 `json:"exported"`
|
||||
@@ -51,8 +52,13 @@ func LoadJSONL(dir string) *jsonlManifest {
|
||||
}
|
||||
var raw entryWithID
|
||||
if json.Unmarshal([]byte(line), &raw) == nil && raw.ID != "" {
|
||||
if raw.Path == "" {
|
||||
raw.Path = raw.Filename
|
||||
}
|
||||
m.entries[raw.ID] = Entry{
|
||||
ID: raw.ID,
|
||||
Filename: raw.Filename,
|
||||
Path: raw.Path,
|
||||
Size: raw.Size,
|
||||
Cloud: raw.Cloud,
|
||||
Exported: raw.Exported,
|
||||
@@ -70,18 +76,25 @@ func (m *jsonlManifest) Has(id string) bool {
|
||||
}
|
||||
|
||||
func (m *jsonlManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.AddEntry(newEntry(id, filename, size, cloud))
|
||||
}
|
||||
|
||||
func (m *jsonlManifest) AddEntry(entry Entry) {
|
||||
m.mu.Lock()
|
||||
defer m.mu.Unlock()
|
||||
entry := newEntry(id, filename, size, cloud)
|
||||
m.entries[id] = entry
|
||||
if entry.Path == "" {
|
||||
entry.Path = entry.Filename
|
||||
}
|
||||
m.entries[entry.ID] = entry
|
||||
if m.file != nil {
|
||||
data, _ := json.Marshal(struct {
|
||||
ID string `json:"id"`
|
||||
Filename string `json:"filename"`
|
||||
Path string `json:"path,omitempty"`
|
||||
Size int64 `json:"size"`
|
||||
Cloud string `json:"cloud"`
|
||||
Exported int64 `json:"exported"`
|
||||
}{ID: id, Filename: entry.Filename, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported})
|
||||
}{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported})
|
||||
m.file.Write(data)
|
||||
m.file.Write([]byte("\n"))
|
||||
}
|
||||
|
||||
@@ -5,6 +5,7 @@ import "time"
|
||||
type Entry struct {
|
||||
ID string
|
||||
Filename string
|
||||
Path string
|
||||
Size int64
|
||||
Cloud string
|
||||
Exported int64
|
||||
@@ -13,6 +14,7 @@ type Entry struct {
|
||||
type Manifest interface {
|
||||
Has(id string) bool
|
||||
Add(id string, filename string, size int64, cloud string)
|
||||
AddEntry(entry Entry)
|
||||
Save() error
|
||||
Close()
|
||||
OpenAppend() error
|
||||
@@ -26,8 +28,17 @@ func newEntry(id, filename string, size int64, cloud string) Entry {
|
||||
return Entry{
|
||||
ID: id,
|
||||
Filename: filename,
|
||||
Path: filename,
|
||||
Size: size,
|
||||
Cloud: cloud,
|
||||
Exported: time.Now().Unix(),
|
||||
}
|
||||
}
|
||||
|
||||
func NewEntry(id, filename, path string, size int64, cloud string) Entry {
|
||||
e := newEntry(id, filename, size, cloud)
|
||||
if path != "" {
|
||||
e.Path = path
|
||||
}
|
||||
return e
|
||||
}
|
||||
|
||||
@@ -49,7 +49,8 @@ func ConvertFromJSONL(dir string) (Manifest, error) {
|
||||
}
|
||||
|
||||
for id, e := range src.Entries() {
|
||||
dst.Add(id, e.Filename, e.Size, e.Cloud)
|
||||
e.ID = id
|
||||
dst.AddEntry(e)
|
||||
}
|
||||
|
||||
os.Remove(JSONLPath(dir))
|
||||
@@ -69,7 +70,8 @@ func ConvertFromSQLite(dir string) (Manifest, error) {
|
||||
}
|
||||
|
||||
for id, e := range src.Entries() {
|
||||
dst.Add(id, e.Filename, e.Size, e.Cloud)
|
||||
e.ID = id
|
||||
dst.AddEntry(e)
|
||||
}
|
||||
if err := dst.Save(); err != nil {
|
||||
return nil, fmt.Errorf("save jsonl: %w", err)
|
||||
|
||||
@@ -61,6 +61,7 @@ func (m *sqliteManifest) OpenAppend() error {
|
||||
_, err = execFn(`CREATE TABLE IF NOT EXISTS downloads (
|
||||
id TEXT PRIMARY KEY,
|
||||
filename TEXT NOT NULL DEFAULT '',
|
||||
path TEXT NOT NULL DEFAULT '',
|
||||
size INTEGER NOT NULL DEFAULT 0,
|
||||
cloud TEXT NOT NULL DEFAULT '',
|
||||
exported INTEGER NOT NULL DEFAULT 0
|
||||
@@ -69,6 +70,7 @@ func (m *sqliteManifest) OpenAppend() error {
|
||||
db.Close()
|
||||
return fmt.Errorf("create table: %w", err)
|
||||
}
|
||||
_, _ = execFn(`ALTER TABLE downloads ADD COLUMN path TEXT NOT NULL DEFAULT ''`)
|
||||
_, err = execFn(`CREATE INDEX IF NOT EXISTS idx_downloads_id ON downloads(id)`)
|
||||
if err != nil {
|
||||
db.Close()
|
||||
@@ -91,12 +93,18 @@ func (m *sqliteManifest) Has(id string) bool {
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) Add(id string, filename string, size int64, cloud string) {
|
||||
m.AddEntry(newEntry(id, filename, size, cloud))
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) AddEntry(entry Entry) {
|
||||
if m.db == nil {
|
||||
return
|
||||
}
|
||||
entry := newEntry(id, filename, size, cloud)
|
||||
m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, size, cloud, exported) VALUES (?, ?, ?, ?, ?)`,
|
||||
id, entry.Filename, entry.Size, entry.Cloud, entry.Exported)
|
||||
if entry.Path == "" {
|
||||
entry.Path = entry.Filename
|
||||
}
|
||||
m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, exported) VALUES (?, ?, ?, ?, ?, ?)`,
|
||||
entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Exported)
|
||||
}
|
||||
|
||||
func (m *sqliteManifest) Save() error {
|
||||
@@ -119,14 +127,17 @@ func (m *sqliteManifest) Entries() map[string]Entry {
|
||||
return nil
|
||||
}
|
||||
out := make(map[string]Entry)
|
||||
rows, err := m.db.Query(`SELECT id, filename, size, cloud, exported FROM downloads`)
|
||||
rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, exported FROM downloads`)
|
||||
if err != nil {
|
||||
return out
|
||||
}
|
||||
defer rows.Close()
|
||||
for rows.Next() {
|
||||
var e Entry
|
||||
if err := rows.Scan(&e.ID, &e.Filename, &e.Size, &e.Cloud, &e.Exported); err == nil {
|
||||
if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Exported); err == nil {
|
||||
if e.Path == "" {
|
||||
e.Path = e.Filename
|
||||
}
|
||||
out[e.ID] = e
|
||||
}
|
||||
}
|
||||
|
||||
@@ -249,10 +249,10 @@ func TestSQLiteCreateIndexError(t *testing.T) {
|
||||
return nil, err
|
||||
}
|
||||
m.execFunc = func(query string, args ...any) (sql.Result, error) {
|
||||
callCount++
|
||||
if callCount == 2 {
|
||||
if strings.Contains(query, "CREATE INDEX") {
|
||||
return nil, fmt.Errorf("injected CREATE INDEX error")
|
||||
}
|
||||
callCount++
|
||||
return db.Exec(query, args...)
|
||||
}
|
||||
return db, nil
|
||||
|
||||
Reference in New Issue
Block a user