tools/photocli
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases 31 Wiki Activity
Files
cf922fee03515659b660e17b9cc66e9ef47eb8ef
photocli/cmd/photoscli/main.go
T
Ein Anderssono c9ac014473
pipeline / test (push) Has been cancelled
Details
pipeline / build (push) Has been cancelled
Details
v0.10.0: ports and adapters refactor 
- Extract shared manifest types into internal/manifest/types leaf package.
- Extract SQLite adapter into internal/manifest/sqlite.
- Extract JSONL adapter into internal/manifest/jsonl.
- Isolate modernc.org/sqlite import to sqlite/adapter.go.
- Add adapter-backed registry with manifest.Default.
- Adapter-agnostic ConvertManifest in types/.
- MemoryAdapter for in-memory manifest testing.
- CLI uses manifest.Default registry directly.
- SQLite LogWriter type assertion moved into SQLiteAdapter.
- Manifest interface includes Entries(); EntryReader removed.
- No behavior changes. 100% coverage across all 6 packages.
2026-06-15 08:27:38 +02:00

2779 lines
83 KiB
Go
Raw Blame History

package main
import (
"bytes"
"crypto/sha256"
"encoding/hex"
"encoding/json"
"encoding/xml"
"fmt"
"io"
"io/fs"
"os"
"path/filepath"
"sort"
"strconv"
"strings"
"sync"
"time"
"gitea.k3s.k0.nu/tools/photocli/internal/manifest"
"gitea.k3s.k0.nu/tools/photocli/internal/photos"
)
var (
progressPollInterval = 100 * time.Millisecond
exportTimeout = 2 * time.Second
configValues map[string]string
configLoaded bool
mkdirTempFunc = os.MkdirTemp
createTempFunc = os.CreateTemp
writeFileFunc = os.WriteFile
readFileFunc = os.ReadFile
statFunc = os.Stat
renameFunc = os.Rename
openFileFunc = os.OpenFile
removeFunc = os.Remove
reg = manifest.Default
)
type exportOptions struct {
dryRun bool
retry int
onlyFavorites bool
media string
jsonOut bool
verify bool
format string
sidecar string
checksum string
xmpPrivacy string
xmpKeywords string
xmpRating string
metadataOnly bool
reverseGeocode bool
minSize int64
maxSize int64
dateTemplate string
}
type commandSummary struct {
Exported int `json:"exported"`
Failed int `json:"failed"`
Total int `json:"total"`
}
const (
exitOK = 0
exitErr = 1
exitPartial = 2
exitAuth = 3
)
func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
if len(args) < 1 {
usage(stderr)
return exitErr
}
cmd := args[0]
switch cmd {
case "albums":
return cmdAlbums(stdout, stderr, bridge)
case "photos":
return cmdPhotos(args[1:], stdout, stderr, bridge)
case "tree":
return cmdTree(stdout, stderr, bridge)
case "backup-all":
return cmdBackupAll(args[1:], stdout, stderr, bridge)
case "export":
return cmdExport(args[1:], stdout, stderr, bridge)
case "report":
return cmdReport(args[1:], stdout, stderr)
case "diff":
return cmdDiff(args[1:], stdout, stderr, bridge)
case "verify":
return cmdVerify(args[1:], stdout, stderr)
case "manifest":
return cmdManifest(args[1:], stdout, stderr)
case "cleanup":
return cmdCleanup(args[1:], stdout, stderr)
case "doctor":
return cmdDoctor(args[1:], stdout, stderr, bridge)
case "retry-failed":
return cmdRetryFailed(args[1:], stdout, stderr, bridge)
case "failures":
return cmdFailures(args[1:], stdout, stderr)
case "status":
return cmdStatus(args[1:], stdout, stderr)
case "sidecar":
return cmdSidecar(args[1:], stdout, stderr)
case "version", "--version", "-v":
fmt.Fprintln(stdout, version)
return exitOK
case "help", "--help", "-h":
usage(stderr)
return exitOK
default:
fmt.Fprintf(stderr, "unknown command: %s\n", cmd)
usage(stderr)
return exitErr
}
}
func usage(w io.Writer) {
fmt.Fprintln(w, `photoscli - export and verify Apple Photos backups
DESCRIPTION
photoscli is a macOS-only Apple Photos exporter. It uses PhotoKit through a
small Objective-C bridge to list albums, inspect assets, export previews or
originals, keep resumable manifests, log structured export events, and verify
backup integrity.
Prebuilt releases target Apple Silicon Macs only (darwin/arm64: M1/M2/M3/M4
or newer). Intel Macs are not currently a supported release target.
The tool is intended for repeatable backups. By default it records exported
asset IDs in a manifest so later runs can skip work already completed.
USAGE
photoscli albums
photoscli photos --album-id <id-or-title>
photoscli tree
photoscli export --album-id <id-or-title> --out <dir> [flags]
photoscli backup-all --out <dir> [flags]
photoscli report --out <dir> [--manifest jsonl|sqlite]
photoscli diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
photoscli verify --out <dir> [--manifest jsonl|sqlite]
photoscli manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
photoscli cleanup --out <dir> [--manifest jsonl|sqlite] [--dry-run]
photoscli doctor [--out <dir>] [--manifest jsonl|sqlite] [--json]
photoscli retry-failed --out <dir>
photoscli retry-failed --out <dir> --clear-on-success
photoscli failures list --out <dir>
photoscli failures clear --out <dir>
photoscli sidecar inspect <file.xmp> [--json]
photoscli status --out <dir> [--json]
photoscli version
photoscli help
COMMANDS
albums
Request Photos access and list user-created albums as:
<album-id><TAB><album-title>
photos --album-id <id-or-title>
List assets in one album. The album can be a PhotoKit local identifier or
an exact album title. Output includes asset ID, filename, cloud state,
media type, dimensions, optional creation date, optional duration, and a
trailing * for favorites.
tree
Print the Photos folder/album hierarchy as an indented tree.
export --album-id <id-or-title> --out <dir> [flags]
Export assets from one album. Preview JPEGs are exported by default.
Use --originals to export original files instead.
backup-all --out <dir> [flags]
Walk the Photos tree and export every album into a matching directory
structure. Duplicate sibling album names are disambiguated with album IDs.
report --out <dir> [--manifest jsonl|sqlite]
Print manifest entry count and failure count.
diff --album-id <id-or-title> --out <dir> [--manifest jsonl|sqlite]
Compare album assets against the manifest. Missing assets are printed as
<asset-id><TAB><filename>. Exits 2 when differences are found.
verify --out <dir> [--manifest jsonl|sqlite]
Verify that manifest entries point to files that exist on disk. Missing
files are printed as <asset-id><TAB><filename>. Exits 2 on missing files.
Add --sidecar to verify expected XMP sidecars too. Add --strict with
--sidecar to require photoscli schema/generator and exported filename metadata.
Add --deep to verify manifest SHA-256 checksums when present.
manifest repair --out <dir> [--manifest jsonl|sqlite] [--checksum sha256] [--dry-run]
Fill missing manifest size/checksum metadata from files that exist on disk.
cleanup --out <dir> [--manifest jsonl|sqlite] [--dry-run]
Remove files not referenced by the manifest. Use --dry-run first.
doctor [--out <dir>] [--manifest jsonl|sqlite] [--json]
Check Photos access and optional backup/manifest health without changing data.
retry-failed --out <dir>
Retry assets previously written to failures.jsonl.
failures list|clear --out <dir>
List or clear deduplicated failure records.
sidecar inspect <file.xmp> [--json]
Read a generated XMP sidecar and print key photoscli metadata.
status --out <dir> [--manifest jsonl|sqlite] [--json]
Show manifest type, entry count, and failure count for a backup.
COMMON EXPORT FLAGS
--out <dir>
Destination directory. Required for export, backup-all, report, diff,
verify, and retry-failed.
--album-id <id-or-title>
Album PhotoKit local identifier or exact album title. Required by export,
photos, and diff.
--size <px>
Target longest-side preview size in pixels. Default: 1024. Ignored when
--originals is used.
--quality <1-100>
JPEG preview compression quality. Default: 85. Ignored when --originals is
used.
--originals
Export original files instead of preview JPEGs.
--concurrency <N>
Number of parallel export workers. Default: 3. Values above the bridge's
progress slot count are capped automatically.
--retry <N>
Retry failed exports N times with a small backoff. Default: 0.
--dry-run
Print assets that would be exported without writing files, manifests, or
logs. Useful before large backup-all runs.
--json
Print a machine-readable summary to stdout:
{"exported":N,"failed":N,"total":N}
--verify
Run manifest/file verification after export or backup-all.
--checksum none|sha256
Store optional file checksum metadata in the manifest. Default: none.
--sidecar none|xmp|json|xmp,json
Write opt-in metadata sidecars next to each exported file. Default: none.
If sidecar writing fails, the asset is counted as failed.
--xmp-privacy keep|strip-location|strip-address
Control location/address metadata in generated XMP sidecars. Default: keep.
--xmp-keywords album-path|album|none
Control dc:subject keywords in generated XMP sidecars. Default: album-path.
--xmp-rating favorite|none
Control favorite-to-rating mapping in generated XMP sidecars. Default: favorite.
--metadata-only
With --sidecar xmp, write or refresh XMP sidecars for files already in
the manifest without exporting media files. Requires a manifest.
--reverse-geocode
With --sidecar xmp, use Apple MapKit on macOS 26+ to add address metadata
for assets with GPS coordinates. Results are cached under .photoscli.
FILTERING AND SELECTION
--since <date>
Include only assets on or after a date. Accepts YYYY-MM-DD or RFC3339.
--sort oldest|newest
Sort assets by creation date. Default: oldest.
--only-favorites
Export only favorite assets.
--media photos|videos|all
Select media type. Default: photos.
--include-videos
Compatibility shortcut that includes videos/audio.
--exclude-album <pattern>
backup-all only. Repeatable. Excludes album names by exact match or glob
pattern, for example --exclude-album "Temp*".
--min-size <n>
Include only assets with estimated pixel count >= n.
--max-size <n>
Include only assets with estimated pixel count <= n.
OUTPUT LAYOUT AND FORMAT
--date-template <template>
Append date folders based on asset creation date. Supported tokens:
YYYY, MM, DD. Example: --date-template YYYY/MM/DD.
--format jpeg|heic|png
Preview output format hint. Currently validated by the CLI; the bridge's
preview export path still writes the existing preview output format.
MANIFESTS
--manifest jsonl|sqlite
Manifest backend. Default: jsonl.
jsonl -> downloads.jsonl
sqlite -> downloads.db
--no-manifest
Disable manifest reads and writes. This makes export stateless and removes
resumable skip behavior.
LOGGING AND FAILURES
--log
Enable structured export logs. With JSONL/no-manifest mode logs are
written to export.log. With SQLite manifests logs are written to a logs
table in downloads.db.
failures.jsonl
Failed exports are deduplicated by asset ID and can be retried with
retry-failed. Use retry-failed --clear-on-success to remove successful
retries from the failure list.
CONFIGURATION
Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG.
Keys match long flag names without leading dashes:
size = 2048
quality = 90
concurrency = 8
manifest = "sqlite"
sort = "newest"
retry = 3
log = true
sidecar = "xmp"
reverse-geocode = true
Command-line flags override config values.
EXAMPLES
photoscli albums
photoscli photos --album-id "Vacation"
photoscli export --album-id "Vacation" --out ./export --size 2048 --quality 92
photoscli export --album-id "Favorites" --out ./favorites --only-favorites
photoscli export --album-id "Archive" --out ./archive --originals --retry 3
photoscli backup-all --out ./backup --manifest sqlite --log --concurrency 8
photoscli backup-all --out ./backup --dry-run --json
photoscli backup-all --out ./backup --exclude-album "Recently Deleted" --exclude-album "Temp*"
photoscli backup-all --out ./backup --since 2024-01-01 --date-template YYYY/MM/DD
photoscli report --out ./backup --manifest sqlite
photoscli diff --album-id "Vacation" --out ./backup
photoscli verify --out ./backup --manifest sqlite
photoscli retry-failed --out ./backup
EXIT CODES
0 Success.
1 Error, invalid arguments, or runtime failure.
2 Partial failure, missing diff entries, or verify failures.
3 Photos access denied.
PERMISSIONS
On first use, macOS may prompt for Photos access. If denied, grant access in:
System Settings > Privacy & Security > Photos`)
}
func mustAuth(stderr io.Writer, bridge photos.Bridge) int {
fmt.Fprintln(stderr, "requesting photo library access...")
if err := bridge.RequestAccess(); err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitAuth
}
fmt.Fprintln(stderr, "access granted")
return exitOK
}
func cmdAlbums(stdout, stderr io.Writer, bridge photos.Bridge) int {
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
fmt.Fprintln(stderr, "loading albums...")
albums, err := bridge.ListAlbums()
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
for _, a := range albums {
fmt.Fprintf(stdout, "%s\t%s\n", a.ID, a.Title)
}
return exitOK
}
func resolveAlbumID(bridge photos.Bridge, idOrName string) (string, error) {
albums, listErr := bridge.ListAlbums()
if listErr != nil {
return idOrName, listErr
}
for _, a := range albums {
if a.Title == idOrName {
return a.ID, nil
}
}
_, _, err := bridge.ListAssets(idOrName)
if err == nil {
return idOrName, nil
}
return idOrName, fmt.Errorf("album not found: %s", idOrName)
}
func cmdPhotos(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
albumID := flagVal(args, "--album-id")
if albumID == "" {
fmt.Fprintln(stderr, "error: --album-id is required")
return exitErr
}
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
resolved, err := resolveAlbumID(bridge, albumID)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
assets, _, err := bridge.ListAssets(resolved)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
for _, a := range assets {
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%dx%d", a.ID, a.Filename, a.Cloud, a.MediaType, a.PixelWidth, a.PixelHeight)
if a.CreationDate != nil {
fmt.Fprintf(stdout, "\t%s", *a.CreationDate)
}
if a.Duration > 0 {
fmt.Fprintf(stdout, "\t%.1fs", a.Duration)
}
if a.IsFavorite {
fmt.Fprintf(stdout, "\t*")
}
fmt.Fprintln(stdout)
}
return exitOK
}
func cmdTree(stdout, stderr io.Writer, bridge photos.Bridge) int {
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
nodes, err := bridge.ListTree()
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
for _, node := range nodes {
printNode(stdout, node, 0)
}
return exitOK
}
func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
albumID := flagVal(args, "--album-id")
outDir := flagVal(args, "--out")
originals := hasFlag(args, "--originals")
skipVideos := !hasFlag(args, "--include-videos")
noManifest := hasFlag(args, "--no-manifest")
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
sortOrder := flagValWithDefault(args, "--sort", "oldest")
sizeStr := flagValWithDefault(args, "--size", "1024")
qualityStr := flagValWithDefault(args, "--quality", "85")
concurrencyStr := flagValWithDefault(args, "--concurrency", "3")
sinceStr := flagVal(args, "--since")
enableLog := hasFlag(args, "--log")
opts, ok := parseExportOptions(args, stderr)
if !ok {
return exitErr
}
if hasFlag(args, "--include-videos") {
opts.media = "all"
}
if albumID == "" {
fmt.Fprintln(stderr, "error: --album-id is required")
return exitErr
}
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
if opts.metadataOnly && noManifest {
fmt.Fprintln(stderr, "error: --metadata-only requires a manifest")
return exitErr
}
mf, mfErr := reg.ParseFormat(manifestFmt)
if mfErr != nil {
fmt.Fprintf(stderr, "error: %v\n", mfErr)
return exitErr
}
sortNewest := sortOrder == "newest"
if sortOrder != "oldest" && sortOrder != "newest" {
fmt.Fprintf(stderr, "error: --sort must be newest or oldest, got %q\n", sortOrder)
return exitErr
}
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
resolved, err := resolveAlbumID(bridge, albumID)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
var size int
if !originals {
if _, err2 := fmt.Sscanf(sizeStr, "%d", &size); err2 != nil || size <= 0 {
fmt.Fprintf(stderr, "error: --size must be a positive integer, got %q\n", sizeStr)
return exitErr
}
}
var quality int
if !originals {
if _, err2 := fmt.Sscanf(qualityStr, "%d", &quality); err2 != nil || quality < 1 || quality > 100 {
fmt.Fprintf(stderr, "error: --quality must be 1-100, got %q\n", qualityStr)
return exitErr
}
}
var concurrency int
if _, err2 := fmt.Sscanf(concurrencyStr, "%d", &concurrency); err2 != nil || concurrency < 1 {
fmt.Fprintf(stderr, "error: --concurrency must be a positive integer, got %q\n", concurrencyStr)
return exitErr
}
maxSlots := photos.GetProgressSlotCount()
if concurrency > maxSlots {
concurrency = maxSlots
}
fmt.Fprintf(stderr, "loading assets for album %s...\n", albumID)
assets, total, err := bridge.ListAssets(resolved)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
if hasFlag(args, "--include-videos") {
opts.media = "all"
}
if skipVideos && opts.media == "photos" {
assets, total = filterVideos(assets)
}
assets = applyAssetFilters(assets, opts)
total = len(assets)
if sinceStr != "" {
since, err := parseSinceDate(sinceStr)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
assets = filterBySince(assets, since)
total = len(assets)
}
if sortNewest {
sort.Slice(assets, func(i, j int) bool {
di := assets[i].CreationDate
dj := assets[j].CreationDate
if di == nil && dj == nil {
return assets[i].ID < assets[j].ID
}
if di == nil {
return false
}
if dj == nil {
return true
}
return *di > *dj
})
}
if opts.dryRun {
for _, a := range assets {
fmt.Fprintf(stdout, "%s\t%s\n", a.ID, a.Filename)
}
if opts.jsonOut {
writeJSONSummary(stdout, commandSummary{Total: total})
}
fmt.Fprintf(stderr, "dry-run: %d assets would be exported to %s\n", total, outDir)
return exitOK
}
var exported, failed int
if opts.metadataOnly {
m, _ := reg.Open(outDir, mf)
defer m.Close()
entries := manifestEntries(m)
fmt.Fprintf(stderr, "writing metadata for %d assets to %s...\n", total, outDir)
exported, failed = metadataOnlyAssets(assets, outDir, originals, "", entries, opts, bridge)
} else {
fmt.Fprintf(stderr, "exporting %d assets (%s) to %s...\n", total, exportMode(originals), outDir)
exported, failed = exportAssets(assets, outDir, size, quality, concurrency, originals, total, stderr, bridge, "", noManifest, mf, enableLog, opts)
}
if opts.jsonOut {
writeJSONSummary(stdout, commandSummary{Exported: exported, Failed: failed, Total: total})
}
if opts.verify && !noManifest {
if rc := cmdVerify([]string{"--out", outDir, "--manifest", manifestFmt}, stdout, stderr); rc != exitOK && failed == 0 {
failed++
}
}
if exported == 0 && failed > 0 {
fmt.Fprintf(stderr, "\nerror: all exports failed\n")
return exitErr
}
if opts.metadataOnly {
fmt.Fprintf(stderr, "\nwrote %d metadata sidecars to %s", exported, outDir)
} else if originals {
fmt.Fprintf(stderr, "\nexported %d original files to %s", exported, outDir)
} else {
fmt.Fprintf(stderr, "\nexported %d photos to %s", exported, outDir)
}
if failed > 0 {
fmt.Fprintf(stderr, " (%d failed)", failed)
}
fmt.Fprintln(stderr)
if failed > 0 && exported > 0 {
return exitPartial
}
return exitOK
}
func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
outDir := flagVal(args, "--out")
originals := hasFlag(args, "--originals")
skipVideos := !hasFlag(args, "--include-videos")
noManifest := hasFlag(args, "--no-manifest")
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
sortOrder := flagValWithDefault(args, "--sort", "oldest")
sizeStr := flagValWithDefault(args, "--size", "1024")
qualityStr := flagValWithDefault(args, "--quality", "85")
concurrencyStr := flagValWithDefault(args, "--concurrency", "3")
excludeAlbums := flagVals(args, "--exclude-album")
sinceStr := flagVal(args, "--since")
enableLog := hasFlag(args, "--log")
opts, ok := parseExportOptions(args, stderr)
if !ok {
return exitErr
}
if hasFlag(args, "--include-videos") {
opts.media = "all"
}
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
if opts.metadataOnly && noManifest {
fmt.Fprintln(stderr, "error: --metadata-only requires a manifest")
return exitErr
}
mf, mfErr := reg.ParseFormat(manifestFmt)
if mfErr != nil {
fmt.Fprintf(stderr, "error: %v\n", mfErr)
return exitErr
}
sortNewest := sortOrder == "newest"
if sortOrder != "oldest" && sortOrder != "newest" {
fmt.Fprintf(stderr, "error: --sort must be newest or oldest, got %q\n", sortOrder)
return exitErr
}
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
var size int
if !originals {
if _, err := fmt.Sscanf(sizeStr, "%d", &size); err != nil || size <= 0 {
fmt.Fprintf(stderr, "error: --size must be a positive integer, got %q\n", sizeStr)
return exitErr
}
}
var quality int
if !originals {
if _, err := fmt.Sscanf(qualityStr, "%d", &quality); err != nil || quality < 1 || quality > 100 {
fmt.Fprintf(stderr, "error: --quality must be 1-100, got %q\n", qualityStr)
return exitErr
}
}
var concurrency int
if _, err := fmt.Sscanf(concurrencyStr, "%d", &concurrency); err != nil || concurrency < 1 {
fmt.Fprintf(stderr, "error: --concurrency must be a positive integer, got %q\n", concurrencyStr)
return exitErr
}
maxSlots := photos.GetProgressSlotCount()
if concurrency > maxSlots {
concurrency = maxSlots
}
var sinceTime time.Time
if sinceStr != "" {
var err error
sinceTime, err = parseSinceDate(sinceStr)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
}
fmt.Fprintln(stderr, "loading photo library tree...")
nodes, err := bridge.ListTree()
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
albumCount := countAlbums(nodes)
fmt.Fprintf(stderr, "found %d albums, building index...\n", albumCount)
if opts.dryRun {
pending, skipped := collectPendingAssets(nodes, outDir, bridge, skipVideos, originals, nil, nil, sortNewest, excludeAlbums, sinceTime, opts)
for _, pa := range pending {
fmt.Fprintf(stdout, "%s\t%s\t%s\n", pa.asset.ID, pa.album, pa.asset.Filename)
}
if opts.jsonOut {
writeJSONSummary(stdout, commandSummary{Total: len(pending)})
}
fmt.Fprintf(stderr, "dry-run: %d assets would be exported (%d skipped)\n", len(pending), skipped)
return exitOK
}
var totalAssets, failed int
if opts.metadataOnly {
m, _ := reg.Open(outDir, mf)
entries := manifestEntries(m)
m.Close()
pending, skipped := collectPendingAssets(nodes, outDir, bridge, skipVideos, originals, nil, nil, sortNewest, excludeAlbums, sinceTime, opts)
fmt.Fprintf(stderr, " indexed %d metadata entries (%d skipped), writing sidecars to %s...\n", len(pending), skipped, outDir)
totalAssets, failed = metadataOnlyPending(pending, entries, originals, opts, bridge)
} else {
totalAssets, failed, err = backupTree(nodes, outDir, size, quality, concurrency, originals, skipVideos, stderr, bridge, noManifest, mf, sortNewest, excludeAlbums, sinceTime, enableLog, opts)
}
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
if opts.metadataOnly {
fmt.Fprintf(stderr, "\nwrote %d metadata sidecars across %d albums to %s", totalAssets, albumCount, outDir)
} else if originals {
fmt.Fprintf(stderr, "\nexported %d original files across %d albums to %s", totalAssets, albumCount, outDir)
} else {
fmt.Fprintf(stderr, "\nexported %d preview files across %d albums to %s", totalAssets, albumCount, outDir)
}
if failed > 0 {
fmt.Fprintf(stderr, " (%d failed)", failed)
}
fmt.Fprintln(stderr)
if opts.jsonOut {
writeJSONSummary(stdout, commandSummary{Exported: totalAssets, Failed: failed, Total: totalAssets + failed})
}
if opts.verify && !noManifest {
if rc := cmdVerify([]string{"--out", outDir, "--manifest", manifestFmt}, stdout, stderr); rc != exitOK && failed == 0 {
failed++
}
}
if failed > 0 && totalAssets > 0 {
return exitPartial
}
return exitOK
}
type pendingAsset struct {
asset photos.Asset
root string
path string
album string
}
type collectProgress struct {
pending int
skipped int
album string
err error
}
func logEntry(event, level, assetID, album, filename, cloud string, size int64, durationMs int64, message string) manifest.LogEntry {
return manifest.LogEntry{
Timestamp: time.Now().Unix(),
Level: level,
Event: event,
AssetID: assetID,
Album: album,
Filename: filename,
Size: size,
Cloud: cloud,
DurationMs: durationMs,
Message: message,
}
}
func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, checksum string) {
if m == nil {
return
}
root := pa.root
if root == "" {
root = pa.path
}
fullPath := filepath.Join(pa.path, result.Filename)
relPath, err := filepath.Rel(root, fullPath)
if err != nil || strings.HasPrefix(relPath, "..") {
relPath = result.Filename
}
m.AddEntry(manifest.NewEntryWithChecksum(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud, checksum))
}
func addManifestEntryForResult(m manifest.Manifest, pa pendingAsset, result photos.ExportResult, opts exportOptions) error {
checksum := ""
if opts.checksum == "sha256" && !result.Skipped {
var err error
checksum, err = fileSHA256(filepath.Join(pa.path, result.Filename))
if err != nil {
return err
}
}
addManifestEntry(m, pa, result, checksum)
return nil
}
func fileSHA256(path string) (string, error) {
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
h := sha256.New()
if _, err := io.Copy(h, f); err != nil {
return "", err
}
return hex.EncodeToString(h.Sum(nil)), nil
}
type xmpSidecarData struct {
AssetID string
OriginalFilename string
ExportedFilename string
Album string
AlbumPath string
Keywords []string
ManifestPath string
MediaType string
MediaSubtypes []string
SourceType string
PlaybackStyle string
PixelWidth int
PixelHeight int
Duration float64
IsFavorite bool
IsHidden bool
HasAdjustments bool
Cloud string
ExportMode string
PhotoscliVersion string
ExportedAt string
Size int64
CreateDate string
ModifyDate string
Location *photos.AssetLocation
Placemark *photos.Placemark
BurstIdentifier string
RepresentsBurst bool
BurstSelectionTypes []string
AdjustmentInfo *photos.AdjustmentInfo
Resources []photos.AssetResource
}
type geocodeCache struct {
path string
mu sync.Mutex
items map[string]photos.Placemark
}
type geocodeCacheEntry struct {
Key string `json:"key"`
Placemark photos.Placemark `json:"placemark"`
}
func newGeocodeCache(root string) *geocodeCache {
c := &geocodeCache{path: filepath.Join(root, ".photoscli", "geocode-cache.jsonl"), items: map[string]photos.Placemark{}}
data, err := os.ReadFile(c.path)
if err != nil {
return c
}
for _, line := range strings.Split(string(data), "\n") {
if strings.TrimSpace(line) == "" {
continue
}
var e geocodeCacheEntry
if json.Unmarshal([]byte(line), &e) == nil && e.Key != "" {
c.items[e.Key] = e.Placemark
}
}
return c
}
func geocodeKey(lat, lon float64) string { return fmt.Sprintf("%.5f,%.5f", lat, lon) }
func (c *geocodeCache) lookup(lat, lon float64, bridge photos.Bridge) *photos.Placemark {
if c == nil {
return nil
}
key := geocodeKey(lat, lon)
c.mu.Lock()
if p, ok := c.items[key]; ok {
c.mu.Unlock()
return &p
}
c.mu.Unlock()
p, err := bridge.ReverseGeocode(lat, lon)
if err != nil {
return nil
}
c.mu.Lock()
c.items[key] = p
_ = os.MkdirAll(filepath.Dir(c.path), 0755)
if f, err := openFileFunc(c.path, os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644); err == nil {
_ = json.NewEncoder(f).Encode(geocodeCacheEntry{Key: key, Placemark: p})
_ = f.Close()
}
c.mu.Unlock()
return &p
}
func sidecarPath(exportedPath string) string {
ext := filepath.Ext(exportedPath)
return strings.TrimSuffix(exportedPath, ext) + ".xmp"
}
func jsonSidecarPath(exportedPath string) string {
ext := filepath.Ext(exportedPath)
return strings.TrimSuffix(exportedPath, ext) + ".json"
}
func sidecarEnabled(sidecar, format string) bool {
for _, part := range strings.Split(sidecar, ",") {
if strings.TrimSpace(part) == format {
return true
}
}
return false
}
func renderXMP(d xmpSidecarData) []byte {
attrs := []struct{ key, val string }{
{"photoscli:xmpSchemaVersion", "2"},
{"photoscli:assetID", d.AssetID},
{"photoscli:originalFilename", d.OriginalFilename},
{"photoscli:exportedFilename", d.ExportedFilename},
{"photoscli:album", d.Album},
{"photoscli:albumPath", d.AlbumPath},
{"photoscli:manifestPath", d.ManifestPath},
{"photoscli:mediaType", d.MediaType},
{"photoscli:sourceType", d.SourceType},
{"photoscli:playbackStyle", d.PlaybackStyle},
{"photoscli:pixelWidth", fmt.Sprintf("%d", d.PixelWidth)},
{"photoscli:pixelHeight", fmt.Sprintf("%d", d.PixelHeight)},
{"photoscli:duration", fmt.Sprintf("%.3f", d.Duration)},
{"photoscli:isFavorite", fmt.Sprintf("%t", d.IsFavorite)},
{"photoscli:isHidden", fmt.Sprintf("%t", d.IsHidden)},
{"photoscli:hasAdjustments", fmt.Sprintf("%t", d.HasAdjustments)},
{"photoscli:cloud", d.Cloud},
{"photoscli:burstIdentifier", d.BurstIdentifier},
{"photoscli:representsBurst", fmt.Sprintf("%t", d.RepresentsBurst)},
{"photoscli:exportMode", d.ExportMode},
{"photoscli:photoscliVersion", d.PhotoscliVersion},
{"photoscli:exportedAt", d.ExportedAt},
{"photoscli:size", fmt.Sprintf("%d", d.Size)},
{"dc:title", d.ExportedFilename},
{"xmp:MetadataDate", d.ExportedAt},
{"photoscli:sidecarGeneratedAt", d.ExportedAt},
{"photoscli:sidecarGenerator", "photoscli " + d.PhotoscliVersion},
}
if d.IsFavorite {
attrs = append(attrs, struct{ key, val string }{"xmp:Rating", "5"})
}
if d.CreateDate != "" {
attrs = append(attrs,
struct{ key, val string }{"xmp:CreateDate", d.CreateDate},
struct{ key, val string }{"photoshop:DateCreated", d.CreateDate},
)
}
if d.ModifyDate != "" {
attrs = append(attrs, struct{ key, val string }{"xmp:ModifyDate", d.ModifyDate})
}
if d.Location != nil {
attrs = append(attrs,
struct{ key, val string }{"photoscli:latitude", fmt.Sprintf("%.8f", d.Location.Latitude)},
struct{ key, val string }{"photoscli:longitude", fmt.Sprintf("%.8f", d.Location.Longitude)},
struct{ key, val string }{"photoscli:altitude", fmt.Sprintf("%.3f", d.Location.Altitude)},
struct{ key, val string }{"photoscli:horizontalAccuracy", fmt.Sprintf("%.3f", d.Location.HorizontalAccuracy)},
struct{ key, val string }{"exif:GPSLatitude", fmt.Sprintf("%.8f", d.Location.Latitude)},
struct{ key, val string }{"exif:GPSLongitude", fmt.Sprintf("%.8f", d.Location.Longitude)},
struct{ key, val string }{"exif:GPSAltitude", fmt.Sprintf("%.3f", d.Location.Altitude)},
)
}
if d.Placemark != nil {
attrs = append(attrs,
struct{ key, val string }{"photoscli:addressName", d.Placemark.Name},
struct{ key, val string }{"photoscli:addressCountry", d.Placemark.Country},
struct{ key, val string }{"photoscli:addressCountryCode", d.Placemark.CountryCode},
struct{ key, val string }{"photoscli:addressRegion", d.Placemark.AdministrativeArea},
struct{ key, val string }{"photoscli:addressCity", d.Placemark.Locality},
struct{ key, val string }{"photoscli:addressSubLocality", d.Placemark.SubLocality},
struct{ key, val string }{"photoscli:addressStreet", strings.TrimSpace(d.Placemark.Thoroughfare + " " + d.Placemark.SubThoroughfare)},
struct{ key, val string }{"photoscli:addressPostalCode", d.Placemark.PostalCode},
struct{ key, val string }{"photoscli:addressFormatted", d.Placemark.FormattedAddress},
struct{ key, val string }{"photoscli:reverseGeocoder", "MapKit"},
)
}
if d.AdjustmentInfo != nil {
attrs = append(attrs,
struct{ key, val string }{"photoscli:adjustmentFormatIdentifier", d.AdjustmentInfo.FormatIdentifier},
struct{ key, val string }{"photoscli:adjustmentFormatVersion", d.AdjustmentInfo.FormatVersion},
struct{ key, val string }{"photoscli:adjustmentBaseFilename", d.AdjustmentInfo.BaseFilename},
)
}
var sb strings.Builder
sb.WriteString("<?xpacket begin=\"\" id=\"W5M0MpCehiHzreSzNTczkc9d\"?>\n")
sb.WriteString("<x:xmpmeta xmlns:x=\"adobe:ns:meta/\">\n")
sb.WriteString(" <rdf:RDF xmlns:rdf=\"http://www.w3.org/1999/02/22-rdf-syntax-ns#\">\n")
sb.WriteString(" <rdf:Description xmlns:photoscli=\"https://gitea.k3s.k0.nu/tools/photocli/ns/1.0/\" xmlns:dc=\"http://purl.org/dc/elements/1.1/\" xmlns:xmp=\"http://ns.adobe.com/xap/1.0/\" xmlns:exif=\"http://ns.adobe.com/exif/1.0/\" xmlns:photoshop=\"http://ns.adobe.com/photoshop/1.0/\"")
for _, a := range attrs {
sb.WriteString("\n ")
sb.WriteString(a.key)
sb.WriteString("=\"")
xml.EscapeText(&sb, []byte(a.val))
sb.WriteString("\"")
}
sb.WriteString(" >\n")
writeStringSeq(&sb, "dc:subject", d.Keywords)
writeStringSeq(&sb, "photoscli:mediaSubtypes", d.MediaSubtypes)
writeStringSeq(&sb, "photoscli:burstSelectionTypes", d.BurstSelectionTypes)
writeStringSeq(&sb, "photoscli:areasOfInterest", func() []string {
if d.Placemark == nil {
return nil
}
return d.Placemark.AreasOfInterest
}())
writeResourceSeq(&sb, d.Resources)
sb.WriteString(" </rdf:Description>\n")
sb.WriteString(" </rdf:RDF>\n")
sb.WriteString("</x:xmpmeta>\n")
sb.WriteString("<?xpacket end=\"w\"?>\n")
return []byte(sb.String())
}
func keywordsFromAlbumPath(album, albumPath string) []string {
seen := map[string]bool{}
var out []string
add := func(v string) {
v = strings.TrimSpace(v)
if v == "" || v == "." || seen[v] {
return
}
seen[v] = true
out = append(out, v)
}
add(album)
for _, part := range strings.Split(filepath.ToSlash(albumPath), "/") {
add(part)
}
return out
}
func writeStringSeq(sb *strings.Builder, name string, vals []string) {
if len(vals) == 0 {
return
}
sb.WriteString("\n <" + name + "><rdf:Seq>")
for _, v := range vals {
sb.WriteString("<rdf:li>")
xml.EscapeText(sb, []byte(v))
sb.WriteString("</rdf:li>")
}
sb.WriteString("</rdf:Seq></" + name + ">")
}
func writeResourceSeq(sb *strings.Builder, resources []photos.AssetResource) {
if len(resources) == 0 {
return
}
sb.WriteString("\n <photoscli:resources><rdf:Seq>")
for _, r := range resources {
sb.WriteString("<rdf:li><rdf:Description")
for _, a := range []struct{ key, val string }{{"photoscli:resourceType", r.Type}, {"photoscli:resourceFilename", r.Filename}, {"photoscli:resourceUTI", r.UTI}, {"photoscli:resourceLocal", fmt.Sprintf("%t", r.Local)}, {"photoscli:resourceSize", fmt.Sprintf("%d", r.Size)}} {
sb.WriteString(" " + a.key + "=\"")
xml.EscapeText(sb, []byte(a.val))
sb.WriteString("\"")
}
sb.WriteString(" /></rdf:li>")
}
sb.WriteString("</rdf:Seq></photoscli:resources>")
}
func writeXMPSidecar(path string, data xmpSidecarData) error {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
f, err := createTempFunc(filepath.Dir(path), ".*.xmp.tmp")
if err != nil {
return err
}
tmp := f.Name()
_ = f.Close()
if err := writeFileFunc(tmp, renderXMP(data), 0644); err != nil {
os.Remove(tmp)
return err
}
if err := renameFunc(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}
func writeJSONSidecar(path string, data xmpSidecarData) error {
if err := os.MkdirAll(filepath.Dir(path), 0755); err != nil {
return err
}
f, err := createTempFunc(filepath.Dir(path), ".*.json.tmp")
if err != nil {
return err
}
tmp := f.Name()
_ = f.Close()
payload, _ := json.MarshalIndent(data, "", " ")
payload = append(payload, '\n')
if err := writeFileFunc(tmp, payload, 0644); err != nil {
os.Remove(tmp)
return err
}
if err := renameFunc(tmp, path); err != nil {
os.Remove(tmp)
return err
}
return nil
}
func writeSidecarIfNeeded(pa pendingAsset, result photos.ExportResult, originals bool, opts exportOptions, cache *geocodeCache, bridge photos.Bridge) error {
if opts.sidecar == "none" || opts.sidecar == "" {
return nil
}
mode := "preview"
if originals {
mode = "original"
}
root := pa.root
if root == "" {
root = pa.path
}
fullPath := filepath.Join(pa.path, result.Filename)
relPath, err := filepath.Rel(root, fullPath)
if err != nil || strings.HasPrefix(relPath, "..") {
relPath = result.Filename
}
relDir, err := filepath.Rel(root, pa.path)
if err != nil || strings.HasPrefix(relDir, "..") || relDir == "." {
relDir = ""
}
createDate := ""
if pa.asset.CreationDate != nil {
createDate = *pa.asset.CreationDate
}
modifyDate := ""
if pa.asset.ModificationDate != nil {
modifyDate = *pa.asset.ModificationDate
}
location := pa.asset.Location
xmpPrivacy := opts.xmpPrivacy
if xmpPrivacy == "" {
xmpPrivacy = "keep"
}
xmpKeywords := opts.xmpKeywords
if xmpKeywords == "" {
xmpKeywords = "album-path"
}
xmpRating := opts.xmpRating
if xmpRating == "" {
xmpRating = "favorite"
}
var placemark *photos.Placemark
if opts.reverseGeocode && location != nil && cache != nil && xmpPrivacy == "keep" {
placemark = cache.lookup(pa.asset.Location.Latitude, pa.asset.Location.Longitude, bridge)
}
if xmpPrivacy == "strip-location" {
location = nil
placemark = nil
}
if xmpPrivacy == "strip-address" {
placemark = nil
}
keywords := keywordsFromAlbumPath(pa.album, relDir)
if xmpKeywords == "album" {
keywords = keywordsFromAlbumPath(pa.album, "")
}
if xmpKeywords == "none" {
keywords = nil
}
isFavorite := pa.asset.IsFavorite
if xmpRating == "none" {
isFavorite = false
}
data := xmpSidecarData{
AssetID: pa.asset.ID,
OriginalFilename: pa.asset.Filename,
ExportedFilename: result.Filename,
Album: pa.album,
AlbumPath: pa.path,
Keywords: keywords,
ManifestPath: relPath,
MediaType: pa.asset.MediaType,
MediaSubtypes: pa.asset.MediaSubtypes,
SourceType: pa.asset.SourceType,
PlaybackStyle: pa.asset.PlaybackStyle,
PixelWidth: pa.asset.PixelWidth,
PixelHeight: pa.asset.PixelHeight,
Duration: pa.asset.Duration,
IsFavorite: isFavorite,
IsHidden: pa.asset.IsHidden,
HasAdjustments: pa.asset.HasAdjustments,
Cloud: result.Cloud,
ExportMode: mode,
PhotoscliVersion: version,
ExportedAt: time.Now().UTC().Format(time.RFC3339),
Size: result.Size,
CreateDate: createDate,
ModifyDate: modifyDate,
Location: location,
Placemark: placemark,
BurstIdentifier: pa.asset.BurstIdentifier,
RepresentsBurst: pa.asset.RepresentsBurst,
BurstSelectionTypes: pa.asset.BurstSelectionTypes,
AdjustmentInfo: pa.asset.AdjustmentInfo,
Resources: pa.asset.Resources,
}
if sidecarEnabled(opts.sidecar, "xmp") {
if err := writeXMPSidecar(sidecarPath(fullPath), data); err != nil {
return err
}
}
if sidecarEnabled(opts.sidecar, "json") {
if err := writeJSONSidecar(jsonSidecarPath(fullPath), data); err != nil {
return err
}
}
return nil
}
func writeMetadataOnlySidecar(pa pendingAsset, entry manifest.Entry, originals bool, opts exportOptions, cache *geocodeCache, bridge photos.Bridge) error {
checkPath := entry.Path
if checkPath == "" {
checkPath = entry.Filename
}
if checkPath == "" {
return fmt.Errorf("manifest entry has no path")
}
root := pa.root
if root == "" {
root = pa.path
}
fullPath := filepath.Join(root, checkPath)
info, err := statFunc(fullPath)
if err != nil {
return fmt.Errorf("metadata target missing: %s", checkPath)
}
if info.Size() == 0 {
return fmt.Errorf("metadata target zero-byte: %s", checkPath)
}
pa.path = filepath.Dir(fullPath)
result := photos.ExportResult{Filename: filepath.Base(fullPath), Size: info.Size(), Cloud: entry.Cloud}
return writeSidecarIfNeeded(pa, result, originals, opts, cache, bridge)
}
func manifestEntries(m manifest.Manifest) map[string]manifest.Entry {
return m.Entries()
}
func metadataOnlyAssets(assets []photos.Asset, outDir string, originals bool, dirPrefix string, entries map[string]manifest.Entry, opts exportOptions, bridge photos.Bridge) (int, int) {
pending := make([]pendingAsset, 0, len(assets))
for _, a := range assets {
pending = append(pending, pendingAsset{asset: a, root: outDir, path: outDir, album: dirPrefix})
}
return metadataOnlyPending(pending, entries, originals, opts, bridge)
}
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
var items []pendingAsset
var skipped int
collectNodes(nodes, outDir, bridge, skipVideos, originals, &items, &skipped, onProgress, m, exclude, opts)
if sortNewest {
sort.Slice(items, func(i, j int) bool {
di := items[i].asset.CreationDate
dj := items[j].asset.CreationDate
if di == nil && dj == nil {
return items[i].asset.ID < items[j].asset.ID
}
if di == nil {
return false
}
if dj == nil {
return true
}
return *di > *dj
})
}
if !since.IsZero() {
filtered := make([]pendingAsset, 0, len(items))
for _, pa := range items {
if pa.asset.CreationDate != nil {
t, err := time.Parse(time.RFC3339, *pa.asset.CreationDate)
if err == nil && t.Before(since) {
skipped++
continue
}
}
filtered = append(filtered, pa)
}
items = filtered
}
return items, skipped
}
func metadataOnlyPending(pending []pendingAsset, entries map[string]manifest.Entry, originals bool, opts exportOptions, bridge photos.Bridge) (int, int) {
var cache *geocodeCache
if opts.reverseGeocode && len(pending) > 0 {
root := pending[0].root
if root == "" {
root = pending[0].path
}
cache = newGeocodeCache(root)
}
written, failed := 0, 0
for _, pa := range pending {
entry, ok := entries[pa.asset.ID]
if !ok {
continue
}
if err := writeMetadataOnlySidecar(pa, entry, originals, opts, cache, bridge); err != nil {
failed++
continue
}
written++
}
return written, failed
}
func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, items *[]pendingAsset, skipped *int, onProgress func(collectProgress), m manifest.Manifest, exclude []string, opts exportOptions) {
names := make(map[string]int)
for _, node := range nodes {
names[node.Name]++
}
for _, node := range nodes {
if bridge.IsCancelled() {
return
}
name := node.Name
if names[node.Name] > 1 && node.ID != "" {
name = fmt.Sprintf("%s (%s)", node.Name, sanitizePathComponent(node.ID))
}
path := outDir + "/" + sanitizePathComponent(name)
if node.Kind == "folder" {
collectNodes(node.Children, path, bridge, skipVideos, originals, items, skipped, onProgress, m, exclude, opts)
continue
}
if node.Kind == "album" && node.ID != "" {
if isExcluded(node.Name, exclude) {
if onProgress != nil {
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
}
continue
}
assets, _, err := bridge.ListAssets(node.ID)
if err != nil {
if onProgress != nil {
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name, err: err})
}
continue
}
if skipVideos && opts.media == "photos" {
assets, _ = filterVideos(assets)
}
assets = applyAssetFilters(assets, opts)
for _, a := range assets {
if m != nil && m.Has(a.ID) && !opts.metadataOnly {
*skipped++
continue
}
*items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name})
}
if onProgress != nil {
onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name})
}
}
}
}
func backupTree(nodes []photos.CollectionNode, outDir string, targetSize, quality, concurrency int, originals bool, skipVideos bool, stderr io.Writer, bridge photos.Bridge, noManifest bool, mf manifest.Format, sortNewest bool, exclude []string, since time.Time, enableLog bool, opts exportOptions) (int, int, error) {
var m manifest.Manifest
if !noManifest {
var err error
m, err = reg.Open(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, " warning: could not open manifest: %v\n", err)
} else {
if err := m.OpenAppend(); err != nil {
fmt.Fprintf(stderr, " warning: could not open manifest: %v\n", err)
}
defer m.Close()
}
}
var lw manifest.LogWriter = manifest.NoopLogWriter
if enableLog {
var err error
lw, err = reg.OpenLogWriter(m, outDir, mf)
if err != nil {
fmt.Fprintf(stderr, " warning: could not open log writer: %v\n", err)
lw = manifest.NoopLogWriter
} else {
defer lw.Close()
}
}
pending, skipped := collectPendingAssets(nodes, outDir, bridge, skipVideos, originals, func(p collectProgress) {
if p.err != nil {
fmt.Fprintf(stderr, " \u26a0 album %s: %v\n", p.album, p.err)
} else {
fmt.Fprintf(stderr, " indexing: %d files (%d skipped)\r", p.pending, p.skipped)
}
}, m, sortNewest, exclude, since, opts)
if bridge.IsCancelled() {
return 0, 0, fmt.Errorf("cancelled")
}
total := len(pending)
fmt.Fprintf(stderr, " indexed %d files (%d skipped), exporting to %s...\n", total, skipped, outDir)
bar := newProgressBar(stderr, concurrency)
lw.Log(logEntry("session_start", "info", "", "", "", "", 0, 0, fmt.Sprintf("version=%s size=%d quality=%d concurrency=%d", version, targetSize, quality, concurrency)))
exported, failed := exportPending(pending, targetSize, quality, originals, total, bar, bridge, m, concurrency, lw, opts)
bar.clear()
if m != nil {
if err := m.Save(); err != nil {
fmt.Fprintf(stderr, " warning: could not save manifest: %v\n", err)
}
}
lw.Log(logEntry("session_end", "info", "", "", "", "", 0, 0, fmt.Sprintf("exported=%d failed=%d skipped=%d", exported, failed, skipped)))
return exported, failed, nil
}
func exportPending(pending []pendingAsset, targetSize, quality int, originals bool, total int, bar *progressBar, bridge photos.Bridge, m manifest.Manifest, concurrency int, lw manifest.LogWriter, opts exportOptions) (int, int) {
var cache *geocodeCache
if opts.reverseGeocode && len(pending) > 0 {
root := pending[0].root
if root == "" {
root = pending[0].path
}
cache = newGeocodeCache(root)
}
if len(pending) < 4 {
return exportPendingSerial(pending, targetSize, quality, originals, total, bar, bridge, m, lw, opts, cache)
}
return exportPendingParallel(pending, targetSize, quality, originals, total, bar, bridge, concurrency, m, lw, opts, cache)
}
func exportPendingSerial(pending []pendingAsset, targetSize, quality int, originals bool, total int, bar *progressBar, bridge photos.Bridge, m manifest.Manifest, lw manifest.LogWriter, opts exportOptions, cache ...*geocodeCache) (int, int) {
var geo *geocodeCache
if len(cache) > 0 {
geo = cache[0]
}
done := 0
failed := 0
var totalBytes int64
var totalDur time.Duration
for i, pa := range pending {
if bridge.IsCancelled() {
break
}
bar.setWorker(0, pa.asset.Filename, 0, pa.asset.Cloud, "exporting")
bar.setTotal(done+failed, total, "")
bar.setAlbum(pa.album, 0, 0)
bar.draw()
start := time.Now()
result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry)
dur := time.Since(start)
isErr := exportErr != nil
isSkipped := result.Skipped
if !isErr && !isSkipped {
totalBytes += result.Size
totalDur += dur
}
if isErr {
failed++
appendFailure(pa.path, pa, exportErr)
} else if isSkipped {
_ = addManifestEntryForResult(m, pa, result, opts)
} else {
if sidecarErr := writeSidecarIfNeeded(pa, result, originals, opts, geo, bridge); sidecarErr != nil {
failed++
exportErr = sidecarErr
isErr = true
appendFailure(pa.path, pa, sidecarErr)
} else if checksumErr := addManifestEntryForResult(m, pa, result, opts); checksumErr != nil {
failed++
exportErr = checksumErr
isErr = true
appendFailure(pa.path, pa, checksumErr)
} else {
done++
}
}
avgSpeed := float64(0)
if totalDur > 0 {
avgSpeed = float64(totalBytes) / totalDur.Seconds()
}
bar.setTotal(done+failed, total, "")
bar.setAlbum(pa.album, 0, 0)
bar.setWorker(0, "", 0, "", "")
var logLine string
if isErr {
logLine = fmt.Sprintf("\u274c %s: %v", pa.asset.Filename, exportErr)
lw.Log(logEntry("export_fail", "error", pa.asset.ID, pa.album, pa.asset.Filename, "", result.Size, dur.Milliseconds(), exportErr.Error()))
} else if isSkipped {
logLine = fmt.Sprintf("\u23ed %s", result.Filename)
lw.Log(logEntry("export_skip", "info", pa.asset.ID, pa.album, result.Filename, result.Cloud, result.Size, 0, ""))
} else if result.Cloud == "cloud" {
logLine = fmt.Sprintf("\u2601 %s - %s - downloaded %s", result.Filename, formatSize(result.Size), formatSpeed(avgSpeed))
lw.Log(logEntry("export_done", "info", pa.asset.ID, pa.album, result.Filename, result.Cloud, result.Size, dur.Milliseconds(), ""))
} else {
logLine = fmt.Sprintf("\u2705 %s - %s - copied", result.Filename, formatSize(result.Size))
lw.Log(logEntry("export_done", "info", pa.asset.ID, pa.album, result.Filename, result.Cloud, result.Size, dur.Milliseconds(), ""))
}
bar.logCompleted(logLine)
}
return done, failed
}
func exportPendingParallel(pending []pendingAsset, targetSize, quality int, originals bool, total int, bar *progressBar, bridge photos.Bridge, workers int, m manifest.Manifest, lw manifest.LogWriter, opts exportOptions, cache ...*geocodeCache) (int, int) {
var geo *geocodeCache
if len(cache) > 0 {
geo = cache[0]
}
type resultEntry struct {
result photos.ExportResult
err error
pa pendingAsset
dur time.Duration
}
completed := make(chan resultEntry, len(pending))
jobs := make(chan int, len(pending))
var wg sync.WaitGroup
for w := 0; w < workers; w++ {
wg.Add(1)
go func(workerID int) {
defer wg.Done()
for i := range jobs {
if bridge.IsCancelled() {
completed <- resultEntry{err: fmt.Errorf("cancelled"), pa: pending[i]}
continue
}
bar.setWorker(workerID, pending[i].asset.Filename, 0, pending[i].asset.Cloud, "exporting")
start := time.Now()
var result photos.ExportResult
var exportErr error
result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry)
dur := time.Since(start)
bar.setWorker(workerID, "", 0, "", "")
completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
}
}(w)
}
go func() {
for i := range pending {
if bridge.IsCancelled() {
break
}
jobs <- i
}
close(jobs)
}()
slots := photos.GetProgressSlots()
pollDone := make(chan struct{})
wg.Add(1)
go func() {
defer wg.Done()
ticker := time.NewTicker(progressPollInterval)
defer ticker.Stop()
for {
select {
case <-ticker.C:
if bridge.IsCancelled() {
return
}
slots = photos.GetProgressSlots()
for i := 0; i < workers && i < len(slots); i++ {
bar.updateWorkerProgress(i, slots[i].Progress, slots[i].BytesDone, slots[i].BytesTotal)
}
bar.draw()
case <-pollDone:
return
}
}
}()
done := 0
failed := 0
var totalBytes int64
var totalDur time.Duration
for n := 0; n < len(pending); n++ {
var entry resultEntry
select {
case entry = <-completed:
case <-time.After(exportTimeout):
if bridge.IsCancelled() {
close(pollDone)
wg.Wait()
return done, failed
}
continue
}
if bridge.IsCancelled() {
close(pollDone)
wg.Wait()
return done, failed
}
isErr := entry.err != nil
isSkipped := entry.result.Skipped
if !isErr && !isSkipped {
totalBytes += entry.result.Size
totalDur += entry.dur
}
if isErr {
failed++
appendFailure(entry.pa.path, entry.pa, entry.err)
} else if isSkipped {
_ = addManifestEntryForResult(m, entry.pa, entry.result, opts)
} else {
if sidecarErr := writeSidecarIfNeeded(entry.pa, entry.result, originals, opts, geo, bridge); sidecarErr != nil {
failed++
entry.err = sidecarErr
isErr = true
appendFailure(entry.pa.path, entry.pa, sidecarErr)
} else if checksumErr := addManifestEntryForResult(m, entry.pa, entry.result, opts); checksumErr != nil {
failed++
entry.err = checksumErr
isErr = true
appendFailure(entry.pa.path, entry.pa, checksumErr)
} else {
done++
}
}
avgSpeed := float64(0)
if totalDur > 0 {
avgSpeed = float64(totalBytes) / totalDur.Seconds()
}
bar.setTotal(done+failed, total, "")
bar.setAlbum(entry.pa.album, 0, 0)
var logLine string
if isErr {
logLine = fmt.Sprintf("\u274c %s: %v", entry.pa.asset.Filename, entry.err)
lw.Log(logEntry("export_fail", "error", entry.pa.asset.ID, entry.pa.album, entry.pa.asset.Filename, "", entry.result.Size, entry.dur.Milliseconds(), entry.err.Error()))
} else if isSkipped {
logLine = fmt.Sprintf("\u23ed %s", entry.result.Filename)
lw.Log(logEntry("export_skip", "info", entry.pa.asset.ID, entry.pa.album, entry.result.Filename, entry.result.Cloud, entry.result.Size, 0, ""))
} else if entry.result.Cloud == "cloud" {
logLine = fmt.Sprintf("\u2601 %s - %s - downloaded %s", entry.result.Filename, formatSize(entry.result.Size), formatSpeed(avgSpeed))
lw.Log(logEntry("export_done", "info", entry.pa.asset.ID, entry.pa.album, entry.result.Filename, entry.result.Cloud, entry.result.Size, entry.dur.Milliseconds(), ""))
} else {
logLine = fmt.Sprintf("\u2705 %s - %s - copied", entry.result.Filename, formatSize(entry.result.Size))
lw.Log(logEntry("export_done", "info", entry.pa.asset.ID, entry.pa.album, entry.result.Filename, entry.result.Cloud, entry.result.Size, entry.dur.Milliseconds(), ""))
}
bar.logCompleted(logLine)
}
close(pollDone)
wg.Wait()
photos.ResetProgressSlots()
return done, failed
}
func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) {
pending := make([]pendingAsset, len(assets))
for i, a := range assets {
pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix}
}
var m manifest.Manifest
if !noManifest {
var err error
m, err = reg.Open(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "warning: could not open manifest: %v\n", err)
} else {
if err := m.OpenAppend(); err != nil {
fmt.Fprintf(stderr, "warning: could not open manifest: %v\n", err)
}
defer m.Close()
}
}
var lw manifest.LogWriter = manifest.NoopLogWriter
if enableLog {
var err error
lw, err = reg.OpenLogWriter(m, outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "warning: could not open log writer: %v\n", err)
lw = manifest.NoopLogWriter
} else {
defer lw.Close()
}
}
bar := newProgressBar(stderr, concurrency)
exported, failed := exportPending(pending, targetSize, quality, originals, len(pending), bar, bridge, m, concurrency, lw, opts)
bar.clear()
if m != nil {
if err := m.Save(); err != nil {
fmt.Fprintf(stderr, "warning: could not save manifest: %v\n", err)
}
}
return exported, failed
}
func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
if originals {
return bridge.ExportOriginal(a.ID, outDir, index)
}
return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index)
}
func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) {
root := pa.root
if root == "" {
root = pa.path
}
stagingRoot := filepath.Join(root, ".photoscli-tmp")
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index)
}
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
if err != nil {
return photos.ExportResult{}, err
}
defer os.RemoveAll(stagingDir)
result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index)
if err != nil || result.Skipped {
return result, err
}
src := filepath.Join(stagingDir, result.Filename)
info, statErr := os.Stat(src)
if statErr != nil {
return result, nil
}
if info.Size() == 0 {
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
}
if err := os.MkdirAll(pa.path, 0755); err != nil {
return result, err
}
dst := filepath.Join(pa.path, result.Filename)
if err := renameFunc(src, dst); err != nil {
return result, err
}
return result, nil
}
func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) {
root := pa.root
if root == "" {
root = pa.path
}
stagingRoot := filepath.Join(root, ".photoscli-tmp")
if err := os.MkdirAll(stagingRoot, 0755); err != nil {
if originals {
return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex)
}
return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex)
}
stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*")
if err != nil {
return photos.ExportResult{}, err
}
defer os.RemoveAll(stagingDir)
var result photos.ExportResult
if originals {
result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex)
} else {
result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex)
}
if err != nil || result.Skipped {
return result, err
}
src := filepath.Join(stagingDir, result.Filename)
info, statErr := os.Stat(src)
if statErr != nil {
return result, nil
}
if info.Size() == 0 {
return result, fmt.Errorf("exported zero-byte file: %s", result.Filename)
}
if err := os.MkdirAll(pa.path, 0755); err != nil {
return result, err
}
dst := filepath.Join(pa.path, result.Filename)
if err := renameFunc(src, dst); err != nil {
return result, err
}
return result, nil
}
func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) {
var result photos.ExportResult
var err error
for attempt := 0; attempt <= retry; attempt++ {
result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index)
if err == nil {
return result, nil
}
time.Sleep(time.Duration(attempt+1) * 10 * time.Millisecond)
}
return result, err
}
func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) {
var result photos.ExportResult
var err error
for attempt := 0; attempt <= retry; attempt++ {
result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex)
if err == nil {
return result, nil
}
time.Sleep(time.Duration(attempt+1) * 10 * time.Millisecond)
}
return result, err
}
func parseSinceDate(s string) (time.Time, error) {
for _, layout := range []string{"2006-01-02", time.RFC3339} {
t, err := time.Parse(layout, s)
if err == nil {
return t, nil
}
}
return time.Time{}, fmt.Errorf("invalid date %q, use YYYY-MM-DD or RFC3339 format", s)
}
func filterBySince(assets []photos.Asset, since time.Time) []photos.Asset {
filtered := make([]photos.Asset, 0, len(assets))
for _, a := range assets {
if a.CreationDate != nil {
t, err := time.Parse(time.RFC3339, *a.CreationDate)
if err == nil && t.Before(since) {
continue
}
}
filtered = append(filtered, a)
}
return filtered
}
func filterVideos(assets []photos.Asset) ([]photos.Asset, int) {
filtered := make([]photos.Asset, 0, len(assets))
for _, a := range assets {
if a.MediaType != "video" && a.MediaType != "audio" {
filtered = append(filtered, a)
}
}
return filtered, len(filtered)
}
// Must stay in sync with sanitized_path_component in bridge/photokit_bridge.m
func sanitizePathComponent(name string) string {
s := strings.TrimSpace(name)
if s == "" {
s = "Untitled"
}
s = strings.ReplaceAll(s, "/", "_")
s = strings.ReplaceAll(s, "\\", "_")
return s
}
func flagVal(args []string, name string) string {
return flagValWithDefault(args, name, "")
}
func flagVals(args []string, name string) []string {
var vals []string
for i, arg := range args {
if arg == name && i+1 < len(args) {
vals = append(vals, args[i+1])
}
}
return vals
}
func isExcluded(name string, exclude []string) bool {
for _, pat := range exclude {
if name == pat {
return true
}
if m, _ := filepath.Match(pat, name); m {
return true
}
}
return false
}
func hasFlag(args []string, name string) bool {
for _, arg := range args {
if arg == name {
return true
}
}
v := configValue(name)
return v == "true" || v == "1" || v == "yes"
}
func printNode(w io.Writer, node photos.CollectionNode, depth int) {
for i := 0; i < depth; i++ {
fmt.Fprint(w, " ")
}
fmt.Fprintln(w, node.Name)
for _, child := range node.Children {
printNode(w, child, depth+1)
}
}
func countAlbums(nodes []photos.CollectionNode) int {
total := 0
for _, node := range nodes {
if node.Kind == "album" {
total++
}
total += countAlbums(node.Children)
}
return total
}
func flagValWithDefault(args []string, name, def string) string {
for i, arg := range args {
if arg == name && i+1 < len(args) {
return args[i+1]
}
}
if v := configValue(name); v != "" {
return v
}
return def
}
func configValue(flag string) string {
if !configLoaded {
configValues = loadConfigFile()
configLoaded = true
}
return configValues[strings.TrimPrefix(flag, "--")]
}
func loadConfigFile() map[string]string {
path := os.Getenv("PHOTOSCLI_CONFIG")
if path == "" {
home, _ := os.UserHomeDir()
path = filepath.Join(home, ".photoscli.toml")
}
data, err := os.ReadFile(path)
if err != nil {
return map[string]string{}
}
out := map[string]string{}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" || strings.HasPrefix(line, "#") || !strings.Contains(line, "=") {
continue
}
parts := strings.SplitN(line, "=", 2)
key := strings.TrimSpace(parts[0])
val := strings.Trim(strings.TrimSpace(parts[1]), `"'`)
out[key] = val
}
return out
}
func exportMode(originals bool) string {
if originals {
return "originals"
}
return "previews"
}
func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) {
opts := exportOptions{
dryRun: hasFlag(args, "--dry-run"),
onlyFavorites: hasFlag(args, "--only-favorites"),
media: flagValWithDefault(args, "--media", "photos"),
jsonOut: hasFlag(args, "--json"),
verify: hasFlag(args, "--verify"),
format: flagValWithDefault(args, "--format", "jpeg"),
sidecar: flagValWithDefault(args, "--sidecar", "none"),
checksum: flagValWithDefault(args, "--checksum", "none"),
xmpPrivacy: flagValWithDefault(args, "--xmp-privacy", "keep"),
xmpKeywords: flagValWithDefault(args, "--xmp-keywords", "album-path"),
xmpRating: flagValWithDefault(args, "--xmp-rating", "favorite"),
metadataOnly: hasFlag(args, "--metadata-only"),
reverseGeocode: hasFlag(args, "--reverse-geocode"),
dateTemplate: flagVal(args, "--date-template"),
}
if opts.media != "photos" && opts.media != "videos" && opts.media != "all" {
fmt.Fprintf(stderr, "error: --media must be photos, videos, or all, got %q\n", opts.media)
return opts, false
}
if opts.format != "jpeg" && opts.format != "heic" && opts.format != "png" {
fmt.Fprintf(stderr, "error: --format must be jpeg, heic, or png, got %q\n", opts.format)
return opts, false
}
if opts.sidecar != "none" && !sidecarEnabled(opts.sidecar, "xmp") && !sidecarEnabled(opts.sidecar, "json") {
fmt.Fprintf(stderr, "error: --sidecar must be none, xmp, json, or xmp,json, got %q\n", opts.sidecar)
return opts, false
}
if opts.sidecar != "none" {
for _, part := range strings.Split(opts.sidecar, ",") {
part = strings.TrimSpace(part)
if part != "xmp" && part != "json" {
fmt.Fprintf(stderr, "error: --sidecar must be none, xmp, json, or xmp,json, got %q\n", opts.sidecar)
return opts, false
}
}
}
if opts.checksum != "none" && opts.checksum != "sha256" {
fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", opts.checksum)
return opts, false
}
if opts.xmpPrivacy != "keep" && opts.xmpPrivacy != "strip-location" && opts.xmpPrivacy != "strip-address" {
fmt.Fprintf(stderr, "error: --xmp-privacy must be keep, strip-location, or strip-address, got %q\n", opts.xmpPrivacy)
return opts, false
}
if opts.xmpKeywords != "album-path" && opts.xmpKeywords != "album" && opts.xmpKeywords != "none" {
fmt.Fprintf(stderr, "error: --xmp-keywords must be album-path, album, or none, got %q\n", opts.xmpKeywords)
return opts, false
}
if opts.xmpRating != "favorite" && opts.xmpRating != "none" {
fmt.Fprintf(stderr, "error: --xmp-rating must be favorite or none, got %q\n", opts.xmpRating)
return opts, false
}
if opts.metadataOnly && opts.sidecar == "none" {
fmt.Fprintln(stderr, "error: --metadata-only requires --sidecar xmp, json, or xmp,json")
return opts, false
}
if v := flagVal(args, "--retry"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
fmt.Fprintf(stderr, "error: --retry must be a non-negative integer, got %q\n", v)
return opts, false
}
opts.retry = n
}
if v := flagVal(args, "--min-size"); v != "" {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil || n < 0 {
fmt.Fprintf(stderr, "error: --min-size must be a non-negative integer, got %q\n", v)
return opts, false
}
opts.minSize = n
}
if v := flagVal(args, "--max-size"); v != "" {
n, err := strconv.ParseInt(v, 10, 64)
if err != nil || n < 0 {
fmt.Fprintf(stderr, "error: --max-size must be a non-negative integer, got %q\n", v)
return opts, false
}
opts.maxSize = n
}
return opts, true
}
func applyAssetFilters(assets []photos.Asset, opts exportOptions) []photos.Asset {
filtered := make([]photos.Asset, 0, len(assets))
for _, a := range assets {
if opts.onlyFavorites && !a.IsFavorite {
continue
}
if opts.media == "photos" && (a.MediaType == "video" || a.MediaType == "audio") {
continue
}
if opts.media == "videos" && a.MediaType != "video" {
continue
}
est := int64(a.PixelWidth) * int64(a.PixelHeight)
if opts.minSize > 0 && est < opts.minSize {
continue
}
if opts.maxSize > 0 && est > opts.maxSize {
continue
}
filtered = append(filtered, a)
}
return filtered
}
func pathWithDateTemplate(base string, a photos.Asset, tmpl string) string {
if tmpl == "" || a.CreationDate == nil {
return base
}
t, err := time.Parse(time.RFC3339, *a.CreationDate)
if err != nil {
return base
}
repl := strings.NewReplacer("YYYY", t.Format("2006"), "MM", t.Format("01"), "DD", t.Format("02"))
return filepath.Join(base, repl.Replace(tmpl))
}
func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest.Entry, error) {
m, _ := manifest.Open(outDir, mf)
if err := m.OpenAppend(); err != nil {
return nil, err
}
defer m.Close()
reader := m.(manifest.EntryReader)
return reader.Entries(), nil
}
func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") }
type failureEntry struct {
ID string `json:"id"`
Filename string `json:"filename"`
Album string `json:"album"`
Path string `json:"path"`
Error string `json:"error"`
FailedAt int64 `json:"failed_at"`
Attempts int `json:"attempts"`
}
func loadFailures(dir string) map[string]failureEntry {
out := map[string]failureEntry{}
data, err := os.ReadFile(failuresPath(dir))
if err != nil {
return out
}
for _, line := range strings.Split(string(data), "\n") {
line = strings.TrimSpace(line)
if line == "" {
continue
}
var f failureEntry
if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" {
out[f.ID] = f
}
}
return out
}
func saveFailures(dir string, failures map[string]failureEntry) error {
if err := os.MkdirAll(dir, 0755); err != nil {
return err
}
f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644)
if err != nil {
return err
}
defer f.Close()
ids := make([]string, 0, len(failures))
for id := range failures {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
data, _ := json.Marshal(failures[id])
f.Write(data)
f.Write([]byte("\n"))
}
return nil
}
func appendFailure(dir string, pa pendingAsset, err error) {
root := pa.root
if root == "" {
root = dir
}
failures := loadFailures(root)
f := failures[pa.asset.ID]
f.ID = pa.asset.ID
f.Filename = pa.asset.Filename
f.Album = pa.album
f.Path = pa.path
f.Error = err.Error()
f.FailedAt = time.Now().Unix()
f.Attempts++
failures[pa.asset.ID] = f
_ = saveFailures(root, failures)
}
func writeJSONSummary(stdout io.Writer, s commandSummary) {
data, _ := json.Marshal(s)
fmt.Fprintln(stdout, string(data))
}
func cmdReport(args []string, stdout, stderr io.Writer) int {
outDir := flagVal(args, "--out")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
failures := 0
if data, err := os.ReadFile(failuresPath(outDir)); err == nil {
for _, line := range strings.Split(strings.TrimSpace(string(data)), "\n") {
if strings.TrimSpace(line) != "" {
failures++
}
}
}
fmt.Fprintf(stdout, "entries\t%d\nfailures\t%d\n", len(entries), failures)
return exitOK
}
func cmdDiff(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
albumID := flagVal(args, "--album-id")
outDir := flagVal(args, "--out")
if albumID == "" || outDir == "" {
fmt.Fprintln(stderr, "error: --album-id and --out are required")
return exitErr
}
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
if rc := mustAuth(stderr, bridge); rc != exitOK {
return rc
}
resolved, err := resolveAlbumID(bridge, albumID)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
assets, _, err := bridge.ListAssets(resolved)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
missing := 0
for _, a := range assets {
if _, ok := entries[a.ID]; !ok {
missing++
fmt.Fprintf(stdout, "%s\t%s\n", a.ID, a.Filename)
}
}
if missing > 0 {
return exitPartial
}
return exitOK
}
func cmdVerify(args []string, stdout, stderr io.Writer) int {
outDir := flagVal(args, "--out")
checkSidecar := hasFlag(args, "--sidecar")
strictSidecar := hasFlag(args, "--strict")
deep := hasFlag(args, "--deep")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
bad := 0
for id, e := range entries {
checkPath := e.Path
if checkPath == "" {
checkPath = e.Filename
}
if checkPath == "" {
continue
}
info, err := os.Stat(filepath.Join(outDir, checkPath))
if err != nil {
bad++
fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath)
continue
}
if info.Size() == 0 {
bad++
fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath)
continue
}
if e.Size > 0 && info.Size() != e.Size {
bad++
fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size())
}
if deep && e.Checksum != "" {
checksum, err := fileSHA256(filepath.Join(outDir, checkPath))
if err != nil {
bad++
fmt.Fprintf(stdout, "%s\t%s\tchecksum-unreadable\n", id, checkPath)
} else if checksum != e.Checksum {
bad++
fmt.Fprintf(stdout, "%s\t%s\tchecksum-mismatch\tmanifest=%s\tdisk=%s\n", id, checkPath, e.Checksum, checksum)
}
}
if checkSidecar {
bad += verifySidecar(stdout, outDir, id, checkPath, strictSidecar)
}
}
if bad > 0 {
return exitPartial
}
fmt.Fprintf(stdout, "verified\t%d\n", len(entries))
return exitOK
}
func verifySidecar(stdout io.Writer, outDir, id, checkPath string, strict bool) int {
xmpPath := sidecarPath(filepath.Join(outDir, checkPath))
rel, err := filepath.Rel(outDir, xmpPath)
if err != nil || strings.HasPrefix(rel, "..") {
rel = xmpPath
}
info, err := statFunc(xmpPath)
if err != nil {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-missing\n", id, rel)
return 1
}
if info.Size() == 0 {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-zero-byte\n", id, rel)
return 1
}
data, err := readFileFunc(xmpPath)
if err != nil {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-unreadable\n", id, rel)
return 1
}
needle := `photoscli:assetID="` + id + `"`
if !strings.Contains(string(data), needle) {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-asset-mismatch\n", id, rel)
return 1
}
if strict {
meta := inspectXMP(data)
if meta["xmpSchemaVersion"] != "2" {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-schema-missing\n", id, rel)
return 1
}
if meta["sidecarGenerator"] == "" {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-generator-missing\n", id, rel)
return 1
}
if meta["exportedFilename"] != filepath.Base(checkPath) {
fmt.Fprintf(stdout, "%s\t%s\tsidecar-filename-mismatch\n", id, rel)
return 1
}
}
return 0
}
func cmdDoctor(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
result := map[string]any{"photos_access": "ok"}
problems := 0
if rc := mustAuth(stderr, bridge); rc != exitOK {
result["photos_access"] = "denied"
problems++
}
outDir := flagVal(args, "--out")
if outDir != "" {
result["out"] = outDir
if info, err := statFunc(outDir); err != nil || !info.IsDir() {
result["backup_dir"] = "missing"
problems++
} else {
result["backup_dir"] = "ok"
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
result["manifest"] = "error"
problems++
} else {
result["manifest"] = string(mf)
result["entries"] = len(entries)
}
failures := loadFailures(outDir)
result["failures"] = len(failures)
if len(failures) > 0 {
problems++
}
}
}
if hasFlag(args, "--json") {
result["problems"] = problems
if err := json.NewEncoder(stdout).Encode(result); err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
} else {
keys := make([]string, 0, len(result))
for k := range result {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprintf(stdout, "%s\t%v\n", k, result[k])
}
fmt.Fprintf(stdout, "problems\t%d\n", problems)
}
if problems > 0 {
return exitPartial
}
return exitOK
}
func cmdCleanup(args []string, stdout, stderr io.Writer) int {
outDir := flagVal(args, "--out")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
keep := map[string]bool{
"downloads.jsonl": true,
"downloads.db": true,
"export.log": true,
"failures.jsonl": true,
}
for _, e := range entries {
checkPath := e.Path
if checkPath == "" {
checkPath = e.Filename
}
if checkPath == "" {
continue
}
clean := filepath.Clean(checkPath)
if clean == "." || strings.HasPrefix(clean, "..") || filepath.IsAbs(clean) {
continue
}
keep[clean] = true
keep[filepath.Clean(sidecarPath(clean))] = true
keep[filepath.Clean(jsonSidecarPath(clean))] = true
}
dryRun := hasFlag(args, "--dry-run")
removed := 0
skipped := 0
_ = filepath.WalkDir(outDir, func(path string, d fs.DirEntry, _ error) error {
if path == outDir {
return nil
}
rel, _ := filepath.Rel(outDir, path)
rel = filepath.Clean(rel)
if d.IsDir() {
if rel == ".photoscli" || strings.HasPrefix(rel, ".photoscli"+string(os.PathSeparator)) {
return filepath.SkipDir
}
return nil
}
if keep[rel] {
return nil
}
fmt.Fprintf(stdout, "%s\torphan\n", rel)
removed++
if !dryRun {
if err := removeFunc(path); err != nil {
skipped++
}
}
return nil
})
fmt.Fprintf(stdout, "removed\t%d\nskipped\t%d\n", removed, skipped)
return exitOK
}
func cmdManifest(args []string, stdout, stderr io.Writer) int {
if len(args) < 1 || args[0] != "repair" {
fmt.Fprintln(stderr, "error: expected manifest repair")
return exitErr
}
outDir := flagVal(args, "--out")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
checksumMode := flagValWithDefault(args, "--checksum", "none")
if checksumMode != "none" && checksumMode != "sha256" {
fmt.Fprintf(stderr, "error: --checksum must be none or sha256, got %q\n", checksumMode)
return exitErr
}
mf, err := reg.ParseFormat(flagValWithDefault(args, "--manifest", "jsonl"))
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
m, err := manifest.Open(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
defer m.Close()
reader := m.(manifest.EntryReader)
if !hasFlag(args, "--dry-run") {
if err := m.OpenAppend(); err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
}
repaired := 0
skipped := 0
for id, entry := range reader.Entries() {
checkPath := entry.Path
if checkPath == "" {
checkPath = entry.Filename
}
if checkPath == "" {
skipped++
continue
}
fullPath := filepath.Join(outDir, checkPath)
info, err := statFunc(fullPath)
if err != nil || info.IsDir() || info.Size() == 0 {
skipped++
continue
}
updated := entry
updated.ID = id
changed := false
if updated.Size != info.Size() {
updated.Size = info.Size()
changed = true
}
if checksumMode == "sha256" && updated.Checksum == "" {
checksum, err := fileSHA256(fullPath)
if err != nil {
skipped++
continue
}
updated.Checksum = checksum
changed = true
}
if !changed {
continue
}
repaired++
fmt.Fprintf(stdout, "%s\t%s\trepaired\n", id, checkPath)
if !hasFlag(args, "--dry-run") {
m.AddEntry(updated)
}
}
if !hasFlag(args, "--dry-run") {
if err := m.Save(); err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
}
fmt.Fprintf(stdout, "repaired\t%d\nskipped\t%d\n", repaired, skipped)
return exitOK
}
func cmdSidecar(args []string, stdout, stderr io.Writer) int {
if len(args) < 1 || args[0] != "inspect" {
fmt.Fprintln(stderr, "error: expected sidecar inspect <file.xmp>")
return exitErr
}
if len(args) < 2 {
fmt.Fprintln(stderr, "error: sidecar inspect requires <file.xmp>")
return exitErr
}
path := args[1]
data, err := readFileFunc(path)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
meta := inspectXMP(data)
if len(meta) == 0 {
fmt.Fprintln(stderr, "error: no photoscli metadata found")
return exitErr
}
if hasFlag(args[2:], "--json") {
if err := json.NewEncoder(stdout).Encode(meta); err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
return exitOK
}
keys := make([]string, 0, len(meta))
for k := range meta {
keys = append(keys, k)
}
sort.Strings(keys)
for _, k := range keys {
fmt.Fprintf(stdout, "%s\t%s\n", k, meta[k])
}
return exitOK
}
func inspectXMP(data []byte) map[string]string {
attrs := map[string]string{}
dec := xml.NewDecoder(bytes.NewReader(data))
for {
tok, err := dec.Token()
if err == io.EOF {
break
}
if err != nil {
return attrs
}
start, ok := tok.(xml.StartElement)
if !ok {
continue
}
for _, a := range start.Attr {
if a.Name.Space == "photoscli" || a.Name.Space == "https://gitea.k3s.k0.nu/tools/photocli/ns/1.0/" {
attrs[a.Name.Local] = a.Value
}
}
}
return attrs
}
func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
outDir := flagVal(args, "--out")
clearOnSuccess := hasFlag(args, "--clear-on-success")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
failures := loadFailures(outDir)
if len(failures) == 0 {
fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir))
return exitErr
}
var pending []pendingAsset
ids := make([]string, 0, len(failures))
for id := range failures {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
f := failures[id]
pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album})
}
bar := newProgressBar(stderr, 1)
done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{})
if clearOnSuccess && done > 0 {
for i := 0; i < done && i < len(pending); i++ {
delete(failures, pending[i].asset.ID)
}
_ = saveFailures(outDir, failures)
}
writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)})
if failed > 0 {
return exitPartial
}
return exitOK
}
func cmdFailures(args []string, stdout, stderr io.Writer) int {
if len(args) < 1 {
fmt.Fprintln(stderr, "error: failures requires list or clear")
return exitErr
}
outDir := flagVal(args, "--out")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
switch args[0] {
case "list":
failures := loadFailures(outDir)
ids := make([]string, 0, len(failures))
for id := range failures {
ids = append(ids, id)
}
sort.Strings(ids)
for _, id := range ids {
f := failures[id]
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts)
}
return exitOK
case "clear":
if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
return exitOK
default:
fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0])
return exitErr
}
}
func cmdStatus(args []string, stdout, stderr io.Writer) int {
outDir := flagVal(args, "--out")
if outDir == "" {
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
manifestFmt := flagValWithDefault(args, "--manifest", "jsonl")
mf, err := reg.ParseFormat(manifestFmt)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
entries, err := loadManifestEntries(outDir, mf)
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
failures := loadFailures(outDir)
if hasFlag(args, "--json") {
data, _ := json.Marshal(struct {
Manifest string `json:"manifest"`
Entries int `json:"entries"`
Failures int `json:"failures"`
}{manifestFmt, len(entries), len(failures)})
fmt.Fprintln(stdout, string(data))
return exitOK
}
fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures))
return exitOK
}
Reference in New Issue View Git Blame Copy Permalink