v0.8.2: add metadata-only sidecars
pipeline / test (push) Has been cancelled
pipeline / build (push) Has been cancelled

This commit is contained in:
Ein Anderssono
2026-06-15 01:48:32 +02:00
parent 9cd702628d
commit a51db37fdb
7 changed files with 266 additions and 16 deletions
+10
View File
@@ -2,6 +2,16 @@
This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page.
## v0.8.2
Metadata-only XMP refresh release.
- Add `--metadata-only` for manifest-based XMP sidecar generation without re-exporting media files.
- Support metadata-only refresh for both `export` and `backup-all` when used with `--sidecar xmp`.
- Require a manifest for metadata-only mode so existing media paths are resolved safely.
- Keep media files untouched while overwriting/regenerating generated XMP sidecars.
- Support `--reverse-geocode` during metadata-only sidecar refresh.
## v0.8.1
XMP standards and sidecar verification release.
+1 -1
View File
@@ -1,6 +1,6 @@
BINARY := ./bin/photoscli
MODULE := gitea.k3s.k0.nu/tools/photocli
VERSION := 0.8.1
VERSION := 0.8.2
RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos-arm64.zip
RELEASE_NOTES := RELEASE_NOTES.md
BRIDGE_DIR := bridge
+8
View File
@@ -21,6 +21,7 @@ For a practical step-by-step manual with recommended backup workflows, recovery
- Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`.
- Verification, reporting, and diff commands for backup integrity.
- Optional XMP sidecar verification with `verify --sidecar`.
- Metadata-only XMP refresh for manifest-backed exports with `--metadata-only`.
- Status command for quick backup summaries.
- Opt-in rich XMP sidecar metadata with `--sidecar xmp`.
- Optional Apple MapKit reverse geocoding for GPS assets on macOS 26+ with `--reverse-geocode`.
@@ -283,6 +284,7 @@ Common flags for `export` and `backup-all`:
- `--max-size <n>`: filter by estimated pixel count.
- `--format jpeg|heic|png`: preview format hint. Current bridge output is still the existing preview path; non-JPEG bridge output is future work.
- `--sidecar none|xmp`: write opt-in XMP metadata sidecars next to exported files.
- `--metadata-only`: with `--sidecar xmp`, refresh XMP sidecars for manifest-backed files without exporting media.
- `--reverse-geocode`: with `--sidecar xmp`, add cached Apple MapKit address metadata for GPS assets on macOS 26+.
- `--date-template <template>`: append date folders based on creation date, for example `YYYY/MM/DD`.
@@ -361,6 +363,12 @@ Verify generated sidecars with:
photoscli verify --out ./backup --sidecar
```
Refresh sidecars for files already present in a manifest-backed export without rewriting media files:
```bash
photoscli backup-all --out ./backup --sidecar xmp --metadata-only
```
## Failure Tracking
Failed exports are deduplicated by asset ID and stored in:
+8 -9
View File
@@ -1,20 +1,19 @@
# v0.8.1
# v0.8.2
This release improves XMP sidecar standards compatibility and adds sidecar verification.
This release adds metadata-only XMP sidecar refresh for existing manifest-backed exports.
## Highlights
- Add `photoscli:xmpSchemaVersion="2"` to generated XMP sidecars.
- Add standard XMP mappings for favorite rating, metadata date, Photoshop date created, and EXIF GPS coordinates.
- Add `dc:subject` keywords from album/folder context.
- Add sidecar generator and generated timestamp metadata.
- Add `verify --sidecar` for missing, zero-byte, unreadable, and asset-ID mismatched XMP sidecars.
- Keep Vision/Core ML people, animal, object, and scene analysis out of this release.
- Add `--metadata-only` for `export` and `backup-all`.
- Use manifest paths to find existing media files and write or refresh `.xmp` sidecars without re-exporting media.
- Require `--sidecar xmp` and an enabled manifest for metadata-only mode.
- Leave media files untouched; only generated XMP sidecars are written.
- Support `--reverse-geocode` during metadata-only refresh.
## Assets
- `photoscli`: Apple Silicon macOS binary (`darwin/arm64`).
- `photoscli-0.8.1-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
- `photoscli-0.8.2-macos-arm64.zip`: Apple Silicon binary plus README, USERGUIDE, and CHANGELOG.
- `USERGUIDE.md`: standalone user guide.
Intel Macs are not currently a supported release target.
+8
View File
@@ -573,6 +573,14 @@ Verify sidecars after an export:
This reports missing, zero-byte, unreadable, or asset-ID mismatched `.xmp` files.
Refresh metadata only for an existing manifest-backed backup:
```bash
./bin/photoscli backup-all --out ./PhotosBackup --sidecar xmp --metadata-only
```
Metadata-only mode does not re-export media files. It uses manifest paths to find existing files and rewrites generated `.xmp` sidecars next to them. It requires `--sidecar xmp` and an enabled manifest.
If you explicitly request `--sidecar xmp` and the XMP file cannot be written, the asset is counted as failed.
## Configuration File
+110 -5
View File
@@ -41,6 +41,7 @@ type exportOptions struct {
verify bool
format string
sidecar string
metadataOnly bool
reverseGeocode bool
minSize int64
maxSize int64
@@ -220,6 +221,10 @@ COMMON EXPORT FLAGS
Write opt-in XMP sidecar metadata next to each exported file. Default:
none. If XMP writing fails, the asset is counted as failed.
--metadata-only
With --sidecar xmp, write or refresh XMP sidecars for files already in
the manifest without exporting media files. Requires a manifest.
--reverse-geocode
With --sidecar xmp, use Apple MapKit on macOS 26+ to add address metadata
for assets with GPS coordinates. Results are cached under .photoscli.
@@ -444,6 +449,10 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
if opts.metadataOnly && noManifest {
fmt.Fprintln(stderr, "error: --metadata-only requires a manifest")
return exitErr
}
mf, mfErr := manifest.ParseFormat(manifestFmt)
if mfErr != nil {
@@ -546,8 +555,17 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
fmt.Fprintf(stderr, "dry-run: %d assets would be exported to %s\n", total, outDir)
return exitOK
}
var exported, failed int
if opts.metadataOnly {
m, _ := manifest.Open(outDir, mf)
defer m.Close()
entries := manifestEntries(m)
fmt.Fprintf(stderr, "writing metadata for %d assets to %s...\n", total, outDir)
exported, failed = metadataOnlyAssets(assets, outDir, originals, "", entries, opts, bridge)
} else {
fmt.Fprintf(stderr, "exporting %d assets (%s) to %s...\n", total, exportMode(originals), outDir)
exported, failed := exportAssets(assets, outDir, size, quality, concurrency, originals, total, stderr, bridge, "", noManifest, mf, enableLog, opts)
exported, failed = exportAssets(assets, outDir, size, quality, concurrency, originals, total, stderr, bridge, "", noManifest, mf, enableLog, opts)
}
if opts.jsonOut {
writeJSONSummary(stdout, commandSummary{Exported: exported, Failed: failed, Total: total})
}
@@ -562,7 +580,9 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
return exitErr
}
if originals {
if opts.metadataOnly {
fmt.Fprintf(stderr, "\nwrote %d metadata sidecars to %s", exported, outDir)
} else if originals {
fmt.Fprintf(stderr, "\nexported %d original files to %s", exported, outDir)
} else {
fmt.Fprintf(stderr, "\nexported %d photos to %s", exported, outDir)
@@ -602,6 +622,10 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
fmt.Fprintln(stderr, "error: --out is required")
return exitErr
}
if opts.metadataOnly && noManifest {
fmt.Fprintln(stderr, "error: --metadata-only requires a manifest")
return exitErr
}
mf, mfErr := manifest.ParseFormat(manifestFmt)
if mfErr != nil {
@@ -675,13 +699,25 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
fmt.Fprintf(stderr, "dry-run: %d assets would be exported (%d skipped)\n", len(pending), skipped)
return exitOK
}
totalAssets, failed, err := backupTree(nodes, outDir, size, quality, concurrency, originals, skipVideos, stderr, bridge, noManifest, mf, sortNewest, excludeAlbums, sinceTime, enableLog, opts)
var totalAssets, failed int
if opts.metadataOnly {
m, _ := manifest.Open(outDir, mf)
entries := manifestEntries(m)
m.Close()
pending, skipped := collectPendingAssets(nodes, outDir, bridge, skipVideos, originals, nil, nil, sortNewest, excludeAlbums, sinceTime, opts)
fmt.Fprintf(stderr, " indexed %d metadata entries (%d skipped), writing sidecars to %s...\n", len(pending), skipped, outDir)
totalAssets, failed = metadataOnlyPending(pending, entries, originals, opts, bridge)
} else {
totalAssets, failed, err = backupTree(nodes, outDir, size, quality, concurrency, originals, skipVideos, stderr, bridge, noManifest, mf, sortNewest, excludeAlbums, sinceTime, enableLog, opts)
}
if err != nil {
fmt.Fprintf(stderr, "error: %v\n", err)
return exitErr
}
if originals {
if opts.metadataOnly {
fmt.Fprintf(stderr, "\nwrote %d metadata sidecars across %d albums to %s", totalAssets, albumCount, outDir)
} else if originals {
fmt.Fprintf(stderr, "\nexported %d original files across %d albums to %s", totalAssets, albumCount, outDir)
} else {
fmt.Fprintf(stderr, "\nexported %d preview files across %d albums to %s", totalAssets, albumCount, outDir)
@@ -1086,6 +1122,46 @@ func writeSidecarIfNeeded(pa pendingAsset, result photos.ExportResult, originals
})
}
func writeMetadataOnlySidecar(pa pendingAsset, entry manifest.Entry, originals bool, opts exportOptions, cache *geocodeCache, bridge photos.Bridge) error {
checkPath := entry.Path
if checkPath == "" {
checkPath = entry.Filename
}
if checkPath == "" {
return fmt.Errorf("manifest entry has no path")
}
root := pa.root
if root == "" {
root = pa.path
}
fullPath := filepath.Join(root, checkPath)
info, err := statFunc(fullPath)
if err != nil {
return fmt.Errorf("metadata target missing: %s", checkPath)
}
if info.Size() == 0 {
return fmt.Errorf("metadata target zero-byte: %s", checkPath)
}
pa.path = filepath.Dir(fullPath)
result := photos.ExportResult{Filename: filepath.Base(fullPath), Size: info.Size(), Cloud: entry.Cloud}
return writeSidecarIfNeeded(pa, result, originals, opts, cache, bridge)
}
func manifestEntries(m manifest.Manifest) map[string]manifest.Entry {
if r, ok := m.(manifest.EntryReader); ok {
return r.Entries()
}
return nil
}
func metadataOnlyAssets(assets []photos.Asset, outDir string, originals bool, dirPrefix string, entries map[string]manifest.Entry, opts exportOptions, bridge photos.Bridge) (int, int) {
pending := make([]pendingAsset, 0, len(assets))
for _, a := range assets {
pending = append(pending, pendingAsset{asset: a, root: outDir, path: outDir, album: dirPrefix})
}
return metadataOnlyPending(pending, entries, originals, opts, bridge)
}
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) {
var items []pendingAsset
var skipped int
@@ -1123,6 +1199,30 @@ func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge p
return items, skipped
}
func metadataOnlyPending(pending []pendingAsset, entries map[string]manifest.Entry, originals bool, opts exportOptions, bridge photos.Bridge) (int, int) {
var cache *geocodeCache
if opts.reverseGeocode && len(pending) > 0 {
root := pending[0].root
if root == "" {
root = pending[0].path
}
cache = newGeocodeCache(root)
}
written, failed := 0, 0
for _, pa := range pending {
entry, ok := entries[pa.asset.ID]
if !ok {
continue
}
if err := writeMetadataOnlySidecar(pa, entry, originals, opts, cache, bridge); err != nil {
failed++
continue
}
written++
}
return written, failed
}
func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, items *[]pendingAsset, skipped *int, onProgress func(collectProgress), m manifest.Manifest, exclude []string, opts exportOptions) {
names := make(map[string]int)
for _, node := range nodes {
@@ -1160,7 +1260,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br
}
assets = applyAssetFilters(assets, opts)
for _, a := range assets {
if m != nil && m.Has(a.ID) {
if m != nil && m.Has(a.ID) && !opts.metadataOnly {
*skipped++
continue
}
@@ -1767,6 +1867,7 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) {
verify: hasFlag(args, "--verify"),
format: flagValWithDefault(args, "--format", "jpeg"),
sidecar: flagValWithDefault(args, "--sidecar", "none"),
metadataOnly: hasFlag(args, "--metadata-only"),
reverseGeocode: hasFlag(args, "--reverse-geocode"),
dateTemplate: flagVal(args, "--date-template"),
}
@@ -1782,6 +1883,10 @@ func parseExportOptions(args []string, stderr io.Writer) (exportOptions, bool) {
fmt.Fprintf(stderr, "error: --sidecar must be none or xmp, got %q\n", opts.sidecar)
return opts, false
}
if opts.metadataOnly && opts.sidecar != "xmp" {
fmt.Fprintln(stderr, "error: --metadata-only requires --sidecar xmp")
return opts, false
}
if v := flagVal(args, "--retry"); v != "" {
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
+120
View File
@@ -35,6 +35,15 @@ type mockBridge struct {
cancelled atomic.Bool
}
type noEntryManifest struct{}
func (noEntryManifest) Has(string) bool { return false }
func (noEntryManifest) Add(string, string, int64, string) {}
func (noEntryManifest) AddEntry(manifest.Entry) {}
func (noEntryManifest) Save() error { return nil }
func (noEntryManifest) Close() {}
func (noEntryManifest) OpenAppend() error { return nil }
func (m *mockBridge) RequestAccess() error { return m.accessErr }
func (m *mockBridge) ListAlbums() ([]photos.Album, error) { return m.albums, m.albumsErr }
func (m *mockBridge) ListAssets(albumID string) ([]photos.Asset, int, error) {
@@ -4505,6 +4514,117 @@ func TestSidecarConfigAndErrors(t *testing.T) {
}
}
func TestMetadataOnlyExport(t *testing.T) {
dir := t.TempDir()
m := manifest.LoadJSONL(dir)
if err := m.OpenAppend(); err != nil {
t.Fatal(err)
}
m.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "photo.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
m.Close()
if err := os.WriteFile(filepath.Join(dir, "photo.jpg"), []byte("data"), 0644); err != nil {
t.Fatal(err)
}
b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "orig.heic", MediaType: "image", IsFavorite: true}}}
b.exportPreviewFn = func(string, string, int, int, int) (photos.ExportResult, error) {
t.Fatal("metadata-only must not export media")
return photos.ExportResult{}, nil
}
out, stderr, rc := runWith([]string{"export", "--album-id", "x", "--out", dir, "--sidecar", "xmp", "--metadata-only"}, b)
if rc != exitOK || out != "" || !strings.Contains(stderr, "wrote 1 metadata sidecars") {
t.Fatalf("metadata-only rc=%d out=%q stderr=%q", rc, out, stderr)
}
data, err := os.ReadFile(filepath.Join(dir, "photo.xmp"))
if err != nil {
t.Fatal(err)
}
if !strings.Contains(string(data), "photoscli:assetID=\"x1\"") || !strings.Contains(string(data), "xmp:Rating=\"5\"") {
t.Fatalf("unexpected metadata sidecar: %s", string(data))
}
}
func TestMetadataOnlyExportErrors(t *testing.T) {
dir := t.TempDir()
b := &mockBridge{assets: []photos.Asset{{ID: "x1", Filename: "photo.jpg"}}}
_, stderr, rc := runWith([]string{"export", "--album-id", "x", "--out", dir, "--metadata-only"}, b)
if rc != exitErr || !strings.Contains(stderr, "--metadata-only requires --sidecar xmp") {
t.Fatalf("expected sidecar requirement rc=%d stderr=%q", rc, stderr)
}
_, stderr, rc = runWith([]string{"export", "--album-id", "x", "--out", dir, "--sidecar", "xmp", "--metadata-only", "--no-manifest"}, b)
if rc != exitErr || !strings.Contains(stderr, "requires a manifest") {
t.Fatalf("expected manifest requirement rc=%d stderr=%q", rc, stderr)
}
_, stderr, rc = runWith([]string{"backup-all", "--out", dir, "--sidecar", "xmp", "--metadata-only", "--no-manifest"}, &mockBridge{})
if rc != exitErr || !strings.Contains(stderr, "requires a manifest") {
t.Fatalf("expected backup manifest requirement rc=%d stderr=%q", rc, stderr)
}
}
func TestMetadataOnlyHelperBranches(t *testing.T) {
dir := t.TempDir()
pa := pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}, path: dir}
if err := writeMetadataOnlySidecar(pa, manifest.Entry{ID: "x1"}, false, exportOptions{sidecar: "xmp"}, nil, &mockBridge{}); err == nil || !strings.Contains(err.Error(), "no path") {
t.Fatalf("expected no path error, got %v", err)
}
if err := writeMetadataOnlySidecar(pa, manifest.Entry{ID: "x1", Path: "missing.jpg"}, false, exportOptions{sidecar: "xmp"}, nil, &mockBridge{}); err == nil || !strings.Contains(err.Error(), "missing") {
t.Fatalf("expected missing error, got %v", err)
}
if err := os.WriteFile(filepath.Join(dir, "zero.jpg"), nil, 0644); err != nil {
t.Fatal(err)
}
if err := writeMetadataOnlySidecar(pa, manifest.Entry{ID: "x1", Path: "zero.jpg"}, false, exportOptions{sidecar: "xmp"}, nil, &mockBridge{}); err == nil || !strings.Contains(err.Error(), "zero-byte") {
t.Fatalf("expected zero-byte error, got %v", err)
}
if entries := manifestEntries(noEntryManifest{}); entries != nil {
t.Fatalf("expected nil entries, got %#v", entries)
}
written, failed := metadataOnlyPending([]pendingAsset{{asset: photos.Asset{ID: "x1"}, path: dir}}, map[string]manifest.Entry{"x1": {ID: "x1", Path: "missing.jpg"}}, false, exportOptions{sidecar: "xmp"}, &mockBridge{})
if written != 0 || failed != 1 {
t.Fatalf("expected failed metadata pending, written=%d failed=%d", written, failed)
}
if err := os.WriteFile(filepath.Join(dir, "photo.jpg"), []byte("data"), 0644); err != nil {
t.Fatal(err)
}
written, failed = metadataOnlyPending([]pendingAsset{{asset: photos.Asset{ID: "x2", Location: &photos.AssetLocation{Latitude: 1, Longitude: 2}}, path: dir}}, map[string]manifest.Entry{"x2": {ID: "x2", Path: "photo.jpg"}}, false, exportOptions{sidecar: "xmp", reverseGeocode: true}, &mockBridge{reverseGeocodeFn: func(float64, float64) (photos.Placemark, error) {
return photos.Placemark{Country: "Sweden"}, nil
}})
if written != 1 || failed != 0 {
t.Fatalf("expected reverse geocode metadata success, written=%d failed=%d", written, failed)
}
}
func TestMetadataOnlyBackupAll(t *testing.T) {
dir := t.TempDir()
m := manifest.LoadJSONL(dir)
if err := m.OpenAppend(); err != nil {
t.Fatal(err)
}
m.AddEntry(manifest.Entry{ID: "x1", Filename: "photo.jpg", Path: "Album/photo.jpg", Size: 4, Cloud: "local", Exported: time.Now().Unix()})
m.Close()
albumDir := filepath.Join(dir, "Album")
if err := os.Mkdir(albumDir, 0755); err != nil {
t.Fatal(err)
}
if err := os.WriteFile(filepath.Join(albumDir, "photo.jpg"), []byte("data"), 0644); err != nil {
t.Fatal(err)
}
b := &mockBridge{
tree: []photos.CollectionNode{{ID: "a1", Name: "Album", Kind: "album"}},
assetsByAlbum: map[string][]photos.Asset{"a1": {{ID: "x1", Filename: "orig.heic", MediaType: "image"}, {ID: "x2", Filename: "missing.heic", MediaType: "image"}}},
}
b.exportPreviewFn = func(string, string, int, int, int) (photos.ExportResult, error) {
t.Fatal("metadata-only backup-all must not export media")
return photos.ExportResult{}, nil
}
_, stderr, rc := runWith([]string{"backup-all", "--out", dir, "--sidecar", "xmp", "--metadata-only"}, b)
if rc != exitOK || !strings.Contains(stderr, "wrote 1 metadata sidecars") {
t.Fatalf("metadata-only backup rc=%d stderr=%q", rc, stderr)
}
if _, err := os.Stat(filepath.Join(albumDir, "photo.xmp")); err != nil {
t.Fatalf("expected metadata-only sidecar: %v", err)
}
}
func TestSidecarAdditionalBranches(t *testing.T) {
dir := t.TempDir()
oldCreateTemp := createTempFunc