From 05188e54515c5005eb3957a769092ace5bbaa508 Mon Sep 17 00:00:00 2001 From: Ein Anderssono Date: Mon, 15 Jun 2026 00:34:32 +0200 Subject: [PATCH] v0.6.0: strengthen backup integrity --- CHANGELOG.md | 121 ++++++-- Makefile | 5 +- README.md | 30 +- RELEASE_NOTES.md | 27 ++ USERGUIDE.md | 19 +- cmd/photoscli/main.go | 359 ++++++++++++++++++---- cmd/photoscli/main_test.go | 271 +++++++++++++++- internal/manifest/entry_test.go | 44 +++ internal/manifest/jsonl.go | 19 +- internal/manifest/manifest.go | 11 + internal/manifest/open.go | 6 +- internal/manifest/sqlite.go | 21 +- internal/manifest/sqlite_internal_test.go | 4 +- 13 files changed, 840 insertions(+), 97 deletions(-) create mode 100644 RELEASE_NOTES.md create mode 100644 internal/manifest/entry_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 2c55ec2..646087b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,48 +1,123 @@ # Changelog +This changelog is maintained from git history plus the published Gitea release series. Future releases should update this file and publish matching release notes on the release page. + +## v0.6.0 + +Backup integrity and recovery release. + +- Add manifest-relative paths for exported files so tree backups can be verified accurately. +- Add SQLite manifest migration for the new `path` column. +- Upgrade `verify` to report missing files, zero-byte files, and size mismatches by default. +- Add atomic staging export writes with per-asset temporary directories before final rename. +- Deduplicate `failures.jsonl` entries by asset ID and track attempts plus latest failure time. +- Add `failures list` and `failures clear` commands. +- Add `retry-failed --clear-on-success`. +- Add `status` command with text and JSON output. +- Update release workflow to publish release notes from `RELEASE_NOTES.md`. +- Keep `CHANGELOG.md` as the canonical release history. + ## v0.5.0 -- Add JSONL and SQLite manifests with resumable skip tracking -- Add structured export logging to export.log or SQLite logs table -- Add --dry-run, --retry, --only-favorites, --media, --json, --verify, --date-template, and size filters -- Add report, diff, verify, and retry-failed commands -- Add failure tracking in failures.jsonl -- Add configurable preview quality and export concurrency -- Add album exclusion, since-date filtering, album collision handling, and config-file defaults -- Expand README and CLI help output -- Maintain 100% test coverage in the release pipeline +Manifest, filtering, logging, and documentation release. + +- Add JSONL and SQLite manifests with resumable skip tracking. +- Add automatic conversion between JSONL and SQLite manifests. +- Add structured export logging to `export.log` or SQLite `logs` table. +- Add explicit exit codes: success, error, partial failure, and access denied. +- Add `--quality` for JPEG preview compression. +- Add `--concurrency` with progress slot cap. +- Add `--exclude-album` with exact/glob matching. +- Add `--since` date filtering. +- Add `--dry-run`. +- Add `--retry`. +- Add `--only-favorites`. +- Add `--media photos|videos|all`. +- Add `--json` command summaries. +- Add `--verify` integration. +- Add `--format jpeg|heic|png` validation/hint. +- Add `--min-size` and `--max-size` estimated pixel-count filters. +- Add `--date-template` folder layout support. +- Add `report`, `diff`, `verify`, and `retry-failed` commands. +- Add failure tracking in `failures.jsonl`. +- Add config-file defaults via `~/.photoscli.toml` or `PHOTOSCLI_CONFIG`. +- Add album name collision handling for duplicate sibling album names. +- Expand `README.md` and verbose CLI help. +- Add 100% test coverage for the CLI and manifest/photos packages. + +## v0.4.1 + +Documentation and release asset follow-up after v0.5.0. + +- Add `USERGUIDE.md` as a practical end-user manual. +- Link the user guide from `README.md`. +- Add release zip packaging with binary, README, USERGUIDE, and CHANGELOG. +- Attach `USERGUIDE.md` and the macOS zip package to the `v0.5.0` release. ## v0.4.0 -- Scroll log for completed exports (copied, downloaded, skipped, failed) -- Worker status lines with live download progress bars for cloud files -- Color gradient progress bar (red to yellow to green) -- Disk skip: files already on disk are skipped during backup-all -- Parallel export with 3 workers for backup-all +Progress and backup-all usability release. + +- Add scroll log for completed exports: copied, downloaded, skipped, and failed. +- Add worker status lines with live download progress bars for cloud files. +- Add color-gradient progress bars from red to yellow to green. +- Add disk skip for files already present during `backup-all`. +- Add parallel export with multiple workers for `backup-all`. +- Increase progress slot support in the bridge. +- Improve progress rendering for large export sessions. + +## v0.3.x + +Published on Gitea as `v0.3.0` through `v0.3.5`. These releases are not represented by local git tags in the current clone, so the exact per-release commit mapping is unavailable here. + +Known purpose of this series: + +- Iterative release packaging and Gitea release workflow improvements. +- Continued progress display and backup/export polish between `v0.2.5` and `v0.4.0`. +- Preparation for the larger `v0.4.0` progress/backup release. + +## v0.2.6 + +Published on Gitea but not represented by a local git tag in the current clone. + +- Post-`v0.2.5` maintenance release in the progress/export series. ## v0.2.5 -- Unicode progress bar with cloud download speed display +- Add Unicode progress bar rendering. +- Add cloud download speed display. ## v0.2.4 -- Stop export loop on Ctrl+C instead of flooding failures +- Stop export loop on Ctrl+C instead of flooding failures. +- Improve graceful cancellation behavior after interrupt. ## v0.2.3 -- Fix export write failures and Ctrl+C cancellation +- Fix export write failures. +- Fix Ctrl+C cancellation behavior. + +## v0.2.2 + +No local tag is present for this version in the current clone. ## v0.2.1 -- Add status messages during export -- Fix parallel export progress display +- Add status messages during export. +- Fix parallel export progress display. ## v0.2.0 -- Semaphore timeouts, error logging, dead code removal -- Parallel exports +- Add semaphore timeouts. +- Add export error logging. +- Remove dead code. +- Add parallel exports. ## Pre-release -- Initial applephotos CLI with progress, cloud status, per-asset export -- Renamed from applephotos to photoscli +- Initial `applephotos` CLI with progress, cloud status, and per-asset export. +- Add `.gitignore` and remove build artifacts from tracking. +- Rename `applephotos` to `photoscli`. +- Update module path to `gitea.k3s.k0.nu/tools/photocli`. +- Add version flag. +- Add Gitea release targets and release pipeline. diff --git a/Makefile b/Makefile index 1839fc5..5559319 100644 --- a/Makefile +++ b/Makefile @@ -1,7 +1,8 @@ BINARY := ./bin/photoscli MODULE := gitea.k3s.k0.nu/tools/photocli -VERSION := 0.5.0 +VERSION := 0.6.0 RELEASE_ZIP := ./bin/photoscli-$(VERSION)-macos.zip +RELEASE_NOTES := RELEASE_NOTES.md BRIDGE_DIR := bridge LDFLAGS := -X main.version=$(VERSION) OBJ := $(BRIDGE_DIR)/photokit_bridge.o @@ -54,7 +55,7 @@ tag: git push origin v$(VERSION) release: package - tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --asset $(BINARY) --asset USERGUIDE.md --asset $(RELEASE_ZIP) + tea releases create --repo $(GITEA_REPO) --tag v$(VERSION) --title "v$(VERSION)" --note-file $(RELEASE_NOTES) --asset $(BINARY) --asset USERGUIDE.md --asset $(RELEASE_ZIP) pipeline: clean test build @echo "--- verifying version ---" diff --git a/README.md b/README.md index d3577d2..de1a99d 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,9 @@ For a practical step-by-step manual with recommended backup workflows, recovery - Dry-run mode for planning large backups. - Filters for favorites, media type, date, estimated size, and excluded albums. - Failure tracking with `failures.jsonl` and `retry-failed`. +- Deduplicated failure management with `failures list`, `failures clear`, and `retry-failed --clear-on-success`. - Verification, reporting, and diff commands for backup integrity. +- Status command for quick backup summaries. - Script-friendly exit codes and optional JSON summaries. - 100% test coverage for the Go CLI and parsing layers. @@ -219,6 +221,26 @@ photoscli retry-failed --out ./backup This is useful after transient iCloud or network failures. +Use `--clear-on-success` to remove successfully retried assets from `failures.jsonl`. + +### `failures` + +Lists or clears deduplicated failure records. + +```bash +photoscli failures list --out ./backup +photoscli failures clear --out ./backup +``` + +### `status` + +Shows a quick backup summary. + +```bash +photoscli status --out ./backup --manifest sqlite +photoscli status --out ./backup --manifest sqlite --json +``` + ## Export Flags Common flags for `export` and `backup-all`: @@ -297,7 +319,7 @@ Logged events include session start/end, completed exports, skipped assets, and ## Failure Tracking -Failed exports are appended to: +Failed exports are deduplicated by asset ID and stored in: ```text failures.jsonl @@ -309,6 +331,12 @@ Retry them with: photoscli retry-failed --out ./backup ``` +Clear successful retries automatically: + +```bash +photoscli retry-failed --out ./backup --clear-on-success +``` + Use `--retry N` during normal exports to handle transient iCloud or network failures automatically. ## Configuration File diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md new file mode 100644 index 0000000..bf4f179 --- /dev/null +++ b/RELEASE_NOTES.md @@ -0,0 +1,27 @@ +# v0.6.0 + +This release focuses on backup integrity, recovery workflows, and clearer operational status. + +## Highlights + +- Manifest entries now include relative paths, so tree backups can be verified accurately. +- SQLite manifests migrate automatically to include the new path column. +- `verify` now detects missing files, zero-byte files, and size mismatches by default. +- Exports use per-asset staging directories and rename completed files into place when possible. +- Failure tracking is deduplicated by asset ID and records attempts and latest error details. +- New `failures list` and `failures clear` commands. +- `retry-failed --clear-on-success` removes successfully retried failures. +- New `status` command with text and JSON output. +- Release workflow now publishes these release notes to the release page. + +## Recommended Upgrade Notes + +- Existing JSONL manifests continue to load; entries without `path` fall back to `filename`. +- Existing SQLite manifests are migrated with `ALTER TABLE downloads ADD COLUMN path ...` during open. +- `verify` is stricter now and may report problems that older versions ignored. + +## Assets + +- `photoscli`: macOS binary. +- `photoscli-0.6.0-macos.zip`: binary plus README, USERGUIDE, and CHANGELOG. +- `USERGUIDE.md`: standalone user guide. diff --git a/USERGUIDE.md b/USERGUIDE.md index 774843a..6fffc5f 100644 --- a/USERGUIDE.md +++ b/USERGUIDE.md @@ -20,6 +20,8 @@ It is especially useful when you want to: - Keep a manifest of already-exported assets. - Retry transient iCloud failures. - Verify that files referenced by a manifest exist on disk. +- Detect missing, zero-byte, and size-mismatched manifest files. +- Inspect or clear deduplicated failure records. - Script Photos exports with stable exit codes. It is not intended to replace Apple Photos, iCloud Photos, or Time Machine. Think of it as an additional file-based export and backup tool. @@ -339,6 +341,8 @@ Failed exports are written to: failures.jsonl ``` +Failure records are deduplicated by asset ID. Repeated failures update the existing record and increment the attempt count. + For transient failures, first use retries during normal export: ```bash @@ -351,6 +355,19 @@ If failures remain, retry later: ./bin/photoscli retry-failed --out ./PhotosBackup ``` +Clear successful retry records automatically: + +```bash +./bin/photoscli retry-failed --out ./PhotosBackup --clear-on-success +``` + +List or clear failure records: + +```bash +./bin/photoscli failures list --out ./PhotosBackup +./bin/photoscli failures clear --out ./PhotosBackup +``` + Typical reasons for failures: - iCloud asset not currently downloadable. @@ -374,7 +391,7 @@ Run verification after large backups: ./bin/photoscli verify --out ./PhotosBackup --manifest sqlite ``` -Verification checks that manifest entries point to files on disk. +Verification checks that manifest entries point to files on disk. It also reports zero-byte files and size mismatches when the manifest has a recorded size. It exits with: diff --git a/cmd/photoscli/main.go b/cmd/photoscli/main.go index b9183e6..f87e60c 100644 --- a/cmd/photoscli/main.go +++ b/cmd/photoscli/main.go @@ -21,6 +21,10 @@ var ( exportTimeout = 2 * time.Second configValues map[string]string configLoaded bool + mkdirTempFunc = os.MkdirTemp + renameFunc = os.Rename + openFileFunc = os.OpenFile + removeFunc = os.Remove ) type exportOptions struct { @@ -75,6 +79,10 @@ func run(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int { return cmdVerify(args[1:], stdout, stderr) case "retry-failed": return cmdRetryFailed(args[1:], stdout, stderr, bridge) + case "failures": + return cmdFailures(args[1:], stdout, stderr) + case "status": + return cmdStatus(args[1:], stdout, stderr) case "version", "--version", "-v": fmt.Fprintln(stdout, version) return exitOK @@ -110,6 +118,10 @@ USAGE photoscli diff --album-id --out [--manifest jsonl|sqlite] photoscli verify --out [--manifest jsonl|sqlite] photoscli retry-failed --out + photoscli retry-failed --out --clear-on-success + photoscli failures list --out + photoscli failures clear --out + photoscli status --out [--json] photoscli version photoscli help @@ -149,6 +161,12 @@ COMMANDS retry-failed --out Retry assets previously written to failures.jsonl. + failures list|clear --out + List or clear deduplicated failure records. + + status --out [--manifest jsonl|sqlite] [--json] + Show manifest type, entry count, and failure count for a backup. + COMMON EXPORT FLAGS --out Destination directory. Required for export, backup-all, report, diff, @@ -240,7 +258,9 @@ LOGGING AND FAILURES table in downloads.db. failures.jsonl - Failed exports are appended here and can be retried with retry-failed. + Failed exports are deduplicated by asset ID and can be retried with + retry-failed. Use retry-failed --clear-on-success to remove successful + retries from the failure list. CONFIGURATION Defaults can be read from ~/.photoscli.toml or from PHOTOSCLI_CONFIG. @@ -665,6 +685,7 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge) type pendingAsset struct { asset photos.Asset + root string path string album string } @@ -691,6 +712,22 @@ func logEntry(event, level, assetID, album, filename, cloud string, size int64, } } +func addManifestEntry(m manifest.Manifest, pa pendingAsset, result photos.ExportResult) { + if m == nil { + return + } + root := pa.root + if root == "" { + root = pa.path + } + fullPath := filepath.Join(pa.path, result.Filename) + relPath, err := filepath.Rel(root, fullPath) + if err != nil || strings.HasPrefix(relPath, "..") { + relPath = result.Filename + } + m.AddEntry(manifest.NewEntry(pa.asset.ID, result.Filename, relPath, result.Size, result.Cloud)) +} + func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, onProgress func(collectProgress), m manifest.Manifest, sortNewest bool, exclude []string, since time.Time, opts exportOptions) ([]pendingAsset, int) { var items []pendingAsset var skipped int @@ -769,7 +806,7 @@ func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Br *skipped++ continue } - *items = append(*items, pendingAsset{asset: a, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name}) + *items = append(*items, pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(path, a, opts.dateTemplate), album: node.Name}) } if onProgress != nil { onProgress(collectProgress{pending: len(*items), skipped: *skipped, album: node.Name}) @@ -850,7 +887,7 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin bar.setAlbum(pa.album, 0, 0) bar.draw() start := time.Now() - result, exportErr := exportOneWithRetry(bridge, pa.asset, pa.path, targetSize, quality, originals, i, opts.retry) + result, exportErr := exportOneWithRetry(bridge, pa, targetSize, quality, originals, i, opts.retry) dur := time.Since(start) isErr := exportErr != nil isSkipped := result.Skipped @@ -862,14 +899,10 @@ func exportPendingSerial(pending []pendingAsset, targetSize, quality int, origin failed++ appendFailure(pa.path, pa, exportErr) } else if isSkipped { - if m != nil { - m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud) - } + addManifestEntry(m, pa, result) } else { done++ - if m != nil { - m.Add(pa.asset.ID, result.Filename, result.Size, result.Cloud) - } + addManifestEntry(m, pa, result) } avgSpeed := float64(0) if totalDur > 0 { @@ -922,7 +955,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig start := time.Now() var result photos.ExportResult var exportErr error - result, exportErr = exportOneWithSlotRetry(bridge, pending[i].asset, pending[i].path, targetSize, quality, originals, i, workerID, opts.retry) + result, exportErr = exportOneWithSlotRetry(bridge, pending[i], targetSize, quality, originals, i, workerID, opts.retry) dur := time.Since(start) bar.setWorker(workerID, "", 0, "", "") completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur} @@ -995,14 +1028,10 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig failed++ appendFailure(entry.pa.path, entry.pa, entry.err) } else if isSkipped { - if m != nil { - m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud) - } + addManifestEntry(m, entry.pa, entry.result) } else { done++ - if m != nil { - m.Add(entry.pa.asset.ID, entry.result.Filename, entry.result.Size, entry.result.Cloud) - } + addManifestEntry(m, entry.pa, entry.result) } avgSpeed := float64(0) if totalDur > 0 { @@ -1036,7 +1065,7 @@ func exportPendingParallel(pending []pendingAsset, targetSize, quality int, orig func exportAssets(assets []photos.Asset, outDir string, targetSize, quality, concurrency int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string, noManifest bool, mf manifest.Format, enableLog bool, opts exportOptions) (int, int) { pending := make([]pendingAsset, len(assets)) for i, a := range assets { - pending[i] = pendingAsset{asset: a, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix} + pending[i] = pendingAsset{asset: a, root: outDir, path: pathWithDateTemplate(outDir, a, opts.dateTemplate), album: dirPrefix} } var m manifest.Manifest if !noManifest { @@ -1080,11 +1109,93 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, return bridge.ExportPreview(a.ID, outDir, targetSize, quality, index) } -func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) { +func exportOneAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index int) (photos.ExportResult, error) { + root := pa.root + if root == "" { + root = pa.path + } + stagingRoot := filepath.Join(root, ".photoscli-tmp") + if err := os.MkdirAll(stagingRoot, 0755); err != nil { + return exportOne(bridge, pa.asset, pa.path, targetSize, quality, originals, index) + } + stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*") + if err != nil { + return photos.ExportResult{}, err + } + defer os.RemoveAll(stagingDir) + + result, err := exportOne(bridge, pa.asset, stagingDir, targetSize, quality, originals, index) + if err != nil || result.Skipped { + return result, err + } + src := filepath.Join(stagingDir, result.Filename) + info, statErr := os.Stat(src) + if statErr != nil { + return result, nil + } + if info.Size() == 0 { + return result, fmt.Errorf("exported zero-byte file: %s", result.Filename) + } + if err := os.MkdirAll(pa.path, 0755); err != nil { + return result, err + } + dst := filepath.Join(pa.path, result.Filename) + if err := renameFunc(src, dst); err != nil { + return result, err + } + return result, nil +} + +func exportOneWithSlotAtomic(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex int) (photos.ExportResult, error) { + root := pa.root + if root == "" { + root = pa.path + } + stagingRoot := filepath.Join(root, ".photoscli-tmp") + if err := os.MkdirAll(stagingRoot, 0755); err != nil { + if originals { + return bridge.ExportOriginalWithSlot(pa.asset.ID, pa.path, index, slotIndex) + } + return bridge.ExportPreviewWithSlot(pa.asset.ID, pa.path, targetSize, quality, index, slotIndex) + } + stagingDir, err := mkdirTempFunc(stagingRoot, sanitizePathComponent(pa.asset.ID)+"-*") + if err != nil { + return photos.ExportResult{}, err + } + defer os.RemoveAll(stagingDir) + + var result photos.ExportResult + if originals { + result, err = bridge.ExportOriginalWithSlot(pa.asset.ID, stagingDir, index, slotIndex) + } else { + result, err = bridge.ExportPreviewWithSlot(pa.asset.ID, stagingDir, targetSize, quality, index, slotIndex) + } + if err != nil || result.Skipped { + return result, err + } + src := filepath.Join(stagingDir, result.Filename) + info, statErr := os.Stat(src) + if statErr != nil { + return result, nil + } + if info.Size() == 0 { + return result, fmt.Errorf("exported zero-byte file: %s", result.Filename) + } + if err := os.MkdirAll(pa.path, 0755); err != nil { + return result, err + } + dst := filepath.Join(pa.path, result.Filename) + if err := renameFunc(src, dst); err != nil { + return result, err + } + return result, nil +} + +func exportOneWithRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, retry int) (photos.ExportResult, error) { var result photos.ExportResult var err error for attempt := 0; attempt <= retry; attempt++ { - result, err = exportOne(bridge, a, outDir, targetSize, quality, originals, index) + result, err = exportOneAtomic(bridge, pa, targetSize, quality, originals, index) if err == nil { return result, nil } @@ -1093,15 +1204,11 @@ func exportOneWithRetry(bridge photos.Bridge, a photos.Asset, outDir string, tar return result, err } -func exportOneWithSlotRetry(bridge photos.Bridge, a photos.Asset, outDir string, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) { +func exportOneWithSlotRetry(bridge photos.Bridge, pa pendingAsset, targetSize, quality int, originals bool, index, slotIndex, retry int) (photos.ExportResult, error) { var result photos.ExportResult var err error for attempt := 0; attempt <= retry; attempt++ { - if originals { - result, err = bridge.ExportOriginalWithSlot(a.ID, outDir, index, slotIndex) - } else { - result, err = bridge.ExportPreviewWithSlot(a.ID, outDir, targetSize, quality, index, slotIndex) - } + result, err = exportOneWithSlotAtomic(bridge, pa, targetSize, quality, originals, index, slotIndex) if err == nil { return result, nil } @@ -1356,22 +1463,73 @@ func loadManifestEntries(outDir string, mf manifest.Format) (map[string]manifest func failuresPath(dir string) string { return filepath.Join(dir, "failures.jsonl") } -func appendFailure(dir string, pa pendingAsset, err error) { - _ = os.MkdirAll(dir, 0755) - f, openErr := os.OpenFile(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_APPEND, 0644) - if openErr != nil { - return +type failureEntry struct { + ID string `json:"id"` + Filename string `json:"filename"` + Album string `json:"album"` + Path string `json:"path"` + Error string `json:"error"` + FailedAt int64 `json:"failed_at"` + Attempts int `json:"attempts"` +} + +func loadFailures(dir string) map[string]failureEntry { + out := map[string]failureEntry{} + data, err := os.ReadFile(failuresPath(dir)) + if err != nil { + return out + } + for _, line := range strings.Split(string(data), "\n") { + line = strings.TrimSpace(line) + if line == "" { + continue + } + var f failureEntry + if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" { + out[f.ID] = f + } + } + return out +} + +func saveFailures(dir string, failures map[string]failureEntry) error { + if err := os.MkdirAll(dir, 0755); err != nil { + return err + } + f, err := openFileFunc(failuresPath(dir), os.O_CREATE|os.O_WRONLY|os.O_TRUNC, 0644) + if err != nil { + return err } defer f.Close() - data, _ := json.Marshal(struct { - ID string `json:"id"` - Filename string `json:"filename"` - Album string `json:"album"` - Path string `json:"path"` - Error string `json:"error"` - }{pa.asset.ID, pa.asset.Filename, pa.album, pa.path, err.Error()}) - f.Write(data) - f.Write([]byte("\n")) + ids := make([]string, 0, len(failures)) + for id := range failures { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + data, _ := json.Marshal(failures[id]) + f.Write(data) + f.Write([]byte("\n")) + } + return nil +} + +func appendFailure(dir string, pa pendingAsset, err error) { + root := pa.root + if root == "" { + root = dir + } + failures := loadFailures(root) + f := failures[pa.asset.ID] + f.ID = pa.asset.ID + f.Filename = pa.asset.Filename + f.Album = pa.album + f.Path = pa.path + f.Error = err.Error() + f.FailedAt = time.Now().Unix() + f.Attempts++ + failures[pa.asset.ID] = f + _ = saveFailures(root, failures) } func writeJSONSummary(stdout io.Writer, s commandSummary) { @@ -1466,17 +1624,32 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int { fmt.Fprintf(stderr, "error: %v\n", err) return exitErr } - missing := 0 + bad := 0 for id, e := range entries { - if e.Filename == "" { + checkPath := e.Path + if checkPath == "" { + checkPath = e.Filename + } + if checkPath == "" { continue } - if _, err := os.Stat(filepath.Join(outDir, e.Filename)); err != nil { - missing++ - fmt.Fprintf(stdout, "%s\t%s\n", id, e.Filename) + info, err := os.Stat(filepath.Join(outDir, checkPath)) + if err != nil { + bad++ + fmt.Fprintf(stdout, "%s\t%s\tmissing\n", id, checkPath) + continue + } + if info.Size() == 0 { + bad++ + fmt.Fprintf(stdout, "%s\t%s\tzero-byte\n", id, checkPath) + continue + } + if e.Size > 0 && info.Size() != e.Size { + bad++ + fmt.Fprintf(stdout, "%s\t%s\tsize-mismatch\tmanifest=%d\tdisk=%d\n", id, checkPath, e.Size, info.Size()) } } - if missing > 0 { + if bad > 0 { return exitPartial } fmt.Fprintf(stdout, "verified\t%d\n", len(entries)) @@ -1485,31 +1658,103 @@ func cmdVerify(args []string, stdout, stderr io.Writer) int { func cmdRetryFailed(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int { outDir := flagVal(args, "--out") + clearOnSuccess := hasFlag(args, "--clear-on-success") if outDir == "" { fmt.Fprintln(stderr, "error: --out is required") return exitErr } - data, err := os.ReadFile(failuresPath(outDir)) - if err != nil { - fmt.Fprintf(stderr, "error: %v\n", err) + failures := loadFailures(outDir) + if len(failures) == 0 { + fmt.Fprintf(stderr, "error: no failures found in %s\n", failuresPath(outDir)) return exitErr } var pending []pendingAsset - for _, line := range strings.Split(string(data), "\n") { - line = strings.TrimSpace(line) - if line == "" { - continue - } - var f struct{ ID, Filename, Album, Path string } - if json.Unmarshal([]byte(line), &f) == nil && f.ID != "" { - pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, path: f.Path, album: f.Album}) - } + ids := make([]string, 0, len(failures)) + for id := range failures { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + f := failures[id] + pending = append(pending, pendingAsset{asset: photos.Asset{ID: f.ID, Filename: f.Filename}, root: outDir, path: f.Path, album: f.Album}) } bar := newProgressBar(stderr, 1) done, failed := exportPendingSerial(pending, 1024, 85, false, len(pending), bar, bridge, nil, manifest.NoopLogWriter, exportOptions{}) + if clearOnSuccess && done > 0 { + for i := 0; i < done && i < len(pending); i++ { + delete(failures, pending[i].asset.ID) + } + _ = saveFailures(outDir, failures) + } writeJSONSummary(stdout, commandSummary{Exported: done, Failed: failed, Total: len(pending)}) if failed > 0 { return exitPartial } return exitOK } + +func cmdFailures(args []string, stdout, stderr io.Writer) int { + if len(args) < 1 { + fmt.Fprintln(stderr, "error: failures requires list or clear") + return exitErr + } + outDir := flagVal(args, "--out") + if outDir == "" { + fmt.Fprintln(stderr, "error: --out is required") + return exitErr + } + switch args[0] { + case "list": + failures := loadFailures(outDir) + ids := make([]string, 0, len(failures)) + for id := range failures { + ids = append(ids, id) + } + sort.Strings(ids) + for _, id := range ids { + f := failures[id] + fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%d\n", f.ID, f.Filename, f.Album, f.Error, f.Attempts) + } + return exitOK + case "clear": + if err := removeFunc(failuresPath(outDir)); err != nil && !os.IsNotExist(err) { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + return exitOK + default: + fmt.Fprintf(stderr, "error: unknown failures command %q\n", args[0]) + return exitErr + } +} + +func cmdStatus(args []string, stdout, stderr io.Writer) int { + outDir := flagVal(args, "--out") + if outDir == "" { + fmt.Fprintln(stderr, "error: --out is required") + return exitErr + } + manifestFmt := flagValWithDefault(args, "--manifest", "jsonl") + mf, err := manifest.ParseFormat(manifestFmt) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + entries, err := loadManifestEntries(outDir, mf) + if err != nil { + fmt.Fprintf(stderr, "error: %v\n", err) + return exitErr + } + failures := loadFailures(outDir) + if hasFlag(args, "--json") { + data, _ := json.Marshal(struct { + Manifest string `json:"manifest"` + Entries int `json:"entries"` + Failures int `json:"failures"` + }{manifestFmt, len(entries), len(failures)}) + fmt.Fprintln(stdout, string(data)) + return exitOK + } + fmt.Fprintf(stdout, "manifest\t%s\nentries\t%d\nfailures\t%d\n", manifestFmt, len(entries), len(failures)) + return exitOK +} diff --git a/cmd/photoscli/main_test.go b/cmd/photoscli/main_test.go index eb10b51..64490a6 100644 --- a/cmd/photoscli/main_test.go +++ b/cmd/photoscli/main_test.go @@ -3815,7 +3815,7 @@ func TestReportDiffVerifyRetryFailedAndConfig(t *testing.T) { t.Fatal(err) } m.Close() - if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("x"), 0644); err != nil { + if err := os.WriteFile(filepath.Join(dir, "file.jpg"), []byte("0123456789"), 0644); err != nil { t.Fatal(err) } appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, path: dir, album: "Album"}, fmt.Errorf("boom")) @@ -4043,3 +4043,272 @@ func TestNewFeatureRemainingBranches(t *testing.T) { t.Fatalf("expected retry partial, got %d", rc) } } + +func TestFailuresAndStatusCommands(t *testing.T) { + dir := t.TempDir() + m := manifest.LoadJSONL(dir) + if err := m.OpenAppend(); err != nil { + t.Fatal(err) + } + m.AddEntry(manifest.NewEntry("x1", "file.jpg", "Album/file.jpg", 4, "local")) + m.Close() + if err := os.MkdirAll(filepath.Join(dir, "Album"), 0755); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "Album", "file.jpg"), []byte("data"), 0644); err != nil { + t.Fatal(err) + } + appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom")) + appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"}, fmt.Errorf("boom2")) + + out, stderr, rc := runWith([]string{"status", "--out", dir, "--json"}, &mockBridge{}) + if rc != exitOK || !strings.Contains(out, "failures") || stderr != "" { + t.Fatalf("status json rc=%d out=%q stderr=%q", rc, out, stderr) + } + out, stderr, rc = runWith([]string{"status", "--out", dir}, &mockBridge{}) + if rc != exitOK || !strings.Contains(out, "entries\t1") || !strings.Contains(out, "failures\t1") || stderr != "" { + t.Fatalf("status rc=%d out=%q stderr=%q", rc, out, stderr) + } + out, stderr, rc = runWith([]string{"failures", "list", "--out", dir}, &mockBridge{}) + if rc != exitOK || !strings.Contains(out, "bad") || !strings.Contains(out, "2") || stderr != "" { + t.Fatalf("failures list rc=%d out=%q stderr=%q", rc, out, stderr) + } + _, stderr, rc = runWith([]string{"failures"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "requires") { + t.Fatalf("failures missing subcommand rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"failures", "bogus", "--out", dir}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "unknown") { + t.Fatalf("failures bogus rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{}) + if rc != exitOK || stderr != "" { + t.Fatalf("failures clear rc=%d stderr=%q", rc, stderr) + } + if len(loadFailures(dir)) != 0 { + t.Fatal("expected failures cleared") + } +} + +func TestAtomicExportHelpers(t *testing.T) { + dir := t.TempDir() + b := &mockBridge{} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "photo.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "photo.jpg", Size: 4, Cloud: "local"}, nil + } + pa := pendingAsset{asset: photos.Asset{ID: "x1", Filename: "photo.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"} + result, err := exportOneAtomic(b, pa, 1024, 85, false, 0) + if err != nil || result.Filename != "photo.jpg" { + t.Fatalf("atomic export result=%+v err=%v", result, err) + } + if _, err := os.Stat(filepath.Join(dir, "Album", "photo.jpg")); err != nil { + t.Fatalf("expected final file: %v", err) + } + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "empty.jpg"}, nil + } + if _, err := exportOneAtomic(b, pa, 1024, 85, false, 1); err == nil || !strings.Contains(err.Error(), "zero-byte") { + t.Fatalf("expected zero-byte error, got %v", err) + } +} + +func TestMoreIntegrityBranches(t *testing.T) { + dir := t.TempDir() + m := manifest.LoadJSONL(dir) + if err := m.OpenAppend(); err != nil { + t.Fatal(err) + } + m.AddEntry(manifest.Entry{ID: "zero", Filename: "zero.jpg", Path: "zero.jpg", Size: 1, Cloud: "local", Exported: time.Now().Unix()}) + m.AddEntry(manifest.Entry{ID: "mismatch", Filename: "mismatch.jpg", Path: "mismatch.jpg", Size: 10, Cloud: "local", Exported: time.Now().Unix()}) + m.Close() + if err := os.WriteFile(filepath.Join(dir, "zero.jpg"), nil, 0644); err != nil { + t.Fatal(err) + } + if err := os.WriteFile(filepath.Join(dir, "mismatch.jpg"), []byte("bad"), 0644); err != nil { + t.Fatal(err) + } + out, _, rc := runWith([]string{"verify", "--out", dir}, &mockBridge{}) + if rc != exitPartial || !strings.Contains(out, "zero-byte") || !strings.Contains(out, "size-mismatch") { + t.Fatalf("verify rc=%d out=%q", rc, out) + } + _, stderr, rc := runWith([]string{"status"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "--out") { + t.Fatalf("status missing out rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"status", "--out", dir, "--manifest", "bad"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "manifest") { + t.Fatalf("status bad manifest rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"status", "--out", "/proc/cannot-create"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "error:") { + t.Fatalf("status load error rc=%d stderr=%q", rc, stderr) + } + _, stderr, rc = runWith([]string{"failures", "list"}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "--out") { + t.Fatalf("failures missing out rc=%d stderr=%q", rc, stderr) + } +} + +func TestRetryFailedClearOnSuccess(t *testing.T) { + dir := t.TempDir() + appendFailure(dir, pendingAsset{asset: photos.Asset{ID: "bad", Filename: "bad.jpg"}, root: dir, path: dir, album: "Album"}, fmt.Errorf("boom")) + out, _, rc := runWith([]string{"retry-failed", "--out", dir, "--clear-on-success"}, &mockBridge{}) + if rc != exitOK || !strings.Contains(out, "exported") { + t.Fatalf("retry clear rc=%d out=%q", rc, out) + } + if len(loadFailures(dir)) != 0 { + t.Fatal("expected successful retry to clear failure") + } +} + +func TestAtomicSlotExportHelper(t *testing.T) { + dir := t.TempDir() + b := &mockBridge{} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "slot.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "slot.jpg", Size: 4, Cloud: "local"}, nil + } + pa := pendingAsset{asset: photos.Asset{ID: "slot", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Album"), album: "Album"} + result, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0) + if err != nil || result.Filename != "slot.jpg" { + t.Fatalf("slot atomic result=%+v err=%v", result, err) + } +} + +func TestInjectedErrorBranchesForCoverage(t *testing.T) { + dir := t.TempDir() + b := &mockBridge{} + pa := pendingAsset{asset: photos.Asset{ID: "x", Filename: "x.jpg"}, root: dir, path: filepath.Join(dir, "Album")} + + oldMkdirTemp := mkdirTempFunc + mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("mkdirtemp") } + if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "mkdirtemp") { + t.Fatalf("expected mkdirtemp error, got %v", err) + } + mkdirTempFunc = oldMkdirTemp + + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil + } + oldRename := renameFunc + renameFunc = func(string, string) error { return fmt.Errorf("rename") } + if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil || !strings.Contains(err.Error(), "rename") { + t.Fatalf("expected rename error, got %v", err) + } + renameFunc = oldRename + + oldOpen := openFileFunc + openFileFunc = func(string, int, os.FileMode) (*os.File, error) { return nil, fmt.Errorf("open") } + if err := saveFailures(dir, map[string]failureEntry{"x": {ID: "x"}}); err == nil || !strings.Contains(err.Error(), "open") { + t.Fatalf("expected open error, got %v", err) + } + openFileFunc = oldOpen + + oldRemove := removeFunc + removeFunc = func(string) error { return fmt.Errorf("remove") } + _, stderr, rc := runWith([]string{"failures", "clear", "--out", dir}, &mockBridge{}) + if rc != exitErr || !strings.Contains(stderr, "remove") { + t.Fatalf("expected remove error, rc=%d stderr=%q", rc, stderr) + } + removeFunc = oldRemove + + mf := &mockManifest{} + addManifestEntry(mf, pendingAsset{asset: photos.Asset{ID: "x"}, root: filepath.Join(dir, "other"), path: dir}, photos.ExportResult{Filename: "file.jpg", Size: 1}) + if mf.last.Path != "file.jpg" { + t.Fatalf("expected fallback rel path, got %+v", mf.last) + } + + badPathRoot := t.TempDir() + badPath := filepath.Join(badPathRoot, "notdir") + if err := os.WriteFile(badPath, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + pa = pendingAsset{asset: photos.Asset{ID: "x2", Filename: "x2.jpg"}, root: badPathRoot, path: badPath} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil + } + if _, err := exportOneAtomic(b, pa, 1024, 85, false, 0); err == nil { + t.Fatal("expected final mkdir error") + } + + slotRootFile := filepath.Join(t.TempDir(), "rootfile") + if err := os.WriteFile(slotRootFile, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + pa = pendingAsset{asset: photos.Asset{ID: "slotfallback", Filename: "slot.jpg"}, root: slotRootFile, path: t.TempDir()} + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, true, 0, 0); err != nil { + t.Fatalf("unexpected original fallback error: %v", err) + } + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err != nil { + t.Fatalf("unexpected preview fallback error: %v", err) + } + + pa = pendingAsset{asset: photos.Asset{ID: "slotzero", Filename: "slot.jpg"}, root: dir, path: filepath.Join(dir, "Slot")} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "empty.jpg"), nil, 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "empty.jpg"}, nil + } + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "zero-byte") { + t.Fatalf("expected slot zero-byte error, got %v", err) + } + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil + } + renameFunc = func(string, string) error { return fmt.Errorf("slot rename") } + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot rename") { + t.Fatalf("expected slot rename error, got %v", err) + } + renameFunc = oldRename + + mkdirTempFunc = func(string, string) (string, error) { return "", fmt.Errorf("slot mkdirtemp") } + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil || !strings.Contains(err.Error(), "slot mkdirtemp") { + t.Fatalf("expected slot mkdirtemp error, got %v", err) + } + mkdirTempFunc = oldMkdirTemp + + badSlotRoot := t.TempDir() + badSlotPath := filepath.Join(badSlotRoot, "notdir") + if err := os.WriteFile(badSlotPath, []byte("x"), 0644); err != nil { + t.Fatal(err) + } + pa = pendingAsset{asset: photos.Asset{ID: "slotbadpath", Filename: "slot.jpg"}, root: badSlotRoot, path: badSlotPath} + b.exportPreviewFn = func(assetID, out string, targetSize, quality, index int) (photos.ExportResult, error) { + if err := os.WriteFile(filepath.Join(out, "file.jpg"), []byte("data"), 0644); err != nil { + return photos.ExportResult{}, err + } + return photos.ExportResult{Filename: "file.jpg", Size: 4}, nil + } + if _, err := exportOneWithSlotAtomic(b, pa, 1024, 85, false, 0, 0); err == nil { + t.Fatal("expected slot final mkdir error") + } +} + +type mockManifest struct{ last manifest.Entry } + +func (m *mockManifest) Has(string) bool { return false } +func (m *mockManifest) Add(id string, filename string, size int64, cloud string) { + m.last = manifest.NewEntry(id, filename, filename, size, cloud) +} +func (m *mockManifest) AddEntry(e manifest.Entry) { m.last = e } +func (m *mockManifest) Save() error { return nil } +func (m *mockManifest) Close() {} +func (m *mockManifest) OpenAppend() error { return nil } diff --git a/internal/manifest/entry_test.go b/internal/manifest/entry_test.go new file mode 100644 index 0000000..cba8e8f --- /dev/null +++ b/internal/manifest/entry_test.go @@ -0,0 +1,44 @@ +package manifest + +import "testing" + +func TestNewEntryPath(t *testing.T) { + e := newEntry("id1", "file.jpg", 123, "local") + if e.ID != "id1" || e.Filename != "file.jpg" || e.Path != "file.jpg" || e.Size != 123 || e.Cloud != "local" || e.Exported == 0 { + t.Fatalf("unexpected entry: %+v", e) + } + e = NewEntry("id2", "file2.jpg", "Album/file2.jpg", 456, "cloud") + if e.Path != "Album/file2.jpg" || e.Filename != "file2.jpg" || e.Size != 456 || e.Cloud != "cloud" { + t.Fatalf("unexpected entry with path: %+v", e) + } +} + +func TestAddEntryDefaultsPath(t *testing.T) { + dir := t.TempDir() + jm := LoadJSONL(dir) + if err := jm.OpenAppend(); err != nil { + t.Fatal(err) + } + jm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"}) + jm.Close() + if got := LoadJSONL(dir).Entries()["x1"].Path; got != "file.jpg" { + t.Fatalf("jsonl path = %q", got) + } + + sdir := t.TempDir() + sm, err := LoadSQLite(sdir) + if err != nil { + t.Fatal(err) + } + if err := sm.OpenAppend(); err != nil { + t.Fatal(err) + } + sm.AddEntry(Entry{ID: "x1", Filename: "file.jpg", Size: 3, Cloud: "local"}) + if _, err := sm.db.Exec(`UPDATE downloads SET path = '' WHERE id = 'x1'`); err != nil { + t.Fatal(err) + } + if got := sm.Entries()["x1"].Path; got != "file.jpg" { + t.Fatalf("sqlite path = %q", got) + } + sm.Close() +} diff --git a/internal/manifest/jsonl.go b/internal/manifest/jsonl.go index 9f4b924..8934af8 100644 --- a/internal/manifest/jsonl.go +++ b/internal/manifest/jsonl.go @@ -40,6 +40,7 @@ func LoadJSONL(dir string) *jsonlManifest { type entryWithID struct { ID string `json:"id"` Filename string `json:"filename"` + Path string `json:"path,omitempty"` Size int64 `json:"size"` Cloud string `json:"cloud"` Exported int64 `json:"exported"` @@ -51,8 +52,13 @@ func LoadJSONL(dir string) *jsonlManifest { } var raw entryWithID if json.Unmarshal([]byte(line), &raw) == nil && raw.ID != "" { + if raw.Path == "" { + raw.Path = raw.Filename + } m.entries[raw.ID] = Entry{ + ID: raw.ID, Filename: raw.Filename, + Path: raw.Path, Size: raw.Size, Cloud: raw.Cloud, Exported: raw.Exported, @@ -70,18 +76,25 @@ func (m *jsonlManifest) Has(id string) bool { } func (m *jsonlManifest) Add(id string, filename string, size int64, cloud string) { + m.AddEntry(newEntry(id, filename, size, cloud)) +} + +func (m *jsonlManifest) AddEntry(entry Entry) { m.mu.Lock() defer m.mu.Unlock() - entry := newEntry(id, filename, size, cloud) - m.entries[id] = entry + if entry.Path == "" { + entry.Path = entry.Filename + } + m.entries[entry.ID] = entry if m.file != nil { data, _ := json.Marshal(struct { ID string `json:"id"` Filename string `json:"filename"` + Path string `json:"path,omitempty"` Size int64 `json:"size"` Cloud string `json:"cloud"` Exported int64 `json:"exported"` - }{ID: id, Filename: entry.Filename, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported}) + }{ID: entry.ID, Filename: entry.Filename, Path: entry.Path, Size: entry.Size, Cloud: entry.Cloud, Exported: entry.Exported}) m.file.Write(data) m.file.Write([]byte("\n")) } diff --git a/internal/manifest/manifest.go b/internal/manifest/manifest.go index f315fa2..01912c0 100644 --- a/internal/manifest/manifest.go +++ b/internal/manifest/manifest.go @@ -5,6 +5,7 @@ import "time" type Entry struct { ID string Filename string + Path string Size int64 Cloud string Exported int64 @@ -13,6 +14,7 @@ type Entry struct { type Manifest interface { Has(id string) bool Add(id string, filename string, size int64, cloud string) + AddEntry(entry Entry) Save() error Close() OpenAppend() error @@ -26,8 +28,17 @@ func newEntry(id, filename string, size int64, cloud string) Entry { return Entry{ ID: id, Filename: filename, + Path: filename, Size: size, Cloud: cloud, Exported: time.Now().Unix(), } } + +func NewEntry(id, filename, path string, size int64, cloud string) Entry { + e := newEntry(id, filename, size, cloud) + if path != "" { + e.Path = path + } + return e +} diff --git a/internal/manifest/open.go b/internal/manifest/open.go index a65f401..ecde369 100644 --- a/internal/manifest/open.go +++ b/internal/manifest/open.go @@ -49,7 +49,8 @@ func ConvertFromJSONL(dir string) (Manifest, error) { } for id, e := range src.Entries() { - dst.Add(id, e.Filename, e.Size, e.Cloud) + e.ID = id + dst.AddEntry(e) } os.Remove(JSONLPath(dir)) @@ -69,7 +70,8 @@ func ConvertFromSQLite(dir string) (Manifest, error) { } for id, e := range src.Entries() { - dst.Add(id, e.Filename, e.Size, e.Cloud) + e.ID = id + dst.AddEntry(e) } if err := dst.Save(); err != nil { return nil, fmt.Errorf("save jsonl: %w", err) diff --git a/internal/manifest/sqlite.go b/internal/manifest/sqlite.go index 6e7f48c..84aea66 100644 --- a/internal/manifest/sqlite.go +++ b/internal/manifest/sqlite.go @@ -61,6 +61,7 @@ func (m *sqliteManifest) OpenAppend() error { _, err = execFn(`CREATE TABLE IF NOT EXISTS downloads ( id TEXT PRIMARY KEY, filename TEXT NOT NULL DEFAULT '', + path TEXT NOT NULL DEFAULT '', size INTEGER NOT NULL DEFAULT 0, cloud TEXT NOT NULL DEFAULT '', exported INTEGER NOT NULL DEFAULT 0 @@ -69,6 +70,7 @@ func (m *sqliteManifest) OpenAppend() error { db.Close() return fmt.Errorf("create table: %w", err) } + _, _ = execFn(`ALTER TABLE downloads ADD COLUMN path TEXT NOT NULL DEFAULT ''`) _, err = execFn(`CREATE INDEX IF NOT EXISTS idx_downloads_id ON downloads(id)`) if err != nil { db.Close() @@ -91,12 +93,18 @@ func (m *sqliteManifest) Has(id string) bool { } func (m *sqliteManifest) Add(id string, filename string, size int64, cloud string) { + m.AddEntry(newEntry(id, filename, size, cloud)) +} + +func (m *sqliteManifest) AddEntry(entry Entry) { if m.db == nil { return } - entry := newEntry(id, filename, size, cloud) - m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, size, cloud, exported) VALUES (?, ?, ?, ?, ?)`, - id, entry.Filename, entry.Size, entry.Cloud, entry.Exported) + if entry.Path == "" { + entry.Path = entry.Filename + } + m.db.Exec(`INSERT OR REPLACE INTO downloads (id, filename, path, size, cloud, exported) VALUES (?, ?, ?, ?, ?, ?)`, + entry.ID, entry.Filename, entry.Path, entry.Size, entry.Cloud, entry.Exported) } func (m *sqliteManifest) Save() error { @@ -119,14 +127,17 @@ func (m *sqliteManifest) Entries() map[string]Entry { return nil } out := make(map[string]Entry) - rows, err := m.db.Query(`SELECT id, filename, size, cloud, exported FROM downloads`) + rows, err := m.db.Query(`SELECT id, filename, path, size, cloud, exported FROM downloads`) if err != nil { return out } defer rows.Close() for rows.Next() { var e Entry - if err := rows.Scan(&e.ID, &e.Filename, &e.Size, &e.Cloud, &e.Exported); err == nil { + if err := rows.Scan(&e.ID, &e.Filename, &e.Path, &e.Size, &e.Cloud, &e.Exported); err == nil { + if e.Path == "" { + e.Path = e.Filename + } out[e.ID] = e } } diff --git a/internal/manifest/sqlite_internal_test.go b/internal/manifest/sqlite_internal_test.go index fc94a20..970c4fa 100644 --- a/internal/manifest/sqlite_internal_test.go +++ b/internal/manifest/sqlite_internal_test.go @@ -249,10 +249,10 @@ func TestSQLiteCreateIndexError(t *testing.T) { return nil, err } m.execFunc = func(query string, args ...any) (sql.Result, error) { - callCount++ - if callCount == 2 { + if strings.Contains(query, "CREATE INDEX") { return nil, fmt.Errorf("injected CREATE INDEX error") } + callCount++ return db.Exec(query, args...) } return db, nil