v0.4.0: scroll log, worker slots, disk skip, color gradient, parallel export
This commit is contained in:
Ein Anderssono
+276
-112
@@ -3,7 +3,10 @@ package main
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"gitea.k3s.k0.nu/tools/photocli/internal/photos"
|
||||
@@ -47,8 +50,8 @@ Usage:
|
||||
photoscli albums
|
||||
photoscli photos --album-id <id>
|
||||
photoscli tree
|
||||
photoscli backup-all --out <dir> [--size <px>] [--originals]
|
||||
photoscli export --album-id <id> --out <dir> [--size <px>] [--originals]
|
||||
photoscli backup-all --out <dir> [--size <px>] [--originals] [--include-videos]
|
||||
photoscli export --album-id <id> --out <dir> [--size <px>] [--originals] [--include-videos]
|
||||
photoscli version
|
||||
|
||||
Commands:
|
||||
@@ -60,10 +63,11 @@ Commands:
|
||||
version Print version
|
||||
|
||||
Flags:
|
||||
--album-id <id> Album local identifier or title (required for photos/export)
|
||||
--out <dir> Output directory (required for export/backup-all)
|
||||
--size <px> Target longest-side in pixels (default: 1024, preview export only)
|
||||
--originals Export original files instead of JPEG previews`)
|
||||
--album-id <id> Album local identifier or title (required for photos/export)
|
||||
--out <dir> Output directory (required for export/backup-all)
|
||||
--size <px> Target longest-side in pixels (default: 1024, preview export only)
|
||||
--originals Export original files instead of JPEG previews
|
||||
--include-videos Include video assets (videos are skipped by default)`)
|
||||
}
|
||||
|
||||
func mustAuth(stderr io.Writer, bridge photos.Bridge) int {
|
||||
@@ -129,7 +133,17 @@ func cmdPhotos(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
|
||||
return 1
|
||||
}
|
||||
for _, a := range assets {
|
||||
fmt.Fprintf(stdout, "%s\t%s\t%s\n", a.ID, a.Filename, a.Cloud)
|
||||
fmt.Fprintf(stdout, "%s\t%s\t%s\t%s\t%dx%d", a.ID, a.Filename, a.Cloud, a.MediaType, a.PixelWidth, a.PixelHeight)
|
||||
if a.CreationDate != nil {
|
||||
fmt.Fprintf(stdout, "\t%s", *a.CreationDate)
|
||||
}
|
||||
if a.Duration > 0 {
|
||||
fmt.Fprintf(stdout, "\t%.1fs", a.Duration)
|
||||
}
|
||||
if a.IsFavorite {
|
||||
fmt.Fprintf(stdout, "\t*")
|
||||
}
|
||||
fmt.Fprintln(stdout)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
@@ -153,6 +167,7 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
|
||||
albumID := flagVal(args, "--album-id")
|
||||
outDir := flagVal(args, "--out")
|
||||
originals := hasFlag(args, "--originals")
|
||||
skipVideos := !hasFlag(args, "--include-videos")
|
||||
sizeStr := flagValWithDefault(args, "--size", "1024")
|
||||
|
||||
if albumID == "" {
|
||||
@@ -189,6 +204,10 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
|
||||
return 1
|
||||
}
|
||||
|
||||
if skipVideos {
|
||||
assets, total = filterVideos(assets)
|
||||
}
|
||||
|
||||
fmt.Fprintf(stderr, "exporting %d assets (%s) to %s...\n", total, exportMode(originals), outDir)
|
||||
exported, failed := exportAssets(assets, outDir, size, originals, total, stderr, bridge, "")
|
||||
|
||||
@@ -212,6 +231,7 @@ func cmdExport(args []string, stdout, stderr io.Writer, bridge photos.Bridge) in
|
||||
func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge) int {
|
||||
outDir := flagVal(args, "--out")
|
||||
originals := hasFlag(args, "--originals")
|
||||
skipVideos := !hasFlag(args, "--include-videos")
|
||||
sizeStr := flagValWithDefault(args, "--size", "1024")
|
||||
|
||||
if outDir == "" {
|
||||
@@ -238,9 +258,9 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
|
||||
return 1
|
||||
}
|
||||
albumCount := countAlbums(nodes)
|
||||
fmt.Fprintf(stderr, "found %d albums, exporting to %s...\n", albumCount, outDir)
|
||||
fmt.Fprintf(stderr, "found %d albums, building index...\n", albumCount)
|
||||
|
||||
totalAssets, _, failed, err := backupTree(nodes, outDir, size, originals, stderr, bridge)
|
||||
totalAssets, failed, err := backupTree(nodes, outDir, size, originals, skipVideos, stderr, bridge)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "error: %v\n", err)
|
||||
return 1
|
||||
@@ -258,72 +278,279 @@ func cmdBackupAll(args []string, stdout, stderr io.Writer, bridge photos.Bridge)
|
||||
return 0
|
||||
}
|
||||
|
||||
func backupTree(nodes []photos.CollectionNode, outDir string, targetSize int, originals bool, stderr io.Writer, bridge photos.Bridge) (int, int, int, error) {
|
||||
exported := 0
|
||||
total := 0
|
||||
failed := 0
|
||||
type pendingAsset struct {
|
||||
asset photos.Asset
|
||||
path string
|
||||
album string
|
||||
}
|
||||
|
||||
func collectPendingAssets(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, stderr io.Writer) ([]pendingAsset, int) {
|
||||
var items []pendingAsset
|
||||
var skipped int
|
||||
collectNodes(nodes, outDir, bridge, skipVideos, originals, &items, &skipped, stderr)
|
||||
return items, skipped
|
||||
}
|
||||
|
||||
func collectNodes(nodes []photos.CollectionNode, outDir string, bridge photos.Bridge, skipVideos bool, originals bool, items *[]pendingAsset, skipped *int, stderr io.Writer) {
|
||||
for _, node := range nodes {
|
||||
if bridge.IsCancelled() {
|
||||
break
|
||||
return
|
||||
}
|
||||
path := outDir + "/" + sanitizePathComponent(node.Name)
|
||||
if node.Kind == "folder" {
|
||||
n, t, f, err := backupTree(node.Children, path, targetSize, originals, stderr, bridge)
|
||||
if err != nil {
|
||||
return exported, total, failed, err
|
||||
}
|
||||
exported += n
|
||||
total += t
|
||||
failed += f
|
||||
collectNodes(node.Children, path, bridge, skipVideos, originals, items, skipped, stderr)
|
||||
continue
|
||||
}
|
||||
if node.Kind == "album" && node.ID != "" {
|
||||
assets, assetTotal, err := bridge.ListAssets(node.ID)
|
||||
assets, _, err := bridge.ListAssets(node.ID)
|
||||
if err != nil {
|
||||
fmt.Fprintf(stderr, "\n skipped album %s: %v\n", node.Name, err)
|
||||
fmt.Fprintf(stderr, " \u26a0 album %s: %v\n", node.Name, err)
|
||||
continue
|
||||
}
|
||||
fmt.Fprintf(stderr, "\nalbum: %s (%d assets)\n", node.Name, assetTotal)
|
||||
total += assetTotal
|
||||
n, f := exportAssets(assets, path, targetSize, originals, total, stderr, bridge, path+"/")
|
||||
exported += n
|
||||
failed += f
|
||||
if skipVideos {
|
||||
assets, _ = filterVideos(assets)
|
||||
}
|
||||
for _, a := range assets {
|
||||
if fileExistsOnDisk(a, path, originals, len(*items)+*skipped) {
|
||||
*skipped++
|
||||
continue
|
||||
}
|
||||
*items = append(*items, pendingAsset{asset: a, path: path, album: node.Name})
|
||||
}
|
||||
fmt.Fprintf(stderr, " indexing: %d files (%d skipped)\r", len(*items), *skipped)
|
||||
}
|
||||
}
|
||||
return exported, total, failed, nil
|
||||
}
|
||||
|
||||
func exportAssets(assets []photos.Asset, outDir string, targetSize int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string) (int, int) {
|
||||
exported := 0
|
||||
func backupTree(nodes []photos.CollectionNode, outDir string, targetSize int, originals bool, skipVideos bool, stderr io.Writer, bridge photos.Bridge) (int, int, error) {
|
||||
pending, skipped := collectPendingAssets(nodes, outDir, bridge, skipVideos, originals, stderr)
|
||||
if bridge.IsCancelled() {
|
||||
return 0, 0, nil
|
||||
}
|
||||
total := len(pending)
|
||||
fmt.Fprintf(stderr, " indexed %d files (%d skipped), exporting to %s...\n", total, skipped, outDir)
|
||||
bar := newProgressBar(stderr, 3)
|
||||
exported, failed := exportPending(pending, targetSize, originals, total, bar, bridge)
|
||||
bar.clear()
|
||||
return exported, failed, nil
|
||||
}
|
||||
|
||||
func exportPending(pending []pendingAsset, targetSize int, originals bool, total int, bar *progressBar, bridge photos.Bridge) (int, int) {
|
||||
if len(pending) < 4 {
|
||||
return exportPendingSerial(pending, targetSize, originals, total, bar, bridge)
|
||||
}
|
||||
return exportPendingParallel(pending, targetSize, originals, total, bar, bridge, 3)
|
||||
}
|
||||
|
||||
func exportPendingSerial(pending []pendingAsset, targetSize int, originals bool, total int, bar *progressBar, bridge photos.Bridge) (int, int) {
|
||||
done := 0
|
||||
failed := 0
|
||||
var totalBytes int64
|
||||
var totalDur time.Duration
|
||||
for i, a := range assets {
|
||||
|
||||
for i, pa := range pending {
|
||||
if bridge.IsCancelled() {
|
||||
break
|
||||
}
|
||||
bar.setWorker(0, pa.asset.Filename, 0, pa.asset.Cloud, "exporting")
|
||||
bar.setTotal(done+failed, total, "")
|
||||
bar.setAlbum(pa.album, 0, 0)
|
||||
bar.draw()
|
||||
start := time.Now()
|
||||
result, exportErr := exportOne(bridge, a, outDir, targetSize, originals, i)
|
||||
result, exportErr := exportOne(bridge, pa.asset, pa.path, targetSize, originals, i)
|
||||
dur := time.Since(start)
|
||||
if exportErr == nil {
|
||||
isErr := exportErr != nil
|
||||
isSkipped := result.Skipped
|
||||
if !isErr && !isSkipped {
|
||||
totalBytes += result.Size
|
||||
totalDur += dur
|
||||
}
|
||||
if isErr {
|
||||
failed++
|
||||
} else {
|
||||
done++
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
avgSpeed = float64(totalBytes) / totalDur.Seconds()
|
||||
}
|
||||
progressBar(stderr, exported+failed+1, total, dirPrefix+result.Filename, result.Size, result.Cloud, avgSpeed, exportErr != nil)
|
||||
if exportErr != nil {
|
||||
fmt.Fprintf(stderr, "\n failed: %s: %v\n", a.Filename, exportErr)
|
||||
failed++
|
||||
bar.setTotal(done+failed, total, "")
|
||||
bar.setAlbum(pa.album, 0, 0)
|
||||
bar.setWorker(0, "", 0, "", "")
|
||||
var logLine string
|
||||
if isErr {
|
||||
logLine = fmt.Sprintf("\u274c %s: %v", pa.asset.Filename, exportErr)
|
||||
} else if isSkipped {
|
||||
logLine = fmt.Sprintf("\u23ed %s", result.Filename)
|
||||
} else if result.Cloud == "cloud" {
|
||||
logLine = fmt.Sprintf("\u2601 %s - %s - downloaded %s", result.Filename, formatSize(result.Size), formatSpeed(avgSpeed))
|
||||
} else {
|
||||
logLine = fmt.Sprintf("\u2705 %s - %s - copied", result.Filename, formatSize(result.Size))
|
||||
}
|
||||
bar.logCompleted(logLine)
|
||||
}
|
||||
return done, failed
|
||||
}
|
||||
|
||||
func exportPendingParallel(pending []pendingAsset, targetSize int, originals bool, total int, bar *progressBar, bridge photos.Bridge, workers int) (int, int) {
|
||||
type resultEntry struct {
|
||||
result photos.ExportResult
|
||||
err error
|
||||
pa pendingAsset
|
||||
dur time.Duration
|
||||
}
|
||||
|
||||
completed := make(chan resultEntry, len(pending))
|
||||
|
||||
jobs := make(chan int, len(pending))
|
||||
var wg sync.WaitGroup
|
||||
for w := 0; w < workers; w++ {
|
||||
wg.Add(1)
|
||||
go func(workerID int) {
|
||||
defer wg.Done()
|
||||
for i := range jobs {
|
||||
if bridge.IsCancelled() {
|
||||
completed <- resultEntry{err: fmt.Errorf("cancelled"), pa: pending[i]}
|
||||
continue
|
||||
}
|
||||
bar.setWorker(workerID, pending[i].asset.Filename, 0, pending[i].asset.Cloud, "exporting")
|
||||
start := time.Now()
|
||||
var result photos.ExportResult
|
||||
var exportErr error
|
||||
if originals {
|
||||
result, exportErr = bridge.ExportOriginalWithSlot(pending[i].asset.ID, pending[i].path, i, workerID)
|
||||
} else {
|
||||
result, exportErr = bridge.ExportPreviewWithSlot(pending[i].asset.ID, pending[i].path, targetSize, i, workerID)
|
||||
}
|
||||
dur := time.Since(start)
|
||||
bar.setWorker(workerID, "", 0, "", "")
|
||||
completed <- resultEntry{result: result, err: exportErr, pa: pending[i], dur: dur}
|
||||
}
|
||||
}(w)
|
||||
}
|
||||
|
||||
go func() {
|
||||
for i := range pending {
|
||||
if bridge.IsCancelled() {
|
||||
break
|
||||
}
|
||||
jobs <- i
|
||||
}
|
||||
close(jobs)
|
||||
}()
|
||||
|
||||
slots := photos.GetProgressSlots()
|
||||
pollDone := make(chan struct{})
|
||||
go func() {
|
||||
ticker := time.NewTicker(100 * time.Millisecond)
|
||||
defer ticker.Stop()
|
||||
for {
|
||||
select {
|
||||
case <-ticker.C:
|
||||
if bridge.IsCancelled() {
|
||||
return
|
||||
}
|
||||
slots = photos.GetProgressSlots()
|
||||
for i := 0; i < workers && i < len(slots); i++ {
|
||||
bar.updateWorkerProgress(i, slots[i].Progress, slots[i].BytesDone, slots[i].BytesTotal)
|
||||
}
|
||||
bar.draw()
|
||||
case <-pollDone:
|
||||
return
|
||||
}
|
||||
}
|
||||
}()
|
||||
|
||||
done := 0
|
||||
failed := 0
|
||||
var totalBytes int64
|
||||
var totalDur time.Duration
|
||||
for n := 0; n < len(pending); n++ {
|
||||
var entry resultEntry
|
||||
select {
|
||||
case entry = <-completed:
|
||||
case <-time.After(2 * time.Second):
|
||||
if bridge.IsCancelled() {
|
||||
close(pollDone)
|
||||
wg.Wait()
|
||||
return done, failed
|
||||
}
|
||||
continue
|
||||
}
|
||||
exported++
|
||||
if bridge.IsCancelled() {
|
||||
close(pollDone)
|
||||
wg.Wait()
|
||||
return done, failed
|
||||
}
|
||||
isErr := entry.err != nil
|
||||
isSkipped := entry.result.Skipped
|
||||
if !isErr && !isSkipped {
|
||||
totalBytes += entry.result.Size
|
||||
totalDur += entry.dur
|
||||
}
|
||||
if isErr {
|
||||
failed++
|
||||
} else {
|
||||
done++
|
||||
}
|
||||
avgSpeed := float64(0)
|
||||
if totalDur > 0 {
|
||||
avgSpeed = float64(totalBytes) / totalDur.Seconds()
|
||||
}
|
||||
bar.setTotal(done+failed, total, "")
|
||||
bar.setAlbum(entry.pa.album, 0, 0)
|
||||
|
||||
var logLine string
|
||||
if isErr {
|
||||
logLine = fmt.Sprintf("\u274c %s: %v", entry.pa.asset.Filename, entry.err)
|
||||
} else if isSkipped {
|
||||
logLine = fmt.Sprintf("\u23ed %s", entry.result.Filename)
|
||||
} else if entry.result.Cloud == "cloud" {
|
||||
logLine = fmt.Sprintf("\u2601 %s - %s - downloaded %s", entry.result.Filename, formatSize(entry.result.Size), formatSpeed(avgSpeed))
|
||||
} else {
|
||||
logLine = fmt.Sprintf("\u2705 %s - %s - copied", entry.result.Filename, formatSize(entry.result.Size))
|
||||
}
|
||||
bar.logCompleted(logLine)
|
||||
}
|
||||
close(pollDone)
|
||||
wg.Wait()
|
||||
photos.ResetProgressSlots()
|
||||
return done, failed
|
||||
}
|
||||
|
||||
func exportAssets(assets []photos.Asset, outDir string, targetSize int, originals bool, total int, stderr io.Writer, bridge photos.Bridge, dirPrefix string) (int, int) {
|
||||
pending := make([]pendingAsset, len(assets))
|
||||
for i, a := range assets {
|
||||
pending[i] = pendingAsset{asset: a, path: outDir, album: dirPrefix}
|
||||
}
|
||||
bar := newProgressBar(stderr, 1)
|
||||
exported, failed := exportPending(pending, targetSize, originals, len(pending), bar, bridge)
|
||||
bar.clear()
|
||||
return exported, failed
|
||||
}
|
||||
|
||||
func fileExistsOnDisk(asset photos.Asset, outDir string, originals bool, index int) bool {
|
||||
var candidates []string
|
||||
if originals {
|
||||
if asset.Filename != "" {
|
||||
candidates = append(candidates, filepath.Join(outDir, asset.Filename))
|
||||
base := strings.TrimSuffix(asset.Filename, filepath.Ext(asset.Filename))
|
||||
ext := filepath.Ext(asset.Filename)
|
||||
candidates = append(candidates, filepath.Join(outDir, fmt.Sprintf("%04d_%s%s", index, base, ext)))
|
||||
}
|
||||
} else {
|
||||
safeID := sanitizePathComponent(asset.ID)
|
||||
candidates = append(candidates, filepath.Join(outDir, fmt.Sprintf("%04d_%s.jpg", index, safeID)))
|
||||
}
|
||||
for _, p := range candidates {
|
||||
info, err := os.Stat(p)
|
||||
if err == nil && info.Size() > 0 {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize int, originals bool, index int) (photos.ExportResult, error) {
|
||||
if originals {
|
||||
return bridge.ExportOriginal(a.ID, outDir, index)
|
||||
@@ -331,6 +558,16 @@ func exportOne(bridge photos.Bridge, a photos.Asset, outDir string, targetSize i
|
||||
return bridge.ExportPreview(a.ID, outDir, targetSize, index)
|
||||
}
|
||||
|
||||
func filterVideos(assets []photos.Asset) ([]photos.Asset, int) {
|
||||
filtered := make([]photos.Asset, 0, len(assets))
|
||||
for _, a := range assets {
|
||||
if a.MediaType != "video" && a.MediaType != "audio" {
|
||||
filtered = append(filtered, a)
|
||||
}
|
||||
}
|
||||
return filtered, len(filtered)
|
||||
}
|
||||
|
||||
// Must stay in sync with sanitized_path_component in bridge/photokit_bridge.m
|
||||
func sanitizePathComponent(name string) string {
|
||||
s := strings.TrimSpace(name)
|
||||
@@ -385,79 +622,6 @@ func flagValWithDefault(args []string, name, def string) string {
|
||||
return def
|
||||
}
|
||||
|
||||
func progressBar(w io.Writer, current, total int, filename string, size int64, cloud string, avgSpeed float64, isErr bool) {
|
||||
pct := 0
|
||||
if total > 0 {
|
||||
pct = current * 100 / total
|
||||
}
|
||||
barWidth := 25
|
||||
filled := pct * barWidth / 100
|
||||
partial := (pct * barWidth % 100) * len(blockPartial) / 100
|
||||
bar := strings.Repeat(string(blockFull), filled)
|
||||
if filled < barWidth {
|
||||
if partial > 0 {
|
||||
bar += string(blockPartial[partial])
|
||||
}
|
||||
bar += strings.Repeat(string(blockEmpty), barWidth-filled-1)
|
||||
}
|
||||
fileSize := formatSize(size)
|
||||
cloudLabel := ""
|
||||
if cloud == "cloud" {
|
||||
cloudLabel = formatSpeed(avgSpeed)
|
||||
if cloudLabel != "" {
|
||||
cloudLabel = " " + cloudLabel
|
||||
}
|
||||
cloudLabel = " ☁" + cloudLabel
|
||||
}
|
||||
name := filename
|
||||
maxName := 30
|
||||
if len(name) > maxName {
|
||||
name = "…" + name[len(name)-maxName+1:]
|
||||
}
|
||||
fmt.Fprintf(w, "\r%s [%s] %3d%% %s%s %s", name, bar, pct, fileSize, cloudLabel, statusLabel(isErr))
|
||||
}
|
||||
|
||||
var blockFull = '█'
|
||||
var blockEmpty = '░'
|
||||
var blockPartial = []rune{'▏', '▎', '▍', '▌', '▋', '▊', '▉', '█'}
|
||||
|
||||
func statusLabel(isErr bool) string {
|
||||
if isErr {
|
||||
return "✗"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func formatSpeed(bytesPerSec float64) string {
|
||||
if bytesPerSec <= 0 {
|
||||
return ""
|
||||
}
|
||||
const kb = 1024
|
||||
const mb = kb * 1024
|
||||
if bytesPerSec >= mb {
|
||||
return fmt.Sprintf("%.1f MB/s", bytesPerSec/mb)
|
||||
}
|
||||
if bytesPerSec >= kb {
|
||||
return fmt.Sprintf("%.1f KB/s", bytesPerSec/kb)
|
||||
}
|
||||
return fmt.Sprintf("%.0f B/s", bytesPerSec)
|
||||
}
|
||||
|
||||
func formatSize(bytes int64) string {
|
||||
if bytes <= 0 {
|
||||
return ""
|
||||
}
|
||||
const kb = 1024
|
||||
const mb = kb * 1024
|
||||
if bytes >= mb {
|
||||
return fmt.Sprintf("%.1f MB", float64(bytes)/float64(mb))
|
||||
}
|
||||
if bytes >= kb {
|
||||
return fmt.Sprintf("%.1f KB", float64(bytes)/float64(kb))
|
||||
}
|
||||
return fmt.Sprintf("%d B", bytes)
|
||||
}
|
||||
|
||||
func exportMode(originals bool) string {
|
||||
if originals {
|
||||
return "originals"
|
||||
|
||||
Reference in New Issue
Block a user