v0.5.0: manifests, filters, logging, docs

internal/manifest/open.go

@@ -0,0 +1,106 @@
+package manifest
+
+import (
+	"fmt"
+	"os"
+	"strings"
+)
+
+type Format string
+
+const (
+	FormatJSONL  Format = "jsonl"
+	FormatSQLite Format = "sqlite"
+)
+
+func Open(dir string, format Format) (Manifest, error) {
+	jsonlPath := JSONLPath(dir)
+	sqlitePath := SQLitePath(dir)
+	jsonlExists := FileExists(jsonlPath)
+	sqliteExists := FileExists(sqlitePath)
+
+	switch {
+	case format == FormatJSONL && jsonlExists:
+		return LoadJSONL(dir), nil
+	case format == FormatSQLite && sqliteExists:
+		return LoadSQLite(dir)
+	case format == FormatJSONL && sqliteExists:
+		return ConvertFromSQLite(dir)
+	case format == FormatSQLite && jsonlExists:
+		return ConvertFromJSONL(dir)
+	default:
+		if format == FormatJSONL {
+			return LoadJSONL(dir), nil
+		}
+		return LoadSQLite(dir)
+	}
+}
+
+func ConvertFromJSONL(dir string) (Manifest, error) {
+	src := LoadJSONL(dir)
+	if err := src.OpenAppend(); err != nil {
+		return nil, fmt.Errorf("open jsonl for read: %w", err)
+	}
+	defer src.Close()
+
+	dst, _ := LoadSQLite(dir)
+	if err := dst.OpenAppend(); err != nil {
+		return nil, fmt.Errorf("open sqlite for write: %w", err)
+	}
+
+	for id, e := range src.Entries() {
+		dst.Add(id, e.Filename, e.Size, e.Cloud)
+	}
+
+	os.Remove(JSONLPath(dir))
+	return dst, nil
+}
+
+func ConvertFromSQLite(dir string) (Manifest, error) {
+	src, _ := LoadSQLite(dir)
+	if err := src.OpenAppend(); err != nil {
+		return nil, fmt.Errorf("open sqlite for read: %w", err)
+	}
+	defer src.Close()
+
+	dst := LoadJSONL(dir)
+	if err := dst.OpenAppend(); err != nil {
+		return nil, fmt.Errorf("open jsonl for write: %w", err)
+	}
+
+	for id, e := range src.Entries() {
+		dst.Add(id, e.Filename, e.Size, e.Cloud)
+	}
+	if err := dst.Save(); err != nil {
+		return nil, fmt.Errorf("save jsonl: %w", err)
+	}
+
+	os.Remove(SQLitePath(dir))
+	return dst, nil
+}
+
+func FileExists(path string) bool {
+	info, err := os.Stat(path)
+	if err != nil {
+		return false
+	}
+	return !info.IsDir()
+}
+
+func ParseFormat(s string) (Format, error) {
+	switch strings.ToLower(s) {
+	case "jsonl", "json":
+		return FormatJSONL, nil
+	case "sqlite", "db", "sqlite3":
+		return FormatSQLite, nil
+	default:
+		return "", fmt.Errorf("unknown manifest format %q (use jsonl or sqlite)", s)
+	}
+}
+
+func OpenLogWriter(m Manifest, dir string) (LogWriter, error) {
+	if sm, ok := m.(*sqliteManifest); ok && sm.DB() != nil {
+		return NewSQLiteLogWriter(sm.DB())
+	}
+	return NewFileLogWriter(LogPath(dir))
+}