Ein Anderssono
1050 lines
24 KiB
Go
1050 lines
24 KiB
Go
package manifest_test
|
|
|
|
import (
|
|
"os"
|
|
"path/filepath"
|
|
"testing"
|
|
|
|
"gitea.k3s.k0.nu/tools/photocli/internal/manifest"
|
|
)
|
|
|
|
func TestLoadJSONLEmptyDir(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
if m == nil {
|
|
t.Fatal("expected non-nil manifest")
|
|
}
|
|
}
|
|
|
|
func TestLoadJSONLNonexistentDir(t *testing.T) {
|
|
m := manifest.LoadJSONL(filepath.Join(t.TempDir(), "no-such-dir"))
|
|
if m == nil {
|
|
t.Fatal("expected non-nil manifest even for nonexistent dir")
|
|
}
|
|
}
|
|
|
|
func TestLoadJSONLExistingFile(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := manifest.JSONLPath(dir)
|
|
if err := os.WriteFile(path, []byte(`{"id":"abc","filename":"a.jpg","size":100,"cloud":"gcs","exported":1234}`+"\n"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := manifest.LoadJSONL(dir)
|
|
if !m.Has("abc") {
|
|
t.Fatal("expected to find entry with id abc")
|
|
}
|
|
}
|
|
|
|
func TestJSONLAddAndHas(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
if m.Has("x") {
|
|
t.Fatal("expected Has to return false for unknown id")
|
|
}
|
|
m.Add("x", "photo.jpg", 42, "s3")
|
|
if !m.Has("x") {
|
|
t.Fatal("expected Has to return true after Add")
|
|
}
|
|
if m.Has("y") {
|
|
t.Fatal("expected Has to return false for y")
|
|
}
|
|
}
|
|
|
|
func TestJSONLSaveAndReload(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("id1", "file1.jpg", 10, "aws")
|
|
m.Add("id2", "file2.jpg", 20, "gcs")
|
|
if err := m.Save(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
|
|
m2 := manifest.LoadJSONL(dir)
|
|
if !m2.Has("id1") {
|
|
t.Fatal("expected id1 after reload")
|
|
}
|
|
if !m2.Has("id2") {
|
|
t.Fatal("expected id2 after reload")
|
|
}
|
|
}
|
|
|
|
func TestJSONLOpenAppendCreatesDirs(t *testing.T) {
|
|
dir := t.TempDir()
|
|
subdir := filepath.Join(dir, "a", "b")
|
|
m := manifest.LoadJSONL(subdir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
if _, err := os.Stat(filepath.Dir(manifest.JSONLPath(subdir))); err != nil {
|
|
t.Fatalf("expected dirs to be created: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestJSONLCloseIdempotent(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
m.Close()
|
|
}
|
|
|
|
func TestJSONLOpenAppendIdempotent(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatalf("expected second OpenAppend to succeed: %v", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestJSONLEntries(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
m.Add("e1", "f1.jpg", 1, "c1")
|
|
m.Add("e2", "f2.jpg", 2, "c2")
|
|
entries := m.Entries()
|
|
if len(entries) != 2 {
|
|
t.Fatalf("expected 2 entries, got %d", len(entries))
|
|
}
|
|
if _, ok := entries["e1"]; !ok {
|
|
t.Fatal("missing e1")
|
|
}
|
|
if _, ok := entries["e2"]; !ok {
|
|
t.Fatal("missing e2")
|
|
}
|
|
}
|
|
|
|
func TestLoadSQLiteEmptyDir(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m == nil {
|
|
t.Fatal("expected non-nil manifest")
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestLoadSQLiteNonexistentDir(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(filepath.Join(t.TempDir(), "no-such-dir"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m == nil {
|
|
t.Fatal("expected non-nil manifest even for nonexistent dir")
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteAddHasSaveClose(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("sid1", "sfile.jpg", 99, "azure")
|
|
if !m.Has("sid1") {
|
|
t.Fatal("expected Has to return true after Add")
|
|
}
|
|
if m.Has("nope") {
|
|
t.Fatal("expected Has to return false for unknown id")
|
|
}
|
|
if err := m.Save(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteRoundTrip(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("rid1", "rfile1.jpg", 10, "aws")
|
|
m.Add("rid2", "rfile2.jpg", 20, "gcs")
|
|
if err := m.Save(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
|
|
m2, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m2.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m2.Close()
|
|
if !m2.Has("rid1") {
|
|
t.Fatal("expected rid1 after reload")
|
|
}
|
|
if !m2.Has("rid2") {
|
|
t.Fatal("expected rid2 after reload")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteEntries(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
m.Add("se1", "sf1.jpg", 1, "sc1")
|
|
m.Add("se2", "sf2.jpg", 2, "sc2")
|
|
entries := m.Entries()
|
|
if len(entries) != 2 {
|
|
t.Fatalf("expected 2 entries, got %d", len(entries))
|
|
}
|
|
if _, ok := entries["se1"]; !ok {
|
|
t.Fatal("missing se1")
|
|
}
|
|
if _, ok := entries["se2"]; !ok {
|
|
t.Fatal("missing se2")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteCloseIdempotent(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteOpenAppendIdempotent(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatalf("expected second OpenAppend to succeed: %v", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestOpenJSONLNoFileExists(t *testing.T) {
|
|
m, err := manifest.Open(t.TempDir(), manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestOpenSQLiteNoFileExists(t *testing.T) {
|
|
m, err := manifest.Open(t.TempDir(), manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestOpenJSONLConvertsFromSQLite(t *testing.T) {
|
|
dir := t.TempDir()
|
|
sm, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sm.Add("conv1", "conv1.jpg", 5, "aws")
|
|
sm.Close()
|
|
|
|
m, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
if !m.Has("conv1") {
|
|
t.Fatal("expected converted entry conv1 to exist")
|
|
}
|
|
if _, err := os.Stat(manifest.SQLitePath(dir)); err == nil {
|
|
t.Fatal("expected sqlite file to be removed after conversion")
|
|
}
|
|
}
|
|
|
|
func TestOpenSQLiteConvertsFromJSONL(t *testing.T) {
|
|
dir := t.TempDir()
|
|
jm := manifest.LoadJSONL(dir)
|
|
if err := jm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
jm.Add("jconv1", "jconv1.jpg", 7, "gcs")
|
|
jm.Save()
|
|
jm.Close()
|
|
|
|
m, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
if !m.Has("jconv1") {
|
|
t.Fatal("expected converted entry jconv1 to exist")
|
|
}
|
|
if _, err := os.Stat(manifest.JSONLPath(dir)); err == nil {
|
|
t.Fatal("expected jsonl file to be removed after conversion")
|
|
}
|
|
}
|
|
|
|
func TestOpenBothFormatsPrefersRequested(t *testing.T) {
|
|
dir := t.TempDir()
|
|
|
|
jm := manifest.LoadJSONL(dir)
|
|
if err := jm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
jm.Add("both1", "both1.jpg", 1, "aws")
|
|
jm.Save()
|
|
jm.Close()
|
|
|
|
sm, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sm.Add("both2", "both2.jpg", 2, "gcs")
|
|
sm.Close()
|
|
|
|
m, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
if _, err := os.Stat(manifest.JSONLPath(dir)); err != nil {
|
|
t.Fatal("expected jsonl file to exist after Open with FormatJSONL")
|
|
}
|
|
|
|
jm2 := manifest.LoadJSONL(dir)
|
|
if err := jm2.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
jm2.Add("both1", "both1.jpg", 1, "aws")
|
|
jm2.Save()
|
|
jm2.Close()
|
|
|
|
sm2, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sm2.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sm2.Add("both2", "both2.jpg", 2, "gcs")
|
|
sm2.Close()
|
|
|
|
m2, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m2.Close()
|
|
if _, err := os.Stat(manifest.SQLitePath(dir)); err != nil {
|
|
t.Fatal("expected sqlite file to exist after Open with FormatSQLite")
|
|
}
|
|
}
|
|
|
|
func TestParseFormatValid(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
want manifest.Format
|
|
}{
|
|
{"jsonl", manifest.FormatJSONL},
|
|
{"JSONL", manifest.FormatJSONL},
|
|
{"json", manifest.FormatJSONL},
|
|
{"JSON", manifest.FormatJSONL},
|
|
{"sqlite", manifest.FormatSQLite},
|
|
{"SQLITE", manifest.FormatSQLite},
|
|
{"db", manifest.FormatSQLite},
|
|
{"DB", manifest.FormatSQLite},
|
|
{"sqlite3", manifest.FormatSQLite},
|
|
{"SQLite3", manifest.FormatSQLite},
|
|
}
|
|
for _, tt := range tests {
|
|
got, err := manifest.ParseFormat(tt.input)
|
|
if err != nil {
|
|
t.Errorf("ParseFormat(%q): unexpected error: %v", tt.input, err)
|
|
continue
|
|
}
|
|
if got != tt.want {
|
|
t.Errorf("ParseFormat(%q) = %q, want %q", tt.input, got, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestParseFormatInvalid(t *testing.T) {
|
|
_, err := manifest.ParseFormat("csv")
|
|
if err == nil {
|
|
t.Fatal("expected error for invalid format")
|
|
}
|
|
}
|
|
|
|
func TestOpenAppendReadOnlyPath(t *testing.T) {
|
|
dir := t.TempDir()
|
|
roDir := filepath.Join(dir, "readonly")
|
|
if err := os.MkdirAll(roDir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
path := manifest.JSONLPath(roDir)
|
|
if err := os.WriteFile(path, []byte("{}\n"), 0444); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := manifest.LoadJSONL(roDir)
|
|
err := m.OpenAppend()
|
|
if err == nil {
|
|
t.Fatal("expected error opening read-only path")
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestJSONLEntriesEmpty(t *testing.T) {
|
|
m := manifest.LoadJSONL(t.TempDir())
|
|
entries := m.Entries()
|
|
if len(entries) != 0 {
|
|
t.Fatalf("expected 0 entries, got %d", len(entries))
|
|
}
|
|
}
|
|
|
|
func TestSQLiteEntriesEmpty(t *testing.T) {
|
|
m, err := manifest.LoadSQLite(t.TempDir())
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
entries := m.Entries()
|
|
if len(entries) != 0 {
|
|
t.Fatalf("expected 0 entries, got %d", len(entries))
|
|
}
|
|
}
|
|
|
|
func TestOpenJSONLWithExistingJSONL(t *testing.T) {
|
|
dir := t.TempDir()
|
|
jm := manifest.LoadJSONL(dir)
|
|
if err := jm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
jm.Add("exist1", "exist1.jpg", 10, "aws")
|
|
jm.Save()
|
|
jm.Close()
|
|
|
|
m, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
if !m.Has("exist1") {
|
|
t.Fatal("expected exist1 from existing jsonl")
|
|
}
|
|
}
|
|
|
|
func TestOpenSQLiteWithExistingSQLite(t *testing.T) {
|
|
dir := t.TempDir()
|
|
sm, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := sm.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
sm.Add("sexist1", "sexist1.jpg", 15, "azure")
|
|
sm.Close()
|
|
|
|
m, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
defer m.Close()
|
|
if !m.Has("sexist1") {
|
|
t.Fatal("expected sexist1 from existing sqlite")
|
|
}
|
|
}
|
|
func TestLoadJSONLWithEmptyLines(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "downloads.jsonl")
|
|
os.WriteFile(path, []byte("\n\n{\"id\":\"x1\",\"filename\":\"a.jpg\",\"size\":1024,\"cloud\":\"local\",\"exported\":100}\n\n"), 0644)
|
|
m := manifest.LoadJSONL(dir)
|
|
if !m.Has("x1") {
|
|
t.Error("should skip empty lines and parse valid JSON")
|
|
}
|
|
}
|
|
|
|
func TestLoadJSONLMalformedJSON(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "downloads.jsonl")
|
|
os.WriteFile(path, []byte("not-json\n{\"id\":\"x1\",\"filename\":\"a.jpg\",\"size\":1024,\"cloud\":\"local\",\"exported\":100}\n"), 0644)
|
|
m := manifest.LoadJSONL(dir)
|
|
if !m.Has("x1") {
|
|
t.Error("should skip malformed lines and parse valid ones")
|
|
}
|
|
}
|
|
|
|
func TestLoadJSONLEmptyID(t *testing.T) {
|
|
dir := t.TempDir()
|
|
path := filepath.Join(dir, "downloads.jsonl")
|
|
os.WriteFile(path, []byte("{\"id\":\"\",\"filename\":\"a.jpg\",\"size\":1024,\"cloud\":\"local\",\"exported\":100}\n"), 0644)
|
|
m := manifest.LoadJSONL(dir)
|
|
if m.Has("") {
|
|
t.Error("should skip entries with empty ID")
|
|
}
|
|
}
|
|
|
|
func TestJSONLAddBeforeOpenAppend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
if !m.Has("x1") {
|
|
t.Error("Add before OpenAppend should store in memory")
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
data, err := os.ReadFile(filepath.Join(dir, "downloads.jsonl"))
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if len(data) > 0 {
|
|
t.Error("entries added before OpenAppend should not be in file")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteHasBeforeOpenAppend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.Has("x1") {
|
|
t.Error("Has on unopened sqlite should return false")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteAddBeforeOpenAppend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteEntriesBeforeOpenAppend(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries := m.Entries()
|
|
if entries != nil {
|
|
t.Error("Entries on unopened sqlite should return nil")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteOpenAppendReadOnlyPath(t *testing.T) {
|
|
m, err := manifest.LoadSQLite("/proc/cannot-create-dir-here")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err == nil {
|
|
t.Error("expected error from OpenAppend on read-only path")
|
|
}
|
|
}
|
|
|
|
func TestFileExistsOnDirectory(t *testing.T) {
|
|
dir := t.TempDir()
|
|
if manifest.FileExists(dir) {
|
|
t.Error("manifest.FileExists should return false for directories")
|
|
}
|
|
}
|
|
|
|
func TestConvertFromJSONLOpenAppendError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
os.Chmod(filepath.Join(dir, "downloads.jsonl"), 0444)
|
|
defer os.Chmod(filepath.Join(dir, "downloads.jsonl"), 0644)
|
|
_, err := manifest.ConvertFromJSONL(dir)
|
|
if err == nil {
|
|
t.Error("expected error when JSONL cannot be opened for reading during conversion")
|
|
}
|
|
}
|
|
|
|
func TestConvertFromSQLiteReadError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
result, err := manifest.ConvertFromSQLite(dir)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
result.Close()
|
|
if _, err := os.Stat(manifest.JSONLPath(dir)); err != nil {
|
|
t.Errorf("JSONL file should exist after conversion: %v", err)
|
|
}
|
|
if os.Remove(manifest.SQLitePath(dir)) != nil {
|
|
t.Logf("sqlite file removed as expected")
|
|
}
|
|
}
|
|
|
|
func TestJSONLSaveWithoutFile(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.Save(); err != nil {
|
|
t.Errorf("Save without file should return nil, got %v", err)
|
|
}
|
|
}
|
|
|
|
func TestJSONLOpenAppendTwice(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Error("second OpenAppend should return nil, got", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteHasWithoutOpen(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if m.Has("anything") {
|
|
t.Error("Has on unopened SQLite should return false")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteAddWithoutOpen(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
if m.Has("x1") {
|
|
t.Error("Add without OpenAppend should not persist")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteEntriesWithoutOpen(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
entries := m.Entries()
|
|
if entries != nil {
|
|
t.Errorf("Entries without open should return nil, got %v", entries)
|
|
}
|
|
}
|
|
|
|
func TestSQLiteCloseWithoutOpen(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteOpenAppendCreateDir(t *testing.T) {
|
|
dir := t.TempDir()
|
|
subdir := filepath.Join(dir, "subdir")
|
|
m, err := manifest.LoadSQLite(subdir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Errorf("OpenAppend should create dir, got %v", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteSave(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
if err := m.Save(); err != nil {
|
|
t.Errorf("SQLite Save should return nil, got %v", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestOpenDefaultSQLite(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestOpenDefaultJSONLNoFile(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestParseFormat(t *testing.T) {
|
|
tests := []struct {
|
|
input string
|
|
want manifest.Format
|
|
err bool
|
|
}{
|
|
{"jsonl", manifest.FormatJSONL, false},
|
|
{"json", manifest.FormatJSONL, false},
|
|
{"sqlite", manifest.FormatSQLite, false},
|
|
{"db", manifest.FormatSQLite, false},
|
|
{"sqlite3", manifest.FormatSQLite, false},
|
|
{"invalid", "", true},
|
|
}
|
|
for _, tt := range tests {
|
|
got, err := manifest.ParseFormat(tt.input)
|
|
if tt.err && err == nil {
|
|
t.Errorf("ParseFormat(%q) expected error", tt.input)
|
|
}
|
|
if !tt.err && got != tt.want {
|
|
t.Errorf("ParseFormat(%q) = %v, want %v", tt.input, got, tt.want)
|
|
}
|
|
}
|
|
}
|
|
|
|
func TestJSONLSaveWithFileSync(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
if err := m.Save(); err != nil {
|
|
t.Errorf("Save with open file should succeed, got %v", err)
|
|
}
|
|
m.Close()
|
|
}
|
|
|
|
func TestConvertFromJSONLSuccess(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("c1", "photo.jpg", 1024, "cloud")
|
|
m.Close()
|
|
result, err := manifest.ConvertFromJSONL(dir)
|
|
if err != nil {
|
|
t.Fatalf("unexpected error: %v", err)
|
|
}
|
|
if !result.Has("c1") {
|
|
t.Error("converted SQLite manifest should have c1")
|
|
}
|
|
result.Close()
|
|
if manifest.FileExists(manifest.JSONLPath(dir)) {
|
|
t.Error("JSONL file should be deleted after conversion")
|
|
}
|
|
}
|
|
|
|
func TestConvertFromJSONLSQLiteOpenError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
subdir := filepath.Join(dir, "readonly")
|
|
if err := os.MkdirAll(subdir, 0755); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
srcPath := manifest.JSONLPath(dir)
|
|
dstPath := filepath.Join(subdir, "downloads.jsonl")
|
|
data, err := os.ReadFile(srcPath)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := os.WriteFile(dstPath, data, 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
}
|
|
|
|
func TestJSONLOpenAppendMkdirError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
jsonlFile := filepath.Join(dir, "downloads.jsonl")
|
|
if err := os.WriteFile(jsonlFile, []byte("{}\n"), 0644); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m := manifest.LoadJSONL(dir)
|
|
m.Close()
|
|
}
|
|
|
|
func TestSQLiteOpenAppendMkdirError(t *testing.T) {
|
|
m, err := manifest.LoadSQLite("/proc/cannot-write")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err == nil {
|
|
t.Error("expected error creating dir under /proc")
|
|
m.Close()
|
|
}
|
|
}
|
|
|
|
func TestSQLiteHasAfterClose(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
if m.Has("x1") {
|
|
t.Error("Has should return false after Close")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteEntriesAfterClose(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
entries := m.Entries()
|
|
if entries != nil {
|
|
t.Errorf("Entries after Close should return nil, got %d entries", len(entries))
|
|
}
|
|
}
|
|
|
|
func TestConvertFromJSONLNoFile(t *testing.T) {
|
|
dir := t.TempDir()
|
|
result, err := manifest.ConvertFromJSONL(dir)
|
|
if err != nil {
|
|
t.Fatalf("ConvertFromJSONL with no existing file should work: %v", err)
|
|
}
|
|
result.Close()
|
|
}
|
|
|
|
func TestConvertFromSQLiteNoFile(t *testing.T) {
|
|
dir := t.TempDir()
|
|
result, err := manifest.ConvertFromSQLite(dir)
|
|
if err != nil {
|
|
t.Fatalf("ConvertFromSQLite with no existing file should work: %v", err)
|
|
}
|
|
result.Close()
|
|
if _, err := os.Stat(manifest.JSONLPath(dir)); err != nil {
|
|
t.Errorf("JSONL file should exist after conversion: %v", err)
|
|
}
|
|
}
|
|
|
|
func TestConvertFromJSONLCloseSrcError(t *testing.T) {
|
|
}
|
|
|
|
func TestConvertFromSQLiteSaveError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
jsonlPath := manifest.JSONLPath(dir)
|
|
os.WriteFile(jsonlPath, []byte(""), 0444)
|
|
defer os.Chmod(jsonlPath, 0644)
|
|
_, err = manifest.ConvertFromSQLite(dir)
|
|
if err == nil {
|
|
t.Error("expected error from dst.Save during ConvertFromSQLite")
|
|
}
|
|
}
|
|
|
|
func TestOpenExistingJSONL(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Save()
|
|
m.Close()
|
|
m2, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !m2.Has("x1") {
|
|
t.Error("should have x1 after reopening")
|
|
}
|
|
m2.Close()
|
|
}
|
|
|
|
func TestOpenExistingSQLite(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
m2, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m2.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !m2.Has("x1") {
|
|
t.Error("should have x1 after reopening")
|
|
}
|
|
m2.Close()
|
|
}
|
|
|
|
func TestOpenConvertJSONLToSQLite(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m := manifest.LoadJSONL(dir)
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
m2, err := manifest.Open(dir, manifest.FormatSQLite)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !m2.Has("x1") {
|
|
t.Error("should have x1 after conversion")
|
|
}
|
|
m2.Close()
|
|
if manifest.FileExists(manifest.JSONLPath(dir)) {
|
|
t.Error("JSONL file should be deleted after conversion")
|
|
}
|
|
}
|
|
|
|
func TestOpenConvertSQLiteToJSONL(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
m2, err := manifest.Open(dir, manifest.FormatJSONL)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if !m2.Has("x1") {
|
|
t.Error("should have x1 after conversion")
|
|
}
|
|
m2.Close()
|
|
if manifest.FileExists(manifest.SQLitePath(dir)) {
|
|
t.Error("SQLite file should be deleted after conversion")
|
|
}
|
|
}
|
|
|
|
func TestSQLiteOpenAppendInvalidPath(t *testing.T) {
|
|
m, err := manifest.LoadSQLite("/dev/null/impossible/path")
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err == nil {
|
|
t.Error("expected error from OpenAppend with invalid path")
|
|
m.Close()
|
|
}
|
|
}
|
|
|
|
func TestConvertFromSQLiteJSONLOpenAppendError(t *testing.T) {
|
|
dir := t.TempDir()
|
|
m, err := manifest.LoadSQLite(dir)
|
|
if err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
if err := m.OpenAppend(); err != nil {
|
|
t.Fatal(err)
|
|
}
|
|
m.Add("x1", "photo.jpg", 1024, "local")
|
|
m.Close()
|
|
jsonlPath := manifest.JSONLPath(dir)
|
|
os.WriteFile(jsonlPath, []byte("{}\n"), 0444)
|
|
defer os.Chmod(jsonlPath, 0644)
|
|
_, err = manifest.ConvertFromSQLite(dir)
|
|
if err == nil {
|
|
t.Error("expected error from dst.OpenAppend during ConvertFromSQLite")
|
|
}
|
|
}
|
|
|
|
func TestJSONLOpenAppendMkdirAllError(t *testing.T) {
|
|
m := manifest.LoadJSONL("/dev/null/impossible/path/downloads.jsonl")
|
|
if err := m.OpenAppend(); err == nil {
|
|
t.Error("expected error from OpenAppend with impossible path")
|
|
m.Close()
|
|
}
|
|
}
|