feat: lazysecrets list prints every source, vault and secret, for fzf and editor pickers

internal/cachefile/cachefile.go

@@ -0,0 +1,146 @@
+// Package cachefile is the cache of what lazysecrets has loaded
+// (cache.json): the Sources lists, vaults, secrets' metadata and where
+// the cursor was in each, never values. The TUI and lazysecrets list
+// both read it and save to it; a save merges with what's on disk, so
+// neither undoes what the other loaded.
+package cachefile
+
+import (
+	"encoding/json"
+	"maps"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+)
+
+// Version is the file's format; a file of another version is ignored,
+// and replaced on the next save.
+const Version = 1
+
+// MaxAge drops what hasn't been loaded again for this long.
+const MaxAge = 30 * 24 * time.Hour
+
+// File is the cache. Every key starts with the provider ID: Sources are
+// keyed by it, the rest by Row, Objects' keys and the like.
+type File struct {
+	Version    int                                      `json:"version"`
+	Sources    map[string]Stamped[[]provider.Source]    `json:"sources,omitempty"` // by provider ID
+	Containers map[string]Stamped[[]provider.Container] `json:"containers,omitempty"`
+	Objects    map[string]Stamped[[]provider.Object]    `json:"objects,omitempty"`
+	Details    map[string]Stamped[provider.Object]      `json:"details,omitempty"`
+	Cursor     map[string]string                        `json:"cursor,omitempty"`
+}
+
+// Stamped is a list or detail with when it was loaded.
+type Stamped[T any] struct {
+	At    time.Time `json:"at"`
+	Value T         `json:"value"`
+}
+
+// New is an empty cache.
+func New() File {
+	return File{
+		Version:    Version,
+		Sources:    map[string]Stamped[[]provider.Source]{},
+		Containers: map[string]Stamped[[]provider.Container]{},
+		Objects:    map[string]Stamped[[]provider.Object]{},
+		Details:    map[string]Stamped[provider.Object]{},
+		Cursor:     map[string]string{},
+	}
+}
+
+// Row keys a source's vaults or scopes.
+func Row(providerID, sourceID string) string { return providerID + "|" + sourceID }
+
+// ContainerKey keys what's cached about a vault or scope.
+func ContainerKey(c provider.Container) string { return c.Source.Provider + "|" + c.ID }
+
+// ObjectsKey keys a vault's secrets, keys or certificates.
+func ObjectsKey(c provider.Container, t provider.ObjectType) string {
+	return ContainerKey(c) + "|" + string(t)
+}
+
+// ProviderOf is the provider ID a key starts with.
+func ProviderOf(key string) string {
+	id, _, _ := strings.Cut(key, "|")
+	return id
+}
+
+// Read reads path; a missing, unreadable or other version's file is an
+// empty cache.
+func Read(path string) File {
+	f := New()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return f
+	}
+	var d File
+	if json.Unmarshal(b, &d) != nil || d.Version != Version {
+		return f
+	}
+	maps.Copy(f.Sources, d.Sources)
+	maps.Copy(f.Containers, d.Containers)
+	maps.Copy(f.Objects, d.Objects)
+	maps.Copy(f.Details, d.Details)
+	maps.Copy(f.Cursor, d.Cursor)
+	return f
+}
+
+// Save writes f to path, merged with the file there: of a list in both,
+// the one loaded last wins, and f's cursors win. Entries older than
+// MaxAge, and those of providers keep says no to (accounts signed out
+// of), are dropped, from the file's side as well.
+func Save(path string, f File, keep func(providerID string) bool, now time.Time) error {
+	out := New()
+	ok := func(key string, at time.Time) bool { return keep(ProviderOf(key)) && now.Sub(at) < MaxAge }
+	disk := Read(path)
+	merge(out.Sources, disk.Sources, f.Sources, ok)
+	merge(out.Containers, disk.Containers, f.Containers, ok)
+	merge(out.Objects, disk.Objects, f.Objects, ok)
+	merge(out.Details, disk.Details, f.Details, ok)
+	for _, m := range []map[string]string{disk.Cursor, f.Cursor} {
+		for k, v := range m {
+			if keep(ProviderOf(k)) {
+				out.Cursor[k] = v
+			}
+		}
+	}
+	return write(path, out)
+}
+
+func merge[T any](out, disk, ours map[string]Stamped[T], ok func(string, time.Time) bool) {
+	for _, m := range []map[string]Stamped[T]{disk, ours} {
+		for k, e := range m {
+			if old, seen := out[k]; ok(k, e.At) && (!seen || !e.At.Before(old.At)) {
+				out[k] = e
+			}
+		}
+	}
+}
+
+// write writes f atomically, readable by the user only.
+func write(path string, f File) error {
+	b, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".cache-*.json")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name()) // after a successful rename, nothing there
+	if _, err := tmp.Write(b); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

internal/cachefile/cachefile_test.go

@@ -0,0 +1,91 @@
+package cachefile
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+)
+
+func list(at time.Time, names ...string) Stamped[[]provider.Container] {
+	var cs []provider.Container
+	for _, n := range names {
+		cs = append(cs, provider.Container{Name: n})
+	}
+	return Stamped[[]provider.Container]{At: at, Value: cs}
+}
+
+func TestSaveMerges(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	now := time.Now()
+	all := func(string) bool { return true }
+
+	// The TUI saved two lists.
+	tui := New()
+	tui.Containers["az|a"] = list(now.Add(-time.Hour), "kv-a-old")
+	tui.Containers["az|b"] = list(now.Add(-time.Minute), "kv-b")
+	tui.Cursor["az|b"] = "kv-b"
+	if err := Save(path, tui, all, now); err != nil {
+		t.Fatal(err)
+	}
+	// lazysecrets list loads a again, and c.
+	listed := Read(path)
+	listed.Containers["az|a"] = list(now, "kv-a-new")
+	listed.Containers["az|c"] = list(now, "kv-c")
+	if err := Save(path, listed, all, now); err != nil {
+		t.Fatal(err)
+	}
+	// The TUI saves what it has, older: list's lists stay.
+	if err := Save(path, tui, all, now); err != nil {
+		t.Fatal(err)
+	}
+	f := Read(path)
+	for key, want := range map[string]string{"az|a": "kv-a-new", "az|b": "kv-b", "az|c": "kv-c"} {
+		if got := f.Containers[key].Value; len(got) != 1 || got[0].Name != want {
+			t.Errorf("%s = %v, want %s", key, got, want)
+		}
+	}
+	if f.Cursor["az|b"] != "kv-b" {
+		t.Error("cursor lost")
+	}
+}
+
+func TestSaveDrops(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	now := time.Now()
+	f := New()
+	f.Containers["az|old"] = list(now.Add(-MaxAge-time.Hour), "kv-old")
+	f.Containers["gone|x"] = list(now, "kv-gone")
+	f.Containers["az|x"] = list(now, "kv-x")
+	if err := Save(path, f, func(id string) bool { return true }, now); err != nil {
+		t.Fatal(err)
+	}
+	// Signed out of "gone" since: dropped from the file too.
+	if err := Save(path, New(), func(id string) bool { return id == "az" }, now); err != nil {
+		t.Fatal(err)
+	}
+	got := Read(path).Containers
+	if _, ok := got["az|old"]; ok {
+		t.Error("kept a list older than MaxAge")
+	}
+	if _, ok := got["gone|x"]; ok {
+		t.Error("kept a signed-out account's list")
+	}
+	if _, ok := got["az|x"]; !ok {
+		t.Error("dropped a list on disk the save didn't have")
+	}
+}
+
+func TestReadOtherVersion(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	f := New()
+	f.Version = Version + 1
+	f.Containers["az|x"] = list(time.Now(), "kv-x")
+	if err := write(path, f); err != nil {
+		t.Fatal(err)
+	}
+	if got := Read(path); len(got.Containers) != 0 || got.Version != Version {
+		t.Errorf("read another version's file: %+v", got)
+	}
+}

internal/listing/listing.go

@@ -0,0 +1,285 @@
+// Package listing lists what lazysecrets can find, for lazysecrets list:
+// what Go to searches, without the TUI. What's cached comes first, at
+// once; then lists that are missing or older than MaxAge load in the
+// background, a few at a time, and what's new in them follows as they
+// arrive. Repos are always read, from disk. Values are never listed.
+package listing
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+	"git.kebcloud.no/keb/lazysecrets/internal/ref"
+)
+
+// Workers is how many lists load at once, as in Go to.
+const Workers = 4
+
+// Entry is a source, vault or item, by its lazysecrets:// link.
+type Entry struct {
+	Link string `json:"link"`
+	// Kind is source (a subscription, workspace or repo), vault (a Key
+	// Vault, secret scope or repo file), secret, key or certificate.
+	Kind string `json:"kind"`
+	// Provider is azure, databricks or repo, as in links.
+	Provider string `json:"provider"`
+	// Path names it: its source's, vault's and own names.
+	Path []string `json:"path"`
+	// Account is the az account it's seen through, when there are several.
+	Account string `json:"account,omitempty"`
+}
+
+type Options struct {
+	Providers []provider.Provider
+	// Cache is what was loaded before; Run adds what it loads.
+	Cache cachefile.File
+	// MaxAge: cached lists loaded longer ago are loaded again.
+	MaxAge time.Duration
+	// Cached lists only what's cached (and repos): nothing is loaded.
+	Cached  bool
+	Timeout func(provider.Kind) time.Duration
+	Now     func() time.Time
+}
+
+// Run lists everything, calling emit for each entry once, from this
+// goroutine. An error from emit (the reader went away) stops it. It
+// returns the cache with what was loaded, and the lists that couldn't be.
+func Run(ctx context.Context, o Options, emit func(Entry) error) (cachefile.File, []error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	l := &lister{o: o, emit: emit, seen: map[string]bool{}, pending: map[string]bool{}, results: make(chan result)}
+	if l.o.Now == nil {
+		l.o.Now = time.Now
+	}
+	for _, p := range o.Providers {
+		if l.stopped != nil {
+			break
+		}
+		l.provider(p)
+	}
+	l.start(ctx)
+	for l.active > 0 {
+		r := <-l.results
+		l.active--
+		if l.stopped == nil {
+			l.handle(r)
+			l.start(ctx)
+		}
+		if l.stopped != nil {
+			cancel()
+		}
+	}
+	return l.o.Cache, l.errs
+}
+
+type job struct {
+	p   provider.Provider
+	src *provider.Source    // list its vaults
+	c   *provider.Container // list its secrets
+	key string              // where the list is cached
+}
+
+type result struct {
+	job
+	sources    []provider.Source
+	containers []provider.Container
+	objects    []provider.Object
+	err        error
+}
+
+type lister struct {
+	o       Options
+	emit    func(Entry) error
+	seen    map[string]bool // provider ID + link of entries emitted
+	pending map[string]bool // cache keys queued or loading
+	queue   []job
+	active  int
+	results chan result
+	errs    []error
+	stopped error // emit's error
+}
+
+func (l *lister) cacheable(p provider.Provider) bool { return p.Kind() != provider.KindRepo }
+
+// load reports whether to load a list: a repo's always (from disk), else
+// unless Cached, when it isn't cached or is older than MaxAge.
+func (l *lister) load(p provider.Provider, cached bool, at time.Time) bool {
+	if !l.cacheable(p) {
+		return true
+	}
+	return !l.o.Cached && (!cached || l.o.Now().Sub(at) >= l.o.MaxAge)
+}
+
+func (l *lister) provider(p provider.Provider) {
+	e, ok := l.o.Cache.Sources[p.ID()]
+	if l.cacheable(p) && ok {
+		for _, src := range sorted(e.Value, func(s provider.Source) string { return s.Name }) {
+			l.source(p, src)
+		}
+	}
+	if l.load(p, ok, e.At) {
+		l.enqueue(job{p: p, key: p.ID()})
+	}
+}
+
+// source emits src, and what's cached under it, and queues what's not.
+func (l *lister) source(p provider.Provider, src provider.Source) {
+	src.Provider, src.ProviderLabel = p.ID(), p.Label()
+	if r, err := ref.ForSource(src); err == nil {
+		l.put(p, Entry{Link: r.String(), Kind: "source", Provider: kindOf(src.Kind), Path: []string{src.Name}, Account: accountOf(src)})
+	}
+	row := cachefile.Row(p.ID(), src.ID)
+	e, ok := l.o.Cache.Containers[row]
+	if l.cacheable(p) && ok {
+		l.containers(p, src, e.Value)
+	}
+	if l.load(p, ok, e.At) {
+		l.enqueue(job{p: p, src: &src, key: row})
+	}
+}
+
+func (l *lister) containers(p provider.Provider, src provider.Source, cs []provider.Container) {
+	for _, c := range sorted(cs, func(c provider.Container) string { return c.Name }) {
+		c.Source = src
+		if r, err := ref.ForContainer(c, provider.TypeSecret); err == nil {
+			l.put(p, Entry{Link: r.String(), Kind: "vault", Provider: kindOf(src.Kind), Path: []string{src.Name, c.Name}, Account: accountOf(src)})
+		}
+		key := cachefile.ObjectsKey(c, provider.TypeSecret)
+		e, ok := l.o.Cache.Objects[key]
+		if l.cacheable(p) {
+			// Keys and certificates only as far as they were opened in
+			// the TUI: Go to doesn't load them either.
+			for _, t := range []provider.ObjectType{provider.TypeSecret, provider.TypeKey, provider.TypeCertificate} {
+				l.objects(p, c, l.o.Cache.Objects[cachefile.ObjectsKey(c, t)].Value)
+			}
+		}
+		if l.load(p, ok, e.At) {
+			l.enqueue(job{p: p, c: &c, key: key})
+		}
+	}
+}
+
+func (l *lister) objects(p provider.Provider, c provider.Container, objs []provider.Object) {
+	for _, o := range sorted(objs, func(o provider.Object) string { return o.Name }) {
+		o.Container = c
+		r, err := ref.ForObject(o, "")
+		if err != nil {
+			continue
+		}
+		l.put(p, Entry{Link: r.String(), Kind: strings.ToLower(string(o.Type)), Provider: kindOf(c.Source.Kind),
+			Path: []string{c.Source.Name, c.Name, o.Name}, Account: accountOf(c.Source)})
+	}
+}
+
+// put emits e, unless it was already (or emitting has failed).
+func (l *lister) put(p provider.Provider, e Entry) {
+	id := p.ID() + "\x00" + e.Link
+	if l.stopped != nil || l.seen[id] {
+		return
+	}
+	l.seen[id] = true
+	l.stopped = l.emit(e)
+}
+
+func (l *lister) enqueue(j job) {
+	if l.pending[j.key] {
+		return
+	}
+	l.pending[j.key] = true
+	l.queue = append(l.queue, j)
+}
+
+func (l *lister) start(ctx context.Context) {
+	for l.stopped == nil && l.active < Workers && len(l.queue) > 0 {
+		j := l.queue[0]
+		l.queue = l.queue[1:]
+		l.active++
+		go func() { l.results <- l.fetch(ctx, j) }()
+	}
+}
+
+func (l *lister) fetch(ctx context.Context, j job) result {
+	kind := j.p.Kind()
+	ctx, cancel := context.WithTimeout(ctx, l.o.Timeout(kind))
+	defer cancel()
+	r := result{job: j}
+	switch {
+	case j.c != nil:
+		r.objects, r.err = j.p.Objects(ctx, *j.c, provider.TypeSecret)
+	case j.src != nil:
+		r.containers, r.err = j.p.Containers(ctx, *j.src)
+	default:
+		r.sources, r.err = j.p.Sources(ctx)
+	}
+	return r
+}
+
+func (l *lister) handle(r result) {
+	delete(l.pending, r.key)
+	now := l.o.Now()
+	if r.err != nil {
+		l.errs = append(l.errs, fmt.Errorf("%s: %w", r.label(), r.err))
+		return
+	}
+	cache := l.cacheable(r.p)
+	switch {
+	case r.c != nil:
+		if cache {
+			l.o.Cache.Objects[r.key] = cachefile.Stamped[[]provider.Object]{At: now, Value: r.objects}
+		}
+		l.objects(r.p, *r.c, r.objects)
+	case r.src != nil:
+		r.containers = slices.Clone(r.containers) // the provider's may be shared
+		for i := range r.containers {
+			r.containers[i].Source = *r.src
+		}
+		if cache {
+			l.o.Cache.Containers[r.key] = cachefile.Stamped[[]provider.Container]{At: now, Value: r.containers}
+		}
+		l.containers(r.p, *r.src, r.containers)
+	default:
+		if cache {
+			l.o.Cache.Sources[r.key] = cachefile.Stamped[[]provider.Source]{At: now, Value: r.sources}
+		}
+		for _, src := range r.sources {
+			l.source(r.p, src)
+		}
+	}
+}
+
+// label names a list that couldn't be loaded.
+func (r result) label() string {
+	switch {
+	case r.c != nil:
+		return r.c.Source.Name + " / " + r.c.Name
+	case r.src != nil:
+		return r.src.Name
+	}
+	return r.p.Label()
+}
+
+func kindOf(k provider.Kind) string { return strings.ToLower(string(k)) }
+
+// accountOf is the account part of a source's provider label ("alice" in
+// "Azure · alice"), or "" when there's a single account.
+func accountOf(src provider.Source) string {
+	account, ok := strings.CutPrefix(src.ProviderLabel, string(src.Kind)+" · ")
+	if !ok {
+		return ""
+	}
+	return account
+}
+
+func sorted[T any](s []T, name func(T) string) []T {
+	s = slices.Clone(s)
+	slices.SortStableFunc(s, func(a, b T) int {
+		return cmp.Compare(strings.ToLower(name(a)), strings.ToLower(name(b)))
+	})
+	return s
+}

internal/listing/listing_test.go

@@ -0,0 +1,176 @@
+package listing
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider/fake"
+)
+
+// counting counts a provider's requests.
+type counting struct {
+	provider.Provider
+	calls atomic.Int32
+}
+
+func (c *counting) Sources(ctx context.Context) ([]provider.Source, error) {
+	c.calls.Add(1)
+	return c.Provider.Sources(ctx)
+}
+
+func (c *counting) Containers(ctx context.Context, src provider.Source) ([]provider.Container, error) {
+	c.calls.Add(1)
+	return c.Provider.Containers(ctx, src)
+}
+
+func (c *counting) Objects(ctx context.Context, ct provider.Container, t provider.ObjectType) ([]provider.Object, error) {
+	c.calls.Add(1)
+	return c.Provider.Objects(ctx, ct, t)
+}
+
+func run(t *testing.T, o Options) ([]Entry, cachefile.File, []error) {
+	t.Helper()
+	if o.Cache.Version == 0 {
+		o.Cache = cachefile.New()
+	}
+	o.Timeout = func(provider.Kind) time.Duration { return 5 * time.Second }
+	var got []Entry
+	f, errs := Run(context.Background(), o, func(e Entry) error {
+		got = append(got, e)
+		return nil
+	})
+	return got, f, errs
+}
+
+func has(es []Entry, kind, path string) bool {
+	for _, e := range es {
+		if e.Kind == kind && strings.Join(e.Path, "/") == path {
+			return true
+		}
+	}
+	return false
+}
+
+func TestListLoadsAndCaches(t *testing.T) {
+	t.Parallel()
+	az := &counting{Provider: fake.Azure(0)}
+	got, f, errs := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, MaxAge: time.Minute})
+	for _, want := range [][2]string{
+		{"source", "platform-prod"}, {"vault", "platform-prod/kv-app-prod"}, {"secret", "platform-prod/kv-app-prod/db-password"},
+		{"source", "lazysecrets-demo"}, {"secret", "lazysecrets-demo/.env/DATABASE_URL"},
+	} {
+		if !has(got, want[0], want[1]) {
+			t.Errorf("no %s %s", want[0], want[1])
+		}
+	}
+	if len(errs) != 1 || !strings.Contains(errs[0].Error(), "kv-restricted: 403") {
+		t.Errorf("errs = %v", errs)
+	}
+	seen := map[string]bool{}
+	for _, e := range got {
+		if seen[e.Link] {
+			t.Errorf("%s listed twice", e.Link)
+		}
+		seen[e.Link] = true
+		if e.Provider != "azure" && e.Provider != "repo" {
+			t.Errorf("provider %q", e.Provider)
+		}
+	}
+	if len(f.Sources) != 1 || len(f.Containers) == 0 || len(f.Objects) == 0 {
+		t.Errorf("not cached: %d sources, %d containers, %d objects", len(f.Sources), len(f.Containers), len(f.Objects))
+	}
+	for k := range f.Containers {
+		if cachefile.ProviderOf(k) != az.ID() {
+			t.Errorf("cached a repo: %s", k)
+		}
+	}
+
+	// Again with that cache: the same, without asking Azure.
+	before := az.calls.Load()
+	again, _, _ := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, Cache: f, MaxAge: time.Minute})
+	if len(again) != len(got) {
+		t.Errorf("%d entries from the cache, %d loaded", len(again), len(got))
+	}
+	// The vault that couldn't be read wasn't cached: it's asked again.
+	if n := az.calls.Load() - before; n != 1 {
+		t.Errorf("%d requests with a fresh cache, want 1 (kv-restricted)", n)
+	}
+
+	// Stale: loaded again, but each entry is still listed once.
+	before = az.calls.Load()
+	stale, _, _ := run(t, Options{Providers: []provider.Provider{az}, Cache: f, MaxAge: time.Minute,
+		Now: func() time.Time { return time.Now().Add(time.Hour) }})
+	if az.calls.Load()-before < 3 {
+		t.Error("stale lists weren't loaded again")
+	}
+	links := map[string]bool{}
+	for _, e := range stale {
+		if links[e.Link] {
+			t.Errorf("%s listed twice", e.Link)
+		}
+		links[e.Link] = true
+	}
+}
+
+func TestListCachedOnly(t *testing.T) {
+	t.Parallel()
+	az := &counting{Provider: fake.Azure(0)}
+	_, f, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute})
+	before := az.calls.Load()
+	got, _, errs := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, Cache: f, Cached: true,
+		Now: func() time.Time { return time.Now().Add(24 * time.Hour) }})
+	if az.calls.Load() != before || len(errs) != 0 {
+		t.Errorf("--cached asked Azure: %d requests, %v", az.calls.Load()-before, errs)
+	}
+	if !has(got, "secret", "platform-prod/kv-app-prod/db-password") || !has(got, "secret", "lazysecrets-demo/.env/DATABASE_URL") {
+		t.Error("cached secrets or repos missing")
+	}
+}
+
+func TestListCachedFirst(t *testing.T) {
+	t.Parallel()
+	az := fake.Azure(0)
+	_, f, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute})
+	// Forget one vault's secrets: they load, after everything cached.
+	var dropped string
+	for k := range f.Objects {
+		if strings.Contains(k, "kv-app-prod") && strings.HasSuffix(k, "|Secret") {
+			dropped = k
+			delete(f.Objects, k)
+			break
+		}
+	}
+	if dropped == "" {
+		t.Fatalf("no kv-app-prod in %v", f.Objects)
+	}
+	got, _, _ := run(t, Options{Providers: []provider.Provider{az}, Cache: f, MaxAge: time.Minute})
+	last := got[len(got)-1]
+	if last.Kind != "secret" || last.Path[1] != "kv-app-prod" {
+		t.Errorf("last = %+v, want a kv-app-prod secret, loaded after the cache", last)
+	}
+}
+
+func TestListStopsWhenTheReaderGoes(t *testing.T) {
+	t.Parallel()
+	gone := errors.New("broken pipe")
+	n := 0
+	_, errs := Run(context.Background(), Options{
+		Providers: []provider.Provider{fake.Azure(0), fake.Repos()}, Cache: cachefile.New(), MaxAge: time.Minute,
+		Timeout: func(provider.Kind) time.Duration { return time.Second },
+	}, func(Entry) error {
+		n++
+		if n == 2 {
+			return gone
+		}
+		return nil
+	})
+	if n != 2 || len(errs) != 0 {
+		t.Errorf("emitted %d after the reader went, errs %v", n, errs)
+	}
+}

internal/ui/app/persist.go

@@ -1,14 +1,12 @@
 package app
 
 import (
-	"encoding/json"
 	"errors"
-	"os"
-	"path/filepath"
 	"time"
 
 	tea "charm.land/bubbletea/v2"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/ref"
 	"git.kebcloud.no/keb/lazysecrets/internal/store"
@@ -23,58 +21,35 @@ import (
 // And where lazysecrets was left, per folder it ran in, goes in the store,
 // to go back there on the next start.
 
-// cacheFileVersion is the file's format; a file of another version is
-// ignored, and replaced on the next save.
-const cacheFileVersion = 1
-
-// cacheMaxAge drops what hasn't been loaded again for this long.
-const cacheMaxAge = 30 * 24 * time.Hour
-
 // persistEvery is how often a changed cache is saved while running, so
 // little is lost if lazysecrets is killed rather than quit.
 const persistEvery = 30 * time.Second
 
-type cacheFile struct {
-	Version    int                                      `json:"version"`
-	Sources    map[string]stamped[[]provider.Source]    `json:"sources,omitempty"` // by provider ID
-	Containers map[string]stamped[[]provider.Container] `json:"containers,omitempty"`
-	Objects    map[string]stamped[[]provider.Object]    `json:"objects,omitempty"`
-	Details    map[string]stamped[provider.Object]      `json:"details,omitempty"`
-	Cursor     map[string]string                        `json:"cursor,omitempty"`
-}
-
-type stamped[T any] struct {
-	At    time.Time `json:"at"`
-	Value T         `json:"value"`
-}
-
 type persistTickMsg struct{}
 
 type persistedMsg struct{ err error }
 
 // keepCached reports whether a cache key's account still exists and
 // isn't a repo: every key starts with the provider ID.
-func (m *Model) keepCached(key string) bool {
-	id, _ := splitSourceRow(key)
-	p, ok := m.byID[id]
-	return ok && p.Kind() != provider.KindRepo
+func (m *Model) keepCached(key string) bool { return m.keepProvider(cachefile.ProviderOf(key)) }
+
+func (m *Model) keepProvider(id string) bool { return keepIn(m.byID)(id) }
+
+func keepIn(byID map[string]provider.Provider) func(id string) bool {
+	return func(id string) bool {
+		p, ok := byID[id]
+		return ok && p.Kind() != provider.KindRepo
+	}
 }
 
 // loadCacheFile fills the cache from the file, leaving out what's too
 // old or belongs to accounts that aren't signed in any more.
 func (m *Model) loadCacheFile() {
-	b, err := os.ReadFile(m.cachePath)
-	if err != nil {
-		return
-	}
-	var f cacheFile
-	if json.Unmarshal(b, &f) != nil || f.Version != cacheFileVersion {
-		return
-	}
+	f := cachefile.Read(m.cachePath)
 	now := m.now()
-	recent := func(at time.Time) bool { return now.Sub(at) < cacheMaxAge }
+	recent := func(at time.Time) bool { return now.Sub(at) < cachefile.MaxAge }
 	for id, e := range f.Sources {
-		if m.keepCached(id+"|") && recent(e.At) {
+		if m.keepProvider(id) && recent(e.At) {
 			m.sourceResults[id] = sourcesMsg{gen: m.sourcesGen, provider: id, sources: e.Value}
 			m.sourcesAt[id] = e.At
 		}
@@ -102,33 +77,26 @@ func (m *Model) loadCacheFile() {
 }
 
 // cacheSnapshot is the cache as the file holds it.
-func (m *Model) cacheSnapshot() cacheFile {
-	f := cacheFile{
-		Version:    cacheFileVersion,
-		Sources:    map[string]stamped[[]provider.Source]{},
-		Containers: map[string]stamped[[]provider.Container]{},
-		Objects:    map[string]stamped[[]provider.Object]{},
-		Details:    map[string]stamped[provider.Object]{},
-		Cursor:     map[string]string{},
-	}
+func (m *Model) cacheSnapshot() cachefile.File {
+	f := cachefile.New()
 	for id, r := range m.sourceResults {
-		if r.err == nil && m.keepCached(id+"|") {
-			f.Sources[id] = stamped[[]provider.Source]{m.sourcesAt[id], r.sources}
+		if r.err == nil && m.keepProvider(id) {
+			f.Sources[id] = cachefile.Stamped[[]provider.Source]{At: m.sourcesAt[id], Value: r.sources}
 		}
 	}
 	for k, e := range m.cache.containers {
 		if m.keepCached(k) {
-			f.Containers[k] = stamped[[]provider.Container]{e.at, e.value}
+			f.Containers[k] = cachefile.Stamped[[]provider.Container]{At: e.at, Value: e.value}
 		}
 	}
 	for k, e := range m.cache.objects {
 		if m.keepCached(k) {
-			f.Objects[k] = stamped[[]provider.Object]{e.at, e.value}
+			f.Objects[k] = cachefile.Stamped[[]provider.Object]{At: e.at, Value: e.value}
 		}
 	}
 	for k, e := range m.cache.details {
 		if m.keepCached(k) {
-			f.Details[k] = stamped[provider.Object]{e.at, e.value}
+			f.Details[k] = cachefile.Stamped[provider.Object]{At: e.at, Value: e.value}
 		}
 	}
 	for k, v := range m.cache.cursor {
@@ -139,28 +107,11 @@ func (m *Model) cacheSnapshot() cacheFile {
 	return f
 }
 
-// writeCacheFile writes f atomically, readable by the user only.
-func writeCacheFile(path string, f cacheFile) error {
-	b, err := json.Marshal(f)
-	if err != nil {
-		return err
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
-		return err
-	}
-	tmp, err := os.CreateTemp(filepath.Dir(path), ".cache-*.json")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tmp.Name()) // after a successful rename, nothing there
-	if _, err := tmp.Write(b); err != nil {
-		tmp.Close()
-		return err
-	}
-	if err := tmp.Close(); err != nil {
-		return err
-	}
-	return os.Rename(tmp.Name(), path)
+// saveCache saves f, merged with the file (lazysecrets list may have
+// loaded more since), from any goroutine.
+func (m *Model) saveCache(f cachefile.File) func() error {
+	path, keep, now := m.cachePath, keepIn(m.byID), m.now()
+	return func() error { return cachefile.Save(path, f, keep, now) }
 }
 
 // persistTick saves the cache, and where lazysecrets is, in the
@@ -169,8 +120,8 @@ func (m *Model) persistTick() tea.Cmd {
 	cmds := []tea.Cmd{tea.Tick(persistEvery, func(time.Time) tea.Msg { return persistTickMsg{} })}
 	if m.cacheDirty && m.cachePath != "" {
 		m.cacheDirty = false
-		f, path := m.cacheSnapshot(), m.cachePath
-		cmds = append(cmds, func() tea.Msg { return persistedMsg{writeCacheFile(path, f)} })
+		save := m.saveCache(m.cacheSnapshot())
+		cmds = append(cmds, func() tea.Msg { return persistedMsg{save()} })
 	}
 	if p, ok := m.place(); ok && m.placeDir != "" && p.Link != m.savedPlace {
 		m.savedPlace = p.Link
@@ -185,7 +136,7 @@ func (m *Model) persistTick() tea.Cmd {
 func (m Model) Persist() error {
 	var errs []error
 	if m.cachePath != "" {
-		errs = append(errs, writeCacheFile(m.cachePath, m.cacheSnapshot()))
+		errs = append(errs, m.saveCache(m.cacheSnapshot())())
 	}
 	if m.placeDir != "" {
 		if p, ok := m.place(); ok {

internal/ui/app/persist_test.go

@@ -1,12 +1,14 @@
 package app
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider/fake"
 	"git.kebcloud.no/keb/lazysecrets/internal/ref"
@@ -80,15 +82,16 @@ func TestCacheFileDropsOldAndSignedOut(t *testing.T) {
 	file := filepath.Join(t.TempDir(), "cache.json")
 	az := fake.Azure(0)
 	now := time.Now()
-	f := cacheFile{
-		Version: cacheFileVersion,
-		Containers: map[string]stamped[[]provider.Container]{
-			az.ID() + "|recent":     {now.Add(-time.Hour), []provider.Container{{Name: "kv-recent"}}},
-			az.ID() + "|old":        {now.Add(-cacheMaxAge - time.Hour), []provider.Container{{Name: "kv-old"}}},
-			"Azure:gone@x/t|recent": {now, []provider.Container{{Name: "kv-gone"}}},
+	f := cachefile.File{
+		Version: cachefile.Version,
+		Containers: map[string]cachefile.Stamped[[]provider.Container]{
+			az.ID() + "|recent":     {At: now.Add(-time.Hour), Value: []provider.Container{{Name: "kv-recent"}}},
+			az.ID() + "|old":        {At: now.Add(-cachefile.MaxAge - time.Hour), Value: []provider.Container{{Name: "kv-old"}}},
+			"Azure:gone@x/t|recent": {At: now, Value: []provider.Container{{Name: "kv-gone"}}},
 		},
 	}
-	if err := writeCacheFile(file, f); err != nil {
+	b, _ := json.Marshal(f) // as an older lazysecrets left it
+	if err := os.WriteFile(file, b, 0o600); err != nil {
 		t.Fatal(err)
 	}
 	d := newDriverOpts(t, Options{CacheFile: file}, az)
@@ -97,7 +100,7 @@ func TestCacheFileDropsOldAndSignedOut(t *testing.T) {
 		t.Error("dropped a recent entry")
 	}
 	if _, ok := c[az.ID()+"|old"]; ok {
-		t.Error("kept an entry older than cacheMaxAge")
+		t.Error("kept an entry older than cachefile.MaxAge")
 	}
 	if _, ok := c["Azure:gone@x/t|recent"]; ok {
 		t.Error("kept an entry of an account that isn't signed in")

internal/ui/app/route.go

@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"slices"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 )
 
 // containerKey identifies a container in the UI and caches. The same vault
 // seen through two accounts must not share cached lists, since each
 // account may see different things.
-func containerKey(c provider.Container) string { return c.Source.Provider + "|" + c.ID }
+func containerKey(c provider.Container) string { return cachefile.ContainerKey(c) }
 
 // providerFor finds the provider instance a source came from. Sources
 // remembered from an earlier run (favourites) may name an account that

internal/ui/app/sources.go

@@ -6,6 +6,7 @@ import (
 	"slices"
 	"strings"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/store"
 	"git.kebcloud.no/keb/lazysecrets/internal/ui/icons"
@@ -39,7 +40,7 @@ type sourceRow struct {
 
 // sourceRowKey joins a provider ID and a source ID. "|" doesn't occur in
 // either; provider IDs contain "/" (Azure:user/tenant).
-func sourceRowKey(providerID, sourceID string) string { return providerID + "|" + sourceID }
+func sourceRowKey(providerID, sourceID string) string { return cachefile.Row(providerID, sourceID) }
 
 func splitSourceRow(row string) (providerID, sourceID string) {
 	providerID, sourceID, _ = strings.Cut(row, "|")