feat: lazysecrets list prints every source, vault and secret, for fzf and editor pickers

README.md

@@ -20,7 +20,8 @@ A terminal UI for finding, reading and sharing secrets. Inspired by the fantasti
 - **Careful with values**: values stay hidden until you reveal them, and
   copied values are cleared from the clipboard after 30 seconds.
 - **Scriptable**: `lazysecrets get <link>` prints a value for scripts and
-  pipes.
+  pipes, and `lazysecrets list` everything it can find, for fzf and editor
+  pickers.
 - **Favourites**: star the secrets you use most for easy access.
 - **Expiry warnings**: secrets, keys and certificates that expire soon are
   marked, listed together in the status pane.
@@ -86,7 +87,7 @@ use most:
 - [Azure Key Vault](docs/azure.md): accounts, permissions, pinned vaults
 - [Databricks](docs/databricks.md): profiles and Key Vault-backed scopes
 - [SOPS and .env files](docs/repos.md): which repos and files, decrypting
-- [Links](docs/links.md): sharing, opening, `lazysecrets get` in scripts,
+- [Links](docs/links.md): sharing, opening, `lazysecrets get` and `list` in scripts,
   and opening links from the browser and chat
 - [Custom commands](docs/custom-commands.md): run your own commands on a
   secret

cmd/lazysecrets/list.go

@@ -0,0 +1,97 @@
+package main
+
+import (
+	"context"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"io"
+	"os"
+	"os/signal"
+	"strings"
+	"syscall"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
+	"git.kebcloud.no/keb/lazysecrets/internal/listing"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+)
+
+// listCmd prints every source, vault and secret lazysecrets can find, one
+// per line, for fzf, editor pickers and scripts:
+//
+//	lazysecrets list | fzf --delimiter='\t' --with-nth=3 | cut -f1
+//
+// What's cached prints at once; what's missing or older than --max-age
+// follows as it loads. Lists that can't be read are reported on stderr.
+func listCmd(providers []provider.Provider, cachePath string, timeout func(provider.Kind) time.Duration, args []string, out, errOut io.Writer) error {
+	fs := flag.NewFlagSet("list", flag.ContinueOnError)
+	cached := fs.Bool("cached", false, "only what's cached, and repos: load nothing from Azure or Databricks")
+	maxAge := fs.Duration("max-age", 2*time.Minute, "load cached lists older than this again")
+	format := fs.String("format", "text", "text: link, kind and name, tab-separated; json: an object per line")
+	if err := fs.Parse(args); err != nil {
+		return err
+	}
+	if fs.NArg() > 0 {
+		return fmt.Errorf("usage: lazysecrets list [--cached] [--max-age 2m] [--format text|json]")
+	}
+	var write func(listing.Entry) error
+	switch *format {
+	case "text":
+		write = func(e listing.Entry) error {
+			name := strings.Join(e.Path, " / ")
+			if e.Account != "" {
+				name += " · " + e.Account
+			}
+			_, err := fmt.Fprintf(out, "%s\t%s\t%s\n", e.Link, e.Kind, name)
+			return err
+		}
+	case "json":
+		enc := json.NewEncoder(out)
+		enc.SetEscapeHTML(false)
+		write = func(e listing.Entry) error { return enc.Encode(e) }
+	default:
+		return fmt.Errorf("--format: %q is not text or json", *format)
+	}
+
+	// When the reader goes away (a pick in fzf), writing fails instead of
+	// killing the process, so what loaded is still saved.
+	signal.Notify(make(chan os.Signal, 1), syscall.SIGPIPE)
+	var gone error
+	emit := func(e listing.Entry) error {
+		if err := write(e); err != nil {
+			gone = err
+			return err
+		}
+		return nil
+	}
+
+	f := cachefile.New()
+	if cachePath != "" {
+		f = cachefile.Read(cachePath)
+	}
+	f, errs := listing.Run(context.Background(), listing.Options{
+		Providers: providers, Cache: f, MaxAge: *maxAge, Cached: *cached, Timeout: timeout,
+	}, emit)
+	if cachePath != "" && !*cached {
+		byID := map[string]provider.Provider{}
+		for _, p := range providers {
+			byID[p.ID()] = p
+		}
+		keep := func(id string) bool { p, ok := byID[id]; return ok && p.Kind() != provider.KindRepo }
+		if err := cachefile.Save(cachePath, f, keep, time.Now()); err != nil {
+			errs = append(errs, err)
+		}
+	}
+	if gone != nil {
+		if errors.Is(gone, syscall.EPIPE) {
+			return nil // the reader has what it wanted
+		}
+		return gone
+	}
+	for _, err := range errs {
+		fmt.Fprintln(errOut, "lazysecrets: list:", err)
+	}
+	return nil
+}

cmd/lazysecrets/main.go

@@ -121,6 +121,10 @@ func run() error {
                                   {{secrets/scope/key}}
   lazysecrets [flags] get <ref>   print the value a lazysecrets:// link points at,
                                   for scripts: VAR=$(lazysecrets get <ref>)
+  lazysecrets [flags] list [--cached] [--max-age 2m] [--format text|json]
+                                  print every source, vault and secret it can
+                                  find, one per line, for fzf and pickers:
+                                  what's cached at once, then what loads
   lazysecrets [flags] share [--key <secret>] <env file>...
                                   print a link to env files of a repo, encrypted
                                   with a secret's value (default: sharing.key)
@@ -146,7 +150,7 @@ Flags:
 	}
 
 	var open, getRef *ref.Ref
-	var shareArgs []string
+	var shareArgs, listArgs []string
 	printKeys := false
 	switch args := flag.Args(); {
 	case len(args) > 0 && args[0] == "share":
@@ -154,6 +158,8 @@ Flags:
 		if len(shareArgs) == 0 {
 			shareArgs = []string{} // still sharing: says how
 		}
+	case len(args) > 0 && args[0] == "list":
+		listArgs = append([]string{}, args[1:]...) // not nil: listing
 	case len(args) == 1 && args[0] == "keybindings":
 		printKeys = true
 	case len(args) == 2 && args[0] == "get":
@@ -345,6 +351,13 @@ Flags:
 		defer cancel()
 		return get(ctx, providers, st, *getRef)
 	}
+	if listArgs != nil {
+		cachePath := ""
+		if d, err := config.CacheDir(); err == nil && !*demo && config.On(cfg.Storage.Cache) {
+			cachePath = filepath.Join(d, "cache.json")
+		}
+		return listCmd(providers, cachePath, timeoutFor, listArgs, os.Stdout, os.Stderr)
+	}
 	if shareArgs != nil {
 		// The files from a repo, the key from whichever provider it's
 		// in: the longest timeout.

docs/config.md

@@ -140,7 +140,8 @@ you: the Sources lists, vaults and scopes, and secrets' names and metadata
 once, and anything older than two minutes reloads in the background when
 you go to it, or when Go to searches it. While it does, the pane's
 bottom border says how old what it shows is (`cached 3h ago`). Revealing
-and copying always fetch the value live.
+and copying always fetch the value live. [`lazysecrets list`](links.md#listing-for-fzf-and-pickers)
+reads it and adds to it too.
 
 Lists not loaded for 30 days are dropped, and so is everything from an
 account you're no longer signed in to. Repos aren't cached: their files

docs/links.md

@@ -136,6 +136,35 @@ it first), the Databricks profile for the workspace, or sops in your clone
 of the repo; a version in the link reads that version. The value ends with
 a newline only on a terminal.
 
+### Listing, for fzf and pickers
+
+`lazysecrets list` prints every source, vault and secret, one per line:
+its link, its kind (`source`, `vault`, `secret`, `key`, `certificate`)
+and its name, tab-separated. It's what Go to searches, so fzf, or an
+editor's picker, can search it instead:
+
+```sh
+lazysecrets list | fzf --delimiter='\t' --with-nth=3 \
+  --bind 'enter:become(lazysecrets open {1})' \
+  --bind 'ctrl-y:execute-silent(lazysecrets get {1} | pbcopy)'
+```
+
+What's [cached](config.md#cache) prints at once. Then, like Go to, it loads
+what's missing or older than `--max-age` (2m), and what's new in it follows
+as it arrives: fzf shows it while you type. `--cached` loads nothing from
+Azure or Databricks, for an answer at once. Repos are always read, from
+disk. What it loads is cached for the TUI too. Lists it can't read are
+reported on stderr; it stops when the reader goes, after a pick in fzf.
+
+`--format json` prints an object per line instead, for editor plugins:
+
+```json
+{"link":"lazysecrets://azure/kv-app-prod.vault.azure.net/secrets/db-password","kind":"secret","provider":"azure","path":["platform-prod","kv-app-prod","db-password"],"account":"alice"}
+```
+
+`account` is there when several az accounts are signed in. Values are never
+listed.
+
 ## From the browser and chat
 
 ```sh

internal/cachefile/cachefile.go

@@ -0,0 +1,146 @@
+// Package cachefile is the cache of what lazysecrets has loaded
+// (cache.json): the Sources lists, vaults, secrets' metadata and where
+// the cursor was in each, never values. The TUI and lazysecrets list
+// both read it and save to it; a save merges with what's on disk, so
+// neither undoes what the other loaded.
+package cachefile
+
+import (
+	"encoding/json"
+	"maps"
+	"os"
+	"path/filepath"
+	"strings"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+)
+
+// Version is the file's format; a file of another version is ignored,
+// and replaced on the next save.
+const Version = 1
+
+// MaxAge drops what hasn't been loaded again for this long.
+const MaxAge = 30 * 24 * time.Hour
+
+// File is the cache. Every key starts with the provider ID: Sources are
+// keyed by it, the rest by Row, Objects' keys and the like.
+type File struct {
+	Version    int                                      `json:"version"`
+	Sources    map[string]Stamped[[]provider.Source]    `json:"sources,omitempty"` // by provider ID
+	Containers map[string]Stamped[[]provider.Container] `json:"containers,omitempty"`
+	Objects    map[string]Stamped[[]provider.Object]    `json:"objects,omitempty"`
+	Details    map[string]Stamped[provider.Object]      `json:"details,omitempty"`
+	Cursor     map[string]string                        `json:"cursor,omitempty"`
+}
+
+// Stamped is a list or detail with when it was loaded.
+type Stamped[T any] struct {
+	At    time.Time `json:"at"`
+	Value T         `json:"value"`
+}
+
+// New is an empty cache.
+func New() File {
+	return File{
+		Version:    Version,
+		Sources:    map[string]Stamped[[]provider.Source]{},
+		Containers: map[string]Stamped[[]provider.Container]{},
+		Objects:    map[string]Stamped[[]provider.Object]{},
+		Details:    map[string]Stamped[provider.Object]{},
+		Cursor:     map[string]string{},
+	}
+}
+
+// Row keys a source's vaults or scopes.
+func Row(providerID, sourceID string) string { return providerID + "|" + sourceID }
+
+// ContainerKey keys what's cached about a vault or scope.
+func ContainerKey(c provider.Container) string { return c.Source.Provider + "|" + c.ID }
+
+// ObjectsKey keys a vault's secrets, keys or certificates.
+func ObjectsKey(c provider.Container, t provider.ObjectType) string {
+	return ContainerKey(c) + "|" + string(t)
+}
+
+// ProviderOf is the provider ID a key starts with.
+func ProviderOf(key string) string {
+	id, _, _ := strings.Cut(key, "|")
+	return id
+}
+
+// Read reads path; a missing, unreadable or other version's file is an
+// empty cache.
+func Read(path string) File {
+	f := New()
+	b, err := os.ReadFile(path)
+	if err != nil {
+		return f
+	}
+	var d File
+	if json.Unmarshal(b, &d) != nil || d.Version != Version {
+		return f
+	}
+	maps.Copy(f.Sources, d.Sources)
+	maps.Copy(f.Containers, d.Containers)
+	maps.Copy(f.Objects, d.Objects)
+	maps.Copy(f.Details, d.Details)
+	maps.Copy(f.Cursor, d.Cursor)
+	return f
+}
+
+// Save writes f to path, merged with the file there: of a list in both,
+// the one loaded last wins, and f's cursors win. Entries older than
+// MaxAge, and those of providers keep says no to (accounts signed out
+// of), are dropped, from the file's side as well.
+func Save(path string, f File, keep func(providerID string) bool, now time.Time) error {
+	out := New()
+	ok := func(key string, at time.Time) bool { return keep(ProviderOf(key)) && now.Sub(at) < MaxAge }
+	disk := Read(path)
+	merge(out.Sources, disk.Sources, f.Sources, ok)
+	merge(out.Containers, disk.Containers, f.Containers, ok)
+	merge(out.Objects, disk.Objects, f.Objects, ok)
+	merge(out.Details, disk.Details, f.Details, ok)
+	for _, m := range []map[string]string{disk.Cursor, f.Cursor} {
+		for k, v := range m {
+			if keep(ProviderOf(k)) {
+				out.Cursor[k] = v
+			}
+		}
+	}
+	return write(path, out)
+}
+
+func merge[T any](out, disk, ours map[string]Stamped[T], ok func(string, time.Time) bool) {
+	for _, m := range []map[string]Stamped[T]{disk, ours} {
+		for k, e := range m {
+			if old, seen := out[k]; ok(k, e.At) && (!seen || !e.At.Before(old.At)) {
+				out[k] = e
+			}
+		}
+	}
+}
+
+// write writes f atomically, readable by the user only.
+func write(path string, f File) error {
+	b, err := json.Marshal(f)
+	if err != nil {
+		return err
+	}
+	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
+		return err
+	}
+	tmp, err := os.CreateTemp(filepath.Dir(path), ".cache-*.json")
+	if err != nil {
+		return err
+	}
+	defer os.Remove(tmp.Name()) // after a successful rename, nothing there
+	if _, err := tmp.Write(b); err != nil {
+		tmp.Close()
+		return err
+	}
+	if err := tmp.Close(); err != nil {
+		return err
+	}
+	return os.Rename(tmp.Name(), path)
+}

internal/cachefile/cachefile_test.go

@@ -0,0 +1,91 @@
+package cachefile
+
+import (
+	"path/filepath"
+	"testing"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+)
+
+func list(at time.Time, names ...string) Stamped[[]provider.Container] {
+	var cs []provider.Container
+	for _, n := range names {
+		cs = append(cs, provider.Container{Name: n})
+	}
+	return Stamped[[]provider.Container]{At: at, Value: cs}
+}
+
+func TestSaveMerges(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	now := time.Now()
+	all := func(string) bool { return true }
+
+	// The TUI saved two lists.
+	tui := New()
+	tui.Containers["az|a"] = list(now.Add(-time.Hour), "kv-a-old")
+	tui.Containers["az|b"] = list(now.Add(-time.Minute), "kv-b")
+	tui.Cursor["az|b"] = "kv-b"
+	if err := Save(path, tui, all, now); err != nil {
+		t.Fatal(err)
+	}
+	// lazysecrets list loads a again, and c.
+	listed := Read(path)
+	listed.Containers["az|a"] = list(now, "kv-a-new")
+	listed.Containers["az|c"] = list(now, "kv-c")
+	if err := Save(path, listed, all, now); err != nil {
+		t.Fatal(err)
+	}
+	// The TUI saves what it has, older: list's lists stay.
+	if err := Save(path, tui, all, now); err != nil {
+		t.Fatal(err)
+	}
+	f := Read(path)
+	for key, want := range map[string]string{"az|a": "kv-a-new", "az|b": "kv-b", "az|c": "kv-c"} {
+		if got := f.Containers[key].Value; len(got) != 1 || got[0].Name != want {
+			t.Errorf("%s = %v, want %s", key, got, want)
+		}
+	}
+	if f.Cursor["az|b"] != "kv-b" {
+		t.Error("cursor lost")
+	}
+}
+
+func TestSaveDrops(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	now := time.Now()
+	f := New()
+	f.Containers["az|old"] = list(now.Add(-MaxAge-time.Hour), "kv-old")
+	f.Containers["gone|x"] = list(now, "kv-gone")
+	f.Containers["az|x"] = list(now, "kv-x")
+	if err := Save(path, f, func(id string) bool { return true }, now); err != nil {
+		t.Fatal(err)
+	}
+	// Signed out of "gone" since: dropped from the file too.
+	if err := Save(path, New(), func(id string) bool { return id == "az" }, now); err != nil {
+		t.Fatal(err)
+	}
+	got := Read(path).Containers
+	if _, ok := got["az|old"]; ok {
+		t.Error("kept a list older than MaxAge")
+	}
+	if _, ok := got["gone|x"]; ok {
+		t.Error("kept a signed-out account's list")
+	}
+	if _, ok := got["az|x"]; !ok {
+		t.Error("dropped a list on disk the save didn't have")
+	}
+}
+
+func TestReadOtherVersion(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "cache.json")
+	f := New()
+	f.Version = Version + 1
+	f.Containers["az|x"] = list(time.Now(), "kv-x")
+	if err := write(path, f); err != nil {
+		t.Fatal(err)
+	}
+	if got := Read(path); len(got.Containers) != 0 || got.Version != Version {
+		t.Errorf("read another version's file: %+v", got)
+	}
+}

internal/listing/listing.go

@@ -0,0 +1,285 @@
+// Package listing lists what lazysecrets can find, for lazysecrets list:
+// what Go to searches, without the TUI. What's cached comes first, at
+// once; then lists that are missing or older than MaxAge load in the
+// background, a few at a time, and what's new in them follows as they
+// arrive. Repos are always read, from disk. Values are never listed.
+package listing
+
+import (
+	"cmp"
+	"context"
+	"fmt"
+	"slices"
+	"strings"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+	"git.kebcloud.no/keb/lazysecrets/internal/ref"
+)
+
+// Workers is how many lists load at once, as in Go to.
+const Workers = 4
+
+// Entry is a source, vault or item, by its lazysecrets:// link.
+type Entry struct {
+	Link string `json:"link"`
+	// Kind is source (a subscription, workspace or repo), vault (a Key
+	// Vault, secret scope or repo file), secret, key or certificate.
+	Kind string `json:"kind"`
+	// Provider is azure, databricks or repo, as in links.
+	Provider string `json:"provider"`
+	// Path names it: its source's, vault's and own names.
+	Path []string `json:"path"`
+	// Account is the az account it's seen through, when there are several.
+	Account string `json:"account,omitempty"`
+}
+
+type Options struct {
+	Providers []provider.Provider
+	// Cache is what was loaded before; Run adds what it loads.
+	Cache cachefile.File
+	// MaxAge: cached lists loaded longer ago are loaded again.
+	MaxAge time.Duration
+	// Cached lists only what's cached (and repos): nothing is loaded.
+	Cached  bool
+	Timeout func(provider.Kind) time.Duration
+	Now     func() time.Time
+}
+
+// Run lists everything, calling emit for each entry once, from this
+// goroutine. An error from emit (the reader went away) stops it. It
+// returns the cache with what was loaded, and the lists that couldn't be.
+func Run(ctx context.Context, o Options, emit func(Entry) error) (cachefile.File, []error) {
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+	l := &lister{o: o, emit: emit, seen: map[string]bool{}, pending: map[string]bool{}, results: make(chan result)}
+	if l.o.Now == nil {
+		l.o.Now = time.Now
+	}
+	for _, p := range o.Providers {
+		if l.stopped != nil {
+			break
+		}
+		l.provider(p)
+	}
+	l.start(ctx)
+	for l.active > 0 {
+		r := <-l.results
+		l.active--
+		if l.stopped == nil {
+			l.handle(r)
+			l.start(ctx)
+		}
+		if l.stopped != nil {
+			cancel()
+		}
+	}
+	return l.o.Cache, l.errs
+}
+
+type job struct {
+	p   provider.Provider
+	src *provider.Source    // list its vaults
+	c   *provider.Container // list its secrets
+	key string              // where the list is cached
+}
+
+type result struct {
+	job
+	sources    []provider.Source
+	containers []provider.Container
+	objects    []provider.Object
+	err        error
+}
+
+type lister struct {
+	o       Options
+	emit    func(Entry) error
+	seen    map[string]bool // provider ID + link of entries emitted
+	pending map[string]bool // cache keys queued or loading
+	queue   []job
+	active  int
+	results chan result
+	errs    []error
+	stopped error // emit's error
+}
+
+func (l *lister) cacheable(p provider.Provider) bool { return p.Kind() != provider.KindRepo }
+
+// load reports whether to load a list: a repo's always (from disk), else
+// unless Cached, when it isn't cached or is older than MaxAge.
+func (l *lister) load(p provider.Provider, cached bool, at time.Time) bool {
+	if !l.cacheable(p) {
+		return true
+	}
+	return !l.o.Cached && (!cached || l.o.Now().Sub(at) >= l.o.MaxAge)
+}
+
+func (l *lister) provider(p provider.Provider) {
+	e, ok := l.o.Cache.Sources[p.ID()]
+	if l.cacheable(p) && ok {
+		for _, src := range sorted(e.Value, func(s provider.Source) string { return s.Name }) {
+			l.source(p, src)
+		}
+	}
+	if l.load(p, ok, e.At) {
+		l.enqueue(job{p: p, key: p.ID()})
+	}
+}
+
+// source emits src, and what's cached under it, and queues what's not.
+func (l *lister) source(p provider.Provider, src provider.Source) {
+	src.Provider, src.ProviderLabel = p.ID(), p.Label()
+	if r, err := ref.ForSource(src); err == nil {
+		l.put(p, Entry{Link: r.String(), Kind: "source", Provider: kindOf(src.Kind), Path: []string{src.Name}, Account: accountOf(src)})
+	}
+	row := cachefile.Row(p.ID(), src.ID)
+	e, ok := l.o.Cache.Containers[row]
+	if l.cacheable(p) && ok {
+		l.containers(p, src, e.Value)
+	}
+	if l.load(p, ok, e.At) {
+		l.enqueue(job{p: p, src: &src, key: row})
+	}
+}
+
+func (l *lister) containers(p provider.Provider, src provider.Source, cs []provider.Container) {
+	for _, c := range sorted(cs, func(c provider.Container) string { return c.Name }) {
+		c.Source = src
+		if r, err := ref.ForContainer(c, provider.TypeSecret); err == nil {
+			l.put(p, Entry{Link: r.String(), Kind: "vault", Provider: kindOf(src.Kind), Path: []string{src.Name, c.Name}, Account: accountOf(src)})
+		}
+		key := cachefile.ObjectsKey(c, provider.TypeSecret)
+		e, ok := l.o.Cache.Objects[key]
+		if l.cacheable(p) {
+			// Keys and certificates only as far as they were opened in
+			// the TUI: Go to doesn't load them either.
+			for _, t := range []provider.ObjectType{provider.TypeSecret, provider.TypeKey, provider.TypeCertificate} {
+				l.objects(p, c, l.o.Cache.Objects[cachefile.ObjectsKey(c, t)].Value)
+			}
+		}
+		if l.load(p, ok, e.At) {
+			l.enqueue(job{p: p, c: &c, key: key})
+		}
+	}
+}
+
+func (l *lister) objects(p provider.Provider, c provider.Container, objs []provider.Object) {
+	for _, o := range sorted(objs, func(o provider.Object) string { return o.Name }) {
+		o.Container = c
+		r, err := ref.ForObject(o, "")
+		if err != nil {
+			continue
+		}
+		l.put(p, Entry{Link: r.String(), Kind: strings.ToLower(string(o.Type)), Provider: kindOf(c.Source.Kind),
+			Path: []string{c.Source.Name, c.Name, o.Name}, Account: accountOf(c.Source)})
+	}
+}
+
+// put emits e, unless it was already (or emitting has failed).
+func (l *lister) put(p provider.Provider, e Entry) {
+	id := p.ID() + "\x00" + e.Link
+	if l.stopped != nil || l.seen[id] {
+		return
+	}
+	l.seen[id] = true
+	l.stopped = l.emit(e)
+}
+
+func (l *lister) enqueue(j job) {
+	if l.pending[j.key] {
+		return
+	}
+	l.pending[j.key] = true
+	l.queue = append(l.queue, j)
+}
+
+func (l *lister) start(ctx context.Context) {
+	for l.stopped == nil && l.active < Workers && len(l.queue) > 0 {
+		j := l.queue[0]
+		l.queue = l.queue[1:]
+		l.active++
+		go func() { l.results <- l.fetch(ctx, j) }()
+	}
+}
+
+func (l *lister) fetch(ctx context.Context, j job) result {
+	kind := j.p.Kind()
+	ctx, cancel := context.WithTimeout(ctx, l.o.Timeout(kind))
+	defer cancel()
+	r := result{job: j}
+	switch {
+	case j.c != nil:
+		r.objects, r.err = j.p.Objects(ctx, *j.c, provider.TypeSecret)
+	case j.src != nil:
+		r.containers, r.err = j.p.Containers(ctx, *j.src)
+	default:
+		r.sources, r.err = j.p.Sources(ctx)
+	}
+	return r
+}
+
+func (l *lister) handle(r result) {
+	delete(l.pending, r.key)
+	now := l.o.Now()
+	if r.err != nil {
+		l.errs = append(l.errs, fmt.Errorf("%s: %w", r.label(), r.err))
+		return
+	}
+	cache := l.cacheable(r.p)
+	switch {
+	case r.c != nil:
+		if cache {
+			l.o.Cache.Objects[r.key] = cachefile.Stamped[[]provider.Object]{At: now, Value: r.objects}
+		}
+		l.objects(r.p, *r.c, r.objects)
+	case r.src != nil:
+		r.containers = slices.Clone(r.containers) // the provider's may be shared
+		for i := range r.containers {
+			r.containers[i].Source = *r.src
+		}
+		if cache {
+			l.o.Cache.Containers[r.key] = cachefile.Stamped[[]provider.Container]{At: now, Value: r.containers}
+		}
+		l.containers(r.p, *r.src, r.containers)
+	default:
+		if cache {
+			l.o.Cache.Sources[r.key] = cachefile.Stamped[[]provider.Source]{At: now, Value: r.sources}
+		}
+		for _, src := range r.sources {
+			l.source(r.p, src)
+		}
+	}
+}
+
+// label names a list that couldn't be loaded.
+func (r result) label() string {
+	switch {
+	case r.c != nil:
+		return r.c.Source.Name + " / " + r.c.Name
+	case r.src != nil:
+		return r.src.Name
+	}
+	return r.p.Label()
+}
+
+func kindOf(k provider.Kind) string { return strings.ToLower(string(k)) }
+
+// accountOf is the account part of a source's provider label ("alice" in
+// "Azure · alice"), or "" when there's a single account.
+func accountOf(src provider.Source) string {
+	account, ok := strings.CutPrefix(src.ProviderLabel, string(src.Kind)+" · ")
+	if !ok {
+		return ""
+	}
+	return account
+}
+
+func sorted[T any](s []T, name func(T) string) []T {
+	s = slices.Clone(s)
+	slices.SortStableFunc(s, func(a, b T) int {
+		return cmp.Compare(strings.ToLower(name(a)), strings.ToLower(name(b)))
+	})
+	return s
+}

internal/listing/listing_test.go

@@ -0,0 +1,176 @@
+package listing
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider"
+	"git.kebcloud.no/keb/lazysecrets/internal/provider/fake"
+)
+
+// counting counts a provider's requests.
+type counting struct {
+	provider.Provider
+	calls atomic.Int32
+}
+
+func (c *counting) Sources(ctx context.Context) ([]provider.Source, error) {
+	c.calls.Add(1)
+	return c.Provider.Sources(ctx)
+}
+
+func (c *counting) Containers(ctx context.Context, src provider.Source) ([]provider.Container, error) {
+	c.calls.Add(1)
+	return c.Provider.Containers(ctx, src)
+}
+
+func (c *counting) Objects(ctx context.Context, ct provider.Container, t provider.ObjectType) ([]provider.Object, error) {
+	c.calls.Add(1)
+	return c.Provider.Objects(ctx, ct, t)
+}
+
+func run(t *testing.T, o Options) ([]Entry, cachefile.File, []error) {
+	t.Helper()
+	if o.Cache.Version == 0 {
+		o.Cache = cachefile.New()
+	}
+	o.Timeout = func(provider.Kind) time.Duration { return 5 * time.Second }
+	var got []Entry
+	f, errs := Run(context.Background(), o, func(e Entry) error {
+		got = append(got, e)
+		return nil
+	})
+	return got, f, errs
+}
+
+func has(es []Entry, kind, path string) bool {
+	for _, e := range es {
+		if e.Kind == kind && strings.Join(e.Path, "/") == path {
+			return true
+		}
+	}
+	return false
+}
+
+func TestListLoadsAndCaches(t *testing.T) {
+	t.Parallel()
+	az := &counting{Provider: fake.Azure(0)}
+	got, f, errs := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, MaxAge: time.Minute})
+	for _, want := range [][2]string{
+		{"source", "platform-prod"}, {"vault", "platform-prod/kv-app-prod"}, {"secret", "platform-prod/kv-app-prod/db-password"},
+		{"source", "lazysecrets-demo"}, {"secret", "lazysecrets-demo/.env/DATABASE_URL"},
+	} {
+		if !has(got, want[0], want[1]) {
+			t.Errorf("no %s %s", want[0], want[1])
+		}
+	}
+	if len(errs) != 1 || !strings.Contains(errs[0].Error(), "kv-restricted: 403") {
+		t.Errorf("errs = %v", errs)
+	}
+	seen := map[string]bool{}
+	for _, e := range got {
+		if seen[e.Link] {
+			t.Errorf("%s listed twice", e.Link)
+		}
+		seen[e.Link] = true
+		if e.Provider != "azure" && e.Provider != "repo" {
+			t.Errorf("provider %q", e.Provider)
+		}
+	}
+	if len(f.Sources) != 1 || len(f.Containers) == 0 || len(f.Objects) == 0 {
+		t.Errorf("not cached: %d sources, %d containers, %d objects", len(f.Sources), len(f.Containers), len(f.Objects))
+	}
+	for k := range f.Containers {
+		if cachefile.ProviderOf(k) != az.ID() {
+			t.Errorf("cached a repo: %s", k)
+		}
+	}
+
+	// Again with that cache: the same, without asking Azure.
+	before := az.calls.Load()
+	again, _, _ := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, Cache: f, MaxAge: time.Minute})
+	if len(again) != len(got) {
+		t.Errorf("%d entries from the cache, %d loaded", len(again), len(got))
+	}
+	// The vault that couldn't be read wasn't cached: it's asked again.
+	if n := az.calls.Load() - before; n != 1 {
+		t.Errorf("%d requests with a fresh cache, want 1 (kv-restricted)", n)
+	}
+
+	// Stale: loaded again, but each entry is still listed once.
+	before = az.calls.Load()
+	stale, _, _ := run(t, Options{Providers: []provider.Provider{az}, Cache: f, MaxAge: time.Minute,
+		Now: func() time.Time { return time.Now().Add(time.Hour) }})
+	if az.calls.Load()-before < 3 {
+		t.Error("stale lists weren't loaded again")
+	}
+	links := map[string]bool{}
+	for _, e := range stale {
+		if links[e.Link] {
+			t.Errorf("%s listed twice", e.Link)
+		}
+		links[e.Link] = true
+	}
+}
+
+func TestListCachedOnly(t *testing.T) {
+	t.Parallel()
+	az := &counting{Provider: fake.Azure(0)}
+	_, f, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute})
+	before := az.calls.Load()
+	got, _, errs := run(t, Options{Providers: []provider.Provider{az, fake.Repos()}, Cache: f, Cached: true,
+		Now: func() time.Time { return time.Now().Add(24 * time.Hour) }})
+	if az.calls.Load() != before || len(errs) != 0 {
+		t.Errorf("--cached asked Azure: %d requests, %v", az.calls.Load()-before, errs)
+	}
+	if !has(got, "secret", "platform-prod/kv-app-prod/db-password") || !has(got, "secret", "lazysecrets-demo/.env/DATABASE_URL") {
+		t.Error("cached secrets or repos missing")
+	}
+}
+
+func TestListCachedFirst(t *testing.T) {
+	t.Parallel()
+	az := fake.Azure(0)
+	_, f, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute})
+	// Forget one vault's secrets: they load, after everything cached.
+	var dropped string
+	for k := range f.Objects {
+		if strings.Contains(k, "kv-app-prod") && strings.HasSuffix(k, "|Secret") {
+			dropped = k
+			delete(f.Objects, k)
+			break
+		}
+	}
+	if dropped == "" {
+		t.Fatalf("no kv-app-prod in %v", f.Objects)
+	}
+	got, _, _ := run(t, Options{Providers: []provider.Provider{az}, Cache: f, MaxAge: time.Minute})
+	last := got[len(got)-1]
+	if last.Kind != "secret" || last.Path[1] != "kv-app-prod" {
+		t.Errorf("last = %+v, want a kv-app-prod secret, loaded after the cache", last)
+	}
+}
+
+func TestListStopsWhenTheReaderGoes(t *testing.T) {
+	t.Parallel()
+	gone := errors.New("broken pipe")
+	n := 0
+	_, errs := Run(context.Background(), Options{
+		Providers: []provider.Provider{fake.Azure(0), fake.Repos()}, Cache: cachefile.New(), MaxAge: time.Minute,
+		Timeout: func(provider.Kind) time.Duration { return time.Second },
+	}, func(Entry) error {
+		n++
+		if n == 2 {
+			return gone
+		}
+		return nil
+	})
+	if n != 2 || len(errs) != 0 {
+		t.Errorf("emitted %d after the reader went, errs %v", n, errs)
+	}
+}

internal/ui/app/persist.go

@@ -1,14 +1,12 @@
 package app
 
 import (
-	"encoding/json"
 	"errors"
-	"os"
-	"path/filepath"
 	"time"
 
 	tea "charm.land/bubbletea/v2"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/ref"
 	"git.kebcloud.no/keb/lazysecrets/internal/store"
@@ -23,58 +21,35 @@ import (
 // And where lazysecrets was left, per folder it ran in, goes in the store,
 // to go back there on the next start.
 
-// cacheFileVersion is the file's format; a file of another version is
-// ignored, and replaced on the next save.
-const cacheFileVersion = 1
-
-// cacheMaxAge drops what hasn't been loaded again for this long.
-const cacheMaxAge = 30 * 24 * time.Hour
-
 // persistEvery is how often a changed cache is saved while running, so
 // little is lost if lazysecrets is killed rather than quit.
 const persistEvery = 30 * time.Second
 
-type cacheFile struct {
-	Version    int                                      `json:"version"`
-	Sources    map[string]stamped[[]provider.Source]    `json:"sources,omitempty"` // by provider ID
-	Containers map[string]stamped[[]provider.Container] `json:"containers,omitempty"`
-	Objects    map[string]stamped[[]provider.Object]    `json:"objects,omitempty"`
-	Details    map[string]stamped[provider.Object]      `json:"details,omitempty"`
-	Cursor     map[string]string                        `json:"cursor,omitempty"`
-}
-
-type stamped[T any] struct {
-	At    time.Time `json:"at"`
-	Value T         `json:"value"`
-}
-
 type persistTickMsg struct{}
 
 type persistedMsg struct{ err error }
 
 // keepCached reports whether a cache key's account still exists and
 // isn't a repo: every key starts with the provider ID.
-func (m *Model) keepCached(key string) bool {
+func (m *Model) keepCached(key string) bool { return m.keepProvider(cachefile.ProviderOf(key)) }
-	id, _ := splitSourceRow(key)
+
-	p, ok := m.byID[id]
+func (m *Model) keepProvider(id string) bool { return keepIn(m.byID)(id) }
+
+func keepIn(byID map[string]provider.Provider) func(id string) bool {
+	return func(id string) bool {
+		p, ok := byID[id]
 		return ok && p.Kind() != provider.KindRepo
 	}
+}
 
 // loadCacheFile fills the cache from the file, leaving out what's too
 // old or belongs to accounts that aren't signed in any more.
 func (m *Model) loadCacheFile() {
-	b, err := os.ReadFile(m.cachePath)
+	f := cachefile.Read(m.cachePath)
-	if err != nil {
-		return
-	}
-	var f cacheFile
-	if json.Unmarshal(b, &f) != nil || f.Version != cacheFileVersion {
-		return
-	}
 	now := m.now()
-	recent := func(at time.Time) bool { return now.Sub(at) < cacheMaxAge }
+	recent := func(at time.Time) bool { return now.Sub(at) < cachefile.MaxAge }
 	for id, e := range f.Sources {
-		if m.keepCached(id+"|") && recent(e.At) {
+		if m.keepProvider(id) && recent(e.At) {
 			m.sourceResults[id] = sourcesMsg{gen: m.sourcesGen, provider: id, sources: e.Value}
 			m.sourcesAt[id] = e.At
 		}
@@ -102,33 +77,26 @@ func (m *Model) loadCacheFile() {
 }
 
 // cacheSnapshot is the cache as the file holds it.
-func (m *Model) cacheSnapshot() cacheFile {
+func (m *Model) cacheSnapshot() cachefile.File {
-	f := cacheFile{
+	f := cachefile.New()
-		Version:    cacheFileVersion,
-		Sources:    map[string]stamped[[]provider.Source]{},
-		Containers: map[string]stamped[[]provider.Container]{},
-		Objects:    map[string]stamped[[]provider.Object]{},
-		Details:    map[string]stamped[provider.Object]{},
-		Cursor:     map[string]string{},
-	}
 	for id, r := range m.sourceResults {
-		if r.err == nil && m.keepCached(id+"|") {
+		if r.err == nil && m.keepProvider(id) {
-			f.Sources[id] = stamped[[]provider.Source]{m.sourcesAt[id], r.sources}
+			f.Sources[id] = cachefile.Stamped[[]provider.Source]{At: m.sourcesAt[id], Value: r.sources}
 		}
 	}
 	for k, e := range m.cache.containers {
 		if m.keepCached(k) {
-			f.Containers[k] = stamped[[]provider.Container]{e.at, e.value}
+			f.Containers[k] = cachefile.Stamped[[]provider.Container]{At: e.at, Value: e.value}
 		}
 	}
 	for k, e := range m.cache.objects {
 		if m.keepCached(k) {
-			f.Objects[k] = stamped[[]provider.Object]{e.at, e.value}
+			f.Objects[k] = cachefile.Stamped[[]provider.Object]{At: e.at, Value: e.value}
 		}
 	}
 	for k, e := range m.cache.details {
 		if m.keepCached(k) {
-			f.Details[k] = stamped[provider.Object]{e.at, e.value}
+			f.Details[k] = cachefile.Stamped[provider.Object]{At: e.at, Value: e.value}
 		}
 	}
 	for k, v := range m.cache.cursor {
@@ -139,28 +107,11 @@ func (m *Model) cacheSnapshot() cacheFile {
 	return f
 }
 
-// writeCacheFile writes f atomically, readable by the user only.
+// saveCache saves f, merged with the file (lazysecrets list may have
-func writeCacheFile(path string, f cacheFile) error {
+// loaded more since), from any goroutine.
-	b, err := json.Marshal(f)
+func (m *Model) saveCache(f cachefile.File) func() error {
-	if err != nil {
+	path, keep, now := m.cachePath, keepIn(m.byID), m.now()
-		return err
+	return func() error { return cachefile.Save(path, f, keep, now) }
-	}
-	if err := os.MkdirAll(filepath.Dir(path), 0o700); err != nil {
-		return err
-	}
-	tmp, err := os.CreateTemp(filepath.Dir(path), ".cache-*.json")
-	if err != nil {
-		return err
-	}
-	defer os.Remove(tmp.Name()) // after a successful rename, nothing there
-	if _, err := tmp.Write(b); err != nil {
-		tmp.Close()
-		return err
-	}
-	if err := tmp.Close(); err != nil {
-		return err
-	}
-	return os.Rename(tmp.Name(), path)
 }
 
 // persistTick saves the cache, and where lazysecrets is, in the
@@ -169,8 +120,8 @@ func (m *Model) persistTick() tea.Cmd {
 	cmds := []tea.Cmd{tea.Tick(persistEvery, func(time.Time) tea.Msg { return persistTickMsg{} })}
 	if m.cacheDirty && m.cachePath != "" {
 		m.cacheDirty = false
-		f, path := m.cacheSnapshot(), m.cachePath
+		save := m.saveCache(m.cacheSnapshot())
-		cmds = append(cmds, func() tea.Msg { return persistedMsg{writeCacheFile(path, f)} })
+		cmds = append(cmds, func() tea.Msg { return persistedMsg{save()} })
 	}
 	if p, ok := m.place(); ok && m.placeDir != "" && p.Link != m.savedPlace {
 		m.savedPlace = p.Link
@@ -185,7 +136,7 @@ func (m *Model) persistTick() tea.Cmd {
 func (m Model) Persist() error {
 	var errs []error
 	if m.cachePath != "" {
-		errs = append(errs, writeCacheFile(m.cachePath, m.cacheSnapshot()))
+		errs = append(errs, m.saveCache(m.cacheSnapshot())())
 	}
 	if m.placeDir != "" {
 		if p, ok := m.place(); ok {

internal/ui/app/persist_test.go

@@ -1,12 +1,14 @@
 package app
 
 import (
+	"encoding/json"
 	"os"
 	"path/filepath"
 	"strings"
 	"testing"
 	"time"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider/fake"
 	"git.kebcloud.no/keb/lazysecrets/internal/ref"
@@ -80,15 +82,16 @@ func TestCacheFileDropsOldAndSignedOut(t *testing.T) {
 	file := filepath.Join(t.TempDir(), "cache.json")
 	az := fake.Azure(0)
 	now := time.Now()
-	f := cacheFile{
+	f := cachefile.File{
-		Version: cacheFileVersion,
+		Version: cachefile.Version,
-		Containers: map[string]stamped[[]provider.Container]{
+		Containers: map[string]cachefile.Stamped[[]provider.Container]{
-			az.ID() + "|recent":     {now.Add(-time.Hour), []provider.Container{{Name: "kv-recent"}}},
+			az.ID() + "|recent":     {At: now.Add(-time.Hour), Value: []provider.Container{{Name: "kv-recent"}}},
-			az.ID() + "|old":        {now.Add(-cacheMaxAge - time.Hour), []provider.Container{{Name: "kv-old"}}},
+			az.ID() + "|old":        {At: now.Add(-cachefile.MaxAge - time.Hour), Value: []provider.Container{{Name: "kv-old"}}},
-			"Azure:gone@x/t|recent": {now, []provider.Container{{Name: "kv-gone"}}},
+			"Azure:gone@x/t|recent": {At: now, Value: []provider.Container{{Name: "kv-gone"}}},
 		},
 	}
-	if err := writeCacheFile(file, f); err != nil {
+	b, _ := json.Marshal(f) // as an older lazysecrets left it
+	if err := os.WriteFile(file, b, 0o600); err != nil {
 		t.Fatal(err)
 	}
 	d := newDriverOpts(t, Options{CacheFile: file}, az)
@@ -97,7 +100,7 @@ func TestCacheFileDropsOldAndSignedOut(t *testing.T) {
 		t.Error("dropped a recent entry")
 	}
 	if _, ok := c[az.ID()+"|old"]; ok {
-		t.Error("kept an entry older than cacheMaxAge")
+		t.Error("kept an entry older than cachefile.MaxAge")
 	}
 	if _, ok := c["Azure:gone@x/t|recent"]; ok {
 		t.Error("kept an entry of an account that isn't signed in")

internal/ui/app/route.go

@@ -5,13 +5,14 @@ import (
 	"fmt"
 	"slices"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 )
 
 // containerKey identifies a container in the UI and caches. The same vault
 // seen through two accounts must not share cached lists, since each
 // account may see different things.
-func containerKey(c provider.Container) string { return c.Source.Provider + "|" + c.ID }
+func containerKey(c provider.Container) string { return cachefile.ContainerKey(c) }
 
 // providerFor finds the provider instance a source came from. Sources
 // remembered from an earlier run (favourites) may name an account that

internal/ui/app/sources.go

@@ -6,6 +6,7 @@ import (
 	"slices"
 	"strings"
 
+	"git.kebcloud.no/keb/lazysecrets/internal/cachefile"
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 	"git.kebcloud.no/keb/lazysecrets/internal/store"
 	"git.kebcloud.no/keb/lazysecrets/internal/ui/icons"
@@ -39,7 +40,7 @@ type sourceRow struct {
 
 // sourceRowKey joins a provider ID and a source ID. "|" doesn't occur in
 // either; provider IDs contain "/" (Azure:user/tenant).
-func sourceRowKey(providerID, sourceID string) string { return providerID + "|" + sourceID }
+func sourceRowKey(providerID, sourceID string) string { return cachefile.Row(providerID, sourceID) }
 
 func splitSourceRow(row string) (providerID, sourceID string) {
 	providerID, sourceID, _ = strings.Cut(row, "|")