feat: lazysecrets list shows unreadable lists as red ✗ lines, and filters by kind

internal/listing/listing.go

@@ -8,7 +8,6 @@ package listing
 import (
 	"cmp"
 	"context"
-	"fmt"
 	"slices"
 	"strings"
 	"time"
@@ -21,6 +20,9 @@ import (
 // Workers is how many lists load at once, as in Go to.
 const Workers = 4
 
+// Kinds are the kinds of entry, from sources down.
+var Kinds = []string{"source", "vault", "secret", "key", "certificate"}
+
 // Entry is a source, vault or item, by its lazysecrets:// link.
 type Entry struct {
 	Link string `json:"link"`
@@ -33,6 +35,10 @@ type Entry struct {
 	Path []string `json:"path"`
 	// Account is the az account it's seen through, when there are several.
 	Account string `json:"account,omitempty"`
+	// Error says why a source's vaults or a vault's secrets couldn't be
+	// listed. The entry is then that source or vault, again, or with an
+	// account's sources unreadable, the account without a link.
+	Error string `json:"error,omitempty"`
 }
 
 type Options struct {
@@ -42,15 +48,20 @@ type Options struct {
 	// MaxAge: cached lists loaded longer ago are loaded again.
 	MaxAge time.Duration
 	// Cached lists only what's cached (and repos): nothing is loaded.
-	Cached  bool
+	Cached bool
+	// Kinds are the kinds of entry to list (Kinds' names); none is all.
+	// Lists only needed for other kinds aren't loaded: sources and vaults
+	// alone don't load any secrets. Lists that fail are always listed.
+	Kinds   []string
 	Timeout func(provider.Kind) time.Duration
 	Now     func() time.Time
 }
 
 // Run lists everything, calling emit for each entry once, from this
-// goroutine. An error from emit (the reader went away) stops it. It
-// returns the cache with what was loaded, and the lists that couldn't be.
-func Run(ctx context.Context, o Options, emit func(Entry) error) (cachefile.File, []error) {
+// goroutine, and for each list that can't be loaded, as it fails. An
+// error from emit (the reader went away) stops it. It returns the cache
+// with what was loaded.
+func Run(ctx context.Context, o Options, emit func(Entry) error) cachefile.File {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
 	l := &lister{o: o, emit: emit, seen: map[string]bool{}, pending: map[string]bool{}, results: make(chan result)}
@@ -75,14 +86,15 @@ func Run(ctx context.Context, o Options, emit func(Entry) error) (cachefile.File
 			cancel()
 		}
 	}
-	return l.o.Cache, l.errs
+	return l.o.Cache
 }
 
 type job struct {
 	p   provider.Provider
 	src *provider.Source    // list its vaults
-	c   *provider.Container // list its secrets
-	key string              // where the list is cached
+	c   *provider.Container // list its secrets, keys or certificates (t)
+	t   provider.ObjectType
+	key string // where the list is cached
 }
 
 type result struct {
@@ -101,7 +113,6 @@ type lister struct {
 	queue   []job
 	active  int
 	results chan result
-	errs    []error
 	stopped error // emit's error
 }
 
@@ -131,36 +142,38 @@ func (l *lister) provider(p provider.Provider) {
 // source emits src, and what's cached under it, and queues what's not.
 func (l *lister) source(p provider.Provider, src provider.Source) {
 	src.Provider, src.ProviderLabel = p.ID(), p.Label()
-	if r, err := ref.ForSource(src); err == nil {
-		l.put(p, Entry{Link: r.String(), Kind: "source", Provider: kindOf(src.Kind), Path: []string{src.Name}, Account: accountOf(src)})
-	}
+	l.put(p, sourceEntry(src))
 	row := cachefile.Row(p.ID(), src.ID)
 	e, ok := l.o.Cache.Containers[row]
 	if l.cacheable(p) && ok {
 		l.containers(p, src, e.Value)
 	}
 	if l.load(p, ok, e.At) {
-		l.enqueue(job{p: p, src: &src, key: row})
+		if l.wants("vault", "secret", "key", "certificate") {
+			l.enqueue(job{p: p, src: &src, key: row})
+		}
 	}
 }
 
 func (l *lister) containers(p provider.Provider, src provider.Source, cs []provider.Container) {
 	for _, c := range sorted(cs, func(c provider.Container) string { return c.Name }) {
 		c.Source = src
-		if r, err := ref.ForContainer(c, provider.TypeSecret); err == nil {
-			l.put(p, Entry{Link: r.String(), Kind: "vault", Provider: kindOf(src.Kind), Path: []string{src.Name, c.Name}, Account: accountOf(src)})
-		}
-		key := cachefile.ObjectsKey(c, provider.TypeSecret)
-		e, ok := l.o.Cache.Objects[key]
-		if l.cacheable(p) {
-			// Keys and certificates only as far as they were opened in
-			// the TUI: Go to doesn't load them either.
-			for _, t := range []provider.ObjectType{provider.TypeSecret, provider.TypeKey, provider.TypeCertificate} {
-				l.objects(p, c, l.o.Cache.Objects[cachefile.ObjectsKey(c, t)].Value)
+		l.put(p, containerEntry(c))
+		for _, t := range c.ObjectTypes() {
+			key := cachefile.ObjectsKey(c, t)
+			e, ok := l.o.Cache.Objects[key]
+			if l.cacheable(p) {
+				l.objects(p, c, e.Value)
+			}
+			// Keys and certificates load only when asked for; else,
+			// like Go to, only as far as the TUI opened them.
+			asked := l.wants(strings.ToLower(string(t)))
+			if t != provider.TypeSecret {
+				asked = len(l.o.Kinds) > 0 && asked
+			}
+			if asked && l.load(p, ok, e.At) {
+				l.enqueue(job{p: p, c: &c, t: t, key: key})
 			}
-		}
-		if l.load(p, ok, e.At) {
-			l.enqueue(job{p: p, c: &c, key: key})
 		}
 	}
 }
@@ -177,9 +190,29 @@ func (l *lister) objects(p provider.Provider, c provider.Container, objs []provi
 	}
 }
 
-// put emits e, unless it was already (or emitting has failed).
+func sourceEntry(src provider.Source) Entry {
+	e := Entry{Kind: "source", Provider: kindOf(src.Kind), Path: []string{src.Name}, Account: accountOf(src)}
+	if r, err := ref.ForSource(src); err == nil {
+		e.Link = r.String()
+	}
+	return e
+}
+
+func containerEntry(c provider.Container) Entry {
+	e := Entry{Kind: "vault", Provider: kindOf(c.Source.Kind), Path: []string{c.Source.Name, c.Name}, Account: accountOf(c.Source)}
+	if r, err := ref.ForContainer(c, provider.TypeSecret); err == nil {
+		e.Link = r.String()
+	}
+	return e
+}
+
+// put emits e, unless it was already (or emitting has failed). Entries
+// without a link (one couldn't be made) are left out, but not errors.
 func (l *lister) put(p provider.Provider, e Entry) {
-	id := p.ID() + "\x00" + e.Link
+	if (e.Link == "" || !l.wants(e.Kind)) && e.Error == "" {
+		return
+	}
+	id := p.ID() + "\x00" + e.Link + "\x00" + e.Error
 	if l.stopped != nil || l.seen[id] {
 		return
 	}
@@ -187,6 +220,19 @@ func (l *lister) put(p provider.Provider, e Entry) {
 	l.stopped = l.emit(e)
 }
 
+// wants reports whether any of kinds is listed.
+func (l *lister) wants(kinds ...string) bool {
+	if len(l.o.Kinds) == 0 {
+		return true
+	}
+	for _, k := range kinds {
+		if slices.Contains(l.o.Kinds, k) {
+			return true
+		}
+	}
+	return false
+}
+
 func (l *lister) enqueue(j job) {
 	if l.pending[j.key] {
 		return
@@ -211,7 +257,7 @@ func (l *lister) fetch(ctx context.Context, j job) result {
 	r := result{job: j}
 	switch {
 	case j.c != nil:
-		r.objects, r.err = j.p.Objects(ctx, *j.c, provider.TypeSecret)
+		r.objects, r.err = j.p.Objects(ctx, *j.c, j.t)
 	case j.src != nil:
 		r.containers, r.err = j.p.Containers(ctx, *j.src)
 	default:
@@ -224,7 +270,7 @@ func (l *lister) handle(r result) {
 	delete(l.pending, r.key)
 	now := l.o.Now()
 	if r.err != nil {
-		l.errs = append(l.errs, fmt.Errorf("%s: %w", r.label(), r.err))
+		l.put(r.p, r.failed())
 		return
 	}
 	cache := l.cacheable(r.p)
@@ -253,15 +299,21 @@ func (l *lister) handle(r result) {
 	}
 }
 
-// label names a list that couldn't be loaded.
-func (r result) label() string {
+// failed is the entry for a list that couldn't be loaded.
+func (r result) failed() Entry {
+	var e Entry
 	switch {
 	case r.c != nil:
-		return r.c.Source.Name + " / " + r.c.Name
+		e = containerEntry(*r.c)
 	case r.src != nil:
-		return r.src.Name
+		src := *r.src
+		src.Provider, src.ProviderLabel = r.p.ID(), r.p.Label()
+		e = sourceEntry(src)
+	default:
+		e = Entry{Kind: "source", Provider: kindOf(r.p.Kind()), Path: []string{r.p.Label()}}
 	}
-	return r.p.Label()
+	e.Error = r.err.Error()
+	return e
 }
 
 func kindOf(k provider.Kind) string { return strings.ToLower(string(k)) }

internal/listing/listing_test.go

@@ -34,18 +34,23 @@ func (c *counting) Objects(ctx context.Context, ct provider.Container, t provide
 	return c.Provider.Objects(ctx, ct, t)
 }
 
-func run(t *testing.T, o Options) ([]Entry, cachefile.File, []error) {
+// run lists, returning the entries and, apart, the lists that failed.
+func run(t *testing.T, o Options) ([]Entry, cachefile.File, []Entry) {
 	t.Helper()
 	if o.Cache.Version == 0 {
 		o.Cache = cachefile.New()
 	}
 	o.Timeout = func(provider.Kind) time.Duration { return 5 * time.Second }
-	var got []Entry
-	f, errs := Run(context.Background(), o, func(e Entry) error {
-		got = append(got, e)
+	var got, failed []Entry
+	f := Run(context.Background(), o, func(e Entry) error {
+		if e.Error != "" {
+			failed = append(failed, e)
+		} else {
+			got = append(got, e)
+		}
 		return nil
 	})
-	return got, f, errs
+	return got, f, failed
 }
 
 func has(es []Entry, kind, path string) bool {
@@ -69,8 +74,8 @@ func TestListLoadsAndCaches(t *testing.T) {
 			t.Errorf("no %s %s", want[0], want[1])
 		}
 	}
-	if len(errs) != 1 || !strings.Contains(errs[0].Error(), "kv-restricted: 403") {
-		t.Errorf("errs = %v", errs)
+	if len(errs) != 1 || errs[0].Kind != "vault" || errs[0].Path[1] != "kv-restricted" || !strings.Contains(errs[0].Error, "403") || errs[0].Link == "" {
+		t.Errorf("failed = %+v", errs)
 	}
 	seen := map[string]bool{}
 	for _, e := range got {
@@ -160,10 +165,14 @@ func TestListStopsWhenTheReaderGoes(t *testing.T) {
 	t.Parallel()
 	gone := errors.New("broken pipe")
 	n := 0
-	_, errs := Run(context.Background(), Options{
+	var errs []Entry
+	Run(context.Background(), Options{
 		Providers: []provider.Provider{fake.Azure(0), fake.Repos()}, Cache: cachefile.New(), MaxAge: time.Minute,
 		Timeout: func(provider.Kind) time.Duration { return time.Second },
-	}, func(Entry) error {
+	}, func(e Entry) error {
+		if e.Error != "" {
+			errs = append(errs, e)
+		}
 		n++
 		if n == 2 {
 			return gone
@@ -174,3 +183,31 @@ func TestListStopsWhenTheReaderGoes(t *testing.T) {
 		t.Errorf("emitted %d after the reader went, errs %v", n, errs)
 	}
 }
+
+func TestListKinds(t *testing.T) {
+	t.Parallel()
+	// Sources only: nothing under them is asked for.
+	az := &counting{Provider: fake.Azure(0)}
+	got, _, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute, Kinds: []string{"source"}})
+	if len(got) == 0 || az.calls.Load() != 1 {
+		t.Errorf("%d sources, %d requests, want just the sources'", len(got), az.calls.Load())
+	}
+	for _, e := range got {
+		if e.Kind != "source" {
+			t.Errorf("listed a %s", e.Kind)
+		}
+	}
+	// Keys, asked for: loaded, though Go to wouldn't; no secrets.
+	got, _, failed := run(t, Options{Providers: []provider.Provider{fake.Azure(0)}, MaxAge: time.Minute, Kinds: []string{"key"}})
+	if !has(got, "key", "platform-prod/kv-app-prod/signing-key") {
+		t.Errorf("no keys: %+v", got)
+	}
+	for _, e := range got {
+		if e.Kind != "key" {
+			t.Errorf("listed a %s", e.Kind)
+		}
+	}
+	if len(failed) == 0 {
+		t.Error("a list that failed wasn't listed")
+	}
+}