fix: review findings for lazysecrets list: recent means the last 90 days, cache saves lock and keep deletions and the TUI's cursors, failed lists aren't retried, one expiry rule

internal/listing/listing.go

@@ -74,25 +74,37 @@ type Options struct {
 	// Lists only needed for other kinds aren't loaded: sources and vaults
 	// alone don't load any secrets. Lists that fail are always listed.
 	Kinds []string
-	// Mark tells whether the entry with link is starred for account,
-	// when it was last used and how often (zero: not lately), and its
-	// note; nil marks nothing.
-	Mark    func(link, account string) (starred bool, lastUsed time.Time, uses int, note string)
-	Timeout func(provider.Kind) time.Duration
+	// Mark tells what's remembered about the entry with link, seen
+	// through account; nil marks nothing.
+	Mark    func(link, account string) Marks
+	Timeout func(provider.Kind) time.Duration // nil: 30s
 	Now     func() time.Time
 }
 
+// Marks are what's remembered about an entry: starred for the account,
+// used lately (Recent) and when and how often, and its note.
+type Marks struct {
+	Starred, Recent bool
+	LastUsed        time.Time
+	Uses            int
+	Note            string
+}
+
 // Run lists everything, calling emit for each entry once, from this
 // goroutine, and for each list that can't be loaded, as it fails. An
 // error from emit (the reader went away) stops it. It returns the cache
-// with what was loaded.
-func Run(ctx context.Context, o Options, emit func(Entry) error) cachefile.File {
+// with what was loaded, and whether anything was.
+func Run(ctx context.Context, o Options, emit func(Entry) error) (cachefile.File, bool) {
 	ctx, cancel := context.WithCancel(ctx)
 	defer cancel()
-	l := &lister{o: o, emit: emit, seen: map[string]bool{}, pending: map[string]bool{}, results: make(chan result)}
+	l := &lister{o: o, emit: emit, seen: map[string]bool{}, pending: map[string]bool{}, failed: map[string]bool{},
+		results: make(chan result)}
 	if l.o.Now == nil {
 		l.o.Now = time.Now
 	}
+	if l.o.Timeout == nil {
+		l.o.Timeout = func(provider.Kind) time.Duration { return 30 * time.Second }
+	}
 	for _, p := range o.Providers {
 		if l.stopped != nil {
 			break
@@ -111,7 +123,7 @@ func Run(ctx context.Context, o Options, emit func(Entry) error) cachefile.File
 			cancel()
 		}
 	}
-	return l.o.Cache
+	return l.o.Cache, l.loaded
 }
 
 type job struct {
@@ -135,6 +147,8 @@ type lister struct {
 	emit    func(Entry) error
 	seen    map[string]bool // provider ID + link of entries emitted
 	pending map[string]bool // cache keys queued or loading
+	failed  map[string]bool // cache keys that couldn't be loaded: not again
+	loaded  bool            // something was cached
 	queue   []job
 	active  int
 	results chan result
@@ -240,8 +254,11 @@ func (l *lister) put(p provider.Provider, e Entry) {
 	}
 	e.Account = p.ID()
 	if l.o.Mark != nil && e.Link != "" {
-		e.Starred, e.LastUsed, e.Uses, e.Note = l.o.Mark(e.Link, e.Account)
-		e.Recent = !e.LastUsed.IsZero()
+		m := l.o.Mark(e.Link, e.Account)
+		e.Starred, e.Recent, e.Note = m.Starred, m.Recent, m.Note
+		if m.Recent {
+			e.LastUsed, e.Uses = m.LastUsed, m.Uses
+		}
 	}
 	id := p.ID() + "\x00" + e.Link + "\x00" + e.Error
 	if l.stopped != nil || l.seen[id] {
@@ -265,7 +282,7 @@ func (l *lister) wants(kinds ...string) bool {
 }
 
 func (l *lister) enqueue(j job) {
-	if l.pending[j.key] {
+	if l.pending[j.key] || l.failed[j.key] {
 		return
 	}
 	l.pending[j.key] = true
@@ -301,9 +318,11 @@ func (l *lister) handle(r result) {
 	delete(l.pending, r.key)
 	now := l.o.Now()
 	if r.err != nil {
+		l.failed[r.key] = true
 		l.put(r.p, r.failed())
 		return
 	}
+	l.loaded = l.loaded || l.cacheable(r.p)
 	cache := l.cacheable(r.p)
 	switch {
 	case r.c != nil:

internal/listing/listing_test.go

@@ -42,7 +42,7 @@ func run(t *testing.T, o Options) ([]Entry, cachefile.File, []Entry) {
 	}
 	o.Timeout = func(provider.Kind) time.Duration { return 5 * time.Second }
 	var got, failed []Entry
-	f := Run(context.Background(), o, func(e Entry) error {
+	f, _ := Run(context.Background(), o, func(e Entry) error {
 		if e.Error != "" {
 			failed = append(failed, e)
 		} else {
@@ -234,14 +234,17 @@ func TestListMarks(t *testing.T) {
 	az := fake.Azure(0)
 	var starredLink string
 	got, _, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute, Kinds: []string{"secret"},
-		Mark: func(link, account string) (bool, time.Time, int, string) {
+		Mark: func(link, account string) Marks {
 			if starredLink == "" && strings.HasSuffix(link, "/db-password") {
 				starredLink = link
 			}
 			if strings.HasSuffix(link, "/api-key") {
-				return false, time.Now(), 3, "rotate in May"
+				return Marks{Recent: true, LastUsed: time.Now(), Uses: 3, Note: "rotate in May"}
 			}
-			return link == starredLink && account == az.ID(), time.Time{}, 0, ""
+			if strings.HasSuffix(link, "/legacy-token") { // used, but too long ago
+				return Marks{LastUsed: time.Now().AddDate(-2, 0, 0), Uses: 9}
+			}
+			return Marks{Starred: link == starredLink && account == az.ID()}
 		}})
 	var starred, recent int
 	for _, e := range got {
@@ -267,3 +270,28 @@ func TestListMarks(t *testing.T) {
 		t.Errorf("%d starred, %d recent", starred, recent)
 	}
 }
+
+func TestFailedListNotRetried(t *testing.T) {
+	t.Parallel()
+	az := &counting{Provider: fake.Azure(0)}
+	_, f, _ := run(t, Options{Providers: []provider.Provider{az}, MaxAge: time.Minute})
+	// Everything stale: sources reload, and their vaults are queued again
+	// as each list arrives. kv-restricted fails once, and isn't asked again.
+	before := az.calls.Load()
+	_, _, failed := run(t, Options{Providers: []provider.Provider{az}, Cache: f, MaxAge: time.Minute,
+		Now: func() time.Time { return time.Now().Add(time.Hour) }})
+	if len(failed) != 1 {
+		t.Errorf("failed: %+v", failed)
+	}
+	fresh := &counting{Provider: fake.Azure(0)}
+	run(t, Options{Providers: []provider.Provider{fresh}, MaxAge: time.Minute})
+	if n := az.calls.Load() - before; n > fresh.calls.Load() {
+		t.Errorf("%d requests with a stale cache, more than %d from scratch: something was asked twice", n, fresh.calls.Load())
+	}
+}
+
+func TestRunWithoutTimeout(t *testing.T) {
+	t.Parallel()
+	Run(context.Background(), Options{Providers: []provider.Provider{fake.Repos()}, Cache: cachefile.New()},
+		func(Entry) error { return nil }) // no Timeout: a default, not a panic
+}