feat: secret patterns for all of a repo's config files, with per-type overrides (#82)

Co-authored-by: Karl-Einar Bergstrøm <karl-einarb@outlook.com>

cmd/lazysecrets/main.go

@@ -346,7 +346,8 @@ Flags:
 		if config.On(pc.Repos.Enabled) {
 			repos := repo.New(repo.Settings{
 				Dir: startDir, Paths: pc.Repos.Paths, Roots: valueOr(pc.Repos.Roots, repo.DefaultRoots),
-				Starred: starredRepos(st), Include: pc.Repos.Include, Exclude: pc.Repos.Exclude, Spring: pc.Repos.Spring,
+				Starred: starredRepos(st), Include: pc.Repos.Include, Exclude: pc.Repos.Exclude,
+				SecretPatterns: pc.Repos.SecretPatterns, Types: map[string]repo.FileType{"spring": pc.Repos.Spring},
 			})
 			// The repo it started in (shown on top as Repo) and the
 			// others, checked for secret files in the background like

docs/repos.md

@@ -83,21 +83,28 @@ A file with none is hidden too, and a repo with only such files. `H` shows
 what's hidden, dimmed, in the panes and in Go to (`lazysecrets list
 --hidden` lists it); a pane's counter says how many it hides. A file's
 details count its hidden keys, and `e` opens it whole. A `${DB_PASSWORD}`
-value is marked as a reference: the secret is set elsewhere. A tracked file is marked in red only when a listed key has its
+value is marked as a reference: the secret is set elsewhere. A tracked
-value in plain text.
+file is marked in red only when a listed key has its value in plain text.
 
-Each list of regexps replaces its default; `keys: [".*"]` lists every key:
+The patterns are regexps, for all of a repo's config files and per type of
+file; each list replaces the one it falls back to, and `keys: [".*"]`
+lists every key:
 
 ```yaml
 providers:
   repos:
-    spring:
+    secretPatterns:                             # all config files
-      keys: ["(?i)(password|secret|token|key)$", "bearer$"]   # matched against the key's name
+      keys: ["(?i)(password|secret|token|key)$"]  # matched against the key's name
       values: ["^ENC\\(.+\\)$"]                   # and against its value
+    spring:
+      secretPatterns:                           # Spring's: only what differs
+        keys: ["(?i)(password|secret)$", "bearer$"]
 ```
 
-A repo's `.lazysecrets.yaml` can set `spring` too, for everyone, and wins
+A repo's `.lazysecrets.yaml` can set them too, the same way
-over the config, one list at a time.
+(`secretPatterns:`, `spring: secretPatterns:`), for everyone. Each list
+comes from the first that sets it: the repo's for the type, the repo's,
+the config's for the type, the config's, the defaults.
 
 Keys are named as Spring names them, `spring.datasource.password`; in a
 file with several documents (`---`, or `#---` in `.properties`), a later

internal/config/config.go

@@ -154,8 +154,10 @@ type Repos struct {
 	// a repo's .sops.yaml creation_rules say where to look.
 	Include []string `yaml:"include"`
 	Exclude []string `yaml:"exclude"`
-	// Spring says which keys of Spring Boot config files are secrets.
+	// SecretPatterns say which keys of config files are secrets; Spring
-	Spring repo.SpringPatterns `yaml:"spring"`
+	// overrides them for Spring Boot's, list by list.
+	SecretPatterns repo.SecretPatterns `yaml:"secretPatterns"`
+	Spring         repo.FileType       `yaml:"spring"`
 }
 
 type Azure struct {
@@ -295,8 +297,9 @@ func (c Config) validate(path string) error {
 			return fmt.Errorf("%s: %s: %s is negative", path, name, d)
 		}
 	}
-	if err := c.Providers.Repos.Spring.Check(); err != nil {
+	if err := errors.Join(c.Providers.Repos.SecretPatterns.Check("providers.repos.secretPatterns"),
-		return fmt.Errorf("%s: providers.repos.%w", path, err)
+		c.Providers.Repos.Spring.SecretPatterns.Check("providers.repos.spring.secretPatterns")); err != nil {
+		return fmt.Errorf("%s: %w", path, err)
 	}
 	for i, v := range c.Providers.Azure.Vaults {
 		if !strings.HasPrefix(v, "https://") {

internal/config/config_test.go

@@ -84,7 +84,8 @@ func TestLoadRejectsMistakes(t *testing.T) {
 		"workspace":  "sharing:\n  key: \"{{secrets/team/env-share}}\"\n",
 		"short keys": "sharing:\n  keyRules:\n    minLength: 8\n",
 		"classes":    "sharing:\n  keyRules:\n    minClasses: 5\n",
-		"regexp":     "providers:\n  repos:\n    spring:\n      keys: [\"(pass\"]\n",
+		"regexp":     "providers:\n  repos:\n    secretPatterns:\n      keys: [\"(pass\"]\n",
+		"spring re":  "providers:\n  repos:\n    spring:\n      secretPatterns:\n        values: [\"(\"]\n",
 	} {
 		if _, err := LoadAll([]string{write(t, content)}); err == nil {
 			t.Errorf("%s: no error", name)

internal/provider/repo/patterns.go

@@ -0,0 +1,113 @@
+package repo
+
+import (
+	"errors"
+	"fmt"
+	"path/filepath"
+	"regexp"
+)
+
+// Config files (Spring's) hold all of an app's config, most of it not
+// secret: of their keys, those named like a secret or with a value that
+// looks like one are listed, the rest hidden (H shows them). The patterns
+// can be set for all of a repo's files and per file type, in the config
+// and in a repo's .lazysecrets.yaml.
+
+// DefaultSecretKeys and DefaultSecretValues are the patterns used when
+// nothing sets them. The key pattern is anchored at the end of the name:
+// spring.datasource.password, client-secret, DB_PASSWORD and checkoutKey,
+// not the oauth2.*, keycloak.* and token-uri keys every Spring app has.
+// passord is Norwegian. They err on listing too much: a row too many is
+// cheap, a secret missed isn't.
+var (
+	DefaultSecretKeys = []string{
+		`(?i)(pass(word|wd|phrase|ord)?|pwd|secret|token|key|credentials?|salt|connection-?string)$`,
+	}
+	DefaultSecretValues = []string{
+		`^ENC\(.+\)$`,            // Jasypt
+		`://[^/\s:@]*:[^/\s@]+@`, // a URL with a password: jdbc:postgresql://user:pass@host
+		`(?i)(AccountKey|SharedAccessKey|Password|Pwd)=`, // a connection string with one
+	}
+)
+
+// SecretPatterns are regexps matched against a key's name and its value:
+// a key is a secret when either matches. A list that isn't set is taken
+// from the next place that sets it (see filterFor).
+type SecretPatterns struct {
+	Keys   *[]string `yaml:"keys"`
+	Values *[]string `yaml:"values"`
+}
+
+// FileType holds the settings of one type of file (spring:), for now its
+// own secret patterns.
+type FileType struct {
+	SecretPatterns SecretPatterns `yaml:"secretPatterns"`
+}
+
+// Check says which pattern doesn't compile; field is where they're set
+// (spring.secretPatterns), for the message.
+func (p SecretPatterns) Check(field string) error {
+	var errs []error
+	for name, list := range map[string]*[]string{"keys": p.Keys, "values": p.Values} {
+		if list == nil {
+			continue
+		}
+		for i, s := range *list {
+			if _, err := regexp.Compile(s); err != nil {
+				errs = append(errs, fmt.Errorf("%s.%s[%d]: %w", field, name, i, err))
+			}
+		}
+	}
+	return errors.Join(errs...)
+}
+
+type secretFilter struct{ keys, values []*regexp.Regexp }
+
+func (f secretFilter) secret(key, value string) bool {
+	return anyMatch(f.keys, key) || anyMatch(f.values, value)
+}
+
+// filterFor is the patterns for a repo's files of one type (spring), each
+// list from the first that sets it: the repo's .lazysecrets.yaml for the
+// type, then for all its files, the config for the type, then for all
+// repos, then the defaults.
+func (p *Provider) filterFor(root, fileType string) (secretFilter, error) {
+	f, err := readLazysecretsFile(root)
+	if err != nil {
+		return secretFilter{}, err
+	}
+	own := filepath.Join(root, ".lazysecrets.yaml") + ": "
+	type source struct {
+		field    string
+		patterns SecretPatterns
+	}
+	sources := []source{
+		{own + fileType + ".secretPatterns", f.types()[fileType].SecretPatterns},
+		{own + "secretPatterns", f.SecretPatterns},
+		{"providers.repos." + fileType + ".secretPatterns", p.s.Types[fileType].SecretPatterns},
+		{"providers.repos.secretPatterns", p.s.SecretPatterns},
+	}
+	pick := func(name string, list func(SecretPatterns) *[]string, defaults []string) ([]*regexp.Regexp, error) {
+		for _, s := range sources {
+			if l := list(s.patterns); l != nil {
+				return compileAll(*l, s.field+"."+name)
+			}
+		}
+		return compileAll(defaults, "default "+name)
+	}
+	keys, kerr := pick("keys", func(p SecretPatterns) *[]string { return p.Keys }, DefaultSecretKeys)
+	values, verr := pick("values", func(p SecretPatterns) *[]string { return p.Values }, DefaultSecretValues)
+	return secretFilter{keys, values}, errors.Join(kerr, verr)
+}
+
+func compileAll(patterns []string, field string) ([]*regexp.Regexp, error) {
+	out := make([]*regexp.Regexp, 0, len(patterns))
+	for i, s := range patterns {
+		re, err := regexp.Compile(s)
+		if err != nil {
+			return nil, fmt.Errorf("%s[%d]: %w", field, i, err)
+		}
+		out = append(out, re)
+	}
+	return out, nil
+}

internal/provider/repo/patterns_test.go

@@ -0,0 +1,49 @@
+package repo
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+// Each list comes from the first that sets it: the repo's spring, the
+// repo's, the config's spring, the config's, the defaults.
+func TestSecretPatternsOrder(t *testing.T) {
+	list := func(s ...string) *[]string { return &s }
+	root := t.TempDir()
+	write := func(body string) {
+		if err := os.WriteFile(filepath.Join(root, ".lazysecrets.yaml"), []byte(body), 0o644); err != nil {
+			t.Fatal(err)
+		}
+	}
+	p := New(Settings{
+		SecretPatterns: SecretPatterns{Keys: list("^config$"), Values: list("^config$")},
+		Types:          map[string]FileType{"spring": {SecretPatterns: SecretPatterns{Keys: list("^config-spring$")}}},
+	})
+	secret := func(key, value string) bool {
+		t.Helper()
+		f, err := p.filterFor(root, "spring")
+		if err != nil {
+			t.Fatal(err)
+		}
+		return f.secret(key, value)
+	}
+
+	if !secret("config-spring", "") || secret("config", "") || !secret("", "config") {
+		t.Error("the config's spring keys, its shared values")
+	}
+	write("secretPatterns:\n  keys: [\"^repo$\"]\n")
+	if !secret("repo", "") || secret("config-spring", "") || !secret("", "config") {
+		t.Error("the repo's shared keys over the config's")
+	}
+	write("secretPatterns:\n  keys: [\"^repo$\"]\nspring:\n  secretPatterns:\n    keys: [\"^repo-spring$\"]\n    values: []\n")
+	if !secret("repo-spring", "") || secret("repo", "") || secret("", "config") {
+		t.Error("the repo's spring over everything; an empty list matches nothing")
+	}
+
+	p = New(Settings{})
+	write("")
+	if !secret("spring.datasource.password", "") || secret("server.port", "") {
+		t.Error("the defaults")
+	}
+}

internal/provider/repo/repo.go

@@ -51,9 +51,11 @@ type Settings struct {
 	// Include and Exclude are globs relative to a repo's root (** matches
 	// any depth), for repos without their own .lazysecrets.yaml.
 	Include, Exclude []string
-	// Spring says which keys of Spring Boot config files are secrets, for
+	// SecretPatterns say which keys of config files are secrets, Types
-	// repos whose .lazysecrets.yaml doesn't.
+	// per type of file (spring), for repos whose .lazysecrets.yaml
-	Spring SpringPatterns
+	// doesn't.
+	SecretPatterns SecretPatterns
+	Types          map[string]FileType
 	// Sops is the sops binary; "sops" (on PATH) when empty.
 	Sops string
 }
@@ -633,7 +635,8 @@ func normalizeRemote(remote string) (string, string) {
 type lazysecretsFile struct {
 	Include        []string       `yaml:"include"`
 	Exclude        []string       `yaml:"exclude"`
-	Spring  SpringPatterns `yaml:"spring"`
+	SecretPatterns SecretPatterns `yaml:"secretPatterns"`
+	Spring         FileType       `yaml:"spring"`
 	Sharing        struct {
 		// Key is the link to the secret that encrypts the repo's
 		// shared env files (package share).
@@ -641,6 +644,10 @@ type lazysecretsFile struct {
 	} `yaml:"sharing"`
 }
 
+func (f lazysecretsFile) types() map[string]FileType {
+	return map[string]FileType{"spring": f.Spring}
+}
+
 // ShareKey is the sharing.key of the repo's .lazysecrets.yaml, if any.
 func ShareKey(root string) (string, error) {
 	f, err := readLazysecretsFile(root)

internal/provider/repo/spring.go

@@ -20,91 +20,8 @@ import (
 	"git.kebcloud.no/keb/lazysecrets/internal/provider"
 )
 
-// Spring Boot config files (application*.properties/.yml, bootstrap*) hold
+// Spring Boot config files: application*.properties/.yml and bootstrap*,
-// all of an app's config, most of it not secret: of their keys, only those
+// their keys filtered by the secret patterns (patterns.go).
-// named like a secret or with a value that looks like one are listed. The
-// patterns are configurable, and the file's details count the rest, so a
-// secret the patterns miss isn't hidden without a trace.
-
-// DefaultSpringKeys and DefaultSpringValues are the patterns used when
-// neither the config nor the repo's .lazysecrets.yaml sets them. The key
-// pattern is anchored at the end of the name: spring.datasource.password,
-// client-secret and checkoutKey, not the oauth2.*, keycloak.* and
-// token-uri keys every Spring app has. passord is Norwegian. The file's
-// details count the keys left out.
-var (
-	DefaultSpringKeys = []string{
-		`(?i)(pass(word|wd|phrase|ord)?|pwd|secret|token|key|credentials?|salt|connection-?string)$`,
-	}
-	DefaultSpringValues = []string{
-		`^ENC\(.+\)$`,            // Jasypt
-		`://[^/\s:@]*:[^/\s@]+@`, // a URL with a password: jdbc:postgresql://user:pass@host
-		`(?i)(AccountKey|SharedAccessKey|Password|Pwd)=`, // a connection string with one
-	}
-)
-
-// SpringPatterns are regexps matched against a Spring config key's name
-// and its value: a key is listed when either matches. A list that isn't
-// set is DefaultSpringKeys or DefaultSpringValues.
-type SpringPatterns struct {
-	Keys   *[]string `yaml:"keys"`
-	Values *[]string `yaml:"values"`
-}
-
-// Check says which pattern doesn't compile.
-func (p SpringPatterns) Check() error {
-	_, err := compileSpring(p)
-	return err
-}
-
-type springFilter struct{ keys, values []*regexp.Regexp }
-
-func compileSpring(p SpringPatterns) (springFilter, error) {
-	var f springFilter
-	var errs []error
-	add := func(field string, patterns *[]string, defaults []string, out *[]*regexp.Regexp) {
-		list := defaults
-		if patterns != nil {
-			list = *patterns
-		}
-		for i, s := range list {
-			re, err := regexp.Compile(s)
-			if err != nil {
-				errs = append(errs, fmt.Errorf("spring.%s[%d]: %w", field, i, err))
-				continue
-			}
-			*out = append(*out, re)
-		}
-	}
-	add("keys", p.Keys, DefaultSpringKeys, &f.keys)
-	add("values", p.Values, DefaultSpringValues, &f.values)
-	return f, errors.Join(errs...)
-}
-
-func (f springFilter) secret(k springKey) bool {
-	return anyMatch(f.keys, k.key) || anyMatch(f.values, k.value)
-}
-
-// springFilterFor is the repo's patterns: its .lazysecrets.yaml's, else the
-// config's, each list on its own.
-func (p *Provider) springFilterFor(root string) (springFilter, error) {
-	f, err := readLazysecretsFile(root)
-	if err != nil {
-		return springFilter{}, err
-	}
-	patterns := p.s.Spring
-	if f.Spring.Keys != nil {
-		patterns.Keys = f.Spring.Keys
-	}
-	if f.Spring.Values != nil {
-		patterns.Values = f.Spring.Values
-	}
-	filter, err := compileSpring(patterns)
-	if err != nil && (f.Spring.Keys != nil || f.Spring.Values != nil) {
-		err = fmt.Errorf("%s/.lazysecrets.yaml: %w", root, err)
-	}
-	return filter, err
-}
 
 // springName is application.yml, application-prod.properties, bootstrap.yaml
 // and the like; not applications.yaml (Argo CD's) or application.yml.example.
@@ -339,11 +256,11 @@ func (p *Provider) springHidden(root, file string) bool {
 	if err != nil {
 		return false
 	}
-	filter, err := p.springFilterFor(root)
+	filter, err := p.filterFor(root, "spring")
 	if err != nil {
 		return false
 	}
-	return !slices.ContainsFunc(keys, filter.secret)
+	return !slices.ContainsFunc(keys, func(k springKey) bool { return filter.secret(k.key, k.value) })
 }
 
 // springObjects lists the file's keys, those that don't look like secrets
@@ -353,7 +270,7 @@ func (p *Provider) springObjects(c provider.Container, b []byte) ([]provider.Obj
 	if err != nil {
 		return nil, fmt.Errorf("%s: %w", c.Name, err)
 	}
-	filter, err := p.springFilterFor(c.Source.ID)
+	filter, err := p.filterFor(c.Source.ID, "spring")
 	if err != nil {
 		return nil, err
 	}
@@ -369,7 +286,7 @@ func (p *Provider) springObjects(c provider.Container, b []byte) ([]provider.Obj
 		}
 		out = append(out, provider.Object{
 			Container: c, Type: provider.TypeSecret, ID: c.ID + "\x00" + k.name, Name: k.name, Detail: detail,
-			Hidden: !filter.secret(k), Attrs: []provider.Attr{format, state},
+			Hidden: !filter.secret(k.key, k.value), Attrs: []provider.Attr{format, state},
 		})
 	}
 	return out, nil
@@ -403,13 +320,13 @@ func (p *Provider) springDetails(c provider.Container) (attrs []provider.Attr, p
 	if err != nil {
 		return []provider.Attr{{Label: "keys", Value: err.Error(), Warn: true}}, false
 	}
-	filter, err := p.springFilterFor(c.Source.ID)
+	filter, err := p.filterFor(c.Source.ID, "spring")
 	if err != nil {
 		return []provider.Attr{{Label: "keys", Value: err.Error(), Warn: true}}, false
 	}
 	secrets, jasypt := 0, 0
 	for _, k := range keys {
-		if filter.secret(k) {
+		if filter.secret(k.key, k.value) {
 			secrets++
 			plain = plain || k.plain()
 		}

internal/provider/repo/spring_test.go

@@ -98,7 +98,7 @@ other: y
 }
 
 func TestSpringDefaults(t *testing.T) {
-	f, err := compileSpring(SpringPatterns{})
+	f, err := (&Provider{}).filterFor(t.TempDir(), "spring")
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -112,7 +112,7 @@ func TestSpringDefaults(t *testing.T) {
 		"app.servicebus.connection":                                  "Endpoint=sb://x/;SharedAccessKeyName=a;SharedAccessKey=b",
 		"app.name":                                                   "ENC(abc==)",
 	} {
-		if !f.secret(springKey{key: key, value: value}) {
+		if !f.secret(key, value) {
 			t.Errorf("%s=%s isn't listed", key, value)
 		}
 	}
@@ -123,7 +123,7 @@ func TestSpringDefaults(t *testing.T) {
 		"server.port":                                                           "8080",
 		"spring.datasource.url":                                                 "jdbc:postgresql://db:5432/app",
 	} {
-		if f.secret(springKey{key: key, value: value}) {
+		if f.secret(key, value) {
 			t.Errorf("%s=%s is listed", key, value)
 		}
 	}
@@ -211,17 +211,17 @@ app:
 
 	// The config's patterns, and the repo's over them.
 	all := []string{".*"}
-	p = New(Settings{Dir: root, Spring: SpringPatterns{Keys: &all}})
+	p = New(Settings{Dir: root, SecretPatterns: SecretPatterns{Keys: &all}})
 	if keys, _ := p.Objects(ctx, yml, provider.TypeSecret); len(shown(keys)) != 5 {
 		t.Errorf("every key: %d", len(shown(keys)))
 	}
 	lazysecrets := filepath.Join(root, ".lazysecrets.yaml")
-	os.WriteFile(lazysecrets, []byte("spring:\n  keys: [\"url$\"]\n  values: []\n"), 0o644)
+	os.WriteFile(lazysecrets, []byte("spring:\n  secretPatterns:\n    keys: [\"url$\"]\n    values: []\n"), 0o644)
 	if keys, _ := p.Objects(ctx, yml, provider.TypeSecret); len(shown(keys)) != 1 || shown(keys)[0].Name != "spring.datasource.url" {
 		t.Errorf("repo's patterns: %v", keys)
 	}
-	os.WriteFile(lazysecrets, []byte("spring:\n  keys: [\"(\"]\n"), 0o644)
+	os.WriteFile(lazysecrets, []byte("spring:\n  secretPatterns:\n    keys: [\"(\"]\n"), 0o644)
-	if _, err := p.Objects(ctx, yml, provider.TypeSecret); err == nil || !strings.Contains(err.Error(), ".lazysecrets.yaml: spring.keys[0]") {
+	if _, err := p.Objects(ctx, yml, provider.TypeSecret); err == nil || !strings.Contains(err.Error(), ".lazysecrets.yaml: spring.secretPatterns.keys[0]") {
 		t.Errorf("bad pattern: %v", err)
 	}
 }