From 0947e30739aa88a7e05ddce01c589918a1f2683e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Karl-Einar=20Bergstr=C3=B8m?= Date: Thu, 15 Oct 2026 14:05:15 +0200 Subject: [PATCH] feat: secret patterns for all of a repo's config files, with per-type overrides providers.repos.secretPatterns (keys, values) apply to every config file type; spring.secretPatterns overrides them list by list. A repo's .lazysecrets.yaml takes the same layout. Each list comes from the first that sets it: the repo's for the type, the repo's, the config's for the type, the config's, the defaults. Replaces the unreleased providers.repos.spring.keys/values. --- cmd/lazysecrets/main.go | 3 +- docs/repos.md | 21 +++-- internal/config/config.go | 11 ++- internal/config/config_test.go | 3 +- internal/provider/repo/patterns.go | 113 ++++++++++++++++++++++++ internal/provider/repo/patterns_test.go | 49 ++++++++++ internal/provider/repo/repo.go | 21 +++-- internal/provider/repo/spring.go | 99 ++------------------- internal/provider/repo/spring_test.go | 14 +-- 9 files changed, 216 insertions(+), 118 deletions(-) create mode 100644 internal/provider/repo/patterns.go create mode 100644 internal/provider/repo/patterns_test.go diff --git a/cmd/lazysecrets/main.go b/cmd/lazysecrets/main.go index bf3fd4b..55a382d 100644 --- a/cmd/lazysecrets/main.go +++ b/cmd/lazysecrets/main.go @@ -346,7 +346,8 @@ Flags: if config.On(pc.Repos.Enabled) { repos := repo.New(repo.Settings{ Dir: startDir, Paths: pc.Repos.Paths, Roots: valueOr(pc.Repos.Roots, repo.DefaultRoots), - Starred: starredRepos(st), Include: pc.Repos.Include, Exclude: pc.Repos.Exclude, Spring: pc.Repos.Spring, + Starred: starredRepos(st), Include: pc.Repos.Include, Exclude: pc.Repos.Exclude, + SecretPatterns: pc.Repos.SecretPatterns, Types: map[string]repo.FileType{"spring": pc.Repos.Spring}, }) // The repo it started in (shown on top as Repo) and the // others, checked for secret files in the background like diff --git a/docs/repos.md b/docs/repos.md index c36f48b..041b9e4 100644 --- a/docs/repos.md +++ b/docs/repos.md @@ -83,21 +83,28 @@ A file with none is hidden too, and a repo with only such files. `H` shows what's hidden, dimmed, in the panes and in Go to (`lazysecrets list --hidden` lists it); a pane's counter says how many it hides. A file's details count its hidden keys, and `e` opens it whole. A `${DB_PASSWORD}` -value is marked as a reference: the secret is set elsewhere. A tracked file is marked in red only when a listed key has its -value in plain text. +value is marked as a reference: the secret is set elsewhere. A tracked +file is marked in red only when a listed key has its value in plain text. -Each list of regexps replaces its default; `keys: [".*"]` lists every key: +The patterns are regexps, for all of a repo's config files and per type of +file; each list replaces the one it falls back to, and `keys: [".*"]` +lists every key: ```yaml providers: repos: + secretPatterns: # all config files + keys: ["(?i)(password|secret|token|key)$"] # matched against the key's name + values: ["^ENC\\(.+\\)$"] # and against its value spring: - keys: ["(?i)(password|secret|token|key)$", "bearer$"] # matched against the key's name - values: ["^ENC\\(.+\\)$"] # and against its value + secretPatterns: # Spring's: only what differs + keys: ["(?i)(password|secret)$", "bearer$"] ``` -A repo's `.lazysecrets.yaml` can set `spring` too, for everyone, and wins -over the config, one list at a time. +A repo's `.lazysecrets.yaml` can set them too, the same way +(`secretPatterns:`, `spring: secretPatterns:`), for everyone. Each list +comes from the first that sets it: the repo's for the type, the repo's, +the config's for the type, the config's, the defaults. Keys are named as Spring names them, `spring.datasource.password`; in a file with several documents (`---`, or `#---` in `.properties`), a later diff --git a/internal/config/config.go b/internal/config/config.go index 98c63c4..9a3a5ea 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -154,8 +154,10 @@ type Repos struct { // a repo's .sops.yaml creation_rules say where to look. Include []string `yaml:"include"` Exclude []string `yaml:"exclude"` - // Spring says which keys of Spring Boot config files are secrets. - Spring repo.SpringPatterns `yaml:"spring"` + // SecretPatterns say which keys of config files are secrets; Spring + // overrides them for Spring Boot's, list by list. + SecretPatterns repo.SecretPatterns `yaml:"secretPatterns"` + Spring repo.FileType `yaml:"spring"` } type Azure struct { @@ -295,8 +297,9 @@ func (c Config) validate(path string) error { return fmt.Errorf("%s: %s: %s is negative", path, name, d) } } - if err := c.Providers.Repos.Spring.Check(); err != nil { - return fmt.Errorf("%s: providers.repos.%w", path, err) + if err := errors.Join(c.Providers.Repos.SecretPatterns.Check("providers.repos.secretPatterns"), + c.Providers.Repos.Spring.SecretPatterns.Check("providers.repos.spring.secretPatterns")); err != nil { + return fmt.Errorf("%s: %w", path, err) } for i, v := range c.Providers.Azure.Vaults { if !strings.HasPrefix(v, "https://") { diff --git a/internal/config/config_test.go b/internal/config/config_test.go index 58e2f73..e764660 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -84,7 +84,8 @@ func TestLoadRejectsMistakes(t *testing.T) { "workspace": "sharing:\n key: \"{{secrets/team/env-share}}\"\n", "short keys": "sharing:\n keyRules:\n minLength: 8\n", "classes": "sharing:\n keyRules:\n minClasses: 5\n", - "regexp": "providers:\n repos:\n spring:\n keys: [\"(pass\"]\n", + "regexp": "providers:\n repos:\n secretPatterns:\n keys: [\"(pass\"]\n", + "spring re": "providers:\n repos:\n spring:\n secretPatterns:\n values: [\"(\"]\n", } { if _, err := LoadAll([]string{write(t, content)}); err == nil { t.Errorf("%s: no error", name) diff --git a/internal/provider/repo/patterns.go b/internal/provider/repo/patterns.go new file mode 100644 index 0000000..7779b18 --- /dev/null +++ b/internal/provider/repo/patterns.go @@ -0,0 +1,113 @@ +package repo + +import ( + "errors" + "fmt" + "path/filepath" + "regexp" +) + +// Config files (Spring's) hold all of an app's config, most of it not +// secret: of their keys, those named like a secret or with a value that +// looks like one are listed, the rest hidden (H shows them). The patterns +// can be set for all of a repo's files and per file type, in the config +// and in a repo's .lazysecrets.yaml. + +// DefaultSecretKeys and DefaultSecretValues are the patterns used when +// nothing sets them. The key pattern is anchored at the end of the name: +// spring.datasource.password, client-secret, DB_PASSWORD and checkoutKey, +// not the oauth2.*, keycloak.* and token-uri keys every Spring app has. +// passord is Norwegian. They err on listing too much: a row too many is +// cheap, a secret missed isn't. +var ( + DefaultSecretKeys = []string{ + `(?i)(pass(word|wd|phrase|ord)?|pwd|secret|token|key|credentials?|salt|connection-?string)$`, + } + DefaultSecretValues = []string{ + `^ENC\(.+\)$`, // Jasypt + `://[^/\s:@]*:[^/\s@]+@`, // a URL with a password: jdbc:postgresql://user:pass@host + `(?i)(AccountKey|SharedAccessKey|Password|Pwd)=`, // a connection string with one + } +) + +// SecretPatterns are regexps matched against a key's name and its value: +// a key is a secret when either matches. A list that isn't set is taken +// from the next place that sets it (see filterFor). +type SecretPatterns struct { + Keys *[]string `yaml:"keys"` + Values *[]string `yaml:"values"` +} + +// FileType holds the settings of one type of file (spring:), for now its +// own secret patterns. +type FileType struct { + SecretPatterns SecretPatterns `yaml:"secretPatterns"` +} + +// Check says which pattern doesn't compile; field is where they're set +// (spring.secretPatterns), for the message. +func (p SecretPatterns) Check(field string) error { + var errs []error + for name, list := range map[string]*[]string{"keys": p.Keys, "values": p.Values} { + if list == nil { + continue + } + for i, s := range *list { + if _, err := regexp.Compile(s); err != nil { + errs = append(errs, fmt.Errorf("%s.%s[%d]: %w", field, name, i, err)) + } + } + } + return errors.Join(errs...) +} + +type secretFilter struct{ keys, values []*regexp.Regexp } + +func (f secretFilter) secret(key, value string) bool { + return anyMatch(f.keys, key) || anyMatch(f.values, value) +} + +// filterFor is the patterns for a repo's files of one type (spring), each +// list from the first that sets it: the repo's .lazysecrets.yaml for the +// type, then for all its files, the config for the type, then for all +// repos, then the defaults. +func (p *Provider) filterFor(root, fileType string) (secretFilter, error) { + f, err := readLazysecretsFile(root) + if err != nil { + return secretFilter{}, err + } + own := filepath.Join(root, ".lazysecrets.yaml") + ": " + type source struct { + field string + patterns SecretPatterns + } + sources := []source{ + {own + fileType + ".secretPatterns", f.types()[fileType].SecretPatterns}, + {own + "secretPatterns", f.SecretPatterns}, + {"providers.repos." + fileType + ".secretPatterns", p.s.Types[fileType].SecretPatterns}, + {"providers.repos.secretPatterns", p.s.SecretPatterns}, + } + pick := func(name string, list func(SecretPatterns) *[]string, defaults []string) ([]*regexp.Regexp, error) { + for _, s := range sources { + if l := list(s.patterns); l != nil { + return compileAll(*l, s.field+"."+name) + } + } + return compileAll(defaults, "default "+name) + } + keys, kerr := pick("keys", func(p SecretPatterns) *[]string { return p.Keys }, DefaultSecretKeys) + values, verr := pick("values", func(p SecretPatterns) *[]string { return p.Values }, DefaultSecretValues) + return secretFilter{keys, values}, errors.Join(kerr, verr) +} + +func compileAll(patterns []string, field string) ([]*regexp.Regexp, error) { + out := make([]*regexp.Regexp, 0, len(patterns)) + for i, s := range patterns { + re, err := regexp.Compile(s) + if err != nil { + return nil, fmt.Errorf("%s[%d]: %w", field, i, err) + } + out = append(out, re) + } + return out, nil +} diff --git a/internal/provider/repo/patterns_test.go b/internal/provider/repo/patterns_test.go new file mode 100644 index 0000000..1928122 --- /dev/null +++ b/internal/provider/repo/patterns_test.go @@ -0,0 +1,49 @@ +package repo + +import ( + "os" + "path/filepath" + "testing" +) + +// Each list comes from the first that sets it: the repo's spring, the +// repo's, the config's spring, the config's, the defaults. +func TestSecretPatternsOrder(t *testing.T) { + list := func(s ...string) *[]string { return &s } + root := t.TempDir() + write := func(body string) { + if err := os.WriteFile(filepath.Join(root, ".lazysecrets.yaml"), []byte(body), 0o644); err != nil { + t.Fatal(err) + } + } + p := New(Settings{ + SecretPatterns: SecretPatterns{Keys: list("^config$"), Values: list("^config$")}, + Types: map[string]FileType{"spring": {SecretPatterns: SecretPatterns{Keys: list("^config-spring$")}}}, + }) + secret := func(key, value string) bool { + t.Helper() + f, err := p.filterFor(root, "spring") + if err != nil { + t.Fatal(err) + } + return f.secret(key, value) + } + + if !secret("config-spring", "") || secret("config", "") || !secret("", "config") { + t.Error("the config's spring keys, its shared values") + } + write("secretPatterns:\n keys: [\"^repo$\"]\n") + if !secret("repo", "") || secret("config-spring", "") || !secret("", "config") { + t.Error("the repo's shared keys over the config's") + } + write("secretPatterns:\n keys: [\"^repo$\"]\nspring:\n secretPatterns:\n keys: [\"^repo-spring$\"]\n values: []\n") + if !secret("repo-spring", "") || secret("repo", "") || secret("", "config") { + t.Error("the repo's spring over everything; an empty list matches nothing") + } + + p = New(Settings{}) + write("") + if !secret("spring.datasource.password", "") || secret("server.port", "") { + t.Error("the defaults") + } +} diff --git a/internal/provider/repo/repo.go b/internal/provider/repo/repo.go index 0b3fc14..db2f510 100644 --- a/internal/provider/repo/repo.go +++ b/internal/provider/repo/repo.go @@ -51,9 +51,11 @@ type Settings struct { // Include and Exclude are globs relative to a repo's root (** matches // any depth), for repos without their own .lazysecrets.yaml. Include, Exclude []string - // Spring says which keys of Spring Boot config files are secrets, for - // repos whose .lazysecrets.yaml doesn't. - Spring SpringPatterns + // SecretPatterns say which keys of config files are secrets, Types + // per type of file (spring), for repos whose .lazysecrets.yaml + // doesn't. + SecretPatterns SecretPatterns + Types map[string]FileType // Sops is the sops binary; "sops" (on PATH) when empty. Sops string } @@ -631,16 +633,21 @@ func normalizeRemote(remote string) (string, string) { // lazysecretsFile is a repo's own .lazysecrets.yaml. type lazysecretsFile struct { - Include []string `yaml:"include"` - Exclude []string `yaml:"exclude"` - Spring SpringPatterns `yaml:"spring"` - Sharing struct { + Include []string `yaml:"include"` + Exclude []string `yaml:"exclude"` + SecretPatterns SecretPatterns `yaml:"secretPatterns"` + Spring FileType `yaml:"spring"` + Sharing struct { // Key is the link to the secret that encrypts the repo's // shared env files (package share). Key string `yaml:"key"` } `yaml:"sharing"` } +func (f lazysecretsFile) types() map[string]FileType { + return map[string]FileType{"spring": f.Spring} +} + // ShareKey is the sharing.key of the repo's .lazysecrets.yaml, if any. func ShareKey(root string) (string, error) { f, err := readLazysecretsFile(root) diff --git a/internal/provider/repo/spring.go b/internal/provider/repo/spring.go index 82f8687..0f3735a 100644 --- a/internal/provider/repo/spring.go +++ b/internal/provider/repo/spring.go @@ -20,91 +20,8 @@ import ( "git.kebcloud.no/keb/lazysecrets/internal/provider" ) -// Spring Boot config files (application*.properties/.yml, bootstrap*) hold -// all of an app's config, most of it not secret: of their keys, only those -// named like a secret or with a value that looks like one are listed. The -// patterns are configurable, and the file's details count the rest, so a -// secret the patterns miss isn't hidden without a trace. - -// DefaultSpringKeys and DefaultSpringValues are the patterns used when -// neither the config nor the repo's .lazysecrets.yaml sets them. The key -// pattern is anchored at the end of the name: spring.datasource.password, -// client-secret and checkoutKey, not the oauth2.*, keycloak.* and -// token-uri keys every Spring app has. passord is Norwegian. The file's -// details count the keys left out. -var ( - DefaultSpringKeys = []string{ - `(?i)(pass(word|wd|phrase|ord)?|pwd|secret|token|key|credentials?|salt|connection-?string)$`, - } - DefaultSpringValues = []string{ - `^ENC\(.+\)$`, // Jasypt - `://[^/\s:@]*:[^/\s@]+@`, // a URL with a password: jdbc:postgresql://user:pass@host - `(?i)(AccountKey|SharedAccessKey|Password|Pwd)=`, // a connection string with one - } -) - -// SpringPatterns are regexps matched against a Spring config key's name -// and its value: a key is listed when either matches. A list that isn't -// set is DefaultSpringKeys or DefaultSpringValues. -type SpringPatterns struct { - Keys *[]string `yaml:"keys"` - Values *[]string `yaml:"values"` -} - -// Check says which pattern doesn't compile. -func (p SpringPatterns) Check() error { - _, err := compileSpring(p) - return err -} - -type springFilter struct{ keys, values []*regexp.Regexp } - -func compileSpring(p SpringPatterns) (springFilter, error) { - var f springFilter - var errs []error - add := func(field string, patterns *[]string, defaults []string, out *[]*regexp.Regexp) { - list := defaults - if patterns != nil { - list = *patterns - } - for i, s := range list { - re, err := regexp.Compile(s) - if err != nil { - errs = append(errs, fmt.Errorf("spring.%s[%d]: %w", field, i, err)) - continue - } - *out = append(*out, re) - } - } - add("keys", p.Keys, DefaultSpringKeys, &f.keys) - add("values", p.Values, DefaultSpringValues, &f.values) - return f, errors.Join(errs...) -} - -func (f springFilter) secret(k springKey) bool { - return anyMatch(f.keys, k.key) || anyMatch(f.values, k.value) -} - -// springFilterFor is the repo's patterns: its .lazysecrets.yaml's, else the -// config's, each list on its own. -func (p *Provider) springFilterFor(root string) (springFilter, error) { - f, err := readLazysecretsFile(root) - if err != nil { - return springFilter{}, err - } - patterns := p.s.Spring - if f.Spring.Keys != nil { - patterns.Keys = f.Spring.Keys - } - if f.Spring.Values != nil { - patterns.Values = f.Spring.Values - } - filter, err := compileSpring(patterns) - if err != nil && (f.Spring.Keys != nil || f.Spring.Values != nil) { - err = fmt.Errorf("%s/.lazysecrets.yaml: %w", root, err) - } - return filter, err -} +// Spring Boot config files: application*.properties/.yml and bootstrap*, +// their keys filtered by the secret patterns (patterns.go). // springName is application.yml, application-prod.properties, bootstrap.yaml // and the like; not applications.yaml (Argo CD's) or application.yml.example. @@ -339,11 +256,11 @@ func (p *Provider) springHidden(root, file string) bool { if err != nil { return false } - filter, err := p.springFilterFor(root) + filter, err := p.filterFor(root, "spring") if err != nil { return false } - return !slices.ContainsFunc(keys, filter.secret) + return !slices.ContainsFunc(keys, func(k springKey) bool { return filter.secret(k.key, k.value) }) } // springObjects lists the file's keys, those that don't look like secrets @@ -353,7 +270,7 @@ func (p *Provider) springObjects(c provider.Container, b []byte) ([]provider.Obj if err != nil { return nil, fmt.Errorf("%s: %w", c.Name, err) } - filter, err := p.springFilterFor(c.Source.ID) + filter, err := p.filterFor(c.Source.ID, "spring") if err != nil { return nil, err } @@ -369,7 +286,7 @@ func (p *Provider) springObjects(c provider.Container, b []byte) ([]provider.Obj } out = append(out, provider.Object{ Container: c, Type: provider.TypeSecret, ID: c.ID + "\x00" + k.name, Name: k.name, Detail: detail, - Hidden: !filter.secret(k), Attrs: []provider.Attr{format, state}, + Hidden: !filter.secret(k.key, k.value), Attrs: []provider.Attr{format, state}, }) } return out, nil @@ -403,13 +320,13 @@ func (p *Provider) springDetails(c provider.Container) (attrs []provider.Attr, p if err != nil { return []provider.Attr{{Label: "keys", Value: err.Error(), Warn: true}}, false } - filter, err := p.springFilterFor(c.Source.ID) + filter, err := p.filterFor(c.Source.ID, "spring") if err != nil { return []provider.Attr{{Label: "keys", Value: err.Error(), Warn: true}}, false } secrets, jasypt := 0, 0 for _, k := range keys { - if filter.secret(k) { + if filter.secret(k.key, k.value) { secrets++ plain = plain || k.plain() } diff --git a/internal/provider/repo/spring_test.go b/internal/provider/repo/spring_test.go index da28ded..70a8412 100644 --- a/internal/provider/repo/spring_test.go +++ b/internal/provider/repo/spring_test.go @@ -98,7 +98,7 @@ other: y } func TestSpringDefaults(t *testing.T) { - f, err := compileSpring(SpringPatterns{}) + f, err := (&Provider{}).filterFor(t.TempDir(), "spring") if err != nil { t.Fatal(err) } @@ -112,7 +112,7 @@ func TestSpringDefaults(t *testing.T) { "app.servicebus.connection": "Endpoint=sb://x/;SharedAccessKeyName=a;SharedAccessKey=b", "app.name": "ENC(abc==)", } { - if !f.secret(springKey{key: key, value: value}) { + if !f.secret(key, value) { t.Errorf("%s=%s isn't listed", key, value) } } @@ -123,7 +123,7 @@ func TestSpringDefaults(t *testing.T) { "server.port": "8080", "spring.datasource.url": "jdbc:postgresql://db:5432/app", } { - if f.secret(springKey{key: key, value: value}) { + if f.secret(key, value) { t.Errorf("%s=%s is listed", key, value) } } @@ -211,17 +211,17 @@ app: // The config's patterns, and the repo's over them. all := []string{".*"} - p = New(Settings{Dir: root, Spring: SpringPatterns{Keys: &all}}) + p = New(Settings{Dir: root, SecretPatterns: SecretPatterns{Keys: &all}}) if keys, _ := p.Objects(ctx, yml, provider.TypeSecret); len(shown(keys)) != 5 { t.Errorf("every key: %d", len(shown(keys))) } lazysecrets := filepath.Join(root, ".lazysecrets.yaml") - os.WriteFile(lazysecrets, []byte("spring:\n keys: [\"url$\"]\n values: []\n"), 0o644) + os.WriteFile(lazysecrets, []byte("spring:\n secretPatterns:\n keys: [\"url$\"]\n values: []\n"), 0o644) if keys, _ := p.Objects(ctx, yml, provider.TypeSecret); len(shown(keys)) != 1 || shown(keys)[0].Name != "spring.datasource.url" { t.Errorf("repo's patterns: %v", keys) } - os.WriteFile(lazysecrets, []byte("spring:\n keys: [\"(\"]\n"), 0o644) - if _, err := p.Objects(ctx, yml, provider.TypeSecret); err == nil || !strings.Contains(err.Error(), ".lazysecrets.yaml: spring.keys[0]") { + os.WriteFile(lazysecrets, []byte("spring:\n secretPatterns:\n keys: [\"(\"]\n"), 0o644) + if _, err := p.Objects(ctx, yml, provider.TypeSecret); err == nil || !strings.Contains(err.Error(), ".lazysecrets.yaml: spring.secretPatterns.keys[0]") { t.Errorf("bad pattern: %v", err) } }