Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>
This commit is contained in:
commit
6fab41703e
111 files changed
+22988
No files matched your search
@@ -0,0 +1,987 @@
|
||||
// Package repo browses the encrypted secret files in git repositories:
|
||||
// a repo is a source, its secret files the containers (vault level), and
|
||||
// their keys the secrets. SOPS files are the one type so far.
|
||||
//
|
||||
// Listing never decrypts: SOPS encrypts values only, so a file's keys and
|
||||
// metadata are read directly. A value is decrypted when it's revealed or
|
||||
// copied, one at a time, by the sops binary, with the keys sops itself
|
||||
// uses (age, Azure Key Vault through az, AWS/GCP KMS, PGP).
|
||||
//
|
||||
// Which files: tracked by git, then, in order of preference, the include
|
||||
// and exclude globs of the repo's .lazysecrets.yaml, those of the config,
|
||||
// or where the repo's .sops.yaml creation_rules say SOPS files go; a repo
|
||||
// with none of those is searched whole.
|
||||
package repo
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"runtime"
|
||||
"slices"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"go.yaml.in/yaml/v3"
|
||||
|
||||
"git.kebcloud.no/keb/lazysecrets/internal/provider"
|
||||
)
|
||||
|
||||
type Settings struct {
|
||||
// Dir is where lazysecrets started: the repo containing it, if any,
|
||||
// is listed first and marked Here, with its files under Dir first.
|
||||
Dir string
|
||||
// Paths are repos to show.
|
||||
Paths []string
|
||||
// Roots are folders of repos (like ~/git): their repos, and repos one
|
||||
// level further down (org/repo).
|
||||
Roots []string
|
||||
// Starred are repos the user starred something in, shown wherever
|
||||
// they are.
|
||||
Starred []string
|
||||
// Include and Exclude are globs relative to a repo's root (** matches
|
||||
// any depth), for repos without their own .lazysecrets.yaml.
|
||||
Include, Exclude []string
|
||||
// Sops is the sops binary; "sops" (on PATH) when empty.
|
||||
Sops string
|
||||
}
|
||||
|
||||
type Provider struct {
|
||||
s Settings
|
||||
repos []repoInfo
|
||||
err error
|
||||
once sync.Once
|
||||
|
||||
// here is the repo containing Dir, found on its own so it can show
|
||||
// before the roots are scanned.
|
||||
here *repoInfo
|
||||
hereOnce sync.Once
|
||||
}
|
||||
|
||||
var (
|
||||
_ provider.Provider = (*Provider)(nil)
|
||||
_ provider.Locator = (*Provider)(nil)
|
||||
_ provider.Current = (*Provider)(nil)
|
||||
)
|
||||
|
||||
type repoInfo struct {
|
||||
root string // absolute
|
||||
name string // the folder, or its path under a root (org/repo)
|
||||
id string // the remote without scheme or .git, else the root path
|
||||
web string // the remote's web page, if it has one
|
||||
prefix string // Dir relative to root ("" when Dir isn't in it)
|
||||
here bool // the repo lazysecrets started in
|
||||
}
|
||||
|
||||
func New(s Settings) *Provider {
|
||||
if s.Sops == "" {
|
||||
s.Sops = "sops"
|
||||
}
|
||||
return &Provider{s: s}
|
||||
}
|
||||
|
||||
func (p *Provider) Kind() provider.Kind { return provider.KindRepo }
|
||||
func (p *Provider) ID() string { return "Repos" }
|
||||
func (p *Provider) Label() string { return "Repos" }
|
||||
|
||||
func (p *Provider) load() {
|
||||
p.once.Do(func() {
|
||||
seen := map[string]bool{}
|
||||
var roots []string
|
||||
names := map[string]string{} // path under a root (org/repo)
|
||||
add := func(root string) {
|
||||
if !seen[root] {
|
||||
seen[root] = true
|
||||
roots = append(roots, root)
|
||||
}
|
||||
}
|
||||
// The repo lazysecrets started in, if any: marked, so the UI can
|
||||
// put it on top, with its files under Dir first.
|
||||
here, inRepo := p.current()
|
||||
if inRepo {
|
||||
seen[here.root] = true
|
||||
}
|
||||
for _, dir := range p.s.Paths {
|
||||
root, err := gitRoot(expandHome(dir))
|
||||
if err != nil {
|
||||
p.err = errors.Join(p.err, fmt.Errorf("%s: not a git repository", dir))
|
||||
continue
|
||||
}
|
||||
add(root)
|
||||
}
|
||||
for _, dir := range p.s.Roots {
|
||||
base, _ := filepath.EvalSymlinks(expandHome(dir))
|
||||
for _, root := range reposIn(expandHome(dir), 2) {
|
||||
if name, err := filepath.Rel(base, root); err == nil && !strings.HasPrefix(name, "..") {
|
||||
names[root] = filepath.ToSlash(name)
|
||||
}
|
||||
add(root)
|
||||
}
|
||||
}
|
||||
// Repos you starred something in, wherever they are.
|
||||
for _, dir := range p.s.Starred {
|
||||
if root, err := gitRoot(dir); err == nil {
|
||||
add(root)
|
||||
}
|
||||
}
|
||||
// A git call each for the remote: dozens under ~/git, so several
|
||||
// at a time.
|
||||
infos := make([]repoInfo, len(roots))
|
||||
parallel(len(roots), func(i int) {
|
||||
infos[i] = info(roots[i], "")
|
||||
if n := names[roots[i]]; n != "" {
|
||||
infos[i].name = n
|
||||
}
|
||||
})
|
||||
if inRepo {
|
||||
infos = append([]repoInfo{here}, infos...)
|
||||
}
|
||||
p.repos = infos
|
||||
})
|
||||
}
|
||||
|
||||
// current is the repo lazysecrets started in.
|
||||
func (p *Provider) current() (repoInfo, bool) {
|
||||
p.hereOnce.Do(func() {
|
||||
root, err := gitRoot(p.s.Dir)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
rel, _ := filepath.Rel(root, absPath(p.s.Dir))
|
||||
if rel == "." {
|
||||
rel = ""
|
||||
}
|
||||
r := info(root, filepath.ToSlash(rel))
|
||||
r.here = true
|
||||
p.here = &r
|
||||
})
|
||||
if p.here == nil {
|
||||
return repoInfo{}, false
|
||||
}
|
||||
return *p.here, true
|
||||
}
|
||||
|
||||
// Current is the repo lazysecrets started in, if it has secret files:
|
||||
// one repo to check, so it can show before the others are.
|
||||
func (p *Provider) Current(ctx context.Context) (provider.Source, bool) {
|
||||
r, ok := p.current()
|
||||
if !ok {
|
||||
return provider.Source{}, false
|
||||
}
|
||||
src := p.source(r)
|
||||
if files, err := p.Containers(ctx, src); err == nil && len(files) == 0 {
|
||||
return provider.Source{}, false
|
||||
}
|
||||
return src, true
|
||||
}
|
||||
|
||||
func (p *Provider) source(r repoInfo) provider.Source {
|
||||
return provider.Source{
|
||||
Kind: provider.KindRepo, Provider: p.ID(), ProviderLabel: p.Label(),
|
||||
ID: r.root, Name: r.name, Detail: r.id, URL: r.web, Here: r.here,
|
||||
}
|
||||
}
|
||||
|
||||
// parallel runs f for 0..n-1, several at a time.
|
||||
func parallel(n int, f func(i int)) {
|
||||
var wg sync.WaitGroup
|
||||
sem := make(chan struct{}, max(8, runtime.NumCPU()))
|
||||
for i := range n {
|
||||
wg.Add(1)
|
||||
go func() {
|
||||
defer wg.Done()
|
||||
sem <- struct{}{}
|
||||
defer func() { <-sem }()
|
||||
f(i)
|
||||
}()
|
||||
}
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
// reposIn finds git repositories in dir, depth levels down at most,
|
||||
// without looking inside a repo or hidden folders.
|
||||
func reposIn(dir string, depth int) []string {
|
||||
entries, err := os.ReadDir(dir)
|
||||
if err != nil || depth == 0 {
|
||||
return nil
|
||||
}
|
||||
var out []string
|
||||
for _, e := range entries {
|
||||
if !e.IsDir() || strings.HasPrefix(e.Name(), ".") {
|
||||
continue
|
||||
}
|
||||
path := filepath.Join(dir, e.Name())
|
||||
if _, err := os.Stat(filepath.Join(path, ".git")); err == nil {
|
||||
if abs, err := filepath.EvalSymlinks(path); err == nil {
|
||||
path = abs
|
||||
}
|
||||
out = append(out, path)
|
||||
continue
|
||||
}
|
||||
out = append(out, reposIn(path, depth-1)...)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func info(root, prefix string) repoInfo {
|
||||
id, web := remoteID(root)
|
||||
if id == "" {
|
||||
id = root
|
||||
}
|
||||
return repoInfo{root: root, name: filepath.Base(root), id: id, web: web, prefix: prefix}
|
||||
}
|
||||
|
||||
// Sources lists the repos that have secret files: one without any isn't
|
||||
// worth a row.
|
||||
func (p *Provider) Sources(ctx context.Context) ([]provider.Source, error) {
|
||||
p.load()
|
||||
// Each repo is checked for secret files, several at a time: under
|
||||
// ~/git that's dozens of repos.
|
||||
keep := make([]bool, len(p.repos))
|
||||
srcs := make([]provider.Source, len(p.repos))
|
||||
for i, r := range p.repos {
|
||||
srcs[i] = p.source(r)
|
||||
}
|
||||
parallel(len(srcs), func(i int) {
|
||||
files, err := p.Containers(ctx, srcs[i])
|
||||
keep[i] = err != nil || len(files) > 0
|
||||
})
|
||||
var out []provider.Source
|
||||
for i, src := range srcs {
|
||||
if keep[i] {
|
||||
out = append(out, src)
|
||||
}
|
||||
}
|
||||
if len(out) == 0 && p.err != nil {
|
||||
return nil, p.err
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Provider) repo(id string) (repoInfo, bool) {
|
||||
if r, ok := p.current(); ok && r.root == id {
|
||||
return r, true // without waiting for the others
|
||||
}
|
||||
p.load()
|
||||
for _, r := range p.repos {
|
||||
if r.root == id {
|
||||
return r, true
|
||||
}
|
||||
}
|
||||
return repoInfo{}, false
|
||||
}
|
||||
|
||||
// Containers lists a repo's secret files: those under where lazysecrets
|
||||
// started first, then the rest, each by path.
|
||||
func (p *Provider) Containers(ctx context.Context, src provider.Source) ([]provider.Container, error) {
|
||||
r, ok := p.repo(src.ID)
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("unknown repo %s", src.ID)
|
||||
}
|
||||
files, err := tracked(ctx, r.root)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
user, sopsWhere, err := p.selector(r.root, files)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// SOPS files are tracked, where .sops.yaml says they go. Env files
|
||||
// usually aren't tracked (they're in .gitignore), so they're looked
|
||||
// for by name among untracked and ignored files too.
|
||||
envs := envFiles(r.root, files)
|
||||
seen := map[string]bool{}
|
||||
var out []provider.Container
|
||||
add := func(f string) {
|
||||
if seen[f] || !user(f) {
|
||||
return
|
||||
}
|
||||
seen[f] = true
|
||||
path := filepath.Join(r.root, f)
|
||||
c := provider.Container{Source: src, ID: path, Name: f, Types: []provider.ObjectType{provider.TypeSecret}}
|
||||
switch meta, isSops := readMeta(path); {
|
||||
case isSops && sopsWhere(f):
|
||||
c.Detail = "sops · " + strings.Join(meta.methods, ", ")
|
||||
case isSops:
|
||||
return // a SOPS file where .sops.yaml says SOPS files don't go
|
||||
case isEnv(f):
|
||||
c.Detail = "env · plain text"
|
||||
default:
|
||||
return
|
||||
}
|
||||
out = append(out, c)
|
||||
}
|
||||
for _, f := range envs {
|
||||
add(f)
|
||||
}
|
||||
for _, f := range files {
|
||||
if !inPackages(f) {
|
||||
add(f)
|
||||
}
|
||||
}
|
||||
under := func(c provider.Container) bool {
|
||||
return r.prefix == "" || strings.HasPrefix(c.Name, r.prefix+"/")
|
||||
}
|
||||
slices.SortStableFunc(out, func(a, b provider.Container) int {
|
||||
if ua, ub := under(a), under(b); ua != ub {
|
||||
if ua {
|
||||
return -1
|
||||
}
|
||||
return 1
|
||||
}
|
||||
return strings.Compare(a.Name, b.Name)
|
||||
})
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Provider) Objects(_ context.Context, c provider.Container, t provider.ObjectType) ([]provider.Object, error) {
|
||||
if t != provider.TypeSecret {
|
||||
return nil, nil
|
||||
}
|
||||
b, err := os.ReadFile(c.ID)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
meta, isSops := readMeta(c.ID)
|
||||
keys, err := parseKeys(c.ID, b)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%s: %w", c.Name, err)
|
||||
}
|
||||
if !isSops { // a plain env file
|
||||
out := make([]provider.Object, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
out = append(out, provider.Object{
|
||||
Container: c, Type: provider.TypeSecret, ID: c.ID + "\x00" + k.extract, Name: k.name,
|
||||
Attrs: []provider.Attr{{Label: "format", Value: "env"}, {Label: "encrypted", Value: "no: plain text in the file"}},
|
||||
})
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
out := make([]provider.Object, 0, len(keys))
|
||||
for _, k := range keys {
|
||||
o := provider.Object{
|
||||
Container: c, Type: provider.TypeSecret,
|
||||
ID: c.ID + "\x00" + k.extract, Name: k.name, Updated: meta.modified,
|
||||
Attrs: []provider.Attr{{Label: "format", Value: "sops " + formatOf(c.ID)}, {Label: "encryption", Value: strings.Join(meta.methods, ", ")}},
|
||||
}
|
||||
if !k.encrypted {
|
||||
o.Attrs = append(o.Attrs, provider.Attr{Label: "encrypted", Value: "no (left in plain text by the file's sops rules)"})
|
||||
}
|
||||
out = append(out, o)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func (p *Provider) Describe(_ context.Context, o provider.Object) (provider.Object, error) {
|
||||
return o, nil
|
||||
}
|
||||
|
||||
func (p *Provider) Versions(context.Context, provider.Object) ([]provider.Object, error) {
|
||||
return nil, provider.ErrUnsupported
|
||||
}
|
||||
|
||||
// Value decrypts one key with sops.
|
||||
func (p *Provider) Value(ctx context.Context, o provider.Object) (provider.SecretValue, error) {
|
||||
file, extract, ok := strings.Cut(o.ID, "\x00")
|
||||
if !ok { // made from a link: the repo's root, the file's path, the key
|
||||
file, extract = o.Container.ID, extractFor(o.Name)
|
||||
if file == "" {
|
||||
file = filepath.Join(o.Container.Source.ID, filepath.FromSlash(o.Container.Name))
|
||||
}
|
||||
}
|
||||
if _, isSops := readMeta(file); !isSops {
|
||||
return plainValue(file, o.Name)
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, p.s.Sops, "decrypt", "--extract", extract, file)
|
||||
cmd.Dir = filepath.Dir(file) // sops looks for .sops.yaml from here
|
||||
var stdout, stderr bytes.Buffer
|
||||
cmd.Stdout, cmd.Stderr = &stdout, &stderr
|
||||
if err := cmd.Run(); err != nil {
|
||||
if errors.Is(err, exec.ErrNotFound) {
|
||||
return provider.SecretValue{}, errors.New("sops isn't installed: decrypting needs it (brew install sops)")
|
||||
}
|
||||
return provider.SecretValue{}, errors.New(sopsError(stderr.String(), err))
|
||||
}
|
||||
return provider.NewSecretValue(stdout.Bytes()), nil
|
||||
}
|
||||
|
||||
// sopsError is sops' own message, whole: which keys it tried and why
|
||||
// each failed can all matter.
|
||||
func sopsError(stderr string, err error) string {
|
||||
msg := strings.TrimSpace(stderr)
|
||||
if msg == "" {
|
||||
msg = err.Error()
|
||||
}
|
||||
return "sops couldn't decrypt it:\n" + msg
|
||||
}
|
||||
|
||||
// Locate finds the repo a link names, by its remote (or root path).
|
||||
func (p *Provider) Locate(_ context.Context, host string) (string, bool, error) {
|
||||
p.load()
|
||||
for _, r := range p.repos {
|
||||
if strings.EqualFold(r.id, host) {
|
||||
return r.root, true, nil
|
||||
}
|
||||
}
|
||||
return "", false, nil
|
||||
}
|
||||
|
||||
// --- git ---
|
||||
|
||||
func gitRoot(dir string) (string, error) {
|
||||
if dir == "" {
|
||||
return "", errors.New("no directory")
|
||||
}
|
||||
out, err := exec.Command("git", "-C", dir, "rev-parse", "--show-toplevel").Output()
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return strings.TrimSpace(string(out)), nil
|
||||
}
|
||||
|
||||
func tracked(ctx context.Context, root string) ([]string, error) {
|
||||
out, err := exec.CommandContext(ctx, "git", "-C", root, "ls-files", "-z").Output()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("git ls-files in %s: %w", root, err)
|
||||
}
|
||||
var files []string
|
||||
for _, f := range strings.Split(string(out), "\x00") {
|
||||
if f != "" {
|
||||
files = append(files, f)
|
||||
}
|
||||
}
|
||||
return files, nil
|
||||
}
|
||||
|
||||
// remoteID turns the origin remote into an ID ("git.kebcloud.no/keb/repo")
|
||||
// and a web URL, both empty without a remote.
|
||||
func remoteID(root string) (string, string) {
|
||||
// From .git/config directly: a git process per repo is most of what
|
||||
// listing dozens of them costs. git answers for worktrees and
|
||||
// submodules (.git is a file) and remotes it rewrites (insteadOf).
|
||||
if url, ok := originURL(filepath.Join(root, ".git", "config")); ok {
|
||||
if url == "" {
|
||||
return "", ""
|
||||
}
|
||||
return normalizeRemote(url)
|
||||
}
|
||||
out, err := exec.Command("git", "-C", root, "remote", "get-url", "origin").Output()
|
||||
if err != nil {
|
||||
return "", ""
|
||||
}
|
||||
return normalizeRemote(strings.TrimSpace(string(out)))
|
||||
}
|
||||
|
||||
// originURL reads [remote "origin"] url from a git config file. It isn't
|
||||
// ok when the file is missing or uses insteadOf, which only git resolves.
|
||||
func originURL(config string) (string, bool) {
|
||||
b, err := os.ReadFile(config)
|
||||
if err != nil || bytes.Contains(b, []byte("insteadOf")) || bytes.Contains(b, []byte("insteadof")) {
|
||||
return "", false
|
||||
}
|
||||
inOrigin := false
|
||||
for line := range strings.SplitSeq(string(b), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
if strings.HasPrefix(line, "[") {
|
||||
inOrigin = line == `[remote "origin"]`
|
||||
continue
|
||||
}
|
||||
if k, v, ok := strings.Cut(line, "="); ok && inOrigin && strings.TrimSpace(k) == "url" {
|
||||
return strings.TrimSpace(v), true
|
||||
}
|
||||
}
|
||||
return "", true // no origin
|
||||
}
|
||||
|
||||
// normalizeRemote handles https://host/path(.git), ssh://user@host[:port]/path
|
||||
// and scp-style user@host:path.
|
||||
func normalizeRemote(remote string) (string, string) {
|
||||
u := strings.TrimSuffix(remote, ".git")
|
||||
scheme := false
|
||||
for _, s := range []string{"https://", "http://", "ssh://", "git://"} {
|
||||
if rest, ok := strings.CutPrefix(u, s); ok {
|
||||
u, scheme = rest, true
|
||||
}
|
||||
}
|
||||
host, rest, _ := strings.Cut(u, "/")
|
||||
if !scheme {
|
||||
// scp-style: user@host:path
|
||||
if h, r, ok := strings.Cut(u, ":"); ok {
|
||||
host, rest = h, r
|
||||
}
|
||||
}
|
||||
if at := strings.LastIndex(host, "@"); at >= 0 {
|
||||
host = host[at+1:]
|
||||
}
|
||||
host, _, _ = strings.Cut(host, ":") // a port
|
||||
if host == "" || rest == "" {
|
||||
return "", ""
|
||||
}
|
||||
id := strings.ToLower(host + "/" + strings.Trim(rest, "/"))
|
||||
if strings.Contains(host, ".") {
|
||||
return id, "https://" + host + "/" + strings.Trim(rest, "/")
|
||||
}
|
||||
return id, ""
|
||||
}
|
||||
|
||||
// --- which files ---
|
||||
|
||||
// lazysecretsFile is a repo's own .lazysecrets.yaml.
|
||||
type lazysecretsFile struct {
|
||||
Include []string `yaml:"include"`
|
||||
Exclude []string `yaml:"exclude"`
|
||||
}
|
||||
|
||||
// selector says which files to look at: user (the include/exclude of the
|
||||
// repo's .lazysecrets.yaml or the config) for every file, and sopsWhere
|
||||
// (.sops.yaml's creation_rules, when there's no include) for SOPS files.
|
||||
func (p *Provider) selector(root string, files []string) (user, sopsWhere func(string) bool, err error) {
|
||||
include, exclude := p.s.Include, p.s.Exclude
|
||||
if b, err := os.ReadFile(filepath.Join(root, ".lazysecrets.yaml")); err == nil {
|
||||
var f lazysecretsFile
|
||||
if err := yaml.Unmarshal(b, &f); err != nil {
|
||||
return nil, nil, fmt.Errorf("%s/.lazysecrets.yaml: %w", root, err)
|
||||
}
|
||||
if len(f.Include) > 0 || len(f.Exclude) > 0 {
|
||||
include, exclude = f.Include, f.Exclude
|
||||
}
|
||||
}
|
||||
inc, err := globs(include)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
exc, err := globs(exclude)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
var rules []sopsRule
|
||||
if len(inc) == 0 {
|
||||
rules = sopsRules(root, files)
|
||||
}
|
||||
user = func(f string) bool {
|
||||
return !anyMatch(exc, f) && (len(inc) == 0 || anyMatch(inc, f))
|
||||
}
|
||||
sopsWhere = func(f string) bool {
|
||||
return len(rules) == 0 || slices.ContainsFunc(rules, func(r sopsRule) bool { return r.matches(f) })
|
||||
}
|
||||
return user, sopsWhere, nil
|
||||
}
|
||||
|
||||
// envFiles lists env files by name (.env, .env.*, *.env), tracked or
|
||||
// not, ignored ones included, leaving out examples and dependencies.
|
||||
// tracked is the repo's tracked files. Untracked ones are looked for in
|
||||
// the folders that have tracked files, not by walking the whole tree: an
|
||||
// ignored .env sits next to the code it's for, and the walk would go
|
||||
// through node_modules and the like, which is most of the time a scan
|
||||
// takes.
|
||||
func envFiles(root string, tracked []string) []string {
|
||||
var out []string
|
||||
isTracked := make(map[string]bool, len(tracked))
|
||||
dirs := map[string]bool{".": true}
|
||||
for _, f := range tracked {
|
||||
isTracked[f] = true
|
||||
if isEnv(f) {
|
||||
out = append(out, f)
|
||||
}
|
||||
if d := path.Dir(f); !dirs[d] && !inPackages(f) {
|
||||
dirs[d] = true
|
||||
}
|
||||
}
|
||||
for d := range dirs {
|
||||
entries, err := os.ReadDir(filepath.Join(root, filepath.FromSlash(d)))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, e := range entries {
|
||||
f := path.Join(d, e.Name())
|
||||
if e.Type().IsRegular() && !isTracked[f] && isEnv(f) {
|
||||
out = append(out, f)
|
||||
}
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
// packages are folders of dependencies, caches and build output: never
|
||||
// looked in for secret files.
|
||||
var packages = map[string]bool{
|
||||
"node_modules": true, "bower_components": true, "jspm_packages": true,
|
||||
"vendor": true, "Pods": true, "Carthage": true,
|
||||
".venv": true, "venv": true, "__pycache__": true, ".tox": true, ".nox": true,
|
||||
".mypy_cache": true, ".pytest_cache": true, ".ruff_cache": true,
|
||||
".terraform": true, ".terragrunt-cache": true,
|
||||
".gradle": true, ".m2": true, ".yarn": true, ".pnpm-store": true,
|
||||
".npm": true, ".cache": true, ".next": true, ".nuxt": true, ".svelte-kit": true,
|
||||
".git": true,
|
||||
}
|
||||
|
||||
// inPackages reports whether f is inside one of the packages folders.
|
||||
func inPackages(f string) bool {
|
||||
for d := range strings.SplitSeq(path.Dir(f), "/") {
|
||||
if packages[d] {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
// isEnv reports whether a file is named like an env file with real values:
|
||||
// .env, .env.local, prod.env; not .env.example and the like, nor one in
|
||||
// dependencies.
|
||||
func isEnv(f string) bool {
|
||||
if inPackages(f) {
|
||||
return false
|
||||
}
|
||||
base := strings.ToLower(path.Base(f))
|
||||
for _, ext := range []string{".example", ".sample", ".template", ".dist", ".tmpl"} {
|
||||
if strings.HasSuffix(base, ext) {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return base == ".env" || strings.HasPrefix(base, ".env.") || strings.HasSuffix(base, ".env")
|
||||
}
|
||||
|
||||
// plainValue reads a key from a plain env file.
|
||||
func plainValue(file, name string) (provider.SecretValue, error) {
|
||||
b, err := os.ReadFile(file)
|
||||
if err != nil {
|
||||
return provider.SecretValue{}, err
|
||||
}
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
if k, v, ok := envLine(line); ok && k == name {
|
||||
return provider.NewSecretValue([]byte(v)), nil
|
||||
}
|
||||
}
|
||||
return provider.SecretValue{}, fmt.Errorf("%s isn't in %s", name, filepath.Base(file))
|
||||
}
|
||||
|
||||
// envLine reads KEY=value (or export KEY=value), unquoting the value and
|
||||
// dropping a comment after an unquoted one.
|
||||
func envLine(line string) (string, string, bool) {
|
||||
line = strings.TrimSpace(line)
|
||||
if line == "" || strings.HasPrefix(line, "#") {
|
||||
return "", "", false
|
||||
}
|
||||
line = strings.TrimPrefix(line, "export ")
|
||||
k, v, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
return "", "", false
|
||||
}
|
||||
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||||
if len(v) >= 2 && (v[0] == '"' || v[0] == '\'') {
|
||||
if end := strings.LastIndexByte(v, v[0]); end > 0 {
|
||||
inner := v[1:end]
|
||||
if v[0] == '"' {
|
||||
inner = strings.NewReplacer(`\n`, "\n", `\"`, `"`, `\\`, `\`).Replace(inner)
|
||||
}
|
||||
return k, inner, true
|
||||
}
|
||||
}
|
||||
if i := strings.Index(v, " #"); i >= 0 {
|
||||
v = strings.TrimSpace(v[:i])
|
||||
}
|
||||
return k, v, true
|
||||
}
|
||||
|
||||
// sopsRule is a creation rule of a .sops.yaml, which applies to files
|
||||
// under the directory it's in.
|
||||
type sopsRule struct {
|
||||
dir string // relative to the repo root, "" for the root
|
||||
re *regexp.Regexp
|
||||
}
|
||||
|
||||
func (r sopsRule) matches(f string) bool {
|
||||
rel := f
|
||||
if r.dir != "" {
|
||||
var ok bool
|
||||
if rel, ok = strings.CutPrefix(f, r.dir+"/"); !ok {
|
||||
return false
|
||||
}
|
||||
}
|
||||
return r.re == nil || r.re.MatchString(rel)
|
||||
}
|
||||
|
||||
func sopsRules(root string, files []string) []sopsRule {
|
||||
var rules []sopsRule
|
||||
for _, f := range files {
|
||||
if path.Base(f) != ".sops.yaml" {
|
||||
continue
|
||||
}
|
||||
b, err := os.ReadFile(filepath.Join(root, f))
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
var cfg struct {
|
||||
CreationRules []struct {
|
||||
PathRegex string `yaml:"path_regex"`
|
||||
} `yaml:"creation_rules"`
|
||||
}
|
||||
if yaml.Unmarshal(b, &cfg) != nil {
|
||||
continue
|
||||
}
|
||||
dir := path.Dir(f)
|
||||
if dir == "." {
|
||||
dir = ""
|
||||
}
|
||||
for _, cr := range cfg.CreationRules {
|
||||
r := sopsRule{dir: dir}
|
||||
if cr.PathRegex != "" {
|
||||
re, err := regexp.Compile(cr.PathRegex)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
r.re = re
|
||||
}
|
||||
rules = append(rules, r)
|
||||
}
|
||||
}
|
||||
return rules
|
||||
}
|
||||
|
||||
// globs compiles globs: * and ? within a path segment, ** across them.
|
||||
func globs(patterns []string) ([]*regexp.Regexp, error) {
|
||||
var out []*regexp.Regexp
|
||||
for _, g := range patterns {
|
||||
var b strings.Builder
|
||||
b.WriteString("^")
|
||||
for i := 0; i < len(g); i++ {
|
||||
switch c := g[i]; {
|
||||
case strings.HasPrefix(g[i:], "**/"):
|
||||
b.WriteString("(.*/)?")
|
||||
i += 2
|
||||
case strings.HasPrefix(g[i:], "**"):
|
||||
b.WriteString(".*")
|
||||
i++
|
||||
case c == '*':
|
||||
b.WriteString("[^/]*")
|
||||
case c == '?':
|
||||
b.WriteString("[^/]")
|
||||
default:
|
||||
b.WriteString(regexp.QuoteMeta(string(c)))
|
||||
}
|
||||
}
|
||||
b.WriteString("$")
|
||||
re, err := regexp.Compile(b.String())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("glob %q: %w", g, err)
|
||||
}
|
||||
out = append(out, re)
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
func anyMatch(res []*regexp.Regexp, f string) bool {
|
||||
return slices.ContainsFunc(res, func(re *regexp.Regexp) bool { return re.MatchString(f) })
|
||||
}
|
||||
|
||||
// --- sops files ---
|
||||
|
||||
type sopsMeta struct {
|
||||
methods []string // age, azure_kv, …
|
||||
modified time.Time
|
||||
}
|
||||
|
||||
const maxFileSize = 4 << 20
|
||||
|
||||
// readMeta reads a file's sops metadata; ok is false if it isn't a sops
|
||||
// file of a supported format.
|
||||
func readMeta(file string) (sopsMeta, bool) {
|
||||
fi, err := os.Stat(file)
|
||||
if err != nil || fi.Size() > maxFileSize {
|
||||
return sopsMeta{}, false
|
||||
}
|
||||
if formatOf(file) == "" {
|
||||
return sopsMeta{}, false // not a format sops writes: skip reading
|
||||
}
|
||||
b, err := os.ReadFile(file)
|
||||
if err != nil || !bytes.Contains(b, []byte("sops")) {
|
||||
return sopsMeta{}, false // cheap: most files aren't sops, no parsing
|
||||
}
|
||||
var m sopsMeta
|
||||
known := []string{"age", "azure_kv", "kms", "gcp_kms", "hc_vault", "pgp"}
|
||||
switch formatOf(file) {
|
||||
case "yaml", "json":
|
||||
var doc struct {
|
||||
Sops map[string]any `yaml:"sops"`
|
||||
}
|
||||
if yaml.Unmarshal(b, &doc) != nil || doc.Sops == nil || doc.Sops["mac"] == nil {
|
||||
return m, false
|
||||
}
|
||||
for _, k := range known {
|
||||
if v, ok := doc.Sops[k].([]any); ok && len(v) > 0 {
|
||||
m.methods = append(m.methods, k)
|
||||
}
|
||||
}
|
||||
if s, ok := doc.Sops["lastmodified"].(string); ok {
|
||||
m.modified, _ = time.Parse(time.RFC3339, s)
|
||||
}
|
||||
if t, ok := doc.Sops["lastmodified"].(time.Time); ok {
|
||||
m.modified = t
|
||||
}
|
||||
case "dotenv", "ini":
|
||||
text := string(b)
|
||||
if !strings.Contains(text, "sops_mac") && !strings.Contains(text, "sops_version") {
|
||||
return m, false
|
||||
}
|
||||
for _, k := range known {
|
||||
if strings.Contains(text, "sops_"+k+"__") {
|
||||
m.methods = append(m.methods, k)
|
||||
}
|
||||
}
|
||||
if i := strings.Index(text, "sops_lastmodified"); i >= 0 {
|
||||
line, _, _ := strings.Cut(text[i:], "\n")
|
||||
if _, v, ok := strings.Cut(line, "="); ok {
|
||||
m.modified, _ = time.Parse(time.RFC3339, strings.TrimSpace(v))
|
||||
}
|
||||
}
|
||||
default:
|
||||
return m, false
|
||||
}
|
||||
if len(m.methods) == 0 {
|
||||
m.methods = []string{"unknown keys"}
|
||||
}
|
||||
return m, true
|
||||
}
|
||||
|
||||
func formatOf(file string) string {
|
||||
switch strings.ToLower(filepath.Ext(file)) {
|
||||
case ".yaml", ".yml":
|
||||
return "yaml"
|
||||
case ".json":
|
||||
return "json"
|
||||
case ".env":
|
||||
return "dotenv"
|
||||
case ".ini":
|
||||
return "ini"
|
||||
}
|
||||
if strings.HasPrefix(filepath.Base(file), ".env") {
|
||||
return "dotenv"
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
type key struct {
|
||||
name string // db.password, list[0]
|
||||
extract string // sops --extract: ["db"]["password"]
|
||||
encrypted bool
|
||||
}
|
||||
|
||||
// parseKeys lists a sops file's keys without decrypting anything.
|
||||
func parseKeys(file string, b []byte) ([]key, error) {
|
||||
switch formatOf(file) {
|
||||
case "yaml", "json":
|
||||
var doc yaml.Node
|
||||
if err := yaml.Unmarshal(b, &doc); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if len(doc.Content) == 0 {
|
||||
return nil, nil
|
||||
}
|
||||
var out []key
|
||||
walk(doc.Content[0], "", "", true, &out)
|
||||
return out, nil
|
||||
case "dotenv":
|
||||
var out []key
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
k, v, ok := envLine(line)
|
||||
if !ok || strings.HasPrefix(k, "sops_") {
|
||||
continue
|
||||
}
|
||||
out = append(out, key{name: k, extract: quote(k), encrypted: strings.HasPrefix(v, "ENC[")})
|
||||
}
|
||||
return out, nil
|
||||
case "ini":
|
||||
var out []key
|
||||
section := ""
|
||||
for _, line := range strings.Split(string(b), "\n") {
|
||||
line = strings.TrimSpace(line)
|
||||
switch {
|
||||
case line == "" || strings.HasPrefix(line, "#") || strings.HasPrefix(line, ";"):
|
||||
case strings.HasPrefix(line, "[") && strings.HasSuffix(line, "]"):
|
||||
section = strings.TrimSpace(line[1 : len(line)-1])
|
||||
case section != "sops":
|
||||
k, v, ok := strings.Cut(line, "=")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
k, v = strings.TrimSpace(k), strings.TrimSpace(v)
|
||||
out = append(out, key{name: section + "." + k, extract: quote(section) + quote(k), encrypted: strings.HasPrefix(v, "ENC[")})
|
||||
}
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
return nil, fmt.Errorf("unsupported format")
|
||||
}
|
||||
|
||||
// walk collects the leaves of a YAML/JSON tree, skipping sops' own block.
|
||||
func walk(n *yaml.Node, name, extract string, top bool, out *[]key) {
|
||||
switch n.Kind {
|
||||
case yaml.MappingNode:
|
||||
for i := 0; i+1 < len(n.Content); i += 2 {
|
||||
k := n.Content[i].Value
|
||||
if top && k == "sops" {
|
||||
continue
|
||||
}
|
||||
child := k
|
||||
if name != "" {
|
||||
child = name + "." + k
|
||||
}
|
||||
walk(n.Content[i+1], child, extract+quote(k), false, out)
|
||||
}
|
||||
case yaml.SequenceNode:
|
||||
for i, c := range n.Content {
|
||||
walk(c, fmt.Sprintf("%s[%d]", name, i), fmt.Sprintf("%s[%d]", extract, i), false, out)
|
||||
}
|
||||
case yaml.AliasNode:
|
||||
if n.Alias != nil {
|
||||
walk(n.Alias, name, extract, false, out)
|
||||
}
|
||||
default:
|
||||
*out = append(*out, key{name: name, extract: extract, encrypted: strings.HasPrefix(n.Value, "ENC[")})
|
||||
}
|
||||
}
|
||||
|
||||
// quote is a sops --extract path element.
|
||||
func quote(k string) string {
|
||||
return `["` + strings.NewReplacer(`\`, `\\`, `"`, `\"`).Replace(k) + `"]`
|
||||
}
|
||||
|
||||
// extractFor rebuilds the extract path from a key name, for objects made
|
||||
// from a link (db.password, list[0]). Names with dots in keys can't be
|
||||
// told apart from nesting: those come from the listing instead.
|
||||
func extractFor(name string) string {
|
||||
var b strings.Builder
|
||||
for _, part := range strings.Split(name, ".") {
|
||||
base, idx, _ := strings.Cut(part, "[")
|
||||
if base != "" {
|
||||
b.WriteString(quote(base))
|
||||
}
|
||||
if idx != "" {
|
||||
b.WriteString("[" + idx)
|
||||
}
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func absPath(p string) string {
|
||||
if a, err := filepath.Abs(p); err == nil {
|
||||
return a
|
||||
}
|
||||
return p
|
||||
}
|
||||
|
||||
func expandHome(p string) string {
|
||||
if rest, ok := strings.CutPrefix(p, "~/"); ok {
|
||||
if home, err := os.UserHomeDir(); err == nil {
|
||||
return filepath.Join(home, rest)
|
||||
}
|
||||
}
|
||||
return p
|
||||
}
|
||||
@@ -0,0 +1,437 @@
|
||||
package repo
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"slices"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"git.kebcloud.no/keb/lazysecrets/internal/provider"
|
||||
)
|
||||
|
||||
// A sops-shaped YAML file: listing reads its keys and metadata only.
|
||||
const sopsYAML = `db:
|
||||
password: ENC[AES256_GCM,data:x,iv:x,tag:x,type:str]
|
||||
port: ENC[AES256_GCM,data:x,iv:x,tag:x,type:int]
|
||||
list:
|
||||
- ENC[AES256_GCM,data:x,iv:x,tag:x,type:str]
|
||||
public_unencrypted: hello
|
||||
sops:
|
||||
age:
|
||||
- recipient: age1xxxx
|
||||
enc: x
|
||||
lastmodified: "2026-10-09T10:05:29Z"
|
||||
mac: ENC[AES256_GCM,data:x,iv:x,tag:x,type:str]
|
||||
version: 3.13.3
|
||||
`
|
||||
|
||||
const sopsEnv = "API_KEY=ENC[AES256_GCM,data:x,iv:x,tag:x,type:str]\n" +
|
||||
"sops_age__list_0__map_recipient=age1xxxx\nsops_mac=ENC[x]\nsops_lastmodified=2026-10-09T10:05:29Z\nsops_version=3.13.3\n"
|
||||
|
||||
// gitRepo makes a git repository with files, all committed, and returns
|
||||
// its root.
|
||||
func gitRepo(t *testing.T, files map[string]string) string {
|
||||
t.Helper()
|
||||
if _, err := exec.LookPath("git"); err != nil {
|
||||
t.Skip("no git")
|
||||
}
|
||||
root := t.TempDir()
|
||||
root, _ = filepath.EvalSymlinks(root) // macOS: /var is /private/var
|
||||
for name, body := range files {
|
||||
p := filepath.Join(root, name)
|
||||
if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err := os.WriteFile(p, []byte(body), 0o644); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
for _, args := range [][]string{
|
||||
{"init", "-q"}, {"add", "."},
|
||||
{"-c", "user.name=t", "-c", "user.email=t@t", "commit", "-qm", "files"},
|
||||
{"remote", "add", "origin", "git@git.example.com:team/infra.git"},
|
||||
} {
|
||||
if out, err := exec.Command("git", append([]string{"-C", root}, args...)...).CombinedOutput(); err != nil {
|
||||
t.Fatalf("git %v: %v %s", args, err, out)
|
||||
}
|
||||
}
|
||||
return root
|
||||
}
|
||||
|
||||
func names[T any](items []T, name func(T) string) []string {
|
||||
var out []string
|
||||
for _, it := range items {
|
||||
out = append(out, name(it))
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func TestRepoFilesAndKeys(t *testing.T) {
|
||||
root := gitRepo(t, map[string]string{
|
||||
"secrets/app.yaml": sopsYAML,
|
||||
"svc/billing/prod.env": sopsEnv,
|
||||
"svc/billing/plain.yaml": "not: sops\n",
|
||||
"README.md": "# hi\n",
|
||||
})
|
||||
p := New(Settings{Dir: filepath.Join(root, "svc", "billing")})
|
||||
ctx := context.Background()
|
||||
|
||||
srcs, err := p.Sources(ctx)
|
||||
if err != nil || len(srcs) != 1 {
|
||||
t.Fatalf("sources %v, %v", srcs, err)
|
||||
}
|
||||
src := srcs[0]
|
||||
if src.Name != filepath.Base(root) || src.Detail != "git.example.com/team/infra" || src.URL != "https://git.example.com/team/infra" {
|
||||
t.Errorf("source %+v", src)
|
||||
}
|
||||
|
||||
files, err := p.Containers(ctx, src)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// Only sops files; those under where it started first.
|
||||
if got := names(files, func(c provider.Container) string { return c.Name }); !slices.Equal(got, []string{"svc/billing/prod.env", "secrets/app.yaml"}) {
|
||||
t.Errorf("files %v", got)
|
||||
}
|
||||
if files[1].Detail != "sops · age" {
|
||||
t.Errorf("detail %q", files[1].Detail)
|
||||
}
|
||||
|
||||
keys, err := p.Objects(ctx, files[1], provider.TypeSecret)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if got := names(keys, func(o provider.Object) string { return o.Name }); !slices.Equal(got, []string{"db.password", "db.port", "list[0]", "public_unencrypted"}) {
|
||||
t.Errorf("keys %v", got)
|
||||
}
|
||||
if keys[0].Updated.IsZero() {
|
||||
t.Error("lastmodified not read")
|
||||
}
|
||||
if _, extract, _ := strings.Cut(keys[2].ID, "\x00"); extract != `["list"][0]` {
|
||||
t.Errorf("extract %q", extract)
|
||||
}
|
||||
|
||||
env, _ := p.Objects(ctx, files[0], provider.TypeSecret)
|
||||
if got := names(env, func(o provider.Object) string { return o.Name }); !slices.Equal(got, []string{"API_KEY"}) {
|
||||
t.Errorf("env keys %v", got)
|
||||
}
|
||||
|
||||
if id, ok, _ := p.Locate(ctx, "git.example.com/team/infra"); !ok || id != root {
|
||||
t.Errorf("locate: %q %v", id, ok)
|
||||
}
|
||||
}
|
||||
|
||||
func TestWhichFiles(t *testing.T) {
|
||||
files := map[string]string{
|
||||
"secrets/a.yaml": sopsYAML,
|
||||
"other/b.yaml": sopsYAML,
|
||||
"other/testdata/c.yaml": sopsYAML,
|
||||
}
|
||||
list := func(root string, s Settings) []string {
|
||||
s.Dir = root
|
||||
p := New(s)
|
||||
srcs, _ := p.Sources(context.Background())
|
||||
cs, err := p.Containers(context.Background(), srcs[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := names(cs, func(c provider.Container) string { return c.Name })
|
||||
slices.Sort(got)
|
||||
return got
|
||||
}
|
||||
|
||||
// No rules: every tracked sops file.
|
||||
root := gitRepo(t, files)
|
||||
if got := list(root, Settings{}); len(got) != 3 {
|
||||
t.Errorf("whole repo: %v", got)
|
||||
}
|
||||
// The config's globs.
|
||||
if got := list(root, Settings{Include: []string{"**/*.yaml"}, Exclude: []string{"**/testdata/**"}}); !slices.Equal(got, []string{"other/b.yaml", "secrets/a.yaml"}) {
|
||||
t.Errorf("config globs: %v", got)
|
||||
}
|
||||
|
||||
// .sops.yaml says where sops files go.
|
||||
withRules := map[string]string{".sops.yaml": "creation_rules:\n - path_regex: secrets/.*\\.yaml$\n age: age1x\n"}
|
||||
for k, v := range files {
|
||||
withRules[k] = v
|
||||
}
|
||||
root = gitRepo(t, withRules)
|
||||
if got := list(root, Settings{}); !slices.Equal(got, []string{"secrets/a.yaml"}) {
|
||||
t.Errorf(".sops.yaml rules: %v", got)
|
||||
}
|
||||
|
||||
// The repo's own .lazysecrets.yaml wins over both.
|
||||
withRules[".lazysecrets.yaml"] = "include: [\"other/**\"]\nexclude: [\"**/testdata/**\"]\n"
|
||||
root = gitRepo(t, withRules)
|
||||
if got := list(root, Settings{Include: []string{"secrets/**"}}); !slices.Equal(got, []string{"other/b.yaml"}) {
|
||||
t.Errorf(".lazysecrets.yaml: %v", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestNormalizeRemote(t *testing.T) {
|
||||
for in, want := range map[string]string{
|
||||
"https://git.kebcloud.no/keb/nix-config.git": "git.kebcloud.no/keb/nix-config",
|
||||
"git@github.com:Team/Repo.git": "github.com/team/repo",
|
||||
"ssh://git@git.example.com:2222/team/infra": "git.example.com/team/infra",
|
||||
"https://user@dev.azure.com/org/proj/_git/x": "dev.azure.com/org/proj/_git/x",
|
||||
} {
|
||||
if got, _ := normalizeRemote(in); got != want {
|
||||
t.Errorf("%s: %q, want %q", in, got, want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// With sops and age installed: real encryption and decryption, with a
|
||||
// throwaway key.
|
||||
func TestDecrypt(t *testing.T) {
|
||||
for _, bin := range []string{"sops", "age-keygen"} {
|
||||
if _, err := exec.LookPath(bin); err != nil {
|
||||
t.Skip("needs " + bin)
|
||||
}
|
||||
}
|
||||
dir := t.TempDir()
|
||||
keyFile := filepath.Join(dir, "key.txt")
|
||||
if out, err := exec.Command("age-keygen", "-o", keyFile).CombinedOutput(); err != nil {
|
||||
t.Fatalf("age-keygen: %v %s", err, out)
|
||||
}
|
||||
b, _ := os.ReadFile(keyFile)
|
||||
var recipient string
|
||||
for _, f := range strings.Fields(string(b)) {
|
||||
if strings.HasPrefix(f, "age1") {
|
||||
recipient = f
|
||||
}
|
||||
}
|
||||
plain := filepath.Join(dir, "plain.yaml")
|
||||
os.WriteFile(plain, []byte("db:\n password: hunter2\nlist:\n - first\n"), 0o600)
|
||||
enc, err := exec.Command("sops", "encrypt", "--age", recipient, plain).Output()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
root := gitRepo(t, map[string]string{"secrets.yaml": string(enc)})
|
||||
t.Setenv("SOPS_AGE_KEY_FILE", keyFile)
|
||||
|
||||
p := New(Settings{Dir: root})
|
||||
ctx := context.Background()
|
||||
srcs, _ := p.Sources(ctx)
|
||||
cs, _ := p.Containers(ctx, srcs[0])
|
||||
keys, _ := p.Objects(ctx, cs[0], provider.TypeSecret)
|
||||
for _, k := range keys {
|
||||
v, err := p.Value(ctx, k)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %v", k.Name, err)
|
||||
}
|
||||
want := map[string]string{"db.password": "hunter2", "list[0]": "first"}[k.Name]
|
||||
if v.Reveal() != want {
|
||||
t.Errorf("%s = %q, want %q", k.Name, v.Reveal(), want)
|
||||
}
|
||||
}
|
||||
|
||||
// From a link, without the listing: the extract path from the name.
|
||||
v, err := p.Value(ctx, provider.Object{Container: cs[0], Name: "list[0]"})
|
||||
if err != nil || v.Reveal() != "first" {
|
||||
t.Errorf("from a link: %q %v", v.Reveal(), err)
|
||||
}
|
||||
|
||||
// Without the key: sops' own error.
|
||||
t.Setenv("SOPS_AGE_KEY_FILE", filepath.Join(dir, "missing.txt"))
|
||||
if _, err := p.Value(ctx, keys[0]); err == nil || !strings.Contains(err.Error(), "no identity matched any of the recipients") {
|
||||
t.Errorf("no key: %v", err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRepoWithoutSecretsIsntListed(t *testing.T) {
|
||||
root := gitRepo(t, map[string]string{"README.md": "# no secrets\n", "config.yaml": "plain: yaml\n"})
|
||||
srcs, err := New(Settings{Dir: root}).Sources(context.Background())
|
||||
if err != nil || len(srcs) != 0 {
|
||||
t.Errorf("sources %v, %v", srcs, err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestPlainEnvFiles(t *testing.T) {
|
||||
root := gitRepo(t, map[string]string{
|
||||
".gitignore": ".env\n.env.local\n",
|
||||
".env.example": "TOKEN=changeme\n",
|
||||
"secrets/app.yaml": sopsYAML,
|
||||
})
|
||||
// Ignored, as they usually are: written after the commit.
|
||||
os.WriteFile(filepath.Join(root, ".env"), []byte("# local\nexport TOKEN=\"s3cr3t value\"\nURL=https://x.example # the API\nQUOTED='a#b'\n"), 0o600)
|
||||
os.MkdirAll(filepath.Join(root, "node_modules", "pkg"), 0o755)
|
||||
os.WriteFile(filepath.Join(root, "node_modules", "pkg", ".env"), []byte("X=1\n"), 0o600)
|
||||
|
||||
p := New(Settings{Dir: root})
|
||||
ctx := context.Background()
|
||||
srcs, _ := p.Sources(ctx)
|
||||
files, err := p.Containers(ctx, srcs[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := map[string]string{}
|
||||
for _, c := range files {
|
||||
got[c.Name] = c.Detail
|
||||
}
|
||||
if len(got) != 2 || got[".env"] != "env · plain text" || got["secrets/app.yaml"] != "sops · age" {
|
||||
t.Fatalf("files %v (want .env and the sops file; not the example or node_modules)", got)
|
||||
}
|
||||
|
||||
var env provider.Container
|
||||
for _, c := range files {
|
||||
if c.Name == ".env" {
|
||||
env = c
|
||||
}
|
||||
}
|
||||
keys, _ := p.Objects(ctx, env, provider.TypeSecret)
|
||||
want := map[string]string{"TOKEN": "s3cr3t value", "URL": "https://x.example", "QUOTED": "a#b"}
|
||||
if len(keys) != len(want) {
|
||||
t.Fatalf("keys %v", names(keys, func(o provider.Object) string { return o.Name }))
|
||||
}
|
||||
for _, k := range keys {
|
||||
v, err := p.Value(ctx, k) // no sops involved
|
||||
if err != nil || v.Reveal() != want[k.Name] {
|
||||
t.Errorf("%s = %q, %v; want %q", k.Name, v.Reveal(), err, want[k.Name])
|
||||
}
|
||||
}
|
||||
|
||||
// .sops.yaml rules limit SOPS files, not plain env files.
|
||||
os.WriteFile(filepath.Join(root, ".sops.yaml"), []byte("creation_rules:\n - path_regex: nowhere/.*\n"), 0o644)
|
||||
exec.Command("git", "-C", root, "add", ".sops.yaml").Run()
|
||||
files, _ = New(Settings{Dir: root}).Containers(ctx, srcs[0])
|
||||
if names := names(files, func(c provider.Container) string { return c.Name }); !slices.Equal(names, []string{".env"}) {
|
||||
t.Errorf("with rules elsewhere: %v", names)
|
||||
}
|
||||
}
|
||||
|
||||
func TestRoots(t *testing.T) {
|
||||
withSecrets := gitRepo(t, map[string]string{"secrets/a.yaml": sopsYAML})
|
||||
without := gitRepo(t, map[string]string{"README.md": "# none\n"})
|
||||
nested := gitRepo(t, map[string]string{".env.example": "X=y\n", "prod.env": sopsEnv})
|
||||
|
||||
// A folder of repos: two at the top, one under an org folder.
|
||||
root, _ := filepath.EvalSymlinks(t.TempDir())
|
||||
os.Rename(withSecrets, filepath.Join(root, "infra"))
|
||||
os.Rename(without, filepath.Join(root, "docs"))
|
||||
os.MkdirAll(filepath.Join(root, "acme"), 0o755)
|
||||
os.Rename(nested, filepath.Join(root, "acme", "billing"))
|
||||
|
||||
outside := t.TempDir() // not in a repo
|
||||
srcs, err := New(Settings{Dir: outside, Roots: []string{root}}).Sources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := names(srcs, func(s provider.Source) string { return s.Name })
|
||||
slices.Sort(got)
|
||||
if !slices.Equal(got, []string{"acme/billing", "infra"}) {
|
||||
t.Errorf("repos under the root: %v (want the two with secret files)", got)
|
||||
}
|
||||
|
||||
// Inside a repo: the same repos, that one first and marked Here.
|
||||
srcs, _ = New(Settings{Dir: filepath.Join(root, "acme", "billing"), Roots: []string{root}}).Sources(context.Background())
|
||||
got = names(srcs, func(s provider.Source) string { return s.Name })
|
||||
if !slices.Equal(got, []string{"billing", "infra"}) || !srcs[0].Here || srcs[1].Here {
|
||||
t.Errorf("inside a repo: %v, want billing (here) then infra", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarredReposAlongside(t *testing.T) {
|
||||
here := gitRepo(t, map[string]string{"secrets/a.yaml": sopsYAML})
|
||||
starred := gitRepo(t, map[string]string{"prod.env": sopsEnv})
|
||||
other := gitRepo(t, map[string]string{"secrets/b.yaml": sopsYAML})
|
||||
_ = other
|
||||
|
||||
srcs, err := New(Settings{Dir: here, Starred: []string{starred, here}}).Sources(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := names(srcs, func(s provider.Source) string { return s.ID })
|
||||
if !slices.Equal(got, []string{here, starred}) {
|
||||
t.Errorf("sources %v, want this repo then the starred one (not under a root)", got)
|
||||
}
|
||||
}
|
||||
|
||||
// sops' output when no key opens the file, as sops 3.13 prints it.
|
||||
const noKeyStderr = `Failed to get the data key required to decrypt the SOPS file.
|
||||
|
||||
Group 0: FAILED
|
||||
age1el8zh94re2488t5rrpqxk5xcxhs2cg6xfcccntz6l4ggv0rsf9yq44c0cs: FAILED
|
||||
- | failed to create reader for decrypting sops data key with
|
||||
| age: no identity matched any of the recipients. Did not find
|
||||
| keys in locations 'SOPS_AGE_SSH_PRIVATE_KEY_FILE'.
|
||||
https://kv-app.vault.azure.net/keys/sops/abc: FAILED
|
||||
- | failed to decrypt sops data key with Azure Key Vault key
|
||||
| 'https://kv-app.vault.azure.net/keys/sops/abc': DefaultAzureCredential: failed to acquire a token.
|
||||
|
||||
Recovery failed because no master key was able to decrypt the file. In
|
||||
order for SOPS to recover the file, at least one key has to be successful,
|
||||
but none were.
|
||||
`
|
||||
|
||||
func TestSopsError(t *testing.T) {
|
||||
got := sopsError(noKeyStderr, errors.New("exit status 128"))
|
||||
if got != "sops couldn't decrypt it:\n"+strings.TrimSpace(noKeyStderr) {
|
||||
t.Errorf("not sops' whole message:\n%s", got)
|
||||
}
|
||||
if got := sopsError("", errors.New("exit status 1")); got != "sops couldn't decrypt it:\nexit status 1" {
|
||||
t.Errorf("without output: %q", got)
|
||||
}
|
||||
}
|
||||
|
||||
func TestOriginURL(t *testing.T) {
|
||||
dir := t.TempDir()
|
||||
write := func(s string) string {
|
||||
p := filepath.Join(dir, "config")
|
||||
os.WriteFile(p, []byte(s), 0o644)
|
||||
return p
|
||||
}
|
||||
for _, c := range []struct {
|
||||
config, url string
|
||||
ok bool
|
||||
}{
|
||||
{"[core]\n\tbare = false\n[remote \"origin\"]\n\turl = git@git.example.com:team/infra.git\n\tfetch = +refs/heads/*:refs/remotes/origin/*\n", "git@git.example.com:team/infra.git", true},
|
||||
{"[remote \"upstream\"]\n\turl = https://a/b\n[remote \"origin\"]\n\turl=https://git.example.com/x.git\n", "https://git.example.com/x.git", true},
|
||||
{"[core]\n\tbare = false\n", "", true}, // no origin
|
||||
{"[url \"git@host:\"]\n\tinsteadOf = https://host/\n[remote \"origin\"]\n\turl = https://host/x\n", "", false},
|
||||
} {
|
||||
url, ok := originURL(write(c.config))
|
||||
if url != c.url || ok != c.ok {
|
||||
t.Errorf("%q: %q %v, want %q %v", c.config, url, ok, c.url, c.ok)
|
||||
}
|
||||
}
|
||||
if _, ok := originURL(filepath.Join(dir, "missing")); ok {
|
||||
t.Error("a missing config should leave it to git")
|
||||
}
|
||||
}
|
||||
|
||||
func TestEnvFilesSkipPackages(t *testing.T) {
|
||||
root := gitRepo(t, map[string]string{
|
||||
"app/main.go": "package main\n",
|
||||
"vendor/lib/x.go": "package lib\n",
|
||||
"web/node_modules/pkg/index.js": "x\n",
|
||||
"secrets/sops.yaml": sopsYAML,
|
||||
"web/node_modules/pkg/sops.yaml": sopsYAML,
|
||||
})
|
||||
// Untracked, as .env files usually are: next to tracked code, in a
|
||||
// package folder, and in a folder with nothing tracked.
|
||||
for f, body := range map[string]string{
|
||||
"app/.env": "A=1\n",
|
||||
"web/node_modules/pkg/.env": "B=2\n",
|
||||
"vendor/lib/.env": "C=3\n",
|
||||
"scratch/.env": "D=4\n",
|
||||
} {
|
||||
os.MkdirAll(filepath.Dir(filepath.Join(root, f)), 0o755)
|
||||
os.WriteFile(filepath.Join(root, f), []byte(body), 0o644)
|
||||
}
|
||||
p := New(Settings{Dir: root})
|
||||
srcs, _ := p.Sources(context.Background())
|
||||
files, err := p.Containers(context.Background(), srcs[0])
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
got := names(files, func(c provider.Container) string { return c.Name })
|
||||
slices.Sort(got)
|
||||
if want := []string{"app/.env", "secrets/sops.yaml"}; !slices.Equal(got, want) {
|
||||
t.Errorf("files %v, want %v (nothing from node_modules or vendor)", got, want)
|
||||
}
|
||||
}
|
||||
Reference in new issue
Block a user