feat(env): add nestedUnderJQPath for extracting nested subtrees by jq-style paths
Introduce `nestedUnderJQPath`, enabling subtree extraction using jq-style path parsing. Add associated utility functions (`splitJQPath`, `peelRedundantPathLeaf`, `buildNestedTreeAtPath`) and test coverage. Extend `.env Codec` to support path-specific subtree merges. Update `WithFile` to accept an optional jqPath for targeted merges.
This commit is contained in:
Vendored
+28
-12
@@ -17,9 +17,14 @@ import (
|
|||||||
"github.com/joho/godotenv"
|
"github.com/joho/godotenv"
|
||||||
)
|
)
|
||||||
|
|
||||||
|
type loadedLayer struct {
|
||||||
|
load func(context.Context) (map[string]string, error)
|
||||||
|
jqPath string // empty: merge full nested tree; e.g. ".service" merges only that subtree
|
||||||
|
}
|
||||||
|
|
||||||
// Codec loads environment-style key/value data from multiple sources.
|
// Codec loads environment-style key/value data from multiple sources.
|
||||||
type Codec struct {
|
type Codec struct {
|
||||||
layers []func(context.Context) (map[string]string, error)
|
layers []loadedLayer
|
||||||
prefix string
|
prefix string
|
||||||
normalizer keymap.Normalizer
|
normalizer keymap.Normalizer
|
||||||
sliceStrat merge.SliceStrategy
|
sliceStrat merge.SliceStrategy
|
||||||
@@ -55,11 +60,15 @@ func (c *Codec) Map(ctx context.Context) (map[string]any, error) {
|
|||||||
}
|
}
|
||||||
acc := make(map[string]any)
|
acc := make(map[string]any)
|
||||||
for _, layer := range c.layers {
|
for _, layer := range c.layers {
|
||||||
flat, err := layer(ctx)
|
flat, err := layer.load(ctx)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
nested := nestedFromFlat(flat, c.prefix)
|
nested := nestedFromFlat(flat, c.prefix)
|
||||||
|
if segs := splitJQPath(layer.jqPath); len(segs) > 0 {
|
||||||
|
sub := nestedUnderJQPath(nested, layer.jqPath)
|
||||||
|
nested = buildNestedTreeAtPath(segs, sub)
|
||||||
|
}
|
||||||
merge.DeepMerge(acc, nested, c.mergeOpts...)
|
merge.DeepMerge(acc, nested, c.mergeOpts...)
|
||||||
}
|
}
|
||||||
if c.normalizer != nil {
|
if c.normalizer != nil {
|
||||||
@@ -156,17 +165,24 @@ func flatFromEnviron(environ []string) map[string]string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
func withSource(s source.Source, label string) func(*Codec) {
|
func withSource(s source.Source, label string) func(*Codec) {
|
||||||
|
return withSourceAtJQ(s, label, "")
|
||||||
|
}
|
||||||
|
|
||||||
|
func withSourceAtJQ(s source.Source, label, jqPath string) func(*Codec) {
|
||||||
return func(c *Codec) {
|
return func(c *Codec) {
|
||||||
c.layers = append(c.layers, func(ctx context.Context) (map[string]string, error) {
|
c.layers = append(c.layers, loadedLayer{
|
||||||
b, err := bytesutil.ReadAll(ctx, s)
|
jqPath: jqPath,
|
||||||
if err != nil {
|
load: func(ctx context.Context) (map[string]string, error) {
|
||||||
return nil, fmt.Errorf("env: read %s: %w", label, err)
|
b, err := bytesutil.ReadAll(ctx, s)
|
||||||
}
|
if err != nil {
|
||||||
m, err := godotenv.UnmarshalBytes(b)
|
return nil, fmt.Errorf("env: read %s: %w", label, err)
|
||||||
if err != nil {
|
}
|
||||||
return nil, fmt.Errorf("env: parse %s: %w", label, err)
|
m, err := godotenv.UnmarshalBytes(b)
|
||||||
}
|
if err != nil {
|
||||||
return m, nil
|
return nil, fmt.Errorf("env: parse %s: %w", label, err)
|
||||||
|
}
|
||||||
|
return m, nil
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Vendored
+109
@@ -0,0 +1,109 @@
|
|||||||
|
package env
|
||||||
|
|
||||||
|
import (
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
// nestedUnderJQPath returns the value at jqPath inside nested as a sub-tree
|
||||||
|
// (just the inner fields — no enclosing key matching the path leaf). It
|
||||||
|
// implements a tiny subset of jq path selection: dotted segments matched on
|
||||||
|
// lowercased keys, consistent with nestedFromFlat / insertPath.
|
||||||
|
//
|
||||||
|
// Examples (jqPath = ".service"):
|
||||||
|
//
|
||||||
|
// {"service": {"name": "x", "db": {...}}} → {"name": "x", "db": {...}}
|
||||||
|
// {"service": {"service": {"name": "inner"}}} → {"name": "inner"} (redundant
|
||||||
|
// single-key wrapper repeating
|
||||||
|
// the path leaf is peeled)
|
||||||
|
// {"other": "v"} → {} (missing)
|
||||||
|
// {"service": "scalar"} → {} (scalar at
|
||||||
|
// single-segment path is ignored)
|
||||||
|
//
|
||||||
|
// jqPath uses "." segments; a leading "." is optional. Empty / "." returns
|
||||||
|
// the input map untouched. The caller is responsible for re-wrapping the
|
||||||
|
// result at jqPath if it wants to merge the sub-tree at the same path in a
|
||||||
|
// larger document (see Codec.Map).
|
||||||
|
func nestedUnderJQPath(nested map[string]any, jqPath string) map[string]any {
|
||||||
|
segs := splitJQPath(jqPath)
|
||||||
|
if len(segs) == 0 {
|
||||||
|
return nested
|
||||||
|
}
|
||||||
|
var cur any = nested
|
||||||
|
for _, seg := range segs {
|
||||||
|
m, ok := cur.(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
cur = m[seg]
|
||||||
|
if cur == nil {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
leaf := segs[len(segs)-1]
|
||||||
|
switch v := cur.(type) {
|
||||||
|
case map[string]any:
|
||||||
|
return peelRedundantPathLeaf(v, leaf)
|
||||||
|
default:
|
||||||
|
if len(segs) == 1 {
|
||||||
|
return map[string]any{}
|
||||||
|
}
|
||||||
|
return buildNestedTreeAtPath(segs, cur)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// splitJQPath parses a jq-style dotted path into lowercased segments. Empty
|
||||||
|
// path or "." yields a nil slice (interpreted as "no path").
|
||||||
|
func splitJQPath(jqPath string) []string {
|
||||||
|
jqPath = strings.TrimSpace(jqPath)
|
||||||
|
if jqPath == "" || jqPath == "." {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
jqPath = strings.TrimPrefix(jqPath, ".")
|
||||||
|
if jqPath == "" {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
parts := strings.Split(jqPath, ".")
|
||||||
|
segs := make([]string, 0, len(parts))
|
||||||
|
for _, s := range parts {
|
||||||
|
s = strings.TrimSpace(s)
|
||||||
|
if s == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
segs = append(segs, strings.ToLower(s))
|
||||||
|
}
|
||||||
|
if len(segs) == 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
return segs
|
||||||
|
}
|
||||||
|
|
||||||
|
// peelRedundantPathLeaf strips outer {"<leaf>": {...}} shells while the only
|
||||||
|
// key in the current map matches pathLeaf. Stops as soon as the map has more
|
||||||
|
// than one key or the single key differs from pathLeaf.
|
||||||
|
func peelRedundantPathLeaf(m map[string]any, pathLeaf string) map[string]any {
|
||||||
|
for len(m) == 1 {
|
||||||
|
inner, ok := m[pathLeaf].(map[string]any)
|
||||||
|
if !ok {
|
||||||
|
break
|
||||||
|
}
|
||||||
|
m = inner
|
||||||
|
}
|
||||||
|
return m
|
||||||
|
}
|
||||||
|
|
||||||
|
// buildNestedTreeAtPath wraps leaf inside a chain of single-key maps following
|
||||||
|
// segments, e.g. (["a","b"], 1) → {"a": {"b": 1}}.
|
||||||
|
func buildNestedTreeAtPath(segments []string, leaf any) map[string]any {
|
||||||
|
root := make(map[string]any)
|
||||||
|
cur := root
|
||||||
|
for i, seg := range segments {
|
||||||
|
if i == len(segments)-1 {
|
||||||
|
cur[seg] = leaf
|
||||||
|
break
|
||||||
|
}
|
||||||
|
next := make(map[string]any)
|
||||||
|
cur[seg] = next
|
||||||
|
cur = next
|
||||||
|
}
|
||||||
|
return root
|
||||||
|
}
|
||||||
Vendored
+69
@@ -0,0 +1,69 @@
|
|||||||
|
package env
|
||||||
|
|
||||||
|
import (
|
||||||
|
"reflect"
|
||||||
|
"testing"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestNestedUnderJQPath(t *testing.T) {
|
||||||
|
nested := map[string]any{
|
||||||
|
"noise": map[string]any{"x": "ignore"},
|
||||||
|
"service": map[string]any{
|
||||||
|
"name": "my-service",
|
||||||
|
"database": map[string]any{
|
||||||
|
"url": "postgres://db",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
got := nestedUnderJQPath(nested, ".service")
|
||||||
|
want := map[string]any{
|
||||||
|
"name": "my-service",
|
||||||
|
"database": map[string]any{
|
||||||
|
"url": "postgres://db",
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %#v want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedUnderJQPath_EmptyPathNoop(t *testing.T) {
|
||||||
|
n := map[string]any{"a": "1"}
|
||||||
|
if got := nestedUnderJQPath(n, ""); !reflect.DeepEqual(got, n) {
|
||||||
|
t.Fatalf("%#v", got)
|
||||||
|
}
|
||||||
|
if got := nestedUnderJQPath(n, "."); !reflect.DeepEqual(got, n) {
|
||||||
|
t.Fatalf("%#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedUnderJQPath_MissingPath(t *testing.T) {
|
||||||
|
got := nestedUnderJQPath(map[string]any{"other": "v"}, ".service")
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("got %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedUnderJQPath_PeelsRedundantServiceWrapper(t *testing.T) {
|
||||||
|
nested := map[string]any{
|
||||||
|
"service": map[string]any{
|
||||||
|
"service": map[string]any{
|
||||||
|
"name": "inner",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
got := nestedUnderJQPath(nested, ".service")
|
||||||
|
want := map[string]any{
|
||||||
|
"name": "inner",
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(got, want) {
|
||||||
|
t.Fatalf("got %#v want %#v", got, want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNestedUnderJQPath_ScalarAtServiceIgnored(t *testing.T) {
|
||||||
|
got := nestedUnderJQPath(map[string]any{"service": "x"}, ".service")
|
||||||
|
if len(got) != 0 {
|
||||||
|
t.Fatalf("got %#v", got)
|
||||||
|
}
|
||||||
|
}
|
||||||
Vendored
+16
-5
@@ -18,15 +18,26 @@ type Option func(*Codec)
|
|||||||
// WithCurrentEnvironment appends the process environment as a source (read at Map time).
|
// WithCurrentEnvironment appends the process environment as a source (read at Map time).
|
||||||
func WithCurrentEnvironment() Option {
|
func WithCurrentEnvironment() Option {
|
||||||
return func(c *Codec) {
|
return func(c *Codec) {
|
||||||
c.layers = append(c.layers, func(_ context.Context) (map[string]string, error) {
|
c.layers = append(c.layers, loadedLayer{
|
||||||
return flatFromEnviron(os.Environ()), nil
|
load: func(_ context.Context) (map[string]string, error) {
|
||||||
|
return flatFromEnviron(os.Environ()), nil
|
||||||
|
},
|
||||||
})
|
})
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithFile appends a dotenv file path as a source.
|
// WithFile appends a dotenv file path as a source.
|
||||||
func WithFile(path string) Option {
|
// Optional jqPath selects only that subtree before merging (jq-style path, e.g. ".service"):
|
||||||
return withSource(source.File{Path: path}, path)
|
// only the object at that path is merged into the config at the same path (its fields,
|
||||||
|
// not a scalar binding for the whole branch). Keys outside that path in the file are
|
||||||
|
// ignored. A redundant single-key wrapper repeating the path leaf (e.g. service.service.*)
|
||||||
|
// is flattened so fields merge directly under service.
|
||||||
|
func WithFile(path string, jqPath ...string) Option {
|
||||||
|
jp := ""
|
||||||
|
if len(jqPath) > 0 {
|
||||||
|
jp = jqPath[0]
|
||||||
|
}
|
||||||
|
return withSourceAtJQ(source.File{Path: path}, path, jp)
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithBytes appends raw dotenv bytes as a source.
|
// WithBytes appends raw dotenv bytes as a source.
|
||||||
@@ -77,7 +88,7 @@ func WithKeyNormalizer(n keymap.Normalizer) Option {
|
|||||||
return func(c *Codec) { c.normalizer = n }
|
return func(c *Codec) { c.normalizer = n }
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithSliceMerge sets slice merge strategy when merging sources.
|
// WithSliceMerge sets a slice merge strategy when merging sources.
|
||||||
func WithSliceMerge(s merge.SliceStrategy) Option {
|
func WithSliceMerge(s merge.SliceStrategy) Option {
|
||||||
return func(c *Codec) { c.sliceStrat = s }
|
return func(c *Codec) { c.sliceStrat = s }
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -17,6 +17,20 @@ type motivationConfig struct {
|
|||||||
Database struct {
|
Database struct {
|
||||||
URL string `mapstructure:"url"`
|
URL string `mapstructure:"url"`
|
||||||
} `mapstructure:"database"`
|
} `mapstructure:"database"`
|
||||||
|
|
||||||
|
Service struct {
|
||||||
|
Database struct {
|
||||||
|
Poolsize string `json:"poolsize"`
|
||||||
|
Url string `json:"url"`
|
||||||
|
} `json:"database"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Subservice struct {
|
||||||
|
Enabled string `json:"enabled"`
|
||||||
|
Key string `json:"key"`
|
||||||
|
Name string `json:"name"`
|
||||||
|
Timeout string `json:"timeout"`
|
||||||
|
} `json:"subservice"`
|
||||||
|
} `json:"service"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestMotivationConfig_YAMLThenEnvThenProcessEnv(t *testing.T) {
|
func TestMotivationConfig_YAMLThenEnvThenProcessEnv(t *testing.T) {
|
||||||
@@ -37,6 +51,7 @@ database:
|
|||||||
HTTP_LISTEN=:9090
|
HTTP_LISTEN=:9090
|
||||||
DATABASE_URL=postgres://dotenv
|
DATABASE_URL=postgres://dotenv
|
||||||
`)),
|
`)),
|
||||||
|
env.WithFile("fixtures/identity/service.env", ".service"),
|
||||||
env.WithCurrentEnvironment(),
|
env.WithCurrentEnvironment(),
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -49,15 +64,21 @@ DATABASE_URL=postgres://dotenv
|
|||||||
t.Fatalf("env marshal returned nil map")
|
t.Fatalf("env marshal returned nil map")
|
||||||
}
|
}
|
||||||
|
|
||||||
// convert m to json and back to map
|
|
||||||
js, err := json.MarshalIndent(m, "", " ")
|
js, err := json.MarshalIndent(m, "", " ")
|
||||||
if err != nil {
|
if err != nil {
|
||||||
t.Fatalf("json marshal: %v", err)
|
t.Fatalf("json marshal: %v", err)
|
||||||
}
|
}
|
||||||
m2 := make(map[string]interface{})
|
var roundTrip map[string]interface{}
|
||||||
if err := json.Unmarshal(js, &m2); err != nil {
|
if err := json.Unmarshal(js, &roundTrip); err != nil {
|
||||||
t.Fatalf("json unmarshal: %v", err)
|
t.Fatalf("json unmarshal: %v", err)
|
||||||
}
|
}
|
||||||
|
svc, ok := roundTrip["service"].(map[string]interface{})
|
||||||
|
if !ok {
|
||||||
|
t.Fatalf("expected service in merged map, got %s", string(js))
|
||||||
|
}
|
||||||
|
if svc["name"] != "my-service" {
|
||||||
|
t.Fatalf("service.name=%v", svc["name"])
|
||||||
|
}
|
||||||
|
|
||||||
var cfg motivationConfig
|
var cfg motivationConfig
|
||||||
if err := yamlCfg.Unmarshal(&cfg); err != nil {
|
if err := yamlCfg.Unmarshal(&cfg); err != nil {
|
||||||
@@ -76,4 +97,16 @@ DATABASE_URL=postgres://dotenv
|
|||||||
if cfg.Database.URL != "postgres://process-env" {
|
if cfg.Database.URL != "postgres://process-env" {
|
||||||
t.Fatalf("Database.URL=%q", cfg.Database.URL)
|
t.Fatalf("Database.URL=%q", cfg.Database.URL)
|
||||||
}
|
}
|
||||||
|
if cfg.Service.Name != "my-service" {
|
||||||
|
t.Fatalf("Service.Name=%q", cfg.Service.Name)
|
||||||
|
}
|
||||||
|
if cfg.Service.Database.Url != "postgres://localhost:5432/db" {
|
||||||
|
t.Fatalf("Service.Database.Url=%q", cfg.Service.Database.Url)
|
||||||
|
}
|
||||||
|
if cfg.Service.Database.Poolsize != "10" {
|
||||||
|
t.Fatalf("Service.Database.Poolsize=%q", cfg.Service.Database.Poolsize)
|
||||||
|
}
|
||||||
|
if cfg.Service.Subservice.Name != "abc" {
|
||||||
|
t.Fatalf("Service.Subservice.Name=%q", cfg.Service.Subservice.Name)
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
Reference in New Issue
Block a user