diff --git a/AGENTS.md b/AGENTS.md index 5123227..0312748 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -4,11 +4,11 @@ This module is part of the eSlider `go-*` library standard (inventar ASR-0008). ## Purpose -Convert **env**, **YAML**, **JSON**, and **INI** into nested `map[string]any` and Go structs (and back), with multi-source merging and the `envc` CLI. +Convert **env**, **YAML**, **JSON**, **TOML**, and **INI** into nested `map[string]any` and Go structs (and back), with multi-source merging and the `envc` CLI. ## Public API surface -- Subpackages: `env`, `yaml`, `json`, `ini` — each exports `New`, `(*Codec).Map`, `Unmarshal`, `UnmarshalContext`, `Marshal`, `WriteTo`, and format-specific options. +- Subpackages: `env`, `yaml`, `json`, `toml`, `ini` — each exports `New`, `(*Codec).Map`, `Unmarshal`, `UnmarshalContext`, `Marshal`, `WriteTo`, and format-specific options. - `cmd/envc` — binary `envc`: `convert`, `get`, `merge`. - Internals under `internal/` are not stable API. diff --git a/README.md b/README.md index 0668fc1..8517833 100644 --- a/README.md +++ b/README.md @@ -8,14 +8,14 @@ [![Go Report Card](https://goreportcard.com/badge/github.com/eslider/go-config)](https://goreportcard.com/report/github.com/eslider/go-config) [![GitHub Stars](https://img.shields.io/github/stars/eSlider/go-config?style=social)](https://github.com/eSlider/go-config/stargazers) -Convert **env**, **YAML**, **JSON**, and **INI** to and from Go `map[string]any` and structs. Multi-source inputs merge with **deep map merge**: nested maps combine, **scalar leaves are last-write-wins**, and **slices** default to **replace** (opt-in **concat** via `WithSliceMerge`). Keys are normalized with a configurable **lower+alnum** rule so `sub-service`, `SUB_SERVICE`, and `SubService` line up across formats. Built on [go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). +Convert **env**, **YAML**, **JSON**, **TOML**, and **INI** to and from Go `map[string]any` and structs. Multi-source inputs merge with **deep map merge**: nested maps combine, **scalar leaves are last-write-wins**, and **slices** default to **replace** (opt-in **concat** via `WithSliceMerge`). Keys are normalized with a configurable **lower+alnum** rule so `sub-service`, `SUB_SERVICE`, and `SubService` line up across formats. Built on [go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure). ## Architecture ```mermaid flowchart TB Sources["Sources\nbytes reader file URL process env"] - Parser["Parser\ngodotenv yaml ini json"] + Parser["Parser\ngodotenv yaml json toml ini"] Norm["keymap.Walk\nNormalizer"] MergeOp["merge.DeepMerge"] Map["map string any"] @@ -88,8 +88,8 @@ os.WriteFile("out.json", b, 0o644) ## CLI: `envc` Install the binary from [**Install**](#install) (`go install …/cmd/envc@latest`). Every -subcommand uses **`--from`** and **`--to`** with one of **`yaml`**, **`json`**, **`ini`**, -**`env`**. Snippets use **`bash`** so you can copy-paste; replace paths and URLs with yours. +subcommand uses **`--from`** and **`--to`** with one of **`yaml`**, **`json`**, **`toml`**, +**`ini`**, **`env`**. Snippets use **`bash`** so you can copy-paste; replace paths and URLs with yours. ### Help and version @@ -121,6 +121,9 @@ envc convert --from json --to yaml --input ./settings.json --output ./settings.y # Windows-style INI → JSON for a one-off jq filter envc convert --from ini --to json --input ./odbc.ini --output ./odbc.json +# TOML (e.g. app / tool config) → YAML for a stack that only reads YAML +envc convert --from toml --to yaml --input ./config.toml --output ./config.yaml + # Remote YAML → materialize .env for `docker compose --env-file` or similar envc convert --from yaml --to env \ --input "https://raw.githubusercontent.com/org/stack/main/config.yaml" \ @@ -186,6 +189,11 @@ set -a source <(envc convert --from ini --to env --input ./legacy.ini) set +a +# TOML tool manifest or stack file → env in the shell +set -a +source <(envc convert --from toml --to env --input ./stack.toml) +set +a + # Inline YAML here-doc → env → source (CI or local; no intermediate file) set -a source <(cat <<'YAML' | envc convert --from yaml --to env @@ -213,7 +221,7 @@ envc convert --from json --to yaml \ --output ./snapshot.yaml ``` -## API (all codecs) +## API (format codecs) | Method | Description | | ----------------------------------------------- | --------------------------------- | @@ -234,6 +242,8 @@ Shared options (each subpackage): `WithBytes`, `WithReader`, `WithFile`, `WithUR INI uses dotted sections, e.g. `[service.subservice]` with `name=...`. +TOML uses explicit tables, e.g. `[service]`, `[service.subservice]`, with `name = "..."`. + ## Related libraries | Module | Role | diff --git a/cmd/envc/convert.go b/cmd/envc/convert.go index b432c83..0c2ad6c 100644 --- a/cmd/envc/convert.go +++ b/cmd/envc/convert.go @@ -11,6 +11,7 @@ import ( "github.com/eslider/go-config/ini" "github.com/eslider/go-config/internal/bytesutil" libjson "github.com/eslider/go-config/json" + "github.com/eslider/go-config/toml" "github.com/eslider/go-config/yaml" yaml3 "gopkg.in/yaml.v3" ) @@ -19,8 +20,8 @@ import ( func RunConvert(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("convert", flag.ContinueOnError) fs.SetOutput(stderr) - from := fs.String("from", "", "source format: yaml|json|ini|env") - to := fs.String("to", "", "target format: yaml|json|ini|env") + from := fs.String("from", "", "source format: yaml|json|toml|ini|env") + to := fs.String("to", "", "target format: yaml|json|toml|ini|env") input := fs.String("input", "-", "input path, URL, or - for stdin") output := fs.String("output", "-", "output path or - for stdout") if err := fs.Parse(args); err != nil { @@ -75,6 +76,8 @@ func loadMapFromBytes(ctx context.Context, format string, b []byte) (map[string] return ini.New(ini.WithBytes(b)).Map(ctx) case "env": return env.New(env.WithBytes(b)).Map(ctx) + case "toml": + return toml.New(toml.WithBytes(b)).Map(ctx) default: return nil, fmt.Errorf("unknown format %q", format) } @@ -90,6 +93,8 @@ func marshalMap(format string, m map[string]any) ([]byte, error) { return ini.New().Marshal(m) case "env": return env.New().Marshal(m) + case "toml": + return toml.New().Marshal(m) default: return nil, fmt.Errorf("unknown format %q", format) } diff --git a/cmd/envc/convert_test.go b/cmd/envc/convert_test.go index 063e255..5423168 100644 --- a/cmd/envc/convert_test.go +++ b/cmd/envc/convert_test.go @@ -8,6 +8,18 @@ import ( "testing" ) +func TestRunConvert_TOMLToJSON(t *testing.T) { + var stdout, stderr bytes.Buffer + stdin := strings.NewReader("[app]\nport = 8080\n") + code := RunConvert([]string{"--from", "toml", "--to", "json", "--input", "-"}, stdin, &stdout, &stderr) + if code != 0 { + t.Fatalf("stderr: %s", stderr.String()) + } + if !strings.Contains(stdout.String(), `"port"`) || !strings.Contains(stdout.String(), "8080") { + t.Fatalf("out: %s", stdout.String()) + } +} + func TestRunConvert_YAMLToJSON(t *testing.T) { var stdout, stderr bytes.Buffer stdin := strings.NewReader("a: 1\n") diff --git a/cmd/envc/doc.go b/cmd/envc/doc.go index 8bfa6b0..210f1a2 100644 --- a/cmd/envc/doc.go +++ b/cmd/envc/doc.go @@ -1,3 +1,3 @@ // Command envc converts, queries, and merges configuration between env, YAML, -// JSON, and INI formats. +// JSON, TOML, and INI formats. package main diff --git a/cmd/envc/get.go b/cmd/envc/get.go index f124ada..1303ef0 100644 --- a/cmd/envc/get.go +++ b/cmd/envc/get.go @@ -16,7 +16,7 @@ import ( func RunGet(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("get", flag.ContinueOnError) fs.SetOutput(stderr) - from := fs.String("from", "", "format: yaml|json|ini|env") + from := fs.String("from", "", "format: yaml|json|toml|ini|env") path := fs.String("path", "", "dot-separated path (e.g. service.subservice.name)") if err := fs.Parse(args); err != nil { return 2 diff --git a/cmd/envc/main.go b/cmd/envc/main.go index 7b2009f..114e3f1 100644 --- a/cmd/envc/main.go +++ b/cmd/envc/main.go @@ -16,7 +16,7 @@ var ( // printUsage writes the root help text to w. func printUsage(w io.Writer) { - _, _ = io.WriteString(w, `envc — convert, query, and merge configuration across env, YAML, JSON, and INI. + _, _ = io.WriteString(w, `envc — convert, query, and merge configuration across env, YAML, JSON, TOML, and INI. Usage: envc [arguments] diff --git a/cmd/envc/merge.go b/cmd/envc/merge.go index f3034fa..404ff37 100644 --- a/cmd/envc/merge.go +++ b/cmd/envc/merge.go @@ -15,8 +15,8 @@ import ( func RunMerge(args []string, stdin io.Reader, stdout, stderr io.Writer) int { fs := flag.NewFlagSet("merge", flag.ContinueOnError) fs.SetOutput(stderr) - from := fs.String("from", "", "input format: yaml|json|ini|env") - to := fs.String("to", "", "output format: yaml|json|ini|env") + from := fs.String("from", "", "input format: yaml|json|toml|ini|env") + to := fs.String("to", "", "output format: yaml|json|toml|ini|env") output := fs.String("output", "-", "output path or - for stdout") if err := fs.Parse(args); err != nil { return 2 diff --git a/doc.go b/doc.go index 9d3fe2a..be7060d 100644 --- a/doc.go +++ b/doc.go @@ -1,6 +1,6 @@ // Package config is the module root for go-config (import path -// github.com/eslider/go-config). Use the subpackages [env], [yaml], [json], and -// [ini] for format-specific codecs and the [cmd/envc] command for CLI +// github.com/eslider/go-config). Use the subpackages [env], [yaml], [json], [toml], +// and [ini] for format-specific codecs and the [cmd/envc] command for CLI // conversion. // // Hero workflow (YAML + layered dotenv + process env): @@ -19,5 +19,6 @@ // [env]: https://pkg.go.dev/github.com/eslider/go-config/env // [yaml]: https://pkg.go.dev/github.com/eslider/go-config/yaml // [json]: https://pkg.go.dev/github.com/eslider/go-config/json +// [toml]: https://pkg.go.dev/github.com/eslider/go-config/toml // [ini]: https://pkg.go.dev/github.com/eslider/go-config/ini package config diff --git a/fixtures/identity/service.toml b/fixtures/identity/service.toml new file mode 100644 index 0000000..29b5add --- /dev/null +++ b/fixtures/identity/service.toml @@ -0,0 +1,12 @@ +[service] +name = "my-service" + +[service.sub-service] +name = "abc" +key = "abs" +timeout = 30 +enabled = true + +[service.database] +url = "postgres://localhost:5432/db" +pool-size = 10 diff --git a/go.mod b/go.mod index 98a9a24..ea336af 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.22.2 require ( github.com/go-viper/mapstructure/v2 v2.5.0 github.com/joho/godotenv v1.5.1 + github.com/pelletier/go-toml/v2 v2.2.3 gopkg.in/ini.v1 v1.67.0 gopkg.in/yaml.v3 v3.0.1 ) diff --git a/go.sum b/go.sum index 5e4d7de..01ed52a 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M= +github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U= diff --git a/toml/codec.go b/toml/codec.go new file mode 100644 index 0000000..e6ecae3 --- /dev/null +++ b/toml/codec.go @@ -0,0 +1,117 @@ +package toml + +import ( + "bytes" + "context" + "fmt" + "io" + "net/http" + + "github.com/eslider/go-config/internal/bytesutil" + "github.com/eslider/go-config/internal/keymap" + "github.com/eslider/go-config/internal/merge" + "github.com/eslider/go-config/internal/source" + "github.com/eslider/go-config/internal/structconv" + toml2 "github.com/pelletier/go-toml/v2" +) + +// Codec loads TOML from multiple sources. +type Codec struct { + sources []source.Source + normalizer keymap.Normalizer + sliceStrat merge.SliceStrategy + structOpts structconv.Options + mergeOpts []merge.Option + httpClient *http.Client + urlHeader http.Header +} + +// New creates a TOML codec. +func New(opts ...Option) *Codec { + c := &Codec{ + normalizer: keymap.LowerAlnum, + sliceStrat: merge.Replace, + structOpts: structconv.Options{ + TagName: "mapstructure", + WeaklyTyped: true, + Trim: true, + }, + } + for _, o := range opts { + o(c) + } + c.mergeOpts = []merge.Option{merge.WithSliceStrategy(c.sliceStrat)} + return c +} + +// Map merges all TOML sources into one map. +func (c *Codec) Map(ctx context.Context) (map[string]any, error) { + if len(c.sources) == 0 { + return nil, fmt.Errorf("toml: no sources") + } + acc := make(map[string]any) + for _, s := range c.sources { + b, err := bytesutil.ReadAll(ctx, s) + if err != nil { + return nil, fmt.Errorf("toml: read %s: %w", s.String(), err) + } + var parsed map[string]any + if err := toml2.Unmarshal(b, &parsed); err != nil { + return nil, fmt.Errorf("toml: parse %s: %w", s.String(), err) + } + if parsed == nil { + parsed = map[string]any{} + } + merge.DeepMerge(acc, parsed, c.mergeOpts...) + } + if c.normalizer != nil { + keymap.Walk(acc, c.normalizer) + } + return acc, nil +} + +// Unmarshal decodes using context.Background. +func (c *Codec) Unmarshal(dst any) error { + return c.UnmarshalContext(context.Background(), dst) +} + +// UnmarshalContext decodes merged TOML into dst. +func (c *Codec) UnmarshalContext(ctx context.Context, dst any) error { + m, err := c.Map(ctx) + if err != nil { + return err + } + return structconv.Decode(m, dst, c.structOpts) +} + +// Marshal encodes src to TOML bytes (struct or map[string]any). +func (c *Codec) Marshal(src any) ([]byte, error) { + m, err := encodeToMap(src) + if err != nil { + return nil, fmt.Errorf("toml: encode: %w", err) + } + var buf bytes.Buffer + enc := toml2.NewEncoder(&buf) + enc.SetIndentTables(true) + if err := enc.Encode(m); err != nil { + return nil, fmt.Errorf("toml: marshal: %w", err) + } + return buf.Bytes(), nil +} + +// WriteTo writes TOML to w. +func (c *Codec) WriteTo(w io.Writer, src any) (int64, error) { + b, err := c.Marshal(src) + if err != nil { + return 0, err + } + n, err := w.Write(b) + return int64(n), err +} + +func encodeToMap(src any) (map[string]any, error) { + if m, ok := src.(map[string]any); ok { + return m, nil + } + return structconv.Encode(src) +} diff --git a/toml/codec_test.go b/toml/codec_test.go new file mode 100644 index 0000000..011709d --- /dev/null +++ b/toml/codec_test.go @@ -0,0 +1,41 @@ +package toml + +import ( + "context" + "testing" + + "github.com/eslider/go-config/internal/testfixtures" +) + +func TestCodec_IdentityTOMLFixture(t *testing.T) { + c := New(WithBytes(testfixtures.Load(t, "identity", "service.toml"))) + m, err := c.Map(context.Background()) + if err != nil { + t.Fatal(err) + } + if m["service"].(map[string]any)["name"] != "my-service" { + t.Fatalf("%#v", m) + } +} + +func TestCodec_MarshalRoundTripMap(t *testing.T) { + in := map[string]any{ + "app": map[string]any{ + "name": "demo", + "port": int64(8080), + }, + } + c := New() + b, err := c.Marshal(in) + if err != nil { + t.Fatal(err) + } + c2 := New(WithBytes(b)) + m, err := c2.Map(context.Background()) + if err != nil { + t.Fatal(err) + } + if m["app"].(map[string]any)["name"] != "demo" { + t.Fatalf("%#v", m) + } +} diff --git a/toml/doc.go b/toml/doc.go new file mode 100644 index 0000000..0d2a353 --- /dev/null +++ b/toml/doc.go @@ -0,0 +1,3 @@ +// Package toml loads and writes TOML configuration using the shared codec pattern +// (sources, deep merge, key normalization, struct decode via mapstructure). +package toml diff --git a/toml/options.go b/toml/options.go new file mode 100644 index 0000000..c726c9a --- /dev/null +++ b/toml/options.go @@ -0,0 +1,87 @@ +package toml + +import ( + "io" + "net/http" + + "github.com/eslider/go-config/internal/keymap" + "github.com/eslider/go-config/internal/merge" + "github.com/eslider/go-config/internal/source" + "github.com/go-viper/mapstructure/v2" +) + +// Option configures a Codec. +type Option func(*Codec) + +// WithBytes appends TOML from bytes. +func WithBytes(b []byte) Option { + return func(c *Codec) { c.sources = append(c.sources, source.Bytes{Data: b, Name: "bytes"}) } +} + +// WithReader appends TOML from r. +func WithReader(r io.Reader) Option { + return func(c *Codec) { c.sources = append(c.sources, source.Reader{R: r, Name: "reader"}) } +} + +// WithFile appends a TOML file path. +func WithFile(path string) Option { + return func(c *Codec) { c.sources = append(c.sources, source.File{Path: path}) } +} + +// WithURL appends TOML from an HTTP(S) URL. +func WithURL(raw string) Option { + return func(c *Codec) { + var hdr http.Header + if c.urlHeader != nil { + hdr = c.urlHeader.Clone() + } + c.sources = append(c.sources, source.URL{Raw: raw, Header: hdr, HTTPClient: c.httpClient}) + } +} + +// WithHTTPClient sets the client for subsequent WithURL sources. +func WithHTTPClient(client *http.Client) Option { + return func(c *Codec) { c.httpClient = client } +} + +// WithHTTPHeader adds a header for subsequent WithURL sources. +func WithHTTPHeader(k, v string) Option { + return func(c *Codec) { + if c.urlHeader == nil { + c.urlHeader = make(http.Header) + } + c.urlHeader.Add(k, v) + } +} + +// WithKeyNormalizer sets key normalizer after merge (nil disables). +func WithKeyNormalizer(n keymap.Normalizer) Option { + return func(c *Codec) { c.normalizer = n } +} + +// WithSliceMerge sets slice merge strategy across sources. +func WithSliceMerge(s merge.SliceStrategy) Option { + return func(c *Codec) { c.sliceStrat = s } +} + +// WithTrim toggles string trim on struct decode. +func WithTrim(enable bool) Option { + return func(c *Codec) { c.structOpts.Trim = enable } +} + +// WithWeaklyTyped toggles weak typing for struct decode. +func WithWeaklyTyped(enable bool) Option { + return func(c *Codec) { c.structOpts.WeaklyTyped = enable } +} + +// WithTagName sets mapstructure tag name. +func WithTagName(name string) Option { + return func(c *Codec) { c.structOpts.TagName = name } +} + +// WithDecodeHook appends a decode hook. +func WithDecodeHook(h mapstructure.DecodeHookFunc) Option { + return func(c *Codec) { + c.structOpts.ExtraHooks = append(c.structOpts.ExtraHooks, h) + } +}