feat: add TOML codec and envc format support

New toml package (pelletier/go-toml/v2) matching yaml/json options and merge flow.
envc convert/get/merge accept --from/--to toml; fixture and CLI tests; docs refresh.

Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
2026-05-03 10:42:21 +01:00
co-authored by Cursor
parent 2d7b97f5ae
commit 474cd88739
16 changed files with 307 additions and 16 deletions
+2 -2
View File
@@ -4,11 +4,11 @@ This module is part of the eSlider `go-*` library standard (inventar ASR-0008).
## Purpose
Convert **env**, **YAML**, **JSON**, and **INI** into nested `map[string]any` and Go structs (and back), with multi-source merging and the `envc` CLI.
Convert **env**, **YAML**, **JSON**, **TOML**, and **INI** into nested `map[string]any` and Go structs (and back), with multi-source merging and the `envc` CLI.
## Public API surface
- Subpackages: `env`, `yaml`, `json`, `ini` — each exports `New`, `(*Codec).Map`, `Unmarshal`, `UnmarshalContext`, `Marshal`, `WriteTo`, and format-specific options.
- Subpackages: `env`, `yaml`, `json`, `toml`, `ini` — each exports `New`, `(*Codec).Map`, `Unmarshal`, `UnmarshalContext`, `Marshal`, `WriteTo`, and format-specific options.
- `cmd/envc` — binary `envc`: `convert`, `get`, `merge`.
- Internals under `internal/` are not stable API.
+15 -5
View File
@@ -8,14 +8,14 @@
[![Go Report Card](https://goreportcard.com/badge/github.com/eslider/go-config)](https://goreportcard.com/report/github.com/eslider/go-config)
[![GitHub Stars](https://img.shields.io/github/stars/eSlider/go-config?style=social)](https://github.com/eSlider/go-config/stargazers)
Convert **env**, **YAML**, **JSON**, and **INI** to and from Go `map[string]any` and structs. Multi-source inputs merge with **deep map merge**: nested maps combine, **scalar leaves are last-write-wins**, and **slices** default to **replace** (opt-in **concat** via `WithSliceMerge`). Keys are normalized with a configurable **lower+alnum** rule so `sub-service`, `SUB_SERVICE`, and `SubService` line up across formats. Built on [go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure).
Convert **env**, **YAML**, **JSON**, **TOML**, and **INI** to and from Go `map[string]any` and structs. Multi-source inputs merge with **deep map merge**: nested maps combine, **scalar leaves are last-write-wins**, and **slices** default to **replace** (opt-in **concat** via `WithSliceMerge`). Keys are normalized with a configurable **lower+alnum** rule so `sub-service`, `SUB_SERVICE`, and `SubService` line up across formats. Built on [go-viper/mapstructure/v2](https://github.com/go-viper/mapstructure).
## Architecture
```mermaid
flowchart TB
Sources["Sources\nbytes reader file URL process env"]
Parser["Parser\ngodotenv yaml ini json"]
Parser["Parser\ngodotenv yaml json toml ini"]
Norm["keymap.Walk\nNormalizer"]
MergeOp["merge.DeepMerge"]
Map["map string any"]
@@ -88,8 +88,8 @@ os.WriteFile("out.json", b, 0o644)
## CLI: `envc`
Install the binary from [**Install**](#install) (`go install …/cmd/envc@latest`). Every
subcommand uses **`--from`** and **`--to`** with one of **`yaml`**, **`json`**, **`ini`**,
**`env`**. Snippets use **`bash`** so you can copy-paste; replace paths and URLs with yours.
subcommand uses **`--from`** and **`--to`** with one of **`yaml`**, **`json`**, **`toml`**,
**`ini`**, **`env`**. Snippets use **`bash`** so you can copy-paste; replace paths and URLs with yours.
### Help and version
@@ -121,6 +121,9 @@ envc convert --from json --to yaml --input ./settings.json --output ./settings.y
# Windows-style INI → JSON for a one-off jq filter
envc convert --from ini --to json --input ./odbc.ini --output ./odbc.json
# TOML (e.g. app / tool config) → YAML for a stack that only reads YAML
envc convert --from toml --to yaml --input ./config.toml --output ./config.yaml
# Remote YAML → materialize .env for `docker compose --env-file` or similar
envc convert --from yaml --to env \
--input "https://raw.githubusercontent.com/org/stack/main/config.yaml" \
@@ -186,6 +189,11 @@ set -a
source <(envc convert --from ini --to env --input ./legacy.ini)
set +a
# TOML tool manifest or stack file → env in the shell
set -a
source <(envc convert --from toml --to env --input ./stack.toml)
set +a
# Inline YAML here-doc → env → source (CI or local; no intermediate file)
set -a
source <(cat <<'YAML' | envc convert --from yaml --to env
@@ -213,7 +221,7 @@ envc convert --from json --to yaml \
--output ./snapshot.yaml
```
## API (all codecs)
## API (format codecs)
| Method | Description |
| ----------------------------------------------- | --------------------------------- |
@@ -234,6 +242,8 @@ Shared options (each subpackage): `WithBytes`, `WithReader`, `WithFile`, `WithUR
INI uses dotted sections, e.g. `[service.subservice]` with `name=...`.
TOML uses explicit tables, e.g. `[service]`, `[service.subservice]`, with `name = "..."`.
## Related libraries
| Module | Role |
+7 -2
View File
@@ -11,6 +11,7 @@ import (
"github.com/eslider/go-config/ini"
"github.com/eslider/go-config/internal/bytesutil"
libjson "github.com/eslider/go-config/json"
"github.com/eslider/go-config/toml"
"github.com/eslider/go-config/yaml"
yaml3 "gopkg.in/yaml.v3"
)
@@ -19,8 +20,8 @@ import (
func RunConvert(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("convert", flag.ContinueOnError)
fs.SetOutput(stderr)
from := fs.String("from", "", "source format: yaml|json|ini|env")
to := fs.String("to", "", "target format: yaml|json|ini|env")
from := fs.String("from", "", "source format: yaml|json|toml|ini|env")
to := fs.String("to", "", "target format: yaml|json|toml|ini|env")
input := fs.String("input", "-", "input path, URL, or - for stdin")
output := fs.String("output", "-", "output path or - for stdout")
if err := fs.Parse(args); err != nil {
@@ -75,6 +76,8 @@ func loadMapFromBytes(ctx context.Context, format string, b []byte) (map[string]
return ini.New(ini.WithBytes(b)).Map(ctx)
case "env":
return env.New(env.WithBytes(b)).Map(ctx)
case "toml":
return toml.New(toml.WithBytes(b)).Map(ctx)
default:
return nil, fmt.Errorf("unknown format %q", format)
}
@@ -90,6 +93,8 @@ func marshalMap(format string, m map[string]any) ([]byte, error) {
return ini.New().Marshal(m)
case "env":
return env.New().Marshal(m)
case "toml":
return toml.New().Marshal(m)
default:
return nil, fmt.Errorf("unknown format %q", format)
}
+12
View File
@@ -8,6 +8,18 @@ import (
"testing"
)
func TestRunConvert_TOMLToJSON(t *testing.T) {
var stdout, stderr bytes.Buffer
stdin := strings.NewReader("[app]\nport = 8080\n")
code := RunConvert([]string{"--from", "toml", "--to", "json", "--input", "-"}, stdin, &stdout, &stderr)
if code != 0 {
t.Fatalf("stderr: %s", stderr.String())
}
if !strings.Contains(stdout.String(), `"port"`) || !strings.Contains(stdout.String(), "8080") {
t.Fatalf("out: %s", stdout.String())
}
}
func TestRunConvert_YAMLToJSON(t *testing.T) {
var stdout, stderr bytes.Buffer
stdin := strings.NewReader("a: 1\n")
+1 -1
View File
@@ -1,3 +1,3 @@
// Command envc converts, queries, and merges configuration between env, YAML,
// JSON, and INI formats.
// JSON, TOML, and INI formats.
package main
+1 -1
View File
@@ -16,7 +16,7 @@ import (
func RunGet(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("get", flag.ContinueOnError)
fs.SetOutput(stderr)
from := fs.String("from", "", "format: yaml|json|ini|env")
from := fs.String("from", "", "format: yaml|json|toml|ini|env")
path := fs.String("path", "", "dot-separated path (e.g. service.subservice.name)")
if err := fs.Parse(args); err != nil {
return 2
+1 -1
View File
@@ -16,7 +16,7 @@ var (
// printUsage writes the root help text to w.
func printUsage(w io.Writer) {
_, _ = io.WriteString(w, `envc — convert, query, and merge configuration across env, YAML, JSON, and INI.
_, _ = io.WriteString(w, `envc — convert, query, and merge configuration across env, YAML, JSON, TOML, and INI.
Usage:
envc <command> [arguments]
+2 -2
View File
@@ -15,8 +15,8 @@ import (
func RunMerge(args []string, stdin io.Reader, stdout, stderr io.Writer) int {
fs := flag.NewFlagSet("merge", flag.ContinueOnError)
fs.SetOutput(stderr)
from := fs.String("from", "", "input format: yaml|json|ini|env")
to := fs.String("to", "", "output format: yaml|json|ini|env")
from := fs.String("from", "", "input format: yaml|json|toml|ini|env")
to := fs.String("to", "", "output format: yaml|json|toml|ini|env")
output := fs.String("output", "-", "output path or - for stdout")
if err := fs.Parse(args); err != nil {
return 2
+3 -2
View File
@@ -1,6 +1,6 @@
// Package config is the module root for go-config (import path
// github.com/eslider/go-config). Use the subpackages [env], [yaml], [json], and
// [ini] for format-specific codecs and the [cmd/envc] command for CLI
// github.com/eslider/go-config). Use the subpackages [env], [yaml], [json], [toml],
// and [ini] for format-specific codecs and the [cmd/envc] command for CLI
// conversion.
//
// Hero workflow (YAML + layered dotenv + process env):
@@ -19,5 +19,6 @@
// [env]: https://pkg.go.dev/github.com/eslider/go-config/env
// [yaml]: https://pkg.go.dev/github.com/eslider/go-config/yaml
// [json]: https://pkg.go.dev/github.com/eslider/go-config/json
// [toml]: https://pkg.go.dev/github.com/eslider/go-config/toml
// [ini]: https://pkg.go.dev/github.com/eslider/go-config/ini
package config
+12
View File
@@ -0,0 +1,12 @@
[service]
name = "my-service"
[service.sub-service]
name = "abc"
key = "abs"
timeout = 30
enabled = true
[service.database]
url = "postgres://localhost:5432/db"
pool-size = 10
+1
View File
@@ -5,6 +5,7 @@ go 1.22.2
require (
github.com/go-viper/mapstructure/v2 v2.5.0
github.com/joho/godotenv v1.5.1
github.com/pelletier/go-toml/v2 v2.2.3
gopkg.in/ini.v1 v1.67.0
gopkg.in/yaml.v3 v3.0.1
)
+2
View File
@@ -4,6 +4,8 @@ github.com/go-viper/mapstructure/v2 v2.5.0 h1:vM5IJoUAy3d7zRSVtIwQgBj7BiWtMPfmPE
github.com/go-viper/mapstructure/v2 v2.5.0/go.mod h1:oJDH3BJKyqBA2TXFhDsKDGDTlndYOZ6rGS0BRZIxGhM=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/pelletier/go-toml/v2 v2.2.3 h1:YmeHyLY8mFWbdkNWwpr+qIL2bEqT0o95WSdkNHvL12M=
github.com/pelletier/go-toml/v2 v2.2.3/go.mod h1:MfCQTFTvCcUyyvvwm1+G6H/jORL20Xlb6rzQu9GuUkc=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+117
View File
@@ -0,0 +1,117 @@
package toml
import (
"bytes"
"context"
"fmt"
"io"
"net/http"
"github.com/eslider/go-config/internal/bytesutil"
"github.com/eslider/go-config/internal/keymap"
"github.com/eslider/go-config/internal/merge"
"github.com/eslider/go-config/internal/source"
"github.com/eslider/go-config/internal/structconv"
toml2 "github.com/pelletier/go-toml/v2"
)
// Codec loads TOML from multiple sources.
type Codec struct {
sources []source.Source
normalizer keymap.Normalizer
sliceStrat merge.SliceStrategy
structOpts structconv.Options
mergeOpts []merge.Option
httpClient *http.Client
urlHeader http.Header
}
// New creates a TOML codec.
func New(opts ...Option) *Codec {
c := &Codec{
normalizer: keymap.LowerAlnum,
sliceStrat: merge.Replace,
structOpts: structconv.Options{
TagName: "mapstructure",
WeaklyTyped: true,
Trim: true,
},
}
for _, o := range opts {
o(c)
}
c.mergeOpts = []merge.Option{merge.WithSliceStrategy(c.sliceStrat)}
return c
}
// Map merges all TOML sources into one map.
func (c *Codec) Map(ctx context.Context) (map[string]any, error) {
if len(c.sources) == 0 {
return nil, fmt.Errorf("toml: no sources")
}
acc := make(map[string]any)
for _, s := range c.sources {
b, err := bytesutil.ReadAll(ctx, s)
if err != nil {
return nil, fmt.Errorf("toml: read %s: %w", s.String(), err)
}
var parsed map[string]any
if err := toml2.Unmarshal(b, &parsed); err != nil {
return nil, fmt.Errorf("toml: parse %s: %w", s.String(), err)
}
if parsed == nil {
parsed = map[string]any{}
}
merge.DeepMerge(acc, parsed, c.mergeOpts...)
}
if c.normalizer != nil {
keymap.Walk(acc, c.normalizer)
}
return acc, nil
}
// Unmarshal decodes using context.Background.
func (c *Codec) Unmarshal(dst any) error {
return c.UnmarshalContext(context.Background(), dst)
}
// UnmarshalContext decodes merged TOML into dst.
func (c *Codec) UnmarshalContext(ctx context.Context, dst any) error {
m, err := c.Map(ctx)
if err != nil {
return err
}
return structconv.Decode(m, dst, c.structOpts)
}
// Marshal encodes src to TOML bytes (struct or map[string]any).
func (c *Codec) Marshal(src any) ([]byte, error) {
m, err := encodeToMap(src)
if err != nil {
return nil, fmt.Errorf("toml: encode: %w", err)
}
var buf bytes.Buffer
enc := toml2.NewEncoder(&buf)
enc.SetIndentTables(true)
if err := enc.Encode(m); err != nil {
return nil, fmt.Errorf("toml: marshal: %w", err)
}
return buf.Bytes(), nil
}
// WriteTo writes TOML to w.
func (c *Codec) WriteTo(w io.Writer, src any) (int64, error) {
b, err := c.Marshal(src)
if err != nil {
return 0, err
}
n, err := w.Write(b)
return int64(n), err
}
func encodeToMap(src any) (map[string]any, error) {
if m, ok := src.(map[string]any); ok {
return m, nil
}
return structconv.Encode(src)
}
+41
View File
@@ -0,0 +1,41 @@
package toml
import (
"context"
"testing"
"github.com/eslider/go-config/internal/testfixtures"
)
func TestCodec_IdentityTOMLFixture(t *testing.T) {
c := New(WithBytes(testfixtures.Load(t, "identity", "service.toml")))
m, err := c.Map(context.Background())
if err != nil {
t.Fatal(err)
}
if m["service"].(map[string]any)["name"] != "my-service" {
t.Fatalf("%#v", m)
}
}
func TestCodec_MarshalRoundTripMap(t *testing.T) {
in := map[string]any{
"app": map[string]any{
"name": "demo",
"port": int64(8080),
},
}
c := New()
b, err := c.Marshal(in)
if err != nil {
t.Fatal(err)
}
c2 := New(WithBytes(b))
m, err := c2.Map(context.Background())
if err != nil {
t.Fatal(err)
}
if m["app"].(map[string]any)["name"] != "demo" {
t.Fatalf("%#v", m)
}
}
+3
View File
@@ -0,0 +1,3 @@
// Package toml loads and writes TOML configuration using the shared codec pattern
// (sources, deep merge, key normalization, struct decode via mapstructure).
package toml
+87
View File
@@ -0,0 +1,87 @@
package toml
import (
"io"
"net/http"
"github.com/eslider/go-config/internal/keymap"
"github.com/eslider/go-config/internal/merge"
"github.com/eslider/go-config/internal/source"
"github.com/go-viper/mapstructure/v2"
)
// Option configures a Codec.
type Option func(*Codec)
// WithBytes appends TOML from bytes.
func WithBytes(b []byte) Option {
return func(c *Codec) { c.sources = append(c.sources, source.Bytes{Data: b, Name: "bytes"}) }
}
// WithReader appends TOML from r.
func WithReader(r io.Reader) Option {
return func(c *Codec) { c.sources = append(c.sources, source.Reader{R: r, Name: "reader"}) }
}
// WithFile appends a TOML file path.
func WithFile(path string) Option {
return func(c *Codec) { c.sources = append(c.sources, source.File{Path: path}) }
}
// WithURL appends TOML from an HTTP(S) URL.
func WithURL(raw string) Option {
return func(c *Codec) {
var hdr http.Header
if c.urlHeader != nil {
hdr = c.urlHeader.Clone()
}
c.sources = append(c.sources, source.URL{Raw: raw, Header: hdr, HTTPClient: c.httpClient})
}
}
// WithHTTPClient sets the client for subsequent WithURL sources.
func WithHTTPClient(client *http.Client) Option {
return func(c *Codec) { c.httpClient = client }
}
// WithHTTPHeader adds a header for subsequent WithURL sources.
func WithHTTPHeader(k, v string) Option {
return func(c *Codec) {
if c.urlHeader == nil {
c.urlHeader = make(http.Header)
}
c.urlHeader.Add(k, v)
}
}
// WithKeyNormalizer sets key normalizer after merge (nil disables).
func WithKeyNormalizer(n keymap.Normalizer) Option {
return func(c *Codec) { c.normalizer = n }
}
// WithSliceMerge sets slice merge strategy across sources.
func WithSliceMerge(s merge.SliceStrategy) Option {
return func(c *Codec) { c.sliceStrat = s }
}
// WithTrim toggles string trim on struct decode.
func WithTrim(enable bool) Option {
return func(c *Codec) { c.structOpts.Trim = enable }
}
// WithWeaklyTyped toggles weak typing for struct decode.
func WithWeaklyTyped(enable bool) Option {
return func(c *Codec) { c.structOpts.WeaklyTyped = enable }
}
// WithTagName sets mapstructure tag name.
func WithTagName(name string) Option {
return func(c *Codec) { c.structOpts.TagName = name }
}
// WithDecodeHook appends a decode hook.
func WithDecodeHook(h mapstructure.DecodeHookFunc) Option {
return func(c *Codec) {
c.structOpts.ExtraHooks = append(c.structOpts.ExtraHooks, h)
}
}