feat: markdown import splits H2 leafs in Go (#26)

Drop the Python ExecFile wrapper. Conversion still does not write
Ladybug; add a dry-run fixture check for the index adapter split.

internal/mdleaves/leaf.go

@@ -0,0 +1,146 @@
+package mdleaves
+
+import (
+	"encoding/json"
+	"os"
+	"path/filepath"
+	"regexp"
+	"strings"
+)
+
+type Leaf struct {
+	Source  string `json:"source"`
+	Repo    string `json:"repo"`
+	Heading string `json:"heading"`
+	Text    string `json:"text"`
+	Type    string `json:"type"`
+	Status  string `json:"status"`
+	Related string `json:"related"`
+}
+
+type chunk struct {
+	Heading string
+	Text    string
+}
+
+var (
+	h1 = regexp.MustCompile(`^# \S`)
+	h2 = regexp.MustCompile(`^## \S`)
+)
+
+func ExtractFrontmatter(text string) (map[string]string, string) {
+	if !strings.HasPrefix(text, "---") {
+		return map[string]string{}, text
+	}
+	end := strings.Index(text[3:], "\n---")
+	if end == -1 {
+		return map[string]string{}, text
+	}
+	fm := strings.TrimSpace(text[3 : 3+end])
+	body := text[3+end+4:]
+	meta := map[string]string{}
+	for _, line := range strings.Split(fm, "\n") {
+		key, value, ok := strings.Cut(line, ":")
+		if !ok {
+			continue
+		}
+		meta[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`)
+	}
+	return meta, body
+}
+
+func SplitLeafs(meta map[string]string, body string) []chunk {
+	lines := strings.Split(body, "\n")
+	title := ""
+	type hdr struct {
+		heading string
+		start   int
+	}
+	var headers []hdr
+	for i, line := range lines {
+		switch {
+		case h1.MatchString(line):
+			title = strings.TrimSpace(strings.TrimLeft(line, "#"))
+		case h2.MatchString(line):
+			headers = append(headers, hdr{strings.TrimSpace(strings.TrimLeft(line, "#")), i})
+		}
+	}
+	if len(headers) == 0 {
+		var kept []string
+		for _, l := range lines {
+			if strings.TrimSpace(l) != "" {
+				kept = append(kept, l)
+			}
+		}
+		return []chunk{{Heading: title, Text: strings.TrimSpace(strings.Join(kept, "\n"))}}
+	}
+	out := make([]chunk, 0, len(headers))
+	for idx, h := range headers {
+		end := len(lines)
+		if idx+1 < len(headers) {
+			end = headers[idx+1].start
+		}
+		var kept []string
+		for _, l := range lines[h.start:end] {
+			if strings.TrimSpace(l) != "" {
+				kept = append(kept, l)
+			}
+		}
+		text := strings.Join(kept, "\n")
+		if idx == 0 && title != "" {
+			text = title + "\n\n" + text
+		}
+		out = append(out, chunk{Heading: h.heading, Text: strings.TrimSpace(text)})
+	}
+	return out
+}
+
+func ToAll(text, path, repo string) []Leaf {
+	meta, body := ExtractFrontmatter(text)
+	if meta["type"] == "" {
+		meta["type"] = "reference"
+	}
+	if meta["status"] == "" {
+		meta["status"] = "current"
+	}
+	chunks := SplitLeafs(meta, body)
+	out := make([]Leaf, 0, len(chunks))
+	for _, c := range chunks {
+		out = append(out, Leaf{
+			Source:  path,
+			Repo:    repo,
+			Heading: c.Heading,
+			Text:    c.Text,
+			Type:    meta["type"],
+			Status:  meta["status"],
+			Related: meta["related"],
+		})
+	}
+	return out
+}
+
+func WalkMarkdown(root string) ([]string, error) {
+	var out []string
+	err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+		if info.IsDir() {
+			return nil
+		}
+		ext := strings.ToLower(filepath.Ext(p))
+		if ext == ".md" || ext == ".markdown" {
+			out = append(out, p)
+		}
+		return nil
+	})
+	return out, err
+}
+
+func EncodeJSON(leafs []Leaf) (string, error) {
+	raw, err := json.MarshalIndent(leafs, "", "  ")
+	if err != nil {
+		return "", err
+	}
+	return string(raw) + "\n", nil
+}

internal/mdleaves/leaf_test.go

@@ -0,0 +1,73 @@
+package mdleaves
+
+import (
+	"strings"
+	"testing"
+)
+
+func TestSplitLeafsOnH2(t *testing.T) {
+	body := "# Title\n\n## One\n\nalpha\n\n## Two\n\nbeta\n"
+	leafs := SplitLeafs(map[string]string{}, body)
+	if len(leafs) != 2 {
+		t.Fatalf("n=%d", len(leafs))
+	}
+	if leafs[0].Heading != "One" || !strings.Contains(leafs[0].Text, "Title") {
+		t.Fatalf("first=%+v", leafs[0])
+	}
+	if leafs[1].Heading != "Two" || strings.Contains(leafs[1].Text, "Title") {
+		t.Fatalf("second=%+v", leafs[1])
+	}
+}
+
+func TestSplitLeafsNoH2IsWholeDoc(t *testing.T) {
+	body := "# Title\n\njust a paragraph\n"
+	leafs := SplitLeafs(nil, body)
+	if len(leafs) != 1 {
+		t.Fatalf("n=%d", len(leafs))
+	}
+	if leafs[0].Heading != "Title" {
+		t.Fatalf("heading=%q", leafs[0].Heading)
+	}
+	if !strings.Contains(leafs[0].Text, "just a paragraph") {
+		t.Fatalf("text=%q", leafs[0].Text)
+	}
+}
+
+func TestFrontmatterAndToAll(t *testing.T) {
+	raw := "---\ntype: howto\nstatus: current\nrelated: docs/design.md\n---\n# Doc\n\n## Step\n\ndo it\n"
+	got := ToAll(raw, "docs/x.md", "eSlider/2dph")
+	if len(got) != 1 {
+		t.Fatalf("n=%d", len(got))
+	}
+	if got[0].Type != "howto" || got[0].Status != "current" {
+		t.Fatalf("%+v", got[0])
+	}
+	if got[0].Source != "docs/x.md" || got[0].Repo != "eSlider/2dph" {
+		t.Fatalf("%+v", got[0])
+	}
+	if got[0].Related != "docs/design.md" {
+		t.Fatalf("related=%q", got[0].Related)
+	}
+}
+
+func TestEncodeJSONAndYAML(t *testing.T) {
+	leafs := ToAll("# Hi\n\nbody\n", "a.md", "")
+	js, err := EncodeJSON(leafs)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !strings.Contains(js, `"heading": "Hi"`) {
+		t.Fatalf("json=%s", js)
+	}
+	y := EncodeYAML(leafs)
+	if !strings.Contains(y, "heading: Hi") {
+		t.Fatalf("yaml=%s", y)
+	}
+}
+
+func TestDefaultsTypeAndStatus(t *testing.T) {
+	got := ToAll("# Hi\n\nbody\n", "a.md", "")
+	if got[0].Type != "reference" || got[0].Status != "current" {
+		t.Fatalf("%+v", got[0])
+	}
+}

internal/mdleaves/yaml.go

@@ -0,0 +1,36 @@
+package mdleaves
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+)
+
+func EncodeYAML(leafs []Leaf) string {
+	if len(leafs) == 0 {
+		return "[]\n"
+	}
+	var b strings.Builder
+	for _, lf := range leafs {
+		b.WriteString("-\n")
+		writeKV(&b, "source", lf.Source)
+		writeKV(&b, "repo", lf.Repo)
+		writeKV(&b, "heading", lf.Heading)
+		writeKV(&b, "text", lf.Text)
+		writeKV(&b, "type", lf.Type)
+		writeKV(&b, "status", lf.Status)
+		writeKV(&b, "related", lf.Related)
+	}
+	return b.String()
+}
+
+func writeKV(b *strings.Builder, k, v string) {
+	fmt.Fprintf(b, "  %s: %s\n", k, yamlScalar(v))
+}
+
+func yamlScalar(s string) string {
+	if strings.Contains(s, "\n") || s == "" || strings.ContainsAny(s, ":#'\"[]{}&*!|>%@`") || s != strings.TrimSpace(s) {
+		return strconv.Quote(s)
+	}
+	return s
+}