feat: markdown import splits H2 leafs in Go (#26)
Drop the Python ExecFile wrapper. Conversion still does not write Ladybug; add a dry-run fixture check for the index adapter split.
This commit is contained in:
GitHub
@@ -42,7 +42,7 @@ bin/ self-describing tools bin/{subject}/{method}.go (shebang)
|
||||
bin/brain/ search.go serve.go index.go get.go stats.go eval.go watch.go
|
||||
bin/chats/ sync.go import.go facts.go apply.go; libs in internal/chats
|
||||
bin/mail/ sync.go import.go (index_mail → brain/index.go)
|
||||
bin/markdown/ import.go (mistune leafs)
|
||||
bin/markdown/ import.go (H2 leaf split; Python bin/md/import fallback)
|
||||
bin/postgres/ query.go (read-only YAML)
|
||||
bin/git/ import.go (go-git history; Python shim execs it)
|
||||
bin/web/ search.go (SearXNG; Python shim execs it)
|
||||
@@ -92,7 +92,7 @@ bin/brain/get.go <id> [--body] [--json] # Go read; Python bin/kb/get CI
|
||||
bin/brain/stats.go [--json]
|
||||
bin/brain/eval.go [--json] # recall@5; questions in internal/brain/rank
|
||||
bin/brain/serve.go # HTTP :8630; GET /openapi.json POST /mcp
|
||||
bin/markdown/import.go [dir] # mistune leaves → YAML
|
||||
bin/markdown/import.go [dir] # H2 leafs → YAML; Python bin/md/import fallback
|
||||
bin/git/import.go [REPO] [--json] [--limit N] # go-git history → commit leafs
|
||||
bin/web/search.go "query" [--json] # SearXNG; throttled ≠ absence
|
||||
bin/reasoner/bakeoff.go [--model ID] [--json] # D18 CPU tool-call bake-off
|
||||
|
||||
@@ -63,7 +63,7 @@ detective method: **a fact needs ≥2 independent sources or it is
|
||||
brain/serve.go HTTP API in-process + OpenAPI/MCP (D20); Zig CGO (D21)
|
||||
cgo/zig zcc zc++ CGO toolchain (zig cc, not gcc)
|
||||
mail/import.go JSON → markdown (no brain write)
|
||||
markdown/import.go mistune leaves
|
||||
markdown/import.go H2 leaf split (Go); Python bin/md/import fallback
|
||||
postgres/query.go read-only YAML (wraps bin/db/psql-yq)
|
||||
git/import.go go-git history (no git binary; conversion only)
|
||||
web/search.go SearXNG client (throttled ≠ absence)
|
||||
|
||||
@@ -31,7 +31,7 @@ graph TB
|
||||
EX["bin/facts/extract.go<br/>2-source pairing"]
|
||||
AU["bin/facts/audit.go<br/>confidence + staleness"]
|
||||
IDX["bin/brain/index.go<br/>chunk + embed"]
|
||||
MD["bin/markdown/import.go<br/>mistune leaves"]
|
||||
MD["bin/markdown/import.go<br/>H2 leaf split"]
|
||||
SR["bin/brain/search.go<br/>deduction"]
|
||||
end
|
||||
|
||||
|
||||
+85
-5
@@ -1,20 +1,100 @@
|
||||
//usr/bin/env go run -tags=markdown_import "$0" "$@"; exit
|
||||
//go:build markdown_import
|
||||
//usr/bin/env go run "$0" "$@"; exit
|
||||
//
|
||||
// bin/markdown/import.go - split markdown into leafs (mistune).
|
||||
// bin/markdown/import.go - split markdown into leafs (H2 boundaries).
|
||||
//
|
||||
// ./bin/markdown/import.go [dir]
|
||||
// ./bin/markdown/import.go --files a.md,b.md --json
|
||||
//
|
||||
// Conversion only. Brain write is bin/brain/index.go.
|
||||
// Python bin/md/import remains as a fallback.
|
||||
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"strings"
|
||||
|
||||
"github.com/eSlider/2dph/internal/cmdbin"
|
||||
"github.com/eSlider/2dph/internal/mdleaves"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(cmdbin.ExecFile("bin/md/import", os.Args[1:]))
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
jsonOut := false
|
||||
files := ""
|
||||
root := "."
|
||||
for i := 0; i < len(args); i++ {
|
||||
a := args[i]
|
||||
switch {
|
||||
case a == "--json":
|
||||
jsonOut = true
|
||||
case a == "--files" && i+1 < len(args):
|
||||
i++
|
||||
files = args[i]
|
||||
case strings.HasPrefix(a, "--files="):
|
||||
files = strings.TrimPrefix(a, "--files=")
|
||||
case a == "-h" || a == "--help":
|
||||
fmt.Fprintln(os.Stderr, "bin/markdown/import.go [dir] [--files a.md,b.md] [--json]")
|
||||
return 0
|
||||
case strings.HasPrefix(a, "-"):
|
||||
fmt.Fprintln(os.Stderr, "unknown arg:", a)
|
||||
return 2
|
||||
default:
|
||||
root = a
|
||||
}
|
||||
}
|
||||
|
||||
var paths []string
|
||||
if files != "" {
|
||||
for _, f := range strings.Split(files, ",") {
|
||||
f = strings.TrimSpace(f)
|
||||
if f != "" {
|
||||
paths = append(paths, f)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
st, err := os.Stat(root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "md/import: no such path %s\n", root)
|
||||
return 2
|
||||
}
|
||||
if !st.IsDir() {
|
||||
paths = []string{root}
|
||||
} else {
|
||||
var err error
|
||||
paths, err = mdleaves.WalkMarkdown(root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "md/import: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
}
|
||||
}
|
||||
if len(paths) == 0 {
|
||||
fmt.Fprintln(os.Stderr, "md/import: no markdown files")
|
||||
return 1
|
||||
}
|
||||
|
||||
var all []mdleaves.Leaf
|
||||
for _, p := range paths {
|
||||
raw, err := os.ReadFile(p)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "md/import: %s: %v\n", p, err)
|
||||
continue
|
||||
}
|
||||
all = append(all, mdleaves.ToAll(string(raw), p, "")...)
|
||||
}
|
||||
if jsonOut {
|
||||
s, err := mdleaves.EncodeJSON(all)
|
||||
if err != nil {
|
||||
fmt.Fprintln(os.Stderr, err)
|
||||
return 1
|
||||
}
|
||||
fmt.Print(s)
|
||||
return 0
|
||||
}
|
||||
fmt.Print(mdleaves.EncodeYAML(all))
|
||||
return 0
|
||||
}
|
||||
|
||||
@@ -125,8 +125,28 @@ class BinLayoutTest(unittest.TestCase):
|
||||
"index_mail must point at bin/brain/index.go",
|
||||
)
|
||||
|
||||
def test_markdown_import_is_shebang(self) -> None:
|
||||
def test_markdown_import_is_go_not_python_exec(self) -> None:
|
||||
self._assert_shebang("bin/markdown/import.go")
|
||||
text = (ROOT / "bin" / "markdown" / "import.go").read_text()
|
||||
self.assertNotIn("ExecFile", text)
|
||||
self.assertNotIn("cmdbin", text)
|
||||
self.assertIn("internal/mdleaves", text)
|
||||
self.assertNotIn("kb.lbug", text)
|
||||
|
||||
def test_import_adapters_do_not_write_ladybug(self) -> None:
|
||||
for rel in (
|
||||
"bin/mail/import.go",
|
||||
"bin/mail/import",
|
||||
"bin/markdown/import.go",
|
||||
"bin/chats/import.go",
|
||||
"bin/git/import.go",
|
||||
):
|
||||
text = (ROOT / rel).read_text()
|
||||
self.assertNotIn("upsert_leaf", text, rel)
|
||||
self.assertNotIn("kb.lbug", text, rel)
|
||||
self.assertNotIn("var/brain.lbug", text, rel)
|
||||
index = (ROOT / "bin" / "brain" / "index.go").read_text()
|
||||
self.assertIn("bin/kb/index", index)
|
||||
|
||||
def test_postgres_query_is_shebang(self) -> None:
|
||||
self._assert_shebang("bin/postgres/query.go")
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
"""Import adapters write files only. Index rebuild is brain/index (D14 / Gitea #7)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
|
||||
|
||||
class IndexAdapterTest(unittest.TestCase):
|
||||
def test_dry_run_fixture_corpus_does_not_write_lbug(self) -> None:
|
||||
tmp = Path(tempfile.mkdtemp())
|
||||
(tmp / "note.md").write_text("# Fixture\n\n## Leaf\n\nhello corpus\n", encoding="utf-8")
|
||||
lbug = tmp / "kb.lbug"
|
||||
try:
|
||||
import ladybug # noqa: F401
|
||||
except ImportError:
|
||||
venv_py = ROOT / ".venv" / "bin" / "python"
|
||||
if not venv_py.is_file():
|
||||
self.skipTest("ladybug missing")
|
||||
py = str(venv_py)
|
||||
else:
|
||||
py = sys.executable
|
||||
proc = subprocess.run(
|
||||
[py, str(ROOT / "bin" / "kb" / "index"), "--dry-run", "--json", "--corpus", str(tmp)],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=os.environ.copy(),
|
||||
check=False,
|
||||
)
|
||||
self.assertEqual(proc.returncode, 0, proc.stderr)
|
||||
msg = json.loads(proc.stdout)
|
||||
self.assertTrue(msg.get("dry_run"))
|
||||
self.assertGreaterEqual(msg.get("corpus_total", 0), 1)
|
||||
self.assertFalse(lbug.exists(), "dry-run must not create a Ladybug file")
|
||||
@@ -0,0 +1,146 @@
|
||||
package mdleaves
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
type Leaf struct {
|
||||
Source string `json:"source"`
|
||||
Repo string `json:"repo"`
|
||||
Heading string `json:"heading"`
|
||||
Text string `json:"text"`
|
||||
Type string `json:"type"`
|
||||
Status string `json:"status"`
|
||||
Related string `json:"related"`
|
||||
}
|
||||
|
||||
type chunk struct {
|
||||
Heading string
|
||||
Text string
|
||||
}
|
||||
|
||||
var (
|
||||
h1 = regexp.MustCompile(`^# \S`)
|
||||
h2 = regexp.MustCompile(`^## \S`)
|
||||
)
|
||||
|
||||
func ExtractFrontmatter(text string) (map[string]string, string) {
|
||||
if !strings.HasPrefix(text, "---") {
|
||||
return map[string]string{}, text
|
||||
}
|
||||
end := strings.Index(text[3:], "\n---")
|
||||
if end == -1 {
|
||||
return map[string]string{}, text
|
||||
}
|
||||
fm := strings.TrimSpace(text[3 : 3+end])
|
||||
body := text[3+end+4:]
|
||||
meta := map[string]string{}
|
||||
for _, line := range strings.Split(fm, "\n") {
|
||||
key, value, ok := strings.Cut(line, ":")
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
meta[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`)
|
||||
}
|
||||
return meta, body
|
||||
}
|
||||
|
||||
func SplitLeafs(meta map[string]string, body string) []chunk {
|
||||
lines := strings.Split(body, "\n")
|
||||
title := ""
|
||||
type hdr struct {
|
||||
heading string
|
||||
start int
|
||||
}
|
||||
var headers []hdr
|
||||
for i, line := range lines {
|
||||
switch {
|
||||
case h1.MatchString(line):
|
||||
title = strings.TrimSpace(strings.TrimLeft(line, "#"))
|
||||
case h2.MatchString(line):
|
||||
headers = append(headers, hdr{strings.TrimSpace(strings.TrimLeft(line, "#")), i})
|
||||
}
|
||||
}
|
||||
if len(headers) == 0 {
|
||||
var kept []string
|
||||
for _, l := range lines {
|
||||
if strings.TrimSpace(l) != "" {
|
||||
kept = append(kept, l)
|
||||
}
|
||||
}
|
||||
return []chunk{{Heading: title, Text: strings.TrimSpace(strings.Join(kept, "\n"))}}
|
||||
}
|
||||
out := make([]chunk, 0, len(headers))
|
||||
for idx, h := range headers {
|
||||
end := len(lines)
|
||||
if idx+1 < len(headers) {
|
||||
end = headers[idx+1].start
|
||||
}
|
||||
var kept []string
|
||||
for _, l := range lines[h.start:end] {
|
||||
if strings.TrimSpace(l) != "" {
|
||||
kept = append(kept, l)
|
||||
}
|
||||
}
|
||||
text := strings.Join(kept, "\n")
|
||||
if idx == 0 && title != "" {
|
||||
text = title + "\n\n" + text
|
||||
}
|
||||
out = append(out, chunk{Heading: h.heading, Text: strings.TrimSpace(text)})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func ToAll(text, path, repo string) []Leaf {
|
||||
meta, body := ExtractFrontmatter(text)
|
||||
if meta["type"] == "" {
|
||||
meta["type"] = "reference"
|
||||
}
|
||||
if meta["status"] == "" {
|
||||
meta["status"] = "current"
|
||||
}
|
||||
chunks := SplitLeafs(meta, body)
|
||||
out := make([]Leaf, 0, len(chunks))
|
||||
for _, c := range chunks {
|
||||
out = append(out, Leaf{
|
||||
Source: path,
|
||||
Repo: repo,
|
||||
Heading: c.Heading,
|
||||
Text: c.Text,
|
||||
Type: meta["type"],
|
||||
Status: meta["status"],
|
||||
Related: meta["related"],
|
||||
})
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func WalkMarkdown(root string) ([]string, error) {
|
||||
var out []string
|
||||
err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
ext := strings.ToLower(filepath.Ext(p))
|
||||
if ext == ".md" || ext == ".markdown" {
|
||||
out = append(out, p)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
return out, err
|
||||
}
|
||||
|
||||
func EncodeJSON(leafs []Leaf) (string, error) {
|
||||
raw, err := json.MarshalIndent(leafs, "", " ")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
return string(raw) + "\n", nil
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package mdleaves
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSplitLeafsOnH2(t *testing.T) {
|
||||
body := "# Title\n\n## One\n\nalpha\n\n## Two\n\nbeta\n"
|
||||
leafs := SplitLeafs(map[string]string{}, body)
|
||||
if len(leafs) != 2 {
|
||||
t.Fatalf("n=%d", len(leafs))
|
||||
}
|
||||
if leafs[0].Heading != "One" || !strings.Contains(leafs[0].Text, "Title") {
|
||||
t.Fatalf("first=%+v", leafs[0])
|
||||
}
|
||||
if leafs[1].Heading != "Two" || strings.Contains(leafs[1].Text, "Title") {
|
||||
t.Fatalf("second=%+v", leafs[1])
|
||||
}
|
||||
}
|
||||
|
||||
func TestSplitLeafsNoH2IsWholeDoc(t *testing.T) {
|
||||
body := "# Title\n\njust a paragraph\n"
|
||||
leafs := SplitLeafs(nil, body)
|
||||
if len(leafs) != 1 {
|
||||
t.Fatalf("n=%d", len(leafs))
|
||||
}
|
||||
if leafs[0].Heading != "Title" {
|
||||
t.Fatalf("heading=%q", leafs[0].Heading)
|
||||
}
|
||||
if !strings.Contains(leafs[0].Text, "just a paragraph") {
|
||||
t.Fatalf("text=%q", leafs[0].Text)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFrontmatterAndToAll(t *testing.T) {
|
||||
raw := "---\ntype: howto\nstatus: current\nrelated: docs/design.md\n---\n# Doc\n\n## Step\n\ndo it\n"
|
||||
got := ToAll(raw, "docs/x.md", "eSlider/2dph")
|
||||
if len(got) != 1 {
|
||||
t.Fatalf("n=%d", len(got))
|
||||
}
|
||||
if got[0].Type != "howto" || got[0].Status != "current" {
|
||||
t.Fatalf("%+v", got[0])
|
||||
}
|
||||
if got[0].Source != "docs/x.md" || got[0].Repo != "eSlider/2dph" {
|
||||
t.Fatalf("%+v", got[0])
|
||||
}
|
||||
if got[0].Related != "docs/design.md" {
|
||||
t.Fatalf("related=%q", got[0].Related)
|
||||
}
|
||||
}
|
||||
|
||||
func TestEncodeJSONAndYAML(t *testing.T) {
|
||||
leafs := ToAll("# Hi\n\nbody\n", "a.md", "")
|
||||
js, err := EncodeJSON(leafs)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !strings.Contains(js, `"heading": "Hi"`) {
|
||||
t.Fatalf("json=%s", js)
|
||||
}
|
||||
y := EncodeYAML(leafs)
|
||||
if !strings.Contains(y, "heading: Hi") {
|
||||
t.Fatalf("yaml=%s", y)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDefaultsTypeAndStatus(t *testing.T) {
|
||||
got := ToAll("# Hi\n\nbody\n", "a.md", "")
|
||||
if got[0].Type != "reference" || got[0].Status != "current" {
|
||||
t.Fatalf("%+v", got[0])
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,36 @@
|
||||
package mdleaves
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strconv"
|
||||
"strings"
|
||||
)
|
||||
|
||||
func EncodeYAML(leafs []Leaf) string {
|
||||
if len(leafs) == 0 {
|
||||
return "[]\n"
|
||||
}
|
||||
var b strings.Builder
|
||||
for _, lf := range leafs {
|
||||
b.WriteString("-\n")
|
||||
writeKV(&b, "source", lf.Source)
|
||||
writeKV(&b, "repo", lf.Repo)
|
||||
writeKV(&b, "heading", lf.Heading)
|
||||
writeKV(&b, "text", lf.Text)
|
||||
writeKV(&b, "type", lf.Type)
|
||||
writeKV(&b, "status", lf.Status)
|
||||
writeKV(&b, "related", lf.Related)
|
||||
}
|
||||
return b.String()
|
||||
}
|
||||
|
||||
func writeKV(b *strings.Builder, k, v string) {
|
||||
fmt.Fprintf(b, " %s: %s\n", k, yamlScalar(v))
|
||||
}
|
||||
|
||||
func yamlScalar(s string) string {
|
||||
if strings.Contains(s, "\n") || s == "" || strings.ContainsAny(s, ":#'\"[]{}&*!|>%@`") || s != strings.TrimSpace(s) {
|
||||
return strconv.Quote(s)
|
||||
}
|
||||
return s
|
||||
}
|
||||
Reference in New Issue
Block a user