diff --git a/AGENTS.md b/AGENTS.md index c8bd4ae..f80c5bf 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -42,7 +42,7 @@ bin/ self-describing tools bin/{subject}/{method}.go (shebang) bin/brain/ search.go serve.go index.go get.go stats.go eval.go watch.go bin/chats/ sync.go import.go facts.go apply.go; libs in internal/chats bin/mail/ sync.go import.go (index_mail → brain/index.go) -bin/markdown/ import.go (mistune leafs) +bin/markdown/ import.go (H2 leaf split; Python bin/md/import fallback) bin/postgres/ query.go (read-only YAML) bin/git/ import.go (go-git history; Python shim execs it) bin/web/ search.go (SearXNG; Python shim execs it) @@ -92,7 +92,7 @@ bin/brain/get.go [--body] [--json] # Go read; Python bin/kb/get CI bin/brain/stats.go [--json] bin/brain/eval.go [--json] # recall@5; questions in internal/brain/rank bin/brain/serve.go # HTTP :8630; GET /openapi.json POST /mcp -bin/markdown/import.go [dir] # mistune leaves → YAML +bin/markdown/import.go [dir] # H2 leafs → YAML; Python bin/md/import fallback bin/git/import.go [REPO] [--json] [--limit N] # go-git history → commit leafs bin/web/search.go "query" [--json] # SearXNG; throttled ≠ absence bin/reasoner/bakeoff.go [--model ID] [--json] # D18 CPU tool-call bake-off diff --git a/PLAN.md b/PLAN.md index 6a197cc..fc73587 100644 --- a/PLAN.md +++ b/PLAN.md @@ -63,7 +63,7 @@ detective method: **a fact needs ≥2 independent sources or it is brain/serve.go HTTP API in-process + OpenAPI/MCP (D20); Zig CGO (D21) cgo/zig zcc zc++ CGO toolchain (zig cc, not gcc) mail/import.go JSON → markdown (no brain write) - markdown/import.go mistune leaves + markdown/import.go H2 leaf split (Go); Python bin/md/import fallback postgres/query.go read-only YAML (wraps bin/db/psql-yq) git/import.go go-git history (no git binary; conversion only) web/search.go SearXNG client (throttled ≠ absence) diff --git a/README.md b/README.md index d25dd06..503e73f 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ graph TB EX["bin/facts/extract.go
2-source pairing"] AU["bin/facts/audit.go
confidence + staleness"] IDX["bin/brain/index.go
chunk + embed"] - MD["bin/markdown/import.go
mistune leaves"] + MD["bin/markdown/import.go
H2 leaf split"] SR["bin/brain/search.go
deduction"] end diff --git a/bin/markdown/import.go b/bin/markdown/import.go index 546c562..f730f54 100755 --- a/bin/markdown/import.go +++ b/bin/markdown/import.go @@ -1,20 +1,100 @@ -//usr/bin/env go run -tags=markdown_import "$0" "$@"; exit -//go:build markdown_import +//usr/bin/env go run "$0" "$@"; exit // -// bin/markdown/import.go - split markdown into leafs (mistune). +// bin/markdown/import.go - split markdown into leafs (H2 boundaries). // // ./bin/markdown/import.go [dir] // ./bin/markdown/import.go --files a.md,b.md --json // +// Conversion only. Brain write is bin/brain/index.go. +// Python bin/md/import remains as a fallback. // NOTE: never run `gofmt -w` on this file — it breaks the shebang. package main import ( + "fmt" "os" + "strings" - "github.com/eSlider/2dph/internal/cmdbin" + "github.com/eSlider/2dph/internal/mdleaves" ) func main() { - os.Exit(cmdbin.ExecFile("bin/md/import", os.Args[1:])) + os.Exit(run(os.Args[1:])) +} + +func run(args []string) int { + jsonOut := false + files := "" + root := "." + for i := 0; i < len(args); i++ { + a := args[i] + switch { + case a == "--json": + jsonOut = true + case a == "--files" && i+1 < len(args): + i++ + files = args[i] + case strings.HasPrefix(a, "--files="): + files = strings.TrimPrefix(a, "--files=") + case a == "-h" || a == "--help": + fmt.Fprintln(os.Stderr, "bin/markdown/import.go [dir] [--files a.md,b.md] [--json]") + return 0 + case strings.HasPrefix(a, "-"): + fmt.Fprintln(os.Stderr, "unknown arg:", a) + return 2 + default: + root = a + } + } + + var paths []string + if files != "" { + for _, f := range strings.Split(files, ",") { + f = strings.TrimSpace(f) + if f != "" { + paths = append(paths, f) + } + } + } else { + st, err := os.Stat(root) + if err != nil { + fmt.Fprintf(os.Stderr, "md/import: no such path %s\n", root) + return 2 + } + if !st.IsDir() { + paths = []string{root} + } else { + var err error + paths, err = mdleaves.WalkMarkdown(root) + if err != nil { + fmt.Fprintf(os.Stderr, "md/import: %v\n", err) + return 1 + } + } + } + if len(paths) == 0 { + fmt.Fprintln(os.Stderr, "md/import: no markdown files") + return 1 + } + + var all []mdleaves.Leaf + for _, p := range paths { + raw, err := os.ReadFile(p) + if err != nil { + fmt.Fprintf(os.Stderr, "md/import: %s: %v\n", p, err) + continue + } + all = append(all, mdleaves.ToAll(string(raw), p, "")...) + } + if jsonOut { + s, err := mdleaves.EncodeJSON(all) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 1 + } + fmt.Print(s) + return 0 + } + fmt.Print(mdleaves.EncodeYAML(all)) + return 0 } diff --git a/bin/tools/test_bin_layout.py b/bin/tools/test_bin_layout.py index 32bd70e..b1f50b4 100644 --- a/bin/tools/test_bin_layout.py +++ b/bin/tools/test_bin_layout.py @@ -125,8 +125,28 @@ class BinLayoutTest(unittest.TestCase): "index_mail must point at bin/brain/index.go", ) - def test_markdown_import_is_shebang(self) -> None: + def test_markdown_import_is_go_not_python_exec(self) -> None: self._assert_shebang("bin/markdown/import.go") + text = (ROOT / "bin" / "markdown" / "import.go").read_text() + self.assertNotIn("ExecFile", text) + self.assertNotIn("cmdbin", text) + self.assertIn("internal/mdleaves", text) + self.assertNotIn("kb.lbug", text) + + def test_import_adapters_do_not_write_ladybug(self) -> None: + for rel in ( + "bin/mail/import.go", + "bin/mail/import", + "bin/markdown/import.go", + "bin/chats/import.go", + "bin/git/import.go", + ): + text = (ROOT / rel).read_text() + self.assertNotIn("upsert_leaf", text, rel) + self.assertNotIn("kb.lbug", text, rel) + self.assertNotIn("var/brain.lbug", text, rel) + index = (ROOT / "bin" / "brain" / "index.go").read_text() + self.assertIn("bin/kb/index", index) def test_postgres_query_is_shebang(self) -> None: self._assert_shebang("bin/postgres/query.go") diff --git a/bin/tools/test_index_adapters.py b/bin/tools/test_index_adapters.py new file mode 100644 index 0000000..638b226 --- /dev/null +++ b/bin/tools/test_index_adapters.py @@ -0,0 +1,41 @@ +"""Import adapters write files only. Index rebuild is brain/index (D14 / Gitea #7).""" +from __future__ import annotations + +import json +import os +import subprocess +import sys +import tempfile +import unittest +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[2] + + +class IndexAdapterTest(unittest.TestCase): + def test_dry_run_fixture_corpus_does_not_write_lbug(self) -> None: + tmp = Path(tempfile.mkdtemp()) + (tmp / "note.md").write_text("# Fixture\n\n## Leaf\n\nhello corpus\n", encoding="utf-8") + lbug = tmp / "kb.lbug" + try: + import ladybug # noqa: F401 + except ImportError: + venv_py = ROOT / ".venv" / "bin" / "python" + if not venv_py.is_file(): + self.skipTest("ladybug missing") + py = str(venv_py) + else: + py = sys.executable + proc = subprocess.run( + [py, str(ROOT / "bin" / "kb" / "index"), "--dry-run", "--json", "--corpus", str(tmp)], + cwd=ROOT, + capture_output=True, + text=True, + env=os.environ.copy(), + check=False, + ) + self.assertEqual(proc.returncode, 0, proc.stderr) + msg = json.loads(proc.stdout) + self.assertTrue(msg.get("dry_run")) + self.assertGreaterEqual(msg.get("corpus_total", 0), 1) + self.assertFalse(lbug.exists(), "dry-run must not create a Ladybug file") diff --git a/internal/mdleaves/leaf.go b/internal/mdleaves/leaf.go new file mode 100644 index 0000000..939c4ac --- /dev/null +++ b/internal/mdleaves/leaf.go @@ -0,0 +1,146 @@ +package mdleaves + +import ( + "encoding/json" + "os" + "path/filepath" + "regexp" + "strings" +) + +type Leaf struct { + Source string `json:"source"` + Repo string `json:"repo"` + Heading string `json:"heading"` + Text string `json:"text"` + Type string `json:"type"` + Status string `json:"status"` + Related string `json:"related"` +} + +type chunk struct { + Heading string + Text string +} + +var ( + h1 = regexp.MustCompile(`^# \S`) + h2 = regexp.MustCompile(`^## \S`) +) + +func ExtractFrontmatter(text string) (map[string]string, string) { + if !strings.HasPrefix(text, "---") { + return map[string]string{}, text + } + end := strings.Index(text[3:], "\n---") + if end == -1 { + return map[string]string{}, text + } + fm := strings.TrimSpace(text[3 : 3+end]) + body := text[3+end+4:] + meta := map[string]string{} + for _, line := range strings.Split(fm, "\n") { + key, value, ok := strings.Cut(line, ":") + if !ok { + continue + } + meta[strings.TrimSpace(key)] = strings.Trim(strings.TrimSpace(value), `"'`) + } + return meta, body +} + +func SplitLeafs(meta map[string]string, body string) []chunk { + lines := strings.Split(body, "\n") + title := "" + type hdr struct { + heading string + start int + } + var headers []hdr + for i, line := range lines { + switch { + case h1.MatchString(line): + title = strings.TrimSpace(strings.TrimLeft(line, "#")) + case h2.MatchString(line): + headers = append(headers, hdr{strings.TrimSpace(strings.TrimLeft(line, "#")), i}) + } + } + if len(headers) == 0 { + var kept []string + for _, l := range lines { + if strings.TrimSpace(l) != "" { + kept = append(kept, l) + } + } + return []chunk{{Heading: title, Text: strings.TrimSpace(strings.Join(kept, "\n"))}} + } + out := make([]chunk, 0, len(headers)) + for idx, h := range headers { + end := len(lines) + if idx+1 < len(headers) { + end = headers[idx+1].start + } + var kept []string + for _, l := range lines[h.start:end] { + if strings.TrimSpace(l) != "" { + kept = append(kept, l) + } + } + text := strings.Join(kept, "\n") + if idx == 0 && title != "" { + text = title + "\n\n" + text + } + out = append(out, chunk{Heading: h.heading, Text: strings.TrimSpace(text)}) + } + return out +} + +func ToAll(text, path, repo string) []Leaf { + meta, body := ExtractFrontmatter(text) + if meta["type"] == "" { + meta["type"] = "reference" + } + if meta["status"] == "" { + meta["status"] = "current" + } + chunks := SplitLeafs(meta, body) + out := make([]Leaf, 0, len(chunks)) + for _, c := range chunks { + out = append(out, Leaf{ + Source: path, + Repo: repo, + Heading: c.Heading, + Text: c.Text, + Type: meta["type"], + Status: meta["status"], + Related: meta["related"], + }) + } + return out +} + +func WalkMarkdown(root string) ([]string, error) { + var out []string + err := filepath.Walk(root, func(p string, info os.FileInfo, err error) error { + if err != nil { + return err + } + if info.IsDir() { + return nil + } + ext := strings.ToLower(filepath.Ext(p)) + if ext == ".md" || ext == ".markdown" { + out = append(out, p) + } + return nil + }) + return out, err +} + +func EncodeJSON(leafs []Leaf) (string, error) { + raw, err := json.MarshalIndent(leafs, "", " ") + if err != nil { + return "", err + } + return string(raw) + "\n", nil +} diff --git a/internal/mdleaves/leaf_test.go b/internal/mdleaves/leaf_test.go new file mode 100644 index 0000000..4dfa3f7 --- /dev/null +++ b/internal/mdleaves/leaf_test.go @@ -0,0 +1,73 @@ +package mdleaves + +import ( + "strings" + "testing" +) + +func TestSplitLeafsOnH2(t *testing.T) { + body := "# Title\n\n## One\n\nalpha\n\n## Two\n\nbeta\n" + leafs := SplitLeafs(map[string]string{}, body) + if len(leafs) != 2 { + t.Fatalf("n=%d", len(leafs)) + } + if leafs[0].Heading != "One" || !strings.Contains(leafs[0].Text, "Title") { + t.Fatalf("first=%+v", leafs[0]) + } + if leafs[1].Heading != "Two" || strings.Contains(leafs[1].Text, "Title") { + t.Fatalf("second=%+v", leafs[1]) + } +} + +func TestSplitLeafsNoH2IsWholeDoc(t *testing.T) { + body := "# Title\n\njust a paragraph\n" + leafs := SplitLeafs(nil, body) + if len(leafs) != 1 { + t.Fatalf("n=%d", len(leafs)) + } + if leafs[0].Heading != "Title" { + t.Fatalf("heading=%q", leafs[0].Heading) + } + if !strings.Contains(leafs[0].Text, "just a paragraph") { + t.Fatalf("text=%q", leafs[0].Text) + } +} + +func TestFrontmatterAndToAll(t *testing.T) { + raw := "---\ntype: howto\nstatus: current\nrelated: docs/design.md\n---\n# Doc\n\n## Step\n\ndo it\n" + got := ToAll(raw, "docs/x.md", "eSlider/2dph") + if len(got) != 1 { + t.Fatalf("n=%d", len(got)) + } + if got[0].Type != "howto" || got[0].Status != "current" { + t.Fatalf("%+v", got[0]) + } + if got[0].Source != "docs/x.md" || got[0].Repo != "eSlider/2dph" { + t.Fatalf("%+v", got[0]) + } + if got[0].Related != "docs/design.md" { + t.Fatalf("related=%q", got[0].Related) + } +} + +func TestEncodeJSONAndYAML(t *testing.T) { + leafs := ToAll("# Hi\n\nbody\n", "a.md", "") + js, err := EncodeJSON(leafs) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(js, `"heading": "Hi"`) { + t.Fatalf("json=%s", js) + } + y := EncodeYAML(leafs) + if !strings.Contains(y, "heading: Hi") { + t.Fatalf("yaml=%s", y) + } +} + +func TestDefaultsTypeAndStatus(t *testing.T) { + got := ToAll("# Hi\n\nbody\n", "a.md", "") + if got[0].Type != "reference" || got[0].Status != "current" { + t.Fatalf("%+v", got[0]) + } +} diff --git a/internal/mdleaves/yaml.go b/internal/mdleaves/yaml.go new file mode 100644 index 0000000..29c2ee1 --- /dev/null +++ b/internal/mdleaves/yaml.go @@ -0,0 +1,36 @@ +package mdleaves + +import ( + "fmt" + "strconv" + "strings" +) + +func EncodeYAML(leafs []Leaf) string { + if len(leafs) == 0 { + return "[]\n" + } + var b strings.Builder + for _, lf := range leafs { + b.WriteString("-\n") + writeKV(&b, "source", lf.Source) + writeKV(&b, "repo", lf.Repo) + writeKV(&b, "heading", lf.Heading) + writeKV(&b, "text", lf.Text) + writeKV(&b, "type", lf.Type) + writeKV(&b, "status", lf.Status) + writeKV(&b, "related", lf.Related) + } + return b.String() +} + +func writeKV(b *strings.Builder, k, v string) { + fmt.Fprintf(b, " %s: %s\n", k, yamlScalar(v)) +} + +func yamlScalar(s string) string { + if strings.Contains(s, "\n") || s == "" || strings.ContainsAny(s, ":#'\"[]{}&*!|>%@`") || s != strings.TrimSpace(s) { + return strconv.Quote(s) + } + return s +}