feat: read git history with go-git, not the git binary

bin/git/import

@@ -1,154 +1,26 @@
 #!/usr/bin/env python3
-"""git/import - import git history (commits, authors, files) into the brain.
+"""git/import — deprecated. Use bin/git/import.go (go-git, no git binary).
 
-  bin/git/import [REPO]                 import all commits -> leafs + graph
-  bin/git/import --json                 emit import leafs as JSON, no write
-  bin/git/import --limit 100            cap commits processed
-  bin/git/import --since 2026-01-01     only recent commits
-  bin/git/import --root DIR             run per repo dir under DIR
-  bin/git/import --no-env               never read .env anywhere (default: true)
-
-Reads `git log --no-merges --name-only` from the repo, maps commits to
-`info` leafs (root=info, type=commit) and writes the version graph
-`File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person` into var/kb.lbug.
-Idempotent: leaf MERGE by (source,text via leaf_id), graph MERGE by sha.
+  bin/git/import.go [REPO] [--json] [--limit N] [--since DATE]
 """
 from __future__ import annotations
 
-import json
-import subprocess
+import os
 import sys
 from pathlib import Path
 
 ROOT = Path(__file__).resolve().parents[2]
-sys.path.insert(0, str(ROOT / "bin" / "tools"))
-
-from kblib import (  # noqa: E402
-    connect, ensure_indexes, init_schema, upsert_leaf,
-)
-from gitimport import commits_to_leafs, ensure_git_schema, index_commits, parse_log  # noqa: E402
-
-LOG_FMT = "--format=%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s"
-
-
-def git_log(repo: Path, limit: int = 0, since: str = "") -> str:
-    cmd = ["git", "-C", str(repo), "log", "--no-merges", "--name-only", LOG_FMT]
-    if since:
-        cmd += ["--since", since]
-    if limit:
-        cmd += ["-n", str(limit)]
-    try:
-        out = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
-    except (FileNotFoundError, subprocess.TimeoutExpired):
-        return ""
-    if out.returncode != 0:
-        print(f"git/import: {repo}: {out.stderr.strip()}", file=sys.stderr)
-        return ""
-    return out.stdout
-
-
-def repo_name(repo: Path) -> str:
-    try:
-        out = subprocess.run(
-            ["git", "-C", str(repo), "remote", "get-url", "origin"],
-            capture_output=True, text=True, timeout=20)
-        url = out.stdout.strip()
-        return url.rstrip("/").split("/")[-1].removesuffix(".git") if url else repo.name
-    except (FileNotFoundError, subprocess.TimeoutExpired):
-        return repo.name
-
-
-def embedder():
-    from model2vec import StaticModel
-    model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
-    return lambda text: model.encode([text])[0].astype(float).tolist()
-
-
-def import_repo(conn, repo: Path, embed, limit: int, since: str,
-                no_write: bool = False) -> tuple[int, int]:
-    raw = git_log(repo, limit, since)
-    commits = parse_log(raw)
-    leafs = commits_to_leafs(commits, repo_name(repo))
-    if no_write:
-        return len(commits), 0
-    written = 0
-    for lf in leafs:
-        query = f"{lf['heading']}\n\n{lf['text']}"
-        emb = embed(lf["text"]) if lf["text"] else None
-        upsert_leaf(conn, text=query, root="info", confidence="confirmed",
-                    source=lf["source"], source_rev="git", how="git/import",
-                    loc=lf["source"], type_=lf.get("type", "commit"),
-                    embedding=emb)
-        written += 1
-    index_commits(conn, commits, repo_name(repo))
-    return len(commits), written
 
 
 def main(argv: list[str]) -> int:
-    import argparse
-    p = argparse.ArgumentParser(description="import git history into the brain")
-    p.add_argument("repo", nargs="?", default=None)
-    p.add_argument("--root", default=None, help="directory of repos to import (each git dir separately)")
-    p.add_argument("--limit", type=int, default=0)
-    p.add_argument("--since", default="")
-    p.add_argument("--json", action="store_true")
-    p.add_argument("--dry-run", action="store_true", help="parse + report, no db write")
-    a = p.parse_args(argv)
-
-    repos: list[Path] = []
-    if a.repo:
-        repos = [Path(a.repo)]
-    elif a.root:
-        root = Path(a.root)
-        if root.is_file():
-            repos = [root]
-        else:
-            repos = [dp for dp in sorted(root.iterdir()) if (dp / ".git").exists() or dp.is_file()]
-    else:
-        repos = [ROOT]
-
-    total_commits = 0
-    results: list[dict] = []
-    if a.dry_run:
-        for repo in repos:
-            if not repo.exists():
-                continue
-            commits = parse_log(git_log(repo, a.limit, a.since))
-            name = repo_name(repo)
-            total_commits += len(commits)
-            results.append({"repo": name, "commits": len(commits),
-                            "leafs": len(commits_to_leafs(commits, name)), "path": str(repo)})
-        if a.json:
-            print(json.dumps(results, indent=2))
-        else:
-            for r in results:
-                print(f"{r['repo']:<24} {r['commits']:>5} commits -> {r['leafs']} leafs  {r['path']}")
-        return 0
-
-    # Never DROP FTS/VECTOR (ghost catalog). Upsert while indexes exist is OK;
-    # ensure_indexes only CREATEs when missing.
-    db, conn = connect(ROOT / "var" / "kb.lbug", read_only=False)
-    init_schema(conn)
-    embed = embedder()
-    rows: list[dict] = []
-    for repo in repos:
-        if not repo.exists():
-            continue
-        reached, written = import_repo(conn, repo, embed, a.limit, a.since)
-        total_commits += reached
-        rows.append({"repo": repo_name(repo), "commits": reached, "written": written})
-    ensure_indexes(conn)
-    conn.close()
-    db.close()
-
-    if a.json:
-        print(json.dumps(rows, indent=2))
-    else:
-        for r in rows:
-            print(f"imported {r['commits']:>5} commits -> {r['written']} leafs  {r['repo']}")
-        print(f"total: {total_commits} commits")
-    return 0
+    print(
+        "bin/git/import is deprecated; use bin/git/import.go (go-git)",
+        file=sys.stderr,
+    )
+    target = ROOT / "bin" / "git" / "import.go"
+    os.execvp("go", ["go", "run", str(target), *argv])
+    return 1
 
 
 if __name__ == "__main__":
-    sys.exit(main(sys.argv[1:]))
+    sys.exit(main(sys.argv[1:]))

bin/git/import.go

@@ -0,0 +1,143 @@
+//usr/bin/env go run "$0" "$@"; exit
+//
+// bin/git/import.go - read git history with go-git (no git binary).
+//
+//	./bin/git/import.go [REPO]
+//	./bin/git/import.go --json
+//	./bin/git/import.go --limit 100 --since 2026-01-01
+//	./bin/git/import.go --root DIR
+//
+// Conversion only: prints commit leafs. Brain write is bin/brain/index.go.
+// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/eSlider/2dph/internal/cmdbin"
+	"github.com/eSlider/2dph/internal/gitlog"
+)
+
+func main() {
+	os.Exit(run(os.Args[1:]))
+}
+
+func run(args []string) int {
+	var repo, root, since string
+	limit := 0
+	jsonOut := false
+	i := 0
+	for i < len(args) {
+		a := args[i]
+		switch {
+		case a == "--json":
+			jsonOut = true
+		case a == "--limit" && i+1 < len(args):
+			i++
+			n, err := strconv.Atoi(args[i])
+			if err != nil || n < 0 {
+				fmt.Fprintf(os.Stderr, "git/import: --limit must be a non-negative integer\n")
+				return 2
+			}
+			limit = n
+		case a == "--since" && i+1 < len(args):
+			i++
+			since = args[i]
+		case a == "--root" && i+1 < len(args):
+			i++
+			root = args[i]
+		case a == "-h" || a == "--help":
+			fmt.Fprintln(os.Stderr, `usage: bin/git/import.go [REPO] [--json] [--limit N] [--since DATE] [--root DIR]`)
+			return 0
+		case len(a) > 0 && a[0] != '-':
+			repo = a
+		default:
+			fmt.Fprintf(os.Stderr, "git/import: unknown flag %s\n", a)
+			return 2
+		}
+		i++
+	}
+
+	var sinceT time.Time
+	if since != "" {
+		var err error
+		sinceT, err = parseSince(since)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "git/import: %v\n", err)
+			return 2
+		}
+	}
+
+	repos := []string{}
+	if repo != "" {
+		repos = []string{repo}
+	} else if root != "" {
+		entries, err := os.ReadDir(root)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "git/import: %v\n", err)
+			return 1
+		}
+		for _, e := range entries {
+			p := filepath.Join(root, e.Name())
+			if _, err := os.Stat(filepath.Join(p, ".git")); err == nil {
+				repos = append(repos, p)
+			}
+		}
+	} else {
+		repos = []string{cmdbin.Root()}
+	}
+
+	opt := gitlog.Options{Limit: limit, Since: sinceT}
+	type row struct {
+		Repo    string         `json:"repo"`
+		Path    string         `json:"path"`
+		Commits int            `json:"commits"`
+		Leafs   []gitlog.Leaf  `json:"leafs,omitempty"`
+	}
+	var rows []row
+	for _, p := range repos {
+		name, err := gitlog.RepoName(p)
+		if err != nil && name == "" {
+			fmt.Fprintf(os.Stderr, "git/import: %s: %v\n", p, err)
+			continue
+		}
+		cs, err := gitlog.Log(p, opt)
+		if err != nil {
+			fmt.Fprintf(os.Stderr, "git/import: %s: %v\n", p, err)
+			return 1
+		}
+		leafs := make([]gitlog.Leaf, 0, len(cs))
+		for _, c := range cs {
+			leafs = append(leafs, gitlog.ToLeaf(c, name))
+		}
+		rows = append(rows, row{Repo: name, Path: p, Commits: len(cs), Leafs: leafs})
+	}
+
+	if jsonOut {
+		enc := json.NewEncoder(os.Stdout)
+		enc.SetIndent("", "  ")
+		enc.SetEscapeHTML(false)
+		if err := enc.Encode(rows); err != nil {
+			return 1
+		}
+		return 0
+	}
+	for _, r := range rows {
+		fmt.Printf("%-24s %5d commits  %s\n", r.Repo, r.Commits, r.Path)
+	}
+	return 0
+}
+
+func parseSince(s string) (time.Time, error) {
+	for _, layout := range []string{time.RFC3339, "2006-01-02"} {
+		if t, err := time.Parse(layout, s); err == nil {
+			return t, nil
+		}
+	}
+	return time.Time{}, fmt.Errorf("cannot parse --since %q", s)
+}

bin/tools/gitimport.py

@@ -1,21 +1,12 @@
-"""gitimport - parse `git log` output and turn commits into brain leafs.
+"""gitimport - Ladybug graph writes for Commit/File/Person (no git binary).
 
-Pure, testable functions. Field grammar (see bin/git/import):
-
-  git log --no-merges --name-only \
-      --format='%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s'
-
-  0x1e = record separator, 0x1f = field separator.
-  Files: newline-separated lines following each record's subject.
+Commit records come from bin/git/import.go (go-git). This module only MERGEs
+the version graph File-[:HAS_VERSION]->Commit-[:AUTHORED]->Person.
 """
-
 from __future__ import annotations
 
 from dataclasses import dataclass, field
 
-REC_SEP = "\x1e"
-FIELD_SEP = "\x1f"
-
 
 @dataclass
 class Commit:
@@ -26,54 +17,6 @@ class Commit:
     subject: str
     files: list[str] = field(default_factory=list)
 
-    def leaf_text(self, repo: str) -> str:
-        head = f"commit {self.sha[:12]} in {repo} — {self.subject}"
-        body = [head, f"Author: {self.author} <{self.email}>", f"Date: {self.date}"]
-        if self.files:
-            body.append("Changing: " + ", ".join(self.files))
-        return "\n".join(body)
-
-
-def parse_log(text: str) -> list[Commit]:
-    """Parse `git log` output into Commit records.
-
-    Records are separated by 0x1e. A record is fields joined by 0x1f,
-    followed by optional newline-separated file paths inside the next
-    segment (git emits blank line + files after each record).
-    """
-    commits: list[Commit] = []
-    # field records and file lists alternate; simpler: split on REC_SEP,
-    # each chunk = header line, possibly followed by newline + files.
-    for chunk in text.split(REC_SEP):
-        chunk = chunk.strip("\n")
-        if not chunk:
-            continue
-        lines = chunk.split("\n", 1)
-        header = lines[0].split(FIELD_SEP)
-        if len(header) < 5:
-            continue
-        sha, author, email, date, subject = header[:5]
-        files = [ln.strip() for ln in lines[1].splitlines() if ln.strip()] if len(lines) > 1 else []
-        commits.append(Commit(sha=sha, author=author, email=email,
-                              date=date, subject=subject, files=files))
-    return commits
-
-
-def commits_to_leafs(commits: list[Commit], repo: str) -> list[dict]:
-    """Map commits to the leaf shape bin/kb/index expects (source/repo/...)."""
-    out: list[dict] = []
-    for c in commits:
-        out.append({
-            "source": f"{repo}@{c.sha}",
-            "repo": repo,
-            "heading": f"commit {c.sha[:12]} — {c.subject}",
-            "text": c.leaf_text(repo),
-            "type": "commit",
-            "status": "current",
-            "related": ",".join(c.files),
-        })
-    return out
-
 
 GIT_SCHEMA = (
     "CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
@@ -114,4 +57,4 @@ def index_commits(conn, commits: list[Commit], repo: str) -> int:
             conn.execute("MATCH (f:File {id:$fid}), (c:Commit {id:$sha}) "
                          "MERGE (f)-[:HAS_VERSION]->(c)",
                          parameters={"fid": f"{repo}:{path}", "sha": c.sha})
-    return len(commits)
+    return len(commits)

bin/tools/test_bin_layout.py

@@ -91,3 +91,26 @@ class BinLayoutTest(unittest.TestCase):
 
     def test_postgres_query_is_shebang(self) -> None:
         self._assert_shebang("bin/postgres/query.go")
+
+    def test_git_import_is_gogit_shebang(self) -> None:
+        self._assert_shebang("bin/git/import.go")
+        py = (ROOT / "bin" / "git" / "import").read_text()
+        self.assertNotIn(
+            '["git"',
+            py,
+            "Python git/import must not subprocess the git binary",
+        )
+        self.assertIn("bin/git/import.go", py)
+
+    def test_gitimport_py_has_no_git_binary(self) -> None:
+        py = (ROOT / "bin" / "tools" / "gitimport.py").read_text()
+        self.assertNotIn("subprocess", py)
+        self.assertNotIn("git log", py)
+
+    def test_gogit_is_direct_go_mod_require(self) -> None:
+        text = (ROOT / "go.mod").read_text()
+        first = text.split("require (")[1].split(")")[0]
+        self.assertRegex(first, r"github.com/go-git/go-git/v5\s+v")
+        for line in first.splitlines():
+            if "go-git/go-git" in line:
+                self.assertNotIn("indirect", line)

bin/tools/test_gitgraph.py

@@ -9,12 +9,6 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
 import kblib  # noqa: E402
 import gitimport  # noqa: E402
 
-SAMPLE = (
-    "\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com"
-    + "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit"
-    + "\n\nREADME.md\nsrc/main.c\n"
-)
-
 COMMIT_PERSON_SCHEMA = (
     "CREATE NODE TABLE IF NOT EXISTS Commit (id STRING, repo STRING, subject STRING, "
     "author STRING, email STRING, date STRING, PRIMARY KEY(id))"
@@ -26,6 +20,17 @@ HAS_VERSION_SCHEMA = "CREATE REL TABLE IF NOT EXISTS HAS_VERSION (FROM File TO C
 AUTHORED_SCHEMA = "CREATE REL TABLE IF NOT EXISTS AUTHORED (FROM Commit TO Person)"
 
 
+def sample_commit() -> gitimport.Commit:
+    return gitimport.Commit(
+        sha="a1b2c3d",
+        author="Ada Lovelace",
+        email="ada@example.com",
+        date="2026-08-10T12:00:00+01:00",
+        subject="feat: first commit",
+        files=["README.md", "src/main.c"],
+    )
+
+
 class GitGraphTest(unittest.TestCase):
     def setUp(self):
         self.dir = tempfile.mkdtemp()
@@ -42,14 +47,12 @@ class GitGraphTest(unittest.TestCase):
         self.db.close()
 
     def test_index_commits_creates_nodes_and_edges(self):
-        cs = gitimport.parse_log(SAMPLE)
-        gitimport.index_commits(self.conn, cs, "sample-repo")
+        gitimport.index_commits(self.conn, [sample_commit()], "sample-repo")
         rp = self.conn.execute("MATCH (p:Person) RETURN p.name, p.email").get_all()
         self.assertEqual([tuple(r) for r in rp], [("Ada Lovelace", "ada@example.com")])
         rc = self.conn.execute("MATCH (c:Commit) RETURN c.id, c.repo").get_all()
         self.assertEqual(len(rc), 1)
         self.assertEqual(rc[0][1], "sample-repo")
-        # File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person
         rf = self.conn.execute(
             "MATCH (f:File)-[:HAS_VERSION]->(c:Commit)-[:AUTHORED]->(p:Person) "
             "RETURN f.path, c.id, p.email").get_all()
@@ -58,7 +61,7 @@ class GitGraphTest(unittest.TestCase):
         self.assertTrue(all(r[2] == "ada@example.com" for r in rf))
 
     def test_index_commits_idempotent(self):
-        cs = gitimport.parse_log(SAMPLE)
+        cs = [sample_commit()]
         gitimport.index_commits(self.conn, cs, "sample-repo")
         gitimport.index_commits(self.conn, cs, "sample-repo")
         n = self.conn.execute("MATCH (c:Commit) RETURN count(*)").get_all()[0][0]
@@ -68,4 +71,4 @@ class GitGraphTest(unittest.TestCase):
 
 
 if __name__ == "__main__":
-    unittest.main()
+    unittest.main()

bin/tools/test_gitimport.py

@@ -1,57 +0,0 @@
-import sys
-import unittest
-from pathlib import Path
-
-sys.path.insert(0, str(Path(__file__).resolve().parent))
-
-import gitimport  # noqa: E402
-
-SAMPLE = (
-    "\x1e" + "a1b2c3d" + "\x1f" + "Ada Lovelace" + "\x1f" + "ada@example.com"
-    + "\x1f" + "2026-08-10T12:00:00+01:00" + "\x1f" + "feat: first commit"
-    + "\n\nREADME.md\nsrc/main.c\n"
-    + "\x1e" + "e4f5a6b" + "\x1f" + "Bob Babbage" + "\x1f" + "bob@example.com"
-    + "\x1f" + "2026-08-11T09:30:00+01:00" + "\x1f" + "fix: typo"
-    + "\n\ndocs/notes.md"
-)
-
-
-class GitparseTest(unittest.TestCase):
-    def test_parses_records(self):
-        cs = gitimport.parse_log(SAMPLE)
-        self.assertEqual(len(cs), 2)
-
-    def test_parses_commit_fields(self):
-        cs = gitimport.parse_log(SAMPLE)
-        c = cs[0]
-        self.assertEqual(c.sha, "a1b2c3d")
-        self.assertEqual(c.author, "Ada Lovelace")
-        self.assertEqual(c.email, "ada@example.com")
-        self.assertEqual(c.date, "2026-08-10T12:00:00+01:00")
-        self.assertEqual(c.subject, "feat: first commit")
-
-    def test_parses_changed_files(self):
-        cs = gitimport.parse_log(SAMPLE)
-        self.assertEqual(cs[0].files, ["README.md", "src/main.c"])
-        self.assertEqual(cs[1].files, ["docs/notes.md"])
-
-    def test_ignores_empty(self):
-        self.assertEqual(gitimport.parse_log(""), [])
-
-    def test_skip_malformed_record(self):
-        self.assertEqual(gitimport.parse_log("\x1eweird\x1e"), [])
-
-    def test_commit_leaf_shape(self):
-        leafs = gitimport.commits_to_leafs(gitimport.parse_log(SAMPLE), "sample-repo")
-        self.assertEqual(len(leafs), 2)
-        lf = leafs[0]
-        self.assertEqual(lf["type"], "commit")
-        self.assertEqual(lf["repo"], "sample-repo")
-        self.assertEqual(lf["source"], "sample-repo@a1b2c3d")
-        self.assertIn("Ada Lovelace", lf["text"])
-        self.assertIn("README.md", lf["related"])
-        self.assertIn("feat: first commit", lf["heading"])
-
-
-if __name__ == "__main__":
-    unittest.main()

bin/tools/test_published_docs.py

@@ -39,6 +39,12 @@ class PublishedDocsTest(unittest.TestCase):
             "mail index is a brain write; README must name bin/brain/index.go",
         )
 
+    def test_readme_git_import_is_gogit(self) -> None:
+        text = (ROOT / "README.md").read_text()
+        self.assertIn("bin/git/import.go", text)
+        self.assertIn("go-git", text)
+        self.assertIn("D19", (ROOT / "PLAN.md").read_text())
+
     def test_docs_do_not_claim_hop_walks(self) -> None:
         paths = [
             ROOT / "README.md",