feat: read git history with go-git, not the git binary (#15)
This commit is contained in:
GitHub
+11
-139
@@ -1,154 +1,26 @@
|
||||
#!/usr/bin/env python3
|
||||
"""git/import - import git history (commits, authors, files) into the brain.
|
||||
"""git/import — deprecated. Use bin/git/import.go (go-git, no git binary).
|
||||
|
||||
bin/git/import [REPO] import all commits -> leafs + graph
|
||||
bin/git/import --json emit import leafs as JSON, no write
|
||||
bin/git/import --limit 100 cap commits processed
|
||||
bin/git/import --since 2026-01-01 only recent commits
|
||||
bin/git/import --root DIR run per repo dir under DIR
|
||||
bin/git/import --no-env never read .env anywhere (default: true)
|
||||
|
||||
Reads `git log --no-merges --name-only` from the repo, maps commits to
|
||||
`info` leafs (root=info, type=commit) and writes the version graph
|
||||
`File -[:HAS_VERSION]-> Commit -[:AUTHORED]-> Person` into var/kb.lbug.
|
||||
Idempotent: leaf MERGE by (source,text via leaf_id), graph MERGE by sha.
|
||||
bin/git/import.go [REPO] [--json] [--limit N] [--since DATE]
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[2]
|
||||
sys.path.insert(0, str(ROOT / "bin" / "tools"))
|
||||
|
||||
from kblib import ( # noqa: E402
|
||||
connect, ensure_indexes, init_schema, upsert_leaf,
|
||||
)
|
||||
from gitimport import commits_to_leafs, ensure_git_schema, index_commits, parse_log # noqa: E402
|
||||
|
||||
LOG_FMT = "--format=%x1e%H%x1f%an%x1f%ae%x1f%aI%x1f%s"
|
||||
|
||||
|
||||
def git_log(repo: Path, limit: int = 0, since: str = "") -> str:
|
||||
cmd = ["git", "-C", str(repo), "log", "--no-merges", "--name-only", LOG_FMT]
|
||||
if since:
|
||||
cmd += ["--since", since]
|
||||
if limit:
|
||||
cmd += ["-n", str(limit)]
|
||||
try:
|
||||
out = subprocess.run(cmd, capture_output=True, text=True, timeout=120)
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return ""
|
||||
if out.returncode != 0:
|
||||
print(f"git/import: {repo}: {out.stderr.strip()}", file=sys.stderr)
|
||||
return ""
|
||||
return out.stdout
|
||||
|
||||
|
||||
def repo_name(repo: Path) -> str:
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["git", "-C", str(repo), "remote", "get-url", "origin"],
|
||||
capture_output=True, text=True, timeout=20)
|
||||
url = out.stdout.strip()
|
||||
return url.rstrip("/").split("/")[-1].removesuffix(".git") if url else repo.name
|
||||
except (FileNotFoundError, subprocess.TimeoutExpired):
|
||||
return repo.name
|
||||
|
||||
|
||||
def embedder():
|
||||
from model2vec import StaticModel
|
||||
model = StaticModel.from_pretrained("minishlab/potion-multilingual-128M")
|
||||
return lambda text: model.encode([text])[0].astype(float).tolist()
|
||||
|
||||
|
||||
def import_repo(conn, repo: Path, embed, limit: int, since: str,
|
||||
no_write: bool = False) -> tuple[int, int]:
|
||||
raw = git_log(repo, limit, since)
|
||||
commits = parse_log(raw)
|
||||
leafs = commits_to_leafs(commits, repo_name(repo))
|
||||
if no_write:
|
||||
return len(commits), 0
|
||||
written = 0
|
||||
for lf in leafs:
|
||||
query = f"{lf['heading']}\n\n{lf['text']}"
|
||||
emb = embed(lf["text"]) if lf["text"] else None
|
||||
upsert_leaf(conn, text=query, root="info", confidence="confirmed",
|
||||
source=lf["source"], source_rev="git", how="git/import",
|
||||
loc=lf["source"], type_=lf.get("type", "commit"),
|
||||
embedding=emb)
|
||||
written += 1
|
||||
index_commits(conn, commits, repo_name(repo))
|
||||
return len(commits), written
|
||||
|
||||
|
||||
def main(argv: list[str]) -> int:
|
||||
import argparse
|
||||
p = argparse.ArgumentParser(description="import git history into the brain")
|
||||
p.add_argument("repo", nargs="?", default=None)
|
||||
p.add_argument("--root", default=None, help="directory of repos to import (each git dir separately)")
|
||||
p.add_argument("--limit", type=int, default=0)
|
||||
p.add_argument("--since", default="")
|
||||
p.add_argument("--json", action="store_true")
|
||||
p.add_argument("--dry-run", action="store_true", help="parse + report, no db write")
|
||||
a = p.parse_args(argv)
|
||||
|
||||
repos: list[Path] = []
|
||||
if a.repo:
|
||||
repos = [Path(a.repo)]
|
||||
elif a.root:
|
||||
root = Path(a.root)
|
||||
if root.is_file():
|
||||
repos = [root]
|
||||
else:
|
||||
repos = [dp for dp in sorted(root.iterdir()) if (dp / ".git").exists() or dp.is_file()]
|
||||
else:
|
||||
repos = [ROOT]
|
||||
|
||||
total_commits = 0
|
||||
results: list[dict] = []
|
||||
if a.dry_run:
|
||||
for repo in repos:
|
||||
if not repo.exists():
|
||||
continue
|
||||
commits = parse_log(git_log(repo, a.limit, a.since))
|
||||
name = repo_name(repo)
|
||||
total_commits += len(commits)
|
||||
results.append({"repo": name, "commits": len(commits),
|
||||
"leafs": len(commits_to_leafs(commits, name)), "path": str(repo)})
|
||||
if a.json:
|
||||
print(json.dumps(results, indent=2))
|
||||
else:
|
||||
for r in results:
|
||||
print(f"{r['repo']:<24} {r['commits']:>5} commits -> {r['leafs']} leafs {r['path']}")
|
||||
return 0
|
||||
|
||||
# Never DROP FTS/VECTOR (ghost catalog). Upsert while indexes exist is OK;
|
||||
# ensure_indexes only CREATEs when missing.
|
||||
db, conn = connect(ROOT / "var" / "kb.lbug", read_only=False)
|
||||
init_schema(conn)
|
||||
embed = embedder()
|
||||
rows: list[dict] = []
|
||||
for repo in repos:
|
||||
if not repo.exists():
|
||||
continue
|
||||
reached, written = import_repo(conn, repo, embed, a.limit, a.since)
|
||||
total_commits += reached
|
||||
rows.append({"repo": repo_name(repo), "commits": reached, "written": written})
|
||||
ensure_indexes(conn)
|
||||
conn.close()
|
||||
db.close()
|
||||
|
||||
if a.json:
|
||||
print(json.dumps(rows, indent=2))
|
||||
else:
|
||||
for r in rows:
|
||||
print(f"imported {r['commits']:>5} commits -> {r['written']} leafs {r['repo']}")
|
||||
print(f"total: {total_commits} commits")
|
||||
return 0
|
||||
print(
|
||||
"bin/git/import is deprecated; use bin/git/import.go (go-git)",
|
||||
file=sys.stderr,
|
||||
)
|
||||
target = ROOT / "bin" / "git" / "import.go"
|
||||
os.execvp("go", ["go", "run", str(target), *argv])
|
||||
return 1
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
sys.exit(main(sys.argv[1:]))
|
||||
|
||||
Executable
+143
@@ -0,0 +1,143 @@
|
||||
//usr/bin/env go run "$0" "$@"; exit
|
||||
//
|
||||
// bin/git/import.go - read git history with go-git (no git binary).
|
||||
//
|
||||
// ./bin/git/import.go [REPO]
|
||||
// ./bin/git/import.go --json
|
||||
// ./bin/git/import.go --limit 100 --since 2026-01-01
|
||||
// ./bin/git/import.go --root DIR
|
||||
//
|
||||
// Conversion only: prints commit leafs. Brain write is bin/brain/index.go.
|
||||
// NOTE: never run `gofmt -w` on this file — it breaks the shebang.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"time"
|
||||
|
||||
"github.com/eSlider/2dph/internal/cmdbin"
|
||||
"github.com/eSlider/2dph/internal/gitlog"
|
||||
)
|
||||
|
||||
func main() {
|
||||
os.Exit(run(os.Args[1:]))
|
||||
}
|
||||
|
||||
func run(args []string) int {
|
||||
var repo, root, since string
|
||||
limit := 0
|
||||
jsonOut := false
|
||||
i := 0
|
||||
for i < len(args) {
|
||||
a := args[i]
|
||||
switch {
|
||||
case a == "--json":
|
||||
jsonOut = true
|
||||
case a == "--limit" && i+1 < len(args):
|
||||
i++
|
||||
n, err := strconv.Atoi(args[i])
|
||||
if err != nil || n < 0 {
|
||||
fmt.Fprintf(os.Stderr, "git/import: --limit must be a non-negative integer\n")
|
||||
return 2
|
||||
}
|
||||
limit = n
|
||||
case a == "--since" && i+1 < len(args):
|
||||
i++
|
||||
since = args[i]
|
||||
case a == "--root" && i+1 < len(args):
|
||||
i++
|
||||
root = args[i]
|
||||
case a == "-h" || a == "--help":
|
||||
fmt.Fprintln(os.Stderr, `usage: bin/git/import.go [REPO] [--json] [--limit N] [--since DATE] [--root DIR]`)
|
||||
return 0
|
||||
case len(a) > 0 && a[0] != '-':
|
||||
repo = a
|
||||
default:
|
||||
fmt.Fprintf(os.Stderr, "git/import: unknown flag %s\n", a)
|
||||
return 2
|
||||
}
|
||||
i++
|
||||
}
|
||||
|
||||
var sinceT time.Time
|
||||
if since != "" {
|
||||
var err error
|
||||
sinceT, err = parseSince(since)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "git/import: %v\n", err)
|
||||
return 2
|
||||
}
|
||||
}
|
||||
|
||||
repos := []string{}
|
||||
if repo != "" {
|
||||
repos = []string{repo}
|
||||
} else if root != "" {
|
||||
entries, err := os.ReadDir(root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "git/import: %v\n", err)
|
||||
return 1
|
||||
}
|
||||
for _, e := range entries {
|
||||
p := filepath.Join(root, e.Name())
|
||||
if _, err := os.Stat(filepath.Join(p, ".git")); err == nil {
|
||||
repos = append(repos, p)
|
||||
}
|
||||
}
|
||||
} else {
|
||||
repos = []string{cmdbin.Root()}
|
||||
}
|
||||
|
||||
opt := gitlog.Options{Limit: limit, Since: sinceT}
|
||||
type row struct {
|
||||
Repo string `json:"repo"`
|
||||
Path string `json:"path"`
|
||||
Commits int `json:"commits"`
|
||||
Leafs []gitlog.Leaf `json:"leafs,omitempty"`
|
||||
}
|
||||
var rows []row
|
||||
for _, p := range repos {
|
||||
name, err := gitlog.RepoName(p)
|
||||
if err != nil && name == "" {
|
||||
fmt.Fprintf(os.Stderr, "git/import: %s: %v\n", p, err)
|
||||
continue
|
||||
}
|
||||
cs, err := gitlog.Log(p, opt)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "git/import: %s: %v\n", p, err)
|
||||
return 1
|
||||
}
|
||||
leafs := make([]gitlog.Leaf, 0, len(cs))
|
||||
for _, c := range cs {
|
||||
leafs = append(leafs, gitlog.ToLeaf(c, name))
|
||||
}
|
||||
rows = append(rows, row{Repo: name, Path: p, Commits: len(cs), Leafs: leafs})
|
||||
}
|
||||
|
||||
if jsonOut {
|
||||
enc := json.NewEncoder(os.Stdout)
|
||||
enc.SetIndent("", " ")
|
||||
enc.SetEscapeHTML(false)
|
||||
if err := enc.Encode(rows); err != nil {
|
||||
return 1
|
||||
}
|
||||
return 0
|
||||
}
|
||||
for _, r := range rows {
|
||||
fmt.Printf("%-24s %5d commits %s\n", r.Repo, r.Commits, r.Path)
|
||||
}
|
||||
return 0
|
||||
}
|
||||
|
||||
func parseSince(s string) (time.Time, error) {
|
||||
for _, layout := range []string{time.RFC3339, "2006-01-02"} {
|
||||
if t, err := time.Parse(layout, s); err == nil {
|
||||
return t, nil
|
||||
}
|
||||
}
|
||||
return time.Time{}, fmt.Errorf("cannot parse --since %q", s)
|
||||
}
|
||||
Reference in New Issue
Block a user