diff --git a/AGENTS.md b/AGENTS.md index 46c0aa4..d6f8f23 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -28,7 +28,7 @@ Canonical Go client for OnlyOffice Workspace (Projects + Calendar + CRM) and the - Prefer `ResponseObject` / `postFormObject` / `putFormObject` / `deleteObject` over hand-rolled `json.Unmarshal(responseField(...))` blocks — they exist for DRY, use them. - Domain split is by file, **not** by subpackage. Don't introduce `internal/` or `pkg/*` subpackages inside the library — it flattens the `*Client` call surface for a reason. - CLI commands follow **subject → verb** structure (`oo `), never `oo -`. Add new commands to the existing subject file if one fits; create a new `cmd/oo/.go` for a genuinely new domain. -- **Documents for agents:** prefer Markdown in git; OnlyOffice UI is weak for `.md`. Use `oo docs put-md` (md→docx upload) and `oo docs as-md` (download→OCR if needed→markdown). Local converters live in `internal/docpipe` (pandoc / ocrmypdf / pdftotext). +- **Documents for agents:** prefer Markdown in git; OnlyOffice UI is weak for `.md`. Use `oo docs put-md` (md→docx upload) and `oo docs as-md` (download→OCR if needed→markdown). Local converters live in `internal/docpipe` (pandoc / ocrmypdf / pdftotext / tesseract+go-hocr). For layout-aware extracts: `oo docs hocr` (or `as-md --hocr`). - Every table output goes through `printTable(headers, rows)`; every single-object through `printObject(v)`. Do not `fmt.Println` rows ad-hoc or the `--output json` flag breaks for that command. - No secrets in the repo; use `.env` (gitignored). Commit `.env.example` only. - Follow SemVer on tags; this repo is tagged at GitHub under `git@github.com:eSlider/go-onlyoffice.git`. diff --git a/README.md b/README.md index f3553bf..a288d56 100644 --- a/README.md +++ b/README.md @@ -617,8 +617,10 @@ oo docs tools oo docs convert ./note.md # → note.docx oo docs convert ./note.docx # → note.md oo docs ocr ./scan.jpg --md ./scan.md # searchable PDF + markdown +oo docs hocr ./scan.jpg --lang spa --md ./scan.hocr.md --yaml ./scan.yml oo docs put-md 7 ./OO-HONDA-7-INDEX.md --folder 490 oo docs as-md 2815 --to ./parte.md # download OO file as MD (OCR if needed) +oo docs as-md 307 --hocr --lang spa # OO download via go-hocr structure oo projects files put-md 7 ./note.md # alias oo projects files as-md 2815 # alias oo tasks files list 208 diff --git a/cmd/oo/docs.go b/cmd/oo/docs.go index 426cb7a..8612976 100644 --- a/cmd/oo/docs.go +++ b/cmd/oo/docs.go @@ -24,10 +24,12 @@ func docsCmd() *cobra.Command { OnlyOffice Documents UI is poor for .md — keep Markdown in git, store .docx in OO. Upload Markdown as DOCX: oo docs put-md PROJECT_ID file.md Read an OO file as MD: oo docs as-md FILE_ID -OCR a scan locally: oo docs ocr scan.pdf --md out.md`, +OCR a scan locally: oo docs ocr scan.pdf --md out.md +Structured OCR (hOCR→MD): oo docs hocr scan.jpg --md out.md --yaml out.yml`, } cmd.AddCommand(docsConvertCmd()) cmd.AddCommand(docsOCRCmd()) + cmd.AddCommand(docsHOCRCmd()) cmd.AddCommand(docsAsMDCmd()) cmd.AddCommand(docsPutMDCmd()) cmd.AddCommand(docsToolsCmd()) @@ -140,10 +142,91 @@ func docsOCRCmd() *cobra.Command { return cmd } +func docsHOCRCmd() *cobra.Command { + var mdOut, yamlOut, hocrOut, lang string + var dpi int + var minConf float32 + cmd := &cobra.Command{ + Use: "hocr PATH", + Short: "Tesseract hOCR → structured Markdown/YAML (via go-hocr)", + Long: `Runs tesseract with hOCR output, parses with go-hocr, writes Markdown +(and optional YAML). Better reading order / confidence than plain pdftotext. + +For Spanish scans: --lang spa or spa+eng (needs tesseract-ocr-spa / TESSDATA_PREFIX). +Phone photos: --dpi 200..300.`, + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + in := args[0] + t := docpipe.LookPath() + dir, err := os.MkdirTemp("", "oo-docs-hocr-*") + if err != nil { + return err + } + defer os.RemoveAll(dir) + + res, err := t.ToHOCRMarkdown(in, dir, lang, dpi, minConf) + if err != nil { + return err + } + if mdOut == "" { + base := strings.TrimSuffix(filepath.Base(in), filepath.Ext(in)) + mdOut = filepath.Join(filepath.Dir(in), base+".hocr.md") + } + if err := docpipe.EnsureDir(mdOut); err != nil { + return err + } + if err := os.WriteFile(mdOut, []byte(res.Markdown), 0o644); err != nil { + return err + } + obj := map[string]any{ + "in": in, + "md": mdOut, + "did_ocr": res.DidOCR, + } + if hocrOut != "" { + if err := docpipe.EnsureDir(hocrOut); err != nil { + return err + } + b, err := os.ReadFile(res.HOCRPath) + if err != nil { + return err + } + if err := os.WriteFile(hocrOut, b, 0o644); err != nil { + return err + } + obj["hocr"] = hocrOut + } else { + obj["hocr_tmp"] = res.HOCRPath + } + if yamlOut != "" { + if err := docpipe.EnsureDir(yamlOut); err != nil { + return err + } + if err := os.WriteFile(yamlOut, []byte(res.YAML), 0o644); err != nil { + return err + } + obj["yaml"] = yamlOut + } + printObject(obj) + return nil + }, + } + cmd.Flags().StringVar(&mdOut, "md", "", "Markdown output (default: .hocr.md)") + cmd.Flags().StringVar(&yamlOut, "yaml", "", "also write structured YAML from go-hocr") + cmd.Flags().StringVar(&hocrOut, "hocr", "", "also keep raw .hocr file at this path") + cmd.Flags().StringVar(&lang, "lang", "eng", "tesseract language(s), e.g. spa+eng") + cmd.Flags().IntVar(&dpi, "dpi", 220, "hint DPI for phone photos / scans (0 = tesseract default)") + cmd.Flags().Float32Var(&minConf, "min-conf", 0, "drop words with OCR confidence below this (0 = keep all)") + return cmd +} + func docsAsMDCmd() *cobra.Command { var to, lang string var minChars int var uploadOCR bool + var useHOCR bool + var dpi int + var minConf float32 cmd := &cobra.Command{ Use: "as-md FILE_ID", Short: "Download an OO Documents file and emit Markdown (OCR PDF/image if needed)", @@ -177,9 +260,18 @@ func docsAsMDCmd() *cobra.Command { _ = f.Close() tools := docpipe.LookPath() - res, err := tools.ToMarkdown(local, dir, lang, minChars) - if err != nil { - return err + var res docpipe.Result + if useHOCR { + hr, err := tools.ToHOCRMarkdown(local, dir, lang, dpi, minConf) + if err != nil { + return err + } + res = docpipe.Result{Markdown: hr.Markdown, DidOCR: hr.DidOCR, Source: hr.Source} + } else { + res, err = tools.ToMarkdown(local, dir, lang, minChars) + if err != nil { + return err + } } outPath := to if outPath == "" { @@ -242,6 +334,9 @@ func docsAsMDCmd() *cobra.Command { cmd.Flags().StringVar(&lang, "lang", "eng", "OCR language") cmd.Flags().IntVar(&minChars, "min-chars", docpipe.DefaultMinTextChars, "OCR PDF if text layer shorter than this") cmd.Flags().BoolVar(&uploadOCR, "upload-ocr", false, "upload searchable OCR PDF back into the same OO folder") + cmd.Flags().BoolVar(&useHOCR, "hocr", false, "use tesseract hOCR + go-hocr instead of ocrmypdf/pdftotext") + cmd.Flags().IntVar(&dpi, "dpi", 220, "DPI hint when --hocr (phone photos)") + cmd.Flags().Float32Var(&minConf, "min-conf", 0, "drop low-confidence words when --hocr") return cmd } diff --git a/go.mod b/go.mod index c4b1471..6bb29de 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/charmbracelet/lipgloss v0.12.1 github.com/charmbracelet/x/ansi v0.1.4 github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 + github.com/eslider/go-hocr v0.2.2-0.20260827163626-8ff01582b002 github.com/eslider/go-xls/v2 v2.1.0 github.com/google/go-querystring v1.2.0 github.com/joho/godotenv v1.5.1 diff --git a/go.sum b/go.sum index 16adb49..8f397bd 100644 --- a/go.sum +++ b/go.sum @@ -37,6 +37,8 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3 h1:B9YK+Tck5mTccyDhtxBzWyqGYcFxLyB6+noMNW4/VgI= github.com/emersion/go-vcard v0.0.0-20260618161152-d854b7e0e2d3/go.mod h1:HMJKR5wlh/ziNp+sHEDV2ltblO4JD2+IdDOWtGcQBTM= +github.com/eslider/go-hocr v0.2.2-0.20260827163626-8ff01582b002 h1:LOFxQG4mxvlH7+lffbO2SIn2ThlClJlrHHfs1OqMNXs= +github.com/eslider/go-hocr v0.2.2-0.20260827163626-8ff01582b002/go.mod h1:fIgfH/E1j3rU8du4X4+7mxTD0GPtPQibTzytgitdJWU= github.com/eslider/go-xls/v2 v2.1.0 h1:HszWKqYQbXxACmAXXWdMsfNl1NDBfGVBnJUPtyUHQ7A= github.com/eslider/go-xls/v2 v2.1.0/go.mod h1:xgxO6JrfuBr9jGUB+0z5l/yDmFFZ5diGk0ATGihxlMU= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= diff --git a/internal/docpipe/hocr.go b/internal/docpipe/hocr.go new file mode 100644 index 0000000..c64b2b6 --- /dev/null +++ b/internal/docpipe/hocr.go @@ -0,0 +1,153 @@ +package docpipe + +import ( + "bytes" + "fmt" + "os" + "os/exec" + "path/filepath" + "strings" + + hocr "github.com/eslider/go-hocr" +) + +// HOCRResult is structured OCR output for agents. +type HOCRResult struct { + HOCRPath string + Markdown string + YAML string + DidOCR bool + Source string +} + +// ImageToHOCR runs tesseract hOCR into outBase+".hocr" (tesseract adds the extension). +// outBase must not include ".hocr". dpi 0 uses tesseract default; phone photos often need 200–300. +func (t Tools) ImageToHOCR(imgPath, outBase, lang string, dpi int) (string, error) { + if t.Tesseract == "" { + return "", fmt.Errorf("tesseract not found on PATH") + } + if lang == "" { + lang = "eng" + } + if outBase == "" { + return "", fmt.Errorf("hOCR output base path required") + } + args := []string{imgPath, outBase, "-l", lang} + if dpi > 0 { + args = append(args, "--dpi", fmt.Sprintf("%d", dpi)) + } + args = append(args, resolveHOCRConfig()) + cmd := exec.Command(t.Tesseract, args...) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("tesseract hocr: %w (%s)", err, strings.TrimSpace(stderr.String())) + } + out := outBase + ".hocr" + if _, err := os.Stat(out); err != nil { + // Some builds write .html + alt := outBase + ".html" + if _, err2 := os.Stat(alt); err2 == nil { + return alt, nil + } + return "", fmt.Errorf("tesseract hocr: missing output %s (%s)", out, strings.TrimSpace(stderr.String())) + } + return out, nil +} + +// resolveHOCRConfig returns a tesseract config path that works when TESSDATA_PREFIX +// points at a custom traineddata dir without relative "hocr" configs. +func resolveHOCRConfig() string { + var candidates []string + if p := strings.TrimSpace(os.Getenv("TESSDATA_PREFIX")); p != "" { + candidates = append(candidates, + filepath.Join(p, "configs", "hocr"), + filepath.Join(p, "tessdata", "configs", "hocr"), + ) + } + candidates = append(candidates, + "/usr/share/tesseract-ocr/5/tessdata/configs/hocr", + "/usr/share/tesseract-ocr/4.00/tessdata/configs/hocr", + "/usr/share/tessdata/configs/hocr", + ) + for _, c := range candidates { + if _, err := os.Stat(c); err == nil { + return c + } + } + return "hocr" +} + +// HOCRToMarkdown parses an hOCR file via go-hocr and returns Markdown (+ optional YAML). +// Words with confidence in (0, minConf) are dropped; minConf 0 keeps all. +func HOCRToMarkdown(hocrPath string, minConf float32) (md string, yml string, err error) { + doc, err := hocr.ReadFile(hocrPath) + if err != nil { + return "", "", fmt.Errorf("go-hocr: %w", err) + } + md = doc.ToMarkdown(minConf) + yml, err = doc.ToYaml() + if err != nil { + return md, "", err + } + return md, yml, nil +} + +// ToHOCRMarkdown OCRs an image (or rasterizes first PDF page) to hOCR → Markdown/YAML. +func (t Tools) ToHOCRMarkdown(path, workDir, lang string, dpi int, minConf float32) (HOCRResult, error) { + res := HOCRResult{Source: path} + if workDir == "" { + workDir = os.TempDir() + } + ext := Ext(path) + img := path + switch ext { + case ".jpg", ".jpeg", ".png", ".tif", ".tiff", ".webp", ".gif", ".bmp": + // ok + case ".pdf": + raster, err := t.pdfFirstPagePNG(path, workDir) + if err != nil { + return res, err + } + img = raster + if dpi == 0 { + dpi = 300 + } + default: + return res, fmt.Errorf("hOCR path expects image or PDF, got %q", ext) + } + base := filepath.Join(workDir, trimExt(filepath.Base(path))+".hocr-out") + hocrPath, err := t.ImageToHOCR(img, base, lang, dpi) + if err != nil { + return res, err + } + res.DidOCR = true + res.HOCRPath = hocrPath + md, yml, err := HOCRToMarkdown(hocrPath, minConf) + if err != nil { + return res, err + } + res.Markdown = wrapMD(filepath.Base(path), strings.TrimSpace(md)) + res.YAML = yml + return res, nil +} + +// pdfFirstPagePNG uses pdftoppm when available. +func (t Tools) pdfFirstPagePNG(pdfPath, workDir string) (string, error) { + pdftoppm, err := exec.LookPath("pdftoppm") + if err != nil { + return "", fmt.Errorf("pdftoppm not found (needed to rasterize PDF for hOCR)") + } + outBase := filepath.Join(workDir, trimExt(filepath.Base(pdfPath))+".page") + cmd := exec.Command(pdftoppm, "-png", "-f", "1", "-singlefile", "-r", "200", pdfPath, outBase) + var stderr bytes.Buffer + cmd.Stderr = &stderr + if err := cmd.Run(); err != nil { + return "", fmt.Errorf("pdftoppm: %w (%s)", err, strings.TrimSpace(stderr.String())) + } + png := outBase + ".png" + if _, err := os.Stat(png); err != nil { + return "", fmt.Errorf("pdftoppm: missing %s", png) + } + return png, nil +} diff --git a/internal/docpipe/hocr_test.go b/internal/docpipe/hocr_test.go new file mode 100644 index 0000000..a4bb6be --- /dev/null +++ b/internal/docpipe/hocr_test.go @@ -0,0 +1,49 @@ +package docpipe + +import ( + "os" + "path/filepath" + "strings" + "testing" +) + +func TestHOCRToMarkdown_Fixture(t *testing.T) { + // Minimal hOCR 1.2 snippet + hocrXML := ` + + + + tesseract + + + + + +
+
+

+ + Hello + world + +

+
+
+ +` + dir := t.TempDir() + p := filepath.Join(dir, "sample.hocr") + if err := os.WriteFile(p, []byte(hocrXML), 0o644); err != nil { + t.Fatal(err) + } + md, yml, err := HOCRToMarkdown(p, 0) + if err != nil { + t.Fatal(err) + } + if !strings.Contains(md, "Hello") || !strings.Contains(md, "world") { + t.Fatalf("md=%q", md) + } + if !strings.Contains(yml, "Hello") { + t.Fatalf("yaml missing word: %s", yml) + } +}