feat(search): gator fields in oo_docs_text + field filters (#86 )

- TextDoc carries gator keyword fields (doc_sha, number, seller, total, typ, issued_at, created_at, status, url); content is now omitempty so a partial doc can be written. - oo_docs_text mapping extended with the keyword fields. - ESTextIndex.UpsertFields: bulk update + doc_as_upsert, merging gator fields into an existing text doc without clobbering title/content. - SearchQuery gains Seller/Number/Status term filters; SearchHit.Fields exposes the gator values. Pure builders unit-tested.
2026-10-07 08:20:29 +00:00
7 changed files with 179 additions and 612 deletions
@@ -48,7 +48,6 @@ Structured OCR (hOCR→MD): oo docs hocr scan.jpg --md out.md --yaml out.yml`,
 	cmd.AddCommand(docsPutTxtCmd())
 	cmd.AddCommand(docsPutXlsxCmd())
 	cmd.AddCommand(docsToolsCmd())
 	cmd.AddCommand(docsCommentCmd())
 	return cmd
 }
@@ -1,166 +0,0 @@
 package main
 import (
 	"bytes"
 	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
 	onlyoffice "github.com/eslider/go-onlyoffice"
 	"github.com/spf13/cobra"
 )
 func docsCommentCmd() *cobra.Command {
 	var spec, author, out, basePath string
 	var replace bool
 	cmd := &cobra.Command{
 		Use:   "comment FILE_ID",
 		Short: "Add Word comments to an OO .docx, anchored to paragraph text",
 		Long: `Adds real Word (DOCX) comments to a Documents file and, with --replace,
 uploads the annotated copy back over the same file id.
 OnlyOffice Workspace exposes no comments REST route (GET/POST
 /api/2.0/files/file/{id}/comment answer 404/500), while ONLYOFFICE Docs renders
 DOCX comments natively, so the comments are injected into the file itself.
 The spec is a JSON array of {"anchor":"<paragraph text>","text":"<comment>"}.
 An anchor is matched against the paragraph's concatenated text, so it survives
 text split across runs. At most one comment lands on a paragraph.
  oo docs comment 4379 --spec comments.json --out review.docx
  oo docs comment 4379 --spec comments.json --replace`,
 		Args: cobra.ExactArgs(1),
 		RunE: func(cmd *cobra.Command, args []string) error {
 			if spec == "" {
 				return fmt.Errorf("--spec is required")
 			}
 			raw, err := os.ReadFile(spec)
 			if err != nil {
 				return err
 			}
 			comments, err := onlyoffice.ParseDocxComments(raw)
 			if err != nil {
 				return err
 			}
 			if len(comments) == 0 {
 				return fmt.Errorf("spec %s has no comments", spec)
 			}
 			c, err := newOO(cmd)
 			if err != nil {
 				return err
 			}
 			ctx := cmd.Context()
 			meta, err := c.GetFile(ctx, args[0])
 			if err != nil {
 				return err
 			}
 			title := onlyoffice.FileEntryTitle(meta)
 			if title == "" {
 				title = "file-" + args[0] + ".docx"
 			}
 			if !strings.HasSuffix(strings.ToLower(title), ".docx") {
 				return fmt.Errorf("%s is not a .docx (comments need Word format)", title)
 			}
 			var base []byte
 			if basePath != "" {
 				base, err = os.ReadFile(basePath)
 				if err != nil {
 					return err
 				}
 			} else {
 				var buf bytes.Buffer
 				if _, err := c.DownloadFile(ctx, args[0], &buf); err != nil {
 					return err
 				}
 				base = buf.Bytes()
 			}
 			if author == "" {
 				author = selfDisplayName(cmd)
 			}
 			annotated, applied, err := onlyoffice.InjectDocxComments(base, author, comments)
 			if err != nil {
 				return err
 			}
 			if applied == 0 {
 				return fmt.Errorf("no anchor matched; check the spec against the document text")
 			}
 			dest := out
 			if dest == "" {
 				base := strings.TrimSuffix(onlyoffice.SafeLocalFileName(title), filepath.Ext(onlyoffice.SafeLocalFileName(title)))
 				dest = base + " - comments.docx"
 			}
 			if err := os.WriteFile(dest, annotated, 0o600); err != nil {
 				return err
 			}
 			obj := map[string]any{
 				"file_id":         args[0],
 				"title":           title,
 				"comments":        len(comments),
 				"applied":         applied,
 				"author":          author,
 				"local":           dest,
 				"replaced":        false,
 				"onlyoffice_link": c.FileEditorURL(args[0]),
 			}
 			if replace {
 				tmp, err := os.MkdirTemp("", "oo-docs-comment-*")
 				if err != nil {
 					return err
 				}
 				defer os.RemoveAll(tmp)
 				up := filepath.Join(tmp, onlyoffice.SafeLocalFileName(title))
 				if err := os.WriteFile(up, annotated, 0o600); err != nil {
 					return err
 				}
 				// UpdateFile replaces the content in place, keeping the file id
 				// (so existing links and the CRM entry stay valid).
 				if _, err := c.UpdateFile(ctx, args[0], up); err != nil {
 					return err
 				}
 				obj["replaced"] = true
 				obj["folder_id"] = onlyoffice.FileFolderID(meta)
 			}
 			printJSON(obj)
 			return nil
 		},
 	}
 	cmd.Flags().StringVar(&spec, "spec", "", "JSON file: [{\"anchor\":...,\"text\":...}] (required)")
 	cmd.Flags().StringVar(&author, "author", "", "comment author (default: the authenticated portal user)")
 	cmd.Flags().StringVar(&out, "out", "", "local output path (default: <title> - comments.docx)")
 	cmd.Flags().StringVar(&basePath, "base", "", "local clean .docx to annotate instead of downloading FILE_ID (re-apply without stacking comments)")
 	cmd.Flags().BoolVar(&replace, "replace", false, "upload the annotated copy back over the same file id")
 	return cmd
 }
 // selfDisplayName resolves the authenticated user's display name for the
 // comment author, falling back to first and last name, then to "OnlyOffice".
 func selfDisplayName(cmd *cobra.Command) string {
 	c, err := newOO(cmd)
 	if err != nil {
 		return "OnlyOffice"
 	}
 	ctx := cmd.Context()
 	id, err := c.SelfUserID(ctx)
 	if err != nil || id == "" {
 		return "OnlyOffice"
 	}
 	u, err := c.GetUser(ctx, id)
 	if err != nil {
 		return "OnlyOffice"
 	}
 	if s, ok := u["displayName"].(string); ok && strings.TrimSpace(s) != "" {
 		return s
 	}
 	first, _ := u["firstName"].(string)
 	last, _ := u["lastName"].(string)
 	if name := strings.TrimSpace(first + " " + last); name != "" {
 		return name
 	}
 	return "OnlyOffice"
 }
@@ -1,250 +0,0 @@
 package onlyoffice
 import (
 	"archive/zip"
 	"bytes"
 	"encoding/json"
 	"encoding/xml"
 	"fmt"
 	"io"
 	"regexp"
 	"strings"
 )
 // DocxComment is one Word comment. Anchor is matched against the plain text of
 // a paragraph (entity-decoded, runs concatenated), so it survives text split
 // across runs. At most one comment is placed per paragraph.
 type DocxComment struct {
 	Anchor string `json:"anchor"`
 	Text   string `json:"text"`
 }
 var (
 	docxParaRe = regexp.MustCompile(`(?s)<w:p\b[^>]*>.*?</w:p>`)
 	docxPPrRe  = regexp.MustCompile(`(?s)^<w:p\b[^>]*>\s*<w:pPr>.*?</w:pPr>`)
 	docxOpenRe = regexp.MustCompile(`^<w:p\b[^>]*>`)
 	docxTextRe = regexp.MustCompile(`(?s)<w:t\b[^>]*>(.*?)</w:t>`)
 )
 // ParseDocxComments decodes a JSON array of comments.
 func ParseDocxComments(data []byte) ([]DocxComment, error) {
 	var comments []DocxComment
 	if err := json.Unmarshal(data, &comments); err != nil {
 		return nil, fmt.Errorf("decode comments: %w", err)
 	}
 	return comments, nil
 }
 // DocxParagraphs returns the concatenated text of every paragraph in a .docx
 // document.xml-free input (it takes the raw document.xml bytes).
 func DocxParagraphs(documentXML []byte) []string {
 	out := make([]string, 0, 64)
 	for _, p := range docxParaRe.FindAllString(string(documentXML), -1) {
 		if txt := strings.TrimSpace(docxPlain(p)); txt != "" {
 			out = append(out, txt)
 		}
 	}
 	return out
 }
 // InjectDocxComments returns a copy of the .docx with real Word comments added,
 // each anchored to the first paragraph whose text contains the comment's Anchor.
 // The author name is written into every comment. It reports how many comments
 // were applied; anchors that match nothing are skipped.
 func InjectDocxComments(src []byte, author string, comments []DocxComment) ([]byte, int, error) {
 	parts, order, err := readDocxZip(src)
 	if err != nil {
 		return nil, 0, err
 	}
 	doc, ok := parts["word/document.xml"]
 	if !ok {
 		return nil, 0, fmt.Errorf("not a docx: word/document.xml missing")
 	}
 	if author == "" {
 		author = "Comment"
 	}
 	used := make([]bool, len(comments))
 	applied := make([]DocxComment, 0, len(comments))
 	id := 0
 	newDoc := docxParaRe.ReplaceAllStringFunc(string(doc), func(p string) string {
 		txt := docxPlain(p)
 		for i, c := range comments {
 			if used[i] || c.Anchor == "" || !strings.Contains(txt, c.Anchor) {
 				continue
 			}
 			used[i] = true
 			// Record in document order so comments.xml id N matches the marker
 			// with id N (the spec order may differ from the document order).
 			applied = append(applied, c)
 			p = docxAddMarkers(p, id)
 			id++
 			break
 		}
 		return p
 	})
 	parts["word/document.xml"] = []byte(newDoc)
 	parts["word/comments.xml"] = docxBuildComments(author, applied)
 	parts["[Content_Types].xml"] = docxAddContentType(parts["[Content_Types].xml"])
 	if rels, ok := parts["word/_rels/document.xml.rels"]; ok {
 		parts["word/_rels/document.xml.rels"] = docxAddRel(rels)
 	} else {
 		parts["word/_rels/document.xml.rels"] = docxDefaultRels()
 		order = append(order, "word/_rels/document.xml.rels")
 	}
 	if !containsStr(order, "word/comments.xml") {
 		order = append(order, "word/comments.xml")
 	}
 	out, err := writeDocxZip(parts, order)
 	if err != nil {
 		return nil, 0, err
 	}
 	return out, len(applied), nil
 }
 func docxAddMarkers(p string, id int) string {
 	start := fmt.Sprintf(`<w:commentRangeStart w:id="%d"/>`, id)
 	end := fmt.Sprintf(`<w:commentRangeEnd w:id="%d"/><w:r><w:commentReference w:id="%d"/></w:r>`, id, id)
 	if loc := docxPPrRe.FindStringIndex(p); loc != nil {
 		p = p[:loc[1]] + start + p[loc[1]:]
 	} else if loc := docxOpenRe.FindStringIndex(p); loc != nil {
 		p = p[:loc[1]] + start + p[loc[1]:]
 	}
 	if i := strings.LastIndex(p, "</w:p>"); i >= 0 {
 		p = p[:i] + end + p[i:]
 	}
 	return p
 }
 func docxPlain(p string) string {
 	var b strings.Builder
 	for _, m := range docxTextRe.FindAllStringSubmatch(p, -1) {
 		b.WriteString(docxUnescape(m[1]))
 	}
 	return b.String()
 }
 func docxUnescape(s string) string {
 	var out strings.Builder
 	d := xml.NewDecoder(strings.NewReader("<x>" + s + "</x>"))
 	for {
 		tok, err := d.Token()
 		if err != nil {
 			break
 		}
 		if cd, ok := tok.(xml.CharData); ok {
 			out.Write(cd)
 		}
 	}
 	return out.String()
 }
 func docxBuildComments(author string, comments []DocxComment) []byte {
 	var b strings.Builder
 	b.WriteString(`<?xml version="1.0" encoding="UTF-8" standalone="yes"?>` + "\n")
 	b.WriteString(`<w:comments xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main">`)
 	for i, c := range comments {
 		fmt.Fprintf(&b, `<w:comment w:id="%d" w:author="%s" w:initials="%s" w:date="%s"><w:p><w:r><w:t xml:space="preserve">%s</w:t></w:r></w:p></w:comment>`,
 			i, docxEscape(author), docxInitials(author), "2026-10-07T12:00:00Z", docxEscape(c.Text))
 	}
 	b.WriteString(`</w:comments>`)
 	return []byte(b.String())
 }
 func docxEscape(s string) string {
 	var b bytes.Buffer
 	_ = xml.EscapeText(&b, []byte(s))
 	return b.String()
 }
 // docxInitials derives up to two initials from an author name.
 func docxInitials(author string) string {
 	var out []rune
 	for _, f := range strings.Fields(author) {
 		if r := []rune(f); len(r) > 0 {
 			out = append(out, r[0])
 		}
 		if len(out) == 2 {
 			break
 		}
 	}
 	return strings.ToUpper(string(out))
 }
 func docxAddContentType(ct []byte) []byte {
 	if len(ct) == 0 || strings.Contains(string(ct), "/word/comments.xml") {
 		return ct
 	}
 	const override = `<Override PartName="/word/comments.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.comments+xml"/>`
 	return []byte(strings.Replace(string(ct), "</Types>", override+"</Types>", 1))
 }
 func docxAddRel(rels []byte) []byte {
 	if strings.Contains(string(rels), "relationships/comments") {
 		return rels
 	}
 	const rel = `<Relationship Id="rIdComments" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments" Target="comments.xml"/>`
 	return []byte(strings.Replace(string(rels), "</Relationships>", rel+"</Relationships>", 1))
 }
 func docxDefaultRels() []byte {
 	return []byte(`<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rIdComments" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/comments" Target="comments.xml"/></Relationships>`)
 }
 func readDocxZip(data []byte) (map[string][]byte, []string, error) {
 	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
 	if err != nil {
 		return nil, nil, err
 	}
 	parts := make(map[string][]byte, len(zr.File))
 	order := make([]string, 0, len(zr.File))
 	for _, f := range zr.File {
 		rc, err := f.Open()
 		if err != nil {
 			return nil, nil, err
 		}
 		b, err := io.ReadAll(rc)
 		rc.Close()
 		if err != nil {
 			return nil, nil, err
 		}
 		parts[f.Name] = b
 		order = append(order, f.Name)
 	}
 	return parts, order, nil
 }
 func writeDocxZip(parts map[string][]byte, order []string) ([]byte, error) {
 	var buf bytes.Buffer
 	zw := zip.NewWriter(&buf)
 	written := map[string]bool{}
 	for _, name := range order {
 		data, ok := parts[name]
 		if !ok || written[name] {
 			continue
 		}
 		w, err := zw.Create(name)
 		if err != nil {
 			return nil, err
 		}
 		if _, err := w.Write(data); err != nil {
 			return nil, err
 		}
 		written[name] = true
 	}
 	if err := zw.Close(); err != nil {
 		return nil, err
 	}
 	return buf.Bytes(), nil
 }
 func containsStr(xs []string, s string) bool {
 	for _, x := range xs {
 		if x == s {
 			return true
 		}
 	}
 	return false
 }
@@ -1,181 +0,0 @@
 package onlyoffice
 import (
 	"archive/zip"
 	"bytes"
 	"io"
 	"regexp"
 	"strings"
 	"testing"
 )
 // buildTestDocx returns a minimal valid .docx with two paragraphs.
 func buildTestDocx(t *testing.T) []byte {
 	t.Helper()
 	const (
 		contentTypes = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="xml" ContentType="application/xml"/><Override PartName="/word/document.xml" ContentType="application/vnd.openxmlformats-officedocument.wordprocessingml.document.main+xml"/></Types>`
 		document     = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:pPr><w:jc w:val="both"/></w:pPr><w:r><w:t>The term is two years, ending 8 October 2028.</w:t></w:r></w:p><w:p><w:r><w:t>Nothing relevant here.</w:t></w:r></w:p></w:body></w:document>`
 		rels         = `<?xml version="1.0" encoding="UTF-8" standalone="yes"?><Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"></Relationships>`
 	)
 	var buf bytes.Buffer
 	zw := zip.NewWriter(&buf)
 	for name, body := range map[string]string{
 		"[Content_Types].xml":          contentTypes,
 		"word/document.xml":            document,
 		"word/_rels/document.xml.rels": rels,
 	} {
 		w, err := zw.Create(name)
 		if err != nil {
 			t.Fatalf("zip create: %v", err)
 		}
 		if _, err := w.Write([]byte(body)); err != nil {
 			t.Fatalf("zip write: %v", err)
 		}
 	}
 	if err := zw.Close(); err != nil {
 		t.Fatalf("zip close: %v", err)
 	}
 	return buf.Bytes()
 }
 func unzipPart(t *testing.T, data []byte, name string) string {
 	t.Helper()
 	zr, err := zip.NewReader(bytes.NewReader(data), int64(len(data)))
 	if err != nil {
 		t.Fatalf("zip reader: %v", err)
 	}
 	for _, f := range zr.File {
 		if f.Name != name {
 			continue
 		}
 		rc, err := f.Open()
 		if err != nil {
 			t.Fatalf("open part: %v", err)
 		}
 		defer rc.Close()
 		b, err := io.ReadAll(rc)
 		if err != nil {
 			t.Fatalf("read part: %v", err)
 		}
 		return string(b)
 	}
 	t.Fatalf("part %s not found", name)
 	return ""
 }
 func TestInjectDocxComments(t *testing.T) {
 	src := buildTestDocx(t)
 	comments := []DocxComment{
 		{Anchor: "8 October 2028", Text: "Change to two years."},
 		{Anchor: "two years", Text: "second anchor, same paragraph"},
 		{Anchor: "no such text anywhere", Text: "skipped"},
 	}
 	// First anchor matches the first paragraph; the second also matches that
 	// paragraph but must be skipped (one comment per paragraph).
 	out, applied, err := InjectDocxComments(src, "Andriy Oblivantsev", comments)
 	if err != nil {
 		t.Fatalf("inject: %v", err)
 	}
 	if applied != 1 {
 		t.Fatalf("applied = %d, want 1", applied)
 	}
 	doc := unzipPart(t, out, "word/document.xml")
 	for _, want := range []string{"commentRangeStart", "commentRangeEnd", "commentReference"} {
 		if !strings.Contains(doc, want) {
 			t.Errorf("document.xml missing %s", want)
 		}
 	}
 	if strings.Contains(doc, `<w:pPr><w:commentRangeStart`) {
 		t.Error("commentRangeStart inserted inside pPr")
 	}
 	cmts := unzipPart(t, out, "word/comments.xml")
 	if strings.Count(cmts, "<w:comment ") != 1 {
 		t.Fatalf("comments count = %d, want 1", strings.Count(cmts, "<w:comment "))
 	}
 	if !strings.Contains(cmts, `w:author="Andriy Oblivantsev"`) {
 		t.Error("author missing in comments.xml")
 	}
 	if !strings.Contains(cmts, "Change to two years.") {
 		t.Error("comment text missing")
 	}
 	ct := unzipPart(t, out, "[Content_Types].xml")
 	if !strings.Contains(ct, "/word/comments.xml") {
 		t.Error("content types missing comments override")
 	}
 	rels := unzipPart(t, out, "word/_rels/document.xml.rels")
 	if !strings.Contains(rels, "relationships/comments") {
 		t.Error("document rels missing comments relationship")
 	}
 }
 func TestInjectDocxCommentsNoMatch(t *testing.T) {
 	src := buildTestDocx(t)
 	out, applied, err := InjectDocxComments(src, "X", []DocxComment{{Anchor: "absent", Text: "t"}})
 	if err != nil {
 		t.Fatalf("inject: %v", err)
 	}
 	if applied != 0 {
 		t.Fatalf("applied = %d, want 0", applied)
 	}
 	if out == nil {
 		t.Fatal("output nil")
 	}
 }
 // TestInjectDocxCommentsOrder guards the marker-id ↔ comments.xml pairing when
 // the spec order differs from the document order.
 func TestInjectDocxCommentsOrder(t *testing.T) {
 	src := buildTestDocx(t)
 	spec := []DocxComment{
 		{Anchor: "Nothing relevant", Text: "comment B"},
 		{Anchor: "The term is two years", Text: "comment A"},
 	}
 	out, applied, err := InjectDocxComments(src, "A", spec)
 	if err != nil || applied != 2 {
 		t.Fatalf("inject: applied=%d err=%v", applied, err)
 	}
 	doc := unzipPart(t, out, "word/document.xml")
 	cmts := unzipPart(t, out, "word/comments.xml")
 	// id -> text from comments.xml
 	byID := map[string]string{}
 	for _, m := range regexp.MustCompile(`(?s)<w:comment w:id="(\d+)"[^>]*>(.*?)</w:comment>`).FindAllStringSubmatch(cmts, -1) {
 		txt := strings.TrimSpace(stripTags(m[2]))
 		byID[m[1]] = txt
 	}
 	// paragraph that starts with the term text must point at "comment A"
 	for _, p := range regexp.MustCompile(`(?s)<w:p\b[^>]*>.*?</w:p>`).FindAllString(doc, -1) {
 		if !strings.Contains(stripTags(p), "The term is two years") {
 			continue
 		}
 		m := regexp.MustCompile(`<w:commentRangeStart w:id="(\d+)"/>`).FindStringSubmatch(p)
 		if m == nil {
 			t.Fatal("no commentRangeStart in the term paragraph")
 		}
 		if got := byID[m[1]]; got != "comment A" {
 			t.Fatalf("term paragraph points at id %s = %q, want \"comment A\"", m[1], got)
 		}
 		return
 	}
 	t.Fatal("term paragraph not found")
 }
 func stripTags(s string) string {
 	return regexp.MustCompile(`<[^>]*>`).ReplaceAllString(s, "")
 }
 func TestParseDocxComments(t *testing.T) {
 	got, err := ParseDocxComments([]byte(`[{"anchor":"a","text":"b"}]`))
 	if err != nil {
 		t.Fatalf("parse: %v", err)
 	}
 	if len(got) != 1 || got[0].Anchor != "a" || got[0].Text != "b" {
 		t.Fatalf("unexpected: %+v", got)
 	}
 	if _, err := ParseDocxComments([]byte(`{"anchor":"a"}`)); err == nil {
 		t.Fatal("expected error for object input")
 	}
 }
@@ -82,7 +82,8 @@ type FileStore interface {
 // SearchQuery narrows a Searcher request. InContent asks the backend to match
 // document bodies, not just titles. Substring switches title matching from the
 // analyzer's whole-token match to a case-insensitive "*term*" wildcard and ANDs
-// every whitespace-separated term (e.g. "rechnung 2025").
+// every whitespace-separated term (e.g. "rechnung 2025"). Seller/Number/Status
 // are exact filters over gator fields indexed into oo_docs_text (Option D).
 type SearchQuery struct {
 	Text       string
 	InContent  bool
@@ -90,15 +91,21 @@ type SearchQuery struct {
 	Extensions []string
 	Limit      int
 	Substring  bool
 	Seller     string
 	Number     string
 	Status     string
 }
 // SearchHit is one Searcher result: the matching entry plus backend-specific
-// ranking metadata.
+// ranking metadata. Fields carries backend-specific extra key/values (for
 // oo_docs_text: gator document fields such as rechnungsnummer, lieferant,
 // betrag, status).
 type SearchHit struct {
 	Entry
 	Score     float64
 	Highlight string
 	Path      []string
 	Fields    map[string]string
 }
 // Searcher is the optional content/name search surface. Only some backends
@@ -44,13 +44,24 @@ func ESTextConfigFromEnv() ESTextConfig {
 }
 // TextDoc is one document in the own full-text index. It is keyed by the
-// OnlyOffice file id so hits map straight back to Documents entries.
+// OnlyOffice file id so hits map straight back to Documents entries. The gator
 // fields (Option D) are keyword values filled from the gator corpus; they stay
 // empty for documents gator does not know.
 type TextDoc struct {
 	ID        string `json:"id"`
 	Title     string `json:"title"`
 	FolderID  string `json:"folder,omitempty"`
 	Ext       string `json:"ext,omitempty"`
-	Content  string `json:"content"`
+	Content   string `json:"content,omitempty"`
 	DocSHA    string `json:"doc_sha,omitempty"`
 	Number    string `json:"number,omitempty"`
 	Seller    string `json:"seller,omitempty"`
 	Total     string `json:"total,omitempty"`
 	Typ       string `json:"typ,omitempty"`
 	IssuedAt  string `json:"issued_at,omitempty"`
 	CreatedAt string `json:"created_at,omitempty"`
 	Status    string `json:"status,omitempty"`
 	URL       string `json:"url,omitempty"`
 }
 // TextIndex is the storage/search surface for locally extracted document text.
@@ -103,7 +114,16 @@ const esTextMapping = `{
      "title":      {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}},
      "folder":     {"type": "keyword"},
      "ext":        {"type": "keyword"},
-      "content": {"type": "text"}
+      "content":    {"type": "text"},
      "doc_sha":    {"type": "keyword"},
      "number":     {"type": "keyword"},
      "seller":     {"type": "keyword"},
      "total":      {"type": "keyword"},
      "typ":        {"type": "keyword"},
      "issued_at":  {"type": "keyword"},
      "created_at": {"type": "keyword"},
      "status":     {"type": "keyword"},
      "url":        {"type": "keyword"}
    }
  }
 }`
@@ -147,6 +167,31 @@ func (x *ESTextIndex) Put(ctx context.Context, docs []TextDoc) error {
 	return fmt.Errorf("onlyoffice: bulk index %s: %s", x.cfg.Index, res.firstError())
 }
 // UpsertFields writes only the gator keyword fields for each document via the
 // bulk update API with doc_as_upsert: an existing document keeps its
 // title/content while gaining the gator fields; an unknown id is created with
 // just these fields. Refreshes so results are immediately searchable.
 func (x *ESTextIndex) UpsertFields(ctx context.Context, docs []TextDoc) error {
 	if len(docs) == 0 {
 		return nil
 	}
 	status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_bulk?refresh=true", esUpdateBulkBody(docs), "application/x-ndjson")
 	if err != nil {
 		return err
 	}
 	if status >= 400 {
 		return fmt.Errorf("onlyoffice: bulk update %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
 	}
 	var res esBulkResponse
 	if err := json.Unmarshal(raw, &res); err != nil {
 		return fmt.Errorf("onlyoffice: decode bulk response: %w", err)
 	}
 	if !res.Errors {
 		return nil
 	}
 	return fmt.Errorf("onlyoffice: bulk update %s: %s", x.cfg.Index, res.firstError())
 }
 // Delete removes documents by OnlyOffice file id. A missing index means there
 // is nothing to delete.
 func (x *ESTextIndex) Delete(ctx context.Context, ids []string) error {
@@ -233,6 +278,20 @@ func esTextBulkBody(docs []TextDoc) []byte {
 	return b.Bytes()
 }
 // esUpdateBulkBody renders bulk update actions with doc_as_upsert, so a partial
 // TextDoc (gator fields only) merges into an existing document instead of
 // replacing it. Pure, so it is unit-tested.
 func esUpdateBulkBody(docs []TextDoc) []byte {
 	var b bytes.Buffer
 	enc := json.NewEncoder(&b)
 	enc.SetEscapeHTML(false)
 	for _, d := range docs {
 		_ = enc.Encode(map[string]any{"update": map[string]any{"_id": d.ID}})
 		_ = enc.Encode(map[string]any{"doc": d, "doc_as_upsert": true})
 	}
 	return b.Bytes()
 }
 // esTextSearchRequest builds the own-index query. Pure, so it is unit-tested.
 func esTextSearchRequest(q SearchQuery) esRequest {
 	limit := q.Limit
@@ -252,10 +311,19 @@ func esTextSearchRequest(q SearchQuery) esRequest {
 	if exts := normalizeExtensions(q.Extensions); len(exts) > 0 {
 		filter = append(filter, esClause{Terms: map[string]any{"ext": exts}})
 	}
 	if v := strings.TrimSpace(q.Seller); v != "" {
 		filter = append(filter, esClause{Term: map[string]any{"seller": v}})
 	}
 	if v := strings.TrimSpace(q.Number); v != "" {
 		filter = append(filter, esClause{Term: map[string]any{"number": v}})
 	}
 	if v := strings.TrimSpace(q.Status); v != "" {
 		filter = append(filter, esClause{Term: map[string]any{"status": v}})
 	}
 	return esRequest{
 		Size:      limit,
-		Source:    []string{"id", "title", "folder", "ext"},
+		Source:    []string{"id", "title", "folder", "ext", "doc_sha", "number", "seller", "total", "typ", "issued_at", "created_at", "status", "url"},
 		Query:     esQuery{Bool: esBool{Must: must, Filter: filter}},
 		Highlight: esHighlight{PreTags: []string{"<em>"}, PostTags: []string{"</em>"}, Fields: map[string]struct{}{"title": {}, "content": {}}},
 	}
@@ -330,7 +398,26 @@ func parseESTextResponse(raw []byte) ([]SearchHit, error) {
 			Score:     h.Score,
 			Highlight: esHighlightText(h.HL),
 			Path:      path,
 			Fields:    gatorFields(h.Source),
 		})
 	}
 	return hits, nil
 }
 // gatorFields extracts the non-empty gator keyword fields from a stored doc.
 func gatorFields(d TextDoc) map[string]string {
 	out := map[string]string{}
 	for k, v := range map[string]string{
 		"doc_sha": d.DocSHA, "number": d.Number, "seller": d.Seller, "total": d.Total,
 		"typ": d.Typ, "issued_at": d.IssuedAt, "created_at": d.CreatedAt,
 		"status": d.Status, "url": d.URL,
 	} {
 		if v != "" {
 			out[k] = v
 		}
 	}
 	if len(out) == 0 {
 		return nil
 	}
 	return out
 }
@@ -0,0 +1,71 @@
 package onlyoffice
 import (
 	"encoding/json"
 	"strings"
 	"testing"
 )
 func TestESUpdateBulkBodyDocAsUpsert(t *testing.T) {
 	got := string(esUpdateBulkBody([]TextDoc{{ID: "42", Number: "RE-1", Seller: "ACME"}}))
 	if !strings.Contains(got, `{"update":{"_id":"42"}}`) {
 		t.Errorf("missing update action: %s", got)
 	}
 	if !strings.Contains(got, `"doc_as_upsert":true`) {
 		t.Errorf("missing doc_as_upsert: %s", got)
 	}
 	if !strings.Contains(got, `"number":"RE-1"`) || !strings.Contains(got, `"seller":"ACME"`) {
 		t.Errorf("gator fields missing: %s", got)
 	}
 }
 func TestESTextSearchRequestGatorFilters(t *testing.T) {
 	body, err := json.Marshal(esTextSearchRequest(SearchQuery{
 		Text: "rechnung", Seller: "ACME", Number: "RE-1", Status: "ok",
 	}))
 	if err != nil {
 		t.Fatal(err)
 	}
 	s := string(body)
 	for _, want := range []string{
 		`"seller":"ACME"`, `"number":"RE-1"`, `"status":"ok"`,
 	} {
 		if !strings.Contains(s, want) {
 			t.Errorf("filter %s missing in %s", want, s)
 		}
 	}
 	for _, src := range []string{`"seller"`, `"number"`, `"total"`, `"status"`, `"url"`} {
 		if !strings.Contains(s, src) {
 			t.Errorf("_source field %s missing in %s", src, s)
 		}
 	}
 }
 func TestGatorFields(t *testing.T) {
 	if got := gatorFields(TextDoc{ID: "1", Title: "x"}); got != nil {
 		t.Errorf("empty doc fields = %v, want nil", got)
 	}
 	got := gatorFields(TextDoc{ID: "1", Number: "RE-1", Seller: "ACME", Status: "ok"})
 	if got["number"] != "RE-1" || got["seller"] != "ACME" || got["status"] != "ok" {
 		t.Errorf("gatorFields = %v", got)
 	}
 	if _, ok := got["total"]; ok {
 		t.Errorf("empty total should be omitted: %v", got)
 	}
 }
 func TestParseESTextResponseGatorFields(t *testing.T) {
 	raw := []byte(`{"hits":{"total":{"value":1},"hits":[
 	  {"_id":"42","_score":1.0,"_source":{"id":"42","title":"a.pdf","seller":"ACME","number":"RE-1","status":"ok"}}
 	]}}`)
 	hits, err := parseESTextResponse(raw)
 	if err != nil {
 		t.Fatal(err)
 	}
 	if len(hits) != 1 {
 		t.Fatalf("hits = %d", len(hits))
 	}
 	if hits[0].Fields["seller"] != "ACME" || hits[0].Fields["number"] != "RE-1" {
 		t.Errorf("Fields = %v", hits[0].Fields)
 	}
 }