filestore_core.go

@@ -82,7 +82,8 @@ type FileStore interface {
 // SearchQuery narrows a Searcher request. InContent asks the backend to match
 // document bodies, not just titles. Substring switches title matching from the
 // analyzer's whole-token match to a case-insensitive "*term*" wildcard and ANDs
-// every whitespace-separated term (e.g. "rechnung 2025").
+// every whitespace-separated term (e.g. "rechnung 2025"). Seller/Number/Status
+// are exact filters over gator fields indexed into oo_docs_text (Option D).
 type SearchQuery struct {
 	Text       string
 	InContent  bool
@@ -90,15 +91,21 @@ type SearchQuery struct {
 	Extensions []string
 	Limit      int
 	Substring  bool
+	Seller     string
+	Number     string
+	Status     string
 }
 
 // SearchHit is one Searcher result: the matching entry plus backend-specific
-// ranking metadata.
+// ranking metadata. Fields carries backend-specific extra key/values (for
+// oo_docs_text: gator document fields such as rechnungsnummer, lieferant,
+// betrag, status).
 type SearchHit struct {
 	Entry
 	Score     float64
 	Highlight string
 	Path      []string
+	Fields    map[string]string
 }
 
 // Searcher is the optional content/name search surface. Only some backends

filestore_es_text.go

@@ -44,13 +44,24 @@ func ESTextConfigFromEnv() ESTextConfig {
 }
 
 // TextDoc is one document in the own full-text index. It is keyed by the
-// OnlyOffice file id so hits map straight back to Documents entries.
+// OnlyOffice file id so hits map straight back to Documents entries. The gator
+// fields (Option D) are keyword values filled from the gator corpus; they stay
+// empty for documents gator does not know.
 type TextDoc struct {
 	ID        string `json:"id"`
 	Title     string `json:"title"`
 	FolderID  string `json:"folder,omitempty"`
 	Ext       string `json:"ext,omitempty"`
-	Content  string `json:"content"`
+	Content   string `json:"content,omitempty"`
+	DocSHA    string `json:"doc_sha,omitempty"`
+	Number    string `json:"number,omitempty"`
+	Seller    string `json:"seller,omitempty"`
+	Total     string `json:"total,omitempty"`
+	Typ       string `json:"typ,omitempty"`
+	IssuedAt  string `json:"issued_at,omitempty"`
+	CreatedAt string `json:"created_at,omitempty"`
+	Status    string `json:"status,omitempty"`
+	URL       string `json:"url,omitempty"`
 }
 
 // TextIndex is the storage/search surface for locally extracted document text.
@@ -103,7 +114,16 @@ const esTextMapping = `{
       "title":      {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}},
       "folder":     {"type": "keyword"},
       "ext":        {"type": "keyword"},
-      "content": {"type": "text"}
+      "content":    {"type": "text"},
+      "doc_sha":    {"type": "keyword"},
+      "number":     {"type": "keyword"},
+      "seller":     {"type": "keyword"},
+      "total":      {"type": "keyword"},
+      "typ":        {"type": "keyword"},
+      "issued_at":  {"type": "keyword"},
+      "created_at": {"type": "keyword"},
+      "status":     {"type": "keyword"},
+      "url":        {"type": "keyword"}
     }
   }
 }`
@@ -147,6 +167,31 @@ func (x *ESTextIndex) Put(ctx context.Context, docs []TextDoc) error {
 	return fmt.Errorf("onlyoffice: bulk index %s: %s", x.cfg.Index, res.firstError())
 }
 
+// UpsertFields writes only the gator keyword fields for each document via the
+// bulk update API with doc_as_upsert: an existing document keeps its
+// title/content while gaining the gator fields; an unknown id is created with
+// just these fields. Refreshes so results are immediately searchable.
+func (x *ESTextIndex) UpsertFields(ctx context.Context, docs []TextDoc) error {
+	if len(docs) == 0 {
+		return nil
+	}
+	status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_bulk?refresh=true", esUpdateBulkBody(docs), "application/x-ndjson")
+	if err != nil {
+		return err
+	}
+	if status >= 400 {
+		return fmt.Errorf("onlyoffice: bulk update %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
+	}
+	var res esBulkResponse
+	if err := json.Unmarshal(raw, &res); err != nil {
+		return fmt.Errorf("onlyoffice: decode bulk response: %w", err)
+	}
+	if !res.Errors {
+		return nil
+	}
+	return fmt.Errorf("onlyoffice: bulk update %s: %s", x.cfg.Index, res.firstError())
+}
+
 // Delete removes documents by OnlyOffice file id. A missing index means there
 // is nothing to delete.
 func (x *ESTextIndex) Delete(ctx context.Context, ids []string) error {
@@ -233,6 +278,20 @@ func esTextBulkBody(docs []TextDoc) []byte {
 	return b.Bytes()
 }
 
+// esUpdateBulkBody renders bulk update actions with doc_as_upsert, so a partial
+// TextDoc (gator fields only) merges into an existing document instead of
+// replacing it. Pure, so it is unit-tested.
+func esUpdateBulkBody(docs []TextDoc) []byte {
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	for _, d := range docs {
+		_ = enc.Encode(map[string]any{"update": map[string]any{"_id": d.ID}})
+		_ = enc.Encode(map[string]any{"doc": d, "doc_as_upsert": true})
+	}
+	return b.Bytes()
+}
+
 // esTextSearchRequest builds the own-index query. Pure, so it is unit-tested.
 func esTextSearchRequest(q SearchQuery) esRequest {
 	limit := q.Limit
@@ -252,10 +311,19 @@ func esTextSearchRequest(q SearchQuery) esRequest {
 	if exts := normalizeExtensions(q.Extensions); len(exts) > 0 {
 		filter = append(filter, esClause{Terms: map[string]any{"ext": exts}})
 	}
+	if v := strings.TrimSpace(q.Seller); v != "" {
+		filter = append(filter, esClause{Term: map[string]any{"seller": v}})
+	}
+	if v := strings.TrimSpace(q.Number); v != "" {
+		filter = append(filter, esClause{Term: map[string]any{"number": v}})
+	}
+	if v := strings.TrimSpace(q.Status); v != "" {
+		filter = append(filter, esClause{Term: map[string]any{"status": v}})
+	}
 
 	return esRequest{
 		Size:      limit,
-		Source:    []string{"id", "title", "folder", "ext"},
+		Source:    []string{"id", "title", "folder", "ext", "doc_sha", "number", "seller", "total", "typ", "issued_at", "created_at", "status", "url"},
 		Query:     esQuery{Bool: esBool{Must: must, Filter: filter}},
 		Highlight: esHighlight{PreTags: []string{"<em>"}, PostTags: []string{"</em>"}, Fields: map[string]struct{}{"title": {}, "content": {}}},
 	}
@@ -330,7 +398,26 @@ func parseESTextResponse(raw []byte) ([]SearchHit, error) {
 			Score:     h.Score,
 			Highlight: esHighlightText(h.HL),
 			Path:      path,
+			Fields:    gatorFields(h.Source),
 		})
 	}
 	return hits, nil
 }
+
+// gatorFields extracts the non-empty gator keyword fields from a stored doc.
+func gatorFields(d TextDoc) map[string]string {
+	out := map[string]string{}
+	for k, v := range map[string]string{
+		"doc_sha": d.DocSHA, "number": d.Number, "seller": d.Seller, "total": d.Total,
+		"typ": d.Typ, "issued_at": d.IssuedAt, "created_at": d.CreatedAt,
+		"status": d.Status, "url": d.URL,
+	} {
+		if v != "" {
+			out[k] = v
+		}
+	}
+	if len(out) == 0 {
+		return nil
+	}
+	return out
+}

filestore_es_text_gator_test.go

@@ -0,0 +1,71 @@
+package onlyoffice
+
+import (
+	"encoding/json"
+	"strings"
+	"testing"
+)
+
+func TestESUpdateBulkBodyDocAsUpsert(t *testing.T) {
+	got := string(esUpdateBulkBody([]TextDoc{{ID: "42", Number: "RE-1", Seller: "ACME"}}))
+	if !strings.Contains(got, `{"update":{"_id":"42"}}`) {
+		t.Errorf("missing update action: %s", got)
+	}
+	if !strings.Contains(got, `"doc_as_upsert":true`) {
+		t.Errorf("missing doc_as_upsert: %s", got)
+	}
+	if !strings.Contains(got, `"number":"RE-1"`) || !strings.Contains(got, `"seller":"ACME"`) {
+		t.Errorf("gator fields missing: %s", got)
+	}
+}
+
+func TestESTextSearchRequestGatorFilters(t *testing.T) {
+	body, err := json.Marshal(esTextSearchRequest(SearchQuery{
+		Text: "rechnung", Seller: "ACME", Number: "RE-1", Status: "ok",
+	}))
+	if err != nil {
+		t.Fatal(err)
+	}
+	s := string(body)
+	for _, want := range []string{
+		`"seller":"ACME"`, `"number":"RE-1"`, `"status":"ok"`,
+	} {
+		if !strings.Contains(s, want) {
+			t.Errorf("filter %s missing in %s", want, s)
+		}
+	}
+	for _, src := range []string{`"seller"`, `"number"`, `"total"`, `"status"`, `"url"`} {
+		if !strings.Contains(s, src) {
+			t.Errorf("_source field %s missing in %s", src, s)
+		}
+	}
+}
+
+func TestGatorFields(t *testing.T) {
+	if got := gatorFields(TextDoc{ID: "1", Title: "x"}); got != nil {
+		t.Errorf("empty doc fields = %v, want nil", got)
+	}
+	got := gatorFields(TextDoc{ID: "1", Number: "RE-1", Seller: "ACME", Status: "ok"})
+	if got["number"] != "RE-1" || got["seller"] != "ACME" || got["status"] != "ok" {
+		t.Errorf("gatorFields = %v", got)
+	}
+	if _, ok := got["total"]; ok {
+		t.Errorf("empty total should be omitted: %v", got)
+	}
+}
+
+func TestParseESTextResponseGatorFields(t *testing.T) {
+	raw := []byte(`{"hits":{"total":{"value":1},"hits":[
+	  {"_id":"42","_score":1.0,"_source":{"id":"42","title":"a.pdf","seller":"ACME","number":"RE-1","status":"ok"}}
+	]}}`)
+	hits, err := parseESTextResponse(raw)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if len(hits) != 1 {
+		t.Fatalf("hits = %d", len(hits))
+	}
+	if hits[0].Fields["seller"] != "ACME" || hits[0].Fields["number"] != "RE-1" {
+		t.Errorf("Fields = %v", hits[0].Fields)
+	}
+}