From 3f7505892f3358fb8f6d01e26602ebc340e6d71c Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Wed, 7 Oct 2026 08:20:29 +0000 Subject: [PATCH] feat(search): gator fields in oo_docs_text + field filters (#86) - TextDoc carries gator keyword fields (doc_sha, number, seller, total, typ, issued_at, created_at, status, url); content is now omitempty so a partial doc can be written. - oo_docs_text mapping extended with the keyword fields. - ESTextIndex.UpsertFields: bulk update + doc_as_upsert, merging gator fields into an existing text doc without clobbering title/content. - SearchQuery gains Seller/Number/Status term filters; SearchHit.Fields exposes the gator values. Pure builders unit-tested. --- filestore_core.go | 11 +++- filestore_es_text.go | 111 ++++++++++++++++++++++++++++---- filestore_es_text_gator_test.go | 71 ++++++++++++++++++++ 3 files changed, 179 insertions(+), 14 deletions(-) create mode 100644 filestore_es_text_gator_test.go diff --git a/filestore_core.go b/filestore_core.go index 51003da..c49aca7 100644 --- a/filestore_core.go +++ b/filestore_core.go @@ -82,7 +82,8 @@ type FileStore interface { // SearchQuery narrows a Searcher request. InContent asks the backend to match // document bodies, not just titles. Substring switches title matching from the // analyzer's whole-token match to a case-insensitive "*term*" wildcard and ANDs -// every whitespace-separated term (e.g. "rechnung 2025"). +// every whitespace-separated term (e.g. "rechnung 2025"). Seller/Number/Status +// are exact filters over gator fields indexed into oo_docs_text (Option D). type SearchQuery struct { Text string InContent bool @@ -90,15 +91,21 @@ type SearchQuery struct { Extensions []string Limit int Substring bool + Seller string + Number string + Status string } // SearchHit is one Searcher result: the matching entry plus backend-specific -// ranking metadata. +// ranking metadata. Fields carries backend-specific extra key/values (for +// oo_docs_text: gator document fields such as rechnungsnummer, lieferant, +// betrag, status). type SearchHit struct { Entry Score float64 Highlight string Path []string + Fields map[string]string } // Searcher is the optional content/name search surface. Only some backends diff --git a/filestore_es_text.go b/filestore_es_text.go index 072f00c..80e091b 100644 --- a/filestore_es_text.go +++ b/filestore_es_text.go @@ -44,13 +44,24 @@ func ESTextConfigFromEnv() ESTextConfig { } // TextDoc is one document in the own full-text index. It is keyed by the -// OnlyOffice file id so hits map straight back to Documents entries. +// OnlyOffice file id so hits map straight back to Documents entries. The gator +// fields (Option D) are keyword values filled from the gator corpus; they stay +// empty for documents gator does not know. type TextDoc struct { - ID string `json:"id"` - Title string `json:"title"` - FolderID string `json:"folder,omitempty"` - Ext string `json:"ext,omitempty"` - Content string `json:"content"` + ID string `json:"id"` + Title string `json:"title"` + FolderID string `json:"folder,omitempty"` + Ext string `json:"ext,omitempty"` + Content string `json:"content,omitempty"` + DocSHA string `json:"doc_sha,omitempty"` + Number string `json:"number,omitempty"` + Seller string `json:"seller,omitempty"` + Total string `json:"total,omitempty"` + Typ string `json:"typ,omitempty"` + IssuedAt string `json:"issued_at,omitempty"` + CreatedAt string `json:"created_at,omitempty"` + Status string `json:"status,omitempty"` + URL string `json:"url,omitempty"` } // TextIndex is the storage/search surface for locally extracted document text. @@ -99,11 +110,20 @@ func (x *ESTextIndex) Index() string { return x.cfg.Index } const esTextMapping = `{ "mappings": { "properties": { - "id": {"type": "keyword"}, - "title": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}}, - "folder": {"type": "keyword"}, - "ext": {"type": "keyword"}, - "content": {"type": "text"} + "id": {"type": "keyword"}, + "title": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}}, + "folder": {"type": "keyword"}, + "ext": {"type": "keyword"}, + "content": {"type": "text"}, + "doc_sha": {"type": "keyword"}, + "number": {"type": "keyword"}, + "seller": {"type": "keyword"}, + "total": {"type": "keyword"}, + "typ": {"type": "keyword"}, + "issued_at": {"type": "keyword"}, + "created_at": {"type": "keyword"}, + "status": {"type": "keyword"}, + "url": {"type": "keyword"} } } }` @@ -147,6 +167,31 @@ func (x *ESTextIndex) Put(ctx context.Context, docs []TextDoc) error { return fmt.Errorf("onlyoffice: bulk index %s: %s", x.cfg.Index, res.firstError()) } +// UpsertFields writes only the gator keyword fields for each document via the +// bulk update API with doc_as_upsert: an existing document keeps its +// title/content while gaining the gator fields; an unknown id is created with +// just these fields. Refreshes so results are immediately searchable. +func (x *ESTextIndex) UpsertFields(ctx context.Context, docs []TextDoc) error { + if len(docs) == 0 { + return nil + } + status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_bulk?refresh=true", esUpdateBulkBody(docs), "application/x-ndjson") + if err != nil { + return err + } + if status >= 400 { + return fmt.Errorf("onlyoffice: bulk update %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400)) + } + var res esBulkResponse + if err := json.Unmarshal(raw, &res); err != nil { + return fmt.Errorf("onlyoffice: decode bulk response: %w", err) + } + if !res.Errors { + return nil + } + return fmt.Errorf("onlyoffice: bulk update %s: %s", x.cfg.Index, res.firstError()) +} + // Delete removes documents by OnlyOffice file id. A missing index means there // is nothing to delete. func (x *ESTextIndex) Delete(ctx context.Context, ids []string) error { @@ -233,6 +278,20 @@ func esTextBulkBody(docs []TextDoc) []byte { return b.Bytes() } +// esUpdateBulkBody renders bulk update actions with doc_as_upsert, so a partial +// TextDoc (gator fields only) merges into an existing document instead of +// replacing it. Pure, so it is unit-tested. +func esUpdateBulkBody(docs []TextDoc) []byte { + var b bytes.Buffer + enc := json.NewEncoder(&b) + enc.SetEscapeHTML(false) + for _, d := range docs { + _ = enc.Encode(map[string]any{"update": map[string]any{"_id": d.ID}}) + _ = enc.Encode(map[string]any{"doc": d, "doc_as_upsert": true}) + } + return b.Bytes() +} + // esTextSearchRequest builds the own-index query. Pure, so it is unit-tested. func esTextSearchRequest(q SearchQuery) esRequest { limit := q.Limit @@ -252,10 +311,19 @@ func esTextSearchRequest(q SearchQuery) esRequest { if exts := normalizeExtensions(q.Extensions); len(exts) > 0 { filter = append(filter, esClause{Terms: map[string]any{"ext": exts}}) } + if v := strings.TrimSpace(q.Seller); v != "" { + filter = append(filter, esClause{Term: map[string]any{"seller": v}}) + } + if v := strings.TrimSpace(q.Number); v != "" { + filter = append(filter, esClause{Term: map[string]any{"number": v}}) + } + if v := strings.TrimSpace(q.Status); v != "" { + filter = append(filter, esClause{Term: map[string]any{"status": v}}) + } return esRequest{ Size: limit, - Source: []string{"id", "title", "folder", "ext"}, + Source: []string{"id", "title", "folder", "ext", "doc_sha", "number", "seller", "total", "typ", "issued_at", "created_at", "status", "url"}, Query: esQuery{Bool: esBool{Must: must, Filter: filter}}, Highlight: esHighlight{PreTags: []string{""}, PostTags: []string{""}, Fields: map[string]struct{}{"title": {}, "content": {}}}, } @@ -330,7 +398,26 @@ func parseESTextResponse(raw []byte) ([]SearchHit, error) { Score: h.Score, Highlight: esHighlightText(h.HL), Path: path, + Fields: gatorFields(h.Source), }) } return hits, nil } + +// gatorFields extracts the non-empty gator keyword fields from a stored doc. +func gatorFields(d TextDoc) map[string]string { + out := map[string]string{} + for k, v := range map[string]string{ + "doc_sha": d.DocSHA, "number": d.Number, "seller": d.Seller, "total": d.Total, + "typ": d.Typ, "issued_at": d.IssuedAt, "created_at": d.CreatedAt, + "status": d.Status, "url": d.URL, + } { + if v != "" { + out[k] = v + } + } + if len(out) == 0 { + return nil + } + return out +} diff --git a/filestore_es_text_gator_test.go b/filestore_es_text_gator_test.go new file mode 100644 index 0000000..fdea55e --- /dev/null +++ b/filestore_es_text_gator_test.go @@ -0,0 +1,71 @@ +package onlyoffice + +import ( + "encoding/json" + "strings" + "testing" +) + +func TestESUpdateBulkBodyDocAsUpsert(t *testing.T) { + got := string(esUpdateBulkBody([]TextDoc{{ID: "42", Number: "RE-1", Seller: "ACME"}})) + if !strings.Contains(got, `{"update":{"_id":"42"}}`) { + t.Errorf("missing update action: %s", got) + } + if !strings.Contains(got, `"doc_as_upsert":true`) { + t.Errorf("missing doc_as_upsert: %s", got) + } + if !strings.Contains(got, `"number":"RE-1"`) || !strings.Contains(got, `"seller":"ACME"`) { + t.Errorf("gator fields missing: %s", got) + } +} + +func TestESTextSearchRequestGatorFilters(t *testing.T) { + body, err := json.Marshal(esTextSearchRequest(SearchQuery{ + Text: "rechnung", Seller: "ACME", Number: "RE-1", Status: "ok", + })) + if err != nil { + t.Fatal(err) + } + s := string(body) + for _, want := range []string{ + `"seller":"ACME"`, `"number":"RE-1"`, `"status":"ok"`, + } { + if !strings.Contains(s, want) { + t.Errorf("filter %s missing in %s", want, s) + } + } + for _, src := range []string{`"seller"`, `"number"`, `"total"`, `"status"`, `"url"`} { + if !strings.Contains(s, src) { + t.Errorf("_source field %s missing in %s", src, s) + } + } +} + +func TestGatorFields(t *testing.T) { + if got := gatorFields(TextDoc{ID: "1", Title: "x"}); got != nil { + t.Errorf("empty doc fields = %v, want nil", got) + } + got := gatorFields(TextDoc{ID: "1", Number: "RE-1", Seller: "ACME", Status: "ok"}) + if got["number"] != "RE-1" || got["seller"] != "ACME" || got["status"] != "ok" { + t.Errorf("gatorFields = %v", got) + } + if _, ok := got["total"]; ok { + t.Errorf("empty total should be omitted: %v", got) + } +} + +func TestParseESTextResponseGatorFields(t *testing.T) { + raw := []byte(`{"hits":{"total":{"value":1},"hits":[ + {"_id":"42","_score":1.0,"_source":{"id":"42","title":"a.pdf","seller":"ACME","number":"RE-1","status":"ok"}} + ]}}`) + hits, err := parseESTextResponse(raw) + if err != nil { + t.Fatal(err) + } + if len(hits) != 1 { + t.Fatalf("hits = %d", len(hits)) + } + if hits[0].Fields["seller"] != "ACME" || hits[0].Fields["number"] != "RE-1" { + t.Errorf("Fields = %v", hits[0].Fields) + } +}