feat(search): gator fields in oo_docs_text + field filters (#86)
Release / GoReleaser (push) Skipped
Tests / Secret scan (gitleaks) (push) Skipped
Tests / Test (Go 1.25) (push) Skipped
Tests / Test (Go stable) (push) Skipped
Tests / Test (Go stable) (pull_request) Failing after 3s
Tests / Secret scan (gitleaks) (pull_request) Successful in 4s
Tests / Test (Go 1.25) (pull_request) Successful in 1m30s

- TextDoc carries gator keyword fields (doc_sha, number, seller, total, typ,
  issued_at, created_at, status, url); content is now omitempty so a partial
  doc can be written.
- oo_docs_text mapping extended with the keyword fields.
- ESTextIndex.UpsertFields: bulk update + doc_as_upsert, merging gator fields
  into an existing text doc without clobbering title/content.
- SearchQuery gains Seller/Number/Status term filters; SearchHit.Fields exposes
  the gator values. Pure builders unit-tested.
This commit is contained in:
2026-10-07 08:20:29 +00:00
parent a14466ce7a
commit 3f7505892f
3 changed files with 179 additions and 14 deletions
+99 -12
View File
@@ -44,13 +44,24 @@ func ESTextConfigFromEnv() ESTextConfig {
}
// TextDoc is one document in the own full-text index. It is keyed by the
// OnlyOffice file id so hits map straight back to Documents entries.
// OnlyOffice file id so hits map straight back to Documents entries. The gator
// fields (Option D) are keyword values filled from the gator corpus; they stay
// empty for documents gator does not know.
type TextDoc struct {
ID string `json:"id"`
Title string `json:"title"`
FolderID string `json:"folder,omitempty"`
Ext string `json:"ext,omitempty"`
Content string `json:"content"`
ID string `json:"id"`
Title string `json:"title"`
FolderID string `json:"folder,omitempty"`
Ext string `json:"ext,omitempty"`
Content string `json:"content,omitempty"`
DocSHA string `json:"doc_sha,omitempty"`
Number string `json:"number,omitempty"`
Seller string `json:"seller,omitempty"`
Total string `json:"total,omitempty"`
Typ string `json:"typ,omitempty"`
IssuedAt string `json:"issued_at,omitempty"`
CreatedAt string `json:"created_at,omitempty"`
Status string `json:"status,omitempty"`
URL string `json:"url,omitempty"`
}
// TextIndex is the storage/search surface for locally extracted document text.
@@ -99,11 +110,20 @@ func (x *ESTextIndex) Index() string { return x.cfg.Index }
const esTextMapping = `{
"mappings": {
"properties": {
"id": {"type": "keyword"},
"title": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}},
"folder": {"type": "keyword"},
"ext": {"type": "keyword"},
"content": {"type": "text"}
"id": {"type": "keyword"},
"title": {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}},
"folder": {"type": "keyword"},
"ext": {"type": "keyword"},
"content": {"type": "text"},
"doc_sha": {"type": "keyword"},
"number": {"type": "keyword"},
"seller": {"type": "keyword"},
"total": {"type": "keyword"},
"typ": {"type": "keyword"},
"issued_at": {"type": "keyword"},
"created_at": {"type": "keyword"},
"status": {"type": "keyword"},
"url": {"type": "keyword"}
}
}
}`
@@ -147,6 +167,31 @@ func (x *ESTextIndex) Put(ctx context.Context, docs []TextDoc) error {
return fmt.Errorf("onlyoffice: bulk index %s: %s", x.cfg.Index, res.firstError())
}
// UpsertFields writes only the gator keyword fields for each document via the
// bulk update API with doc_as_upsert: an existing document keeps its
// title/content while gaining the gator fields; an unknown id is created with
// just these fields. Refreshes so results are immediately searchable.
func (x *ESTextIndex) UpsertFields(ctx context.Context, docs []TextDoc) error {
if len(docs) == 0 {
return nil
}
status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_bulk?refresh=true", esUpdateBulkBody(docs), "application/x-ndjson")
if err != nil {
return err
}
if status >= 400 {
return fmt.Errorf("onlyoffice: bulk update %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
}
var res esBulkResponse
if err := json.Unmarshal(raw, &res); err != nil {
return fmt.Errorf("onlyoffice: decode bulk response: %w", err)
}
if !res.Errors {
return nil
}
return fmt.Errorf("onlyoffice: bulk update %s: %s", x.cfg.Index, res.firstError())
}
// Delete removes documents by OnlyOffice file id. A missing index means there
// is nothing to delete.
func (x *ESTextIndex) Delete(ctx context.Context, ids []string) error {
@@ -233,6 +278,20 @@ func esTextBulkBody(docs []TextDoc) []byte {
return b.Bytes()
}
// esUpdateBulkBody renders bulk update actions with doc_as_upsert, so a partial
// TextDoc (gator fields only) merges into an existing document instead of
// replacing it. Pure, so it is unit-tested.
func esUpdateBulkBody(docs []TextDoc) []byte {
var b bytes.Buffer
enc := json.NewEncoder(&b)
enc.SetEscapeHTML(false)
for _, d := range docs {
_ = enc.Encode(map[string]any{"update": map[string]any{"_id": d.ID}})
_ = enc.Encode(map[string]any{"doc": d, "doc_as_upsert": true})
}
return b.Bytes()
}
// esTextSearchRequest builds the own-index query. Pure, so it is unit-tested.
func esTextSearchRequest(q SearchQuery) esRequest {
limit := q.Limit
@@ -252,10 +311,19 @@ func esTextSearchRequest(q SearchQuery) esRequest {
if exts := normalizeExtensions(q.Extensions); len(exts) > 0 {
filter = append(filter, esClause{Terms: map[string]any{"ext": exts}})
}
if v := strings.TrimSpace(q.Seller); v != "" {
filter = append(filter, esClause{Term: map[string]any{"seller": v}})
}
if v := strings.TrimSpace(q.Number); v != "" {
filter = append(filter, esClause{Term: map[string]any{"number": v}})
}
if v := strings.TrimSpace(q.Status); v != "" {
filter = append(filter, esClause{Term: map[string]any{"status": v}})
}
return esRequest{
Size: limit,
Source: []string{"id", "title", "folder", "ext"},
Source: []string{"id", "title", "folder", "ext", "doc_sha", "number", "seller", "total", "typ", "issued_at", "created_at", "status", "url"},
Query: esQuery{Bool: esBool{Must: must, Filter: filter}},
Highlight: esHighlight{PreTags: []string{"<em>"}, PostTags: []string{"</em>"}, Fields: map[string]struct{}{"title": {}, "content": {}}},
}
@@ -330,7 +398,26 @@ func parseESTextResponse(raw []byte) ([]SearchHit, error) {
Score: h.Score,
Highlight: esHighlightText(h.HL),
Path: path,
Fields: gatorFields(h.Source),
})
}
return hits, nil
}
// gatorFields extracts the non-empty gator keyword fields from a stored doc.
func gatorFields(d TextDoc) map[string]string {
out := map[string]string{}
for k, v := range map[string]string{
"doc_sha": d.DocSHA, "number": d.Number, "seller": d.Seller, "total": d.Total,
"typ": d.Typ, "issued_at": d.IssuedAt, "created_at": d.CreatedAt,
"status": d.Status, "url": d.URL,
} {
if v != "" {
out[k] = v
}
}
if len(out) == 0 {
return nil
}
return out
}