chore: move business tooling out of the public library; tidy filestore naming

Keep the public tree project-generic. Business/one-off tools, deployment and business docs move to the private oo-workspace repo. Moved to oo-workspace: - cmd/ooscan, cmd/pdfamount, cmd/kontoblatt, cmd/kontolink - internal/xlspipe (cutover-portugal workbook) -> oow workbook build (drops the --template/--title flags from oo docs put-xlsx) - deploy/docker-compose.rclone-webdav.yml + docs/rclone-webdav.md - docs/crm-associations.md Removed GitHub-era leftovers: - .github/workflows/release-please.yml, release-please-config.json, .release-please-manifest.json (tags are created on Gitea per SemVer) Naming: the FileStore subsystem is now filestore_*.go (was file_*.go) to match files_*.go (project Documents). Docs/AGENTS/README updated.
2026-09-23 09:05:40 +01:00
parent a9a77e110d
commit dc62570a5c
46 changed files with 69 additions and 2390 deletions
@@ -0,0 +1,336 @@
+package onlyoffice
+
+// Own full-text index (epic #34, F6 #42).
+//
+// The OnlyOffice Elasticsearch index (files_file) only holds extracted content
+// for Office formats. FileUtility.CanIndex gates extraction by the server
+// setting files.index.formats, whose default is ".pptx|.xlsx|.docx", so PDFs
+// are indexed by name only. Instead of patching the server (risky: lost on
+// upgrade, forces a full reindex) this file implements a second, independent
+// index (default oo_docs_text) that our own pipeline fills from
+// internal/docpipe (pdftotext + OCR). The OnlyOffice index is never touched.
+//
+// See docs/elasticsearch.md for the decision and the trade-offs.
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"strings"
+	"time"
+)
+
+const defaultESTextIndex = "oo_docs_text"
+
+// ESTextConfig configures the own full-text index.
+type ESTextConfig struct {
+	URL    string // scheme://host:port of the ES HTTP endpoint
+	Index  string // index name, default oo_docs_text
+	Tenant string // reserved for future multi-tenant data; unused for now
+}
+
+// ESTextConfigFromEnv reads ONLYOFFICE_ES_URL and ONLYOFFICE_ES_TEXT_INDEX
+// (default oo_docs_text). The library never loads dotfiles — the CLI does that.
+func ESTextConfigFromEnv() ESTextConfig {
+	return ESTextConfig{
+		URL:    strings.TrimRight(strings.TrimSpace(os.Getenv("ONLYOFFICE_ES_URL")), "/"),
+		Index:  firstNonEmpty(os.Getenv("ONLYOFFICE_ES_TEXT_INDEX"), defaultESTextIndex),
+		Tenant: strings.TrimSpace(os.Getenv("ONLYOFFICE_TENANT")),
+	}
+}
+
+// TextDoc is one document in the own full-text index. It is keyed by the
+// OnlyOffice file id so hits map straight back to Documents entries.
+type TextDoc struct {
+	ID       string `json:"id"`
+	Title    string `json:"title"`
+	FolderID string `json:"folder,omitempty"`
+	Ext      string `json:"ext,omitempty"`
+	Content  string `json:"content"`
+}
+
+// TextIndex is the storage/search surface for locally extracted document text.
+// It complements Searcher: ESSearcher reads OnlyOffice's index, ESTextIndex
+// reads ours.
+type TextIndex interface {
+	Put(ctx context.Context, docs []TextDoc) error
+	Delete(ctx context.Context, ids []string) error
+	Search(ctx context.Context, q SearchQuery) ([]SearchHit, error)
+	Name() string
+}
+
+// ESTextIndex is a TextIndex (and Searcher) over a dedicated Elasticsearch
+// index filled by TextIndexer.
+type ESTextIndex struct {
+	cfg  ESTextConfig
+	http *http.Client
+}
+
+var (
+	_ TextIndex = (*ESTextIndex)(nil)
+	_ Searcher  = (*ESTextIndex)(nil)
+)
+
+// NewESTextIndex returns a searcher/writer for the own full-text index. The URL
+// is required; an empty index falls back to oo_docs_text.
+func NewESTextIndex(cfg ESTextConfig) (*ESTextIndex, error) {
+	if strings.TrimSpace(cfg.URL) == "" {
+		return nil, fmt.Errorf("onlyoffice: elasticsearch URL is empty (set ONLYOFFICE_ES_URL)")
+	}
+	cfg.URL = strings.TrimRight(cfg.URL, "/")
+	if cfg.Index == "" {
+		cfg.Index = defaultESTextIndex
+	}
+	return &ESTextIndex{cfg: cfg, http: &http.Client{Timeout: 120 * time.Second}}, nil
+}
+
+// Name implements Searcher and TextIndex.
+func (x *ESTextIndex) Name() string { return "es-text" }
+
+// Index returns the configured index name.
+func (x *ESTextIndex) Index() string { return x.cfg.Index }
+
+// esTextMapping pins explicit types: content must stay a plain text field (no
+// keyword subfield) and folder/ext stay exact keywords for filters.
+const esTextMapping = `{
+  "mappings": {
+    "properties": {
+      "id":      {"type": "keyword"},
+      "title":   {"type": "text", "fields": {"keyword": {"type": "keyword", "ignore_above": 512}}},
+      "folder":  {"type": "keyword"},
+      "ext":     {"type": "keyword"},
+      "content": {"type": "text"}
+    }
+  }
+}`
+
+// Ensure creates the index with the explicit mapping. A missing index is
+// created; an already existing one is left untouched.
+func (x *ESTextIndex) Ensure(ctx context.Context) error {
+	status, raw, err := x.do(ctx, http.MethodPut, "/"+x.cfg.Index, []byte(esTextMapping), "application/json")
+	if err != nil {
+		return err
+	}
+	if status == http.StatusOK {
+		return nil
+	}
+	if status == http.StatusBadRequest && bytes.Contains(raw, []byte("resource_already_exists_exception")) {
+		return nil
+	}
+	return fmt.Errorf("onlyoffice: create text index %s: %d %s", x.cfg.Index, status, truncate(string(raw), 300))
+}
+
+// Put upserts documents via the bulk API and refreshes so they are immediately
+// searchable.
+func (x *ESTextIndex) Put(ctx context.Context, docs []TextDoc) error {
+	if len(docs) == 0 {
+		return nil
+	}
+	status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_bulk?refresh=true", esTextBulkBody(docs), "application/x-ndjson")
+	if err != nil {
+		return err
+	}
+	if status >= 400 {
+		return fmt.Errorf("onlyoffice: bulk index %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
+	}
+	var res esBulkResponse
+	if err := json.Unmarshal(raw, &res); err != nil {
+		return fmt.Errorf("onlyoffice: decode bulk response: %w", err)
+	}
+	if !res.Errors {
+		return nil
+	}
+	return fmt.Errorf("onlyoffice: bulk index %s: %s", x.cfg.Index, res.firstError())
+}
+
+// Delete removes documents by OnlyOffice file id. A missing index means there
+// is nothing to delete.
+func (x *ESTextIndex) Delete(ctx context.Context, ids []string) error {
+	if len(ids) == 0 {
+		return nil
+	}
+	body, err := json.Marshal(map[string]any{"query": map[string]any{"terms": map[string]any{"id": ids}}})
+	if err != nil {
+		return err
+	}
+	status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_delete_by_query?refresh=true", body, "application/json")
+	if err != nil {
+		return err
+	}
+	if status == http.StatusNotFound {
+		return nil
+	}
+	if status >= 400 {
+		return fmt.Errorf("onlyoffice: delete from %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
+	}
+	return nil
+}
+
+// Search runs a multi_match over title (boosted) and content, with optional
+// folder and extension filters. A missing index yields no hits, not an error.
+func (x *ESTextIndex) Search(ctx context.Context, q SearchQuery) ([]SearchHit, error) {
+	q.Text = strings.TrimSpace(q.Text)
+	if q.Text == "" {
+		return nil, fmt.Errorf("onlyoffice: empty search query")
+	}
+	body, err := json.Marshal(esTextSearchRequest(q))
+	if err != nil {
+		return nil, fmt.Errorf("onlyoffice: build elasticsearch query: %w", err)
+	}
+	status, raw, err := x.do(ctx, http.MethodPost, "/"+x.cfg.Index+"/_search", body, "application/json")
+	if err != nil {
+		return nil, err
+	}
+	if status == http.StatusNotFound {
+		return nil, nil
+	}
+	if status >= 400 {
+		return nil, fmt.Errorf("onlyoffice: search %s: %d %s", x.cfg.Index, status, truncate(string(raw), 400))
+	}
+	return parseESTextResponse(raw)
+}
+
+// do sends one request and returns the status and body (bounded). The caller
+// decides which statuses are errors.
+func (x *ESTextIndex) do(ctx context.Context, method, path string, body []byte, contentType string) (int, []byte, error) {
+	var r io.Reader
+	if body != nil {
+		r = bytes.NewReader(body)
+	}
+	req, err := http.NewRequestWithContext(ctx, method, x.cfg.URL+path, r)
+	if err != nil {
+		return 0, nil, err
+	}
+	req.Header.Set("Accept", "application/json")
+	if contentType != "" {
+		req.Header.Set("Content-Type", contentType)
+	}
+	resp, err := x.http.Do(req)
+	if err != nil {
+		return 0, nil, fmt.Errorf("onlyoffice: elasticsearch %s: %w", method, err)
+	}
+	defer resp.Body.Close()
+	raw, err := io.ReadAll(io.LimitReader(resp.Body, maxESResponseSize))
+	if err != nil {
+		return resp.StatusCode, nil, err
+	}
+	return resp.StatusCode, raw, nil
+}
+
+// esTextBulkBody renders the NDJSON bulk payload. Pure, so it is unit-tested.
+func esTextBulkBody(docs []TextDoc) []byte {
+	var b bytes.Buffer
+	enc := json.NewEncoder(&b)
+	enc.SetEscapeHTML(false)
+	for _, d := range docs {
+		_ = enc.Encode(map[string]any{"index": map[string]any{"_id": d.ID}})
+		_ = enc.Encode(d)
+	}
+	return b.Bytes()
+}
+
+// esTextSearchRequest builds the own-index query. Pure, so it is unit-tested.
+func esTextSearchRequest(q SearchQuery) esRequest {
+	limit := q.Limit
+	if limit <= 0 {
+		limit = defaultESLimit
+	}
+	if limit > maxESLimit {
+		limit = maxESLimit
+	}
+	fields := []string{"title^2", "content"}
+	must := []esClause{{MultiMatch: &esMultiMatch{Query: q.Text, Fields: fields}}}
+
+	var filter []esClause
+	if f := strings.TrimSpace(q.FolderID); f != "" {
+		filter = append(filter, esClause{Term: map[string]any{"folder": f}})
+	}
+	if exts := normalizeExtensions(q.Extensions); len(exts) > 0 {
+		filter = append(filter, esClause{Terms: map[string]any{"ext": exts}})
+	}
+
+	return esRequest{
+		Size:      limit,
+		Source:    []string{"id", "title", "folder", "ext"},
+		Query:     esQuery{Bool: esBool{Must: must, Filter: filter}},
+		Highlight: esHighlight{PreTags: []string{"<em>"}, PostTags: []string{"</em>"}, Fields: map[string]struct{}{"title": {}, "content": {}}},
+	}
+}
+
+// esBulkResponse is the subset of an ES bulk response we consume.
+type esBulkResponse struct {
+	Errors bool `json:"errors"`
+	Items  []map[string]struct {
+		ID     string `json:"_id"`
+		Status int    `json:"status"`
+		Error  *struct {
+			Type   string `json:"type"`
+			Reason string `json:"reason"`
+		} `json:"error"`
+	} `json:"items"`
+}
+
+// firstError returns a compact description of the first failed bulk item.
+func (r esBulkResponse) firstError() string {
+	for _, item := range r.Items {
+		for op, res := range item {
+			if res.Error != nil {
+				return fmt.Sprintf("%s %s: %s %s", op, res.ID, res.Error.Type, res.Error.Reason)
+			}
+		}
+	}
+	return "unknown bulk error"
+}
+
+// esTextResponse is the subset of an own-index search response we consume.
+type esTextResponse struct {
+	Hits struct {
+		Total struct {
+			Value int `json:"value"`
+		} `json:"total"`
+		Hits []struct {
+			ID     string              `json:"_id"`
+			Score  float64             `json:"_score"`
+			Source TextDoc             `json:"_source"`
+			HL     map[string][]string `json:"highlight"`
+		} `json:"hits"`
+	} `json:"hits"`
+}
+
+// parseESTextResponse converts an own-index search response into SearchHit
+// values. Pure, so it is unit-tested.
+func parseESTextResponse(raw []byte) ([]SearchHit, error) {
+	var r esTextResponse
+	if err := json.Unmarshal(raw, &r); err != nil {
+		return nil, fmt.Errorf("onlyoffice: decode elasticsearch response: %w", err)
+	}
+	hits := make([]SearchHit, 0, len(r.Hits.Hits))
+	for _, h := range r.Hits.Hits {
+		id := h.Source.ID
+		if id == "" {
+			id = h.ID
+		}
+		parent := h.Source.FolderID
+		var path []string
+		if parent != "" {
+			path = []string{parent}
+		}
+		hits = append(hits, SearchHit{
+			Entry: Entry{
+				ID:       id,
+				ParentID: parent,
+				Title:    h.Source.Title,
+				Kind:     File,
+				Provider: "es-text",
+			},
+			Score:     h.Score,
+			Highlight: esHighlightText(h.HL),
+			Path:      path,
+		})
+	}
+	return hits, nil
+}