feat(search): PDF content via own ES index and oo index (#42)

2026-09-16 17:28:05 +00:00
parent 6d93ab5b1f
commit 3fe43ee82c
12 changed files with 1411 additions and 7 deletions
@@ -0,0 +1,158 @@
+package main
+
+import (
+	"strings"
+
+	onlyoffice "github.com/eslider/go-onlyoffice"
+	"github.com/spf13/cobra"
+)
+
+func init() {
+	rootCmd.AddCommand(indexCmd())
+}
+
+// indexFlags are shared by the `oo index folder` and `oo index files` verbs.
+type indexFlags struct {
+	recursive bool
+	exts      string
+	limit     int
+	workers   int
+	lang      string
+	minChars  int
+	workDir   string
+	backend   string
+	dryRun    bool
+	asJSON    bool
+}
+
+// indexCmd populates the own full-text index (oo_docs_text) that makes PDF and
+// scanned content searchable. The OnlyOffice index is left untouched.
+func indexCmd() *cobra.Command {
+	f := &indexFlags{}
+	cmd := &cobra.Command{
+		Use:   "index",
+		Short: "Populate the own full-text index for PDF/scan content",
+		Long: "Index document text that the OnlyOffice Elasticsearch index does not\n" +
+			"cover (PDFs and scans) into a separate index (ONLYOFFICE_ES_TEXT_INDEX,\n" +
+			"default oo_docs_text). Text is extracted with docpipe (pdftotext, OCR)\n" +
+			"and the OnlyOffice server is never modified.\n\n" +
+			"Requires ONLYOFFICE_URL/USER/PASS (to download files) and ONLYOFFICE_ES_URL\n" +
+			"(to write the index). See docs/elasticsearch.md.",
+	}
+	cmd.PersistentFlags().BoolVar(&f.recursive, "recursive", false, "folder: descend into subfolders")
+	cmd.PersistentFlags().StringVar(&f.exts, "exts", "pdf", "comma-separated extensions to index")
+	cmd.PersistentFlags().IntVar(&f.limit, "limit", 0, "maximum number of files to index (0 = all)")
+	cmd.PersistentFlags().IntVar(&f.workers, "workers", 3, "parallel downloads/extractions")
+	cmd.PersistentFlags().StringVar(&f.lang, "lang", "deu+eng", "OCR language(s)")
+	cmd.PersistentFlags().IntVar(&f.minChars, "min-chars", 0, "text-layer threshold below which OCR runs")
+	cmd.PersistentFlags().StringVar(&f.workDir, "work-dir", "", "temp dir for downloads (default: system temp)")
+	cmd.PersistentFlags().StringVar(&f.backend, "backend", "rest", "file backend: rest|dav")
+	cmd.PersistentFlags().BoolVar(&f.dryRun, "dry-run", false, "list what would be indexed, without changes")
+	cmd.PersistentFlags().BoolVar(&f.asJSON, "json", false, "shorthand for --output json")
+
+	cmd.AddCommand(indexFolderCmd(f), indexFilesCmd(f))
+	return cmd
+}
+
+func indexFolderCmd(f *indexFlags) *cobra.Command {
+	return &cobra.Command{
+		Use:   "folder FOLDER_ID",
+		Short: "Index every matching file in a Documents folder",
+		Args:  cobra.ExactArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runIndex(cmd, f, args[0], nil)
+		},
+	}
+}
+
+func indexFilesCmd(f *indexFlags) *cobra.Command {
+	return &cobra.Command{
+		Use:   "files FILE_ID...",
+		Short: "Index specific Documents files",
+		Args:  cobra.MinimumNArgs(1),
+		RunE: func(cmd *cobra.Command, args []string) error {
+			return runIndex(cmd, f, "", args)
+		},
+	}
+}
+
+func runIndex(cmd *cobra.Command, f *indexFlags, folderID string, ids []string) error {
+	if f.asJSON {
+		outputFormat = "json"
+	}
+	c, err := newOO(cmd)
+	if err != nil {
+		return err
+	}
+	idx, err := onlyoffice.NewESTextIndex(onlyoffice.ESTextConfigFromEnv())
+	if err != nil {
+		return err
+	}
+	ti := onlyoffice.NewTextIndexer(c.FileStore(f.backend), idx)
+	ti.WorkDir = f.workDir
+	opts := onlyoffice.IndexOptions{
+		Recursive:  f.recursive,
+		Extensions: splitList(f.exts),
+		Limit:      f.limit,
+		Lang:       f.lang,
+		MinChars:   f.minChars,
+		Workers:    f.workers,
+	}
+	ctx := cmd.Context()
+
+	if f.dryRun {
+		var entries []onlyoffice.Entry
+		if folderID != "" {
+			entries, err = ti.PlanFolder(ctx, folderID, opts)
+		} else {
+			entries, err = ti.PlanFiles(ctx, ids, opts)
+		}
+		if err != nil {
+			return err
+		}
+		rows := make([]map[string]any, 0, len(entries))
+		for _, e := range entries {
+			rows = append(rows, map[string]any{
+				"id":     e.ID,
+				"title":  e.Title,
+				"folder": e.ParentID,
+			})
+		}
+		printTable([]string{"id", "title", "folder"}, rows)
+		return nil
+	}
+
+	if err := ti.Ensure(ctx); err != nil {
+		return err
+	}
+	var res onlyoffice.IndexResult
+	if folderID != "" {
+		res, err = ti.IndexFolder(ctx, folderID, opts)
+	} else {
+		res, err = ti.IndexFiles(ctx, ids, opts)
+	}
+	if err != nil {
+		return err
+	}
+	printObject(map[string]any{
+		"index":   idx.Index(),
+		"scanned": res.Scanned,
+		"indexed": res.Indexed,
+		"skipped": res.Skipped,
+		"failed":  res.Failed,
+		"errors":  res.Errors,
+	})
+	return nil
+}
+
+// splitList parses a comma-separated flag value, dropping blanks.
+func splitList(s string) []string {
+	parts := strings.Split(s, ",")
+	out := make([]string, 0, len(parts))
+	for _, p := range parts {
+		if p = strings.TrimSpace(p); p != "" {
+			out = append(out, p)
+		}
+	}
+	return out
+}
@@ -0,0 +1,55 @@
+package main
+
+import (
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestIndexCommandRegistered(t *testing.T) {
+	cmd, _, err := rootCmd.Find([]string{"index"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cmd.Name() != "index" {
+		t.Fatalf("index resolved to %q", cmd.Name())
+	}
+	for _, name := range []string{"exts", "limit", "workers", "lang", "min-chars", "work-dir", "backend", "dry-run", "recursive", "json"} {
+		if cmd.PersistentFlags().Lookup(name) == nil {
+			t.Errorf("index: missing --%s flag", name)
+		}
+	}
+	if cmd.PersistentFlags().Lookup("exts").DefValue != "pdf" {
+		t.Errorf("--exts default = %q, want pdf", cmd.PersistentFlags().Lookup("exts").DefValue)
+	}
+	for _, verb := range []string{"index folder", "index files"} {
+		sub, _, err := rootCmd.Find(strings.Fields(verb))
+		if err != nil {
+			t.Fatalf("%s: %v", verb, err)
+		}
+		if sub.Name() != strings.Fields(verb)[1] {
+			t.Errorf("%s resolved to %q", verb, sub.Name())
+		}
+	}
+}
+
+func TestIndexSearchBackendFlag(t *testing.T) {
+	cmd, _, err := rootCmd.Find([]string{"search"})
+	if err != nil {
+		t.Fatal(err)
+	}
+	if cmd.Flags().Lookup("backend") == nil {
+		t.Fatal("search: missing --backend flag")
+	}
+	if cmd.Flags().Lookup("backend").DefValue != "oo" {
+		t.Errorf("--backend default = %q, want oo", cmd.Flags().Lookup("backend").DefValue)
+	}
+}
+
+func TestSplitList(t *testing.T) {
+	got := splitList(" pdf , .PDF, docx ,, ")
+	want := []string{"pdf", ".PDF", "docx"}
+	if !reflect.DeepEqual(got, want) {
+		t.Errorf("splitList = %v, want %v", got, want)
+	}
+}
@@ -18,7 +18,8 @@
 //	oo docs          tools | convert | optimize | ocr | hocr | as-md | put-md | put-txt | put-xlsx
 //	oo catalog       match | merge | apply | scan-contacts | scan-projects | scan-thunderbird
 //	oo dav           ls | move | copy | mkdir | rename-file | rename-folder | download | fileops
-//	oo search        QUERY [--content] [--folder ID] [--limit N] [--json]
+//	oo search        QUERY [--content] [--folder ID] [--limit N] [--backend oo|own] [--json]
+//	oo index         folder FOLDER_ID | files FILE_ID... [--recursive] [--exts pdf] [--dry-run]
 //
 // CRM association rules: docs/crm-associations.md
 //
@@ -1,6 +1,9 @@
 package main

 import (
+	"fmt"
+	"strings"
+
 	onlyoffice "github.com/eslider/go-onlyoffice"
 	"github.com/eslider/go-onlyoffice/cmd/internal/bootstrap"
 	"github.com/spf13/cobra"
@@ -18,6 +21,7 @@ func searchCmd() *cobra.Command {
 		content bool
 		folder  string
 		limit   int
+		backend string
 		asJSON  bool
 	)
 	cmd := &cobra.Command{
@@ -27,6 +31,9 @@ func searchCmd() *cobra.Command {
 			"By default only file names are matched. With --content the query also\n" +
 			"matches extracted document text (document.attachment.content); this covers\n" +
 			"Office formats (docx/xlsx/pptx) and is slower.\n\n" +
+			"--backend own queries the separate index populated by `oo index`\n" +
+			"(ONLYOFFICE_ES_TEXT_INDEX, default oo_docs_text) instead, which also holds\n" +
+			"PDFs and scans (see docs/elasticsearch.md).\n\n" +
 			"Requires ONLYOFFICE_ES_URL (and optionally ONLYOFFICE_ES_INDEX,\n" +
 			"ONLYOFFICE_TENANT). See docs/elasticsearch.md for the tunnel setup.",
 		Args: cobra.ExactArgs(1),
@@ -36,7 +43,18 @@ func searchCmd() *cobra.Command {
 			}
 			bootstrap.LoadEnv()
 			c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials())
-			searcher, err := c.Files().Search()
+			var (
+				searcher onlyoffice.Searcher
+				err      error
+			)
+			switch strings.ToLower(strings.TrimSpace(backend)) {
+			case "", "oo", "elasticsearch":
+				searcher, err = c.Files().Search()
+			case "own", "es-text":
+				searcher, err = onlyoffice.NewESTextIndex(onlyoffice.ESTextConfigFromEnv())
+			default:
+				return fmt.Errorf("unknown search backend %q (want oo|own)", backend)
+			}
 			if err != nil {
 				return err
 			}
@@ -70,6 +88,7 @@ func searchCmd() *cobra.Command {
 	cmd.Flags().BoolVar(&content, "content", false, "also match extracted document content")
 	cmd.Flags().StringVar(&folder, "folder", "", "limit to a Documents folder id")
 	cmd.Flags().IntVar(&limit, "limit", 20, "maximum number of results")
+	cmd.Flags().StringVar(&backend, "backend", "oo", "index to query: oo (OnlyOffice) | own (oo index)")
 	cmd.Flags().BoolVar(&asJSON, "json", false, "shorthand for --output json")
 	return cmd
 }