diff --git a/internal/docpipe/pdfattach.go b/internal/docpipe/pdfattach.go index 11fd414..0808118 100644 --- a/internal/docpipe/pdfattach.go +++ b/internal/docpipe/pdfattach.go @@ -9,11 +9,14 @@ import ( "bytes" "encoding/xml" "fmt" + "io" "os" "os/exec" "path/filepath" "strconv" "strings" + "unicode" + "unicode/utf8" ) // PDFAttachment is one embedded file in a PDF. @@ -193,15 +196,59 @@ func (t Tools) attachmentMarkdown(path, workDir, lang string, minChars int) (str return "", err } return xmlToText(raw), nil - case ".json", ".csv": + case ".json", ".csv", ".yaml", ".yml", ".toml", ".txt", ".md", ".markdown": raw, err := os.ReadFile(path) if err != nil { return "", err } return string(raw), nil default: + return textFallback(path) + } +} + +// textFallback reads an attachment of an unknown or missing extension as plain +// text when it looks textual (valid UTF-8, mostly printable runes). Binary +// payloads (images, archives, NUL-padded blobs) are rejected with an error so +// the caller skips them instead of poisoning the index. Classified digitised +// PDFs (Scanner-*.ocr.pdf) carry .yaml/.md attachments; some exporters omit the +// extension, which this covers. +func textFallback(path string) (string, error) { + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + raw, err := io.ReadAll(io.LimitReader(f, 1<<20)) + if err != nil { + return "", err + } + if !utf8.Valid(raw) { return "", fmt.Errorf("unsupported attachment type %q", Ext(path)) } + if !mostlyPrintable(raw) { + return "", fmt.Errorf("unsupported attachment type %q", Ext(path)) + } + return string(raw), nil +} + +// mostlyPrintable reports whether at least 90% of the runes are printable text +// (newlines, carriage returns and tabs count as text). Pure, so it is tested. +func mostlyPrintable(b []byte) bool { + if len(b) == 0 { + return false + } + printable, total := 0, 0 + for _, r := range string(b) { + if r == utf8.RuneError { + continue + } + total++ + if unicode.IsPrint(r) || r == '\n' || r == '\r' || r == '\t' { + printable++ + } + } + return total > 0 && printable*10 >= total*9 } // xmlToText returns the character data of an XML/HTML document: element text diff --git a/internal/docpipe/pdfattach_test.go b/internal/docpipe/pdfattach_test.go index ad8ab6e..d2a3665 100644 --- a/internal/docpipe/pdfattach_test.go +++ b/internal/docpipe/pdfattach_test.go @@ -135,6 +135,32 @@ func TestAttachmentMarkdownFallback(t *testing.T) { t.Errorf("xml attachment text = %q, want S1063", got) } + // Classified digitised PDFs (Scanner-*.ocr.pdf) carry .yaml metadata. + yamlPath := filepath.Join(dir, "Scanner-123-003.ocr.yaml") + if err := os.WriteFile(yamlPath, []byte("document:\n type: Rechnung\nnumber: S1063\n"), 0o644); err != nil { + t.Fatal(err) + } + gotYAML, err := (Tools{}).attachmentMarkdown(yamlPath, dir, "", 0) + if err != nil { + t.Fatalf("attachmentMarkdown(yaml): %v", err) + } + if !strings.Contains(gotYAML, "S1063") { + t.Errorf("yaml attachment text = %q, want S1063", gotYAML) + } + + // Extensionless textual attachment falls back to raw text. + noExt := filepath.Join(dir, "attachment") + if err := os.WriteFile(noExt, []byte("plain attachment token goonoext"), 0o644); err != nil { + t.Fatal(err) + } + gotNoExt, err := (Tools{}).attachmentMarkdown(noExt, dir, "", 0) + if err != nil { + t.Fatalf("attachmentMarkdown(no extension): %v", err) + } + if !strings.Contains(gotNoExt, "goonoext") { + t.Errorf("extensionless attachment text = %q, want goonoext", gotNoExt) + } + binPath := filepath.Join(dir, "data.bin") if err := os.WriteFile(binPath, []byte{0, 1, 2, 3}, 0o644); err != nil { t.Fatal(err)