fix(docpipe): index .yaml and extensionless PDF attachments (#47)

internal/docpipe/pdfattach.go

@@ -9,11 +9,14 @@ import (
 	"bytes"
 	"encoding/xml"
 	"fmt"
+	"io"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"strconv"
 	"strings"
+	"unicode"
+	"unicode/utf8"
 )
 
 // PDFAttachment is one embedded file in a PDF.
@@ -193,15 +196,59 @@ func (t Tools) attachmentMarkdown(path, workDir, lang string, minChars int) (str
 			return "", err
 		}
 		return xmlToText(raw), nil
-	case ".json", ".csv":
+	case ".json", ".csv", ".yaml", ".yml", ".toml", ".txt", ".md", ".markdown":
 		raw, err := os.ReadFile(path)
 		if err != nil {
 			return "", err
 		}
 		return string(raw), nil
 	default:
+		return textFallback(path)
+	}
+}
+
+// textFallback reads an attachment of an unknown or missing extension as plain
+// text when it looks textual (valid UTF-8, mostly printable runes). Binary
+// payloads (images, archives, NUL-padded blobs) are rejected with an error so
+// the caller skips them instead of poisoning the index. Classified digitised
+// PDFs (Scanner-*.ocr.pdf) carry .yaml/.md attachments; some exporters omit the
+// extension, which this covers.
+func textFallback(path string) (string, error) {
+	f, err := os.Open(path)
+	if err != nil {
+		return "", err
+	}
+	defer f.Close()
+	raw, err := io.ReadAll(io.LimitReader(f, 1<<20))
+	if err != nil {
+		return "", err
+	}
+	if !utf8.Valid(raw) {
 		return "", fmt.Errorf("unsupported attachment type %q", Ext(path))
 	}
+	if !mostlyPrintable(raw) {
+		return "", fmt.Errorf("unsupported attachment type %q", Ext(path))
+	}
+	return string(raw), nil
+}
+
+// mostlyPrintable reports whether at least 90% of the runes are printable text
+// (newlines, carriage returns and tabs count as text). Pure, so it is tested.
+func mostlyPrintable(b []byte) bool {
+	if len(b) == 0 {
+		return false
+	}
+	printable, total := 0, 0
+	for _, r := range string(b) {
+		if r == utf8.RuneError {
+			continue
+		}
+		total++
+		if unicode.IsPrint(r) || r == '\n' || r == '\r' || r == '\t' {
+			printable++
+		}
+	}
+	return total > 0 && printable*10 >= total*9
 }
 
 // xmlToText returns the character data of an XML/HTML document: element text

internal/docpipe/pdfattach_test.go

@@ -135,6 +135,32 @@ func TestAttachmentMarkdownFallback(t *testing.T) {
 		t.Errorf("xml attachment text = %q, want S1063", got)
 	}
 
+	// Classified digitised PDFs (Scanner-*.ocr.pdf) carry .yaml metadata.
+	yamlPath := filepath.Join(dir, "Scanner-123-003.ocr.yaml")
+	if err := os.WriteFile(yamlPath, []byte("document:\n  type: Rechnung\nnumber: S1063\n"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	gotYAML, err := (Tools{}).attachmentMarkdown(yamlPath, dir, "", 0)
+	if err != nil {
+		t.Fatalf("attachmentMarkdown(yaml): %v", err)
+	}
+	if !strings.Contains(gotYAML, "S1063") {
+		t.Errorf("yaml attachment text = %q, want S1063", gotYAML)
+	}
+
+	// Extensionless textual attachment falls back to raw text.
+	noExt := filepath.Join(dir, "attachment")
+	if err := os.WriteFile(noExt, []byte("plain attachment token goonoext"), 0o644); err != nil {
+		t.Fatal(err)
+	}
+	gotNoExt, err := (Tools{}).attachmentMarkdown(noExt, dir, "", 0)
+	if err != nil {
+		t.Fatalf("attachmentMarkdown(no extension): %v", err)
+	}
+	if !strings.Contains(gotNoExt, "goonoext") {
+		t.Errorf("extensionless attachment text = %q, want goonoext", gotNoExt)
+	}
+
 	binPath := filepath.Join(dir, "data.bin")
 	if err := os.WriteFile(binPath, []byte{0, 1, 2, 3}, 0o644); err != nil {
 		t.Fatal(err)