Merge branch 'main' into feat/oo-automation#152
Release Please / Release Please (push) Skipped
Release / GoReleaser (push) Skipped
Tests / Secret scan (gitleaks) (push) Skipped
Tests / Test (Go 1.25) (push) Skipped
Tests / Test (Go stable) (push) Skipped
Tests / Secret scan (gitleaks) (pull_request) Successful in 4s
Tests / Test (Go stable) (pull_request) Successful in 1m49s
Tests / Test (Go 1.25) (pull_request) Successful in 2m20s
Release Please / Release Please (push) Skipped
Release / GoReleaser (push) Skipped
Tests / Secret scan (gitleaks) (push) Skipped
Tests / Test (Go 1.25) (push) Skipped
Tests / Test (Go stable) (push) Skipped
Tests / Secret scan (gitleaks) (pull_request) Successful in 4s
Tests / Test (Go stable) (pull_request) Successful in 1m49s
Tests / Test (Go 1.25) (pull_request) Successful in 2m20s
This commit is contained in:
@@ -25,3 +25,12 @@ ONLYOFFICE_PROJECT_ID=33
|
||||
# cmd/office TUI — optional Document Server for DOCX→HTML preview:
|
||||
# ONLYOFFICE_DOCS_URL=https://docs.example.com
|
||||
# ONLYOFFICE_DOCS_SECRET=
|
||||
|
||||
# MinIO download fallback for the portal's stale AWS S3 consumer (older
|
||||
# Documents folders). When the portal redirects to amazonaws.com with access
|
||||
# key "minio" (403 InvalidAccessKeyId), files are fetched from the local MinIO
|
||||
# store instead. Without a key/secret the fallback is disabled.
|
||||
# MINIO_ENDPOINT=http://192.168.188.10:9000
|
||||
# MINIO_BUCKET=office
|
||||
# MINIO_ACCESS_KEY=
|
||||
# MINIO_SECRET_KEY=
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
// Command kontoblatt builds a summary ("сводная таблица") of a Kontoblatt XLSX
|
||||
// (Datum, Gegenkonto, Buchungstext, Beleg, Soll, Haben, Bemerkung) and uploads
|
||||
// it back to the same OnlyOffice folder as the source file.
|
||||
//
|
||||
// Usage: kontoblatt <FILE_ID> <LOCAL_XLSX>
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"regexp"
|
||||
"sort"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
onlyoffice "github.com/eslider/go-onlyoffice"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
type agg struct {
|
||||
count int
|
||||
soll float64
|
||||
haben float64
|
||||
reFehlt int
|
||||
}
|
||||
|
||||
type rec struct {
|
||||
date, month, konto, text string
|
||||
soll, haben float64
|
||||
reFehlt bool
|
||||
}
|
||||
|
||||
var dateRe = regexp.MustCompile(`^\d{2}\.\d{2}\.\d{4}$`)
|
||||
|
||||
func parseAmount(s string) float64 {
|
||||
s = strings.TrimSpace(s)
|
||||
s = strings.ReplaceAll(s, "€", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
s = strings.ReplaceAll(s, ",", "") // German thousands separator
|
||||
s = strings.TrimSpace(s)
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
func cell(row []string, i int) string {
|
||||
if i < len(row) {
|
||||
return strings.TrimSpace(row[i])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 3 {
|
||||
fmt.Fprintln(os.Stderr, "usage: kontoblatt <FILE_ID> <LOCAL_XLSX>")
|
||||
os.Exit(2)
|
||||
}
|
||||
fileID, path := os.Args[1], os.Args[2]
|
||||
ctx := context.Background()
|
||||
|
||||
f, err := excelize.OpenFile(path)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
var recs []rec
|
||||
for _, sh := range f.GetSheetList() {
|
||||
rows, err := f.GetRows(sh)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
for _, r := range rows {
|
||||
d := cell(r, 0)
|
||||
if !dateRe.MatchString(d) {
|
||||
continue
|
||||
}
|
||||
text := cell(r, 2)
|
||||
recs = append(recs, rec{
|
||||
date: d,
|
||||
month: d[3:10],
|
||||
konto: cell(r, 1),
|
||||
text: text,
|
||||
soll: parseAmount(cell(r, 4)),
|
||||
haben: parseAmount(cell(r, 5)),
|
||||
reFehlt: strings.Contains(strings.ToUpper(text), "FEHLT"),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
byKonto := map[string]*agg{}
|
||||
byMonth := map[string]*agg{}
|
||||
getK := func(k string) *agg {
|
||||
if byKonto[k] == nil {
|
||||
byKonto[k] = &agg{}
|
||||
}
|
||||
return byKonto[k]
|
||||
}
|
||||
getM := func(k string) *agg {
|
||||
if byMonth[k] == nil {
|
||||
byMonth[k] = &agg{}
|
||||
}
|
||||
return byMonth[k]
|
||||
}
|
||||
var tot agg
|
||||
for _, r := range recs {
|
||||
k := getK(r.konto)
|
||||
k.count++
|
||||
k.soll += r.soll
|
||||
k.haben += r.haben
|
||||
if r.reFehlt {
|
||||
k.reFehlt++
|
||||
}
|
||||
m := getM(r.month)
|
||||
m.count++
|
||||
m.soll += r.soll
|
||||
m.haben += r.haben
|
||||
if r.reFehlt {
|
||||
m.reFehlt++
|
||||
}
|
||||
tot.count++
|
||||
tot.soll += r.soll
|
||||
tot.haben += r.haben
|
||||
if r.reFehlt {
|
||||
tot.reFehlt++
|
||||
}
|
||||
}
|
||||
|
||||
out := excelize.NewFile()
|
||||
defer out.Close()
|
||||
writeSheet(out, "Nach Gegenkonto", "Gegenkonto", byKonto, tot)
|
||||
writeSheet(out, "Nach Monat", "Monat", byMonth, tot)
|
||||
outPath := "/tmp/opencode/kontoblatt-zusammenfassung.xlsx"
|
||||
if err := out.SaveAs(outPath); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// upload next to the source file
|
||||
creds := onlyoffice.GetEnvironmentCredentials()
|
||||
c := onlyoffice.NewClient(creds)
|
||||
var src *onlyoffice.FileEntry
|
||||
if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
var err error
|
||||
src, err = c.GetFile(ctx, fileID)
|
||||
return err
|
||||
}); derr != nil {
|
||||
panic(derr)
|
||||
}
|
||||
folder := ""
|
||||
if src.FolderID != nil {
|
||||
folder = src.FolderID.String()
|
||||
}
|
||||
title := ""
|
||||
if src.Title != nil {
|
||||
title = *src.Title
|
||||
}
|
||||
fmt.Printf("source: id=%s title=%q folder=%s\n", fileID, title, folder)
|
||||
|
||||
name := "Kontoblatt-1591-2025-Zusammenfassung.xlsx"
|
||||
tmp := "/tmp/opencode/" + name
|
||||
data, _ := os.ReadFile(outPath)
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var entry *onlyoffice.FileEntry
|
||||
if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
var err error
|
||||
entry, _, err = c.UploadToFolderReplacing(ctx, folder, tmp)
|
||||
return err
|
||||
}); derr != nil {
|
||||
panic(derr)
|
||||
}
|
||||
fmt.Printf("uploaded: %s -> folder %s (id %v)\n", name, folder, entry.ID)
|
||||
|
||||
// print the summary
|
||||
printAgg("Nach Gegenkonto", byKonto, tot)
|
||||
printAgg("Nach Monat", byMonth, tot)
|
||||
}
|
||||
|
||||
func writeSheet(f *excelize.File, sheet, key string, m map[string]*agg, tot agg) {
|
||||
f.NewSheet(sheet)
|
||||
rows := [][]any{{key, "Anzahl", "Soll", "Haben", "Saldo", `davon "fehlt"`}}
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
for _, k := range keys {
|
||||
a := m[k]
|
||||
rows = append(rows, []any{k, a.count, a.soll, a.haben, a.soll - a.haben, a.reFehlt})
|
||||
}
|
||||
rows = append(rows, []any{"GESAMT", tot.count, tot.soll, tot.haben, tot.soll - tot.haben, tot.reFehlt})
|
||||
for i, row := range rows {
|
||||
for j, v := range row {
|
||||
cellRef, _ := excelize.CoordinatesToCellName(j+1, i+1)
|
||||
_ = f.SetCellValue(sheet, cellRef, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func printAgg(title string, m map[string]*agg, tot agg) {
|
||||
fmt.Printf("\n== %s ==\n", title)
|
||||
keys := make([]string, 0, len(m))
|
||||
for k := range m {
|
||||
keys = append(keys, k)
|
||||
}
|
||||
sort.Strings(keys)
|
||||
fmt.Printf("%-12s %6s %12s %12s %12s %7s\n", "key", "count", "soll", "haben", "saldo", "fehlt")
|
||||
for _, k := range keys {
|
||||
a := m[k]
|
||||
fmt.Printf("%-12s %6d %12.2f %12.2f %12.2f %7d\n", k, a.count, a.soll, a.haben, a.soll-a.haben, a.reFehlt)
|
||||
}
|
||||
fmt.Printf("%-12s %6d %12.2f %12.2f %12.2f %7d\n", "GESAMT", tot.count, tot.soll, tot.haben, tot.soll-tot.haben, tot.reFehlt)
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
// Command kontolink fills the "Link" column of a Kontoblatt ("ungeklärte
|
||||
// Posten") XLSX by matching each row to an OnlyOffice document.
|
||||
//
|
||||
// Strategy (deterministic, conservative — no LLM):
|
||||
// 1. Beleg token (letters/digits from the "Beleg" column) appears in the file
|
||||
// title; among candidates prefer (a) the row's month, (b) real invoices over
|
||||
// copies/dupes, and require the result to be unique;
|
||||
// 2. else supplier + row month + "rechnung", again unique.
|
||||
//
|
||||
// A file is linked at most once (rows already carrying a link are kept and their
|
||||
// file counts as used). Ambiguous rows are left UNLINKED for manual review.
|
||||
//
|
||||
// Usage: kontolink <IN_XLSX> <INDEX_TSV> <OUT_XLSX>
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
onlyoffice "github.com/eslider/go-onlyoffice"
|
||||
"github.com/xuri/excelize/v2"
|
||||
)
|
||||
|
||||
var (
|
||||
dateRe = regexp.MustCompile(`^\d{2}\.\d{2}\.\d{4}$`)
|
||||
nonAln = regexp.MustCompile(`[^0-9a-z]+`)
|
||||
fileID = regexp.MustCompile(`fileid=(\d+)`)
|
||||
)
|
||||
|
||||
func parseDay(s string) (time.Time, bool) {
|
||||
t, err := time.Parse("02.01.2006", strings.TrimSpace(s))
|
||||
return t, err == nil
|
||||
}
|
||||
|
||||
func titleDay(title string) (time.Time, bool) {
|
||||
if len(title) >= 10 {
|
||||
if t, err := time.Parse("2006-01-02", title[:10]); err == nil {
|
||||
return t, true
|
||||
}
|
||||
}
|
||||
return time.Time{}, false
|
||||
}
|
||||
|
||||
// nearest picks the candidate whose title date is closest to rd. Ties and
|
||||
// undated candidates (when >1) are rejected.
|
||||
func nearest(cands []entry, rd time.Time) (entry, bool) {
|
||||
if len(cands) == 1 {
|
||||
return cands[0], true
|
||||
}
|
||||
best, bestD, tie := -1, 0.0, false
|
||||
for i, e := range cands {
|
||||
td, ok := titleDay(e.title)
|
||||
if !ok {
|
||||
continue
|
||||
}
|
||||
d := td.Sub(rd).Hours() / 24
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if best < 0 || d < bestD {
|
||||
best, bestD, tie = i, d, false
|
||||
} else if d == bestD {
|
||||
tie = true
|
||||
}
|
||||
}
|
||||
if best < 0 || tie {
|
||||
return entry{}, false
|
||||
}
|
||||
return cands[best], true
|
||||
}
|
||||
|
||||
const linkPrefix = "https://office.pro-dukt.de/Products/Files/DocEditor.aspx?fileid="
|
||||
|
||||
type entry struct {
|
||||
id, path, title, norm string
|
||||
}
|
||||
|
||||
func norm(s string) string { return nonAln.ReplaceAllString(strings.ToLower(s), "") }
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 4 {
|
||||
fmt.Fprintln(os.Stderr, "usage: kontolink <IN_XLSX> <INDEX_TSV> <OUT_XLSX>")
|
||||
os.Exit(2)
|
||||
}
|
||||
in, idxPath, out := os.Args[1], os.Args[2], os.Args[3]
|
||||
|
||||
idxRaw, err := os.ReadFile(idxPath)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
var entries []entry
|
||||
for _, line := range strings.Split(string(idxRaw), "\n") {
|
||||
parts := strings.Split(line, "\t")
|
||||
if len(parts) < 4 || parts[0] == "" {
|
||||
continue
|
||||
}
|
||||
entries = append(entries, entry{id: parts[0], path: parts[2], title: parts[3], norm: norm(parts[3])})
|
||||
}
|
||||
|
||||
f, err := excelize.OpenFile(in)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer f.Close()
|
||||
sheet := f.GetSheetList()[0]
|
||||
rows, err := f.GetRows(sheet)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
|
||||
// optional 5th arg: amounts TSV "file_id\ttitle\tamount" (see cmd/pdfamount)
|
||||
var amts []amtEntry
|
||||
if len(os.Args) >= 6 && os.Args[5] != "" {
|
||||
amts = loadAmounts(os.Args[5])
|
||||
}
|
||||
|
||||
used := map[string]bool{}
|
||||
for _, r := range rows {
|
||||
if m := fileID.FindStringSubmatch(cell(r, 7)); m != nil {
|
||||
used[m[1]] = true
|
||||
}
|
||||
}
|
||||
|
||||
var linked, byBeleg, bySupplier, byAmount, unmatched, ambiguous int
|
||||
for i, r := range rows {
|
||||
if i == 0 || !dateRe.MatchString(cell(r, 0)) || strings.TrimSpace(cell(r, 7)) != "" {
|
||||
continue
|
||||
}
|
||||
beleg := norm(cell(r, 3))
|
||||
supplier := supplierNorm(cell(r, 2))
|
||||
month := monthYear(cell(r, 0))
|
||||
rd, _ := parseDay(cell(r, 0))
|
||||
|
||||
e, kind, ok := pick(entries, used, beleg, supplier, month, rd)
|
||||
if !ok {
|
||||
if ae, aok := amountPick(amts, used, supplier, rowAmount(r), rd); aok {
|
||||
e, kind, ok = entry{id: ae.id, title: ae.title}, "amount", true
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
if beleg != "" {
|
||||
ambiguous++
|
||||
} else {
|
||||
unmatched++
|
||||
}
|
||||
continue
|
||||
}
|
||||
ref, _ := excelize.CoordinatesToCellName(8, i+1)
|
||||
if err := f.SetCellValue(sheet, ref, linkPrefix+e.id); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
used[e.id] = true
|
||||
linked++
|
||||
switch kind {
|
||||
case "beleg":
|
||||
byBeleg++
|
||||
case "supplier":
|
||||
bySupplier++
|
||||
case "amount":
|
||||
byAmount++
|
||||
}
|
||||
fmt.Printf("row %3d %-30s -> %s [%s]\n", i+1, cell(r, 2), e.title, kind)
|
||||
}
|
||||
|
||||
if err := f.SaveAs(out); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
fmt.Printf("\nlinked=%d (beleg=%d, supplier=%d, amount=%d), ambiguous=%d, no-candidate=%d\n",
|
||||
linked, byBeleg, bySupplier, byAmount, ambiguous, unmatched)
|
||||
|
||||
// Optional 4th arg: source OnlyOffice file id. Try to update it in place;
|
||||
// if it is locked (OnlyOffice 500), upload a "(links)" copy next to it.
|
||||
if len(os.Args) >= 5 && os.Args[4] != "" {
|
||||
c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials())
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second)
|
||||
defer cancel()
|
||||
var src *onlyoffice.FileEntry
|
||||
if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
var err error
|
||||
src, err = c.GetFile(ctx, os.Args[4])
|
||||
return err
|
||||
}); derr != nil {
|
||||
panic(derr)
|
||||
}
|
||||
folder, title := "", ""
|
||||
if src.FolderID != nil {
|
||||
folder = src.FolderID.String()
|
||||
}
|
||||
if src.Title != nil {
|
||||
title = *src.Title
|
||||
}
|
||||
uderr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
_, err := c.UpdateFile(ctx, os.Args[4], out)
|
||||
return err
|
||||
})
|
||||
if uderr == nil {
|
||||
fmt.Printf("updated file %s in place\n", os.Args[4])
|
||||
return
|
||||
}
|
||||
fmt.Printf("in-place update failed (locked?); uploading a copy to folder %s\n", folder)
|
||||
ext := filepath.Ext(title)
|
||||
name := strings.TrimSuffix(title, ext) + " (links)" + ext
|
||||
tmp := filepath.Join(os.TempDir(), name)
|
||||
data, _ := os.ReadFile(out)
|
||||
if err := os.WriteFile(tmp, data, 0o600); err != nil {
|
||||
panic(err)
|
||||
}
|
||||
if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
_, _, err := c.UploadToFolderReplacing(ctx, folder, tmp)
|
||||
return err
|
||||
}); derr != nil {
|
||||
panic(derr)
|
||||
}
|
||||
fmt.Printf("uploaded copy: %s -> folder %s\n", name, folder)
|
||||
}
|
||||
}
|
||||
|
||||
func cell(r []string, i int) string {
|
||||
if i < len(r) {
|
||||
return strings.TrimSpace(r[i])
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
func supplierNorm(s string) string {
|
||||
s = strings.ToUpper(s)
|
||||
if i := strings.Index(s, ","); i >= 0 {
|
||||
s = s[:i]
|
||||
}
|
||||
for _, w := range []string{"RE FEHLT", "GS FEHLT", "WOFR", "WOFÜR"} {
|
||||
s = strings.ReplaceAll(s, w, "")
|
||||
}
|
||||
return norm(s)
|
||||
}
|
||||
|
||||
func monthYear(date string) string {
|
||||
if len(date) == 10 {
|
||||
return date[6:10] + "-" + date[3:5]
|
||||
}
|
||||
return ""
|
||||
}
|
||||
|
||||
// pick returns an unused candidate. Beleg match wins; supplier+month is a
|
||||
// fallback. When several candidates qualify, the one closest in time to the row
|
||||
// date wins; a tie is rejected (ambiguous) rather than guessed.
|
||||
func pick(entries []entry, used map[string]bool, beleg, supplier, month string, rd time.Time) (entry, string, bool) {
|
||||
free := func(e entry) bool { return !used[e.id] }
|
||||
|
||||
if len(beleg) >= 5 {
|
||||
var inMonth []entry
|
||||
for _, e := range entries {
|
||||
if free(e) && belegMatches(e.norm, beleg) &&
|
||||
(month == "" || strings.Contains(e.title, month)) {
|
||||
inMonth = append(inMonth, e)
|
||||
}
|
||||
}
|
||||
if supplier != "" {
|
||||
var s []entry
|
||||
for _, e := range inMonth {
|
||||
if strings.Contains(e.norm, supplier) {
|
||||
s = append(s, e)
|
||||
}
|
||||
}
|
||||
if len(s) > 0 {
|
||||
inMonth = s
|
||||
}
|
||||
}
|
||||
inMonth = topRank(inMonth)
|
||||
if e, ok := nearest(inMonth, rd); ok {
|
||||
return e, "beleg", true
|
||||
}
|
||||
// A Beleg is present but no file carries it: do NOT fall back to a
|
||||
// supplier guess (that links the wrong invoice).
|
||||
return entry{}, "", false
|
||||
}
|
||||
|
||||
if supplier != "" && month != "" {
|
||||
var c []entry
|
||||
for _, e := range entries {
|
||||
if free(e) && strings.Contains(e.norm, supplier) &&
|
||||
strings.Contains(e.title, month) && strings.Contains(e.norm, "rechnung") {
|
||||
c = append(c, e)
|
||||
}
|
||||
}
|
||||
c = topRank(c)
|
||||
if e, ok := nearest(c, rd); ok {
|
||||
return e, "supplier", true
|
||||
}
|
||||
}
|
||||
return entry{}, "", false
|
||||
}
|
||||
|
||||
// topRank keeps only the highest-ranked candidates (real invoice over copy /
|
||||
// dupe / op), so a tie with a duplicate does not mask the real file.
|
||||
func topRank(cands []entry) []entry {
|
||||
if len(cands) < 2 {
|
||||
return cands
|
||||
}
|
||||
best := 0
|
||||
for _, e := range cands {
|
||||
if rank(e) > best {
|
||||
best = rank(e)
|
||||
}
|
||||
}
|
||||
out := cands[:0]
|
||||
for _, e := range cands {
|
||||
if rank(e) == best {
|
||||
out = append(out, e)
|
||||
}
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rank(e entry) int {
|
||||
s := 0
|
||||
if strings.Contains(e.path, "/2025") || strings.Contains(e.path, "/2024") {
|
||||
s += 4
|
||||
}
|
||||
if strings.Contains(e.norm, "rechnung") {
|
||||
s += 2
|
||||
}
|
||||
if strings.Contains(e.norm, "dupe") || strings.Contains(e.norm, "copy") ||
|
||||
strings.Contains(e.norm, "op") {
|
||||
s--
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
// belegMatches reports whether a Beleg identifies the file: the whole normalized
|
||||
// Beleg appears, or (for long numeric Belege, e.g. "24/641393110") an 8-digit
|
||||
// window of its longest digit run appears.
|
||||
func belegMatches(titleNorm, beleg string) bool {
|
||||
if strings.Contains(titleNorm, beleg) {
|
||||
return true
|
||||
}
|
||||
run := longestDigitRun(beleg)
|
||||
for i := 0; i+8 <= len(run); i++ {
|
||||
if strings.Contains(titleNorm, run[i:i+8]) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func longestDigitRun(s string) string {
|
||||
var best, cur strings.Builder
|
||||
for _, r := range s {
|
||||
if r >= '0' && r <= '9' {
|
||||
cur.WriteRune(r)
|
||||
if cur.Len() > best.Len() {
|
||||
best.Reset()
|
||||
best.WriteString(cur.String())
|
||||
}
|
||||
} else {
|
||||
cur.Reset()
|
||||
}
|
||||
}
|
||||
return best.String()
|
||||
}
|
||||
|
||||
type amtEntry struct {
|
||||
id string
|
||||
title string
|
||||
norm string
|
||||
amount float64
|
||||
date time.Time
|
||||
hasDate bool
|
||||
}
|
||||
|
||||
func loadAmounts(path string) []amtEntry {
|
||||
raw, err := os.ReadFile(path)
|
||||
if err != nil {
|
||||
return nil
|
||||
}
|
||||
var out []amtEntry
|
||||
for _, line := range strings.Split(string(raw), "\n") {
|
||||
p := strings.Split(line, "\t")
|
||||
if len(p) < 3 {
|
||||
continue
|
||||
}
|
||||
v, err := strconv.ParseFloat(strings.TrimSpace(p[2]), 64)
|
||||
if err != nil {
|
||||
continue
|
||||
}
|
||||
e := amtEntry{id: p[0], title: p[1], norm: norm(p[1]), amount: v}
|
||||
if len(p[1]) >= 10 {
|
||||
if t, err := time.Parse("2006-01-02", p[1][:10]); err == nil {
|
||||
e.date, e.hasDate = t, true
|
||||
}
|
||||
}
|
||||
out = append(out, e)
|
||||
}
|
||||
return out
|
||||
}
|
||||
|
||||
func rowAmount(r []string) float64 {
|
||||
if v := parseAmount(cell(r, 4)); v != 0 {
|
||||
return v
|
||||
}
|
||||
return parseAmount(cell(r, 5))
|
||||
}
|
||||
|
||||
func parseAmount(s string) float64 {
|
||||
s = strings.ReplaceAll(s, "€", "")
|
||||
s = strings.ReplaceAll(s, " ", "")
|
||||
s = strings.ReplaceAll(s, ",", ".")
|
||||
if s == "" {
|
||||
return 0
|
||||
}
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return v
|
||||
}
|
||||
|
||||
// amountPick matches a row to an O2 invoice by amount + nearest date. Scoped to
|
||||
// Telefonica/O2 rows and O2 files, so it cannot cross-link other suppliers.
|
||||
func amountPick(amts []amtEntry, used map[string]bool, supplier string, amt float64, rd time.Time) (amtEntry, bool) {
|
||||
if amt <= 0 || len(amts) == 0 {
|
||||
return amtEntry{}, false
|
||||
}
|
||||
if !strings.Contains(supplier, "telefonica") && !strings.Contains(supplier, "o2") {
|
||||
return amtEntry{}, false
|
||||
}
|
||||
var cands []amtEntry
|
||||
for _, a := range amts {
|
||||
if used[a.id] || !strings.Contains(a.norm, "o2") {
|
||||
continue
|
||||
}
|
||||
d := a.amount - amt
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if d > 0.005 {
|
||||
continue
|
||||
}
|
||||
if a.hasDate && !rd.IsZero() {
|
||||
days := a.date.Sub(rd).Hours() / 24
|
||||
if days < 0 {
|
||||
days = -days
|
||||
}
|
||||
if days > 75 {
|
||||
continue
|
||||
}
|
||||
}
|
||||
cands = append(cands, a)
|
||||
}
|
||||
if len(cands) == 1 {
|
||||
return cands[0], true
|
||||
}
|
||||
best, bestD, tie := -1, 0.0, false
|
||||
for i, a := range cands {
|
||||
if !a.hasDate {
|
||||
continue
|
||||
}
|
||||
d := a.date.Sub(rd).Hours() / 24
|
||||
if d < 0 {
|
||||
d = -d
|
||||
}
|
||||
if best < 0 || d < bestD {
|
||||
best, bestD, tie = i, d, false
|
||||
} else if d == bestD {
|
||||
tie = true
|
||||
}
|
||||
}
|
||||
if best < 0 || tie {
|
||||
return amtEntry{}, false
|
||||
}
|
||||
return cands[best], true
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
// Command ooscan recursively lists OnlyOffice Documents folders into a TSV
|
||||
// index: file_id, folder_id, path, title.
|
||||
//
|
||||
// Usage: ooscan <FOLDER_ID> [<FOLDER_ID>...]
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"time"
|
||||
|
||||
onlyoffice "github.com/eslider/go-onlyoffice"
|
||||
)
|
||||
|
||||
func main() {
|
||||
ctx := context.Background()
|
||||
c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials())
|
||||
seen := map[string]bool{}
|
||||
for _, root := range os.Args[1:] {
|
||||
walk(ctx, c, root, "", 0, seen)
|
||||
}
|
||||
}
|
||||
|
||||
func walk(ctx context.Context, c *onlyoffice.Client, folderID, path string, depth int, seen map[string]bool) {
|
||||
if depth > 8 || seen[folderID] {
|
||||
return
|
||||
}
|
||||
seen[folderID] = true
|
||||
// Throttle: OnlyOffice rate-limits (429) and the host must not be flooded.
|
||||
time.Sleep(350 * time.Millisecond)
|
||||
ctx, cancel := context.WithTimeout(ctx, 60*time.Second)
|
||||
defer cancel()
|
||||
var l *onlyoffice.DavListing
|
||||
derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
var err error
|
||||
l, err = c.ListDavFolder(ctx, folderID)
|
||||
return err
|
||||
})
|
||||
if derr != nil {
|
||||
fmt.Fprintf(os.Stderr, "list %s (%s): %v\n", path, folderID, derr)
|
||||
return
|
||||
}
|
||||
for _, f := range l.Files {
|
||||
fmt.Printf("%s\t%s\t%s\t%s\n", f.ID, folderID, path, f.Title)
|
||||
}
|
||||
for _, sub := range l.Folders {
|
||||
walk(ctx, c, sub.ID, path+"/"+sub.Title, depth+1, seen)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,124 @@
|
||||
// Command pdfamount walks a Documents folder, downloads matching PDFs and
|
||||
// extracts the payable amount, printing "file_id\ttitle\tamount".
|
||||
//
|
||||
// Usage: pdfamount <FOLDER_ID> [TITLE_FILTER_REGEX]
|
||||
package main
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
onlyoffice "github.com/eslider/go-onlyoffice"
|
||||
)
|
||||
|
||||
var amountRe = regexp.MustCompile(`(?i)(zu zahlender betrag|rechnungsbetrag)\s*[:\s]*([0-9][0-9.]*,[0-9]{2})`)
|
||||
|
||||
func main() {
|
||||
if len(os.Args) < 2 {
|
||||
fmt.Fprintln(os.Stderr, "usage: pdfamount <FOLDER_ID> [TITLE_FILTER_REGEX]")
|
||||
os.Exit(2)
|
||||
}
|
||||
folder := os.Args[1]
|
||||
filter := regexp.MustCompile(`(?i)rechnung`)
|
||||
if len(os.Args) >= 3 {
|
||||
filter = regexp.MustCompile(os.Args[2])
|
||||
}
|
||||
ctx := context.Background()
|
||||
c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials())
|
||||
|
||||
files := listAll(ctx, c, folder)
|
||||
for _, f := range files {
|
||||
if !filter.MatchString(f.title) {
|
||||
continue
|
||||
}
|
||||
if !strings.HasSuffix(strings.ToLower(f.title), ".pdf") {
|
||||
continue
|
||||
}
|
||||
amount, err := pdfAmount(ctx, c, f.id)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "%s: %v\n", f.title, err)
|
||||
continue
|
||||
}
|
||||
if amount == "" {
|
||||
continue
|
||||
}
|
||||
fmt.Printf("%s\t%s\t%s\n", f.id, f.title, amount)
|
||||
}
|
||||
}
|
||||
|
||||
type file struct{ id, title string }
|
||||
|
||||
func listAll(ctx context.Context, c *onlyoffice.Client, folder string) []file {
|
||||
seen := map[string]bool{}
|
||||
var out []file
|
||||
var walk func(string)
|
||||
walk = func(id string) {
|
||||
if seen[id] {
|
||||
return
|
||||
}
|
||||
seen[id] = true
|
||||
time.Sleep(300 * time.Millisecond)
|
||||
l, err := c.ListDavFolder(ctx, id)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "list %s: %v\n", id, err)
|
||||
return
|
||||
}
|
||||
for _, f := range l.Files {
|
||||
out = append(out, file{f.ID, f.Title})
|
||||
}
|
||||
for _, sub := range l.Folders {
|
||||
walk(sub.ID)
|
||||
}
|
||||
}
|
||||
walk(folder)
|
||||
return out
|
||||
}
|
||||
|
||||
func pdfAmount(ctx context.Context, c *onlyoffice.Client, id string) (string, error) {
|
||||
time.Sleep(time.Second)
|
||||
tmp, err := os.CreateTemp("", "pdf-*.pdf")
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
defer os.Remove(tmp.Name())
|
||||
derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error {
|
||||
_ = tmp.Truncate(0)
|
||||
_, _ = tmp.Seek(0, 0)
|
||||
_, err := c.DownloadFile(ctx, id, tmp)
|
||||
return err
|
||||
})
|
||||
if derr != nil {
|
||||
tmp.Close()
|
||||
return "", derr
|
||||
}
|
||||
tmp.Close()
|
||||
var buf bytes.Buffer
|
||||
cmd := exec.CommandContext(ctx, "pdftotext", "-layout", tmp.Name(), "-")
|
||||
cmd.Stdout = &buf
|
||||
if err := cmd.Run(); err != nil {
|
||||
return "", err
|
||||
}
|
||||
m := amountRe.FindStringSubmatch(buf.String())
|
||||
if m == nil {
|
||||
return "", nil
|
||||
}
|
||||
return parseDe(m[2]), nil
|
||||
}
|
||||
|
||||
// parseDe turns "1.234,56" into 1234.56.
|
||||
func parseDe(s string) string {
|
||||
s = strings.ReplaceAll(s, ".", "")
|
||||
s = strings.ReplaceAll(s, ",", ".")
|
||||
v, err := strconv.ParseFloat(s, 64)
|
||||
if err != nil {
|
||||
return s
|
||||
}
|
||||
return strconv.FormatFloat(v, 'f', 2, 64)
|
||||
}
|
||||
@@ -5,7 +5,6 @@ import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"path"
|
||||
"strconv"
|
||||
@@ -383,36 +382,15 @@ func FileFolderID(f *FileEntry) string {
|
||||
}
|
||||
|
||||
// DownloadFile streams file bytes from the file's viewUrl using the same auth
|
||||
// as API calls. Writes into dst.
|
||||
// as API calls. Writes into dst. When the portal serves the file from its stale
|
||||
// AWS S3 consumer, the bytes are fetched from the local MinIO store instead
|
||||
// (see storage_fallback.go).
|
||||
func (c *Client) DownloadFile(ctx context.Context, fileID string, dst io.Writer) (int64, error) {
|
||||
f, err := c.GetFile(ctx, fileID)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if f.ViewURL == nil || *f.ViewURL == "" {
|
||||
return 0, fmt.Errorf("file %s has no viewUrl", fileID)
|
||||
}
|
||||
downloadURL := c.resolveAPIURL(*f.ViewURL)
|
||||
auth, err := c.authHeader()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Authorization", auth)
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400))
|
||||
}
|
||||
n, err := io.Copy(dst, resp.Body)
|
||||
return n, err
|
||||
return c.downloadFileEntry(ctx, f, dst)
|
||||
}
|
||||
|
||||
func (c *Client) resolveAPIURL(ref string) string {
|
||||
|
||||
+9
-33
@@ -259,34 +259,10 @@ func (c *Client) UploadDavFile(ctx context.Context, folderID, fileName string, s
|
||||
return env.Response, nil
|
||||
}
|
||||
|
||||
// DownloadDavFile streams the file identified by id to w, returning bytes copied.
|
||||
// DownloadDavFile streams the file identified by id to w, returning bytes
|
||||
// copied. It shares the MinIO stale-S3 fallback with DownloadFile.
|
||||
func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) {
|
||||
file, err := c.GetFile(ctx, id)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if file.ViewURL == nil || *file.ViewURL == "" {
|
||||
return 0, fmt.Errorf("onlyoffice: file %s has no viewUrl", id)
|
||||
}
|
||||
u := c.resolveAPIURL(*file.ViewURL)
|
||||
auth, err := c.authHeader()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Authorization", auth)
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
return 0, fmt.Errorf("onlyoffice: download: %d", resp.StatusCode)
|
||||
}
|
||||
return io.Copy(w, resp.Body)
|
||||
return c.DownloadFile(ctx, id, w)
|
||||
}
|
||||
|
||||
// --- internal helpers -------------------------------------------------------
|
||||
@@ -434,12 +410,12 @@ func (f *DavFolder) UnmarshalJSON(b []byte) error {
|
||||
// UnmarshalJSON decodes a file row, capturing size and timestamps.
|
||||
func (f *DavFile) UnmarshalJSON(b []byte) error {
|
||||
var raw struct {
|
||||
ID *json.Number `json:"id"`
|
||||
Title *string `json:"title"`
|
||||
PureSize *int64 `json:"pureContentLength"`
|
||||
SizeStr *string `json:"contentLength"`
|
||||
Updated *string `json:"updated"`
|
||||
ViewURL *string `json:"viewUrl"`
|
||||
ID *json.Number `json:"id"`
|
||||
Title *string `json:"title"`
|
||||
PureSize *int64 `json:"pureContentLength"`
|
||||
SizeStr *string `json:"contentLength"`
|
||||
Updated *string `json:"updated"`
|
||||
ViewURL *string `json:"viewUrl"`
|
||||
}
|
||||
if err := json.Unmarshal(b, &raw); err != nil {
|
||||
return err
|
||||
|
||||
@@ -4,6 +4,7 @@ go 1.25.0
|
||||
|
||||
require (
|
||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||
github.com/charmbracelet/bubbles v0.18.0
|
||||
github.com/charmbracelet/bubbletea v0.25.0
|
||||
github.com/charmbracelet/glamour v0.8.0
|
||||
@@ -26,6 +27,7 @@ require (
|
||||
github.com/JohannesKaufmann/dom v0.3.1 // indirect
|
||||
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
|
||||
github.com/atotto/clipboard v0.1.4 // indirect
|
||||
github.com/aws/smithy-go v1.24.0 // indirect
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||
github.com/aymerick/douceur v0.2.0 // indirect
|
||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||
|
||||
@@ -10,6 +10,10 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc
|
||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
||||
|
||||
@@ -0,0 +1,60 @@
|
||||
package onlyoffice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"regexp"
|
||||
"time"
|
||||
)
|
||||
|
||||
// RetryPolicy controls deterministic retries against OnlyOffice: fixed linear
|
||||
// backoff without jitter, so repeated runs wait exactly the same schedule.
|
||||
// OnlyOffice throttles bulk reads/writes with 429 (and occasional 502/503/504
|
||||
// from openresty), so every bulk tool routes API calls through DoRetry.
|
||||
type RetryPolicy struct {
|
||||
Attempts int // total attempts, including the first try
|
||||
Base time.Duration // wait before retry N is N*Base
|
||||
Max time.Duration // per-wait cap
|
||||
}
|
||||
|
||||
// DefaultRetryPolicy retries up to 5 times with 1s, 2s, 3s, 4s waits.
|
||||
func DefaultRetryPolicy() RetryPolicy {
|
||||
return RetryPolicy{Attempts: 5, Base: time.Second, Max: 30 * time.Second}
|
||||
}
|
||||
|
||||
var transientRe = regexp.MustCompile(`:\s*(429|502|503|504)\b`)
|
||||
|
||||
// Transient reports whether err looks like a transient OnlyOffice answer
|
||||
// (an HTTP 429/502/503/504 surfaced as "...: <code> ...").
|
||||
func Transient(err error) bool {
|
||||
if err == nil {
|
||||
return false
|
||||
}
|
||||
return transientRe.MatchString(err.Error())
|
||||
}
|
||||
|
||||
// DoRetry runs fn until it succeeds, fails non-transiently, or attempts run
|
||||
// out. Waits are deterministic: N*Base capped at Max, no jitter.
|
||||
func DoRetry(ctx context.Context, p RetryPolicy, fn func() error) error {
|
||||
if p.Attempts < 1 {
|
||||
p.Attempts = 1
|
||||
}
|
||||
var err error
|
||||
for attempt := 1; attempt <= p.Attempts; attempt++ {
|
||||
if ctx.Err() != nil {
|
||||
return ctx.Err()
|
||||
}
|
||||
if err = fn(); err == nil || !Transient(err) || attempt == p.Attempts {
|
||||
return err
|
||||
}
|
||||
wait := time.Duration(attempt) * p.Base
|
||||
if wait > p.Max {
|
||||
wait = p.Max
|
||||
}
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return ctx.Err()
|
||||
case <-time.After(wait):
|
||||
}
|
||||
}
|
||||
return err
|
||||
}
|
||||
@@ -0,0 +1,200 @@
|
||||
package onlyoffice
|
||||
|
||||
// MinIO download fallback for the portal's stale AWS S3 consumer.
|
||||
//
|
||||
// On the Fibu EDL portal some older Documents files live in S3/MinIO, but the
|
||||
// portal's storage consumer still points at s3.us-east-1.amazonaws.com with
|
||||
// access key "minio". Downloads of those files answer 403 InvalidAccessKeyId.
|
||||
// The bytes are present in the local MinIO store under a deterministic object
|
||||
// key, so the client retries the GET there.
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/sha256"
|
||||
"encoding/hex"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/aws/aws-sdk-go-v2/aws"
|
||||
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||
)
|
||||
|
||||
const (
|
||||
defaultMinioEndpoint = "http://192.168.188.10:9000"
|
||||
defaultMinioBucket = "office"
|
||||
minioRegion = "us-east-1"
|
||||
)
|
||||
|
||||
// minioObjectKey is the fallback object key layout the portal's S3 consumer
|
||||
// writes for Documents files: 00/00/01/files/folder_<folderId>/file_<fileId>/v1/content.pdf.
|
||||
// Prefer minioObjectKeyFromURL: the portal stores all files below its storage
|
||||
// root folder, which is not the API folderId returned by GetFile.
|
||||
func minioObjectKey(fileID, folderID string) string {
|
||||
return "00/00/01/files/folder_" + folderID + "/file_" + fileID + "/v1/content.pdf"
|
||||
}
|
||||
|
||||
// minioObjectKeyFromURL extracts the object key from an S3 download URL. Path
|
||||
// style URLs (bucket as first path segment) have that segment removed; virtual
|
||||
// host style URLs are returned as-is. This is authoritative: the portal signs
|
||||
// the exact key, so no folder-id guessing is needed.
|
||||
func minioObjectKeyFromURL(rawURL, bucket string) (string, bool) {
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || u.Path == "" {
|
||||
return "", false
|
||||
}
|
||||
segs := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||
// Path-style URLs carry the bucket as leading segment; the portal's S3
|
||||
// consumer can emit it twice (serviceurl already includes the bucket), so
|
||||
// strip every leading segment equal to the bucket.
|
||||
for len(segs) > 0 && bucket != "" && segs[0] == bucket {
|
||||
segs = segs[1:]
|
||||
}
|
||||
if len(segs) == 0 {
|
||||
return "", false
|
||||
}
|
||||
for _, s := range segs {
|
||||
if s == "" || s == "." || s == ".." {
|
||||
return "", false
|
||||
}
|
||||
}
|
||||
return strings.Join(segs, "/"), true
|
||||
}
|
||||
|
||||
// isStaleS3Redirect reports whether a download landed on the portal's stale AWS
|
||||
// S3 consumer. Such responses either carry an S3 InvalidAccessKeyId XML body or
|
||||
// point at amazonaws.com with the "minio" access key id in the query.
|
||||
func isStaleS3Redirect(rawURL string, body []byte) bool {
|
||||
if strings.Contains(strings.ToLower(string(body)), "invalidaccesskeyid") {
|
||||
return true
|
||||
}
|
||||
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||
if err != nil || u.Host == "" {
|
||||
return false
|
||||
}
|
||||
host := strings.ToLower(u.Host)
|
||||
if !strings.Contains(host, "amazonaws.com") {
|
||||
return false
|
||||
}
|
||||
q := strings.ToLower(u.RawQuery)
|
||||
return strings.Contains(q, "accesskeyid=minio") || strings.Contains(q, "x-amz-credential=minio")
|
||||
}
|
||||
|
||||
// minioConfig is the runtime configuration for the local MinIO fallback.
|
||||
type minioConfig struct {
|
||||
Endpoint string
|
||||
Bucket string
|
||||
AccessKey string
|
||||
SecretKey string
|
||||
}
|
||||
|
||||
// loadMinioConfig reads the fallback configuration from the environment.
|
||||
// Secrets are never defaulted; without access/secret keys the fallback is off.
|
||||
func loadMinioConfig() minioConfig {
|
||||
return minioConfig{
|
||||
Endpoint: strings.TrimRight(firstNonEmpty(os.Getenv("MINIO_ENDPOINT"), defaultMinioEndpoint), "/"),
|
||||
Bucket: firstNonEmpty(os.Getenv("MINIO_BUCKET"), defaultMinioBucket),
|
||||
AccessKey: os.Getenv("MINIO_ACCESS_KEY"),
|
||||
SecretKey: os.Getenv("MINIO_SECRET_KEY"),
|
||||
}
|
||||
}
|
||||
|
||||
// downloadFileEntry downloads f's bytes to dst. It transparently falls back to
|
||||
// the local MinIO store when the portal redirects the download to its stale AWS
|
||||
// S3 consumer.
|
||||
func (c *Client) downloadFileEntry(ctx context.Context, f *FileEntry, dst io.Writer) (int64, error) {
|
||||
if f == nil {
|
||||
return 0, fmt.Errorf("onlyoffice: download: nil file entry")
|
||||
}
|
||||
if f.ViewURL == nil || *f.ViewURL == "" {
|
||||
return 0, fmt.Errorf("onlyoffice: file has no viewUrl")
|
||||
}
|
||||
downloadURL := c.resolveAPIURL(*f.ViewURL)
|
||||
auth, err := c.authHeader()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
req.Header.Set("Authorization", auth)
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
if resp.StatusCode >= 400 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||
finalURL := downloadURL
|
||||
if resp.Request != nil && resp.Request.URL != nil {
|
||||
finalURL = resp.Request.URL.String()
|
||||
}
|
||||
if isStaleS3Redirect(finalURL, b) {
|
||||
key, ok := minioObjectKeyFromURL(finalURL, loadMinioConfig().Bucket)
|
||||
if !ok {
|
||||
fileID := ""
|
||||
if f.ID != nil {
|
||||
fileID = f.ID.String()
|
||||
}
|
||||
key = minioObjectKey(fileID, FileFolderID(f))
|
||||
}
|
||||
n, merr := c.downloadFromMinio(ctx, key, dst)
|
||||
if merr == nil {
|
||||
return n, nil
|
||||
}
|
||||
return 0, fmt.Errorf("GET viewUrl: %d (stale S3) and minio fallback: %w", resp.StatusCode, merr)
|
||||
}
|
||||
return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400))
|
||||
}
|
||||
return io.Copy(dst, resp.Body)
|
||||
}
|
||||
|
||||
// downloadFromMinio streams objectKey from the configured MinIO bucket.
|
||||
func (c *Client) downloadFromMinio(ctx context.Context, objectKey string, dst io.Writer) (int64, error) {
|
||||
if objectKey == "" {
|
||||
return 0, fmt.Errorf("onlyoffice: minio fallback: empty object key")
|
||||
}
|
||||
cfg := loadMinioConfig()
|
||||
if cfg.AccessKey == "" || cfg.SecretKey == "" {
|
||||
return 0, fmt.Errorf("onlyoffice: minio fallback: MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set")
|
||||
}
|
||||
base, err := url.Parse(cfg.Endpoint)
|
||||
if err != nil || base.Host == "" {
|
||||
return 0, fmt.Errorf("onlyoffice: minio fallback: bad MINIO_ENDPOINT %q", cfg.Endpoint)
|
||||
}
|
||||
u := *base
|
||||
u.Path = "/" + cfg.Bucket + "/" + objectKey
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
if err := signMinioRequest(ctx, cfg, req); err != nil {
|
||||
return 0, err
|
||||
}
|
||||
resp, err := c.client.Do(req)
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("onlyoffice: minio fallback: %w", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode >= 400 {
|
||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||
return 0, fmt.Errorf("onlyoffice: minio fallback: %d %s", resp.StatusCode, truncate(string(b), 300))
|
||||
}
|
||||
return io.Copy(dst, resp.Body)
|
||||
}
|
||||
|
||||
// signMinioRequest signs req with AWS Signature V4 for the S3 service.
|
||||
func signMinioRequest(ctx context.Context, cfg minioConfig, req *http.Request) error {
|
||||
sum := sha256.Sum256(nil)
|
||||
creds := aws.Credentials{AccessKeyID: cfg.AccessKey, SecretAccessKey: cfg.SecretKey}
|
||||
if err := v4.NewSigner().SignHTTP(ctx, creds, req, hex.EncodeToString(sum[:]), "s3", minioRegion, time.Now()); err != nil {
|
||||
return fmt.Errorf("onlyoffice: minio fallback: sign: %w", err)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,45 @@
|
||||
//go:build integration
|
||||
|
||||
package onlyoffice
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
// TestIntegrationMinioFallback downloads known stale-S3 files through the local
|
||||
// MinIO fallback. Requires ONLYOFFICE_URL/USER/PASS (as all integration tests),
|
||||
// MINIO_ACCESS_KEY/MINIO_SECRET_KEY and MINIO_TEST_FILE_IDS="3785,3859,3666";
|
||||
// skips when any of those are missing.
|
||||
func TestIntegrationMinioFallback(t *testing.T) {
|
||||
if os.Getenv("MINIO_ACCESS_KEY") == "" || os.Getenv("MINIO_SECRET_KEY") == "" {
|
||||
t.Skip("MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set — skipping integration test")
|
||||
}
|
||||
raw := strings.TrimSpace(os.Getenv("MINIO_TEST_FILE_IDS"))
|
||||
if raw == "" {
|
||||
t.Skip("MINIO_TEST_FILE_IDS not set — skipping integration test")
|
||||
}
|
||||
c := liveClient(t)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||
defer cancel()
|
||||
for _, id := range strings.Split(raw, ",") {
|
||||
id = strings.TrimSpace(id)
|
||||
if id == "" {
|
||||
continue
|
||||
}
|
||||
n, err := c.DownloadFile(ctx, id, io.Discard)
|
||||
if err != nil {
|
||||
t.Errorf("DownloadFile(%s): %v", id, err)
|
||||
continue
|
||||
}
|
||||
if n == 0 {
|
||||
t.Errorf("DownloadFile(%s): 0 bytes", id)
|
||||
} else {
|
||||
t.Logf("DownloadFile(%s): %d bytes", id, n)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,219 @@
|
||||
package onlyoffice
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
)
|
||||
|
||||
func TestMinioObjectKey(t *testing.T) {
|
||||
cases := []struct {
|
||||
fileID string
|
||||
folderID string
|
||||
want string
|
||||
}{
|
||||
{"3785", "652", "00/00/01/files/folder_652/file_3785/v1/content.pdf"},
|
||||
{"1", "2", "00/00/01/files/folder_2/file_1/v1/content.pdf"},
|
||||
{"3666", "4000", "00/00/01/files/folder_4000/file_3666/v1/content.pdf"},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
if got := minioObjectKey(tc.fileID, tc.folderID); got != tc.want {
|
||||
t.Errorf("minioObjectKey(%q, %q) = %q, want %q", tc.fileID, tc.folderID, got, tc.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMinioObjectKeyFromURL(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
bucket string
|
||||
want string
|
||||
ok bool
|
||||
}{
|
||||
{
|
||||
name: "path style drops bucket segment",
|
||||
url: "https://s3.us-east-1.amazonaws.com/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio",
|
||||
bucket: "office",
|
||||
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "doubled bucket segment (portal serviceurl includes bucket)",
|
||||
url: "https://s3.us-east-1.amazonaws.com/office/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio",
|
||||
bucket: "office",
|
||||
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "virtual host style keeps path",
|
||||
url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||
bucket: "office",
|
||||
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||
ok: true,
|
||||
},
|
||||
{
|
||||
name: "foreign first segment kept",
|
||||
url: "https://example.com/other/file_1/v1/content.pdf",
|
||||
bucket: "office",
|
||||
want: "other/file_1/v1/content.pdf",
|
||||
ok: true,
|
||||
},
|
||||
{name: "empty path", url: "https://example.com", bucket: "office", ok: false},
|
||||
{name: "traversal", url: "https://example.com/office/../etc/passwd", bucket: "office", ok: false},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
got, ok := minioObjectKeyFromURL(tc.url, tc.bucket)
|
||||
if ok != tc.ok || got != tc.want {
|
||||
t.Errorf("minioObjectKeyFromURL(%q, %q) = (%q, %v), want (%q, %v)", tc.url, tc.bucket, got, ok, tc.want, tc.ok)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsStaleS3Redirect(t *testing.T) {
|
||||
cases := []struct {
|
||||
name string
|
||||
url string
|
||||
body []byte
|
||||
want bool
|
||||
}{
|
||||
{
|
||||
name: "aws redirect with minio access key",
|
||||
url: "https://s3.us-east-1.amazonaws.com/office/x/file_1?AWSAccessKeyId=minio&Expires=1",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "aws redirect with minio x-amz-credential",
|
||||
url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_1/file_1?X-Amz-Credential=minio%2F20260914",
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "invalid access key xml body",
|
||||
url: "https://portal.internal/download/1",
|
||||
body: []byte(`<?xml version="1.0"?><Error><Code>InvalidAccessKeyId</Code><AWSAccessKeyId>minio</AWSAccessKeyId></Error>`),
|
||||
want: true,
|
||||
},
|
||||
{
|
||||
name: "regular pdf from portal",
|
||||
url: "https://portal.internal/download/1",
|
||||
body: []byte("%PDF-1.7 data"),
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "aws redirect with foreign key",
|
||||
url: "https://s3.us-east-1.amazonaws.com/office/x?AWSAccessKeyId=other",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "amazonaws in path but foreign host",
|
||||
url: "https://example.com/amazonaws.com/file?AWSAccessKeyId=minio",
|
||||
want: false,
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
want: false,
|
||||
},
|
||||
}
|
||||
for _, tc := range cases {
|
||||
t.Run(tc.name, func(t *testing.T) {
|
||||
if got := isStaleS3Redirect(tc.url, tc.body); got != tc.want {
|
||||
t.Errorf("isStaleS3Redirect(%q, %q) = %v, want %v", tc.url, tc.body, got, tc.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
const staleS3Body = `<?xml version="1.0" encoding="UTF-8"?>` +
|
||||
`<Error><Code>InvalidAccessKeyId</Code>` +
|
||||
`<Message>The AWS Access Key Id you provided does not exist in our records.</Message>` +
|
||||
`<AWSAccessKeyId>minio</AWSAccessKeyId></Error>`
|
||||
|
||||
func TestDownloadFileMinioFallback(t *testing.T) {
|
||||
const payload = "PDFDATA-3785"
|
||||
|
||||
portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/2.0/files/file/3785.json":
|
||||
w.Header().Set("Content-Type", "application/json")
|
||||
io.WriteString(w, `{"response":{"id":3785,"title":"04.pdf","folderId":655,"viewUrl":"/download/3785"}}`)
|
||||
case "/download/3785":
|
||||
http.Redirect(w, r, "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", http.StatusTemporaryRedirect)
|
||||
case "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf":
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
io.WriteString(w, staleS3Body)
|
||||
default:
|
||||
http.NotFound(w, r)
|
||||
}
|
||||
}))
|
||||
defer portal.Close()
|
||||
|
||||
var minioPath, minioAuth string
|
||||
minio := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
minioPath, minioAuth = r.URL.Path, r.Header.Get("Authorization")
|
||||
io.WriteString(w, payload)
|
||||
}))
|
||||
defer minio.Close()
|
||||
|
||||
t.Setenv("MINIO_ENDPOINT", minio.URL)
|
||||
t.Setenv("MINIO_BUCKET", "office")
|
||||
t.Setenv("MINIO_ACCESS_KEY", "testkey")
|
||||
t.Setenv("MINIO_SECRET_KEY", "testsecret")
|
||||
|
||||
c := &Client{
|
||||
client: portal.Client(),
|
||||
credentials: &Credentials{Url: portal.URL},
|
||||
token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))},
|
||||
}
|
||||
|
||||
var buf bytes.Buffer
|
||||
n, err := c.DownloadFile(context.Background(), "3785", &buf)
|
||||
if err != nil {
|
||||
t.Fatalf("DownloadFile: %v", err)
|
||||
}
|
||||
if n != int64(len(payload)) || buf.String() != payload {
|
||||
t.Fatalf("got %d bytes %q, want %d bytes %q", n, buf.String(), len(payload), payload)
|
||||
}
|
||||
if want := "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf"; minioPath != want {
|
||||
t.Errorf("minio path = %q, want %q", minioPath, want)
|
||||
}
|
||||
if !strings.HasPrefix(minioAuth, "AWS4-HMAC-SHA256") {
|
||||
t.Errorf("minio request not SigV4-signed; Authorization=%q", minioAuth)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadFileMinioFallbackWithoutCreds(t *testing.T) {
|
||||
portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
switch r.URL.Path {
|
||||
case "/api/2.0/files/file/3785.json":
|
||||
io.WriteString(w, `{"response":{"id":3785,"folderId":652,"viewUrl":"/download/3785"}}`)
|
||||
default:
|
||||
w.WriteHeader(http.StatusForbidden)
|
||||
io.WriteString(w, staleS3Body)
|
||||
}
|
||||
}))
|
||||
defer portal.Close()
|
||||
|
||||
t.Setenv("MINIO_ACCESS_KEY", "")
|
||||
t.Setenv("MINIO_SECRET_KEY", "")
|
||||
|
||||
c := &Client{
|
||||
client: portal.Client(),
|
||||
credentials: &Credentials{Url: portal.URL},
|
||||
token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))},
|
||||
}
|
||||
|
||||
_, err := c.DownloadFile(context.Background(), "3785", io.Discard)
|
||||
if err == nil {
|
||||
t.Fatal("expected error without minio credentials")
|
||||
}
|
||||
if !strings.Contains(err.Error(), "MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") {
|
||||
t.Fatalf("unexpected error: %v", err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user