From 8ac777c031f72700b06ce54ea59f7f1d6af2b559 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Mon, 14 Sep 2026 15:41:39 +0000 Subject: [PATCH 1/2] feat(files): MinIO fallback for stale S3 downloads (#152) --- .env.example | 9 ++ files.go | 30 +--- files_webdav.go | 30 +--- go.mod | 2 + go.sum | 4 + storage_fallback.go | 200 ++++++++++++++++++++++++ storage_fallback_integration_test.go | 45 ++++++ storage_fallback_test.go | 219 +++++++++++++++++++++++++++ 8 files changed, 486 insertions(+), 53 deletions(-) create mode 100644 storage_fallback.go create mode 100644 storage_fallback_integration_test.go create mode 100644 storage_fallback_test.go diff --git a/.env.example b/.env.example index 2e3e49e..bfcff31 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,12 @@ ONLYOFFICE_PROJECT_ID=33 # cmd/office TUI — optional Document Server for DOCX→HTML preview: # ONLYOFFICE_DOCS_URL=https://docs.example.com # ONLYOFFICE_DOCS_SECRET= + +# MinIO download fallback for the portal's stale AWS S3 consumer (older +# Documents folders). When the portal redirects to amazonaws.com with access +# key "minio" (403 InvalidAccessKeyId), files are fetched from the local MinIO +# store instead. Without a key/secret the fallback is disabled. +# MINIO_ENDPOINT=http://192.168.188.10:9000 +# MINIO_BUCKET=office +# MINIO_ACCESS_KEY= +# MINIO_SECRET_KEY= diff --git a/files.go b/files.go index 4d0497b..3fe2ea3 100644 --- a/files.go +++ b/files.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "path" "strconv" @@ -383,36 +382,15 @@ func FileFolderID(f *FileEntry) string { } // DownloadFile streams file bytes from the file's viewUrl using the same auth -// as API calls. Writes into dst. +// as API calls. Writes into dst. When the portal serves the file from its stale +// AWS S3 consumer, the bytes are fetched from the local MinIO store instead +// (see storage_fallback.go). func (c *Client) DownloadFile(ctx context.Context, fileID string, dst io.Writer) (int64, error) { f, err := c.GetFile(ctx, fileID) if err != nil { return 0, err } - if f.ViewURL == nil || *f.ViewURL == "" { - return 0, fmt.Errorf("file %s has no viewUrl", fileID) - } - downloadURL := c.resolveAPIURL(*f.ViewURL) - auth, err := c.authHeader() - if err != nil { - return 0, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", auth) - resp, err := c.client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400)) - } - n, err := io.Copy(dst, resp.Body) - return n, err + return c.downloadFileEntry(ctx, f, dst) } func (c *Client) resolveAPIURL(ref string) string { diff --git a/files_webdav.go b/files_webdav.go index af2c232..8b3cf32 100644 --- a/files_webdav.go +++ b/files_webdav.go @@ -259,34 +259,10 @@ func (c *Client) UploadDavFile(ctx context.Context, folderID, fileName string, s return env.Response, nil } -// DownloadDavFile streams the file identified by id to w, returning bytes copied. +// DownloadDavFile streams the file identified by id to w, returning bytes +// copied. It shares the MinIO stale-S3 fallback with DownloadFile. func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) { - file, err := c.GetFile(ctx, id) - if err != nil { - return 0, err - } - if file.ViewURL == nil || *file.ViewURL == "" { - return 0, fmt.Errorf("onlyoffice: file %s has no viewUrl", id) - } - u := c.resolveAPIURL(*file.ViewURL) - auth, err := c.authHeader() - if err != nil { - return 0, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", auth) - resp, err := c.client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return 0, fmt.Errorf("onlyoffice: download: %d", resp.StatusCode) - } - return io.Copy(w, resp.Body) + return c.DownloadFile(ctx, id, w) } // --- internal helpers ------------------------------------------------------- diff --git a/go.mod b/go.mod index efbac23..e988b59 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 + github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/glamour v0.8.0 @@ -26,6 +27,7 @@ require ( github.com/JohannesKaufmann/dom v0.3.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect diff --git a/go.sum b/go.sum index f3dff4b..0bef785 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,10 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= diff --git a/storage_fallback.go b/storage_fallback.go new file mode 100644 index 0000000..1d5db52 --- /dev/null +++ b/storage_fallback.go @@ -0,0 +1,200 @@ +package onlyoffice + +// MinIO download fallback for the portal's stale AWS S3 consumer. +// +// On the Fibu EDL portal some older Documents files live in S3/MinIO, but the +// portal's storage consumer still points at s3.us-east-1.amazonaws.com with +// access key "minio". Downloads of those files answer 403 InvalidAccessKeyId. +// The bytes are present in the local MinIO store under a deterministic object +// key, so the client retries the GET there. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +const ( + defaultMinioEndpoint = "http://192.168.188.10:9000" + defaultMinioBucket = "office" + minioRegion = "us-east-1" +) + +// minioObjectKey is the fallback object key layout the portal's S3 consumer +// writes for Documents files: 00/00/01/files/folder_/file_/v1/content.pdf. +// Prefer minioObjectKeyFromURL: the portal stores all files below its storage +// root folder, which is not the API folderId returned by GetFile. +func minioObjectKey(fileID, folderID string) string { + return "00/00/01/files/folder_" + folderID + "/file_" + fileID + "/v1/content.pdf" +} + +// minioObjectKeyFromURL extracts the object key from an S3 download URL. Path +// style URLs (bucket as first path segment) have that segment removed; virtual +// host style URLs are returned as-is. This is authoritative: the portal signs +// the exact key, so no folder-id guessing is needed. +func minioObjectKeyFromURL(rawURL, bucket string) (string, bool) { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Path == "" { + return "", false + } + segs := strings.Split(strings.Trim(u.Path, "/"), "/") + // Path-style URLs carry the bucket as leading segment; the portal's S3 + // consumer can emit it twice (serviceurl already includes the bucket), so + // strip every leading segment equal to the bucket. + for len(segs) > 0 && bucket != "" && segs[0] == bucket { + segs = segs[1:] + } + if len(segs) == 0 { + return "", false + } + for _, s := range segs { + if s == "" || s == "." || s == ".." { + return "", false + } + } + return strings.Join(segs, "/"), true +} + +// isStaleS3Redirect reports whether a download landed on the portal's stale AWS +// S3 consumer. Such responses either carry an S3 InvalidAccessKeyId XML body or +// point at amazonaws.com with the "minio" access key id in the query. +func isStaleS3Redirect(rawURL string, body []byte) bool { + if strings.Contains(strings.ToLower(string(body)), "invalidaccesskeyid") { + return true + } + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Host == "" { + return false + } + host := strings.ToLower(u.Host) + if !strings.Contains(host, "amazonaws.com") { + return false + } + q := strings.ToLower(u.RawQuery) + return strings.Contains(q, "accesskeyid=minio") || strings.Contains(q, "x-amz-credential=minio") +} + +// minioConfig is the runtime configuration for the local MinIO fallback. +type minioConfig struct { + Endpoint string + Bucket string + AccessKey string + SecretKey string +} + +// loadMinioConfig reads the fallback configuration from the environment. +// Secrets are never defaulted; without access/secret keys the fallback is off. +func loadMinioConfig() minioConfig { + return minioConfig{ + Endpoint: strings.TrimRight(firstNonEmpty(os.Getenv("MINIO_ENDPOINT"), defaultMinioEndpoint), "/"), + Bucket: firstNonEmpty(os.Getenv("MINIO_BUCKET"), defaultMinioBucket), + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + } +} + +// downloadFileEntry downloads f's bytes to dst. It transparently falls back to +// the local MinIO store when the portal redirects the download to its stale AWS +// S3 consumer. +func (c *Client) downloadFileEntry(ctx context.Context, f *FileEntry, dst io.Writer) (int64, error) { + if f == nil { + return 0, fmt.Errorf("onlyoffice: download: nil file entry") + } + if f.ViewURL == nil || *f.ViewURL == "" { + return 0, fmt.Errorf("onlyoffice: file has no viewUrl") + } + downloadURL := c.resolveAPIURL(*f.ViewURL) + auth, err := c.authHeader() + if err != nil { + return 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return 0, err + } + req.Header.Set("Authorization", auth) + resp, err := c.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + finalURL := downloadURL + if resp.Request != nil && resp.Request.URL != nil { + finalURL = resp.Request.URL.String() + } + if isStaleS3Redirect(finalURL, b) { + key, ok := minioObjectKeyFromURL(finalURL, loadMinioConfig().Bucket) + if !ok { + fileID := "" + if f.ID != nil { + fileID = f.ID.String() + } + key = minioObjectKey(fileID, FileFolderID(f)) + } + n, merr := c.downloadFromMinio(ctx, key, dst) + if merr == nil { + return n, nil + } + return 0, fmt.Errorf("GET viewUrl: %d (stale S3) and minio fallback: %w", resp.StatusCode, merr) + } + return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400)) + } + return io.Copy(dst, resp.Body) +} + +// downloadFromMinio streams objectKey from the configured MinIO bucket. +func (c *Client) downloadFromMinio(ctx context.Context, objectKey string, dst io.Writer) (int64, error) { + if objectKey == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: empty object key") + } + cfg := loadMinioConfig() + if cfg.AccessKey == "" || cfg.SecretKey == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") + } + base, err := url.Parse(cfg.Endpoint) + if err != nil || base.Host == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: bad MINIO_ENDPOINT %q", cfg.Endpoint) + } + u := *base + u.Path = "/" + cfg.Bucket + "/" + objectKey + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return 0, err + } + if err := signMinioRequest(ctx, cfg, req); err != nil { + return 0, err + } + resp, err := c.client.Do(req) + if err != nil { + return 0, fmt.Errorf("onlyoffice: minio fallback: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return 0, fmt.Errorf("onlyoffice: minio fallback: %d %s", resp.StatusCode, truncate(string(b), 300)) + } + return io.Copy(dst, resp.Body) +} + +// signMinioRequest signs req with AWS Signature V4 for the S3 service. +func signMinioRequest(ctx context.Context, cfg minioConfig, req *http.Request) error { + sum := sha256.Sum256(nil) + creds := aws.Credentials{AccessKeyID: cfg.AccessKey, SecretAccessKey: cfg.SecretKey} + if err := v4.NewSigner().SignHTTP(ctx, creds, req, hex.EncodeToString(sum[:]), "s3", minioRegion, time.Now()); err != nil { + return fmt.Errorf("onlyoffice: minio fallback: sign: %w", err) + } + return nil +} diff --git a/storage_fallback_integration_test.go b/storage_fallback_integration_test.go new file mode 100644 index 0000000..9e90f0b --- /dev/null +++ b/storage_fallback_integration_test.go @@ -0,0 +1,45 @@ +//go:build integration + +package onlyoffice + +import ( + "context" + "io" + "os" + "strings" + "testing" + "time" +) + +// TestIntegrationMinioFallback downloads known stale-S3 files through the local +// MinIO fallback. Requires ONLYOFFICE_URL/USER/PASS (as all integration tests), +// MINIO_ACCESS_KEY/MINIO_SECRET_KEY and MINIO_TEST_FILE_IDS="3785,3859,3666"; +// skips when any of those are missing. +func TestIntegrationMinioFallback(t *testing.T) { + if os.Getenv("MINIO_ACCESS_KEY") == "" || os.Getenv("MINIO_SECRET_KEY") == "" { + t.Skip("MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set — skipping integration test") + } + raw := strings.TrimSpace(os.Getenv("MINIO_TEST_FILE_IDS")) + if raw == "" { + t.Skip("MINIO_TEST_FILE_IDS not set — skipping integration test") + } + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + for _, id := range strings.Split(raw, ",") { + id = strings.TrimSpace(id) + if id == "" { + continue + } + n, err := c.DownloadFile(ctx, id, io.Discard) + if err != nil { + t.Errorf("DownloadFile(%s): %v", id, err) + continue + } + if n == 0 { + t.Errorf("DownloadFile(%s): 0 bytes", id) + } else { + t.Logf("DownloadFile(%s): %d bytes", id, n) + } + } +} diff --git a/storage_fallback_test.go b/storage_fallback_test.go new file mode 100644 index 0000000..7928e89 --- /dev/null +++ b/storage_fallback_test.go @@ -0,0 +1,219 @@ +package onlyoffice + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestMinioObjectKey(t *testing.T) { + cases := []struct { + fileID string + folderID string + want string + }{ + {"3785", "652", "00/00/01/files/folder_652/file_3785/v1/content.pdf"}, + {"1", "2", "00/00/01/files/folder_2/file_1/v1/content.pdf"}, + {"3666", "4000", "00/00/01/files/folder_4000/file_3666/v1/content.pdf"}, + } + for _, tc := range cases { + if got := minioObjectKey(tc.fileID, tc.folderID); got != tc.want { + t.Errorf("minioObjectKey(%q, %q) = %q, want %q", tc.fileID, tc.folderID, got, tc.want) + } + } +} + +func TestMinioObjectKeyFromURL(t *testing.T) { + cases := []struct { + name string + url string + bucket string + want string + ok bool + }{ + { + name: "path style drops bucket segment", + url: "https://s3.us-east-1.amazonaws.com/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "doubled bucket segment (portal serviceurl includes bucket)", + url: "https://s3.us-east-1.amazonaws.com/office/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "virtual host style keeps path", + url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_4000/file_3785/v1/content.pdf", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "foreign first segment kept", + url: "https://example.com/other/file_1/v1/content.pdf", + bucket: "office", + want: "other/file_1/v1/content.pdf", + ok: true, + }, + {name: "empty path", url: "https://example.com", bucket: "office", ok: false}, + {name: "traversal", url: "https://example.com/office/../etc/passwd", bucket: "office", ok: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := minioObjectKeyFromURL(tc.url, tc.bucket) + if ok != tc.ok || got != tc.want { + t.Errorf("minioObjectKeyFromURL(%q, %q) = (%q, %v), want (%q, %v)", tc.url, tc.bucket, got, ok, tc.want, tc.ok) + } + }) + } +} + +func TestIsStaleS3Redirect(t *testing.T) { + cases := []struct { + name string + url string + body []byte + want bool + }{ + { + name: "aws redirect with minio access key", + url: "https://s3.us-east-1.amazonaws.com/office/x/file_1?AWSAccessKeyId=minio&Expires=1", + want: true, + }, + { + name: "aws redirect with minio x-amz-credential", + url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_1/file_1?X-Amz-Credential=minio%2F20260914", + want: true, + }, + { + name: "invalid access key xml body", + url: "https://portal.internal/download/1", + body: []byte(`InvalidAccessKeyIdminio`), + want: true, + }, + { + name: "regular pdf from portal", + url: "https://portal.internal/download/1", + body: []byte("%PDF-1.7 data"), + want: false, + }, + { + name: "aws redirect with foreign key", + url: "https://s3.us-east-1.amazonaws.com/office/x?AWSAccessKeyId=other", + want: false, + }, + { + name: "amazonaws in path but foreign host", + url: "https://example.com/amazonaws.com/file?AWSAccessKeyId=minio", + want: false, + }, + { + name: "empty", + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isStaleS3Redirect(tc.url, tc.body); got != tc.want { + t.Errorf("isStaleS3Redirect(%q, %q) = %v, want %v", tc.url, tc.body, got, tc.want) + } + }) + } +} + +const staleS3Body = `` + + `InvalidAccessKeyId` + + `The AWS Access Key Id you provided does not exist in our records.` + + `minio` + +func TestDownloadFileMinioFallback(t *testing.T) { + const payload = "PDFDATA-3785" + + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/files/file/3785.json": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"response":{"id":3785,"title":"04.pdf","folderId":655,"viewUrl":"/download/3785"}}`) + case "/download/3785": + http.Redirect(w, r, "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", http.StatusTemporaryRedirect) + case "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf": + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, staleS3Body) + default: + http.NotFound(w, r) + } + })) + defer portal.Close() + + var minioPath, minioAuth string + minio := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + minioPath, minioAuth = r.URL.Path, r.Header.Get("Authorization") + io.WriteString(w, payload) + })) + defer minio.Close() + + t.Setenv("MINIO_ENDPOINT", minio.URL) + t.Setenv("MINIO_BUCKET", "office") + t.Setenv("MINIO_ACCESS_KEY", "testkey") + t.Setenv("MINIO_SECRET_KEY", "testsecret") + + c := &Client{ + client: portal.Client(), + credentials: &Credentials{Url: portal.URL}, + token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))}, + } + + var buf bytes.Buffer + n, err := c.DownloadFile(context.Background(), "3785", &buf) + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + if n != int64(len(payload)) || buf.String() != payload { + t.Fatalf("got %d bytes %q, want %d bytes %q", n, buf.String(), len(payload), payload) + } + if want := "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf"; minioPath != want { + t.Errorf("minio path = %q, want %q", minioPath, want) + } + if !strings.HasPrefix(minioAuth, "AWS4-HMAC-SHA256") { + t.Errorf("minio request not SigV4-signed; Authorization=%q", minioAuth) + } +} + +func TestDownloadFileMinioFallbackWithoutCreds(t *testing.T) { + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/files/file/3785.json": + io.WriteString(w, `{"response":{"id":3785,"folderId":652,"viewUrl":"/download/3785"}}`) + default: + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, staleS3Body) + } + })) + defer portal.Close() + + t.Setenv("MINIO_ACCESS_KEY", "") + t.Setenv("MINIO_SECRET_KEY", "") + + c := &Client{ + client: portal.Client(), + credentials: &Credentials{Url: portal.URL}, + token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))}, + } + + _, err := c.DownloadFile(context.Background(), "3785", io.Discard) + if err == nil { + t.Fatal("expected error without minio credentials") + } + if !strings.Contains(err.Error(), "MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") { + t.Fatalf("unexpected error: %v", err) + } +} From af8e3b053a949c3c87b2db1f09bbdc2e96b592b1 Mon Sep 17 00:00:00 2001 From: Andriy Oblivantsev Date: Mon, 14 Sep 2026 21:03:39 +0000 Subject: [PATCH 2/2] feat(oo): kontoblatt bulk tools, oo dav, update, retry (#22) --- cmd/kontoblatt/main.go | 220 +++++++++++++++++++ cmd/kontolink/main.go | 478 +++++++++++++++++++++++++++++++++++++++++ cmd/oo/dav.go | 291 +++++++++++++++++++++++++ cmd/oo/main.go | 1 + cmd/ooscan/main.go | 50 +++++ cmd/pdfamount/main.go | 124 +++++++++++ files.go | 28 +++ files_webdav.go | 78 ++++++- retry.go | 60 ++++++ 9 files changed, 1320 insertions(+), 10 deletions(-) create mode 100644 cmd/kontoblatt/main.go create mode 100644 cmd/kontolink/main.go create mode 100644 cmd/oo/dav.go create mode 100644 cmd/ooscan/main.go create mode 100644 cmd/pdfamount/main.go create mode 100644 retry.go diff --git a/cmd/kontoblatt/main.go b/cmd/kontoblatt/main.go new file mode 100644 index 0000000..37a8a0d --- /dev/null +++ b/cmd/kontoblatt/main.go @@ -0,0 +1,220 @@ +// Command kontoblatt builds a summary ("сводная таблица") of a Kontoblatt XLSX +// (Datum, Gegenkonto, Buchungstext, Beleg, Soll, Haben, Bemerkung) and uploads +// it back to the same OnlyOffice folder as the source file. +// +// Usage: kontoblatt +package main + +import ( + "context" + "fmt" + "os" + "regexp" + "sort" + "strconv" + "strings" + + onlyoffice "github.com/eslider/go-onlyoffice" + "github.com/xuri/excelize/v2" +) + +type agg struct { + count int + soll float64 + haben float64 + reFehlt int +} + +type rec struct { + date, month, konto, text string + soll, haben float64 + reFehlt bool +} + +var dateRe = regexp.MustCompile(`^\d{2}\.\d{2}\.\d{4}$`) + +func parseAmount(s string) float64 { + s = strings.TrimSpace(s) + s = strings.ReplaceAll(s, "€", "") + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, ",", "") // German thousands separator + s = strings.TrimSpace(s) + if s == "" { + return 0 + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return v +} + +func cell(row []string, i int) string { + if i < len(row) { + return strings.TrimSpace(row[i]) + } + return "" +} + +func main() { + if len(os.Args) < 3 { + fmt.Fprintln(os.Stderr, "usage: kontoblatt ") + os.Exit(2) + } + fileID, path := os.Args[1], os.Args[2] + ctx := context.Background() + + f, err := excelize.OpenFile(path) + if err != nil { + panic(err) + } + defer f.Close() + + var recs []rec + for _, sh := range f.GetSheetList() { + rows, err := f.GetRows(sh) + if err != nil { + continue + } + for _, r := range rows { + d := cell(r, 0) + if !dateRe.MatchString(d) { + continue + } + text := cell(r, 2) + recs = append(recs, rec{ + date: d, + month: d[3:10], + konto: cell(r, 1), + text: text, + soll: parseAmount(cell(r, 4)), + haben: parseAmount(cell(r, 5)), + reFehlt: strings.Contains(strings.ToUpper(text), "FEHLT"), + }) + } + } + + byKonto := map[string]*agg{} + byMonth := map[string]*agg{} + getK := func(k string) *agg { + if byKonto[k] == nil { + byKonto[k] = &agg{} + } + return byKonto[k] + } + getM := func(k string) *agg { + if byMonth[k] == nil { + byMonth[k] = &agg{} + } + return byMonth[k] + } + var tot agg + for _, r := range recs { + k := getK(r.konto) + k.count++ + k.soll += r.soll + k.haben += r.haben + if r.reFehlt { + k.reFehlt++ + } + m := getM(r.month) + m.count++ + m.soll += r.soll + m.haben += r.haben + if r.reFehlt { + m.reFehlt++ + } + tot.count++ + tot.soll += r.soll + tot.haben += r.haben + if r.reFehlt { + tot.reFehlt++ + } + } + + out := excelize.NewFile() + defer out.Close() + writeSheet(out, "Nach Gegenkonto", "Gegenkonto", byKonto, tot) + writeSheet(out, "Nach Monat", "Monat", byMonth, tot) + outPath := "/tmp/opencode/kontoblatt-zusammenfassung.xlsx" + if err := out.SaveAs(outPath); err != nil { + panic(err) + } + + // upload next to the source file + creds := onlyoffice.GetEnvironmentCredentials() + c := onlyoffice.NewClient(creds) + var src *onlyoffice.FileEntry + if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + var err error + src, err = c.GetFile(ctx, fileID) + return err + }); derr != nil { + panic(derr) + } + folder := "" + if src.FolderID != nil { + folder = src.FolderID.String() + } + title := "" + if src.Title != nil { + title = *src.Title + } + fmt.Printf("source: id=%s title=%q folder=%s\n", fileID, title, folder) + + name := "Kontoblatt-1591-2025-Zusammenfassung.xlsx" + tmp := "/tmp/opencode/" + name + data, _ := os.ReadFile(outPath) + if err := os.WriteFile(tmp, data, 0o600); err != nil { + panic(err) + } + var entry *onlyoffice.FileEntry + if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + var err error + entry, _, err = c.UploadToFolderReplacing(ctx, folder, tmp) + return err + }); derr != nil { + panic(derr) + } + fmt.Printf("uploaded: %s -> folder %s (id %v)\n", name, folder, entry.ID) + + // print the summary + printAgg("Nach Gegenkonto", byKonto, tot) + printAgg("Nach Monat", byMonth, tot) +} + +func writeSheet(f *excelize.File, sheet, key string, m map[string]*agg, tot agg) { + f.NewSheet(sheet) + rows := [][]any{{key, "Anzahl", "Soll", "Haben", "Saldo", `davon "fehlt"`}} + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + for _, k := range keys { + a := m[k] + rows = append(rows, []any{k, a.count, a.soll, a.haben, a.soll - a.haben, a.reFehlt}) + } + rows = append(rows, []any{"GESAMT", tot.count, tot.soll, tot.haben, tot.soll - tot.haben, tot.reFehlt}) + for i, row := range rows { + for j, v := range row { + cellRef, _ := excelize.CoordinatesToCellName(j+1, i+1) + _ = f.SetCellValue(sheet, cellRef, v) + } + } +} + +func printAgg(title string, m map[string]*agg, tot agg) { + fmt.Printf("\n== %s ==\n", title) + keys := make([]string, 0, len(m)) + for k := range m { + keys = append(keys, k) + } + sort.Strings(keys) + fmt.Printf("%-12s %6s %12s %12s %12s %7s\n", "key", "count", "soll", "haben", "saldo", "fehlt") + for _, k := range keys { + a := m[k] + fmt.Printf("%-12s %6d %12.2f %12.2f %12.2f %7d\n", k, a.count, a.soll, a.haben, a.soll-a.haben, a.reFehlt) + } + fmt.Printf("%-12s %6d %12.2f %12.2f %12.2f %7d\n", "GESAMT", tot.count, tot.soll, tot.haben, tot.soll-tot.haben, tot.reFehlt) +} diff --git a/cmd/kontolink/main.go b/cmd/kontolink/main.go new file mode 100644 index 0000000..a1b4dba --- /dev/null +++ b/cmd/kontolink/main.go @@ -0,0 +1,478 @@ +// Command kontolink fills the "Link" column of a Kontoblatt ("ungeklärte +// Posten") XLSX by matching each row to an OnlyOffice document. +// +// Strategy (deterministic, conservative — no LLM): +// 1. Beleg token (letters/digits from the "Beleg" column) appears in the file +// title; among candidates prefer (a) the row's month, (b) real invoices over +// copies/dupes, and require the result to be unique; +// 2. else supplier + row month + "rechnung", again unique. +// +// A file is linked at most once (rows already carrying a link are kept and their +// file counts as used). Ambiguous rows are left UNLINKED for manual review. +// +// Usage: kontolink +package main + +import ( + "context" + + "fmt" + "os" + "path/filepath" + "regexp" + "strconv" + "strings" + "time" + + onlyoffice "github.com/eslider/go-onlyoffice" + "github.com/xuri/excelize/v2" +) + +var ( + dateRe = regexp.MustCompile(`^\d{2}\.\d{2}\.\d{4}$`) + nonAln = regexp.MustCompile(`[^0-9a-z]+`) + fileID = regexp.MustCompile(`fileid=(\d+)`) +) + +func parseDay(s string) (time.Time, bool) { + t, err := time.Parse("02.01.2006", strings.TrimSpace(s)) + return t, err == nil +} + +func titleDay(title string) (time.Time, bool) { + if len(title) >= 10 { + if t, err := time.Parse("2006-01-02", title[:10]); err == nil { + return t, true + } + } + return time.Time{}, false +} + +// nearest picks the candidate whose title date is closest to rd. Ties and +// undated candidates (when >1) are rejected. +func nearest(cands []entry, rd time.Time) (entry, bool) { + if len(cands) == 1 { + return cands[0], true + } + best, bestD, tie := -1, 0.0, false + for i, e := range cands { + td, ok := titleDay(e.title) + if !ok { + continue + } + d := td.Sub(rd).Hours() / 24 + if d < 0 { + d = -d + } + if best < 0 || d < bestD { + best, bestD, tie = i, d, false + } else if d == bestD { + tie = true + } + } + if best < 0 || tie { + return entry{}, false + } + return cands[best], true +} + +const linkPrefix = "https://office.pro-dukt.de/Products/Files/DocEditor.aspx?fileid=" + +type entry struct { + id, path, title, norm string +} + +func norm(s string) string { return nonAln.ReplaceAllString(strings.ToLower(s), "") } + +func main() { + if len(os.Args) < 4 { + fmt.Fprintln(os.Stderr, "usage: kontolink ") + os.Exit(2) + } + in, idxPath, out := os.Args[1], os.Args[2], os.Args[3] + + idxRaw, err := os.ReadFile(idxPath) + if err != nil { + panic(err) + } + var entries []entry + for _, line := range strings.Split(string(idxRaw), "\n") { + parts := strings.Split(line, "\t") + if len(parts) < 4 || parts[0] == "" { + continue + } + entries = append(entries, entry{id: parts[0], path: parts[2], title: parts[3], norm: norm(parts[3])}) + } + + f, err := excelize.OpenFile(in) + if err != nil { + panic(err) + } + defer f.Close() + sheet := f.GetSheetList()[0] + rows, err := f.GetRows(sheet) + if err != nil { + panic(err) + } + + // optional 5th arg: amounts TSV "file_id\ttitle\tamount" (see cmd/pdfamount) + var amts []amtEntry + if len(os.Args) >= 6 && os.Args[5] != "" { + amts = loadAmounts(os.Args[5]) + } + + used := map[string]bool{} + for _, r := range rows { + if m := fileID.FindStringSubmatch(cell(r, 7)); m != nil { + used[m[1]] = true + } + } + + var linked, byBeleg, bySupplier, byAmount, unmatched, ambiguous int + for i, r := range rows { + if i == 0 || !dateRe.MatchString(cell(r, 0)) || strings.TrimSpace(cell(r, 7)) != "" { + continue + } + beleg := norm(cell(r, 3)) + supplier := supplierNorm(cell(r, 2)) + month := monthYear(cell(r, 0)) + rd, _ := parseDay(cell(r, 0)) + + e, kind, ok := pick(entries, used, beleg, supplier, month, rd) + if !ok { + if ae, aok := amountPick(amts, used, supplier, rowAmount(r), rd); aok { + e, kind, ok = entry{id: ae.id, title: ae.title}, "amount", true + } + } + if !ok { + if beleg != "" { + ambiguous++ + } else { + unmatched++ + } + continue + } + ref, _ := excelize.CoordinatesToCellName(8, i+1) + if err := f.SetCellValue(sheet, ref, linkPrefix+e.id); err != nil { + panic(err) + } + used[e.id] = true + linked++ + switch kind { + case "beleg": + byBeleg++ + case "supplier": + bySupplier++ + case "amount": + byAmount++ + } + fmt.Printf("row %3d %-30s -> %s [%s]\n", i+1, cell(r, 2), e.title, kind) + } + + if err := f.SaveAs(out); err != nil { + panic(err) + } + fmt.Printf("\nlinked=%d (beleg=%d, supplier=%d, amount=%d), ambiguous=%d, no-candidate=%d\n", + linked, byBeleg, bySupplier, byAmount, ambiguous, unmatched) + + // Optional 4th arg: source OnlyOffice file id. Try to update it in place; + // if it is locked (OnlyOffice 500), upload a "(links)" copy next to it. + if len(os.Args) >= 5 && os.Args[4] != "" { + c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials()) + ctx, cancel := context.WithTimeout(context.Background(), 120*time.Second) + defer cancel() + var src *onlyoffice.FileEntry + if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + var err error + src, err = c.GetFile(ctx, os.Args[4]) + return err + }); derr != nil { + panic(derr) + } + folder, title := "", "" + if src.FolderID != nil { + folder = src.FolderID.String() + } + if src.Title != nil { + title = *src.Title + } + uderr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + _, err := c.UpdateFile(ctx, os.Args[4], out) + return err + }) + if uderr == nil { + fmt.Printf("updated file %s in place\n", os.Args[4]) + return + } + fmt.Printf("in-place update failed (locked?); uploading a copy to folder %s\n", folder) + ext := filepath.Ext(title) + name := strings.TrimSuffix(title, ext) + " (links)" + ext + tmp := filepath.Join(os.TempDir(), name) + data, _ := os.ReadFile(out) + if err := os.WriteFile(tmp, data, 0o600); err != nil { + panic(err) + } + if derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + _, _, err := c.UploadToFolderReplacing(ctx, folder, tmp) + return err + }); derr != nil { + panic(derr) + } + fmt.Printf("uploaded copy: %s -> folder %s\n", name, folder) + } +} + +func cell(r []string, i int) string { + if i < len(r) { + return strings.TrimSpace(r[i]) + } + return "" +} + +func supplierNorm(s string) string { + s = strings.ToUpper(s) + if i := strings.Index(s, ","); i >= 0 { + s = s[:i] + } + for _, w := range []string{"RE FEHLT", "GS FEHLT", "WOFR", "WOFÜR"} { + s = strings.ReplaceAll(s, w, "") + } + return norm(s) +} + +func monthYear(date string) string { + if len(date) == 10 { + return date[6:10] + "-" + date[3:5] + } + return "" +} + +// pick returns an unused candidate. Beleg match wins; supplier+month is a +// fallback. When several candidates qualify, the one closest in time to the row +// date wins; a tie is rejected (ambiguous) rather than guessed. +func pick(entries []entry, used map[string]bool, beleg, supplier, month string, rd time.Time) (entry, string, bool) { + free := func(e entry) bool { return !used[e.id] } + + if len(beleg) >= 5 { + var inMonth []entry + for _, e := range entries { + if free(e) && belegMatches(e.norm, beleg) && + (month == "" || strings.Contains(e.title, month)) { + inMonth = append(inMonth, e) + } + } + if supplier != "" { + var s []entry + for _, e := range inMonth { + if strings.Contains(e.norm, supplier) { + s = append(s, e) + } + } + if len(s) > 0 { + inMonth = s + } + } + inMonth = topRank(inMonth) + if e, ok := nearest(inMonth, rd); ok { + return e, "beleg", true + } + // A Beleg is present but no file carries it: do NOT fall back to a + // supplier guess (that links the wrong invoice). + return entry{}, "", false + } + + if supplier != "" && month != "" { + var c []entry + for _, e := range entries { + if free(e) && strings.Contains(e.norm, supplier) && + strings.Contains(e.title, month) && strings.Contains(e.norm, "rechnung") { + c = append(c, e) + } + } + c = topRank(c) + if e, ok := nearest(c, rd); ok { + return e, "supplier", true + } + } + return entry{}, "", false +} + +// topRank keeps only the highest-ranked candidates (real invoice over copy / +// dupe / op), so a tie with a duplicate does not mask the real file. +func topRank(cands []entry) []entry { + if len(cands) < 2 { + return cands + } + best := 0 + for _, e := range cands { + if rank(e) > best { + best = rank(e) + } + } + out := cands[:0] + for _, e := range cands { + if rank(e) == best { + out = append(out, e) + } + } + return out +} + +func rank(e entry) int { + s := 0 + if strings.Contains(e.path, "/2025") || strings.Contains(e.path, "/2024") { + s += 4 + } + if strings.Contains(e.norm, "rechnung") { + s += 2 + } + if strings.Contains(e.norm, "dupe") || strings.Contains(e.norm, "copy") || + strings.Contains(e.norm, "op") { + s-- + } + return s +} + +// belegMatches reports whether a Beleg identifies the file: the whole normalized +// Beleg appears, or (for long numeric Belege, e.g. "24/641393110") an 8-digit +// window of its longest digit run appears. +func belegMatches(titleNorm, beleg string) bool { + if strings.Contains(titleNorm, beleg) { + return true + } + run := longestDigitRun(beleg) + for i := 0; i+8 <= len(run); i++ { + if strings.Contains(titleNorm, run[i:i+8]) { + return true + } + } + return false +} + +func longestDigitRun(s string) string { + var best, cur strings.Builder + for _, r := range s { + if r >= '0' && r <= '9' { + cur.WriteRune(r) + if cur.Len() > best.Len() { + best.Reset() + best.WriteString(cur.String()) + } + } else { + cur.Reset() + } + } + return best.String() +} + +type amtEntry struct { + id string + title string + norm string + amount float64 + date time.Time + hasDate bool +} + +func loadAmounts(path string) []amtEntry { + raw, err := os.ReadFile(path) + if err != nil { + return nil + } + var out []amtEntry + for _, line := range strings.Split(string(raw), "\n") { + p := strings.Split(line, "\t") + if len(p) < 3 { + continue + } + v, err := strconv.ParseFloat(strings.TrimSpace(p[2]), 64) + if err != nil { + continue + } + e := amtEntry{id: p[0], title: p[1], norm: norm(p[1]), amount: v} + if len(p[1]) >= 10 { + if t, err := time.Parse("2006-01-02", p[1][:10]); err == nil { + e.date, e.hasDate = t, true + } + } + out = append(out, e) + } + return out +} + +func rowAmount(r []string) float64 { + if v := parseAmount(cell(r, 4)); v != 0 { + return v + } + return parseAmount(cell(r, 5)) +} + +func parseAmount(s string) float64 { + s = strings.ReplaceAll(s, "€", "") + s = strings.ReplaceAll(s, " ", "") + s = strings.ReplaceAll(s, ",", ".") + if s == "" { + return 0 + } + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return 0 + } + return v +} + +// amountPick matches a row to an O2 invoice by amount + nearest date. Scoped to +// Telefonica/O2 rows and O2 files, so it cannot cross-link other suppliers. +func amountPick(amts []amtEntry, used map[string]bool, supplier string, amt float64, rd time.Time) (amtEntry, bool) { + if amt <= 0 || len(amts) == 0 { + return amtEntry{}, false + } + if !strings.Contains(supplier, "telefonica") && !strings.Contains(supplier, "o2") { + return amtEntry{}, false + } + var cands []amtEntry + for _, a := range amts { + if used[a.id] || !strings.Contains(a.norm, "o2") { + continue + } + d := a.amount - amt + if d < 0 { + d = -d + } + if d > 0.005 { + continue + } + if a.hasDate && !rd.IsZero() { + days := a.date.Sub(rd).Hours() / 24 + if days < 0 { + days = -days + } + if days > 75 { + continue + } + } + cands = append(cands, a) + } + if len(cands) == 1 { + return cands[0], true + } + best, bestD, tie := -1, 0.0, false + for i, a := range cands { + if !a.hasDate { + continue + } + d := a.date.Sub(rd).Hours() / 24 + if d < 0 { + d = -d + } + if best < 0 || d < bestD { + best, bestD, tie = i, d, false + } else if d == bestD { + tie = true + } + } + if best < 0 || tie { + return amtEntry{}, false + } + return cands[best], true +} diff --git a/cmd/oo/dav.go b/cmd/oo/dav.go new file mode 100644 index 0000000..4683444 --- /dev/null +++ b/cmd/oo/dav.go @@ -0,0 +1,291 @@ +package main + +import ( + "fmt" + "os" + + onlyoffice "github.com/eslider/go-onlyoffice" + "github.com/spf13/cobra" +) + +func init() { + rootCmd.AddCommand(davCmd()) +} + +// davCmd exposes the Documents module through the same Dav calls that back +// oo-webdav (ListDavFolder / MoveDavItems / CopyDavItems / DownloadDavFile). +// MoveDavItems sends resolveType=Skip + holdResult=true, which the legacy +// fileops/move call without those params silently ignores (200 without move). +func davCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "dav", + Short: "Documents module by folder/file id (oo-webdav proven path)", + } + cmd.AddCommand(davLsCmd()) + cmd.AddCommand(davMoveCmd()) + cmd.AddCommand(davCopyCmd()) + cmd.AddCommand(davMkdirCmd()) + cmd.AddCommand(davRenameFileCmd()) + cmd.AddCommand(davRenameFolderCmd()) + cmd.AddCommand(davDownloadCmd()) + cmd.AddCommand(davFileOpsCmd()) + return cmd +} + +func davLsCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "ls FOLDER_ID", + Short: "List a Documents folder (@root for virtual sections)", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + ctx := cmd.Context() + if args[0] == "@root" { + sections, err := c.ListDavSections(ctx) + if err != nil { + return err + } + rows := make([]map[string]any, 0, len(sections)) + for _, s := range sections { + rows = append(rows, map[string]any{ + "id": s.ID, + "title": s.Title, + "filesCount": s.FilesCount, + "foldersCount": s.FoldersCount, + }) + } + printTable([]string{"id", "title", "filesCount", "foldersCount"}, rows) + return nil + } + l, err := c.ListDavFolder(ctx, args[0]) + if err != nil { + return err + } + if outputFormat == "json" { + folders := make([]map[string]any, 0, len(l.Folders)) + for _, f := range l.Folders { + folders = append(folders, map[string]any{ + "id": f.ID, + "title": f.Title, + "filesCount": f.FilesCount, + "foldersCount": f.FoldersCount, + }) + } + files := make([]map[string]any, 0, len(l.Files)) + for _, f := range l.Files { + files = append(files, map[string]any{ + "id": f.ID, + "title": f.Title, + "size": f.Size, + "updated": f.Updated, + }) + } + printObject(map[string]any{"folders": folders, "files": files}) + return nil + } + if len(l.Folders) > 0 { + frows := make([]map[string]any, 0, len(l.Folders)) + for _, f := range l.Folders { + frows = append(frows, map[string]any{ + "id": f.ID, + "title": f.Title, + "filesCount": f.FilesCount, + "foldersCount": f.FoldersCount, + }) + } + if outputFormat == "table" { + fmt.Println("folders:") + } + printTable([]string{"id", "title", "filesCount", "foldersCount"}, frows) + } + rows := make([]map[string]any, 0, len(l.Files)) + for _, f := range l.Files { + rows = append(rows, map[string]any{ + "id": f.ID, + "title": f.Title, + "size": f.Size, + "updated": f.Updated, + }) + } + if outputFormat == "table" { + fmt.Println("files:") + } + printTable([]string{"id", "title", "size", "updated"}, rows) + return nil + }, + } + return cmd +} + +func davMoveCmd() *cobra.Command { + var folderIDs []string + cmd := &cobra.Command{ + Use: "move DEST_FOLDER_ID FILE_ID [FILE_ID...]", + Short: "Move file(s) into a Documents folder (resolveType=Skip, holdResult)", + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + if err := c.MoveDavItems(cmd.Context(), folderIDs, args[1:], args[0]); err != nil { + return err + } + printObject(map[string]any{"moved_files": args[1:], "moved_folders": folderIDs, "dest": args[0]}) + return nil + }, + } + cmd.Flags().StringSliceVar(&folderIDs, "folders", nil, "folder ids to move along with the files") + return cmd +} + +func davCopyCmd() *cobra.Command { + var folderIDs []string + cmd := &cobra.Command{ + Use: "copy DEST_FOLDER_ID FILE_ID [FILE_ID...]", + Short: "Copy file(s) into a Documents folder (conflictResolveType=Skip)", + Args: cobra.MinimumNArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + if err := c.CopyDavItems(cmd.Context(), folderIDs, args[1:], args[0]); err != nil { + return err + } + printObject(map[string]any{"copied_files": args[1:], "copied_folders": folderIDs, "dest": args[0]}) + return nil + }, + } + cmd.Flags().StringSliceVar(&folderIDs, "folders", nil, "folder ids to copy along with the files") + return cmd +} + +func davMkdirCmd() *cobra.Command { + return &cobra.Command{ + Use: "mkdir PARENT_FOLDER_ID TITLE", + Short: "Create a subfolder in Documents", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + f, err := c.CreateDavFolder(cmd.Context(), args[0], args[1]) + if err != nil { + return err + } + printObject(map[string]any{"id": f.ID, "title": f.Title, "parent": args[0]}) + return nil + }, + } +} + +func davRenameFileCmd() *cobra.Command { + return &cobra.Command{ + Use: "rename-file FILE_ID NEW_TITLE", + Short: "Rename a Documents file (include extension in NEW_TITLE)", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + if err := c.RenameDavFile(cmd.Context(), args[0], args[1]); err != nil { + return err + } + printObject(map[string]any{"id": args[0], "title": args[1]}) + return nil + }, + } +} + +func davRenameFolderCmd() *cobra.Command { + return &cobra.Command{ + Use: "rename-folder FOLDER_ID NEW_TITLE", + Short: "Rename a Documents folder", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + if err := c.RenameDavFolder(cmd.Context(), args[0], args[1]); err != nil { + return err + } + printObject(map[string]any{"id": args[0], "title": args[1]}) + return nil + }, + } +} + +func davDownloadCmd() *cobra.Command { + var to string + cmd := &cobra.Command{ + Use: "download FILE_ID", + Short: "Download Documents file bytes (default path: ./)", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + ctx := cmd.Context() + f, err := c.GetFile(ctx, args[0]) + if err != nil { + return err + } + path := to + if path == "" { + path = onlyoffice.SafeLocalFileName(onlyoffice.FileEntryTitle(f)) + } + out, err := os.Create(path) + if err != nil { + return err + } + defer out.Close() + n, err := c.DownloadDavFile(ctx, args[0], out) + if err != nil { + _ = os.Remove(path) + return err + } + printObject(map[string]any{"path": path, "bytes": n}) + return nil + }, + } + cmd.Flags().StringVar(&to, "to", "", "output path (default: ./<server title>)") + return cmd +} + +func davFileOpsCmd() *cobra.Command { + return &cobra.Command{ + Use: "fileops", + Short: "List active file operations (move/copy status polling)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + c, err := newOO(cmd) + if err != nil { + return err + } + ops, err := c.ListFileOps(cmd.Context()) + if err != nil { + return err + } + rows := make([]map[string]any, 0, len(ops)) + for _, op := range ops { + rows = append(rows, map[string]any{ + "id": fmt.Sprint(op["id"]), + "operation": fmt.Sprint(op["operation"]), + "progress": fmt.Sprint(op["progress"]), + "finished": fmt.Sprint(op["finished"]), + "error": fmt.Sprint(op["error"]), + }) + } + printTable([]string{"id", "operation", "progress", "finished", "error"}, rows) + return nil + }, + } +} diff --git a/cmd/oo/main.go b/cmd/oo/main.go index 63c2f83..7422a2d 100644 --- a/cmd/oo/main.go +++ b/cmd/oo/main.go @@ -16,6 +16,7 @@ // oo mails accounts | folders | list | get | download-attachment | draft | attach | draft-invoice | delete // oo invoices list | get | create | update | pdf | pdf-cleanup | status | delete | items … // oo docs tools | convert | ocr | as-md | put-md +// oo dav ls | move | copy | mkdir | rename-file | rename-folder | download | fileops // // CRM association rules: docs/crm-associations.md // diff --git a/cmd/ooscan/main.go b/cmd/ooscan/main.go new file mode 100644 index 0000000..7ef0276 --- /dev/null +++ b/cmd/ooscan/main.go @@ -0,0 +1,50 @@ +// Command ooscan recursively lists OnlyOffice Documents folders into a TSV +// index: file_id, folder_id, path, title. +// +// Usage: ooscan <FOLDER_ID> [<FOLDER_ID>...] +package main + +import ( + "context" + "fmt" + "os" + "time" + + onlyoffice "github.com/eslider/go-onlyoffice" +) + +func main() { + ctx := context.Background() + c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials()) + seen := map[string]bool{} + for _, root := range os.Args[1:] { + walk(ctx, c, root, "", 0, seen) + } +} + +func walk(ctx context.Context, c *onlyoffice.Client, folderID, path string, depth int, seen map[string]bool) { + if depth > 8 || seen[folderID] { + return + } + seen[folderID] = true + // Throttle: OnlyOffice rate-limits (429) and the host must not be flooded. + time.Sleep(350 * time.Millisecond) + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + var l *onlyoffice.DavListing + derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + var err error + l, err = c.ListDavFolder(ctx, folderID) + return err + }) + if derr != nil { + fmt.Fprintf(os.Stderr, "list %s (%s): %v\n", path, folderID, derr) + return + } + for _, f := range l.Files { + fmt.Printf("%s\t%s\t%s\t%s\n", f.ID, folderID, path, f.Title) + } + for _, sub := range l.Folders { + walk(ctx, c, sub.ID, path+"/"+sub.Title, depth+1, seen) + } +} diff --git a/cmd/pdfamount/main.go b/cmd/pdfamount/main.go new file mode 100644 index 0000000..d6b991e --- /dev/null +++ b/cmd/pdfamount/main.go @@ -0,0 +1,124 @@ +// Command pdfamount walks a Documents folder, downloads matching PDFs and +// extracts the payable amount, printing "file_id\ttitle\tamount". +// +// Usage: pdfamount <FOLDER_ID> [TITLE_FILTER_REGEX] +package main + +import ( + "bytes" + "context" + "fmt" + "os" + "os/exec" + "regexp" + "strconv" + "strings" + "time" + + onlyoffice "github.com/eslider/go-onlyoffice" +) + +var amountRe = regexp.MustCompile(`(?i)(zu zahlender betrag|rechnungsbetrag)\s*[:\s]*([0-9][0-9.]*,[0-9]{2})`) + +func main() { + if len(os.Args) < 2 { + fmt.Fprintln(os.Stderr, "usage: pdfamount <FOLDER_ID> [TITLE_FILTER_REGEX]") + os.Exit(2) + } + folder := os.Args[1] + filter := regexp.MustCompile(`(?i)rechnung`) + if len(os.Args) >= 3 { + filter = regexp.MustCompile(os.Args[2]) + } + ctx := context.Background() + c := onlyoffice.NewClient(onlyoffice.GetEnvironmentCredentials()) + + files := listAll(ctx, c, folder) + for _, f := range files { + if !filter.MatchString(f.title) { + continue + } + if !strings.HasSuffix(strings.ToLower(f.title), ".pdf") { + continue + } + amount, err := pdfAmount(ctx, c, f.id) + if err != nil { + fmt.Fprintf(os.Stderr, "%s: %v\n", f.title, err) + continue + } + if amount == "" { + continue + } + fmt.Printf("%s\t%s\t%s\n", f.id, f.title, amount) + } +} + +type file struct{ id, title string } + +func listAll(ctx context.Context, c *onlyoffice.Client, folder string) []file { + seen := map[string]bool{} + var out []file + var walk func(string) + walk = func(id string) { + if seen[id] { + return + } + seen[id] = true + time.Sleep(300 * time.Millisecond) + l, err := c.ListDavFolder(ctx, id) + if err != nil { + fmt.Fprintf(os.Stderr, "list %s: %v\n", id, err) + return + } + for _, f := range l.Files { + out = append(out, file{f.ID, f.Title}) + } + for _, sub := range l.Folders { + walk(sub.ID) + } + } + walk(folder) + return out +} + +func pdfAmount(ctx context.Context, c *onlyoffice.Client, id string) (string, error) { + time.Sleep(time.Second) + tmp, err := os.CreateTemp("", "pdf-*.pdf") + if err != nil { + return "", err + } + defer os.Remove(tmp.Name()) + derr := onlyoffice.DoRetry(ctx, onlyoffice.DefaultRetryPolicy(), func() error { + _ = tmp.Truncate(0) + _, _ = tmp.Seek(0, 0) + _, err := c.DownloadFile(ctx, id, tmp) + return err + }) + if derr != nil { + tmp.Close() + return "", derr + } + tmp.Close() + var buf bytes.Buffer + cmd := exec.CommandContext(ctx, "pdftotext", "-layout", tmp.Name(), "-") + cmd.Stdout = &buf + if err := cmd.Run(); err != nil { + return "", err + } + m := amountRe.FindStringSubmatch(buf.String()) + if m == nil { + return "", nil + } + return parseDe(m[2]), nil +} + +// parseDe turns "1.234,56" into 1234.56. +func parseDe(s string) string { + s = strings.ReplaceAll(s, ".", "") + s = strings.ReplaceAll(s, ",", ".") + v, err := strconv.ParseFloat(s, 64) + if err != nil { + return s + } + return strconv.FormatFloat(v, 'f', 2, 64) +} diff --git a/files.go b/files.go index 43a740e..4d0497b 100644 --- a/files.go +++ b/files.go @@ -321,11 +321,21 @@ func (c *Client) MoveFiles(ctx context.Context, destFolderID int, fileIDs []int) "folderIds": []int{}, "fileIds": fileIDs, "destFolderId": destFolderID, + "resolveType": "Skip", + "holdResult": true, } out, err := c.putJSONObject(ctx, "/api/2.0/files/fileops/move.json", body) if err != nil { out, err = c.putJSONObject(ctx, "/api/2.0/files/fileops/move", body) } + if err != nil { + return nil, err + } + if raw, merr := json.Marshal(out); merr == nil { + if ferr := fileopsError(raw); ferr != nil { + return nil, ferr + } + } return out, err } @@ -346,6 +356,24 @@ func (c *Client) UploadToFolder(ctx context.Context, folderID, localPath string) return decodeResponseFileEntry(raw) } +// UpdateFile uploads a new version of an existing file (same id, name and +// folder). It does not delete and does not create a second file. +func (c *Client) UpdateFile(ctx context.Context, fileID, localPath string) (*FileEntry, error) { + if fileID == "" || localPath == "" { + return nil, fmt.Errorf("file id and local path are required") + } + uploadPath := fmt.Sprintf("/api/2.0/files/%s/update", url.PathEscape(fileID)) + raw, err := c.uploadMultipart(ctx, uploadPath, "file", localPath) + if err != nil { + uploadPath = fmt.Sprintf("/api/2.0/files/%s/update.json", url.PathEscape(fileID)) + raw, err = c.uploadMultipart(ctx, uploadPath, "file", localPath) + if err != nil { + return nil, err + } + } + return decodeResponseFileEntry(raw) +} + // FileFolderID returns the parent folder id string for a file entry, if known. func FileFolderID(f *FileEntry) string { if f == nil || f.FolderID == nil { diff --git a/files_webdav.go b/files_webdav.go index f4d7e32..6775e80 100644 --- a/files_webdav.go +++ b/files_webdav.go @@ -144,27 +144,85 @@ func (c *Client) RenameDavFile(ctx context.Context, id, title string) error { } // MoveDavItems moves the given folders and/or files into destFolderID. +// The fileops API answers 200 with per-operation error strings even when +// nothing moves (e.g. missing permission), so the response is parsed and the +// first operation error is returned instead of a silent nil. func (c *Client) MoveDavItems(ctx context.Context, folderIDs, fileIDs []string, destFolderID string) error { - _, err := c.putJSON(ctx, "/api/2.0/files/fileops/move", map[string]any{ + raw, err := c.putJSON(ctx, "/api/2.0/files/fileops/move", map[string]any{ "folderIds": nums(folderIDs), "fileIds": nums(fileIDs), "destFolderId": num(destFolderID), "resolveType": "Skip", "holdResult": true, }) - return err + if err != nil { + return err + } + return fileopsError(raw) } // CopyDavItems copies the given folders and/or files into destFolderID. +// Per-operation errors are surfaced like in MoveDavItems. func (c *Client) CopyDavItems(ctx context.Context, folderIDs, fileIDs []string, destFolderID string) error { - _, err := c.putJSON(ctx, "/api/2.0/files/fileops/copy", map[string]any{ + raw, err := c.putJSON(ctx, "/api/2.0/files/fileops/copy", map[string]any{ "folderIds": nums(folderIDs), "fileIds": nums(fileIDs), "destFolderId": num(destFolderID), "conflictResolveType": "Skip", "deleteAfter": true, }) - return err + if err != nil { + return err + } + return fileopsError(raw) +} + +// ListFileOps returns the currently active file operations +// (GET /api/2.0/files/fileops) for status polling. +func (c *Client) ListFileOps(ctx context.Context) ([]map[string]any, error) { + raw, err := c.getJSON(ctx, "/api/2.0/files/fileops") + if err != nil { + return nil, err + } + resp, err := responseField(raw, "response") + if err != nil { + return nil, err + } + if len(resp) == 0 || string(resp) == "null" { + return nil, nil + } + var ops []map[string]any + if err := json.Unmarshal(resp, &ops); err != nil { + return nil, err + } + return ops, nil +} + +// fileopsError extracts per-operation "error" strings from a fileops/move or +// fileops/copy envelope. A 200 with error entries means nothing moved. +func fileopsError(raw json.RawMessage) error { + resp, err := responseField(raw, "response") + if err != nil { + return err + } + var ops []struct { + Error *string `json:"error"` + Finished *bool `json:"finished"` + Progress *int `json:"progress"` + } + if err := json.Unmarshal(resp, &ops); err != nil { + return nil // not an operations envelope — nothing to report + } + var errs []string + for _, op := range ops { + if op.Error != nil && *op.Error != "" { + errs = append(errs, *op.Error) + } + } + if len(errs) > 0 { + return fmt.Errorf("onlyoffice: fileops: %s", strings.Join(errs, "; ")) + } + return nil } // DeleteDavItems deletes the given folders and/or files. @@ -376,12 +434,12 @@ func (f *DavFolder) UnmarshalJSON(b []byte) error { // UnmarshalJSON decodes a file row, capturing size and timestamps. func (f *DavFile) UnmarshalJSON(b []byte) error { var raw struct { - ID *json.Number `json:"id"` - Title *string `json:"title"` - PureSize *int64 `json:"pureContentLength"` - SizeStr *string `json:"contentLength"` - Updated *string `json:"updated"` - ViewURL *string `json:"viewUrl"` + ID *json.Number `json:"id"` + Title *string `json:"title"` + PureSize *int64 `json:"pureContentLength"` + SizeStr *string `json:"contentLength"` + Updated *string `json:"updated"` + ViewURL *string `json:"viewUrl"` } if err := json.Unmarshal(b, &raw); err != nil { return err diff --git a/retry.go b/retry.go new file mode 100644 index 0000000..73210e9 --- /dev/null +++ b/retry.go @@ -0,0 +1,60 @@ +package onlyoffice + +import ( + "context" + "regexp" + "time" +) + +// RetryPolicy controls deterministic retries against OnlyOffice: fixed linear +// backoff without jitter, so repeated runs wait exactly the same schedule. +// OnlyOffice throttles bulk reads/writes with 429 (and occasional 502/503/504 +// from openresty), so every bulk tool routes API calls through DoRetry. +type RetryPolicy struct { + Attempts int // total attempts, including the first try + Base time.Duration // wait before retry N is N*Base + Max time.Duration // per-wait cap +} + +// DefaultRetryPolicy retries up to 5 times with 1s, 2s, 3s, 4s waits. +func DefaultRetryPolicy() RetryPolicy { + return RetryPolicy{Attempts: 5, Base: time.Second, Max: 30 * time.Second} +} + +var transientRe = regexp.MustCompile(`:\s*(429|502|503|504)\b`) + +// Transient reports whether err looks like a transient OnlyOffice answer +// (an HTTP 429/502/503/504 surfaced as "...: <code> ..."). +func Transient(err error) bool { + if err == nil { + return false + } + return transientRe.MatchString(err.Error()) +} + +// DoRetry runs fn until it succeeds, fails non-transiently, or attempts run +// out. Waits are deterministic: N*Base capped at Max, no jitter. +func DoRetry(ctx context.Context, p RetryPolicy, fn func() error) error { + if p.Attempts < 1 { + p.Attempts = 1 + } + var err error + for attempt := 1; attempt <= p.Attempts; attempt++ { + if ctx.Err() != nil { + return ctx.Err() + } + if err = fn(); err == nil || !Transient(err) || attempt == p.Attempts { + return err + } + wait := time.Duration(attempt) * p.Base + if wait > p.Max { + wait = p.Max + } + select { + case <-ctx.Done(): + return ctx.Err() + case <-time.After(wait): + } + } + return err +}