diff --git a/.env.example b/.env.example index 2e3e49e..bfcff31 100644 --- a/.env.example +++ b/.env.example @@ -25,3 +25,12 @@ ONLYOFFICE_PROJECT_ID=33 # cmd/office TUI — optional Document Server for DOCX→HTML preview: # ONLYOFFICE_DOCS_URL=https://docs.example.com # ONLYOFFICE_DOCS_SECRET= + +# MinIO download fallback for the portal's stale AWS S3 consumer (older +# Documents folders). When the portal redirects to amazonaws.com with access +# key "minio" (403 InvalidAccessKeyId), files are fetched from the local MinIO +# store instead. Without a key/secret the fallback is disabled. +# MINIO_ENDPOINT=http://192.168.188.10:9000 +# MINIO_BUCKET=office +# MINIO_ACCESS_KEY= +# MINIO_SECRET_KEY= diff --git a/files.go b/files.go index 4d0497b..3fe2ea3 100644 --- a/files.go +++ b/files.go @@ -5,7 +5,6 @@ import ( "encoding/json" "fmt" "io" - "net/http" "net/url" "path" "strconv" @@ -383,36 +382,15 @@ func FileFolderID(f *FileEntry) string { } // DownloadFile streams file bytes from the file's viewUrl using the same auth -// as API calls. Writes into dst. +// as API calls. Writes into dst. When the portal serves the file from its stale +// AWS S3 consumer, the bytes are fetched from the local MinIO store instead +// (see storage_fallback.go). func (c *Client) DownloadFile(ctx context.Context, fileID string, dst io.Writer) (int64, error) { f, err := c.GetFile(ctx, fileID) if err != nil { return 0, err } - if f.ViewURL == nil || *f.ViewURL == "" { - return 0, fmt.Errorf("file %s has no viewUrl", fileID) - } - downloadURL := c.resolveAPIURL(*f.ViewURL) - auth, err := c.authHeader() - if err != nil { - return 0, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", auth) - resp, err := c.client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) - return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400)) - } - n, err := io.Copy(dst, resp.Body) - return n, err + return c.downloadFileEntry(ctx, f, dst) } func (c *Client) resolveAPIURL(ref string) string { diff --git a/files_webdav.go b/files_webdav.go index 6775e80..536071a 100644 --- a/files_webdav.go +++ b/files_webdav.go @@ -259,34 +259,10 @@ func (c *Client) UploadDavFile(ctx context.Context, folderID, fileName string, s return env.Response, nil } -// DownloadDavFile streams the file identified by id to w, returning bytes copied. +// DownloadDavFile streams the file identified by id to w, returning bytes +// copied. It shares the MinIO stale-S3 fallback with DownloadFile. func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) { - file, err := c.GetFile(ctx, id) - if err != nil { - return 0, err - } - if file.ViewURL == nil || *file.ViewURL == "" { - return 0, fmt.Errorf("onlyoffice: file %s has no viewUrl", id) - } - u := c.resolveAPIURL(*file.ViewURL) - auth, err := c.authHeader() - if err != nil { - return 0, err - } - req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil) - if err != nil { - return 0, err - } - req.Header.Set("Authorization", auth) - resp, err := c.client.Do(req) - if err != nil { - return 0, err - } - defer resp.Body.Close() - if resp.StatusCode >= 400 { - return 0, fmt.Errorf("onlyoffice: download: %d", resp.StatusCode) - } - return io.Copy(w, resp.Body) + return c.DownloadFile(ctx, id, w) } // --- internal helpers ------------------------------------------------------- diff --git a/go.mod b/go.mod index efbac23..e988b59 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.25.0 require ( github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2 + github.com/aws/aws-sdk-go-v2 v1.41.1 github.com/charmbracelet/bubbles v0.18.0 github.com/charmbracelet/bubbletea v0.25.0 github.com/charmbracelet/glamour v0.8.0 @@ -26,6 +27,7 @@ require ( github.com/JohannesKaufmann/dom v0.3.1 // indirect github.com/alecthomas/chroma/v2 v2.14.0 // indirect github.com/atotto/clipboard v0.1.4 // indirect + github.com/aws/smithy-go v1.24.0 // indirect github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect github.com/aymerick/douceur v0.2.0 // indirect github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect diff --git a/go.sum b/go.sum index f3dff4b..0bef785 100644 --- a/go.sum +++ b/go.sum @@ -10,6 +10,10 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4= github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4= github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI= +github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU= +github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0= +github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk= +github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0= github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k= github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8= github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8= diff --git a/storage_fallback.go b/storage_fallback.go new file mode 100644 index 0000000..1d5db52 --- /dev/null +++ b/storage_fallback.go @@ -0,0 +1,200 @@ +package onlyoffice + +// MinIO download fallback for the portal's stale AWS S3 consumer. +// +// On the Fibu EDL portal some older Documents files live in S3/MinIO, but the +// portal's storage consumer still points at s3.us-east-1.amazonaws.com with +// access key "minio". Downloads of those files answer 403 InvalidAccessKeyId. +// The bytes are present in the local MinIO store under a deterministic object +// key, so the client retries the GET there. + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "net/http" + "net/url" + "os" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/signer/v4" +) + +const ( + defaultMinioEndpoint = "http://192.168.188.10:9000" + defaultMinioBucket = "office" + minioRegion = "us-east-1" +) + +// minioObjectKey is the fallback object key layout the portal's S3 consumer +// writes for Documents files: 00/00/01/files/folder_/file_/v1/content.pdf. +// Prefer minioObjectKeyFromURL: the portal stores all files below its storage +// root folder, which is not the API folderId returned by GetFile. +func minioObjectKey(fileID, folderID string) string { + return "00/00/01/files/folder_" + folderID + "/file_" + fileID + "/v1/content.pdf" +} + +// minioObjectKeyFromURL extracts the object key from an S3 download URL. Path +// style URLs (bucket as first path segment) have that segment removed; virtual +// host style URLs are returned as-is. This is authoritative: the portal signs +// the exact key, so no folder-id guessing is needed. +func minioObjectKeyFromURL(rawURL, bucket string) (string, bool) { + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Path == "" { + return "", false + } + segs := strings.Split(strings.Trim(u.Path, "/"), "/") + // Path-style URLs carry the bucket as leading segment; the portal's S3 + // consumer can emit it twice (serviceurl already includes the bucket), so + // strip every leading segment equal to the bucket. + for len(segs) > 0 && bucket != "" && segs[0] == bucket { + segs = segs[1:] + } + if len(segs) == 0 { + return "", false + } + for _, s := range segs { + if s == "" || s == "." || s == ".." { + return "", false + } + } + return strings.Join(segs, "/"), true +} + +// isStaleS3Redirect reports whether a download landed on the portal's stale AWS +// S3 consumer. Such responses either carry an S3 InvalidAccessKeyId XML body or +// point at amazonaws.com with the "minio" access key id in the query. +func isStaleS3Redirect(rawURL string, body []byte) bool { + if strings.Contains(strings.ToLower(string(body)), "invalidaccesskeyid") { + return true + } + u, err := url.Parse(strings.TrimSpace(rawURL)) + if err != nil || u.Host == "" { + return false + } + host := strings.ToLower(u.Host) + if !strings.Contains(host, "amazonaws.com") { + return false + } + q := strings.ToLower(u.RawQuery) + return strings.Contains(q, "accesskeyid=minio") || strings.Contains(q, "x-amz-credential=minio") +} + +// minioConfig is the runtime configuration for the local MinIO fallback. +type minioConfig struct { + Endpoint string + Bucket string + AccessKey string + SecretKey string +} + +// loadMinioConfig reads the fallback configuration from the environment. +// Secrets are never defaulted; without access/secret keys the fallback is off. +func loadMinioConfig() minioConfig { + return minioConfig{ + Endpoint: strings.TrimRight(firstNonEmpty(os.Getenv("MINIO_ENDPOINT"), defaultMinioEndpoint), "/"), + Bucket: firstNonEmpty(os.Getenv("MINIO_BUCKET"), defaultMinioBucket), + AccessKey: os.Getenv("MINIO_ACCESS_KEY"), + SecretKey: os.Getenv("MINIO_SECRET_KEY"), + } +} + +// downloadFileEntry downloads f's bytes to dst. It transparently falls back to +// the local MinIO store when the portal redirects the download to its stale AWS +// S3 consumer. +func (c *Client) downloadFileEntry(ctx context.Context, f *FileEntry, dst io.Writer) (int64, error) { + if f == nil { + return 0, fmt.Errorf("onlyoffice: download: nil file entry") + } + if f.ViewURL == nil || *f.ViewURL == "" { + return 0, fmt.Errorf("onlyoffice: file has no viewUrl") + } + downloadURL := c.resolveAPIURL(*f.ViewURL) + auth, err := c.authHeader() + if err != nil { + return 0, err + } + req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil) + if err != nil { + return 0, err + } + req.Header.Set("Authorization", auth) + resp, err := c.client.Do(req) + if err != nil { + return 0, err + } + defer resp.Body.Close() + + if resp.StatusCode >= 400 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096)) + finalURL := downloadURL + if resp.Request != nil && resp.Request.URL != nil { + finalURL = resp.Request.URL.String() + } + if isStaleS3Redirect(finalURL, b) { + key, ok := minioObjectKeyFromURL(finalURL, loadMinioConfig().Bucket) + if !ok { + fileID := "" + if f.ID != nil { + fileID = f.ID.String() + } + key = minioObjectKey(fileID, FileFolderID(f)) + } + n, merr := c.downloadFromMinio(ctx, key, dst) + if merr == nil { + return n, nil + } + return 0, fmt.Errorf("GET viewUrl: %d (stale S3) and minio fallback: %w", resp.StatusCode, merr) + } + return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400)) + } + return io.Copy(dst, resp.Body) +} + +// downloadFromMinio streams objectKey from the configured MinIO bucket. +func (c *Client) downloadFromMinio(ctx context.Context, objectKey string, dst io.Writer) (int64, error) { + if objectKey == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: empty object key") + } + cfg := loadMinioConfig() + if cfg.AccessKey == "" || cfg.SecretKey == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") + } + base, err := url.Parse(cfg.Endpoint) + if err != nil || base.Host == "" { + return 0, fmt.Errorf("onlyoffice: minio fallback: bad MINIO_ENDPOINT %q", cfg.Endpoint) + } + u := *base + u.Path = "/" + cfg.Bucket + "/" + objectKey + req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) + if err != nil { + return 0, err + } + if err := signMinioRequest(ctx, cfg, req); err != nil { + return 0, err + } + resp, err := c.client.Do(req) + if err != nil { + return 0, fmt.Errorf("onlyoffice: minio fallback: %w", err) + } + defer resp.Body.Close() + if resp.StatusCode >= 400 { + b, _ := io.ReadAll(io.LimitReader(resp.Body, 512)) + return 0, fmt.Errorf("onlyoffice: minio fallback: %d %s", resp.StatusCode, truncate(string(b), 300)) + } + return io.Copy(dst, resp.Body) +} + +// signMinioRequest signs req with AWS Signature V4 for the S3 service. +func signMinioRequest(ctx context.Context, cfg minioConfig, req *http.Request) error { + sum := sha256.Sum256(nil) + creds := aws.Credentials{AccessKeyID: cfg.AccessKey, SecretAccessKey: cfg.SecretKey} + if err := v4.NewSigner().SignHTTP(ctx, creds, req, hex.EncodeToString(sum[:]), "s3", minioRegion, time.Now()); err != nil { + return fmt.Errorf("onlyoffice: minio fallback: sign: %w", err) + } + return nil +} diff --git a/storage_fallback_integration_test.go b/storage_fallback_integration_test.go new file mode 100644 index 0000000..9e90f0b --- /dev/null +++ b/storage_fallback_integration_test.go @@ -0,0 +1,45 @@ +//go:build integration + +package onlyoffice + +import ( + "context" + "io" + "os" + "strings" + "testing" + "time" +) + +// TestIntegrationMinioFallback downloads known stale-S3 files through the local +// MinIO fallback. Requires ONLYOFFICE_URL/USER/PASS (as all integration tests), +// MINIO_ACCESS_KEY/MINIO_SECRET_KEY and MINIO_TEST_FILE_IDS="3785,3859,3666"; +// skips when any of those are missing. +func TestIntegrationMinioFallback(t *testing.T) { + if os.Getenv("MINIO_ACCESS_KEY") == "" || os.Getenv("MINIO_SECRET_KEY") == "" { + t.Skip("MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set — skipping integration test") + } + raw := strings.TrimSpace(os.Getenv("MINIO_TEST_FILE_IDS")) + if raw == "" { + t.Skip("MINIO_TEST_FILE_IDS not set — skipping integration test") + } + c := liveClient(t) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) + defer cancel() + for _, id := range strings.Split(raw, ",") { + id = strings.TrimSpace(id) + if id == "" { + continue + } + n, err := c.DownloadFile(ctx, id, io.Discard) + if err != nil { + t.Errorf("DownloadFile(%s): %v", id, err) + continue + } + if n == 0 { + t.Errorf("DownloadFile(%s): 0 bytes", id) + } else { + t.Logf("DownloadFile(%s): %d bytes", id, n) + } + } +} diff --git a/storage_fallback_test.go b/storage_fallback_test.go new file mode 100644 index 0000000..7928e89 --- /dev/null +++ b/storage_fallback_test.go @@ -0,0 +1,219 @@ +package onlyoffice + +import ( + "bytes" + "context" + "io" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestMinioObjectKey(t *testing.T) { + cases := []struct { + fileID string + folderID string + want string + }{ + {"3785", "652", "00/00/01/files/folder_652/file_3785/v1/content.pdf"}, + {"1", "2", "00/00/01/files/folder_2/file_1/v1/content.pdf"}, + {"3666", "4000", "00/00/01/files/folder_4000/file_3666/v1/content.pdf"}, + } + for _, tc := range cases { + if got := minioObjectKey(tc.fileID, tc.folderID); got != tc.want { + t.Errorf("minioObjectKey(%q, %q) = %q, want %q", tc.fileID, tc.folderID, got, tc.want) + } + } +} + +func TestMinioObjectKeyFromURL(t *testing.T) { + cases := []struct { + name string + url string + bucket string + want string + ok bool + }{ + { + name: "path style drops bucket segment", + url: "https://s3.us-east-1.amazonaws.com/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "doubled bucket segment (portal serviceurl includes bucket)", + url: "https://s3.us-east-1.amazonaws.com/office/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "virtual host style keeps path", + url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_4000/file_3785/v1/content.pdf", + bucket: "office", + want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf", + ok: true, + }, + { + name: "foreign first segment kept", + url: "https://example.com/other/file_1/v1/content.pdf", + bucket: "office", + want: "other/file_1/v1/content.pdf", + ok: true, + }, + {name: "empty path", url: "https://example.com", bucket: "office", ok: false}, + {name: "traversal", url: "https://example.com/office/../etc/passwd", bucket: "office", ok: false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + got, ok := minioObjectKeyFromURL(tc.url, tc.bucket) + if ok != tc.ok || got != tc.want { + t.Errorf("minioObjectKeyFromURL(%q, %q) = (%q, %v), want (%q, %v)", tc.url, tc.bucket, got, ok, tc.want, tc.ok) + } + }) + } +} + +func TestIsStaleS3Redirect(t *testing.T) { + cases := []struct { + name string + url string + body []byte + want bool + }{ + { + name: "aws redirect with minio access key", + url: "https://s3.us-east-1.amazonaws.com/office/x/file_1?AWSAccessKeyId=minio&Expires=1", + want: true, + }, + { + name: "aws redirect with minio x-amz-credential", + url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_1/file_1?X-Amz-Credential=minio%2F20260914", + want: true, + }, + { + name: "invalid access key xml body", + url: "https://portal.internal/download/1", + body: []byte(`InvalidAccessKeyIdminio`), + want: true, + }, + { + name: "regular pdf from portal", + url: "https://portal.internal/download/1", + body: []byte("%PDF-1.7 data"), + want: false, + }, + { + name: "aws redirect with foreign key", + url: "https://s3.us-east-1.amazonaws.com/office/x?AWSAccessKeyId=other", + want: false, + }, + { + name: "amazonaws in path but foreign host", + url: "https://example.com/amazonaws.com/file?AWSAccessKeyId=minio", + want: false, + }, + { + name: "empty", + want: false, + }, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + if got := isStaleS3Redirect(tc.url, tc.body); got != tc.want { + t.Errorf("isStaleS3Redirect(%q, %q) = %v, want %v", tc.url, tc.body, got, tc.want) + } + }) + } +} + +const staleS3Body = `` + + `InvalidAccessKeyId` + + `The AWS Access Key Id you provided does not exist in our records.` + + `minio` + +func TestDownloadFileMinioFallback(t *testing.T) { + const payload = "PDFDATA-3785" + + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/files/file/3785.json": + w.Header().Set("Content-Type", "application/json") + io.WriteString(w, `{"response":{"id":3785,"title":"04.pdf","folderId":655,"viewUrl":"/download/3785"}}`) + case "/download/3785": + http.Redirect(w, r, "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", http.StatusTemporaryRedirect) + case "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf": + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, staleS3Body) + default: + http.NotFound(w, r) + } + })) + defer portal.Close() + + var minioPath, minioAuth string + minio := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + minioPath, minioAuth = r.URL.Path, r.Header.Get("Authorization") + io.WriteString(w, payload) + })) + defer minio.Close() + + t.Setenv("MINIO_ENDPOINT", minio.URL) + t.Setenv("MINIO_BUCKET", "office") + t.Setenv("MINIO_ACCESS_KEY", "testkey") + t.Setenv("MINIO_SECRET_KEY", "testsecret") + + c := &Client{ + client: portal.Client(), + credentials: &Credentials{Url: portal.URL}, + token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))}, + } + + var buf bytes.Buffer + n, err := c.DownloadFile(context.Background(), "3785", &buf) + if err != nil { + t.Fatalf("DownloadFile: %v", err) + } + if n != int64(len(payload)) || buf.String() != payload { + t.Fatalf("got %d bytes %q, want %d bytes %q", n, buf.String(), len(payload), payload) + } + if want := "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf"; minioPath != want { + t.Errorf("minio path = %q, want %q", minioPath, want) + } + if !strings.HasPrefix(minioAuth, "AWS4-HMAC-SHA256") { + t.Errorf("minio request not SigV4-signed; Authorization=%q", minioAuth) + } +} + +func TestDownloadFileMinioFallbackWithoutCreds(t *testing.T) { + portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + switch r.URL.Path { + case "/api/2.0/files/file/3785.json": + io.WriteString(w, `{"response":{"id":3785,"folderId":652,"viewUrl":"/download/3785"}}`) + default: + w.WriteHeader(http.StatusForbidden) + io.WriteString(w, staleS3Body) + } + })) + defer portal.Close() + + t.Setenv("MINIO_ACCESS_KEY", "") + t.Setenv("MINIO_SECRET_KEY", "") + + c := &Client{ + client: portal.Client(), + credentials: &Credentials{Url: portal.URL}, + token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))}, + } + + _, err := c.DownloadFile(context.Background(), "3785", io.Discard) + if err == nil { + t.Fatal("expected error without minio credentials") + } + if !strings.Contains(err.Error(), "MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") { + t.Fatalf("unexpected error: %v", err) + } +}