feat(files): MinIO fallback for stale S3 downloads (#152) #21
@@ -25,3 +25,12 @@ ONLYOFFICE_PROJECT_ID=33
|
|||||||
# cmd/office TUI — optional Document Server for DOCX→HTML preview:
|
# cmd/office TUI — optional Document Server for DOCX→HTML preview:
|
||||||
# ONLYOFFICE_DOCS_URL=https://docs.example.com
|
# ONLYOFFICE_DOCS_URL=https://docs.example.com
|
||||||
# ONLYOFFICE_DOCS_SECRET=
|
# ONLYOFFICE_DOCS_SECRET=
|
||||||
|
|
||||||
|
# MinIO download fallback for the portal's stale AWS S3 consumer (older
|
||||||
|
# Documents folders). When the portal redirects to amazonaws.com with access
|
||||||
|
# key "minio" (403 InvalidAccessKeyId), files are fetched from the local MinIO
|
||||||
|
# store instead. Without a key/secret the fallback is disabled.
|
||||||
|
# MINIO_ENDPOINT=http://192.168.188.10:9000
|
||||||
|
# MINIO_BUCKET=office
|
||||||
|
# MINIO_ACCESS_KEY=
|
||||||
|
# MINIO_SECRET_KEY=
|
||||||
|
|||||||
@@ -5,7 +5,6 @@ import (
|
|||||||
"encoding/json"
|
"encoding/json"
|
||||||
"fmt"
|
"fmt"
|
||||||
"io"
|
"io"
|
||||||
"net/http"
|
|
||||||
"net/url"
|
"net/url"
|
||||||
"path"
|
"path"
|
||||||
"strconv"
|
"strconv"
|
||||||
@@ -383,36 +382,15 @@ func FileFolderID(f *FileEntry) string {
|
|||||||
}
|
}
|
||||||
|
|
||||||
// DownloadFile streams file bytes from the file's viewUrl using the same auth
|
// DownloadFile streams file bytes from the file's viewUrl using the same auth
|
||||||
// as API calls. Writes into dst.
|
// as API calls. Writes into dst. When the portal serves the file from its stale
|
||||||
|
// AWS S3 consumer, the bytes are fetched from the local MinIO store instead
|
||||||
|
// (see storage_fallback.go).
|
||||||
func (c *Client) DownloadFile(ctx context.Context, fileID string, dst io.Writer) (int64, error) {
|
func (c *Client) DownloadFile(ctx context.Context, fileID string, dst io.Writer) (int64, error) {
|
||||||
f, err := c.GetFile(ctx, fileID)
|
f, err := c.GetFile(ctx, fileID)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return 0, err
|
return 0, err
|
||||||
}
|
}
|
||||||
if f.ViewURL == nil || *f.ViewURL == "" {
|
return c.downloadFileEntry(ctx, f, dst)
|
||||||
return 0, fmt.Errorf("file %s has no viewUrl", fileID)
|
|
||||||
}
|
|
||||||
downloadURL := c.resolveAPIURL(*f.ViewURL)
|
|
||||||
auth, err := c.authHeader()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
req.Header.Set("Authorization", auth)
|
|
||||||
resp, err := c.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode >= 400 {
|
|
||||||
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
|
||||||
return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400))
|
|
||||||
}
|
|
||||||
n, err := io.Copy(dst, resp.Body)
|
|
||||||
return n, err
|
|
||||||
}
|
}
|
||||||
|
|
||||||
func (c *Client) resolveAPIURL(ref string) string {
|
func (c *Client) resolveAPIURL(ref string) string {
|
||||||
|
|||||||
+3
-27
@@ -259,34 +259,10 @@ func (c *Client) UploadDavFile(ctx context.Context, folderID, fileName string, s
|
|||||||
return env.Response, nil
|
return env.Response, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// DownloadDavFile streams the file identified by id to w, returning bytes copied.
|
// DownloadDavFile streams the file identified by id to w, returning bytes
|
||||||
|
// copied. It shares the MinIO stale-S3 fallback with DownloadFile.
|
||||||
func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) {
|
func (c *Client) DownloadDavFile(ctx context.Context, id string, w io.Writer) (int64, error) {
|
||||||
file, err := c.GetFile(ctx, id)
|
return c.DownloadFile(ctx, id, w)
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
if file.ViewURL == nil || *file.ViewURL == "" {
|
|
||||||
return 0, fmt.Errorf("onlyoffice: file %s has no viewUrl", id)
|
|
||||||
}
|
|
||||||
u := c.resolveAPIURL(*file.ViewURL)
|
|
||||||
auth, err := c.authHeader()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u, nil)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
req.Header.Set("Authorization", auth)
|
|
||||||
resp, err := c.client.Do(req)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
defer resp.Body.Close()
|
|
||||||
if resp.StatusCode >= 400 {
|
|
||||||
return 0, fmt.Errorf("onlyoffice: download: %d", resp.StatusCode)
|
|
||||||
}
|
|
||||||
return io.Copy(w, resp.Body)
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// --- internal helpers -------------------------------------------------------
|
// --- internal helpers -------------------------------------------------------
|
||||||
|
|||||||
@@ -4,6 +4,7 @@ go 1.25.0
|
|||||||
|
|
||||||
require (
|
require (
|
||||||
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
|
github.com/JohannesKaufmann/html-to-markdown/v2 v2.5.2
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.1
|
||||||
github.com/charmbracelet/bubbles v0.18.0
|
github.com/charmbracelet/bubbles v0.18.0
|
||||||
github.com/charmbracelet/bubbletea v0.25.0
|
github.com/charmbracelet/bubbletea v0.25.0
|
||||||
github.com/charmbracelet/glamour v0.8.0
|
github.com/charmbracelet/glamour v0.8.0
|
||||||
@@ -26,6 +27,7 @@ require (
|
|||||||
github.com/JohannesKaufmann/dom v0.3.1 // indirect
|
github.com/JohannesKaufmann/dom v0.3.1 // indirect
|
||||||
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
|
github.com/alecthomas/chroma/v2 v2.14.0 // indirect
|
||||||
github.com/atotto/clipboard v0.1.4 // indirect
|
github.com/atotto/clipboard v0.1.4 // indirect
|
||||||
|
github.com/aws/smithy-go v1.24.0 // indirect
|
||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1 // indirect
|
||||||
github.com/aymerick/douceur v0.2.0 // indirect
|
github.com/aymerick/douceur v0.2.0 // indirect
|
||||||
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
github.com/containerd/console v1.0.4-0.20230313162750-1ae8d489ac81 // indirect
|
||||||
|
|||||||
@@ -10,6 +10,10 @@ github.com/alecthomas/repr v0.4.0 h1:GhI2A8MACjfegCPVq9f1FLvIBS+DrQ2KQBFZP1iFzXc
|
|||||||
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
github.com/alecthomas/repr v0.4.0/go.mod h1:Fr0507jx4eOXV7AlPV6AVZLYrLIuIeSOWtW57eE/O/4=
|
||||||
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
github.com/atotto/clipboard v0.1.4 h1:EH0zSVneZPSuFR11BlR9YppQTVDbh5+16AmcJi4g1z4=
|
||||||
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
github.com/atotto/clipboard v0.1.4/go.mod h1:ZY9tmq7sm5xIbd9bOK4onWV4S6X0u6GY7Vn0Yu86PYI=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.1 h1:ABlyEARCDLN034NhxlRUSZr4l71mh+T5KAeGh6cerhU=
|
||||||
|
github.com/aws/aws-sdk-go-v2 v1.41.1/go.mod h1:MayyLB8y+buD9hZqkCW3kX1AKq07Y5pXxtgB+rRFhz0=
|
||||||
|
github.com/aws/smithy-go v1.24.0 h1:LpilSUItNPFr1eY85RYgTIg5eIEPtvFbskaFcmmIUnk=
|
||||||
|
github.com/aws/smithy-go v1.24.0/go.mod h1:LEj2LM3rBRQJxPZTB4KuzZkaZYnZPnvgIhb4pu07mx0=
|
||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1 h1:HwpRHbFMcZLEVr42D4p7XBqjyuxQH5SMiErDT4WkJ2k=
|
||||||
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
github.com/aymanbagabas/go-osc52/v2 v2.0.1/go.mod h1:uYgXzlJ7ZpABp8OJ+exZzJJhRNQ2ASbcXHWsFqH8hp8=
|
||||||
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
github.com/aymanbagabas/go-udiff v0.2.0 h1:TK0fH4MteXUDspT88n8CKzvK0X9O2xu9yQjWpi6yML8=
|
||||||
|
|||||||
@@ -0,0 +1,200 @@
|
|||||||
|
package onlyoffice
|
||||||
|
|
||||||
|
// MinIO download fallback for the portal's stale AWS S3 consumer.
|
||||||
|
//
|
||||||
|
// On the Fibu EDL portal some older Documents files live in S3/MinIO, but the
|
||||||
|
// portal's storage consumer still points at s3.us-east-1.amazonaws.com with
|
||||||
|
// access key "minio". Downloads of those files answer 403 InvalidAccessKeyId.
|
||||||
|
// The bytes are present in the local MinIO store under a deterministic object
|
||||||
|
// key, so the client retries the GET there.
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"crypto/sha256"
|
||||||
|
"encoding/hex"
|
||||||
|
"fmt"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/url"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"time"
|
||||||
|
|
||||||
|
"github.com/aws/aws-sdk-go-v2/aws"
|
||||||
|
"github.com/aws/aws-sdk-go-v2/aws/signer/v4"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
defaultMinioEndpoint = "http://192.168.188.10:9000"
|
||||||
|
defaultMinioBucket = "office"
|
||||||
|
minioRegion = "us-east-1"
|
||||||
|
)
|
||||||
|
|
||||||
|
// minioObjectKey is the fallback object key layout the portal's S3 consumer
|
||||||
|
// writes for Documents files: 00/00/01/files/folder_<folderId>/file_<fileId>/v1/content.pdf.
|
||||||
|
// Prefer minioObjectKeyFromURL: the portal stores all files below its storage
|
||||||
|
// root folder, which is not the API folderId returned by GetFile.
|
||||||
|
func minioObjectKey(fileID, folderID string) string {
|
||||||
|
return "00/00/01/files/folder_" + folderID + "/file_" + fileID + "/v1/content.pdf"
|
||||||
|
}
|
||||||
|
|
||||||
|
// minioObjectKeyFromURL extracts the object key from an S3 download URL. Path
|
||||||
|
// style URLs (bucket as first path segment) have that segment removed; virtual
|
||||||
|
// host style URLs are returned as-is. This is authoritative: the portal signs
|
||||||
|
// the exact key, so no folder-id guessing is needed.
|
||||||
|
func minioObjectKeyFromURL(rawURL, bucket string) (string, bool) {
|
||||||
|
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||||
|
if err != nil || u.Path == "" {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
segs := strings.Split(strings.Trim(u.Path, "/"), "/")
|
||||||
|
// Path-style URLs carry the bucket as leading segment; the portal's S3
|
||||||
|
// consumer can emit it twice (serviceurl already includes the bucket), so
|
||||||
|
// strip every leading segment equal to the bucket.
|
||||||
|
for len(segs) > 0 && bucket != "" && segs[0] == bucket {
|
||||||
|
segs = segs[1:]
|
||||||
|
}
|
||||||
|
if len(segs) == 0 {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
for _, s := range segs {
|
||||||
|
if s == "" || s == "." || s == ".." {
|
||||||
|
return "", false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return strings.Join(segs, "/"), true
|
||||||
|
}
|
||||||
|
|
||||||
|
// isStaleS3Redirect reports whether a download landed on the portal's stale AWS
|
||||||
|
// S3 consumer. Such responses either carry an S3 InvalidAccessKeyId XML body or
|
||||||
|
// point at amazonaws.com with the "minio" access key id in the query.
|
||||||
|
func isStaleS3Redirect(rawURL string, body []byte) bool {
|
||||||
|
if strings.Contains(strings.ToLower(string(body)), "invalidaccesskeyid") {
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
u, err := url.Parse(strings.TrimSpace(rawURL))
|
||||||
|
if err != nil || u.Host == "" {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
host := strings.ToLower(u.Host)
|
||||||
|
if !strings.Contains(host, "amazonaws.com") {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
q := strings.ToLower(u.RawQuery)
|
||||||
|
return strings.Contains(q, "accesskeyid=minio") || strings.Contains(q, "x-amz-credential=minio")
|
||||||
|
}
|
||||||
|
|
||||||
|
// minioConfig is the runtime configuration for the local MinIO fallback.
|
||||||
|
type minioConfig struct {
|
||||||
|
Endpoint string
|
||||||
|
Bucket string
|
||||||
|
AccessKey string
|
||||||
|
SecretKey string
|
||||||
|
}
|
||||||
|
|
||||||
|
// loadMinioConfig reads the fallback configuration from the environment.
|
||||||
|
// Secrets are never defaulted; without access/secret keys the fallback is off.
|
||||||
|
func loadMinioConfig() minioConfig {
|
||||||
|
return minioConfig{
|
||||||
|
Endpoint: strings.TrimRight(firstNonEmpty(os.Getenv("MINIO_ENDPOINT"), defaultMinioEndpoint), "/"),
|
||||||
|
Bucket: firstNonEmpty(os.Getenv("MINIO_BUCKET"), defaultMinioBucket),
|
||||||
|
AccessKey: os.Getenv("MINIO_ACCESS_KEY"),
|
||||||
|
SecretKey: os.Getenv("MINIO_SECRET_KEY"),
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadFileEntry downloads f's bytes to dst. It transparently falls back to
|
||||||
|
// the local MinIO store when the portal redirects the download to its stale AWS
|
||||||
|
// S3 consumer.
|
||||||
|
func (c *Client) downloadFileEntry(ctx context.Context, f *FileEntry, dst io.Writer) (int64, error) {
|
||||||
|
if f == nil {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: download: nil file entry")
|
||||||
|
}
|
||||||
|
if f.ViewURL == nil || *f.ViewURL == "" {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: file has no viewUrl")
|
||||||
|
}
|
||||||
|
downloadURL := c.resolveAPIURL(*f.ViewURL)
|
||||||
|
auth, err := c.authHeader()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, downloadURL, nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
req.Header.Set("Authorization", auth)
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 4096))
|
||||||
|
finalURL := downloadURL
|
||||||
|
if resp.Request != nil && resp.Request.URL != nil {
|
||||||
|
finalURL = resp.Request.URL.String()
|
||||||
|
}
|
||||||
|
if isStaleS3Redirect(finalURL, b) {
|
||||||
|
key, ok := minioObjectKeyFromURL(finalURL, loadMinioConfig().Bucket)
|
||||||
|
if !ok {
|
||||||
|
fileID := ""
|
||||||
|
if f.ID != nil {
|
||||||
|
fileID = f.ID.String()
|
||||||
|
}
|
||||||
|
key = minioObjectKey(fileID, FileFolderID(f))
|
||||||
|
}
|
||||||
|
n, merr := c.downloadFromMinio(ctx, key, dst)
|
||||||
|
if merr == nil {
|
||||||
|
return n, nil
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("GET viewUrl: %d (stale S3) and minio fallback: %w", resp.StatusCode, merr)
|
||||||
|
}
|
||||||
|
return 0, fmt.Errorf("GET viewUrl: %d %s", resp.StatusCode, truncate(string(b), 400))
|
||||||
|
}
|
||||||
|
return io.Copy(dst, resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// downloadFromMinio streams objectKey from the configured MinIO bucket.
|
||||||
|
func (c *Client) downloadFromMinio(ctx context.Context, objectKey string, dst io.Writer) (int64, error) {
|
||||||
|
if objectKey == "" {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: minio fallback: empty object key")
|
||||||
|
}
|
||||||
|
cfg := loadMinioConfig()
|
||||||
|
if cfg.AccessKey == "" || cfg.SecretKey == "" {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: minio fallback: MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set")
|
||||||
|
}
|
||||||
|
base, err := url.Parse(cfg.Endpoint)
|
||||||
|
if err != nil || base.Host == "" {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: minio fallback: bad MINIO_ENDPOINT %q", cfg.Endpoint)
|
||||||
|
}
|
||||||
|
u := *base
|
||||||
|
u.Path = "/" + cfg.Bucket + "/" + objectKey
|
||||||
|
req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
if err := signMinioRequest(ctx, cfg, req); err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
resp, err := c.client.Do(req)
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("onlyoffice: minio fallback: %w", err)
|
||||||
|
}
|
||||||
|
defer resp.Body.Close()
|
||||||
|
if resp.StatusCode >= 400 {
|
||||||
|
b, _ := io.ReadAll(io.LimitReader(resp.Body, 512))
|
||||||
|
return 0, fmt.Errorf("onlyoffice: minio fallback: %d %s", resp.StatusCode, truncate(string(b), 300))
|
||||||
|
}
|
||||||
|
return io.Copy(dst, resp.Body)
|
||||||
|
}
|
||||||
|
|
||||||
|
// signMinioRequest signs req with AWS Signature V4 for the S3 service.
|
||||||
|
func signMinioRequest(ctx context.Context, cfg minioConfig, req *http.Request) error {
|
||||||
|
sum := sha256.Sum256(nil)
|
||||||
|
creds := aws.Credentials{AccessKeyID: cfg.AccessKey, SecretAccessKey: cfg.SecretKey}
|
||||||
|
if err := v4.NewSigner().SignHTTP(ctx, creds, req, hex.EncodeToString(sum[:]), "s3", minioRegion, time.Now()); err != nil {
|
||||||
|
return fmt.Errorf("onlyoffice: minio fallback: sign: %w", err)
|
||||||
|
}
|
||||||
|
return nil
|
||||||
|
}
|
||||||
@@ -0,0 +1,45 @@
|
|||||||
|
//go:build integration
|
||||||
|
|
||||||
|
package onlyoffice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"os"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
// TestIntegrationMinioFallback downloads known stale-S3 files through the local
|
||||||
|
// MinIO fallback. Requires ONLYOFFICE_URL/USER/PASS (as all integration tests),
|
||||||
|
// MINIO_ACCESS_KEY/MINIO_SECRET_KEY and MINIO_TEST_FILE_IDS="3785,3859,3666";
|
||||||
|
// skips when any of those are missing.
|
||||||
|
func TestIntegrationMinioFallback(t *testing.T) {
|
||||||
|
if os.Getenv("MINIO_ACCESS_KEY") == "" || os.Getenv("MINIO_SECRET_KEY") == "" {
|
||||||
|
t.Skip("MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set — skipping integration test")
|
||||||
|
}
|
||||||
|
raw := strings.TrimSpace(os.Getenv("MINIO_TEST_FILE_IDS"))
|
||||||
|
if raw == "" {
|
||||||
|
t.Skip("MINIO_TEST_FILE_IDS not set — skipping integration test")
|
||||||
|
}
|
||||||
|
c := liveClient(t)
|
||||||
|
ctx, cancel := context.WithTimeout(context.Background(), 5*time.Minute)
|
||||||
|
defer cancel()
|
||||||
|
for _, id := range strings.Split(raw, ",") {
|
||||||
|
id = strings.TrimSpace(id)
|
||||||
|
if id == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
n, err := c.DownloadFile(ctx, id, io.Discard)
|
||||||
|
if err != nil {
|
||||||
|
t.Errorf("DownloadFile(%s): %v", id, err)
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if n == 0 {
|
||||||
|
t.Errorf("DownloadFile(%s): 0 bytes", id)
|
||||||
|
} else {
|
||||||
|
t.Logf("DownloadFile(%s): %d bytes", id, n)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,219 @@
|
|||||||
|
package onlyoffice
|
||||||
|
|
||||||
|
import (
|
||||||
|
"bytes"
|
||||||
|
"context"
|
||||||
|
"io"
|
||||||
|
"net/http"
|
||||||
|
"net/http/httptest"
|
||||||
|
"strings"
|
||||||
|
"testing"
|
||||||
|
"time"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestMinioObjectKey(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
fileID string
|
||||||
|
folderID string
|
||||||
|
want string
|
||||||
|
}{
|
||||||
|
{"3785", "652", "00/00/01/files/folder_652/file_3785/v1/content.pdf"},
|
||||||
|
{"1", "2", "00/00/01/files/folder_2/file_1/v1/content.pdf"},
|
||||||
|
{"3666", "4000", "00/00/01/files/folder_4000/file_3666/v1/content.pdf"},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
if got := minioObjectKey(tc.fileID, tc.folderID); got != tc.want {
|
||||||
|
t.Errorf("minioObjectKey(%q, %q) = %q, want %q", tc.fileID, tc.folderID, got, tc.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestMinioObjectKeyFromURL(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
bucket string
|
||||||
|
want string
|
||||||
|
ok bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "path style drops bucket segment",
|
||||||
|
url: "https://s3.us-east-1.amazonaws.com/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio",
|
||||||
|
bucket: "office",
|
||||||
|
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "doubled bucket segment (portal serviceurl includes bucket)",
|
||||||
|
url: "https://s3.us-east-1.amazonaws.com/office/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio",
|
||||||
|
bucket: "office",
|
||||||
|
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "virtual host style keeps path",
|
||||||
|
url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||||
|
bucket: "office",
|
||||||
|
want: "00/00/01/files/folder_4000/file_3785/v1/content.pdf",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "foreign first segment kept",
|
||||||
|
url: "https://example.com/other/file_1/v1/content.pdf",
|
||||||
|
bucket: "office",
|
||||||
|
want: "other/file_1/v1/content.pdf",
|
||||||
|
ok: true,
|
||||||
|
},
|
||||||
|
{name: "empty path", url: "https://example.com", bucket: "office", ok: false},
|
||||||
|
{name: "traversal", url: "https://example.com/office/../etc/passwd", bucket: "office", ok: false},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
got, ok := minioObjectKeyFromURL(tc.url, tc.bucket)
|
||||||
|
if ok != tc.ok || got != tc.want {
|
||||||
|
t.Errorf("minioObjectKeyFromURL(%q, %q) = (%q, %v), want (%q, %v)", tc.url, tc.bucket, got, ok, tc.want, tc.ok)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestIsStaleS3Redirect(t *testing.T) {
|
||||||
|
cases := []struct {
|
||||||
|
name string
|
||||||
|
url string
|
||||||
|
body []byte
|
||||||
|
want bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "aws redirect with minio access key",
|
||||||
|
url: "https://s3.us-east-1.amazonaws.com/office/x/file_1?AWSAccessKeyId=minio&Expires=1",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "aws redirect with minio x-amz-credential",
|
||||||
|
url: "https://office.s3.us-east-1.amazonaws.com/00/00/01/files/folder_1/file_1?X-Amz-Credential=minio%2F20260914",
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "invalid access key xml body",
|
||||||
|
url: "https://portal.internal/download/1",
|
||||||
|
body: []byte(`<?xml version="1.0"?><Error><Code>InvalidAccessKeyId</Code><AWSAccessKeyId>minio</AWSAccessKeyId></Error>`),
|
||||||
|
want: true,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "regular pdf from portal",
|
||||||
|
url: "https://portal.internal/download/1",
|
||||||
|
body: []byte("%PDF-1.7 data"),
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "aws redirect with foreign key",
|
||||||
|
url: "https://s3.us-east-1.amazonaws.com/office/x?AWSAccessKeyId=other",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "amazonaws in path but foreign host",
|
||||||
|
url: "https://example.com/amazonaws.com/file?AWSAccessKeyId=minio",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
name: "empty",
|
||||||
|
want: false,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tc := range cases {
|
||||||
|
t.Run(tc.name, func(t *testing.T) {
|
||||||
|
if got := isStaleS3Redirect(tc.url, tc.body); got != tc.want {
|
||||||
|
t.Errorf("isStaleS3Redirect(%q, %q) = %v, want %v", tc.url, tc.body, got, tc.want)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const staleS3Body = `<?xml version="1.0" encoding="UTF-8"?>` +
|
||||||
|
`<Error><Code>InvalidAccessKeyId</Code>` +
|
||||||
|
`<Message>The AWS Access Key Id you provided does not exist in our records.</Message>` +
|
||||||
|
`<AWSAccessKeyId>minio</AWSAccessKeyId></Error>`
|
||||||
|
|
||||||
|
func TestDownloadFileMinioFallback(t *testing.T) {
|
||||||
|
const payload = "PDFDATA-3785"
|
||||||
|
|
||||||
|
portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/api/2.0/files/file/3785.json":
|
||||||
|
w.Header().Set("Content-Type", "application/json")
|
||||||
|
io.WriteString(w, `{"response":{"id":3785,"title":"04.pdf","folderId":655,"viewUrl":"/download/3785"}}`)
|
||||||
|
case "/download/3785":
|
||||||
|
http.Redirect(w, r, "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf?AWSAccessKeyId=minio", http.StatusTemporaryRedirect)
|
||||||
|
case "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf":
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
io.WriteString(w, staleS3Body)
|
||||||
|
default:
|
||||||
|
http.NotFound(w, r)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer portal.Close()
|
||||||
|
|
||||||
|
var minioPath, minioAuth string
|
||||||
|
minio := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
minioPath, minioAuth = r.URL.Path, r.Header.Get("Authorization")
|
||||||
|
io.WriteString(w, payload)
|
||||||
|
}))
|
||||||
|
defer minio.Close()
|
||||||
|
|
||||||
|
t.Setenv("MINIO_ENDPOINT", minio.URL)
|
||||||
|
t.Setenv("MINIO_BUCKET", "office")
|
||||||
|
t.Setenv("MINIO_ACCESS_KEY", "testkey")
|
||||||
|
t.Setenv("MINIO_SECRET_KEY", "testsecret")
|
||||||
|
|
||||||
|
c := &Client{
|
||||||
|
client: portal.Client(),
|
||||||
|
credentials: &Credentials{Url: portal.URL},
|
||||||
|
token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))},
|
||||||
|
}
|
||||||
|
|
||||||
|
var buf bytes.Buffer
|
||||||
|
n, err := c.DownloadFile(context.Background(), "3785", &buf)
|
||||||
|
if err != nil {
|
||||||
|
t.Fatalf("DownloadFile: %v", err)
|
||||||
|
}
|
||||||
|
if n != int64(len(payload)) || buf.String() != payload {
|
||||||
|
t.Fatalf("got %d bytes %q, want %d bytes %q", n, buf.String(), len(payload), payload)
|
||||||
|
}
|
||||||
|
if want := "/office/00/00/01/files/folder_4000/file_3785/v1/content.pdf"; minioPath != want {
|
||||||
|
t.Errorf("minio path = %q, want %q", minioPath, want)
|
||||||
|
}
|
||||||
|
if !strings.HasPrefix(minioAuth, "AWS4-HMAC-SHA256") {
|
||||||
|
t.Errorf("minio request not SigV4-signed; Authorization=%q", minioAuth)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestDownloadFileMinioFallbackWithoutCreds(t *testing.T) {
|
||||||
|
portal := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||||
|
switch r.URL.Path {
|
||||||
|
case "/api/2.0/files/file/3785.json":
|
||||||
|
io.WriteString(w, `{"response":{"id":3785,"folderId":652,"viewUrl":"/download/3785"}}`)
|
||||||
|
default:
|
||||||
|
w.WriteHeader(http.StatusForbidden)
|
||||||
|
io.WriteString(w, staleS3Body)
|
||||||
|
}
|
||||||
|
}))
|
||||||
|
defer portal.Close()
|
||||||
|
|
||||||
|
t.Setenv("MINIO_ACCESS_KEY", "")
|
||||||
|
t.Setenv("MINIO_SECRET_KEY", "")
|
||||||
|
|
||||||
|
c := &Client{
|
||||||
|
client: portal.Client(),
|
||||||
|
credentials: &Credentials{Url: portal.URL},
|
||||||
|
token: &Token{Value: "Bearer test", Expires: Time(time.Now().Add(time.Hour))},
|
||||||
|
}
|
||||||
|
|
||||||
|
_, err := c.DownloadFile(context.Background(), "3785", io.Discard)
|
||||||
|
if err == nil {
|
||||||
|
t.Fatal("expected error without minio credentials")
|
||||||
|
}
|
||||||
|
if !strings.Contains(err.Error(), "MINIO_ACCESS_KEY/MINIO_SECRET_KEY not set") {
|
||||||
|
t.Fatalf("unexpected error: %v", err)
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user