diff --git a/files_dedupe.go b/files_dedupe.go index 73e0b83..bd3a3bf 100644 --- a/files_dedupe.go +++ b/files_dedupe.go @@ -154,6 +154,9 @@ func findWithinFolderDuplicates(indexed []ProjectFolderFile) []DedupGroup { byKey[k] = append(byKey[k], it.File) } for k, group := range byKey { + if k == "" { + continue // dotfiles etc. have no stem: never treat as duplicates + } if len(group) < 2 { continue } @@ -178,6 +181,9 @@ func findCrossFolderDuplicates(indexed []ProjectFolderFile) []DedupGroup { } var out []DedupGroup for k, items := range byKey { + if k == "" { + continue // dotfiles etc. have no stem: never treat as duplicates + } if len(items) < 2 { continue } diff --git a/files_dedupe_integration_test.go b/files_dedupe_integration_test.go new file mode 100644 index 0000000..f71ce95 --- /dev/null +++ b/files_dedupe_integration_test.go @@ -0,0 +1,416 @@ +//go:build integration + +package onlyoffice + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "strconv" + "testing" + "time" +) + +// TestIntegrationFileDedup proves on a live OnlyOffice portal that the file +// dedup helpers find real duplicates and delete only the redundant copies. +// It creates a throwaway "go-onlyoffice-test-" project (removed by cleanup), +// places same stem|ext files in two subfolders and in the project root, then +// exercises FindProjectDuplicates, mergeProjectRootForDedupe, +// ApplyDedupGroups and DeleteFilesByDedupKey. Destructive — run only against +// an instance you own. +func TestIntegrationFileDedup(t *testing.T) { + c := liveClient(t) + t.Cleanup(func() { cleanupTestProjects(t, c) }) + ctx := context.Background() + + suffix := time.Now().UTC().Format("20060102-150405") + project, err := c.CreateProject(NewProjectRequest{ + Title: testProjectPrefix + "dedup-" + suffix, + Description: "go-onlyoffice file dedup integration", + }) + if err != nil { + t.Fatalf("CreateProject: %v", err) + } + if project.ID == nil { + t.Fatal("created project without id") + } + pid := strconv.Itoa(*project.ID) + + root := projectFolderEventually(t, ctx, c, pid) + + aID := createFolderLive(t, ctx, c, root, "A-"+suffix) + bID := createFolderLive(t, ctx, c, root, "B-"+suffix) + createFolderLive(t, ctx, c, root, "_trash-"+suffix) + + // Check IsTrashFolderTitle against a real live folder title. + if !IsTrashFolderTitle("_trash-" + suffix) { + t.Fatalf("IsTrashFolderTitle(%q) = false for a live _trash folder", "_trash-"+suffix) + } + + stem := "dedup-" + suffix + local := writeLocalFile(t, stem+".txt", []byte("dedup integration "+suffix+"\n")) + + a1 := uploadFolderLive(t, ctx, c, aID, local) + a2 := uploadFolderLive(t, ctx, c, aID, local) + b1 := uploadFolderLive(t, ctx, c, bID, local) + r1 := uploadFolderLive(t, ctx, c, root, local) + r2 := uploadFolderLive(t, ctx, c, root, local) + t.Logf("uploaded a1=%d a2=%d b1=%d r1=%d r2=%d", + FileEntryNumericID(a1), FileEntryNumericID(a2), FileEntryNumericID(b1), + FileEntryNumericID(r1), FileEntryNumericID(r2)) + + if n := dedupWaitCount(t, ctx, c, aID, stem, ".txt", 2, 30*time.Second); n != 2 { + t.Fatalf("folder A has %d copies after upload, want 2", n) + } + if n := dedupWaitCount(t, ctx, c, bID, stem, ".txt", 1, 30*time.Second); n != 1 { + t.Fatalf("folder B has %d copies after upload, want 1", n) + } + if n := dedupWaitCount(t, ctx, c, root, stem, ".txt", 2, 30*time.Second); n != 2 { + t.Fatalf("project root has %d copies after upload, want 2", n) + } + + // #2 mergeProjectRootForDedupe on the live tree: the project root that + // carries documents must be part of the scan exactly once. + folders, byFolder, rootFiles := liveProjectIndex(t, ctx, c, pid, root) + merged, mergedBy := mergeProjectRootForDedupe(root, folders, byFolder, rootFiles) + rootCount := 0 + for _, folder := range merged { + if folder != nil && folder.ID != nil && folder.ID.String() == root { + rootCount++ + } + } + if rootCount != 1 { + t.Fatalf("mergeProjectRootForDedupe: project root appears %d times in live tree, want 1", rootCount) + } + if got := len(FindFilesByDedupKey(mergedBy[root], stem, ".txt")); got != 2 { + t.Fatalf("mergeProjectRootForDedupe: root carries %d matching files, want 2", got) + } + + // Within-folder scan: a duplicate pair in A and in the project root. + within := FindProjectDuplicates(merged, mergedBy, DedupOptions{}) + removesByFolder := map[string]int{} + for _, g := range within { + removesByFolder[g.FolderID] = len(g.Remove) + } + if len(within) != 2 || removesByFolder[aID] != 1 || removesByFolder[root] != 1 { + t.Fatalf("within-folder groups = %d (%v), want exactly A:1 root:1", len(within), removesByFolder) + } + + // DeleteFilesByDedupKey removes every stem|ext copy in one folder. + removed, err := c.DeleteFilesByDedupKey(ctx, aID, stem, ".txt") + if err != nil { + t.Fatalf("DeleteFilesByDedupKey(A): %v", err) + } + if len(removed) != 2 { + t.Fatalf("DeleteFilesByDedupKey(A) removed %v, want 2 ids", removed) + } + if n := dedupWaitCount(t, ctx, c, aID, stem, ".txt", 0, 30*time.Second); n != 0 { + t.Fatalf("folder A still has %d copies after DeleteFilesByDedupKey", n) + } + + // Re-create the A duplicates so the cross-folder project scan can be + // applied and a single survivor proven by polling. + uploadFolderLive(t, ctx, c, aID, local) + uploadFolderLive(t, ctx, c, aID, local) + if n := dedupWaitCount(t, ctx, c, aID, stem, ".txt", 2, 30*time.Second); n != 2 { + t.Fatalf("folder A has %d re-uploaded copies, want 2", n) + } + + // Cross-folder dry-run over the live project: one key, five copies, and + // the project-root copies must be part of the group (root merge live). + groups, deleted, err := dedupeProjectEventually(t, ctx, c, pid, DedupOptions{CrossFolder: true}, false) + if err != nil { + t.Fatalf("DedupeProject: %v", err) + } + if len(deleted) != 0 { + t.Fatalf("dry-run DedupeProject deleted %v", deleted) + } + if len(groups) != 1 { + t.Fatalf("DedupeProject cross-folder groups = %d, want 1 (%+v)", len(groups), groups) + } + if len(groups[0].Remove) != 4 { + t.Fatalf("cross-folder group removes %d files, want 4", len(groups[0].Remove)) + } + if !dedupGroupHasID(groups[0], FileEntryNumericID(r1)) && !dedupGroupHasID(groups[0], FileEntryNumericID(r2)) { + t.Fatalf("cross-folder group does not include a project-root copy (merge not applied)") + } + + keepID := FileEntryNumericID(groups[0].Keep) + deleted, err = c.ApplyDedupGroups(ctx, groups) + if err != nil { + t.Fatalf("ApplyDedupGroups: %v", err) + } + if len(deleted) != 4 { + t.Fatalf("ApplyDedupGroups deleted %v, want 4 ids", deleted) + } + + survivorID, total := dedupWaitTotal(t, ctx, c, []string{aID, bID, root}, stem, ".txt", 1, 40*time.Second) + if total != 1 { + t.Fatalf("after ApplyDedupGroups %d copies survive, want 1", total) + } + if survivorID != keepID { + t.Fatalf("remaining copy id = %d, want kept id %d", survivorID, keepID) + } + + // The removed ids must really be gone from every folder. + gone := map[int64]bool{ + FileEntryNumericID(a2): true, + FileEntryNumericID(b1): true, + FileEntryNumericID(r1): true, + FileEntryNumericID(r2): true, + } + for _, fid := range []string{aID, bID, root} { + files, err := c.FolderFiles(ctx, fid) + if err != nil { + t.Fatalf("FolderFiles %s: %v", fid, err) + } + for _, f := range FindFilesByDedupKey(files, stem, ".txt") { + id := FileEntryNumericID(f) + if gone[id] { + t.Fatalf("removed id %d still present in folder %s", id, fid) + } + } + } + t.Logf("survivor id=%d keep id=%d, deleted=%v", survivorID, keepID, deleted) +} + +// createFolderLive creates a subfolder (retrying transient 5xx) and returns +// its Documents folder id. +func createFolderLive(t *testing.T, ctx context.Context, c *Client, parentID, title string) string { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ( + m map[string]any + err error + ) + for { + m, err = c.CreateFolder(ctx, parentID, title) + if err == nil || time.Now().After(deadline) { + break + } + time.Sleep(time.Second) + } + if err != nil { + t.Fatalf("CreateFolder %q: %v", title, err) + } + if m == nil { + t.Fatalf("CreateFolder %q: empty response", title) + } + switch v := m["id"].(type) { + case float64: + return strconv.FormatInt(int64(v), 10) + case json.Number: + return v.String() + case string: + if v != "" { + return v + } + } + t.Fatalf("CreateFolder %q: no id in response %#v", title, m) + return "" +} + +// writeLocalFile writes content to a temp file and returns its path. +func writeLocalFile(t *testing.T, name string, content []byte) string { + t.Helper() + p := filepath.Join(t.TempDir(), name) + if err := os.WriteFile(p, content, 0o600); err != nil { + t.Fatal(err) + } + return p +} + +// uploadFolderLive uploads localPath into folderID (retrying the portal's +// transient post-create 500) and returns the file entry. +func uploadFolderLive(t *testing.T, ctx context.Context, c *Client, folderID, localPath string) *FileEntry { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ( + e *FileEntry + err error + ) + for { + e, err = c.UploadToFolder(ctx, folderID, localPath) + if err == nil || time.Now().After(deadline) { + break + } + time.Sleep(time.Second) + } + if err != nil { + t.Fatalf("UploadToFolder %s: %v", folderID, err) + } + if e == nil || e.ID == nil { + t.Fatalf("UploadToFolder %s: no file entry (%+v)", folderID, e) + } + return e +} + +// liveProjectIndex rebuilds the project folder/file index the same way +// DedupeProject does, for direct mergeProjectRootForDedupe assertions. +func liveProjectIndex(t *testing.T, ctx context.Context, c *Client, projectID, rootID string) ([]*FolderEntry, map[string][]*FileEntry, []*FileEntry) { + t.Helper() + pf := getProjectFilesEventually(t, ctx, c, projectID) + rootFiles := folderFilesEventually(t, ctx, c, rootID) + folders := make([]*FolderEntry, 0, len(pf.Folders)+1) + byFolder := make(map[string][]*FileEntry, len(pf.Folders)+1) + for _, folder := range pf.Folders { + if folder == nil || folder.ID == nil { + continue + } + fid := folder.ID.String() + if fid == rootID { + byFolder[fid] = rootFiles + } else { + byFolder[fid] = folderFilesEventually(t, ctx, c, fid) + } + folders = append(folders, folder) + } + return folders, byFolder, rootFiles +} + +// projectFolderEventually resolves the project Documents root id, retrying on +// a transient portal answer. +func projectFolderEventually(t *testing.T, ctx context.Context, c *Client, projectID string) string { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ( + root string + err error + ) + for { + root, err = c.projectFolderID(ctx, projectID) + if err == nil || time.Now().After(deadline) { + break + } + time.Sleep(time.Second) + } + if err != nil { + t.Fatalf("projectFolderID: %v", err) + } + if root == "" { + t.Fatal("projectFolderID returned empty id") + } + return root +} + +// getProjectFilesEventually lists a project's files/folders, retrying on a +// transient portal answer. +func getProjectFilesEventually(t *testing.T, ctx context.Context, c *Client, projectID string) *ProjectFilesResponse { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var last error + for { + pf, err := c.GetProjectFiles(ctx, projectID) + if err == nil { + return pf + } + last = err + if time.Now().After(deadline) { + t.Fatalf("GetProjectFiles %s: %v", projectID, last) + } + time.Sleep(500 * time.Millisecond) + } +} + +// folderFilesEventually lists a folder, retrying while the portal answers +// transiently (a freshly created folder can 500 until its parent map settles). +func folderFilesEventually(t *testing.T, ctx context.Context, c *Client, folderID string) []*FileEntry { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var last error + for { + files, err := c.FolderFiles(ctx, folderID) + if err == nil { + return files + } + last = err + if time.Now().After(deadline) { + t.Fatalf("FolderFiles %s: %v", folderID, last) + } + time.Sleep(500 * time.Millisecond) + } +} + +// dedupeProjectEventually runs a project dedup scan, retrying the whole scan on +// a transient portal error. It is used for dry-runs only (apply must stay a +// single deliberate call). +func dedupeProjectEventually(t *testing.T, ctx context.Context, c *Client, projectID string, opts DedupOptions, apply bool) ([]DedupGroup, []int, error) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + var ( + groups []DedupGroup + deleted []int + err error + ) + for { + groups, deleted, err = c.DedupeProject(ctx, projectID, opts, apply) + if err == nil || time.Now().After(deadline) { + return groups, deleted, err + } + time.Sleep(time.Second) + } +} + +// dedupWaitCount polls folderID until want stem|ext copies are visible. +func dedupWaitCount(t *testing.T, ctx context.Context, c *Client, folderID, stem, ext string, want int, d time.Duration) int { + t.Helper() + deadline := time.Now().Add(d) + got := -1 + for { + if files, err := c.FolderFiles(ctx, folderID); err == nil { + got = len(FindFilesByDedupKey(files, stem, ext)) + if got == want { + return got + } + } + if time.Now().After(deadline) { + return got + } + time.Sleep(500 * time.Millisecond) + } +} + +// dedupWaitTotal polls the given folders until the total number of stem|ext +// copies reaches want, returning the last seen file id and count. +func dedupWaitTotal(t *testing.T, ctx context.Context, c *Client, folderIDs []string, stem, ext string, want int, d time.Duration) (int64, int) { + t.Helper() + deadline := time.Now().Add(d) + var survivor int64 + total := -1 + for { + survivor, total = 0, 0 + for _, fid := range folderIDs { + files, err := c.FolderFiles(ctx, fid) + if err != nil { + continue + } + for _, f := range FindFilesByDedupKey(files, stem, ext) { + total++ + survivor = FileEntryNumericID(f) + } + } + if total == want { + return survivor, total + } + if time.Now().After(deadline) { + return survivor, total + } + time.Sleep(500 * time.Millisecond) + } +} + +func dedupGroupHasID(g DedupGroup, id int64) bool { + if id == 0 { + return false + } + if FileEntryNumericID(g.Keep) == id { + return true + } + for _, f := range g.Remove { + if FileEntryNumericID(f) == id { + return true + } + } + return false +} diff --git a/files_dedupe_test.go b/files_dedupe_test.go index 6d24a85..f75d2a3 100644 --- a/files_dedupe_test.go +++ b/files_dedupe_test.go @@ -84,6 +84,32 @@ func TestMergeProjectRootForDedupe(t *testing.T) { } } +func TestFindDuplicatesSkipsEmptyKey(t *testing.T) { + // Dotfiles (".env", ".gitignore", ".npmrc") normalize to an empty stem, so + // FileDedupKey is "". They are not duplicates of each other and must never + // form a dedup group that would delete one of them. + env := &FileEntry{ID: jsonNum("1"), Title: strPtr(".env")} + gitignore := &FileEntry{ID: jsonNum("2"), Title: strPtr(".gitignore")} + npmrc := &FileEntry{ID: jsonNum("3"), Title: strPtr(".npmrc")} + if FileDedupKey(env) != "" || FileDedupKey(gitignore) != "" || FileDedupKey(npmrc) != "" { + t.Fatalf("dotfiles should have empty dedup key") + } + within := []ProjectFolderFile{ + {FolderID: "500", FolderTitle: "Cfg", File: env}, + {FolderID: "500", FolderTitle: "Cfg", File: gitignore}, + } + if groups := findWithinFolderDuplicates(within); len(groups) != 0 { + t.Fatalf("within-folder empty-key groups = %d, want 0 (%+v)", len(groups), groups) + } + cross := []ProjectFolderFile{ + {FolderID: "500", FolderTitle: "Cfg", File: env}, + {FolderID: "501", FolderTitle: "Other", File: npmrc}, + } + if groups := findCrossFolderDuplicates(cross); len(groups) != 0 { + t.Fatalf("cross-folder empty-key groups = %d, want 0 (%+v)", len(groups), groups) + } +} + func TestIsTrashFolderTitle(t *testing.T) { if !IsTrashFolderTitle("_trash-md") { t.Fatal("expected trash") diff --git a/files_webdav.go b/files_webdav.go index 197c394..a38b732 100644 --- a/files_webdav.go +++ b/files_webdav.go @@ -243,18 +243,27 @@ func fileopsError(raw json.RawMessage) error { func (c *Client) DeleteDavItems(ctx context.Context, folderIDs, fileIDs []string) error { body := map[string]any{"DeleteAfter": true, "Immediately": true} for _, id := range folderIDs { - if _, err := c.deleteJSON(ctx, "/api/2.0/files/folder/"+url.PathEscape(id), body); err != nil { + if err := c.deleteDavItem(ctx, "/api/2.0/files/folder/"+url.PathEscape(id), body); err != nil { return err } } for _, id := range fileIDs { - if _, err := c.deleteJSON(ctx, "/api/2.0/files/file/"+url.PathEscape(id), body); err != nil { + if err := c.deleteDavItem(ctx, "/api/2.0/files/file/"+url.PathEscape(id), body); err != nil { return err } } return nil } +// deleteDavItem deletes one item, retrying transient 429/502/503/504 answers +// through DoRetry like every other bulk path (deletes are idempotent). +func (c *Client) deleteDavItem(ctx context.Context, path string, body any) error { + return DoRetry(ctx, DefaultRetryPolicy(), func() error { + _, err := c.deleteJSON(ctx, path, body) + return err + }) +} + // UploadDavFile uploads src (fileName) into folderID, streaming from src. // // Deprecated: use FileStore.Upload via Client.Files()/Client.FileStore.