fix: workflow file collection (#3207)

packages/netlify-cms-backend-bitbucket/package.json

@@ -22,7 +22,8 @@
   "dependencies": {
     "common-tags": "^1.8.0",
     "js-base64": "^2.5.1",
-    "semaphore": "^1.1.0"
+    "semaphore": "^1.1.0",
+    "what-the-diff": "^0.6.0"
   },
   "peerDependencies": {
     "@emotion/core": "^10.0.9",

packages/netlify-cms-backend-bitbucket/src/API.ts

@@ -25,6 +25,7 @@ import {
   parseContentKey,
 } from 'netlify-cms-lib-util';
 import { oneLine } from 'common-tags';
+import { parse } from 'what-the-diff';
 
 interface Config {
   apiRoot?: string;
@@ -125,21 +126,6 @@ type BitBucketPullRequestStatues = {
   values: BitBucketPullRequestStatus[];
 };
 
-type BitBucketDiffStat = {
-  pagelen: number;
-  page: number;
-  size: number;
-  values: {
-    status: string;
-    lines_removed: number;
-    lines_added: number;
-    new: {
-      path: string;
-      type: 'commit_file';
-    };
-  }[];
-};
-
 type DeleteEntry = {
   path: string;
   delete: true;
@@ -448,13 +434,18 @@ export default class API {
   }
 
   async getDifferences(branch: string) {
-    const diff: BitBucketDiffStat = await this.requestJSON({
+    const rawDiff = await this.requestText({
-      url: `${this.repoURL}/diffstat/${branch}..${this.branch}`,
+      url: `${this.repoURL}/diff/${branch}..${this.branch}`,
       params: {
-        pagelen: 100,
+        binary: false,
       },
     });
-    return diff.values;
+
+    return parse(rawDiff).map(d => ({
+      newPath: d.newPath.replace(/b\//, ''),
+      binary: d.binary || /.svg$/.test(d.newPath),
+      newFile: d.status === 'added',
+    }));
   }
 
   async editorialWorkflowGit(files: (Entry | AssetProxy)[], entry: Entry, options: PersistOptions) {
@@ -478,8 +469,8 @@ export default class API {
       const diffs = await this.getDifferences(branch);
       const toDelete: DeleteEntry[] = [];
       for (const diff of diffs) {
-        if (!files.some(file => file.path === diff.new.path)) {
+        if (!files.some(file => file.path === diff.newPath)) {
-          toDelete.push({ path: diff.new.path, delete: true });
+          toDelete.push({ path: diff.newPath, delete: true });
         }
       }
 
@@ -571,31 +562,37 @@ export default class API {
     const branch = this.branchFromContentKey(contentKey);
     const pullRequest = await this.getBranchPullRequest(branch);
     const diff = await this.getDifferences(branch);
-    const path = diff.find(d => d.new.path.includes(slug))?.new.path as string;
+    const { newPath: path, newFile } = diff.find(d => !d.binary) as {
+      newPath: string;
+      newFile: boolean;
+    };
     // TODO: get real file id
     const mediaFiles = await Promise.all(
-      diff.filter(d => d.new.path !== path).map(d => ({ path: d.new.path, id: null })),
+      diff.filter(d => d.newPath !== path).map(d => ({ path: d.newPath, id: null })),
     );
     const label = await this.getPullRequestLabel(pullRequest.id);
     const status = labelToStatus(label);
-    return { branch, collection, slug, path, status, mediaFiles };
+    return { branch, collection, slug, path, status, newFile, mediaFiles };
   }
 
   async readUnpublishedBranchFile(contentKey: string) {
-    const { branch, collection, slug, path, status, mediaFiles } = await this.retrieveMetadata(
+    const {
-      contentKey,
+      branch,
-    );
+      collection,
+      slug,
+      path,
+      status,
+      newFile,
+      mediaFiles,
+    } = await this.retrieveMetadata(contentKey);
 
-    const [fileData, isModification] = await Promise.all([
+    const fileData = (await this.readFile(path, null, { branch })) as string;
-      this.readFile(path, null, { branch }) as Promise<string>,
-      this.isFileExists(path, this.branch),
-    ]);
 
     return {
       slug,
       metaData: { branch, collection, objects: { entry: { path, mediaFiles } }, status },
       fileData,
-      isModification,
+      isModification: !newFile,
     };
   }
 

packages/netlify-cms-backend-bitbucket/src/types/what-the-diff.d.ts

@@ -0,0 +1,3 @@
+declare module 'what-the-diff' {
+  export const parse: (rawDiff: string) => { newPath: string; binary: boolean; status: string }[];
+}

packages/netlify-cms-backend-gitlab/src/API.ts

@@ -73,6 +73,7 @@ type GitLabCommitDiff = {
   diff: string;
   new_path: string;
   old_path: string;
+  new_file: boolean;
 };
 
 enum GitLabCommitStatuses {
@@ -546,7 +547,10 @@ export default class API {
       },
     });
 
-    return result.diffs;
+    return result.diffs.map(d => ({
+      ...d,
+      binary: d.diff.startsWith('Binary') || /.svg$/.test(d.new_path),
+    }));
   }
 
   async retrieveMetadata(contentKey: string) {
@@ -554,7 +558,10 @@ export default class API {
     const branch = this.branchFromContentKey(contentKey);
     const mergeRequest = await this.getBranchMergeRequest(branch);
     const diff = await this.getDifferences(mergeRequest.sha);
-    const path = diff.find(d => d.old_path.includes(slug))?.old_path as string;
+    const { old_path: path, new_file: newFile } = diff.find(d => !d.binary) as {
+      old_path: string;
+      new_file: boolean;
+    };
     const mediaFiles = await Promise.all(
       diff
         .filter(d => d.old_path !== path)
@@ -566,24 +573,27 @@ export default class API {
     );
     const label = mergeRequest.labels.find(isCMSLabel) as string;
     const status = labelToStatus(label);
-    return { branch, collection, slug, path, status, mediaFiles };
+    return { branch, collection, slug, path, status, newFile, mediaFiles };
   }
 
   async readUnpublishedBranchFile(contentKey: string) {
-    const { branch, collection, slug, path, status, mediaFiles } = await this.retrieveMetadata(
+    const {
-      contentKey,
+      branch,
-    );
+      collection,
+      slug,
+      path,
+      status,
+      newFile,
+      mediaFiles,
+    } = await this.retrieveMetadata(contentKey);
 
-    const [fileData, isModification] = await Promise.all([
+    const fileData = (await this.readFile(path, null, { branch })) as string;
-      this.readFile(path, null, { branch }) as Promise<string>,
-      this.isFileExists(path, this.branch),
-    ]);
 
     return {
       slug,
       metaData: { branch, collection, objects: { entry: { path, mediaFiles } }, status },
       fileData,
-      isModification,
+      isModification: !newFile,
     };
   }
 

yarn.lock

@@ -8134,7 +8134,6 @@ fsevents@^1.2.7:
   dependencies:
     bindings "^1.5.0"
     nan "^2.12.1"
-    node-pre-gyp "*"
 
 fsevents@~2.1.2:
   version "2.1.2"
@@ -17483,6 +17482,11 @@ what-input@^5.1.4:
   resolved "https://registry.yarnpkg.com/what-input/-/what-input-5.2.6.tgz#ac6f003bf8d3592a0031dea7a03565469b00020b"
   integrity sha512-a0BcI5YR7xp87vSzGcbN0IszJKpUQuTmrZaTSQBl7TLDIdKj6rDhluQ7b/7lYGG81gWDvkySsEvwv4BW5an9kg==
 
+what-the-diff@^0.6.0:
+  version "0.6.0"
+  resolved "https://registry.yarnpkg.com/what-the-diff/-/what-the-diff-0.6.0.tgz#445cc56a9d8ee9aea0ee1ed943f4957ae009291e"
+  integrity sha512-8BgQ4uo4cxojRXvCIcqDpH4QHaq0Ksn2P3LYfztylC5LDSwZKuGHf0Wf7sAStjPLTcB8eCB8pJJcPQSWfhZlkg==
+
 whatwg-encoding@^1.0.1, whatwg-encoding@^1.0.3:
   version "1.0.5"
   resolved "https://registry.yarnpkg.com/whatwg-encoding/-/whatwg-encoding-1.0.5.tgz#5abacf777c32166a51d085d6b4f3e7d27113ddb0"