Feat: entry sorting (#3494)

* refactor: typescript search actions, add tests avoid duplicate search

* refactor: switch from promise chain to async/await in loadEntries

* feat: add sorting, initial commit

* fix: set isFetching to true on entries request

* fix: ui improvments and bug fixes

* test: fix tests

* feat(backend-gitlab): cache local tree)

* fix: fix prop type warning

* refactor: code cleanup

* feat(backend-bitbucket): add local tree caching support

* feat: swtich to orderBy and support multiple sort keys

* fix: backoff function

* fix: improve backoff

* feat: infer sortable fields

* feat: fetch file commit metadata - initial commit

* feat: extract file author and date, finalize GitLab & Bitbucket

* refactor: code cleanup

* feat: handle github rate limit errors

* refactor: code cleanup

* fix(github): add missing author and date when traversing cursor

* fix: add missing author and date when traversing cursor

* refactor: code cleanup

* refactor: code cleanup

* refactor: code cleanup

* test: fix tests

* fix: rebuild local tree when head doesn't exist in remote branch

* fix: allow sortable fields to be an empty array

* fix: allow translation of built in sort fields

* build: fix proxy server build

* fix: hide commit author and date fields by default on non git backends

* fix(algolia): add listAllEntries method for alogolia integration

* fix: handle sort fields overflow

* test(bitbucket): re-record some bitbucket e2e tests

* test(bitbucket): fix media library test

* refactor(gitgateway-gitlab): share request code and handle 404 errors

* fix: always show commit date by default

* docs: add sortableFields

* refactor: code cleanup

* improvement: drop multi-sort, rework sort UI

* chore: force main package bumps

Co-authored-by: Shawn Erquhart <shawn@erquh.art>

packages/netlify-cms-backend-github/src/API.ts

@@ -12,6 +12,7 @@ import {
   Entry as LibEntry,
   PersistOptions,
   readFile,
+  readFileMetadata,
   CMS_BRANCH_PREFIX,
   generateContentKey,
   DEFAULT_PR_BODY,
@@ -24,6 +25,9 @@ import {
   labelToStatus,
   statusToLabel,
   contentKeyFromBranch,
+  requestWithBackoff,
+  unsentRequest,
+  ApiRequest,
 } from 'netlify-cms-lib-util';
 import { Octokit } from '@octokit/rest';
 
@@ -276,21 +280,31 @@ export default class API {
     throw new APIError(error.message, responseStatus, API_NAME);
   }
 
+  buildRequest(req: ApiRequest) {
+    return req;
+  }
+
   async request(
     path: string,
     options: Options = {},
-    // eslint-disable-next-line @typescript-eslint/no-explicit-any
     parser = (response: Response) => this.parseResponse(response),
   ) {
     const headers = await this.requestHeaders(options.headers || {});
     const url = this.urlFor(path, options);
-    let responseStatus: number;
-    return fetch(url, { ...options, headers })
-      .then(response => {
-        responseStatus = response.status;
-        return parser(response);
-      })
-      .catch(error => this.handleRequestError(error, responseStatus));
+    let responseStatus = 500;
+
+    try {
+      const req = (unsentRequest.fromFetchArguments(url, {
+        ...options,
+        headers,
+      }) as unknown) as ApiRequest;
+      const response = await requestWithBackoff(this, req);
+      responseStatus = response.status;
+      const parsedResponse = await parser(response);
+      return parsedResponse;
+    } catch (error) {
+      return this.handleRequestError(error, responseStatus);
+    }
   }
 
   nextUrlProcessor() {
@@ -580,6 +594,28 @@ export default class API {
     return content;
   }
 
+  async readFileMetadata(path: string, sha: string) {
+    const fetchFileMetadata = async () => {
+      try {
+        const result: Octokit.ReposListCommitsResponse = await this.request(
+          `${this.originRepoURL}/commits`,
+          {
+            params: { path, sha: this.branch },
+          },
+        );
+        const { commit } = result[0];
+        return {
+          author: commit.author.name || commit.author.email,
+          updatedOn: commit.author.date,
+        };
+      } catch (e) {
+        return { author: '', updatedOn: '' };
+      }
+    };
+    const fileMetadata = await readFileMetadata(sha, fetchFileMetadata, localForage);
+    return fileMetadata;
+  }
+
   async fetchBlobContent({ sha, repoURL, parseText }: BlobArgs) {
     const result: Octokit.GitGetBlobResponse = await this.request(`${repoURL}/git/blobs/${sha}`);
 

packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js

@@ -219,10 +219,12 @@ describe('github backend implementation', () => {
   describe('entriesByFolder', () => {
     const listFiles = jest.fn();
     const readFile = jest.fn();
+    const readFileMetadata = jest.fn(() => Promise.resolve({ author: '', updatedOn: '' }));
 
     const mockAPI = {
       listFiles,
       readFile,
+      readFileMetadata,
       originRepoURL: 'originRepoURL',
     };
 
@@ -245,7 +247,7 @@ describe('github backend implementation', () => {
 
       const expectedEntries = files
         .slice(0, 20)
-        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+        .map(({ id, path }) => ({ data: id, file: { path, id, author: '', updatedOn: '' } }));
 
       const expectedCursor = Cursor.create({
         actions: ['next', 'last'],
@@ -267,11 +269,13 @@ describe('github backend implementation', () => {
   describe('traverseCursor', () => {
     const listFiles = jest.fn();
     const readFile = jest.fn((path, id) => Promise.resolve(`${id}`));
+    const readFileMetadata = jest.fn(() => Promise.resolve({}));
 
     const mockAPI = {
       listFiles,
       readFile,
       originRepoURL: 'originRepoURL',
+      readFileMetadata,
     };
 
     const files = [];

packages/netlify-cms-backend-github/src/implementation.tsx

@@ -20,7 +20,7 @@ import {
   getMediaDisplayURL,
   getMediaAsBlob,
   Credentials,
-  filterByPropExtension,
+  filterByExtension,
   Config,
   ImplementationFile,
   getPreviewStatus,
@@ -104,6 +104,10 @@ export default class GitHub implements Implementation {
     this.lock = asyncLock();
   }
 
+  isGitBackend() {
+    return true;
+  }
+
   authComponent() {
     const wrappedAuthenticationPage = (props: Record<string, unknown>) => (
       <AuthenticationPage {...props} backend={this} />
@@ -319,7 +323,7 @@ export default class GitHub implements Implementation {
         repoURL,
         depth,
       }).then(files => {
-        const filtered = filterByPropExtension(extension, 'path')(files);
+        const filtered = files.filter(file => filterByExtension(file, extension));
         const result = this.getCursorAndFiles(filtered, 1);
         cursor = result.cursor;
         return result.files;
@@ -328,7 +332,12 @@ export default class GitHub implements Implementation {
     const readFile = (path: string, id: string | null | undefined) =>
       this.api!.readFile(path, id, { repoURL }) as Promise<string>;
 
-    const files = await entriesByFolder(listFiles, readFile, API_NAME);
+    const files = await entriesByFolder(
+      listFiles,
+      readFile,
+      this.api!.readFileMetadata.bind(this.api),
+      API_NAME,
+    );
     // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
     // @ts-ignore
     files[CURSOR_COMPATIBILITY_SYMBOL] = cursor;
@@ -342,14 +351,18 @@ export default class GitHub implements Implementation {
       this.api!.listFiles(folder, {
         repoURL,
         depth,
-      }).then(files => {
-        return filterByPropExtension(extension, 'path')(files);
-      });
+      }).then(files => files.filter(file => filterByExtension(file, extension)));
 
-    const readFile = (path: string, id: string | null | undefined) =>
-      this.api!.readFile(path, id, { repoURL }) as Promise<string>;
+    const readFile = (path: string, id: string | null | undefined) => {
+      return this.api!.readFile(path, id, { repoURL }) as Promise<string>;
+    };
 
-    const files = await entriesByFolder(listFiles, readFile, API_NAME);
+    const files = await entriesByFolder(
+      listFiles,
+      readFile,
+      this.api!.readFileMetadata.bind(this.api),
+      API_NAME,
+    );
     return files;
   }
 
@@ -359,7 +372,7 @@ export default class GitHub implements Implementation {
     const readFile = (path: string, id: string | null | undefined) =>
       this.api!.readFile(path, id, { repoURL }).catch(() => '') as Promise<string>;
 
-    return entriesByFiles(files, readFile, 'GitHub');
+    return entriesByFiles(files, readFile, this.api!.readFileMetadata.bind(this.api), API_NAME);
   }
 
   // Fetches a single entry.
@@ -470,17 +483,20 @@ export default class GitHub implements Implementation {
       }
     }
 
+    const readFile = (path: string, id: string | null | undefined) =>
+      this.api!.readFile(path, id, { repoURL: this.api!.originRepoURL }).catch(() => '') as Promise<
+        string
+      >;
+
+    const entries = await entriesByFiles(
+      result.files,
+      readFile,
+      this.api!.readFileMetadata.bind(this.api),
+      API_NAME,
+    );
+
     return {
-      entries: await Promise.all(
-        result.files.map(file =>
-          this.api!.readFile(file.path, file.id, { repoURL: this.api!.originRepoURL }).then(
-            data => ({
-              file,
-              data: data as string,
-            }),
-          ),
-        ),
-      ),
+      entries,
       cursor: result.cursor,
     };
   }