feat(backend-github): add pagination (#3379)

packages/netlify-cms-backend-git-gateway/src/implementation.ts

@@ -258,6 +258,9 @@ export default class GitGateway implements Implementation {
   entriesByFolder(folder: string, extension: string, depth: number) {
     return this.backend!.entriesByFolder(folder, extension, depth);
   }
+  allEntriesByFolder(folder: string, extension: string, depth: number) {
+    return this.backend!.allEntriesByFolder(folder, extension, depth);
+  }
   entriesByFiles(files: ImplementationFile[]) {
     return this.backend!.entriesByFiles(files);
   }
@@ -511,6 +514,6 @@ export default class GitGateway implements Implementation {
     return this.backend!.publishUnpublishedEntry(collection, slug);
   }
   traverseCursor(cursor: Cursor, action: string) {
-    return (this.backend as GitLabBackend | BitbucketBackend).traverseCursor!(cursor, action);
+    return this.backend!.traverseCursor!(cursor, action);
   }
 }

packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js

@@ -1,4 +1,5 @@
 import GitHubImplementation from '../implementation';
+import { Cursor, CURSOR_COMPATIBILITY_SYMBOL } from 'netlify-cms-lib-util';
 
 jest.spyOn(console, 'error').mockImplementation(() => {});
 
@@ -214,4 +215,185 @@ describe('github backend implementation', () => {
       ]);
     });
   });
+
+  describe('entriesByFolder', () => {
+    const listFiles = jest.fn();
+    const readFile = jest.fn();
+
+    const mockAPI = {
+      listFiles,
+      readFile,
+      originRepoURL: 'originRepoURL',
+    };
+
+    it('should return entries and cursor', async () => {
+      const gitHubImplementation = new GitHubImplementation(config);
+      gitHubImplementation.api = mockAPI;
+
+      const files = [];
+      const count = 1501;
+      for (let i = 0; i < count; i++) {
+        const id = `${i}`.padStart(`${count}`.length, '0');
+        files.push({
+          id,
+          path: `posts/post-${id}.md`,
+        });
+      }
+
+      listFiles.mockResolvedValue(files);
+      readFile.mockImplementation((path, id) => Promise.resolve(`${id}`));
+
+      const expectedEntries = files
+        .slice(0, 20)
+        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+
+      const expectedCursor = Cursor.create({
+        actions: ['next', 'last'],
+        meta: { page: 1, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      expectedEntries[CURSOR_COMPATIBILITY_SYMBOL] = expectedCursor;
+
+      const result = await gitHubImplementation.entriesByFolder('posts', 'md', 1);
+
+      expect(result).toEqual(expectedEntries);
+      expect(listFiles).toHaveBeenCalledTimes(1);
+      expect(listFiles).toHaveBeenCalledWith('posts', { depth: 1, repoURL: 'originRepoURL' });
+      expect(readFile).toHaveBeenCalledTimes(20);
+    });
+  });
+
+  describe('traverseCursor', () => {
+    const listFiles = jest.fn();
+    const readFile = jest.fn((path, id) => Promise.resolve(`${id}`));
+
+    const mockAPI = {
+      listFiles,
+      readFile,
+      originRepoURL: 'originRepoURL',
+    };
+
+    const files = [];
+    const count = 1501;
+    for (let i = 0; i < count; i++) {
+      const id = `${i}`.padStart(`${count}`.length, '0');
+      files.push({
+        id,
+        path: `posts/post-${id}.md`,
+      });
+    }
+
+    it('should handle next action', async () => {
+      const gitHubImplementation = new GitHubImplementation(config);
+      gitHubImplementation.api = mockAPI;
+
+      const cursor = Cursor.create({
+        actions: ['next', 'last'],
+        meta: { page: 1, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const expectedEntries = files
+        .slice(20, 40)
+        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+
+      const expectedCursor = Cursor.create({
+        actions: ['prev', 'first', 'next', 'last'],
+        meta: { page: 2, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const result = await gitHubImplementation.traverseCursor(cursor, 'next');
+
+      expect(result).toEqual({
+        entries: expectedEntries,
+        cursor: expectedCursor,
+      });
+    });
+
+    it('should handle prev action', async () => {
+      const gitHubImplementation = new GitHubImplementation(config);
+      gitHubImplementation.api = mockAPI;
+
+      const cursor = Cursor.create({
+        actions: ['prev', 'first', 'next', 'last'],
+        meta: { page: 2, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const expectedEntries = files
+        .slice(0, 20)
+        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+
+      const expectedCursor = Cursor.create({
+        actions: ['next', 'last'],
+        meta: { page: 1, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const result = await gitHubImplementation.traverseCursor(cursor, 'prev');
+
+      expect(result).toEqual({
+        entries: expectedEntries,
+        cursor: expectedCursor,
+      });
+    });
+
+    it('should handle last action', async () => {
+      const gitHubImplementation = new GitHubImplementation(config);
+      gitHubImplementation.api = mockAPI;
+
+      const cursor = Cursor.create({
+        actions: ['next', 'last'],
+        meta: { page: 1, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const expectedEntries = files
+        .slice(1500)
+        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+
+      const expectedCursor = Cursor.create({
+        actions: ['prev', 'first'],
+        meta: { page: 76, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const result = await gitHubImplementation.traverseCursor(cursor, 'last');
+
+      expect(result).toEqual({
+        entries: expectedEntries,
+        cursor: expectedCursor,
+      });
+    });
+
+    it('should handle first action', async () => {
+      const gitHubImplementation = new GitHubImplementation(config);
+      gitHubImplementation.api = mockAPI;
+
+      const cursor = Cursor.create({
+        actions: ['prev', 'first'],
+        meta: { page: 76, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const expectedEntries = files
+        .slice(0, 20)
+        .map(({ id, path }) => ({ data: id, file: { path, id } }));
+
+      const expectedCursor = Cursor.create({
+        actions: ['next', 'last'],
+        meta: { page: 1, count, pageSize: 20, pageCount: 76 },
+        data: { files },
+      });
+
+      const result = await gitHubImplementation.traverseCursor(cursor, 'first');
+
+      expect(result).toEqual({
+        entries: expectedEntries,
+        cursor: expectedCursor,
+      });
+    });
+  });
 });

packages/netlify-cms-backend-github/src/implementation.tsx

@@ -3,6 +3,8 @@ import semaphore, { Semaphore } from 'semaphore';
 import trimStart from 'lodash/trimStart';
 import { stripIndent } from 'common-tags';
 import {
+  CURSOR_COMPATIBILITY_SYMBOL,
+  Cursor,
   asyncLock,
   basename,
   AsyncLock,
@@ -29,13 +31,15 @@ import {
 } from 'netlify-cms-lib-util';
 import AuthenticationPage from './AuthenticationPage';
 import { Octokit } from '@octokit/rest';
-import API, { Entry } from './API';
+import API, { Entry, API_NAME } from './API';
 import GraphQLAPI from './GraphQLAPI';
 
 type GitHubUser = Octokit.UsersGetAuthenticatedResponse;
 
 const MAX_CONCURRENT_DOWNLOADS = 10;
 
+type ApiFile = { id: string; type: string; name: string; path: string; size: number };
+
 export default class GitHub implements Implementation {
   lock: AsyncLock;
   api: API | null;
@@ -281,19 +285,72 @@ export default class GitHub implements Implementation {
     return Promise.resolve(this.token);
   }
 
+  getCursorAndFiles = (files: ApiFile[], page: number) => {
+    const pageSize = 20;
+    const count = files.length;
+    const pageCount = Math.ceil(files.length / pageSize);
+
+    const actions = [] as string[];
+    if (page > 1) {
+      actions.push('prev');
+      actions.push('first');
+    }
+    if (page < pageCount) {
+      actions.push('next');
+      actions.push('last');
+    }
+
+    const cursor = Cursor.create({
+      actions,
+      meta: { page, count, pageSize, pageCount },
+      data: { files },
+    });
+    const pageFiles = files.slice((page - 1) * pageSize, page * pageSize);
+    return { cursor, files: pageFiles };
+  };
+
   async entriesByFolder(folder: string, extension: string, depth: number) {
-    const repoURL = this.useOpenAuthoring ? this.api!.originRepoURL : this.api!.repoURL;
+    const repoURL = this.api!.originRepoURL;
+
+    let cursor: Cursor;
 
     const listFiles = () =>
       this.api!.listFiles(folder, {
         repoURL,
         depth,
-      }).then(filterByPropExtension(extension, 'path'));
+      }).then(files => {
+        const filtered = filterByPropExtension(extension, 'path')(files);
+        const result = this.getCursorAndFiles(filtered, 1);
+        cursor = result.cursor;
+        return result.files;
+      });
 
     const readFile = (path: string, id: string | null | undefined) =>
       this.api!.readFile(path, id, { repoURL }) as Promise<string>;
 
-    return entriesByFolder(listFiles, readFile, 'GitHub');
+    const files = await entriesByFolder(listFiles, readFile, API_NAME);
+    // eslint-disable-next-line @typescript-eslint/ban-ts-ignore
+    // @ts-ignore
+    files[CURSOR_COMPATIBILITY_SYMBOL] = cursor;
+    return files;
+  }
+
+  async allEntriesByFolder(folder: string, extension: string, depth: number) {
+    const repoURL = this.api!.originRepoURL;
+
+    const listFiles = () =>
+      this.api!.listFiles(folder, {
+        repoURL,
+        depth,
+      }).then(files => {
+        return filterByPropExtension(extension, 'path')(files);
+      });
+
+    const readFile = (path: string, id: string | null | undefined) =>
+      this.api!.readFile(path, id, { repoURL }) as Promise<string>;
+
+    const files = await entriesByFolder(listFiles, readFile, API_NAME);
+    return files;
   }
 
   entriesByFiles(files: ImplementationFile[]) {
@@ -385,6 +442,49 @@ export default class GitHub implements Implementation {
     return this.api!.deleteFile(path, commitMessage);
   }
 
+  async traverseCursor(cursor: Cursor, action: string) {
+    const meta = cursor.meta!;
+    const files = cursor.data!.get('files')!.toJS() as ApiFile[];
+
+    let result: { cursor: Cursor; files: ApiFile[] };
+    switch (action) {
+      case 'first': {
+        result = this.getCursorAndFiles(files, 1);
+        break;
+      }
+      case 'last': {
+        result = this.getCursorAndFiles(files, meta.get('pageCount'));
+        break;
+      }
+      case 'next': {
+        result = this.getCursorAndFiles(files, meta.get('page') + 1);
+        break;
+      }
+      case 'prev': {
+        result = this.getCursorAndFiles(files, meta.get('page') - 1);
+        break;
+      }
+      default: {
+        result = this.getCursorAndFiles(files, 1);
+        break;
+      }
+    }
+
+    return {
+      entries: await Promise.all(
+        result.files.map(file =>
+          this.api!.readFile(file.path, file.id, { repoURL: this.api!.originRepoURL }).then(
+            data => ({
+              file,
+              data: data as string,
+            }),
+          ),
+        ),
+      ),
+      cursor: result.cursor,
+    };
+  }
+
   loadMediaFile(branch: string, file: UnpublishedEntryMediaFile) {
     const readFile = (
       path: string,

packages/netlify-cms-lib-util/src/Cursor.ts

@@ -40,6 +40,7 @@ const jsToMap = (obj: {}) => {
 
 const knownMetaKeys = Set([
   'index',
+  'page',
   'count',
   'pageSize',
   'pageCount',
@@ -85,8 +86,10 @@ const getActionHandlers = (store: CursorStore, handler: ActionHandler) =>
 export default class Cursor {
   store?: CursorStore;
   actions?: Set<string>;
-  data?: Map<string, unknown>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
-  meta?: Map<string, unknown>;
+  data?: Map<string, any>;
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  meta?: Map<string, any>;
 
   static create(...args: {}[]) {
     return new Cursor(...args);