From 39f1307e3a36447da8c9b3ca79b1d7db52ea1a19 Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Thu, 5 Mar 2020 11:58:49 +0100 Subject: [PATCH] feat(backend-github): add pagination (#3379) --- .../src/implementation.ts | 5 +- .../src/__tests__/implementation.spec.js | 182 ++++++++++++++++++ .../src/implementation.tsx | 108 ++++++++++- packages/netlify-cms-lib-util/src/Cursor.ts | 7 +- 4 files changed, 295 insertions(+), 7 deletions(-) diff --git a/packages/netlify-cms-backend-git-gateway/src/implementation.ts b/packages/netlify-cms-backend-git-gateway/src/implementation.ts index 2814931f..633c4634 100644 --- a/packages/netlify-cms-backend-git-gateway/src/implementation.ts +++ b/packages/netlify-cms-backend-git-gateway/src/implementation.ts @@ -258,6 +258,9 @@ export default class GitGateway implements Implementation { entriesByFolder(folder: string, extension: string, depth: number) { return this.backend!.entriesByFolder(folder, extension, depth); } + allEntriesByFolder(folder: string, extension: string, depth: number) { + return this.backend!.allEntriesByFolder(folder, extension, depth); + } entriesByFiles(files: ImplementationFile[]) { return this.backend!.entriesByFiles(files); } @@ -511,6 +514,6 @@ export default class GitGateway implements Implementation { return this.backend!.publishUnpublishedEntry(collection, slug); } traverseCursor(cursor: Cursor, action: string) { - return (this.backend as GitLabBackend | BitbucketBackend).traverseCursor!(cursor, action); + return this.backend!.traverseCursor!(cursor, action); } } diff --git a/packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js b/packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js index b51f945e..37bc5f0a 100644 --- a/packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js +++ b/packages/netlify-cms-backend-github/src/__tests__/implementation.spec.js @@ -1,4 +1,5 @@ import GitHubImplementation from '../implementation'; +import { Cursor, CURSOR_COMPATIBILITY_SYMBOL } from 'netlify-cms-lib-util'; jest.spyOn(console, 'error').mockImplementation(() => {}); @@ -214,4 +215,185 @@ describe('github backend implementation', () => { ]); }); }); + + describe('entriesByFolder', () => { + const listFiles = jest.fn(); + const readFile = jest.fn(); + + const mockAPI = { + listFiles, + readFile, + originRepoURL: 'originRepoURL', + }; + + it('should return entries and cursor', async () => { + const gitHubImplementation = new GitHubImplementation(config); + gitHubImplementation.api = mockAPI; + + const files = []; + const count = 1501; + for (let i = 0; i < count; i++) { + const id = `${i}`.padStart(`${count}`.length, '0'); + files.push({ + id, + path: `posts/post-${id}.md`, + }); + } + + listFiles.mockResolvedValue(files); + readFile.mockImplementation((path, id) => Promise.resolve(`${id}`)); + + const expectedEntries = files + .slice(0, 20) + .map(({ id, path }) => ({ data: id, file: { path, id } })); + + const expectedCursor = Cursor.create({ + actions: ['next', 'last'], + meta: { page: 1, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + expectedEntries[CURSOR_COMPATIBILITY_SYMBOL] = expectedCursor; + + const result = await gitHubImplementation.entriesByFolder('posts', 'md', 1); + + expect(result).toEqual(expectedEntries); + expect(listFiles).toHaveBeenCalledTimes(1); + expect(listFiles).toHaveBeenCalledWith('posts', { depth: 1, repoURL: 'originRepoURL' }); + expect(readFile).toHaveBeenCalledTimes(20); + }); + }); + + describe('traverseCursor', () => { + const listFiles = jest.fn(); + const readFile = jest.fn((path, id) => Promise.resolve(`${id}`)); + + const mockAPI = { + listFiles, + readFile, + originRepoURL: 'originRepoURL', + }; + + const files = []; + const count = 1501; + for (let i = 0; i < count; i++) { + const id = `${i}`.padStart(`${count}`.length, '0'); + files.push({ + id, + path: `posts/post-${id}.md`, + }); + } + + it('should handle next action', async () => { + const gitHubImplementation = new GitHubImplementation(config); + gitHubImplementation.api = mockAPI; + + const cursor = Cursor.create({ + actions: ['next', 'last'], + meta: { page: 1, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const expectedEntries = files + .slice(20, 40) + .map(({ id, path }) => ({ data: id, file: { path, id } })); + + const expectedCursor = Cursor.create({ + actions: ['prev', 'first', 'next', 'last'], + meta: { page: 2, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const result = await gitHubImplementation.traverseCursor(cursor, 'next'); + + expect(result).toEqual({ + entries: expectedEntries, + cursor: expectedCursor, + }); + }); + + it('should handle prev action', async () => { + const gitHubImplementation = new GitHubImplementation(config); + gitHubImplementation.api = mockAPI; + + const cursor = Cursor.create({ + actions: ['prev', 'first', 'next', 'last'], + meta: { page: 2, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const expectedEntries = files + .slice(0, 20) + .map(({ id, path }) => ({ data: id, file: { path, id } })); + + const expectedCursor = Cursor.create({ + actions: ['next', 'last'], + meta: { page: 1, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const result = await gitHubImplementation.traverseCursor(cursor, 'prev'); + + expect(result).toEqual({ + entries: expectedEntries, + cursor: expectedCursor, + }); + }); + + it('should handle last action', async () => { + const gitHubImplementation = new GitHubImplementation(config); + gitHubImplementation.api = mockAPI; + + const cursor = Cursor.create({ + actions: ['next', 'last'], + meta: { page: 1, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const expectedEntries = files + .slice(1500) + .map(({ id, path }) => ({ data: id, file: { path, id } })); + + const expectedCursor = Cursor.create({ + actions: ['prev', 'first'], + meta: { page: 76, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const result = await gitHubImplementation.traverseCursor(cursor, 'last'); + + expect(result).toEqual({ + entries: expectedEntries, + cursor: expectedCursor, + }); + }); + + it('should handle first action', async () => { + const gitHubImplementation = new GitHubImplementation(config); + gitHubImplementation.api = mockAPI; + + const cursor = Cursor.create({ + actions: ['prev', 'first'], + meta: { page: 76, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const expectedEntries = files + .slice(0, 20) + .map(({ id, path }) => ({ data: id, file: { path, id } })); + + const expectedCursor = Cursor.create({ + actions: ['next', 'last'], + meta: { page: 1, count, pageSize: 20, pageCount: 76 }, + data: { files }, + }); + + const result = await gitHubImplementation.traverseCursor(cursor, 'first'); + + expect(result).toEqual({ + entries: expectedEntries, + cursor: expectedCursor, + }); + }); + }); }); diff --git a/packages/netlify-cms-backend-github/src/implementation.tsx b/packages/netlify-cms-backend-github/src/implementation.tsx index 5c131855..6f15b5bd 100644 --- a/packages/netlify-cms-backend-github/src/implementation.tsx +++ b/packages/netlify-cms-backend-github/src/implementation.tsx @@ -3,6 +3,8 @@ import semaphore, { Semaphore } from 'semaphore'; import trimStart from 'lodash/trimStart'; import { stripIndent } from 'common-tags'; import { + CURSOR_COMPATIBILITY_SYMBOL, + Cursor, asyncLock, basename, AsyncLock, @@ -29,13 +31,15 @@ import { } from 'netlify-cms-lib-util'; import AuthenticationPage from './AuthenticationPage'; import { Octokit } from '@octokit/rest'; -import API, { Entry } from './API'; +import API, { Entry, API_NAME } from './API'; import GraphQLAPI from './GraphQLAPI'; type GitHubUser = Octokit.UsersGetAuthenticatedResponse; const MAX_CONCURRENT_DOWNLOADS = 10; +type ApiFile = { id: string; type: string; name: string; path: string; size: number }; + export default class GitHub implements Implementation { lock: AsyncLock; api: API | null; @@ -281,19 +285,72 @@ export default class GitHub implements Implementation { return Promise.resolve(this.token); } + getCursorAndFiles = (files: ApiFile[], page: number) => { + const pageSize = 20; + const count = files.length; + const pageCount = Math.ceil(files.length / pageSize); + + const actions = [] as string[]; + if (page > 1) { + actions.push('prev'); + actions.push('first'); + } + if (page < pageCount) { + actions.push('next'); + actions.push('last'); + } + + const cursor = Cursor.create({ + actions, + meta: { page, count, pageSize, pageCount }, + data: { files }, + }); + const pageFiles = files.slice((page - 1) * pageSize, page * pageSize); + return { cursor, files: pageFiles }; + }; + async entriesByFolder(folder: string, extension: string, depth: number) { - const repoURL = this.useOpenAuthoring ? this.api!.originRepoURL : this.api!.repoURL; + const repoURL = this.api!.originRepoURL; + + let cursor: Cursor; const listFiles = () => this.api!.listFiles(folder, { repoURL, depth, - }).then(filterByPropExtension(extension, 'path')); + }).then(files => { + const filtered = filterByPropExtension(extension, 'path')(files); + const result = this.getCursorAndFiles(filtered, 1); + cursor = result.cursor; + return result.files; + }); const readFile = (path: string, id: string | null | undefined) => this.api!.readFile(path, id, { repoURL }) as Promise; - return entriesByFolder(listFiles, readFile, 'GitHub'); + const files = await entriesByFolder(listFiles, readFile, API_NAME); + // eslint-disable-next-line @typescript-eslint/ban-ts-ignore + // @ts-ignore + files[CURSOR_COMPATIBILITY_SYMBOL] = cursor; + return files; + } + + async allEntriesByFolder(folder: string, extension: string, depth: number) { + const repoURL = this.api!.originRepoURL; + + const listFiles = () => + this.api!.listFiles(folder, { + repoURL, + depth, + }).then(files => { + return filterByPropExtension(extension, 'path')(files); + }); + + const readFile = (path: string, id: string | null | undefined) => + this.api!.readFile(path, id, { repoURL }) as Promise; + + const files = await entriesByFolder(listFiles, readFile, API_NAME); + return files; } entriesByFiles(files: ImplementationFile[]) { @@ -385,6 +442,49 @@ export default class GitHub implements Implementation { return this.api!.deleteFile(path, commitMessage); } + async traverseCursor(cursor: Cursor, action: string) { + const meta = cursor.meta!; + const files = cursor.data!.get('files')!.toJS() as ApiFile[]; + + let result: { cursor: Cursor; files: ApiFile[] }; + switch (action) { + case 'first': { + result = this.getCursorAndFiles(files, 1); + break; + } + case 'last': { + result = this.getCursorAndFiles(files, meta.get('pageCount')); + break; + } + case 'next': { + result = this.getCursorAndFiles(files, meta.get('page') + 1); + break; + } + case 'prev': { + result = this.getCursorAndFiles(files, meta.get('page') - 1); + break; + } + default: { + result = this.getCursorAndFiles(files, 1); + break; + } + } + + return { + entries: await Promise.all( + result.files.map(file => + this.api!.readFile(file.path, file.id, { repoURL: this.api!.originRepoURL }).then( + data => ({ + file, + data: data as string, + }), + ), + ), + ), + cursor: result.cursor, + }; + } + loadMediaFile(branch: string, file: UnpublishedEntryMediaFile) { const readFile = ( path: string, diff --git a/packages/netlify-cms-lib-util/src/Cursor.ts b/packages/netlify-cms-lib-util/src/Cursor.ts index 43d3baa7..35ac2922 100644 --- a/packages/netlify-cms-lib-util/src/Cursor.ts +++ b/packages/netlify-cms-lib-util/src/Cursor.ts @@ -40,6 +40,7 @@ const jsToMap = (obj: {}) => { const knownMetaKeys = Set([ 'index', + 'page', 'count', 'pageSize', 'pageCount', @@ -85,8 +86,10 @@ const getActionHandlers = (store: CursorStore, handler: ActionHandler) => export default class Cursor { store?: CursorStore; actions?: Set; - data?: Map; - meta?: Map; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + data?: Map; + // eslint-disable-next-line @typescript-eslint/no-explicit-any + meta?: Map; static create(...args: {}[]) { return new Cursor(...args);