gcg/static-cms
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix: get files by path depth (#2993)

Browse Source 
* fix: get files up to depth specified by colletion path

* test(e2e): update mock data

* chore: fix comment
This commit is contained in:
Erez Rokah
2019-12-22 15:20:42 +02:00
committed by GitHub
parent 982fd7b0f8
commit b27748b54f
75 changed files with 4075 additions and 3714 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
packages/netlify-cms-backend-bitbucket/src/API.js
View File

@ -114,11 +114,11 @@ export default class API {
};
};
listFiles = async path => {
listFiles = async (path, depth) => {
const node = await this.branchCommitSha();
const { entries, cursor } = await flow([
// sort files by filename ascending
unsentRequest.withParams({ sort: '-path', max_depth: 10 }),
unsentRequest.withParams({ sort: '-path', max_depth: depth }),
this.requestJSON,
then(this.getEntriesAndCursor),
])(`${this.repoURL}/src/${node}/${path}`);
@ -135,8 +135,8 @@ export default class API {
})),
])(cursor.data.getIn(['links', action]));
listAllFiles = async path => {
const { cursor: initialCursor, entries: initialEntries } = await this.listFiles(path);
listAllFiles = async (path, depth) => {
const { cursor: initialCursor, entries: initialEntries } = await this.listFiles(path, depth);
const entries = [...initialEntries];
let currentCursor = initialCursor;
while (currentCursor && currentCursor.actions.has('next')) {

8
packages/netlify-cms-backend-bitbucket/src/implementation.js
View File

@ -9,6 +9,7 @@ import {
unsentRequest,
basename,
getBlobSHA,
getCollectionDepth,
} from 'netlify-cms-lib-util';
import { NetlifyAuthenticator } from 'netlify-cms-lib-auth';
import AuthenticationPage from './AuthenticationPage';
@ -165,7 +166,10 @@ export default class BitbucketBackend {
};
entriesByFolder(collection, extension) {
const listPromise = this.api.listFiles(collection.get('folder'));
const listPromise = this.api.listFiles(
collection.get('folder'),
getCollectionDepth(collection),
);
return resolvePromiseProperties({
files: listPromise
.then(({ entries }) => entries)
@ -180,7 +184,7 @@ export default class BitbucketBackend {
allEntriesByFolder(collection, extension) {
return this.api
.listAllFiles(collection.get('folder'))
.listAllFiles(collection.get('folder'), getCollectionDepth(collection))
.then(filterByPropExtension(extension, 'path'))
.then(this.fetchFiles);
}

12
packages/netlify-cms-backend-github/src/API.ts
View File

@ -11,6 +11,7 @@ import {
onlySuccessfulPromises,
resolvePromiseProperties,
ResponseParser,
basename,
} from 'netlify-cms-lib-util';
import {
UsersGetAuthenticatedResponse as GitHubUser,
@ -509,17 +510,20 @@ export default class API {
});
}
async listFiles(path: string, { repoURL = this.repoURL, branch = this.branch } = {}) {
async listFiles(path: string, { repoURL = this.repoURL, branch = this.branch, depth = 1 } = {}) {
const folder = trimStart(path, '/');
return this.request(`${repoURL}/git/trees/${branch}:${folder}`, {
params: { recursive: 10 },
// GitHub API supports recursive=1 for getting the entire recursive tree
// or omitting it to get the non-recursive tree
params: depth > 1 ? { recursive: 1 } : {},
})
.then((res: GitHubTree) =>
res.tree
.filter(file => file.type === 'blob')
// filter only files and up to the required depth
.filter(file => file.type === 'blob' && file.path.split('/').length <= depth)
.map(file => ({
...file,
name: file.path,
name: basename(file.path),
path: `${folder}/${file.path}`,
})),
)

7
packages/netlify-cms-backend-github/src/GraphQLAPI.js
View File

@ -187,7 +187,8 @@ export default class GraphQLAPI extends API {
getAllFiles(entries, path) {
const allFiles = entries.reduce((acc, item) => {
if (item.type === 'tree') {
return [...acc, ...this.getAllFiles(item.object.entries, `${path}/${item.name}`)];
const entries = item.object?.entries || [];
return [...acc, ...this.getAllFiles(entries, `${path}/${item.name}`)];
} else if (item.type === 'blob') {
return [
...acc,
@ -204,10 +205,10 @@ export default class GraphQLAPI extends API {
return allFiles;
}
async listFiles(path, { repoURL = this.repoURL, branch = this.branch } = {}) {
async listFiles(path, { repoURL = this.repoURL, branch = this.branch, depth = 1 } = {}) {
const { owner, name } = this.getOwnerAndNameFromRepoUrl(repoURL);
const { data } = await this.query({
query: queries.files,
query: queries.files(depth),
variables: { owner, name, expression: `${branch}:${path}` },
});

83
packages/netlify-cms-backend-github/src/__tests__/API.spec.js
View File

@ -489,4 +489,87 @@ describe('github API', () => {
);
});
});
describe('listFiles', () => {
it('should get files by depth', async () => {
const api = new API({ branch: 'master', repo: 'owner/repo' });
const tree = [
{
path: 'post.md',
type: 'blob',
},
{
path: 'dir1',
type: 'tree',
},
{
path: 'dir1/nested-post.md',
type: 'blob',
},
{
path: 'dir1/dir2',
type: 'tree',
},
{
path: 'dir1/dir2/nested-post.md',
type: 'blob',
},
];
api.request = jest.fn().mockResolvedValue({ tree });
await expect(api.listFiles('posts', { depth: 1 })).resolves.toEqual([
{
path: 'posts/post.md',
type: 'blob',
name: 'post.md',
},
]);
expect(api.request).toHaveBeenCalledTimes(1);
expect(api.request).toHaveBeenCalledWith('/repos/owner/repo/git/trees/master:posts', {
params: {},
});
jest.clearAllMocks();
await expect(api.listFiles('posts', { depth: 2 })).resolves.toEqual([
{
path: 'posts/post.md',
type: 'blob',
name: 'post.md',
},
{
path: 'posts/dir1/nested-post.md',
type: 'blob',
name: 'nested-post.md',
},
]);
expect(api.request).toHaveBeenCalledTimes(1);
expect(api.request).toHaveBeenCalledWith('/repos/owner/repo/git/trees/master:posts', {
params: { recursive: 1 },
});
jest.clearAllMocks();
await expect(api.listFiles('posts', { depth: 3 })).resolves.toEqual([
{
path: 'posts/post.md',
type: 'blob',
name: 'post.md',
},
{
path: 'posts/dir1/nested-post.md',
type: 'blob',
name: 'nested-post.md',
},
{
path: 'posts/dir1/dir2/nested-post.md',
type: 'blob',
name: 'nested-post.md',
},
]);
expect(api.request).toHaveBeenCalledTimes(1);
expect(api.request).toHaveBeenCalledWith('/repos/owner/repo/git/trees/master:posts', {
params: { recursive: 1 },
});
});
});
});

7
packages/netlify-cms-backend-github/src/implementation.js
View File

@ -2,7 +2,7 @@ import React from 'react';
import trimStart from 'lodash/trimStart';
import semaphore from 'semaphore';
import { stripIndent } from 'common-tags';
import { asyncLock, basename } from 'netlify-cms-lib-util';
import { asyncLock, basename, getCollectionDepth } from 'netlify-cms-lib-util';
import AuthenticationPage from './AuthenticationPage';
import { get } from 'lodash';
import API from './API';
@ -256,7 +256,10 @@ export default class GitHub {
async entriesByFolder(collection, extension) {
const repoURL = this.useOpenAuthoring ? this.api.originRepoURL : this.api.repoURL;
const files = await this.api.listFiles(collection.get('folder'), { repoURL });
const files = await this.api.listFiles(collection.get('folder'), {
repoURL,
depth: getCollectionDepth(collection),
});
const filteredFiles = files.filter(file => file.name.endsWith('.' + extension));
return this.fetchFiles(filteredFiles, { repoURL });
}

8
packages/netlify-cms-backend-github/src/queries.js
View File

@ -93,7 +93,7 @@ export const statues = gql`
${fragments.object}
`;
const buildFilesQuery = (depth = 10) => {
const buildFilesQuery = (depth = 1) => {
const PLACE_HOLDER = 'PLACE_HOLDER';
let query = oneLine`
...ObjectParts
@ -126,14 +126,12 @@ const buildFilesQuery = (depth = 10) => {
return query;
};
const filesQuery = buildFilesQuery();
export const files = gql`
export const files = depth => gql`
query files($owner: String!, $name: String!, $expression: String!) {
repository(owner: $owner, name: $name) {
...RepositoryParts
object(expression: $expression) {
${filesQuery}
${buildFilesQuery(depth)}
}
}
}

8
packages/netlify-cms-backend-gitlab/src/API.js
View File

@ -187,10 +187,10 @@ export default class API {
// while the CMS defaults to sorting by filename _ascending_, at
// least in the current GitHub backend). This should eventually be
// refactored.
listFiles = async path => {
listFiles = async (path, recursive = false) => {
const firstPageCursor = await this.fetchCursor({
url: `${this.repoURL}/repository/tree`,
params: { path, ref: this.branch, recursive: true },
params: { path, ref: this.branch, recursive },
});
const lastPageLink = firstPageCursor.data.getIn(['links', 'last']);
const { entries, cursor } = await this.fetchCursorAndEntries(lastPageLink);
@ -209,12 +209,12 @@ export default class API {
};
};
listAllFiles = async path => {
listAllFiles = async (path, recursive = false) => {
const entries = [];
let { cursor, entries: initialEntries } = await this.fetchCursorAndEntries({
url: `${this.repoURL}/repository/tree`,
// Get the maximum number of entries per page
params: { path, ref: this.branch, per_page: 100 },
params: { path, ref: this.branch, per_page: 100, recursive },
});
entries.push(...initialEntries);
while (cursor && cursor.actions.has('next')) {

28
packages/netlify-cms-backend-gitlab/src/__tests__/gitlab.spec.js
View File

@ -484,6 +484,34 @@ describe('gitlab backend', () => {
});
});
describe('filterFile', () => {
it('should return true for nested file with matching depth', () => {
backend = resolveBackend(defaultConfig);
expect(
backend.implementation.filterFile(
'content/posts',
{ name: 'index.md', path: 'content/posts/dir1/dir2/index.md' },
'md',
3,
),
).toBe(true);
});
it('should return false for nested file with non matching depth', () => {
backend = resolveBackend(defaultConfig);
expect(
backend.implementation.filterFile(
'content/posts',
{ name: 'index.md', path: 'content/posts/dir1/dir2/index.md' },
'md',
2,
),
).toBe(false);
});
});
afterEach(() => {
nock.cleanAll();
authStore.logout();

23
packages/netlify-cms-backend-gitlab/src/implementation.js
View File

@ -1,7 +1,8 @@
import trimStart from 'lodash/trimStart';
import semaphore from 'semaphore';
import { trim } from 'lodash';
import { stripIndent } from 'common-tags';
import { CURSOR_COMPATIBILITY_SYMBOL, basename } from 'netlify-cms-lib-util';
import { CURSOR_COMPATIBILITY_SYMBOL, basename, getCollectionDepth } from 'netlify-cms-lib-util';
import AuthenticationPage from './AuthenticationPage';
import API from './API';
@ -78,9 +79,17 @@ export default class GitLab {
return Promise.resolve(this.token);
}
filterFile(folder, file, extension, depth) {
// gitlab paths include the root folder
const fileFolder = trim(file.path.split(folder)[1] || '/', '/');
return file.name.endsWith('.' + extension) && fileFolder.split('/').length <= depth;
}
entriesByFolder(collection, extension) {
return this.api.listFiles(collection.get('folder')).then(({ files, cursor }) =>
this.fetchFiles(files.filter(file => file.name.endsWith('.' + extension))).then(
const depth = getCollectionDepth(collection);
const folder = collection.get('folder');
return this.api.listFiles(folder, depth > 1).then(({ files, cursor }) =>
this.fetchFiles(files.filter(file => this.filterFile(folder, file, extension, depth))).then(
fetchedFiles => {
const returnedFiles = fetchedFiles;
returnedFiles[CURSOR_COMPATIBILITY_SYMBOL] = cursor;
@ -91,9 +100,13 @@ export default class GitLab {
}
allEntriesByFolder(collection, extension) {
const depth = getCollectionDepth(collection);
const folder = collection.get('folder');
return this.api
.listAllFiles(collection.get('folder'))
.then(files => this.fetchFiles(files.filter(file => file.name.endsWith('.' + extension))));
.listAllFiles(folder, depth > 1)
.then(files =>
this.fetchFiles(files.filter(file => this.filterFile(folder, file, extension, depth))),
);
}
entriesByFiles(collection) {

55
packages/netlify-cms-backend-test/src/__tests__/implementation.spec.js
View File

@ -1,4 +1,4 @@
import TestBackend from '../implementation';
import TestBackend, { getFolderEntries } from '../implementation';
describe('test backend implementation', () => {
beforeEach(() => {
@ -201,4 +201,57 @@ describe('test backend implementation', () => {
});
});
});
describe('getFolderEntries', () => {
it('should get files by depth', () => {
const tree = {
pages: {
'root-page.md': {
content: 'root page content',
},
dir1: {
'nested-page-1.md': {
content: 'nested page 1 content',
},
dir2: {
'nested-page-2.md': {
content: 'nested page 2 content',
},
},
},
},
};
expect(getFolderEntries(tree, 'pages', 'md', 1)).toEqual([
{
file: { path: 'pages/root-page.md' },
data: 'root page content',
},
]);
expect(getFolderEntries(tree, 'pages', 'md', 2)).toEqual([
{
file: { path: 'pages/dir1/nested-page-1.md' },
data: 'nested page 1 content',
},
{
file: { path: 'pages/root-page.md' },
data: 'root page content',
},
]);
expect(getFolderEntries(tree, 'pages', 'md', 3)).toEqual([
{
file: { path: 'pages/dir1/dir2/nested-page-2.md' },
data: 'nested page 2 content',
},
{
file: { path: 'pages/dir1/nested-page-1.md' },
data: 'nested page 1 content',
},
{
file: { path: 'pages/root-page.md' },
data: 'root page content',
},
]);
});
});
});

39
packages/netlify-cms-backend-test/src/implementation.js
View File

@ -5,6 +5,7 @@ import {
Cursor,
CURSOR_COMPATIBILITY_SYMBOL,
basename,
getCollectionDepth,
} from 'netlify-cms-lib-util';
import AuthenticationPage from './AuthenticationPage';
@ -35,14 +36,25 @@ const getCursor = (collection, extension, entries, index) => {
});
};
const getFolderEntries = (folder, extension) => {
return Object.keys(window.repoFiles[folder] || {})
.filter(path => path.endsWith(`.${extension}`))
.map(path => ({
file: { path: `${folder}/${path}` },
data: window.repoFiles[folder][path].content,
}))
.reverse();
export const getFolderEntries = (tree, folder, extension, depth, files = [], path = folder) => {
if (depth <= 0) {
return files;
}
Object.keys(tree[folder] || {}).forEach(key => {
if (key.endsWith(`.${extension}`)) {
const file = tree[folder][key];
files.unshift({
file: { path: `${path}/${key}` },
data: file.content,
});
} else {
const subTree = tree[folder];
return getFolderEntries(subTree, key, extension, depth - 1, files, `${path}/${key}`);
}
});
return files;
};
export default class TestBackend {
@ -89,7 +101,13 @@ export default class TestBackend {
}
})();
// TODO: stop assuming cursors are for collections
const allEntries = getFolderEntries(collection.get('folder'), extension);
const depth = getCollectionDepth(collection);
const allEntries = getFolderEntries(
window.repoFiles,
collection.get('folder'),
extension,
depth,
);
const entries = allEntries.slice(newIndex * pageSize, newIndex * pageSize + pageSize);
const newCursor = getCursor(collection, extension, allEntries, newIndex);
return Promise.resolve({ entries, cursor: newCursor });
@ -97,7 +115,8 @@ export default class TestBackend {
entriesByFolder(collection, extension) {
const folder = collection.get('folder');
const entries = folder ? getFolderEntries(folder, extension) : [];
const depth = getCollectionDepth(collection);
const entries = folder ? getFolderEntries(window.repoFiles, folder, extension, depth) : [];
const cursor = getCursor(collection, extension, entries, 0);
const ret = take(entries, pageSize);
ret[CURSOR_COMPATIBILITY_SYMBOL] = cursor;

13
packages/netlify-cms-lib-util/src/__tests__/backendUtil.spec.js
View File

@ -1,5 +1,6 @@
import { parseLinkHeader, getAllResponses } from '../backendUtil';
import { parseLinkHeader, getAllResponses, getCollectionDepth } from '../backendUtil';
import { oneLine } from 'common-tags';
import { Map } from 'immutable';
import nock from 'nock';
describe('parseLinkHeader', () => {
@ -69,3 +70,13 @@ describe('getAllResponses', () => {
expect(pages[2]).toHaveLength(10);
});
});
describe('getCollectionDepth', () => {
it('should return 1 for collection with no path', () => {
expect(getCollectionDepth(Map({}))).toBe(1);
});
it('should return 2 for collection with path of one nested folder', () => {
expect(getCollectionDepth(Map({ path: '{{year}}/{{slug}}' }))).toBe(2);
});
});

5
packages/netlify-cms-lib-util/src/backendUtil.js
View File

@ -77,3 +77,8 @@ export const getAllResponses = async (url, options = {}, linkHeaderRelName = 'ne
return pageResponses;
};
export const getCollectionDepth = collection => {
const depth = collection.get('path', '').split('/').length;
return depth;
};

3
packages/netlify-cms-lib-util/src/index.js
View File

@ -18,6 +18,7 @@ import {
parseLinkHeader,
parseResponse,
responseParser,
getCollectionDepth,
} from './backendUtil';
import loadScript from './loadScript';
import getBlobSHA from './getBlobSHA';
@ -46,6 +47,7 @@ export const NetlifyCmsLibUtil = {
responseParser,
loadScript,
getBlobSHA,
getCollectionDepth,
};
export {
APIError,
@ -73,4 +75,5 @@ export {
getBlobSHA,
asyncLock,
isAbsolutePath,
getCollectionDepth,
};