gcg/static-cms
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Feat: nested collections (#3716)

Browse Source
This commit is contained in:
Erez Rokah
2020-06-18 10:11:37 +03:00
committed by GitHub
parent b4c47caf59
commit af7bbbd9a9
89 changed files with 8269 additions and 5619 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
packages/netlify-cms-proxy-server/package.json
View File

@ -27,7 +27,8 @@
"dotenv": "^8.2.0",
"express": "^4.17.1",
"morgan": "^1.9.1",
"simple-git": "^2.0.0"
"simple-git": "^2.0.0",
"what-the-diff": "^0.6.0"
},
"devDependencies": {
"@types/cors": "^2.8.6",

87
packages/netlify-cms-proxy-server/src/middlewares/joi/index.spec.ts
View File

@ -5,8 +5,8 @@ import Joi from '@hapi/joi';
const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
const { error } = result;
expect(error).not.toBeNull();
expect(error.details).toHaveLength(1);
const message = error.details.map(({ message }) => message)[0];
expect(error!.details).toHaveLength(1);
const message = error!.details.map(({ message }) => message)[0];
expect(message).toBe(expectedMessage);
};
@ -26,7 +26,7 @@ describe('defaultSchema', () => {
assetFailure(
schema.validate({ action: 'unknown', params: {} }),
'"action" must be one of [info, entriesByFolder, entriesByFiles, getEntry, unpublishedEntries, unpublishedEntry, deleteUnpublishedEntry, persistEntry, updateUnpublishedEntryStatus, publishUnpublishedEntry, getMedia, getMediaFile, persistMedia, deleteFile, getDeployPreview]',
'"action" must be one of [info, entriesByFolder, entriesByFiles, getEntry, unpublishedEntries, unpublishedEntry, unpublishedEntryDataFile, unpublishedEntryMediaFile, deleteUnpublishedEntry, persistEntry, updateUnpublishedEntryStatus, publishUnpublishedEntry, getMedia, getMediaFile, persistMedia, deleteFile, getDeployPreview]',
);
});
@ -157,28 +157,13 @@ describe('defaultSchema', () => {
describe('unpublishedEntry', () => {
it('should fail on invalid params', () => {
const schema = defaultSchema();
assetFailure(
schema.validate({ action: 'unpublishedEntry', params: { ...defaultParams } }),
'"params.collection" is required',
);
assetFailure(
schema.validate({
action: 'unpublishedEntry',
params: { ...defaultParams, collection: 'collection' },
}),
'"params.slug" is required',
);
assetFailure(
schema.validate({
action: 'unpublishedEntry',
params: { ...defaultParams, collection: 'collection', slug: 1 },
}),
'"params.slug" must be a string',
schema.validate({ action: 'unpublishedEntry', params: {} }),
'"params.branch" is required',
);
});
it('should pass on valid params', () => {
it('should pass on valid collection and slug', () => {
const schema = defaultSchema();
const { error } = schema.validate({
action: 'unpublishedEntry',
@ -187,6 +172,66 @@ describe('defaultSchema', () => {
expect(error).toBeUndefined();
});
it('should pass on valid id', () => {
const schema = defaultSchema();
const { error } = schema.validate({
action: 'unpublishedEntry',
params: { ...defaultParams, id: 'id' },
});
expect(error).toBeUndefined();
});
});
['unpublishedEntryDataFile', 'unpublishedEntryMediaFile'].forEach(action => {
describe(action, () => {
it('should fail on invalid params', () => {
const schema = defaultSchema();
assetFailure(
schema.validate({ action, params: { ...defaultParams } }),
'"params.collection" is required',
);
assetFailure(
schema.validate({
action,
params: { ...defaultParams, collection: 'collection' },
}),
'"params.slug" is required',
);
assetFailure(
schema.validate({
action,
params: { ...defaultParams, collection: 'collection', slug: 'slug' },
}),
'"params.id" is required',
);
assetFailure(
schema.validate({
action,
params: { ...defaultParams, collection: 'collection', slug: 'slug', id: 'id' },
}),
'"params.path" is required',
);
});
it('should pass on valid params', () => {
const schema = defaultSchema();
const { error } = schema.validate({
action,
params: {
...defaultParams,
collection: 'collection',
slug: 'slug',
id: 'id',
path: 'path',
},
});
expect(error).toBeUndefined();
});
});
});
describe('deleteUnpublishedEntry', () => {

32
packages/netlify-cms-proxy-server/src/middlewares/joi/index.ts
View File

@ -8,6 +8,8 @@ const allowedActions = [
'getEntry',
'unpublishedEntries',
'unpublishedEntry',
'unpublishedEntryDataFile',
'unpublishedEntryMediaFile',
'deleteUnpublishedEntry',
'persistEntry',
'updateUnpublishedEntryStatus',
@ -75,10 +77,33 @@ export const defaultSchema = ({ path = requiredString } = {}) => {
},
{
is: 'unpublishedEntry',
then: defaultParams
.keys({
id: Joi.string().optional(),
collection: Joi.string().optional(),
slug: Joi.string().optional(),
})
.required(),
},
{
is: 'unpublishedEntryDataFile',
then: defaultParams
.keys({
collection,
slug,
id: requiredString,
path: requiredString,
})
.required(),
},
{
is: 'unpublishedEntryMediaFile',
then: defaultParams
.keys({
collection,
slug,
id: requiredString,
path: requiredString,
})
.required(),
},
@ -95,7 +120,12 @@ export const defaultSchema = ({ path = requiredString } = {}) => {
is: 'persistEntry',
then: defaultParams
.keys({
entry: Joi.object({ slug: requiredString, path, raw: requiredString }).required(),
entry: Joi.object({
slug: requiredString,
path,
raw: requiredString,
newPath: path.optional(),
}).required(),
assets: Joi.array()
.items(asset)
.required(),

4
packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts
View File

@ -5,8 +5,8 @@ import { getSchema } from '.';
const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
const { error } = result;
expect(error).not.toBeNull();
expect(error.details).toHaveLength(1);
const message = error.details.map(({ message }) => message)[0];
expect(error!.details).toHaveLength(1);
const message = error!.details.map(({ message }) => message)[0];
expect(message).toBe(expectedMessage);
};

5
packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts
View File

@ -12,7 +12,7 @@ import {
PersistMediaParams,
DeleteFileParams,
} from '../types';
import { listRepoFiles, deleteFile, writeFile } from '../utils/fs';
import { listRepoFiles, deleteFile, writeFile, move } from '../utils/fs';
import { entriesFromFiles, readMediaFile } from '../utils/entries';
type Options = {
@ -67,6 +67,9 @@ export const localFsMiddleware = ({ repoPath }: Options) => {
writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding)),
),
);
if (entry.newPath) {
await move(path.join(repoPath, entry.path), path.join(repoPath, entry.newPath));
}
res.json({ message: 'entry persisted' });
break;
}

4
packages/netlify-cms-proxy-server/src/middlewares/localGit/index.spec.ts
View File

@ -9,8 +9,8 @@ jest.mock('simple-git/promise');
const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
const { error } = result;
expect(error).not.toBeNull();
expect(error.details).toHaveLength(1);
const message = error.details.map(({ message }) => message)[0];
expect(error!.details).toHaveLength(1);
const message = error!.details.map(({ message }) => message)[0];
expect(message).toBe(expectedMessage);
};

177
packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts
View File

@ -2,15 +2,15 @@ import express from 'express';
import path from 'path';
import { promises as fs } from 'fs';
import {
parseContentKey,
branchFromContentKey,
generateContentKey,
contentKeyFromBranch,
CMS_BRANCH_PREFIX,
statusToLabel,
labelToStatus,
parseContentKey,
} from 'netlify-cms-lib-util/src/APIUtils';
import { parse } from 'what-the-diff';
import { defaultSchema, joi } from '../joi';
import {
EntriesByFolderParams,
@ -27,16 +27,19 @@ import {
UpdateUnpublishedEntryStatusParams,
Entry,
GetMediaFileParams,
DeleteEntryParams,
UnpublishedEntryDataFileParams,
UnpublishedEntryMediaFileParams,
} from '../types';
// eslint-disable-next-line import/default
import simpleGit from 'simple-git/promise';
import { pathTraversal } from '../joi/customValidators';
import { listRepoFiles, writeFile } from '../utils/fs';
import { listRepoFiles, writeFile, move } from '../utils/fs';
import { entriesFromFiles, readMediaFile } from '../utils/entries';
const commit = async (git: simpleGit.SimpleGit, commitMessage: string, files: string[]) => {
await git.add(files);
await git.commit(commitMessage, files, {
const commit = async (git: simpleGit.SimpleGit, commitMessage: string) => {
await git.add('.');
await git.commit(commitMessage, undefined, {
'--no-verify': true,
'--no-gpg-sign': true,
});
@ -62,69 +65,10 @@ const runOnBranch = async <T>(git: simpleGit.SimpleGit, branch: string, func: ()
const branchDescription = (branch: string) => `branch.${branch}.description`;
const getEntryDataFromDiff = async (git: simpleGit.SimpleGit, branch: string, diff: string[]) => {
const contentKey = contentKeyFromBranch(branch);
const { collection, slug } = parseContentKey(contentKey);
const path = diff.find(d => d.includes(slug)) as string;
const mediaFiles = diff.filter(d => d !== path);
const label = await git.raw(['config', branchDescription(branch)]);
const status = label && labelToStatus(label.trim());
return {
slug,
metaData: { branch, collection, objects: { entry: { path, mediaFiles } }, status },
};
};
type Options = {
repoPath: string;
};
const entriesFromDiffs = async (
git: simpleGit.SimpleGit,
branch: string,
repoPath: string,
cmsBranches: string[],
diffs: simpleGit.DiffResult[],
) => {
const entries = [];
for (let i = 0; i < diffs.length; i++) {
const cmsBranch = cmsBranches[i];
const diff = diffs[i];
const data = await getEntryDataFromDiff(
git,
cmsBranch,
diff.files.map(f => f.file),
);
const entryPath = data.metaData.objects.entry.path;
const [entry] = await runOnBranch(git, cmsBranch, () =>
entriesFromFiles(repoPath, [{ path: entryPath }]),
);
const rawDiff = await git.diff([branch, cmsBranch, '--', entryPath]);
entries.push({
...data,
...entry,
isModification: !rawDiff.includes('new file'),
});
}
return entries;
};
const getEntryMediaFiles = async (
git: simpleGit.SimpleGit,
repoPath: string,
cmsBranch: string,
files: string[],
) => {
const mediaFiles = await runOnBranch(git, cmsBranch, async () => {
const serializedFiles = await Promise.all(files.map(file => readMediaFile(repoPath, file)));
return serializedFiles;
});
return mediaFiles;
};
const commitEntry = async (
git: simpleGit.SimpleGit,
repoPath: string,
@ -138,8 +82,12 @@ const commitEntry = async (
await Promise.all(
assets.map(a => writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding))),
);
if (entry.newPath) {
await move(path.join(repoPath, entry.path), path.join(repoPath, entry.newPath));
}
// commits files
await commit(git, commitMessage, [entry.path, ...assets.map(a => a.path)]);
await commit(git, commitMessage);
};
const rebase = async (git: simpleGit.SimpleGit, branch: string) => {
@ -175,6 +123,25 @@ const isBranchExists = async (git: simpleGit.SimpleGit, branch: string) => {
return branchExists;
};
const getDiffs = async (git: simpleGit.SimpleGit, source: string, dest: string) => {
const rawDiff = await git.diff([source, dest]);
const diffs = parse(rawDiff).map(d => {
const oldPath = d.oldPath?.replace(/b\//, '') || '';
const newPath = d.newPath?.replace(/b\//, '') || '';
const path = newPath || (oldPath as string);
return {
oldPath,
newPath,
status: d.status,
newFile: d.status === 'added',
path,
id: path,
binary: d.binary || /.svg$/.test(path),
};
});
return diffs;
};
export const validateRepo = async ({ repoPath }: Options) => {
const git = simpleGit(repoPath).silent(false);
const isRepo = await git.checkIsRepo();
@ -247,36 +214,53 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
const cmsBranches = await git
.branchLocal()
.then(result => result.all.filter(b => b.startsWith(`${CMS_BRANCH_PREFIX}/`)));
const diffs = await Promise.all(
cmsBranches.map(cmsBranch => git.diffSummary([branch, cmsBranch])),
);
const entries = await entriesFromDiffs(git, branch, repoPath, cmsBranches, diffs);
res.json(entries);
res.json(cmsBranches.map(contentKeyFromBranch));
break;
}
case 'unpublishedEntry': {
const { collection, slug } = body.params as UnpublishedEntryParams;
const contentKey = generateContentKey(collection, slug);
let { id, collection, slug } = body.params as UnpublishedEntryParams;
if (id) {
({ collection, slug } = parseContentKey(id));
}
const contentKey = generateContentKey(collection as string, slug as string);
const cmsBranch = branchFromContentKey(contentKey);
const branchExists = await isBranchExists(git, cmsBranch);
if (branchExists) {
const diff = await git.diffSummary([branch, cmsBranch]);
const [entry] = await entriesFromDiffs(git, branch, repoPath, [cmsBranch], [diff]);
const mediaFiles = await getEntryMediaFiles(
git,
repoPath,
cmsBranch,
entry.metaData.objects.entry.mediaFiles,
);
res.json({ ...entry, mediaFiles });
const diffs = await getDiffs(git, branch, cmsBranch);
const label = await git.raw(['config', branchDescription(cmsBranch)]);
const status = label && labelToStatus(label.trim());
const unpublishedEntry = {
collection,
slug,
status,
diffs,
};
res.json(unpublishedEntry);
} else {
return res.status(404).json({ message: 'Not Found' });
}
break;
}
case 'unpublishedEntryDataFile': {
const { path, collection, slug } = body.params as UnpublishedEntryDataFileParams;
const contentKey = generateContentKey(collection as string, slug as string);
const cmsBranch = branchFromContentKey(contentKey);
const [entry] = await runOnBranch(git, cmsBranch, () =>
entriesFromFiles(repoPath, [{ path }]),
);
res.json({ data: entry.data });
break;
}
case 'unpublishedEntryMediaFile': {
const { path, collection, slug } = body.params as UnpublishedEntryMediaFileParams;
const contentKey = generateContentKey(collection as string, slug as string);
const cmsBranch = branchFromContentKey(contentKey);
const file = await runOnBranch(git, cmsBranch, () => readMediaFile(repoPath, path));
res.json(file);
break;
}
case 'deleteUnpublishedEntry': {
const { collection, slug } = body.params as UnpublishedEntryParams;
const { collection, slug } = body.params as DeleteEntryParams;
const contentKey = generateContentKey(collection, slug);
const cmsBranch = branchFromContentKey(contentKey);
const currentBranch = await getCurrentBranch(git);
@ -290,7 +274,7 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
case 'persistEntry': {
const { entry, assets, options } = body.params as PersistEntryParams;
if (!options.useWorkflow) {
runOnBranch(git, branch, async () => {
await runOnBranch(git, branch, async () => {
await commitEntry(git, repoPath, entry, assets, options.commitMessage);
});
} else {
@ -306,28 +290,19 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
await git.checkoutLocalBranch(cmsBranch);
}
await rebase(git, branch);
const diff = await git.diffSummary([branch, cmsBranch]);
const data = await getEntryDataFromDiff(
git,
branch,
diff.files.map(f => f.file),
);
const diffs = await getDiffs(git, branch, cmsBranch);
// delete media files that have been removed from the entry
const toDelete = data.metaData.objects.entry.mediaFiles.filter(
f => !assets.map(a => a.path).includes(f),
const toDelete = diffs.filter(
d => d.binary && !assets.map(a => a.path).includes(d.path),
);
await Promise.all(toDelete.map(f => fs.unlink(path.join(repoPath, f))));
await Promise.all(toDelete.map(f => fs.unlink(path.join(repoPath, f.path))));
await commitEntry(git, repoPath, entry, assets, options.commitMessage);
// add status for new entries
if (!data.metaData.status) {
if (!branchExists) {
const description = statusToLabel(options.status);
await git.addConfig(branchDescription(cmsBranch), description);
}
// set path for new entries
if (!data.metaData.objects.entry.path) {
data.metaData.objects.entry.path = entry.path;
}
});
}
res.json({ message: 'entry persisted' });
@ -382,7 +357,7 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
path.join(repoPath, asset.path),
Buffer.from(asset.content, asset.encoding),
);
await commit(git, commitMessage, [asset.path]);
await commit(git, commitMessage);
return readMediaFile(repoPath, asset.path);
});
res.json(file);
@ -395,7 +370,7 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
} = body.params as DeleteFileParams;
await runOnBranch(git, branch, async () => {
await fs.unlink(path.join(repoPath, filePath));
await commit(git, commitMessage, [filePath]);
await commit(git, commitMessage);
});
res.json({ message: `deleted file ${filePath}` });
break;

22
packages/netlify-cms-proxy-server/src/middlewares/types.ts
View File

@ -17,6 +17,26 @@ export type GetEntryParams = {
};
export type UnpublishedEntryParams = {
id?: string;
collection?: string;
slug?: string;
};
export type UnpublishedEntryDataFileParams = {
collection: string;
slug: string;
id: string;
path: string;
};
export type UnpublishedEntryMediaFileParams = {
collection: string;
slug: string;
id: string;
path: string;
};
export type DeleteEntryParams = {
collection: string;
slug: string;
};
@ -32,7 +52,7 @@ export type PublishUnpublishedEntryParams = {
slug: string;
};
export type Entry = { slug: string; path: string; raw: string };
export type Entry = { slug: string; path: string; raw: string; newPath?: string };
export type Asset = { path: string; content: string; encoding: 'base64' };

16
packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts
View File

@ -40,3 +40,19 @@ export const writeFile = async (filePath: string, content: Buffer | string) => {
export const deleteFile = async (repoPath: string, filePath: string) => {
await fs.unlink(path.join(repoPath, filePath));
};
const moveFile = async (from: string, to: string) => {
await fs.mkdir(path.dirname(to), { recursive: true });
await fs.rename(from, to);
};
export const move = async (from: string, to: string) => {
// move file
await moveFile(from, to);
// move children
const sourceDir = path.dirname(from);
const destDir = path.dirname(to);
const allFiles = await listFiles(sourceDir, '', 100);
await Promise.all(allFiles.map(file => moveFile(file, file.replace(sourceDir, destDir))));
};

5
packages/netlify-cms-proxy-server/src/what-the-diff.d.ts vendored Normal file
View File

@ -0,0 +1,5 @@
declare module 'what-the-diff' {
export const parse: (
rawDiff: string,
) => { oldPath?: string; newPath?: string; binary: boolean; status: string }[];
}