feat(proxy-server): add local fs middleware and make it the default (#3217)

This commit is contained in:
Erez Rokah
2020-02-10 18:07:52 +02:00
committed by GitHub
parent 97bc0c8dc4
commit 31dbd72273
14 changed files with 523 additions and 131 deletions

View File

@ -3,6 +3,7 @@ import express from 'express';
import morgan from 'morgan';
import cors from 'cors';
import { registerMiddleware as registerLocalGit } from './middlewares/localGit';
import { registerMiddleware as registerLocalFs } from './middlewares/localFs';
const app = express();
const port = process.env.PORT || 8081;
@ -13,7 +14,14 @@ const port = process.env.PORT || 8081;
app.use(express.json());
try {
await registerLocalGit(app);
const mode = process.env.MODE || 'fs';
if (mode === 'fs') {
registerLocalFs(app);
} else if (mode === 'git') {
registerLocalGit(app);
} else {
throw new Error(`Unknown proxy mode '${mode}'`);
}
} catch (e) {
console.error(e.message);
process.exit(1);

View File

@ -0,0 +1,17 @@
import Joi from '@hapi/joi';
import path from 'path';
export const pathTraversal = (repoPath: string) =>
Joi.extend({
type: 'path',
base: Joi.string().required(),
messages: {
'path.invalid': '{{#label}} must resolve to a path under the configured repository',
},
validate(value, helpers) {
const resolvedPath = path.join(repoPath, value);
if (!resolvedPath.startsWith(repoPath)) {
return { value, errors: helpers.error('path.invalid') };
}
},
}).path();

View File

@ -0,0 +1,91 @@
/* eslint-disable @typescript-eslint/no-var-requires */
import Joi from '@hapi/joi';
import { getSchema } from '.';
const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
const { error } = result;
expect(error).not.toBeNull();
expect(error.details).toHaveLength(1);
const message = error.details.map(({ message }) => message)[0];
expect(message).toBe(expectedMessage);
};
const defaultParams = {
branch: 'master',
};
describe('localFsMiddleware', () => {
beforeEach(() => {
jest.clearAllMocks();
});
describe('getSchema', () => {
it('should throw on path traversal', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
assetFailure(
schema.validate({
action: 'getEntry',
params: { ...defaultParams, path: '../' },
}),
'"params.path" must resolve to a path under the configured repository',
);
});
it('should not throw on valid path', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
const { error } = schema.validate({
action: 'getEntry',
params: { ...defaultParams, path: 'src/content/posts/title.md' },
});
expect(error).toBeUndefined();
});
it('should throw on folder traversal', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
assetFailure(
schema.validate({
action: 'entriesByFolder',
params: { ...defaultParams, folder: '../', extension: 'md', depth: 1 },
}),
'"params.folder" must resolve to a path under the configured repository',
);
});
it('should not throw on valid folder', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
const { error } = schema.validate({
action: 'entriesByFolder',
params: { ...defaultParams, folder: 'src/posts', extension: 'md', depth: 1 },
});
expect(error).toBeUndefined();
});
it('should throw on media folder traversal', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
assetFailure(
schema.validate({
action: 'getMedia',
params: { ...defaultParams, mediaFolder: '../' },
}),
'"params.mediaFolder" must resolve to a path under the configured repository',
);
});
it('should not throw on valid folder', () => {
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
const { error } = schema.validate({
action: 'getMedia',
params: { ...defaultParams, mediaFolder: 'static/images' },
});
expect(error).toBeUndefined();
});
});
});

View File

@ -0,0 +1,129 @@
import express from 'express';
import path from 'path';
import { defaultSchema, joi } from '../joi';
import { pathTraversal } from '../joi/customValidators';
import {
EntriesByFolderParams,
EntriesByFilesParams,
GetEntryParams,
PersistEntryParams,
GetMediaParams,
GetMediaFileParams,
PersistMediaParams,
DeleteFileParams,
} from '../types';
import { listRepoFiles, deleteFile, writeFile } from '../utils/fs';
import { entriesFromFiles, readMediaFile } from '../utils/entries';
type Options = {
repoPath: string;
};
export const localFsMiddleware = ({ repoPath }: Options) => {
return async function(req: express.Request, res: express.Response) {
try {
const { body } = req;
switch (body.action) {
case 'action': {
res.json({
repo: path.basename(repoPath),
// eslint-disable-next-line @typescript-eslint/camelcase
publish_modes: ['simple'],
type: 'local_fs',
});
break;
}
case 'entriesByFolder': {
const payload = body.params as EntriesByFolderParams;
const { folder, extension, depth } = payload;
const entries = await listRepoFiles(repoPath, folder, extension, depth).then(files =>
entriesFromFiles(repoPath, files),
);
res.json(entries);
break;
}
case 'entriesByFiles': {
const payload = body.params as EntriesByFilesParams;
const entries = await entriesFromFiles(
repoPath,
payload.files.map(file => file.path),
);
res.json(entries);
break;
}
case 'getEntry': {
const payload = body.params as GetEntryParams;
const [entry] = await entriesFromFiles(repoPath, [payload.path]);
res.json(entry);
break;
}
case 'persistEntry': {
const { entry, assets } = body.params as PersistEntryParams;
await writeFile(path.join(repoPath, entry.path), entry.raw);
// save assets
await Promise.all(
assets.map(a =>
writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding)),
),
);
res.json({ message: 'entry persisted' });
break;
}
case 'getMedia': {
const { mediaFolder } = body.params as GetMediaParams;
const files = await listRepoFiles(repoPath, mediaFolder, '', 1);
const mediaFiles = await Promise.all(files.map(file => readMediaFile(repoPath, file)));
res.json(mediaFiles);
break;
}
case 'getMediaFile': {
const { path } = body.params as GetMediaFileParams;
const mediaFile = await readMediaFile(repoPath, path);
res.json(mediaFile);
break;
}
case 'persistMedia': {
const { asset } = body.params as PersistMediaParams;
await writeFile(
path.join(repoPath, asset.path),
Buffer.from(asset.content, asset.encoding),
);
const file = await readMediaFile(repoPath, asset.path);
res.json(file);
break;
}
case 'deleteFile': {
const { path: filePath } = body.params as DeleteFileParams;
await deleteFile(repoPath, filePath);
res.json({ message: `deleted file ${filePath}` });
break;
}
case 'getDeployPreview': {
res.json(null);
break;
}
default: {
const message = `Unknown action ${body.action}`;
res.status(422).json({ error: message });
break;
}
}
} catch (e) {
console.error(`Error handling ${JSON.stringify(req.body)}: ${e.message}`);
res.status(500).json({ error: 'Unknown error' });
}
};
};
export const getSchema = ({ repoPath }: Options) => {
const schema = defaultSchema({ path: pathTraversal(repoPath) });
return schema;
};
export const registerMiddleware = async (app: express.Express) => {
const repoPath = path.resolve(process.env.GIT_REPO_DIRECTORY || process.cwd());
app.post('/api/v1', joi(getSchema({ repoPath })));
app.post('/api/v1', localFsMiddleware({ repoPath }));
console.log(`Netlify CMS File System Proxy Server configured with ${repoPath}`);
};

View File

@ -1,8 +1,6 @@
import express from 'express';
import path from 'path';
import crypto from 'crypto';
import { promises as fs } from 'fs';
import Joi from '@hapi/joi';
import {
parseContentKey,
branchFromContentKey,
@ -32,18 +30,9 @@ import {
} from '../types';
// eslint-disable-next-line import/default
import simpleGit from 'simple-git/promise';
const sha256 = (buffer: Buffer) => {
return crypto
.createHash('sha256')
.update(buffer)
.digest('hex');
};
const writeFile = async (filePath: string, content: Buffer | string) => {
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, content);
};
import { pathTraversal } from '../joi/customValidators';
import { listRepoFiles, writeFile } from '../utils/fs';
import { entriesFromFiles, readMediaFile } from '../utils/entries';
const commit = async (git: simpleGit.SimpleGit, commitMessage: string, files: string[]) => {
await git.add(files);
@ -71,53 +60,6 @@ const runOnBranch = async <T>(git: simpleGit.SimpleGit, branch: string, func: ()
}
};
const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
if (depth <= 0) {
return [];
}
try {
const dirents = await fs.readdir(dir, { withFileTypes: true });
const files = await Promise.all(
dirents.map(dirent => {
const res = path.join(dir, dirent.name);
return dirent.isDirectory()
? listFiles(res, extension, depth - 1)
: [res].filter(f => f.endsWith(extension));
}),
);
return ([] as string[]).concat(...files);
} catch (e) {
return [];
}
};
const listRepoFiles = async (
repoPath: string,
folder: string,
extension: string,
depth: number,
) => {
const files = await listFiles(path.join(repoPath, folder), extension, depth);
return files.map(f => f.substr(repoPath.length + 1));
};
const entriesFromFiles = async (repoPath: string, files: string[]) => {
return Promise.all(
files.map(async file => {
try {
const content = await fs.readFile(path.join(repoPath, file));
return {
data: content.toString(),
file: { path: file, id: sha256(content) },
};
} catch (e) {
return { data: null, file: { path: file, id: null } };
}
}),
);
};
const branchDescription = (branch: string) => `branch.${branch}.description`;
const getEntryDataFromDiff = async (git: simpleGit.SimpleGit, branch: string, diff: string[]) => {
@ -170,20 +112,6 @@ const entriesFromDiffs = async (
return entries;
};
const readMediaFile = async (repoPath: string, file: string) => {
const encoding = 'base64';
const buffer = await fs.readFile(path.join(repoPath, file));
const id = sha256(buffer);
return {
id,
content: buffer.toString(encoding),
encoding,
path: file,
name: path.basename(file),
};
};
const getEntryMediaFiles = async (
git: simpleGit.SimpleGit,
repoPath: string,
@ -256,20 +184,7 @@ export const validateRepo = async ({ repoPath }: Options) => {
};
export const getSchema = ({ repoPath }: Options) => {
const custom = Joi.extend({
type: 'path',
base: Joi.string().required(),
messages: {
'path.invalid': '{{#label}} must resolve to a path under the configured repository',
},
validate(value, helpers) {
const resolvedPath = path.join(repoPath, value);
if (!resolvedPath.startsWith(repoPath)) {
return { value, errors: helpers.error('path.invalid') };
}
},
});
const schema = defaultSchema({ path: custom.path() });
const schema = defaultSchema({ path: pathTraversal(repoPath) });
return schema;
};
@ -280,7 +195,12 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
try {
const { body } = req;
if (body.action === 'info') {
res.json({ repo: path.basename(repoPath) });
res.json({
repo: path.basename(repoPath),
// eslint-disable-next-line @typescript-eslint/camelcase
publish_modes: ['simple', 'editorial_workflow'],
type: 'local_git',
});
return;
}
const { branch } = body.params as DefaultParams;
@ -502,5 +422,5 @@ export const registerMiddleware = async (app: express.Express) => {
await validateRepo({ repoPath });
app.post('/api/v1', joi(getSchema({ repoPath })));
app.post('/api/v1', localGitMiddleware({ repoPath }));
console.log(`Netlify CMS Proxy Server configured with ${repoPath}`);
console.log(`Netlify CMS Git Proxy Server configured with ${repoPath}`);
};

View File

@ -0,0 +1,40 @@
import crypto from 'crypto';
import path from 'path';
import { promises as fs } from 'fs';
const sha256 = (buffer: Buffer) => {
return crypto
.createHash('sha256')
.update(buffer)
.digest('hex');
};
export const entriesFromFiles = async (repoPath: string, files: string[]) => {
return Promise.all(
files.map(async file => {
try {
const content = await fs.readFile(path.join(repoPath, file));
return {
data: content.toString(),
file: { path: file, id: sha256(content) },
};
} catch (e) {
return { data: null, file: { path: file, id: null } };
}
}),
);
};
export const readMediaFile = async (repoPath: string, file: string) => {
const encoding = 'base64';
const buffer = await fs.readFile(path.join(repoPath, file));
const id = sha256(buffer);
return {
id,
content: buffer.toString(encoding),
encoding,
path: file,
name: path.basename(file),
};
};

View File

@ -0,0 +1,42 @@
import path from 'path';
import { promises as fs } from 'fs';
const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
if (depth <= 0) {
return [];
}
try {
const dirents = await fs.readdir(dir, { withFileTypes: true });
const files = await Promise.all(
dirents.map(dirent => {
const res = path.join(dir, dirent.name);
return dirent.isDirectory()
? listFiles(res, extension, depth - 1)
: [res].filter(f => f.endsWith(extension));
}),
);
return ([] as string[]).concat(...files);
} catch (e) {
return [];
}
};
export const listRepoFiles = async (
repoPath: string,
folder: string,
extension: string,
depth: number,
) => {
const files = await listFiles(path.join(repoPath, folder), extension, depth);
return files.map(f => f.substr(repoPath.length + 1));
};
export const writeFile = async (filePath: string, content: Buffer | string) => {
await fs.mkdir(path.dirname(filePath), { recursive: true });
await fs.writeFile(filePath, content);
};
export const deleteFile = async (repoPath: string, filePath: string) => {
await fs.unlink(path.join(repoPath, filePath));
};