feat(proxy-server): add local fs middleware and make it the default (#3217)
This commit is contained in:
@ -3,6 +3,7 @@ import express from 'express';
|
||||
import morgan from 'morgan';
|
||||
import cors from 'cors';
|
||||
import { registerMiddleware as registerLocalGit } from './middlewares/localGit';
|
||||
import { registerMiddleware as registerLocalFs } from './middlewares/localFs';
|
||||
|
||||
const app = express();
|
||||
const port = process.env.PORT || 8081;
|
||||
@ -13,7 +14,14 @@ const port = process.env.PORT || 8081;
|
||||
app.use(express.json());
|
||||
|
||||
try {
|
||||
await registerLocalGit(app);
|
||||
const mode = process.env.MODE || 'fs';
|
||||
if (mode === 'fs') {
|
||||
registerLocalFs(app);
|
||||
} else if (mode === 'git') {
|
||||
registerLocalGit(app);
|
||||
} else {
|
||||
throw new Error(`Unknown proxy mode '${mode}'`);
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(e.message);
|
||||
process.exit(1);
|
||||
|
@ -0,0 +1,17 @@
|
||||
import Joi from '@hapi/joi';
|
||||
import path from 'path';
|
||||
|
||||
export const pathTraversal = (repoPath: string) =>
|
||||
Joi.extend({
|
||||
type: 'path',
|
||||
base: Joi.string().required(),
|
||||
messages: {
|
||||
'path.invalid': '{{#label}} must resolve to a path under the configured repository',
|
||||
},
|
||||
validate(value, helpers) {
|
||||
const resolvedPath = path.join(repoPath, value);
|
||||
if (!resolvedPath.startsWith(repoPath)) {
|
||||
return { value, errors: helpers.error('path.invalid') };
|
||||
}
|
||||
},
|
||||
}).path();
|
@ -0,0 +1,91 @@
|
||||
/* eslint-disable @typescript-eslint/no-var-requires */
|
||||
import Joi from '@hapi/joi';
|
||||
import { getSchema } from '.';
|
||||
|
||||
const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
|
||||
const { error } = result;
|
||||
expect(error).not.toBeNull();
|
||||
expect(error.details).toHaveLength(1);
|
||||
const message = error.details.map(({ message }) => message)[0];
|
||||
expect(message).toBe(expectedMessage);
|
||||
};
|
||||
|
||||
const defaultParams = {
|
||||
branch: 'master',
|
||||
};
|
||||
|
||||
describe('localFsMiddleware', () => {
|
||||
beforeEach(() => {
|
||||
jest.clearAllMocks();
|
||||
});
|
||||
|
||||
describe('getSchema', () => {
|
||||
it('should throw on path traversal', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
|
||||
assetFailure(
|
||||
schema.validate({
|
||||
action: 'getEntry',
|
||||
params: { ...defaultParams, path: '../' },
|
||||
}),
|
||||
'"params.path" must resolve to a path under the configured repository',
|
||||
);
|
||||
});
|
||||
|
||||
it('should not throw on valid path', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
|
||||
const { error } = schema.validate({
|
||||
action: 'getEntry',
|
||||
params: { ...defaultParams, path: 'src/content/posts/title.md' },
|
||||
});
|
||||
|
||||
expect(error).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should throw on folder traversal', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
|
||||
assetFailure(
|
||||
schema.validate({
|
||||
action: 'entriesByFolder',
|
||||
params: { ...defaultParams, folder: '../', extension: 'md', depth: 1 },
|
||||
}),
|
||||
'"params.folder" must resolve to a path under the configured repository',
|
||||
);
|
||||
});
|
||||
|
||||
it('should not throw on valid folder', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
|
||||
const { error } = schema.validate({
|
||||
action: 'entriesByFolder',
|
||||
params: { ...defaultParams, folder: 'src/posts', extension: 'md', depth: 1 },
|
||||
});
|
||||
|
||||
expect(error).toBeUndefined();
|
||||
});
|
||||
|
||||
it('should throw on media folder traversal', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
|
||||
assetFailure(
|
||||
schema.validate({
|
||||
action: 'getMedia',
|
||||
params: { ...defaultParams, mediaFolder: '../' },
|
||||
}),
|
||||
'"params.mediaFolder" must resolve to a path under the configured repository',
|
||||
);
|
||||
});
|
||||
|
||||
it('should not throw on valid folder', () => {
|
||||
const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
|
||||
const { error } = schema.validate({
|
||||
action: 'getMedia',
|
||||
params: { ...defaultParams, mediaFolder: 'static/images' },
|
||||
});
|
||||
|
||||
expect(error).toBeUndefined();
|
||||
});
|
||||
});
|
||||
});
|
@ -0,0 +1,129 @@
|
||||
import express from 'express';
|
||||
import path from 'path';
|
||||
import { defaultSchema, joi } from '../joi';
|
||||
import { pathTraversal } from '../joi/customValidators';
|
||||
import {
|
||||
EntriesByFolderParams,
|
||||
EntriesByFilesParams,
|
||||
GetEntryParams,
|
||||
PersistEntryParams,
|
||||
GetMediaParams,
|
||||
GetMediaFileParams,
|
||||
PersistMediaParams,
|
||||
DeleteFileParams,
|
||||
} from '../types';
|
||||
import { listRepoFiles, deleteFile, writeFile } from '../utils/fs';
|
||||
import { entriesFromFiles, readMediaFile } from '../utils/entries';
|
||||
|
||||
type Options = {
|
||||
repoPath: string;
|
||||
};
|
||||
|
||||
export const localFsMiddleware = ({ repoPath }: Options) => {
|
||||
return async function(req: express.Request, res: express.Response) {
|
||||
try {
|
||||
const { body } = req;
|
||||
|
||||
switch (body.action) {
|
||||
case 'action': {
|
||||
res.json({
|
||||
repo: path.basename(repoPath),
|
||||
// eslint-disable-next-line @typescript-eslint/camelcase
|
||||
publish_modes: ['simple'],
|
||||
type: 'local_fs',
|
||||
});
|
||||
break;
|
||||
}
|
||||
case 'entriesByFolder': {
|
||||
const payload = body.params as EntriesByFolderParams;
|
||||
const { folder, extension, depth } = payload;
|
||||
const entries = await listRepoFiles(repoPath, folder, extension, depth).then(files =>
|
||||
entriesFromFiles(repoPath, files),
|
||||
);
|
||||
res.json(entries);
|
||||
break;
|
||||
}
|
||||
case 'entriesByFiles': {
|
||||
const payload = body.params as EntriesByFilesParams;
|
||||
const entries = await entriesFromFiles(
|
||||
repoPath,
|
||||
payload.files.map(file => file.path),
|
||||
);
|
||||
res.json(entries);
|
||||
break;
|
||||
}
|
||||
case 'getEntry': {
|
||||
const payload = body.params as GetEntryParams;
|
||||
const [entry] = await entriesFromFiles(repoPath, [payload.path]);
|
||||
res.json(entry);
|
||||
break;
|
||||
}
|
||||
case 'persistEntry': {
|
||||
const { entry, assets } = body.params as PersistEntryParams;
|
||||
await writeFile(path.join(repoPath, entry.path), entry.raw);
|
||||
// save assets
|
||||
await Promise.all(
|
||||
assets.map(a =>
|
||||
writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding)),
|
||||
),
|
||||
);
|
||||
res.json({ message: 'entry persisted' });
|
||||
break;
|
||||
}
|
||||
case 'getMedia': {
|
||||
const { mediaFolder } = body.params as GetMediaParams;
|
||||
const files = await listRepoFiles(repoPath, mediaFolder, '', 1);
|
||||
const mediaFiles = await Promise.all(files.map(file => readMediaFile(repoPath, file)));
|
||||
res.json(mediaFiles);
|
||||
break;
|
||||
}
|
||||
case 'getMediaFile': {
|
||||
const { path } = body.params as GetMediaFileParams;
|
||||
const mediaFile = await readMediaFile(repoPath, path);
|
||||
res.json(mediaFile);
|
||||
break;
|
||||
}
|
||||
case 'persistMedia': {
|
||||
const { asset } = body.params as PersistMediaParams;
|
||||
await writeFile(
|
||||
path.join(repoPath, asset.path),
|
||||
Buffer.from(asset.content, asset.encoding),
|
||||
);
|
||||
const file = await readMediaFile(repoPath, asset.path);
|
||||
res.json(file);
|
||||
break;
|
||||
}
|
||||
case 'deleteFile': {
|
||||
const { path: filePath } = body.params as DeleteFileParams;
|
||||
await deleteFile(repoPath, filePath);
|
||||
res.json({ message: `deleted file ${filePath}` });
|
||||
break;
|
||||
}
|
||||
case 'getDeployPreview': {
|
||||
res.json(null);
|
||||
break;
|
||||
}
|
||||
default: {
|
||||
const message = `Unknown action ${body.action}`;
|
||||
res.status(422).json({ error: message });
|
||||
break;
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
console.error(`Error handling ${JSON.stringify(req.body)}: ${e.message}`);
|
||||
res.status(500).json({ error: 'Unknown error' });
|
||||
}
|
||||
};
|
||||
};
|
||||
|
||||
export const getSchema = ({ repoPath }: Options) => {
|
||||
const schema = defaultSchema({ path: pathTraversal(repoPath) });
|
||||
return schema;
|
||||
};
|
||||
|
||||
export const registerMiddleware = async (app: express.Express) => {
|
||||
const repoPath = path.resolve(process.env.GIT_REPO_DIRECTORY || process.cwd());
|
||||
app.post('/api/v1', joi(getSchema({ repoPath })));
|
||||
app.post('/api/v1', localFsMiddleware({ repoPath }));
|
||||
console.log(`Netlify CMS File System Proxy Server configured with ${repoPath}`);
|
||||
};
|
@ -1,8 +1,6 @@
|
||||
import express from 'express';
|
||||
import path from 'path';
|
||||
import crypto from 'crypto';
|
||||
import { promises as fs } from 'fs';
|
||||
import Joi from '@hapi/joi';
|
||||
import {
|
||||
parseContentKey,
|
||||
branchFromContentKey,
|
||||
@ -32,18 +30,9 @@ import {
|
||||
} from '../types';
|
||||
// eslint-disable-next-line import/default
|
||||
import simpleGit from 'simple-git/promise';
|
||||
|
||||
const sha256 = (buffer: Buffer) => {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(buffer)
|
||||
.digest('hex');
|
||||
};
|
||||
|
||||
const writeFile = async (filePath: string, content: Buffer | string) => {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, content);
|
||||
};
|
||||
import { pathTraversal } from '../joi/customValidators';
|
||||
import { listRepoFiles, writeFile } from '../utils/fs';
|
||||
import { entriesFromFiles, readMediaFile } from '../utils/entries';
|
||||
|
||||
const commit = async (git: simpleGit.SimpleGit, commitMessage: string, files: string[]) => {
|
||||
await git.add(files);
|
||||
@ -71,53 +60,6 @@ const runOnBranch = async <T>(git: simpleGit.SimpleGit, branch: string, func: ()
|
||||
}
|
||||
};
|
||||
|
||||
const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
|
||||
if (depth <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const dirents = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files = await Promise.all(
|
||||
dirents.map(dirent => {
|
||||
const res = path.join(dir, dirent.name);
|
||||
return dirent.isDirectory()
|
||||
? listFiles(res, extension, depth - 1)
|
||||
: [res].filter(f => f.endsWith(extension));
|
||||
}),
|
||||
);
|
||||
return ([] as string[]).concat(...files);
|
||||
} catch (e) {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
const listRepoFiles = async (
|
||||
repoPath: string,
|
||||
folder: string,
|
||||
extension: string,
|
||||
depth: number,
|
||||
) => {
|
||||
const files = await listFiles(path.join(repoPath, folder), extension, depth);
|
||||
return files.map(f => f.substr(repoPath.length + 1));
|
||||
};
|
||||
|
||||
const entriesFromFiles = async (repoPath: string, files: string[]) => {
|
||||
return Promise.all(
|
||||
files.map(async file => {
|
||||
try {
|
||||
const content = await fs.readFile(path.join(repoPath, file));
|
||||
return {
|
||||
data: content.toString(),
|
||||
file: { path: file, id: sha256(content) },
|
||||
};
|
||||
} catch (e) {
|
||||
return { data: null, file: { path: file, id: null } };
|
||||
}
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
const branchDescription = (branch: string) => `branch.${branch}.description`;
|
||||
|
||||
const getEntryDataFromDiff = async (git: simpleGit.SimpleGit, branch: string, diff: string[]) => {
|
||||
@ -170,20 +112,6 @@ const entriesFromDiffs = async (
|
||||
return entries;
|
||||
};
|
||||
|
||||
const readMediaFile = async (repoPath: string, file: string) => {
|
||||
const encoding = 'base64';
|
||||
const buffer = await fs.readFile(path.join(repoPath, file));
|
||||
const id = sha256(buffer);
|
||||
|
||||
return {
|
||||
id,
|
||||
content: buffer.toString(encoding),
|
||||
encoding,
|
||||
path: file,
|
||||
name: path.basename(file),
|
||||
};
|
||||
};
|
||||
|
||||
const getEntryMediaFiles = async (
|
||||
git: simpleGit.SimpleGit,
|
||||
repoPath: string,
|
||||
@ -256,20 +184,7 @@ export const validateRepo = async ({ repoPath }: Options) => {
|
||||
};
|
||||
|
||||
export const getSchema = ({ repoPath }: Options) => {
|
||||
const custom = Joi.extend({
|
||||
type: 'path',
|
||||
base: Joi.string().required(),
|
||||
messages: {
|
||||
'path.invalid': '{{#label}} must resolve to a path under the configured repository',
|
||||
},
|
||||
validate(value, helpers) {
|
||||
const resolvedPath = path.join(repoPath, value);
|
||||
if (!resolvedPath.startsWith(repoPath)) {
|
||||
return { value, errors: helpers.error('path.invalid') };
|
||||
}
|
||||
},
|
||||
});
|
||||
const schema = defaultSchema({ path: custom.path() });
|
||||
const schema = defaultSchema({ path: pathTraversal(repoPath) });
|
||||
return schema;
|
||||
};
|
||||
|
||||
@ -280,7 +195,12 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
|
||||
try {
|
||||
const { body } = req;
|
||||
if (body.action === 'info') {
|
||||
res.json({ repo: path.basename(repoPath) });
|
||||
res.json({
|
||||
repo: path.basename(repoPath),
|
||||
// eslint-disable-next-line @typescript-eslint/camelcase
|
||||
publish_modes: ['simple', 'editorial_workflow'],
|
||||
type: 'local_git',
|
||||
});
|
||||
return;
|
||||
}
|
||||
const { branch } = body.params as DefaultParams;
|
||||
@ -502,5 +422,5 @@ export const registerMiddleware = async (app: express.Express) => {
|
||||
await validateRepo({ repoPath });
|
||||
app.post('/api/v1', joi(getSchema({ repoPath })));
|
||||
app.post('/api/v1', localGitMiddleware({ repoPath }));
|
||||
console.log(`Netlify CMS Proxy Server configured with ${repoPath}`);
|
||||
console.log(`Netlify CMS Git Proxy Server configured with ${repoPath}`);
|
||||
};
|
||||
|
@ -0,0 +1,40 @@
|
||||
import crypto from 'crypto';
|
||||
import path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
const sha256 = (buffer: Buffer) => {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(buffer)
|
||||
.digest('hex');
|
||||
};
|
||||
|
||||
export const entriesFromFiles = async (repoPath: string, files: string[]) => {
|
||||
return Promise.all(
|
||||
files.map(async file => {
|
||||
try {
|
||||
const content = await fs.readFile(path.join(repoPath, file));
|
||||
return {
|
||||
data: content.toString(),
|
||||
file: { path: file, id: sha256(content) },
|
||||
};
|
||||
} catch (e) {
|
||||
return { data: null, file: { path: file, id: null } };
|
||||
}
|
||||
}),
|
||||
);
|
||||
};
|
||||
|
||||
export const readMediaFile = async (repoPath: string, file: string) => {
|
||||
const encoding = 'base64';
|
||||
const buffer = await fs.readFile(path.join(repoPath, file));
|
||||
const id = sha256(buffer);
|
||||
|
||||
return {
|
||||
id,
|
||||
content: buffer.toString(encoding),
|
||||
encoding,
|
||||
path: file,
|
||||
name: path.basename(file),
|
||||
};
|
||||
};
|
@ -0,0 +1,42 @@
|
||||
import path from 'path';
|
||||
import { promises as fs } from 'fs';
|
||||
|
||||
const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
|
||||
if (depth <= 0) {
|
||||
return [];
|
||||
}
|
||||
|
||||
try {
|
||||
const dirents = await fs.readdir(dir, { withFileTypes: true });
|
||||
const files = await Promise.all(
|
||||
dirents.map(dirent => {
|
||||
const res = path.join(dir, dirent.name);
|
||||
return dirent.isDirectory()
|
||||
? listFiles(res, extension, depth - 1)
|
||||
: [res].filter(f => f.endsWith(extension));
|
||||
}),
|
||||
);
|
||||
return ([] as string[]).concat(...files);
|
||||
} catch (e) {
|
||||
return [];
|
||||
}
|
||||
};
|
||||
|
||||
export const listRepoFiles = async (
|
||||
repoPath: string,
|
||||
folder: string,
|
||||
extension: string,
|
||||
depth: number,
|
||||
) => {
|
||||
const files = await listFiles(path.join(repoPath, folder), extension, depth);
|
||||
return files.map(f => f.substr(repoPath.length + 1));
|
||||
};
|
||||
|
||||
export const writeFile = async (filePath: string, content: Buffer | string) => {
|
||||
await fs.mkdir(path.dirname(filePath), { recursive: true });
|
||||
await fs.writeFile(filePath, content);
|
||||
};
|
||||
|
||||
export const deleteFile = async (repoPath: string, filePath: string) => {
|
||||
await fs.unlink(path.join(repoPath, filePath));
|
||||
};
|
Reference in New Issue
Block a user