From 31dbd72273b723bb6bbb551641a6e4bcc1f0314b Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Mon, 10 Feb 2020 18:07:52 +0200 Subject: [PATCH] feat(proxy-server): add local fs middleware and make it the default (#3217) --- ...torial_workflow_spec_proxy_git_backend.js} | 5 +- ...> media_library_spec_proxy_git_backend.js} | 5 +- ... simple_workflow_spec_proxy_fs_backend.js} | 5 +- .../simple_workflow_spec_proxy_git_backend.js | 32 +++++ cypress/plugins/proxy.js | 20 ++- .../src/actions/__tests__/config.spec.js | 111 ++++++++++++--- .../netlify-cms-core/src/actions/config.js | 45 ++++-- .../netlify-cms-proxy-server/src/index.ts | 10 +- .../src/middlewares/joi/customValidators.ts | 17 +++ .../src/middlewares/localFs/index.spec.ts | 91 ++++++++++++ .../src/middlewares/localFs/index.ts | 129 ++++++++++++++++++ .../src/middlewares/localGit/index.ts | 102 ++------------ .../src/middlewares/utils/entries.ts | 40 ++++++ .../src/middlewares/utils/fs.ts | 42 ++++++ 14 files changed, 523 insertions(+), 131 deletions(-) rename cypress/integration/{editorial_workflow_spec_proxy_backend.js => editorial_workflow_spec_proxy_git_backend.js} (84%) rename cypress/integration/{media_library_spec_proxy_backend.js => media_library_spec_proxy_git_backend.js} (85%) rename cypress/integration/{simple_workflow_spec_proxy_backend.js => simple_workflow_spec_proxy_fs_backend.js} (78%) create mode 100644 cypress/integration/simple_workflow_spec_proxy_git_backend.js create mode 100644 packages/netlify-cms-proxy-server/src/middlewares/joi/customValidators.ts create mode 100644 packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts create mode 100644 packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts create mode 100644 packages/netlify-cms-proxy-server/src/middlewares/utils/entries.ts create mode 100644 packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts diff --git a/cypress/integration/editorial_workflow_spec_proxy_backend.js b/cypress/integration/editorial_workflow_spec_proxy_git_backend.js similarity index 84% rename from cypress/integration/editorial_workflow_spec_proxy_backend.js rename to cypress/integration/editorial_workflow_spec_proxy_git_backend.js index f7578aab..ce3277bb 100644 --- a/cypress/integration/editorial_workflow_spec_proxy_backend.js +++ b/cypress/integration/editorial_workflow_spec_proxy_git_backend.js @@ -3,12 +3,13 @@ import * as specUtils from './common/spec_utils'; import { entry1, entry2, entry3 } from './common/entries'; const backend = 'proxy'; +const mode = 'git'; -describe.skip('Proxy Backend Editorial Workflow', () => { +describe.skip(`Proxy Backend Editorial Workflow - '${mode}' mode`, () => { let taskResult = { data: {} }; before(() => { - specUtils.before(taskResult, { publish_mode: 'editorial_workflow' }, backend); + specUtils.before(taskResult, { publish_mode: 'editorial_workflow', mode }, backend); Cypress.config('defaultCommandTimeout', 5 * 1000); }); diff --git a/cypress/integration/media_library_spec_proxy_backend.js b/cypress/integration/media_library_spec_proxy_git_backend.js similarity index 85% rename from cypress/integration/media_library_spec_proxy_backend.js rename to cypress/integration/media_library_spec_proxy_git_backend.js index 6f2b6859..3538cb60 100644 --- a/cypress/integration/media_library_spec_proxy_backend.js +++ b/cypress/integration/media_library_spec_proxy_git_backend.js @@ -3,12 +3,13 @@ import * as specUtils from './common/spec_utils'; import { entry1 } from './common/entries'; const backend = 'proxy'; +const mode = 'git'; -describe('Proxy Backend Media Library - REST API', () => { +describe(`Proxy Backend Media Library - '${mode}' mode`, () => { let taskResult = { data: {} }; before(() => { - specUtils.before(taskResult, { publish_mode: 'editorial_workflow' }, backend); + specUtils.before(taskResult, { publish_mode: 'editorial_workflow', mode }, backend); Cypress.config('defaultCommandTimeout', 5 * 1000); }); diff --git a/cypress/integration/simple_workflow_spec_proxy_backend.js b/cypress/integration/simple_workflow_spec_proxy_fs_backend.js similarity index 78% rename from cypress/integration/simple_workflow_spec_proxy_backend.js rename to cypress/integration/simple_workflow_spec_proxy_fs_backend.js index 3638a402..9901fdf7 100644 --- a/cypress/integration/simple_workflow_spec_proxy_backend.js +++ b/cypress/integration/simple_workflow_spec_proxy_fs_backend.js @@ -3,12 +3,13 @@ import * as specUtils from './common/spec_utils'; import { entry1, entry2, entry3 } from './common/entries'; const backend = 'proxy'; +const mode = 'fs'; -describe('Proxy Backend Simple Workflow', () => { +describe(`Proxy Backend Simple Workflow - '${mode}' mode`, () => { let taskResult = { data: {} }; before(() => { - specUtils.before(taskResult, { publish_mode: 'simple' }, backend); + specUtils.before(taskResult, { publish_mode: 'simple', mode }, backend); Cypress.config('defaultCommandTimeout', 5 * 1000); }); diff --git a/cypress/integration/simple_workflow_spec_proxy_git_backend.js b/cypress/integration/simple_workflow_spec_proxy_git_backend.js new file mode 100644 index 00000000..af5d8de4 --- /dev/null +++ b/cypress/integration/simple_workflow_spec_proxy_git_backend.js @@ -0,0 +1,32 @@ +import fixture from './common/simple_workflow'; +import * as specUtils from './common/spec_utils'; +import { entry1, entry2, entry3 } from './common/entries'; + +const backend = 'proxy'; +const mode = 'git'; + +describe(`Proxy Backend Simple Workflow - '${mode}' mode`, () => { + let taskResult = { data: {} }; + + before(() => { + specUtils.before(taskResult, { publish_mode: 'simple', mode }, backend); + Cypress.config('defaultCommandTimeout', 5 * 1000); + }); + + after(() => { + specUtils.after(taskResult, backend); + }); + + beforeEach(() => { + specUtils.beforeEach(taskResult, backend); + }); + + afterEach(() => { + specUtils.afterEach(taskResult, backend); + }); + + fixture({ + entries: [entry1, entry2, entry3], + getUser: () => taskResult.data.user, + }); +}); diff --git a/cypress/plugins/proxy.js b/cypress/plugins/proxy.js index f48ef6ec..0179f437 100644 --- a/cypress/plugins/proxy.js +++ b/cypress/plugins/proxy.js @@ -19,13 +19,21 @@ const initRepo = async dir => { await git.commit('initial commit', readme, { '--no-verify': true, '--no-gpg-sign': true }); }; -const startServer = async repoDir => { +const startServer = async (repoDir, mode) => { const tsNode = path.join(__dirname, '..', '..', 'node_modules', '.bin', 'ts-node'); const serverDir = path.join(__dirname, '..', '..', 'packages', 'netlify-cms-proxy-server'); const distIndex = path.join(serverDir, 'dist', 'index.js'); const tsIndex = path.join(serverDir, 'src', 'index.ts'); - const env = { ...process.env, GIT_REPO_DIRECTORY: path.resolve(repoDir), PORT: 8082 }; + const port = 8082; + const env = { + ...process.env, + GIT_REPO_DIRECTORY: path.resolve(repoDir), + PORT: port, + MODE: mode, + }; + + console.log(`Starting proxy server on port '${port}' with mode ${mode}`); if (await fs.pathExists(distIndex)) { serverProcess = spawn('node', [distIndex], { env, cwd: serverDir }); } else { @@ -58,11 +66,13 @@ async function setupProxy(options) { const testRepoName = `proxy-test-repo-${Date.now()}-${postfix}`; const tempDir = path.join('.temp', testRepoName); + const { mode, ...rest } = options; + await updateConfig(config => { - merge(config, options); + merge(config, rest); }); - return { tempDir }; + return { tempDir, mode }; } async function teardownProxy(taskData) { @@ -76,7 +86,7 @@ async function teardownProxy(taskData) { async function setupProxyTest(taskData) { await initRepo(taskData.tempDir); - serverProcess = await startServer(taskData.tempDir); + serverProcess = await startServer(taskData.tempDir, taskData.mode); return null; } diff --git a/packages/netlify-cms-core/src/actions/__tests__/config.spec.js b/packages/netlify-cms-core/src/actions/__tests__/config.spec.js index f2a6173a..688726db 100644 --- a/packages/netlify-cms-core/src/actions/__tests__/config.spec.js +++ b/packages/netlify-cms-core/src/actions/__tests__/config.spec.js @@ -1,5 +1,5 @@ import { fromJS } from 'immutable'; -import { applyDefaults, detectProxyServer } from '../config'; +import { applyDefaults, detectProxyServer, handleLocalBackend } from '../config'; jest.spyOn(console, 'log').mockImplementation(() => {}); @@ -185,38 +185,46 @@ describe('config', () => { delete window.location; }); - it('should return undefined when not on localhost', async () => { + it('should return empty object when not on localhost', async () => { window.location = { hostname: 'www.netlify.com' }; global.fetch = jest.fn(); - await expect(detectProxyServer()).resolves.toBeUndefined(); + await expect(detectProxyServer()).resolves.toEqual({}); expect(global.fetch).toHaveBeenCalledTimes(0); }); - it('should return undefined when fetch returns an error', async () => { + it('should return empty object when fetch returns an error', async () => { window.location = { hostname: 'localhost' }; global.fetch = jest.fn().mockRejectedValue(new Error()); - await expect(detectProxyServer(true)).resolves.toBeUndefined(); + await expect(detectProxyServer(true)).resolves.toEqual({}); assetFetchCalled(); }); - it('should return undefined when fetch returns an invalid response', async () => { + it('should return empty object when fetch returns an invalid response', async () => { window.location = { hostname: 'localhost' }; global.fetch = jest .fn() .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: [] }) }); - await expect(detectProxyServer(true)).resolves.toBeUndefined(); + await expect(detectProxyServer(true)).resolves.toEqual({}); assetFetchCalled(); }); - it('should return proxyUrl when fetch returns a valid response', async () => { + it('should return result object when fetch returns a valid response', async () => { window.location = { hostname: 'localhost' }; - global.fetch = jest - .fn() - .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: 'test-repo' }) }); - await expect(detectProxyServer(true)).resolves.toBe('http://localhost:8081/api/v1'); + global.fetch = jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + repo: 'test-repo', + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }), + }); + await expect(detectProxyServer(true)).resolves.toEqual({ + proxyUrl: 'http://localhost:8081/api/v1', + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }); assetFetchCalled(); }); @@ -224,12 +232,83 @@ describe('config', () => { it('should use local_backend url', async () => { const url = 'http://localhost:8082/api/v1'; window.location = { hostname: 'localhost' }; - global.fetch = jest - .fn() - .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: 'test-repo' }) }); - await expect(detectProxyServer({ url })).resolves.toBe(url); + global.fetch = jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + repo: 'test-repo', + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }), + }); + await expect(detectProxyServer({ url })).resolves.toEqual({ + proxyUrl: url, + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }); assetFetchCalled(url); }); }); + + describe('handleLocalBackend', () => { + beforeEach(() => { + delete window.location; + }); + + it('should not replace backend config when proxy is not detected', async () => { + window.location = { hostname: 'localhost' }; + global.fetch = jest.fn().mockRejectedValue(new Error()); + + const config = fromJS({ local_backend: true, backend: { name: 'github' } }); + const actual = await handleLocalBackend(config); + + expect(actual).toEqual(config); + }); + + it('should replace backend config when proxy is detected', async () => { + window.location = { hostname: 'localhost' }; + global.fetch = jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + repo: 'test-repo', + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }), + }); + + const config = fromJS({ local_backend: true, backend: { name: 'github' } }); + const actual = await handleLocalBackend(config); + + expect(actual).toEqual( + fromJS({ + local_backend: true, + backend: { name: 'proxy', proxy_url: 'http://localhost:8081/api/v1' }, + }), + ); + }); + + it('should replace publish mode when not supported by proxy', async () => { + window.location = { hostname: 'localhost' }; + global.fetch = jest.fn().mockResolvedValue({ + json: jest.fn().mockResolvedValue({ + repo: 'test-repo', + publish_modes: ['simple'], + type: 'local_fs', + }), + }); + + const config = fromJS({ + local_backend: true, + publish_mode: 'editorial_workflow', + backend: { name: 'github' }, + }); + const actual = await handleLocalBackend(config); + + expect(actual).toEqual( + fromJS({ + local_backend: true, + publish_mode: 'simple', + backend: { name: 'proxy', proxy_url: 'http://localhost:8081/api/v1' }, + }), + ); + }); + }); }); diff --git a/packages/netlify-cms-core/src/actions/config.js b/packages/netlify-cms-core/src/actions/config.js index 93fd878f..678b448f 100644 --- a/packages/netlify-cms-core/src/actions/config.js +++ b/packages/netlify-cms-core/src/actions/config.js @@ -155,19 +155,48 @@ export async function detectProxyServer(localBackend) { } try { console.log(`Looking for Netlify CMS Proxy Server at '${proxyUrl}'`); - const { repo } = await fetch(`${proxyUrl}`, { + const { repo, publish_modes, type } = await fetch(`${proxyUrl}`, { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify({ action: 'info' }), }).then(res => res.json()); - if (typeof repo === 'string') { + if (typeof repo === 'string' && Array.isArray(publish_modes) && typeof type === 'string') { console.log(`Detected Netlify CMS Proxy Server at '${proxyUrl}' with repo: '${repo}'`); - return proxyUrl; + return { proxyUrl, publish_modes, type }; } } catch { console.log(`Netlify CMS Proxy Server not detected at '${proxyUrl}'`); } } + return {}; +} + +export async function handleLocalBackend(mergedConfig) { + if (mergedConfig.has('local_backend')) { + const { proxyUrl, publish_modes, type } = await detectProxyServer( + mergedConfig.toJS().local_backend, + ); + if (proxyUrl) { + mergedConfig = mergePreloadedConfig(mergedConfig, { + backend: { name: 'proxy', proxy_url: proxyUrl }, + }); + if ( + mergedConfig.has('publish_mode') && + !publish_modes.includes(mergedConfig.get('publish_mode')) + ) { + const newPublishMode = publish_modes[0]; + console.log( + `'${mergedConfig.get( + 'publish_mode', + )}' is not supported by '${type}' backend, switching to '${newPublishMode}'`, + ); + mergedConfig = mergePreloadedConfig(mergedConfig, { + publish_mode: newPublishMode, + }); + } + } + } + return mergedConfig; } export function loadConfig() { @@ -193,15 +222,7 @@ export function loadConfig() { validateConfig(mergedConfig.toJS()); - // detect running Netlify CMS proxy - if (mergedConfig.has('local_backend')) { - const proxyUrl = await detectProxyServer(mergedConfig.toJS().local_backend); - if (proxyUrl) { - mergedConfig = mergePreloadedConfig(mergedConfig, { - backend: { name: 'proxy', proxy_url: proxyUrl }, - }); - } - } + mergedConfig = await handleLocalBackend(mergedConfig); const config = applyDefaults(mergedConfig); diff --git a/packages/netlify-cms-proxy-server/src/index.ts b/packages/netlify-cms-proxy-server/src/index.ts index 401760a2..859a454c 100644 --- a/packages/netlify-cms-proxy-server/src/index.ts +++ b/packages/netlify-cms-proxy-server/src/index.ts @@ -3,6 +3,7 @@ import express from 'express'; import morgan from 'morgan'; import cors from 'cors'; import { registerMiddleware as registerLocalGit } from './middlewares/localGit'; +import { registerMiddleware as registerLocalFs } from './middlewares/localFs'; const app = express(); const port = process.env.PORT || 8081; @@ -13,7 +14,14 @@ const port = process.env.PORT || 8081; app.use(express.json()); try { - await registerLocalGit(app); + const mode = process.env.MODE || 'fs'; + if (mode === 'fs') { + registerLocalFs(app); + } else if (mode === 'git') { + registerLocalGit(app); + } else { + throw new Error(`Unknown proxy mode '${mode}'`); + } } catch (e) { console.error(e.message); process.exit(1); diff --git a/packages/netlify-cms-proxy-server/src/middlewares/joi/customValidators.ts b/packages/netlify-cms-proxy-server/src/middlewares/joi/customValidators.ts new file mode 100644 index 00000000..27db9aa4 --- /dev/null +++ b/packages/netlify-cms-proxy-server/src/middlewares/joi/customValidators.ts @@ -0,0 +1,17 @@ +import Joi from '@hapi/joi'; +import path from 'path'; + +export const pathTraversal = (repoPath: string) => + Joi.extend({ + type: 'path', + base: Joi.string().required(), + messages: { + 'path.invalid': '{{#label}} must resolve to a path under the configured repository', + }, + validate(value, helpers) { + const resolvedPath = path.join(repoPath, value); + if (!resolvedPath.startsWith(repoPath)) { + return { value, errors: helpers.error('path.invalid') }; + } + }, + }).path(); diff --git a/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts b/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts new file mode 100644 index 00000000..9abed3b3 --- /dev/null +++ b/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts @@ -0,0 +1,91 @@ +/* eslint-disable @typescript-eslint/no-var-requires */ +import Joi from '@hapi/joi'; +import { getSchema } from '.'; + +const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => { + const { error } = result; + expect(error).not.toBeNull(); + expect(error.details).toHaveLength(1); + const message = error.details.map(({ message }) => message)[0]; + expect(message).toBe(expectedMessage); +}; + +const defaultParams = { + branch: 'master', +}; + +describe('localFsMiddleware', () => { + beforeEach(() => { + jest.clearAllMocks(); + }); + + describe('getSchema', () => { + it('should throw on path traversal', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + + assetFailure( + schema.validate({ + action: 'getEntry', + params: { ...defaultParams, path: '../' }, + }), + '"params.path" must resolve to a path under the configured repository', + ); + }); + + it('should not throw on valid path', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + + const { error } = schema.validate({ + action: 'getEntry', + params: { ...defaultParams, path: 'src/content/posts/title.md' }, + }); + + expect(error).toBeUndefined(); + }); + + it('should throw on folder traversal', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + + assetFailure( + schema.validate({ + action: 'entriesByFolder', + params: { ...defaultParams, folder: '../', extension: 'md', depth: 1 }, + }), + '"params.folder" must resolve to a path under the configured repository', + ); + }); + + it('should not throw on valid folder', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + + const { error } = schema.validate({ + action: 'entriesByFolder', + params: { ...defaultParams, folder: 'src/posts', extension: 'md', depth: 1 }, + }); + + expect(error).toBeUndefined(); + }); + + it('should throw on media folder traversal', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + + assetFailure( + schema.validate({ + action: 'getMedia', + params: { ...defaultParams, mediaFolder: '../' }, + }), + '"params.mediaFolder" must resolve to a path under the configured repository', + ); + }); + + it('should not throw on valid folder', () => { + const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' }); + const { error } = schema.validate({ + action: 'getMedia', + params: { ...defaultParams, mediaFolder: 'static/images' }, + }); + + expect(error).toBeUndefined(); + }); + }); +}); diff --git a/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts b/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts new file mode 100644 index 00000000..16a8a8e5 --- /dev/null +++ b/packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts @@ -0,0 +1,129 @@ +import express from 'express'; +import path from 'path'; +import { defaultSchema, joi } from '../joi'; +import { pathTraversal } from '../joi/customValidators'; +import { + EntriesByFolderParams, + EntriesByFilesParams, + GetEntryParams, + PersistEntryParams, + GetMediaParams, + GetMediaFileParams, + PersistMediaParams, + DeleteFileParams, +} from '../types'; +import { listRepoFiles, deleteFile, writeFile } from '../utils/fs'; +import { entriesFromFiles, readMediaFile } from '../utils/entries'; + +type Options = { + repoPath: string; +}; + +export const localFsMiddleware = ({ repoPath }: Options) => { + return async function(req: express.Request, res: express.Response) { + try { + const { body } = req; + + switch (body.action) { + case 'action': { + res.json({ + repo: path.basename(repoPath), + // eslint-disable-next-line @typescript-eslint/camelcase + publish_modes: ['simple'], + type: 'local_fs', + }); + break; + } + case 'entriesByFolder': { + const payload = body.params as EntriesByFolderParams; + const { folder, extension, depth } = payload; + const entries = await listRepoFiles(repoPath, folder, extension, depth).then(files => + entriesFromFiles(repoPath, files), + ); + res.json(entries); + break; + } + case 'entriesByFiles': { + const payload = body.params as EntriesByFilesParams; + const entries = await entriesFromFiles( + repoPath, + payload.files.map(file => file.path), + ); + res.json(entries); + break; + } + case 'getEntry': { + const payload = body.params as GetEntryParams; + const [entry] = await entriesFromFiles(repoPath, [payload.path]); + res.json(entry); + break; + } + case 'persistEntry': { + const { entry, assets } = body.params as PersistEntryParams; + await writeFile(path.join(repoPath, entry.path), entry.raw); + // save assets + await Promise.all( + assets.map(a => + writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding)), + ), + ); + res.json({ message: 'entry persisted' }); + break; + } + case 'getMedia': { + const { mediaFolder } = body.params as GetMediaParams; + const files = await listRepoFiles(repoPath, mediaFolder, '', 1); + const mediaFiles = await Promise.all(files.map(file => readMediaFile(repoPath, file))); + res.json(mediaFiles); + break; + } + case 'getMediaFile': { + const { path } = body.params as GetMediaFileParams; + const mediaFile = await readMediaFile(repoPath, path); + res.json(mediaFile); + break; + } + case 'persistMedia': { + const { asset } = body.params as PersistMediaParams; + await writeFile( + path.join(repoPath, asset.path), + Buffer.from(asset.content, asset.encoding), + ); + const file = await readMediaFile(repoPath, asset.path); + res.json(file); + break; + } + case 'deleteFile': { + const { path: filePath } = body.params as DeleteFileParams; + await deleteFile(repoPath, filePath); + res.json({ message: `deleted file ${filePath}` }); + break; + } + case 'getDeployPreview': { + res.json(null); + break; + } + default: { + const message = `Unknown action ${body.action}`; + res.status(422).json({ error: message }); + break; + } + } + } catch (e) { + console.error(`Error handling ${JSON.stringify(req.body)}: ${e.message}`); + res.status(500).json({ error: 'Unknown error' }); + } + }; +}; + +export const getSchema = ({ repoPath }: Options) => { + const schema = defaultSchema({ path: pathTraversal(repoPath) }); + return schema; +}; + +export const registerMiddleware = async (app: express.Express) => { + const repoPath = path.resolve(process.env.GIT_REPO_DIRECTORY || process.cwd()); + app.post('/api/v1', joi(getSchema({ repoPath }))); + app.post('/api/v1', localFsMiddleware({ repoPath })); + console.log(`Netlify CMS File System Proxy Server configured with ${repoPath}`); +}; diff --git a/packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts b/packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts index b1bf3bbf..ad103105 100644 --- a/packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts +++ b/packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts @@ -1,8 +1,6 @@ import express from 'express'; import path from 'path'; -import crypto from 'crypto'; import { promises as fs } from 'fs'; -import Joi from '@hapi/joi'; import { parseContentKey, branchFromContentKey, @@ -32,18 +30,9 @@ import { } from '../types'; // eslint-disable-next-line import/default import simpleGit from 'simple-git/promise'; - -const sha256 = (buffer: Buffer) => { - return crypto - .createHash('sha256') - .update(buffer) - .digest('hex'); -}; - -const writeFile = async (filePath: string, content: Buffer | string) => { - await fs.mkdir(path.dirname(filePath), { recursive: true }); - await fs.writeFile(filePath, content); -}; +import { pathTraversal } from '../joi/customValidators'; +import { listRepoFiles, writeFile } from '../utils/fs'; +import { entriesFromFiles, readMediaFile } from '../utils/entries'; const commit = async (git: simpleGit.SimpleGit, commitMessage: string, files: string[]) => { await git.add(files); @@ -71,53 +60,6 @@ const runOnBranch = async (git: simpleGit.SimpleGit, branch: string, func: () } }; -const listFiles = async (dir: string, extension: string, depth: number): Promise => { - if (depth <= 0) { - return []; - } - - try { - const dirents = await fs.readdir(dir, { withFileTypes: true }); - const files = await Promise.all( - dirents.map(dirent => { - const res = path.join(dir, dirent.name); - return dirent.isDirectory() - ? listFiles(res, extension, depth - 1) - : [res].filter(f => f.endsWith(extension)); - }), - ); - return ([] as string[]).concat(...files); - } catch (e) { - return []; - } -}; - -const listRepoFiles = async ( - repoPath: string, - folder: string, - extension: string, - depth: number, -) => { - const files = await listFiles(path.join(repoPath, folder), extension, depth); - return files.map(f => f.substr(repoPath.length + 1)); -}; - -const entriesFromFiles = async (repoPath: string, files: string[]) => { - return Promise.all( - files.map(async file => { - try { - const content = await fs.readFile(path.join(repoPath, file)); - return { - data: content.toString(), - file: { path: file, id: sha256(content) }, - }; - } catch (e) { - return { data: null, file: { path: file, id: null } }; - } - }), - ); -}; - const branchDescription = (branch: string) => `branch.${branch}.description`; const getEntryDataFromDiff = async (git: simpleGit.SimpleGit, branch: string, diff: string[]) => { @@ -170,20 +112,6 @@ const entriesFromDiffs = async ( return entries; }; -const readMediaFile = async (repoPath: string, file: string) => { - const encoding = 'base64'; - const buffer = await fs.readFile(path.join(repoPath, file)); - const id = sha256(buffer); - - return { - id, - content: buffer.toString(encoding), - encoding, - path: file, - name: path.basename(file), - }; -}; - const getEntryMediaFiles = async ( git: simpleGit.SimpleGit, repoPath: string, @@ -256,20 +184,7 @@ export const validateRepo = async ({ repoPath }: Options) => { }; export const getSchema = ({ repoPath }: Options) => { - const custom = Joi.extend({ - type: 'path', - base: Joi.string().required(), - messages: { - 'path.invalid': '{{#label}} must resolve to a path under the configured repository', - }, - validate(value, helpers) { - const resolvedPath = path.join(repoPath, value); - if (!resolvedPath.startsWith(repoPath)) { - return { value, errors: helpers.error('path.invalid') }; - } - }, - }); - const schema = defaultSchema({ path: custom.path() }); + const schema = defaultSchema({ path: pathTraversal(repoPath) }); return schema; }; @@ -280,7 +195,12 @@ export const localGitMiddleware = ({ repoPath }: Options) => { try { const { body } = req; if (body.action === 'info') { - res.json({ repo: path.basename(repoPath) }); + res.json({ + repo: path.basename(repoPath), + // eslint-disable-next-line @typescript-eslint/camelcase + publish_modes: ['simple', 'editorial_workflow'], + type: 'local_git', + }); return; } const { branch } = body.params as DefaultParams; @@ -502,5 +422,5 @@ export const registerMiddleware = async (app: express.Express) => { await validateRepo({ repoPath }); app.post('/api/v1', joi(getSchema({ repoPath }))); app.post('/api/v1', localGitMiddleware({ repoPath })); - console.log(`Netlify CMS Proxy Server configured with ${repoPath}`); + console.log(`Netlify CMS Git Proxy Server configured with ${repoPath}`); }; diff --git a/packages/netlify-cms-proxy-server/src/middlewares/utils/entries.ts b/packages/netlify-cms-proxy-server/src/middlewares/utils/entries.ts new file mode 100644 index 00000000..1bddfe98 --- /dev/null +++ b/packages/netlify-cms-proxy-server/src/middlewares/utils/entries.ts @@ -0,0 +1,40 @@ +import crypto from 'crypto'; +import path from 'path'; +import { promises as fs } from 'fs'; + +const sha256 = (buffer: Buffer) => { + return crypto + .createHash('sha256') + .update(buffer) + .digest('hex'); +}; + +export const entriesFromFiles = async (repoPath: string, files: string[]) => { + return Promise.all( + files.map(async file => { + try { + const content = await fs.readFile(path.join(repoPath, file)); + return { + data: content.toString(), + file: { path: file, id: sha256(content) }, + }; + } catch (e) { + return { data: null, file: { path: file, id: null } }; + } + }), + ); +}; + +export const readMediaFile = async (repoPath: string, file: string) => { + const encoding = 'base64'; + const buffer = await fs.readFile(path.join(repoPath, file)); + const id = sha256(buffer); + + return { + id, + content: buffer.toString(encoding), + encoding, + path: file, + name: path.basename(file), + }; +}; diff --git a/packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts b/packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts new file mode 100644 index 00000000..16e32110 --- /dev/null +++ b/packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts @@ -0,0 +1,42 @@ +import path from 'path'; +import { promises as fs } from 'fs'; + +const listFiles = async (dir: string, extension: string, depth: number): Promise => { + if (depth <= 0) { + return []; + } + + try { + const dirents = await fs.readdir(dir, { withFileTypes: true }); + const files = await Promise.all( + dirents.map(dirent => { + const res = path.join(dir, dirent.name); + return dirent.isDirectory() + ? listFiles(res, extension, depth - 1) + : [res].filter(f => f.endsWith(extension)); + }), + ); + return ([] as string[]).concat(...files); + } catch (e) { + return []; + } +}; + +export const listRepoFiles = async ( + repoPath: string, + folder: string, + extension: string, + depth: number, +) => { + const files = await listFiles(path.join(repoPath, folder), extension, depth); + return files.map(f => f.substr(repoPath.length + 1)); +}; + +export const writeFile = async (filePath: string, content: Buffer | string) => { + await fs.mkdir(path.dirname(filePath), { recursive: true }); + await fs.writeFile(filePath, content); +}; + +export const deleteFile = async (repoPath: string, filePath: string) => { + await fs.unlink(path.join(repoPath, filePath)); +};