feat(proxy-server): add local fs middleware and make it the default (#3217)

packages/netlify-cms-core/src/actions/__tests__/config.spec.js

@@ -1,5 +1,5 @@
 import { fromJS } from 'immutable';
-import { applyDefaults, detectProxyServer } from '../config';
+import { applyDefaults, detectProxyServer, handleLocalBackend } from '../config';
 
 jest.spyOn(console, 'log').mockImplementation(() => {});
 
@@ -185,38 +185,46 @@ describe('config', () => {
       delete window.location;
     });
 
-    it('should return undefined when not on localhost', async () => {
+    it('should return empty object when not on localhost', async () => {
       window.location = { hostname: 'www.netlify.com' };
       global.fetch = jest.fn();
-      await expect(detectProxyServer()).resolves.toBeUndefined();
+      await expect(detectProxyServer()).resolves.toEqual({});
 
       expect(global.fetch).toHaveBeenCalledTimes(0);
     });
 
-    it('should return undefined when fetch returns an error', async () => {
+    it('should return empty object when fetch returns an error', async () => {
       window.location = { hostname: 'localhost' };
       global.fetch = jest.fn().mockRejectedValue(new Error());
-      await expect(detectProxyServer(true)).resolves.toBeUndefined();
+      await expect(detectProxyServer(true)).resolves.toEqual({});
 
       assetFetchCalled();
     });
 
-    it('should return undefined when fetch returns an invalid response', async () => {
+    it('should return empty object when fetch returns an invalid response', async () => {
       window.location = { hostname: 'localhost' };
       global.fetch = jest
         .fn()
         .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: [] }) });
-      await expect(detectProxyServer(true)).resolves.toBeUndefined();
+      await expect(detectProxyServer(true)).resolves.toEqual({});
 
       assetFetchCalled();
     });
 
-    it('should return proxyUrl when fetch returns a valid response', async () => {
+    it('should return result object when fetch returns a valid response', async () => {
       window.location = { hostname: 'localhost' };
-      global.fetch = jest
-        .fn()
-        .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: 'test-repo' }) });
-      await expect(detectProxyServer(true)).resolves.toBe('http://localhost:8081/api/v1');
+      global.fetch = jest.fn().mockResolvedValue({
+        json: jest.fn().mockResolvedValue({
+          repo: 'test-repo',
+          publish_modes: ['simple', 'editorial_workflow'],
+          type: 'local_git',
+        }),
+      });
+      await expect(detectProxyServer(true)).resolves.toEqual({
+        proxyUrl: 'http://localhost:8081/api/v1',
+        publish_modes: ['simple', 'editorial_workflow'],
+        type: 'local_git',
+      });
 
       assetFetchCalled();
     });
@@ -224,12 +232,83 @@ describe('config', () => {
     it('should use local_backend url', async () => {
       const url = 'http://localhost:8082/api/v1';
       window.location = { hostname: 'localhost' };
-      global.fetch = jest
-        .fn()
-        .mockResolvedValue({ json: jest.fn().mockResolvedValue({ repo: 'test-repo' }) });
-      await expect(detectProxyServer({ url })).resolves.toBe(url);
+      global.fetch = jest.fn().mockResolvedValue({
+        json: jest.fn().mockResolvedValue({
+          repo: 'test-repo',
+          publish_modes: ['simple', 'editorial_workflow'],
+          type: 'local_git',
+        }),
+      });
+      await expect(detectProxyServer({ url })).resolves.toEqual({
+        proxyUrl: url,
+        publish_modes: ['simple', 'editorial_workflow'],
+        type: 'local_git',
+      });
 
       assetFetchCalled(url);
     });
   });
+
+  describe('handleLocalBackend', () => {
+    beforeEach(() => {
+      delete window.location;
+    });
+
+    it('should not replace backend config when proxy is not detected', async () => {
+      window.location = { hostname: 'localhost' };
+      global.fetch = jest.fn().mockRejectedValue(new Error());
+
+      const config = fromJS({ local_backend: true, backend: { name: 'github' } });
+      const actual = await handleLocalBackend(config);
+
+      expect(actual).toEqual(config);
+    });
+
+    it('should replace backend config when proxy is detected', async () => {
+      window.location = { hostname: 'localhost' };
+      global.fetch = jest.fn().mockResolvedValue({
+        json: jest.fn().mockResolvedValue({
+          repo: 'test-repo',
+          publish_modes: ['simple', 'editorial_workflow'],
+          type: 'local_git',
+        }),
+      });
+
+      const config = fromJS({ local_backend: true, backend: { name: 'github' } });
+      const actual = await handleLocalBackend(config);
+
+      expect(actual).toEqual(
+        fromJS({
+          local_backend: true,
+          backend: { name: 'proxy', proxy_url: 'http://localhost:8081/api/v1' },
+        }),
+      );
+    });
+
+    it('should replace publish mode when not supported by proxy', async () => {
+      window.location = { hostname: 'localhost' };
+      global.fetch = jest.fn().mockResolvedValue({
+        json: jest.fn().mockResolvedValue({
+          repo: 'test-repo',
+          publish_modes: ['simple'],
+          type: 'local_fs',
+        }),
+      });
+
+      const config = fromJS({
+        local_backend: true,
+        publish_mode: 'editorial_workflow',
+        backend: { name: 'github' },
+      });
+      const actual = await handleLocalBackend(config);
+
+      expect(actual).toEqual(
+        fromJS({
+          local_backend: true,
+          publish_mode: 'simple',
+          backend: { name: 'proxy', proxy_url: 'http://localhost:8081/api/v1' },
+        }),
+      );
+    });
+  });
 });

packages/netlify-cms-core/src/actions/config.js

@@ -155,19 +155,48 @@ export async function detectProxyServer(localBackend) {
     }
     try {
       console.log(`Looking for Netlify CMS Proxy Server at '${proxyUrl}'`);
-      const { repo } = await fetch(`${proxyUrl}`, {
+      const { repo, publish_modes, type } = await fetch(`${proxyUrl}`, {
         method: 'POST',
         headers: { 'Content-Type': 'application/json' },
         body: JSON.stringify({ action: 'info' }),
       }).then(res => res.json());
-      if (typeof repo === 'string') {
+      if (typeof repo === 'string' && Array.isArray(publish_modes) && typeof type === 'string') {
         console.log(`Detected Netlify CMS Proxy Server at '${proxyUrl}' with repo: '${repo}'`);
-        return proxyUrl;
+        return { proxyUrl, publish_modes, type };
       }
     } catch {
       console.log(`Netlify CMS Proxy Server not detected at '${proxyUrl}'`);
     }
   }
+  return {};
+}
+
+export async function handleLocalBackend(mergedConfig) {
+  if (mergedConfig.has('local_backend')) {
+    const { proxyUrl, publish_modes, type } = await detectProxyServer(
+      mergedConfig.toJS().local_backend,
+    );
+    if (proxyUrl) {
+      mergedConfig = mergePreloadedConfig(mergedConfig, {
+        backend: { name: 'proxy', proxy_url: proxyUrl },
+      });
+      if (
+        mergedConfig.has('publish_mode') &&
+        !publish_modes.includes(mergedConfig.get('publish_mode'))
+      ) {
+        const newPublishMode = publish_modes[0];
+        console.log(
+          `'${mergedConfig.get(
+            'publish_mode',
+          )}' is not supported by '${type}' backend, switching to '${newPublishMode}'`,
+        );
+        mergedConfig = mergePreloadedConfig(mergedConfig, {
+          publish_mode: newPublishMode,
+        });
+      }
+    }
+  }
+  return mergedConfig;
 }
 
 export function loadConfig() {
@@ -193,15 +222,7 @@ export function loadConfig() {
 
       validateConfig(mergedConfig.toJS());
 
-      // detect running Netlify CMS proxy
-      if (mergedConfig.has('local_backend')) {
-        const proxyUrl = await detectProxyServer(mergedConfig.toJS().local_backend);
-        if (proxyUrl) {
-          mergedConfig = mergePreloadedConfig(mergedConfig, {
-            backend: { name: 'proxy', proxy_url: proxyUrl },
-          });
-        }
-      }
+      mergedConfig = await handleLocalBackend(mergedConfig);
 
       const config = applyDefaults(mergedConfig);
 

packages/netlify-cms-proxy-server/src/index.ts

@@ -3,6 +3,7 @@ import express from 'express';
 import morgan from 'morgan';
 import cors from 'cors';
 import { registerMiddleware as registerLocalGit } from './middlewares/localGit';
+import { registerMiddleware as registerLocalFs } from './middlewares/localFs';
 
 const app = express();
 const port = process.env.PORT || 8081;
@@ -13,7 +14,14 @@ const port = process.env.PORT || 8081;
   app.use(express.json());
 
   try {
-    await registerLocalGit(app);
+    const mode = process.env.MODE || 'fs';
+    if (mode === 'fs') {
+      registerLocalFs(app);
+    } else if (mode === 'git') {
+      registerLocalGit(app);
+    } else {
+      throw new Error(`Unknown proxy mode '${mode}'`);
+    }
   } catch (e) {
     console.error(e.message);
     process.exit(1);

packages/netlify-cms-proxy-server/src/middlewares/joi/customValidators.ts

@@ -0,0 +1,17 @@
+import Joi from '@hapi/joi';
+import path from 'path';
+
+export const pathTraversal = (repoPath: string) =>
+  Joi.extend({
+    type: 'path',
+    base: Joi.string().required(),
+    messages: {
+      'path.invalid': '{{#label}} must resolve to a path under the configured repository',
+    },
+    validate(value, helpers) {
+      const resolvedPath = path.join(repoPath, value);
+      if (!resolvedPath.startsWith(repoPath)) {
+        return { value, errors: helpers.error('path.invalid') };
+      }
+    },
+  }).path();

packages/netlify-cms-proxy-server/src/middlewares/localFs/index.spec.ts

@@ -0,0 +1,91 @@
+/* eslint-disable @typescript-eslint/no-var-requires */
+import Joi from '@hapi/joi';
+import { getSchema } from '.';
+
+const assetFailure = (result: Joi.ValidationResult, expectedMessage: string) => {
+  const { error } = result;
+  expect(error).not.toBeNull();
+  expect(error.details).toHaveLength(1);
+  const message = error.details.map(({ message }) => message)[0];
+  expect(message).toBe(expectedMessage);
+};
+
+const defaultParams = {
+  branch: 'master',
+};
+
+describe('localFsMiddleware', () => {
+  beforeEach(() => {
+    jest.clearAllMocks();
+  });
+
+  describe('getSchema', () => {
+    it('should throw on path traversal', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+
+      assetFailure(
+        schema.validate({
+          action: 'getEntry',
+          params: { ...defaultParams, path: '../' },
+        }),
+        '"params.path" must resolve to a path under the configured repository',
+      );
+    });
+
+    it('should not throw on valid path', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+
+      const { error } = schema.validate({
+        action: 'getEntry',
+        params: { ...defaultParams, path: 'src/content/posts/title.md' },
+      });
+
+      expect(error).toBeUndefined();
+    });
+
+    it('should throw on folder traversal', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+
+      assetFailure(
+        schema.validate({
+          action: 'entriesByFolder',
+          params: { ...defaultParams, folder: '../', extension: 'md', depth: 1 },
+        }),
+        '"params.folder" must resolve to a path under the configured repository',
+      );
+    });
+
+    it('should not throw on valid folder', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+
+      const { error } = schema.validate({
+        action: 'entriesByFolder',
+        params: { ...defaultParams, folder: 'src/posts', extension: 'md', depth: 1 },
+      });
+
+      expect(error).toBeUndefined();
+    });
+
+    it('should throw on media folder traversal', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+
+      assetFailure(
+        schema.validate({
+          action: 'getMedia',
+          params: { ...defaultParams, mediaFolder: '../' },
+        }),
+        '"params.mediaFolder" must resolve to a path under the configured repository',
+      );
+    });
+
+    it('should not throw on valid folder', () => {
+      const schema = getSchema({ repoPath: '/Users/user/documents/code/repo' });
+      const { error } = schema.validate({
+        action: 'getMedia',
+        params: { ...defaultParams, mediaFolder: 'static/images' },
+      });
+
+      expect(error).toBeUndefined();
+    });
+  });
+});

packages/netlify-cms-proxy-server/src/middlewares/localFs/index.ts

@@ -0,0 +1,129 @@
+import express from 'express';
+import path from 'path';
+import { defaultSchema, joi } from '../joi';
+import { pathTraversal } from '../joi/customValidators';
+import {
+  EntriesByFolderParams,
+  EntriesByFilesParams,
+  GetEntryParams,
+  PersistEntryParams,
+  GetMediaParams,
+  GetMediaFileParams,
+  PersistMediaParams,
+  DeleteFileParams,
+} from '../types';
+import { listRepoFiles, deleteFile, writeFile } from '../utils/fs';
+import { entriesFromFiles, readMediaFile } from '../utils/entries';
+
+type Options = {
+  repoPath: string;
+};
+
+export const localFsMiddleware = ({ repoPath }: Options) => {
+  return async function(req: express.Request, res: express.Response) {
+    try {
+      const { body } = req;
+
+      switch (body.action) {
+        case 'action': {
+          res.json({
+            repo: path.basename(repoPath),
+            // eslint-disable-next-line @typescript-eslint/camelcase
+            publish_modes: ['simple'],
+            type: 'local_fs',
+          });
+          break;
+        }
+        case 'entriesByFolder': {
+          const payload = body.params as EntriesByFolderParams;
+          const { folder, extension, depth } = payload;
+          const entries = await listRepoFiles(repoPath, folder, extension, depth).then(files =>
+            entriesFromFiles(repoPath, files),
+          );
+          res.json(entries);
+          break;
+        }
+        case 'entriesByFiles': {
+          const payload = body.params as EntriesByFilesParams;
+          const entries = await entriesFromFiles(
+            repoPath,
+            payload.files.map(file => file.path),
+          );
+          res.json(entries);
+          break;
+        }
+        case 'getEntry': {
+          const payload = body.params as GetEntryParams;
+          const [entry] = await entriesFromFiles(repoPath, [payload.path]);
+          res.json(entry);
+          break;
+        }
+        case 'persistEntry': {
+          const { entry, assets } = body.params as PersistEntryParams;
+          await writeFile(path.join(repoPath, entry.path), entry.raw);
+          // save assets
+          await Promise.all(
+            assets.map(a =>
+              writeFile(path.join(repoPath, a.path), Buffer.from(a.content, a.encoding)),
+            ),
+          );
+          res.json({ message: 'entry persisted' });
+          break;
+        }
+        case 'getMedia': {
+          const { mediaFolder } = body.params as GetMediaParams;
+          const files = await listRepoFiles(repoPath, mediaFolder, '', 1);
+          const mediaFiles = await Promise.all(files.map(file => readMediaFile(repoPath, file)));
+          res.json(mediaFiles);
+          break;
+        }
+        case 'getMediaFile': {
+          const { path } = body.params as GetMediaFileParams;
+          const mediaFile = await readMediaFile(repoPath, path);
+          res.json(mediaFile);
+          break;
+        }
+        case 'persistMedia': {
+          const { asset } = body.params as PersistMediaParams;
+          await writeFile(
+            path.join(repoPath, asset.path),
+            Buffer.from(asset.content, asset.encoding),
+          );
+          const file = await readMediaFile(repoPath, asset.path);
+          res.json(file);
+          break;
+        }
+        case 'deleteFile': {
+          const { path: filePath } = body.params as DeleteFileParams;
+          await deleteFile(repoPath, filePath);
+          res.json({ message: `deleted file ${filePath}` });
+          break;
+        }
+        case 'getDeployPreview': {
+          res.json(null);
+          break;
+        }
+        default: {
+          const message = `Unknown action ${body.action}`;
+          res.status(422).json({ error: message });
+          break;
+        }
+      }
+    } catch (e) {
+      console.error(`Error handling ${JSON.stringify(req.body)}: ${e.message}`);
+      res.status(500).json({ error: 'Unknown error' });
+    }
+  };
+};
+
+export const getSchema = ({ repoPath }: Options) => {
+  const schema = defaultSchema({ path: pathTraversal(repoPath) });
+  return schema;
+};
+
+export const registerMiddleware = async (app: express.Express) => {
+  const repoPath = path.resolve(process.env.GIT_REPO_DIRECTORY || process.cwd());
+  app.post('/api/v1', joi(getSchema({ repoPath })));
+  app.post('/api/v1', localFsMiddleware({ repoPath }));
+  console.log(`Netlify CMS File System Proxy Server configured with ${repoPath}`);
+};

packages/netlify-cms-proxy-server/src/middlewares/localGit/index.ts

@@ -1,8 +1,6 @@
 import express from 'express';
 import path from 'path';
-import crypto from 'crypto';
 import { promises as fs } from 'fs';
-import Joi from '@hapi/joi';
 import {
   parseContentKey,
   branchFromContentKey,
@@ -32,18 +30,9 @@ import {
 } from '../types';
 // eslint-disable-next-line import/default
 import simpleGit from 'simple-git/promise';
-
-const sha256 = (buffer: Buffer) => {
-  return crypto
-    .createHash('sha256')
-    .update(buffer)
-    .digest('hex');
-};
-
-const writeFile = async (filePath: string, content: Buffer | string) => {
-  await fs.mkdir(path.dirname(filePath), { recursive: true });
-  await fs.writeFile(filePath, content);
-};
+import { pathTraversal } from '../joi/customValidators';
+import { listRepoFiles, writeFile } from '../utils/fs';
+import { entriesFromFiles, readMediaFile } from '../utils/entries';
 
 const commit = async (git: simpleGit.SimpleGit, commitMessage: string, files: string[]) => {
   await git.add(files);
@@ -71,53 +60,6 @@ const runOnBranch = async <T>(git: simpleGit.SimpleGit, branch: string, func: ()
   }
 };
 
-const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
-  if (depth <= 0) {
-    return [];
-  }
-
-  try {
-    const dirents = await fs.readdir(dir, { withFileTypes: true });
-    const files = await Promise.all(
-      dirents.map(dirent => {
-        const res = path.join(dir, dirent.name);
-        return dirent.isDirectory()
-          ? listFiles(res, extension, depth - 1)
-          : [res].filter(f => f.endsWith(extension));
-      }),
-    );
-    return ([] as string[]).concat(...files);
-  } catch (e) {
-    return [];
-  }
-};
-
-const listRepoFiles = async (
-  repoPath: string,
-  folder: string,
-  extension: string,
-  depth: number,
-) => {
-  const files = await listFiles(path.join(repoPath, folder), extension, depth);
-  return files.map(f => f.substr(repoPath.length + 1));
-};
-
-const entriesFromFiles = async (repoPath: string, files: string[]) => {
-  return Promise.all(
-    files.map(async file => {
-      try {
-        const content = await fs.readFile(path.join(repoPath, file));
-        return {
-          data: content.toString(),
-          file: { path: file, id: sha256(content) },
-        };
-      } catch (e) {
-        return { data: null, file: { path: file, id: null } };
-      }
-    }),
-  );
-};
-
 const branchDescription = (branch: string) => `branch.${branch}.description`;
 
 const getEntryDataFromDiff = async (git: simpleGit.SimpleGit, branch: string, diff: string[]) => {
@@ -170,20 +112,6 @@ const entriesFromDiffs = async (
   return entries;
 };
 
-const readMediaFile = async (repoPath: string, file: string) => {
-  const encoding = 'base64';
-  const buffer = await fs.readFile(path.join(repoPath, file));
-  const id = sha256(buffer);
-
-  return {
-    id,
-    content: buffer.toString(encoding),
-    encoding,
-    path: file,
-    name: path.basename(file),
-  };
-};
-
 const getEntryMediaFiles = async (
   git: simpleGit.SimpleGit,
   repoPath: string,
@@ -256,20 +184,7 @@ export const validateRepo = async ({ repoPath }: Options) => {
 };
 
 export const getSchema = ({ repoPath }: Options) => {
-  const custom = Joi.extend({
-    type: 'path',
-    base: Joi.string().required(),
-    messages: {
-      'path.invalid': '{{#label}} must resolve to a path under the configured repository',
-    },
-    validate(value, helpers) {
-      const resolvedPath = path.join(repoPath, value);
-      if (!resolvedPath.startsWith(repoPath)) {
-        return { value, errors: helpers.error('path.invalid') };
-      }
-    },
-  });
-  const schema = defaultSchema({ path: custom.path() });
+  const schema = defaultSchema({ path: pathTraversal(repoPath) });
   return schema;
 };
 
@@ -280,7 +195,12 @@ export const localGitMiddleware = ({ repoPath }: Options) => {
     try {
       const { body } = req;
       if (body.action === 'info') {
-        res.json({ repo: path.basename(repoPath) });
+        res.json({
+          repo: path.basename(repoPath),
+          // eslint-disable-next-line @typescript-eslint/camelcase
+          publish_modes: ['simple', 'editorial_workflow'],
+          type: 'local_git',
+        });
         return;
       }
       const { branch } = body.params as DefaultParams;
@@ -502,5 +422,5 @@ export const registerMiddleware = async (app: express.Express) => {
   await validateRepo({ repoPath });
   app.post('/api/v1', joi(getSchema({ repoPath })));
   app.post('/api/v1', localGitMiddleware({ repoPath }));
-  console.log(`Netlify CMS Proxy Server configured with ${repoPath}`);
+  console.log(`Netlify CMS Git Proxy Server configured with ${repoPath}`);
 };

packages/netlify-cms-proxy-server/src/middlewares/utils/entries.ts

@@ -0,0 +1,40 @@
+import crypto from 'crypto';
+import path from 'path';
+import { promises as fs } from 'fs';
+
+const sha256 = (buffer: Buffer) => {
+  return crypto
+    .createHash('sha256')
+    .update(buffer)
+    .digest('hex');
+};
+
+export const entriesFromFiles = async (repoPath: string, files: string[]) => {
+  return Promise.all(
+    files.map(async file => {
+      try {
+        const content = await fs.readFile(path.join(repoPath, file));
+        return {
+          data: content.toString(),
+          file: { path: file, id: sha256(content) },
+        };
+      } catch (e) {
+        return { data: null, file: { path: file, id: null } };
+      }
+    }),
+  );
+};
+
+export const readMediaFile = async (repoPath: string, file: string) => {
+  const encoding = 'base64';
+  const buffer = await fs.readFile(path.join(repoPath, file));
+  const id = sha256(buffer);
+
+  return {
+    id,
+    content: buffer.toString(encoding),
+    encoding,
+    path: file,
+    name: path.basename(file),
+  };
+};

packages/netlify-cms-proxy-server/src/middlewares/utils/fs.ts

@@ -0,0 +1,42 @@
+import path from 'path';
+import { promises as fs } from 'fs';
+
+const listFiles = async (dir: string, extension: string, depth: number): Promise<string[]> => {
+  if (depth <= 0) {
+    return [];
+  }
+
+  try {
+    const dirents = await fs.readdir(dir, { withFileTypes: true });
+    const files = await Promise.all(
+      dirents.map(dirent => {
+        const res = path.join(dir, dirent.name);
+        return dirent.isDirectory()
+          ? listFiles(res, extension, depth - 1)
+          : [res].filter(f => f.endsWith(extension));
+      }),
+    );
+    return ([] as string[]).concat(...files);
+  } catch (e) {
+    return [];
+  }
+};
+
+export const listRepoFiles = async (
+  repoPath: string,
+  folder: string,
+  extension: string,
+  depth: number,
+) => {
+  const files = await listFiles(path.join(repoPath, folder), extension, depth);
+  return files.map(f => f.substr(repoPath.length + 1));
+};
+
+export const writeFile = async (filePath: string, content: Buffer | string) => {
+  await fs.mkdir(path.dirname(filePath), { recursive: true });
+  await fs.writeFile(filePath, content);
+};
+
+export const deleteFile = async (repoPath: string, filePath: string) => {
+  await fs.unlink(path.join(repoPath, filePath));
+};