test(cypress-github-backend): optionally record tests and run using recorded data (#2776)

.github/workflows/nodejs.yml

@@ -4,7 +4,6 @@ on: [push, pull_request]
 
 jobs:
   build:
-
     runs-on: ubuntu-latest
 
     strategy:
@@ -12,20 +11,55 @@ jobs:
         node-version: [8.x, 10.x]
 
     steps:
-    - uses: actions/checkout@v1
-    - name: Use Node.js ${{ matrix.node-version }}
-      uses: actions/setup-node@v1
-      with:
-        node-version: ${{ matrix.node-version }}
-    - name: npm install, build, and test
-      run: |
-        node --version
-        npm --version
-        yarn --version
-        yarn
-        yarn bootstrap
-        yarn test:all:ci
-      env:
-        CI: true
-        CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
-        NODE_OPTIONS: --max-old-space-size=4096
+      - uses: actions/checkout@v1
+      - name: Use Node.js ${{ matrix.node-version }}
+        uses: actions/setup-node@v1
+        with:
+          node-version: ${{ matrix.node-version }}
+      - name: npm install, unit test and build
+        run: |
+          node --version
+          npm --version
+          yarn --version
+          yarn
+          yarn bootstrap
+          yarn test:ci
+          yarn build:demo
+        env:
+          CI: true
+          NODE_OPTIONS: --max-old-space-size=4096
+      - uses: actions/upload-artifact@master
+        with:
+          name: dev-test-website-node-${{ matrix.node-version }}
+          path: dev-test
+
+  e2e:
+    needs: build
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      matrix:
+        machine: [1, 2, 3, 4, 5]
+
+    steps:
+      - uses: actions/checkout@v1
+      - name: Use Node.js 10.x
+        uses: actions/setup-node@v1
+        with:
+          node-version: 10.x
+      - uses: actions/download-artifact@master
+        with:
+          name: dev-test-website-node-10.x
+          path: dev-test
+      - name: npm install and e2e test
+        run: |
+          node --version
+          npm --version
+          yarn --version
+          yarn
+          yarn test:e2e:run-ci
+        env:
+          CI: true
+          CYPRESS_RECORD_KEY: ${{ secrets.CYPRESS_RECORD_KEY }}
+          NODE_OPTIONS: --max-old-space-size=4096

cypress/Readme.md

@@ -0,0 +1,73 @@
+# Cypress Tests Guide
+
+## Introduction
+
+[Cypress](https://www.cypress.io/) is a JavaScript End to End Testing Framework that runs in the browser.
+
+Cypress tests run with a [local version](../dev-test) of the CMS.
+
+During the setup of a spec file, the relevant `index.html` and `config.yml` are copied from `dev-test/backends/<backend>` to `dev-test`.
+
+Tests for the `test` backend use mock data generated in `dev-test/backends/test/index.html`.
+
+Tests for the `github` backend use previously [recorded data](fixtures) and stub `fetch` [calls](support/commands.js#L82). See more about recording tests data [here](#recording-tests-data).
+
+## Run Tests Locally
+
+```bash
+yarn test:e2e # builds the demo site and runs Cypress in headless mode with mock data
+```
+
+## Debug Tests
+
+```bash
+yarn develop # starts a local dev server with the demo site
+yarn test:e2e:exec # runs Cypress in non-headless mode with mock data
+```
+
+## Recording Tests Data
+
+> Currently only relevant for `github` backend tests.
+
+When recording tests, access to the GitHub API is required, thus one must set up a `.env` file in the root project directory in the following format:
+
+```bash
+GITHUB_REPO_OWNER=owner
+GITHUB_REPO_NAME=repo
+GITHUB_REPO_TOKEN=tokenWithWritePermissions
+GITHUB_OPEN_AUTHORING_OWNER=forkOwner
+GITHUB_OPEN_AUTHORING_TOKEN=tokenWithWritePermissions
+```
+
+> The structure of the repo designated by `GITHUB_REPO_OWNER/GITHUB_REPO_NAME` should match the settings in [`config.yml`](../dev-test/backends/github/config.yml#L1)
+
+To start a recording run the following commands:
+
+```bash
+yarn develop # starts a local dev server with the demo site
+yarn mock:server:start # starts the recording proxy
+yarn test:e2e:record-fixtures:dev # runs Cypress in non-headless and pass data through the recording proxy
+yarn mock:server:stop # stops the recording proxy
+```
+
+> During the recorded process a clone of `GITHUB_REPO_NAME` will be created under `.temp` and reset between tests.
+
+Recordings are sanitized from any possible sensitive data and [transformed](plugins/github.js#L395) into an easier to process format.
+
+To avoid recording all the tests over and over again, a recommended process is to:
+
+1. Mark the specific test as `only` by changing `it("some test...` to `it.only("some test...` for the relevant test.
+2. Run the test in recording mode.
+3. Exit Cypress and stop the proxy.
+4. Run the test normally (with mock data) to verify the recording works.
+
+## Debugging Playback Failures
+
+Most common failures are:
+
+1. The [recorded data](utils/mock-server.js#L17) is not [transformed](plugins/github.js#L395) properly (e.g. [sanitization](plugins/github.js#L283) broke something).
+2. The [stubbed requests and responses](support/commands.js#L82) are not [matched](support/commands.js#L29) properly (e.g. timestamp changes in request body between recording and playback).
+
+Dumping all recorded data as is to a file [here](utils/mock-server.js#L24) and adding a `debugger;` statement [here](support/commands.js#L52) is useful to gain insights.
+
+Also comparing console log messages between recording and playback is very useful (ordering of requests, etc.)

cypress/integration/editorial_workflow_spec_github_backend_graphql.js

@@ -1,5 +1,5 @@
 import fixture from './github/editorial_workflow';
 
-describe.skip('Github Backend Editorial Workflow - GraphQL API', () => {
+describe('Github Backend Editorial Workflow - GraphQL API', () => {
   fixture({ use_graphql: true });
 });

cypress/integration/editorial_workflow_spec_github_backend_graphql_open_authoring.js

@@ -1,5 +1,5 @@
 import fixture from './github/open_authoring';
 
-describe.skip('Github Backend Editorial Workflow - GraphQL API - Open Authoring', () => {
+describe('Github Backend Editorial Workflow - GraphQL API - Open Authoring', () => {
   fixture({ use_graphql: true });
 });

cypress/integration/editorial_workflow_spec_github_backend_rest.js

@@ -1,5 +1,5 @@
 import fixture from './github/editorial_workflow';
 
-describe.skip('Github Backend Editorial Workflow - REST API', () => {
+describe('Github Backend Editorial Workflow - REST API', () => {
   fixture({ use_graphql: false });
 });

cypress/integration/editorial_workflow_spec_github_backend_rest_open_authoring.js

@@ -1,5 +1,5 @@
 import fixture from './github/open_authoring';
 
-describe.skip('Github Backend Editorial Workflow - REST API - Open Authoring', () => {
+describe('Github Backend Editorial Workflow - REST API - Open Authoring', () => {
   fixture({ use_graphql: false });
 });

cypress/integration/editorial_workflow_spec_test_backend.js

@@ -36,7 +36,7 @@ const entry3 = {
 
 describe('Test Backend Editorial Workflow', () => {
   after(() => {
-    cy.task('restoreDefaults');
+    cy.task('teardownBackend', { backend: 'test' });
   });
 
   before(() => {

cypress/integration/github/editorial_workflow.js

@@ -19,29 +19,25 @@ import {
 } from '../../utils/steps';
 import { workflowStatus, editorStatus } from '../../utils/constants';
 import { entry1, entry2, entry3 } from './entries';
+import * as specUtils from './spec_utils';
 
 export default function({ use_graphql }) {
   let taskResult = { data: {} };
 
-  const backend = 'github';
-
   before(() => {
-    Cypress.config('taskTimeout', 1200000);
-    Cypress.config('defaultCommandTimeout', 60000);
-    cy.task('setupBackend', { backend }).then(data => {
-      taskResult.data = data;
-    });
-
-    cy.task('updateBackendOptions', { backend, options: { use_graphql, open_authoring: false } });
+    specUtils.before(taskResult, { use_graphql, open_authoring: false });
   });
 
   after(() => {
-    cy.task('teardownBackend', { backend, ...taskResult.data });
-    cy.task('restoreDefaults');
+    specUtils.after(taskResult);
+  });
+
+  beforeEach(() => {
+    specUtils.beforeEach(taskResult);
   });
 
   afterEach(() => {
-    cy.task('teardownBackendTest', { backend, ...taskResult.data });
+    specUtils.afterEach(taskResult);
   });
 
   it('successfully loads', () => {

cypress/integration/github/open_authoring.js

@@ -10,28 +10,25 @@ import {
 } from '../../utils/steps';
 import { workflowStatus } from '../../utils/constants';
 import { entry1, entry2 } from './entries';
+import * as specUtils from './spec_utils';
 
 export default function({ use_graphql }) {
   let taskResult = { data: {} };
 
-  const backend = 'github';
-
   before(() => {
-    Cypress.config('taskTimeout', 1200000);
-    Cypress.config('defaultCommandTimeout', 60000);
-    cy.task('setupBackend', { backend }).then(data => {
-      taskResult.data = data;
-    });
-    cy.task('updateBackendOptions', { backend, options: { use_graphql, open_authoring: true } });
+    specUtils.before(taskResult, { use_graphql, open_authoring: true });
   });
 
   after(() => {
-    cy.task('teardownBackend', { backend, ...taskResult.data });
-    cy.task('restoreDefaults');
+    specUtils.after(taskResult);
+  });
+
+  beforeEach(() => {
+    specUtils.beforeEach(taskResult);
   });
 
   afterEach(() => {
-    cy.task('teardownBackendTest', { backend, ...taskResult.data });
+    specUtils.afterEach(taskResult);
   });
 
   it('successfully loads', () => {

cypress/integration/github/spec_utils.js

@@ -0,0 +1,51 @@
+const backend = 'github';
+
+export const before = (taskResult, options) => {
+  Cypress.config('taskTimeout', 5 * 60 * 1000);
+  cy.task('setupBackend', { backend, options }).then(data => {
+    taskResult.data = data;
+    Cypress.config('defaultCommandTimeout', data.mockResponses ? 5 * 1000 : 1 * 60 * 1000);
+  });
+};
+
+export const after = taskResult => {
+  cy.task('teardownBackend', {
+    backend,
+    ...taskResult.data,
+  });
+};
+
+export const beforeEach = taskResult => {
+  const spec = Cypress.mocha.getRunner().suite.ctx.currentTest.parent.title;
+  const testName = Cypress.mocha.getRunner().suite.ctx.currentTest.title;
+  cy.task('setupBackendTest', {
+    backend,
+    ...taskResult.data,
+    spec,
+    testName,
+  });
+
+  if (taskResult.data.mockResponses) {
+    const fixture = `${spec}__${testName}.json`;
+    console.log('loading fixture:', fixture);
+    cy.stubFetch({ fixture });
+  }
+
+  return cy.clock(0, ['Date']);
+};
+
+export const afterEach = taskResult => {
+  const spec = Cypress.mocha.getRunner().suite.ctx.currentTest.parent.title;
+  const testName = Cypress.mocha.getRunner().suite.ctx.currentTest.title;
+
+  cy.task('teardownBackendTest', {
+    backend,
+    ...taskResult.data,
+    spec,
+    testName,
+  });
+
+  if (Cypress.mocha.getRunner().suite.ctx.currentTest.state === 'failed') {
+    Cypress.runner.stop();
+  }
+};

cypress/plugins/github.js

@@ -2,8 +2,32 @@ const Octokit = require('@octokit/rest');
 const fs = require('fs-extra');
 const path = require('path');
 const simpleGit = require('simple-git/promise');
+const { updateConfig } = require('../utils/config');
+const { escapeRegExp } = require('../utils/regexp');
+const { retrieveRecordedExpectations, resetMockServerState } = require('../utils/mock-server');
 
 const GIT_SSH_COMMAND = 'ssh -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no';
+const GIT_SSL_NO_VERIFY = true;
+
+const GITHUB_REPO_OWNER_SANITIZED_VALUE = 'owner';
+const GITHUB_REPO_NAME_SANITIZED_VALUE = 'repo';
+const GITHUB_REPO_TOKEN_SANITIZED_VALUE = 'fakeToken';
+const GITHUB_OPEN_AUTHORING_OWNER_SANITIZED_VALUE = 'forkOwner';
+const GITHUB_OPEN_AUTHORING_TOKEN_SANITIZED_VALUE = 'fakeForkToken';
+
+const FAKE_OWNER_USER = {
+  login: 'owner',
+  id: 1,
+  avatar_url: 'https://avatars1.githubusercontent.com/u/7892489?v=4',
+  name: 'owner',
+};
+
+const FAKE_FORK_OWNER_USER = {
+  login: 'forkOwner',
+  id: 2,
+  avatar_url: 'https://avatars1.githubusercontent.com/u/9919?s=200&v=4',
+  name: 'forkOwner',
+};
 
 function getGitHubClient(token) {
   const client = new Octokit({
@@ -47,13 +71,13 @@ async function prepareTestGitHubRepo() {
 
   const tempDir = path.join('.temp', testRepoName);
   await fs.remove(tempDir);
-  let git = simpleGit().env({ ...process.env, GIT_SSH_COMMAND });
+  let git = simpleGit().env({ ...process.env, GIT_SSH_COMMAND, GIT_SSL_NO_VERIFY });
 
   const repoUrl = `git@github.com:${owner}/${repo}.git`;
 
   console.log('Cloning repository', repoUrl);
   await git.clone(repoUrl, tempDir);
-  git = simpleGit(tempDir).env({ ...process.env, GIT_SSH_COMMAND });
+  git = simpleGit(tempDir).env({ ...process.env, GIT_SSH_COMMAND, GIT_SSL_NO_VERIFY });
 
   console.log('Pushing to new repository', testRepoName);
 
@@ -150,7 +174,7 @@ async function resetOriginRepo({ owner, repo, tempDir }) {
   );
 
   console.log('Resetting master');
-  const git = simpleGit(tempDir).env({ ...process.env, GIT_SSH_COMMAND });
+  const git = simpleGit(tempDir).env({ ...process.env, GIT_SSH_COMMAND, GIT_SSL_NO_VERIFY });
   await git.push(['--force', 'origin', 'master']);
   console.log('Done resetting origin repo:', `${owner}/repo`);
 }
@@ -184,10 +208,209 @@ async function resetRepositories({ owner, repo, tempDir }) {
   await resetForkedRepo({ repo });
 }
 
-module.exports = {
-  prepareTestGitHubRepo,
-  deleteRepositories,
-  getUser,
-  getForkUser,
-  resetRepositories,
+async function setupGitHub(options) {
+  if (process.env.RECORD_FIXTURES) {
+    console.log('Running tests in "record" mode - live data with be used!');
+    const [user, forkUser, repoData] = await Promise.all([
+      getUser(),
+      getForkUser(),
+      prepareTestGitHubRepo(),
+    ]);
+
+    const { use_graphql = false, open_authoring = false } = options;
+
+    await updateConfig(config => {
+      config.backend = {
+        ...config.backend,
+        use_graphql,
+        open_authoring,
+        repo: `${repoData.owner}/${repoData.repo}`,
+      };
+    });
+
+    return { ...repoData, user, forkUser, mockResponses: false };
+  } else {
+    console.log('Running tests in "playback" mode - local data with be used');
+
+    await updateConfig(config => {
+      config.backend = {
+        ...config.backend,
+        ...options,
+        repo: `${GITHUB_REPO_OWNER_SANITIZED_VALUE}/${GITHUB_REPO_NAME_SANITIZED_VALUE}`,
+      };
+    });
+
+    return {
+      owner: GITHUB_REPO_OWNER_SANITIZED_VALUE,
+      repo: GITHUB_REPO_NAME_SANITIZED_VALUE,
+      user: { ...FAKE_OWNER_USER, token: GITHUB_REPO_TOKEN_SANITIZED_VALUE, backendName: 'github' },
+      forkUser: {
+        ...FAKE_FORK_OWNER_USER,
+        token: GITHUB_OPEN_AUTHORING_TOKEN_SANITIZED_VALUE,
+        backendName: 'github',
+      },
+      mockResponses: true,
+    };
+  }
+}
+
+async function teardownGitHub(taskData) {
+  if (process.env.RECORD_FIXTURES) {
+    await deleteRepositories(taskData);
+  }
+
+  return null;
+}
+
+const getExpectationsFilename = taskData => {
+  const { spec, testName } = taskData;
+  const basename = `${spec}__${testName}`;
+  const fixtures = path.join(__dirname, '..', 'fixtures');
+  const filename = path.join(fixtures, `${basename}.json`);
+
+  return filename;
+};
+
+async function setupGitHubTest(taskData) {
+  if (process.env.RECORD_FIXTURES) {
+    await resetRepositories(taskData);
+    await resetMockServerState();
+  }
+
+  return null;
+}
+
+const sanitizeString = (
+  str,
+  { owner, repo, token, forkOwner, forkToken, ownerName, forkOwnerName },
+) => {
+  let replaced = str
+    .replace(new RegExp(escapeRegExp(forkOwner), 'g'), GITHUB_OPEN_AUTHORING_OWNER_SANITIZED_VALUE)
+    .replace(new RegExp(escapeRegExp(forkToken), 'g'), GITHUB_OPEN_AUTHORING_TOKEN_SANITIZED_VALUE)
+    .replace(new RegExp(escapeRegExp(owner), 'g'), GITHUB_REPO_OWNER_SANITIZED_VALUE)
+    .replace(new RegExp(escapeRegExp(repo), 'g'), GITHUB_REPO_NAME_SANITIZED_VALUE)
+    .replace(new RegExp(escapeRegExp(token), 'g'), GITHUB_REPO_TOKEN_SANITIZED_VALUE)
+    .replace(new RegExp('https://avatars.+?/u/.+?v=\\d', 'g'), `${FAKE_OWNER_USER.avatar_url}`);
+
+  if (ownerName) {
+    replaced = replaced.replace(new RegExp(escapeRegExp(ownerName), 'g'), FAKE_OWNER_USER.name);
+  }
+
+  if (forkOwnerName) {
+    replaced = replaced.replace(
+      new RegExp(escapeRegExp(forkOwnerName), 'g'),
+      FAKE_FORK_OWNER_USER.name,
+    );
+  }
+
+  return replaced;
+};
+
+const transformRecordedData = (expectation, toSanitize) => {
+  const { httpRequest, httpResponse } = expectation;
+
+  const responseHeaders = {};
+
+  Object.keys(httpResponse.headers).forEach(key => {
+    responseHeaders[key] = httpResponse.headers[key][0];
+  });
+
+  let responseBody = null;
+  if (httpResponse.body && httpResponse.body.string) {
+    responseBody = httpResponse.body.string;
+  }
+
+  // replace recorded user with fake one
+  if (
+    responseBody &&
+    httpRequest.path === '/user' &&
+    httpRequest.headers.Host.includes('api.github.com')
+  ) {
+    const parsed = JSON.parse(responseBody);
+    if (parsed.login === toSanitize.forkOwner) {
+      responseBody = JSON.stringify(FAKE_FORK_OWNER_USER);
+    } else {
+      responseBody = JSON.stringify(FAKE_OWNER_USER);
+    }
+  }
+
+  let queryString;
+  if (httpRequest.queryStringParameters) {
+    const { queryStringParameters } = httpRequest;
+
+    queryString = Object.keys(queryStringParameters)
+      .map(key => `${key}=${queryStringParameters[key]}`)
+      .join('&');
+  }
+
+  let body;
+  if (httpRequest.body && httpRequest.body.string) {
+    const bodyObject = JSON.parse(httpRequest.body.string);
+    if (bodyObject.encoding === 'base64') {
+      // sanitize encoded data
+      const decodedBody = Buffer.from(bodyObject.content, 'base64').toString();
+      bodyObject.content = Buffer.from(sanitizeString(decodedBody, toSanitize)).toString('base64');
+      body = JSON.stringify(bodyObject);
+    } else {
+      body = httpRequest.body.string;
+    }
+  }
+
+  const cypressRouteOptions = {
+    body,
+    method: httpRequest.method,
+    url: queryString ? `${httpRequest.path}?${queryString}` : httpRequest.path,
+    headers: responseHeaders,
+    response: responseBody,
+    status: httpResponse.statusCode,
+  };
+
+  return cypressRouteOptions;
+};
+
+async function teardownGitHubTest(taskData) {
+  if (process.env.RECORD_FIXTURES) {
+    await resetRepositories(taskData);
+
+    try {
+      const filename = getExpectationsFilename(taskData);
+
+      console.log('Persisting recorded data for test:', path.basename(filename));
+
+      const { owner, token, forkOwner, forkToken } = getEnvs();
+
+      const expectations = await retrieveRecordedExpectations();
+
+      const toSanitize = {
+        owner,
+        repo: taskData.repo,
+        token,
+        forkOwner,
+        forkToken,
+        ownerName: taskData.user.name,
+        forkOwnerName: taskData.forkUser.name,
+      };
+      // transform the mock proxy recorded requests into Cypress route format
+      const toPersist = expectations.map(expectation =>
+        transformRecordedData(expectation, toSanitize),
+      );
+
+      const toPersistString = sanitizeString(JSON.stringify(toPersist, null, 2), toSanitize);
+
+      await fs.writeFile(filename, toPersistString);
+    } catch (e) {
+      console.log(e);
+    }
+
+    await resetMockServerState();
+  }
+
+  return null;
+}
+
+module.exports = {
+  setupGitHub,
+  teardownGitHub,
+  setupGitHubTest,
+  teardownGitHubTest,
 };

cypress/plugins/index.js

@@ -11,105 +11,71 @@
 // This function is called when a project is opened or re-opened (e.g. due to
 // the project's config changing)
 require('dotenv').config();
-const fs = require('fs-extra');
-const path = require('path');
-const yaml = require('js-yaml');
-const {
-  prepareTestGitHubRepo,
-  deleteRepositories,
-  getUser,
-  getForkUser,
-  resetRepositories,
-} = require('./github');
-
-const devTestDirectory = path.join(__dirname, '..', '..', 'dev-test');
-const backendsDirectory = path.join(devTestDirectory, 'backends');
-
-async function copyBackendFiles(backend) {
-  for (let file of ['config.yml', 'index.html']) {
-    await fs.copyFile(
-      path.join(backendsDirectory, backend, file),
-      path.join(devTestDirectory, file),
-    );
-  }
-}
-
-async function updateConfig(configModifier) {
-  const configFile = path.join(devTestDirectory, 'config.yml');
-  const configContent = await fs.readFile(configFile);
-  const config = yaml.safeLoad(configContent);
-  await configModifier(config);
-  await fs.writeFileSync(configFile, yaml.safeDump(config));
-  return null;
-}
-
-async function setupGitHub() {
-  const [user, forkUser, repoData] = await Promise.all([
-    getUser(),
-    getForkUser(),
-    prepareTestGitHubRepo(),
-  ]);
-
-  await updateConfig(config => {
-    config.backend.repo = `${repoData.owner}/${repoData.repo}`;
-  });
-
-  return { ...repoData, user, forkUser };
-}
-
-async function teardownGitHub(taskData) {
-  await deleteRepositories(taskData);
-  return null;
-}
-
-async function teardownBackendTest(taskData) {
-  await resetRepositories(taskData);
-  return null;
-}
+const { setupGitHub, teardownGitHub, setupGitHubTest, teardownGitHubTest } = require('./github');
+const { copyBackendFiles } = require('../utils/config');
 
 module.exports = async on => {
   // `on` is used to hook into various events Cypress emits
   on('task', {
-    async setupBackend({ backend }) {
+    async setupBackend({ backend, options }) {
       console.log('Preparing environment for backend', backend);
       await copyBackendFiles(backend);
 
+      let result = null;
       if (backend === 'github') {
-        return await setupGitHub();
+        result = await setupGitHub(options);
       }
 
-      return null;
+      return result;
     },
     async teardownBackend(taskData) {
       const { backend } = taskData;
       console.log('Tearing down backend', backend);
+
       if (backend === 'github') {
-        return await teardownGitHub(taskData);
+        await teardownGitHub(taskData);
       }
 
-      return null;
-    },
-    async teardownBackendTest(taskData) {
-      const { backend } = taskData;
-      console.log('Tearing down single test for backend', backend);
-      if (backend === 'github') {
-        return await teardownBackendTest(taskData);
-      }
-    },
-    async updateBackendOptions({ backend, options }) {
-      console.log('Updating backend', backend, 'with options', options);
-      if (backend === 'github') {
-        return await updateConfig(config => {
-          config.backend = { ...config.backend, ...options };
-        });
-      }
-      return null;
-    },
-    async restoreDefaults() {
       console.log('Restoring defaults');
       await copyBackendFiles('test');
 
       return null;
     },
+    async setupBackendTest(taskData) {
+      const { backend, testName } = taskData;
+      console.log(`Setting up single test '${testName}' for backend`, backend);
+
+      if (backend === 'github') {
+        await setupGitHubTest(taskData);
+      }
+
+      return null;
+    },
+    async teardownBackendTest(taskData) {
+      const { backend, testName } = taskData;
+
+      console.log(`Tearing down single test '${testName}' for backend`, backend);
+
+      if (backend === 'github') {
+        await teardownGitHubTest(taskData);
+      }
+
+      return null;
+    },
+  });
+
+  // to allows usage of a mock proxy
+  on('before:browser:launch', (browser = {}, args) => {
+    if (browser.name === 'chrome') {
+      args.push('--ignore-certificate-errors');
+
+      return args;
+    }
+
+    if (browser.name === 'electron') {
+      args['ignore-certificate-errors'] = true;
+
+      return args;
+    }
   });
 };

cypress/support/commands.js

@@ -23,3 +23,66 @@
 //
 // -- This is will overwrite an existing command --
 // Cypress.Commands.overwrite("visit", (originalFn, url, options) => { ... })
+const { escapeRegExp } = require('../utils/regexp');
+const path = require('path');
+
+const matchRoute = (route, fetchArgs) => {
+  const url = fetchArgs[0];
+  const options = fetchArgs[1];
+
+  const method = options && options.method ? options.method : 'GET';
+  const body = options && options.body;
+
+  // use pattern matching for the timestamp parameter
+  const urlRegex = escapeRegExp(decodeURIComponent(route.url)).replace(
+    /ts=\d{1,15}/,
+    'ts=\\d{1,15}',
+  );
+
+  return (
+    method === route.method &&
+    body === route.body &&
+    decodeURIComponent(url).match(new RegExp(`${urlRegex}`))
+  );
+};
+
+const stubFetch = (win, routes) => {
+  const fetch = win.fetch;
+  cy.stub(win, 'fetch').callsFake((...args) => {
+    const routeIndex = routes.findIndex(r => matchRoute(r, args));
+    if (routeIndex >= 0) {
+      const route = routes.splice(routeIndex, 1)[0];
+      console.log(`matched ${args[0]} to ${route.url} ${route.method} ${route.status}`);
+
+      const response = {
+        status: route.status,
+        headers: new Headers(route.headers),
+        text: () => Promise.resolve(route.response),
+        json: () => Promise.resolve(JSON.parse(route.response)),
+        ok: route.status >= 200 && route.status <= 299,
+      };
+      return Promise.resolve(response);
+    } else if (args[0].includes('api.github.com')) {
+      console.warn(
+        `No route match for github api request. Fetch args: ${JSON.stringify(args)}. Returning 404`,
+      );
+      const response = {
+        status: 404,
+        headers: new Headers(),
+        text: () => Promise.resolve('{}'),
+        json: () => Promise.resolve({}),
+        ok: false,
+      };
+      return Promise.resolve(response);
+    } else {
+      console.log(`No route match for fetch args: ${JSON.stringify(args)}`);
+      return fetch(...args);
+    }
+  });
+};
+
+Cypress.Commands.add('stubFetch', ({ fixture }) => {
+  return cy.readFile(path.join('cypress', 'fixtures', fixture), { log: false }).then(routes => {
+    cy.on('window:before:load', win => stubFetch(win, routes));
+  });
+});

cypress/utils/config.js

@@ -0,0 +1,27 @@
+const fs = require('fs-extra');
+const path = require('path');
+const yaml = require('js-yaml');
+
+const devTestDirectory = path.join(__dirname, '..', '..', 'dev-test');
+const backendsDirectory = path.join(devTestDirectory, 'backends');
+
+async function copyBackendFiles(backend) {
+  await Promise.all(
+    ['config.yml', 'index.html'].map(file => {
+      return fs.copyFile(
+        path.join(backendsDirectory, backend, file),
+        path.join(devTestDirectory, file),
+      );
+    }),
+  );
+}
+
+async function updateConfig(configModifier) {
+  const configFile = path.join(devTestDirectory, 'config.yml');
+  const configContent = await fs.readFile(configFile);
+  const config = yaml.safeLoad(configContent);
+  await configModifier(config);
+  await fs.writeFileSync(configFile, yaml.safeDump(config));
+}
+
+module.exports = { copyBackendFiles, updateConfig };

cypress/utils/mock-server.js

@@ -0,0 +1,49 @@
+const { mockServerClient } = require('mockserver-client');
+const mockserver = require('mockserver-node');
+
+const PROXY_PORT = 1080;
+const PROXY_HOST = 'localhost';
+
+const start = () =>
+  mockserver.start_mockserver({
+    serverPort: PROXY_PORT,
+  });
+
+const stop = () =>
+  mockserver.stop_mockserver({
+    serverPort: PROXY_PORT,
+  });
+
+const retrieveRecordedExpectations = async () => {
+  const promise = new Promise((resolve, reject) => {
+    mockServerClient(PROXY_HOST, PROXY_PORT)
+      .retrieveRecordedExpectations({})
+      .then(resolve, reject);
+  });
+
+  let recorded = await promise;
+  recorded = recorded.filter(({ httpRequest }) => {
+    const { Host = [] } = httpRequest.headers;
+
+    return Host.includes('api.github.com');
+  });
+
+  return recorded;
+};
+
+const resetMockServerState = async () => {
+  const promise = new Promise((resolve, reject) => {
+    mockServerClient(PROXY_HOST, PROXY_PORT)
+      .reset()
+      .then(resolve, reject);
+  });
+
+  await promise;
+};
+
+module.exports = {
+  start,
+  stop,
+  resetMockServerState,
+  retrieveRecordedExpectations,
+};

cypress/utils/regexp.js

@@ -0,0 +1,5 @@
+const escapeRegExp = string => {
+  return string.replace(/[-/\\^$*+?.()|[\]{}]/g, '\\$&');
+};
+
+module.exports = { escapeRegExp };

cypress/utils/steps.js

@@ -5,6 +5,8 @@ function login(user) {
   if (user) {
     cy.visit('/', {
       onBeforeLoad: () => {
+        // https://github.com/cypress-io/cypress/issues/1208
+        window.indexedDB.deleteDatabase('localforage');
         window.localStorage.setItem('netlify-cms-user', JSON.stringify(user));
       },
     });
@@ -12,6 +14,7 @@ function login(user) {
     cy.visit('/');
     cy.contains('button', 'Login').click();
   }
+  cy.contains('a', 'New Post');
 }
 
 function assertNotification(message) {
@@ -170,11 +173,22 @@ function populateEntry(entry) {
     }
   }
 
-  cy.get('input')
-    .first()
-    .click();
-  cy.contains('button', 'Save').click();
-  assertNotification(notifications.saved);
+  cy.clock().then(clock => {
+    // some input fields are de-bounced thus require advancing the clock
+    if (clock) {
+      // https://github.com/cypress-io/cypress/issues/1273
+      clock.tick(150);
+      clock.tick(150);
+      // eslint-disable-next-line cypress/no-unnecessary-waiting
+      cy.wait(500);
+    }
+
+    cy.get('input')
+      .first()
+      .click();
+    cy.contains('button', 'Save').click();
+    assertNotification(notifications.saved);
+  });
 }
 
 function createPost(entry) {

dev-test/backends/test/config.yml

@@ -69,7 +69,13 @@ collections: # A list of collections the CMS should be able to edit
                   max: 10,
                 }
               - { label: 'Default Author', name: author, widget: string }
-              - { label: 'Default Thumbnail', name: thumb, widget: image, class: 'thumb', required: false }
+              - {
+                  label: 'Default Thumbnail',
+                  name: thumb,
+                  widget: image,
+                  class: 'thumb',
+                  required: false,
+                }
 
       - name: 'authors'
         label: 'Authors'

dev-test/config.yml

@@ -69,7 +69,13 @@ collections: # A list of collections the CMS should be able to edit
                   max: 10,
                 }
               - { label: 'Default Author', name: author, widget: string }
-              - { label: 'Default Thumbnail', name: thumb, widget: image, class: 'thumb', required: false }
+              - {
+                  label: 'Default Thumbnail',
+                  name: thumb,
+                  widget: image,
+                  class: 'thumb',
+                  required: false,
+                }
 
       - name: 'authors'
         label: 'Authors'

package.json

@@ -23,10 +23,13 @@
     "test:e2e:dev": "start-test develop 8080 test:e2e:exec-dev",
     "test:e2e:serve": "http-server dev-test",
     "test:e2e:exec": "cypress run",
-    "test:e2e:exec-ci": "cypress run --record",
+    "test:e2e:exec-ci": "cypress run --record --parallel --ci-build-id $GITHUB_SHA --group 'GitHub CI' displayName: 'Run Cypress tests'",
     "test:e2e:exec-dev": "cypress open",
+    "test:e2e:record-fixtures:dev": "HTTP_PROXY=http://localhost:1080 RECORD_FIXTURES=true cypress open",
     "test:e2e:run": "start-test test:e2e:serve 8080 test:e2e:exec",
     "test:e2e:run-ci": "start-test test:e2e:serve 8080 test:e2e:exec-ci",
+    "mock:server:start": "node -e 'require(\"./cypress/utils/mock-server\").start()'",
+    "mock:server:stop": "node -e 'require(\"./cypress/utils/mock-server\").stop()'",
     "lint": "run-p -c --aggregate-output \"lint:*\"",
     "lint-quiet": "run-p -c --aggregate-output \"lint:* -- --quiet\"",
     "lint:css": "stylelint --ignore-path .gitignore \"{packages/**/*.{css,js},website/**/*.css}\"",
@@ -105,6 +108,8 @@
     "jest-dom": "^3.1.3",
     "jest-emotion": "^10.0.9",
     "js-yaml": "^3.13.1",
+    "mockserver-client": "^5.6.1",
+    "mockserver-node": "^5.6.1",
     "ncp": "^2.0.0",
     "nock": "^10.0.4",
     "node-fetch": "^2.3.0",

packages/netlify-cms-backend-github/src/API.js

@@ -147,7 +147,7 @@ export default class API {
   }
 
   checkMetadataRef() {
-    return this.request(`${this.repoURL}/git/refs/meta/_netlify_cms?${Date.now()}`, {
+    return this.request(`${this.repoURL}/git/refs/meta/_netlify_cms`, {
       cache: 'no-store',
     })
       .then(response => response.object)

packages/netlify-cms-backend-github/src/implementation.js

@@ -210,7 +210,7 @@ export default class GitHub {
 
   logout() {
     this.token = null;
-    if (typeof this.api.reset === 'function') {
+    if (this.api && typeof this.api.reset === 'function') {
       return this.api.reset();
     }
     return;
@@ -422,7 +422,11 @@ export default class GitHub {
   }
 
   deleteUnpublishedEntry(collection, slug) {
-    return this.api.deleteUnpublishedEntry(collection, slug);
+    // deleteUnpublishedEntry is a transactional operation
+    return this.runWithLock(
+      () => this.api.deleteUnpublishedEntry(collection, slug),
+      'Failed to acquire delete entry lock',
+    );
   }
 
   publishUnpublishedEntry(collection, slug) {

yarn.lock

@@ -5484,6 +5484,13 @@ focus-group@^0.3.1:
   resolved "https://registry.yarnpkg.com/focus-group/-/focus-group-0.3.1.tgz#e0f32ed86b0dabdd6ffcebdf898ecb32e47fedce"
   integrity sha1-4PMu2GsNq91v/OvfiY7LMuR/7c4=
 
+follow-redirects@1.7.0:
+  version "1.7.0"
+  resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.7.0.tgz#489ebc198dc0e7f64167bd23b03c4c19b5784c76"
+  integrity sha512-m/pZQy4Gj287eNy94nivy5wchN3Kp+Q5WgUPNy5lJSZ3sgkVKSYV/ZChMAQVIgx1SqfZ2zBZtPA2YlXIWxxJOQ==
+  dependencies:
+    debug "^3.2.6"
+
 follow-redirects@^1.0.0:
   version "1.9.0"
   resolved "https://registry.yarnpkg.com/follow-redirects/-/follow-redirects-1.9.0.tgz#8d5bcdc65b7108fe1508649c79c12d732dcedb4f"
@@ -6933,7 +6940,7 @@ is-text-path@^2.0.0:
   dependencies:
     text-extensions "^2.0.0"
 
-is-typedarray@~1.0.0:
+is-typedarray@^1.0.0, is-typedarray@~1.0.0:
   version "1.0.0"
   resolved "https://registry.yarnpkg.com/is-typedarray/-/is-typedarray-1.0.0.tgz#e479c80858df0c1b11ddda6940f96011fcda4a9a"
   integrity sha1-5HnICFjfDBsR3dppQPlgEfzaSpo=
@@ -8431,6 +8438,23 @@ mkdirp@*, mkdirp@0.5.1, mkdirp@^0.5.0, mkdirp@^0.5.1, mkdirp@~0.5.1:
   dependencies:
     minimist "0.0.8"
 
+mockserver-client@^5.6.1:
+  version "5.6.1"
+  resolved "https://registry.yarnpkg.com/mockserver-client/-/mockserver-client-5.6.1.tgz#8d171d4a1b3eec69234849ded52075f58d61190c"
+  integrity sha512-ueupsTHHFVCdES4G1nUfuuBthPdyteKfkBtpQ43yCZEze6VF6j5lAYj/bgUNuxJNcBt2MukXf7BHlN5wqdRH3Q==
+  dependencies:
+    q "~2.0"
+    websocket "^1.0.28"
+
+mockserver-node@^5.6.1:
+  version "5.6.1"
+  resolved "https://registry.yarnpkg.com/mockserver-node/-/mockserver-node-5.6.1.tgz#2682243f42aee5f78bba1b62ce8cc7bae69f980d"
+  integrity sha512-q7HLn+ms9Lk6/VkQFItTL7o+4o3alVVIznwTqZbwES0OhkBjese288tXNOnQ6xstlnT/Y/rXsfCQrBjuZOzLyg==
+  dependencies:
+    follow-redirects "1.7.0"
+    glob "^7.1.3"
+    q "~2.0"
+
 modify-values@^1.0.0:
   version "1.0.1"
   resolved "https://registry.yarnpkg.com/modify-values/-/modify-values-1.0.1.tgz#b3939fa605546474e3e3e3c63d64bd43b4ee6022"
@@ -8510,7 +8534,7 @@ mz@^2.5.0:
     object-assign "^4.0.1"
     thenify-all "^1.0.0"
 
-nan@^2.12.1:
+nan@^2.12.1, nan@^2.14.0:
   version "2.14.0"
   resolved "https://registry.yarnpkg.com/nan/-/nan-2.14.0.tgz#7818f722027b2459a86f0295d434d1fc2336c52c"
   integrity sha512-INOFj37C7k3AfaNTtX8RhsTw7qRy7eLET14cROi9+5HAVbbHuIWUHEauBv5qT4Av2tWasiTY1Jw6puUNqRJXQg==
@@ -9540,6 +9564,11 @@ pn@^1.1.0:
   resolved "https://registry.yarnpkg.com/pn/-/pn-1.1.0.tgz#e2f4cef0e219f463c179ab37463e4e1ecdccbafb"
   integrity sha512-2qHaIQr2VLRFoxe2nASzsV6ef4yOOH+Fi9FBOVH6cqeSgUnoyySPZkxzLuzd+RYOQTRpROA0ztTMqxROKSb/nA==
 
+pop-iterate@^1.0.1:
+  version "1.0.1"
+  resolved "https://registry.yarnpkg.com/pop-iterate/-/pop-iterate-1.0.1.tgz#ceacfdab4abf353d7a0f2aaa2c1fc7b3f9413ba3"
+  integrity sha1-zqz9q0q/NT16DyqqLB/Hs/lBO6M=
+
 portfinder@^1.0.13, portfinder@^1.0.24:
   version "1.0.25"
   resolved "https://registry.yarnpkg.com/portfinder/-/portfinder-1.0.25.tgz#254fd337ffba869f4b9d37edc298059cb4d35eca"
@@ -9898,6 +9927,15 @@ q@^1.1.2, q@^1.5.1:
   resolved "https://registry.yarnpkg.com/q/-/q-1.5.1.tgz#7e32f75b41381291d04611f1bf14109ac00651d7"
   integrity sha1-fjL3W0E4EpHQRhHxvxQQmsAGUdc=
 
+q@~2.0:
+  version "2.0.3"
+  resolved "https://registry.yarnpkg.com/q/-/q-2.0.3.tgz#75b8db0255a1a5af82f58c3f3aaa1efec7d0d134"
+  integrity sha1-dbjbAlWhpa+C9Yw/Oqoe/sfQ0TQ=
+  dependencies:
+    asap "^2.0.0"
+    pop-iterate "^1.0.1"
+    weak-map "^1.0.5"
+
 qs@6.7.0:
   version "6.7.0"
   resolved "https://registry.yarnpkg.com/qs/-/qs-6.7.0.tgz#41dc1a015e3d581f1621776be31afb2876a9b1bc"
@@ -12369,6 +12407,13 @@ type-of@^2.0.1:
   resolved "https://registry.yarnpkg.com/type-of/-/type-of-2.0.1.tgz#e72a1741896568e9f628378d816d6912f7f23972"
   integrity sha1-5yoXQYllaOn2KDeNgW1pEvfyOXI=
 
+typedarray-to-buffer@^3.1.5:
+  version "3.1.5"
+  resolved "https://registry.yarnpkg.com/typedarray-to-buffer/-/typedarray-to-buffer-3.1.5.tgz#a97ee7a9ff42691b9f783ff1bc5112fe3fca9080"
+  integrity sha512-zdu8XMNEDepKKR+XYOXAVPtWui0ly0NtohUscw+UmaHiAWT8hrV1rr//H6V+0DvJ3OQ19S979M0laLfX8rm82Q==
+  dependencies:
+    is-typedarray "^1.0.0"
+
 typedarray@^0.0.6:
   version "0.0.6"
   resolved "https://registry.yarnpkg.com/typedarray/-/typedarray-0.0.6.tgz#867ac74e3864187b1d3d47d996a78ec5c8830777"
@@ -12834,6 +12879,11 @@ wcwidth@^1.0.0:
   dependencies:
     defaults "^1.0.3"
 
+weak-map@^1.0.5:
+  version "1.0.5"
+  resolved "https://registry.yarnpkg.com/weak-map/-/weak-map-1.0.5.tgz#79691584d98607f5070bd3b70a40e6bb22e401eb"
+  integrity sha1-eWkVhNmGB/UHC9O3CkDmuyLkAes=
+
 webidl-conversions@^4.0.2:
   version "4.0.2"
   resolved "https://registry.yarnpkg.com/webidl-conversions/-/webidl-conversions-4.0.2.tgz#a855980b1f0b6b359ba1d5d9fb39ae941faa63ad"
@@ -12965,6 +13015,16 @@ websocket-extensions@>=0.1.1:
   resolved "https://registry.yarnpkg.com/websocket-extensions/-/websocket-extensions-0.1.3.tgz#5d2ff22977003ec687a4b87073dfbbac146ccf29"
   integrity sha512-nqHUnMXmBzT0w570r2JpJxfiSD1IzoI+HGVdd3aZ0yNi3ngvQ4jv1dtHt5VGxfI2yj5yqImPhOK4vmIh2xMbGg==
 
+websocket@^1.0.28:
+  version "1.0.30"
+  resolved "https://registry.yarnpkg.com/websocket/-/websocket-1.0.30.tgz#91d3bd00c3d43e916f0cf962f8f8c451bb0b2373"
+  integrity sha512-aO6klgaTdSMkhfl5VVJzD5fm+Srhh5jLYbS15+OiI1sN6h/RU/XW6WN9J1uVIpUKNmsTvT3Hs35XAFjn9NMfOw==
+  dependencies:
+    debug "^2.2.0"
+    nan "^2.14.0"
+    typedarray-to-buffer "^3.1.5"
+    yaeti "^0.0.6"
+
 what-input@^5.1.4:
   version "5.2.6"
   resolved "https://registry.yarnpkg.com/what-input/-/what-input-5.2.6.tgz#ac6f003bf8d3592a0031dea7a03565469b00020b"
@@ -13173,6 +13233,11 @@ xtend@^4.0.0, xtend@^4.0.1, xtend@~4.0.1:
   resolved "https://registry.yarnpkg.com/y18n/-/y18n-4.0.0.tgz#95ef94f85ecc81d007c264e190a120f0a3c8566b"
   integrity sha512-r9S/ZyXu/Xu9q1tYlpsLIsa3EeLXXk0VwlxqTcFRfg9EhMW+17kbt9G0NrgCmhGb5vT2hyhJZLfDGx+7+5Uj/w==
 
+yaeti@^0.0.6:
+  version "0.0.6"
+  resolved "https://registry.yarnpkg.com/yaeti/-/yaeti-0.0.6.tgz#f26f484d72684cf42bedfb76970aa1608fbf9577"
+  integrity sha1-8m9ITXJoTPQr7ft2lwqhYI+/lXc=
+
 yallist@^3.0.0, yallist@^3.0.2, yallist@^3.0.3:
   version "3.1.1"
   resolved "https://registry.yarnpkg.com/yallist/-/yallist-3.1.1.tgz#dbb7daf9bfd8bac9ab45ebf602b8cbad0d5d08fd"