Add netlify-git backend

package.json

@@ -77,6 +77,7 @@
     "prismjs": "^1.5.1",
     "react-portal": "^2.2.1",
     "selection-position": "^1.0.0",
+    "semaphore": "^1.0.5",
     "slate": "^0.13.6"
   }
 }

src/backends/backend.js

@@ -1,5 +1,6 @@
 import TestRepoBackend from './test-repo/implementation';
 import GitHubBackend from './github/implementation';
+import NetlifyGitBackend from './netlify-git/implementation';
 import { resolveFormat } from '../formats/formats';
 import { createEntry } from '../valueObjects/Entry';
 import { SIMPLE, BRANCH } from './constants';
@@ -152,6 +153,8 @@ export function resolveBackend(config) {
       return new Backend(new TestRepoBackend(config), authStore);
     case 'github':
       return new Backend(new GitHubBackend(config), authStore);
+    case 'netlify-git':
+      return new Backend(new NetlifyGitBackend(config), authStore);
     default:
       throw `Backend not found: ${name}`;
   }

src/backends/github/implementation.js

@@ -1,7 +1,10 @@
+import semaphore from 'semaphore';
 import {createEntry} from '../../valueObjects/Entry';
 import AuthenticationPage from './AuthenticationPage';
 import API from './API';
 
+const MAX_CONCURRENT_DOWNLOADS = 10;
+
 export default class GitHub {
   constructor(config) {
     this.config = config;
@@ -29,13 +32,19 @@ export default class GitHub {
   }
 
   entries(collection) {
-    return this.api.listFiles(collection.get('folder')).then((files) => (
+    return this.api.listFiles(collection.get('folder')).then((files) => {
-      Promise.all(files.map((file) => (
+      const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
-        this.api.readFile(file.path, file.sha).then((data) => {
+      const promises = [];
-          return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data);
+      files.map((file) => {
-        })
+        sem.take(() => {
-      )))
+          promises.push(this.api.readFile(file.path, file.sha).then((data) => {
-    )).then((entries) => ({
+            sem.leave();
+            return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data);
+          }));
+        });
+      });
+      return Promise.all(promises);
+    }).then((entries) => ({
       pagination: {},
       entries
     }));

src/backends/netlify-git/API.js

@@ -0,0 +1,284 @@
+import LocalForage from 'localforage';
+import MediaProxy from '../../valueObjects/MediaProxy';
+import { Base64 } from 'js-base64';
+import { BRANCH } from '../constants';
+
+export default class API {
+  constructor(token, url, branch) {
+    this.token = token;
+    this.url = url;
+    this.branch = branch;
+    this.repoURL = '';
+  }
+
+  requestHeaders(headers = {}) {
+    return {
+      Authorization: `Bearer ${this.token}`,
+      'Content-Type': 'application/json',
+      ...headers
+    };
+  }
+
+  parseJsonResponse(response) {
+    return response.json().then((json) => {
+      if (!response.ok) {
+        return Promise.reject(json);
+      }
+
+      return json;
+    });
+  }
+
+  urlFor(path, options) {
+    const params = [];
+    if (options.params) {
+      for (const key in options.params) {
+        params.push(`${key}=${encodeURIComponent(options.params[key])}`);
+      }
+    }
+    if (params.length) {
+      path += `?${params.join('&')}`;
+    }
+    return this.url + path;
+  }
+
+  request(path, options = {}) {
+    const headers = this.requestHeaders(options.headers || {});
+    const url = this.urlFor(path, options);
+    return fetch(url, { ...options, headers: headers }).then((response) => {
+      if (response.headers.get('Content-Type').match(/json/) && !options.raw) {
+        return this.parseJsonResponse(response);
+      }
+
+      return response.text();
+    });
+  }
+
+  checkMetadataRef() {
+    return this.request(`${this.repoURL}/git/refs/meta/_netlify_cms?${Date.now()}`, {
+      cache: 'no-store',
+    })
+    .then(response => response.object)
+    .catch(error => {
+      // Meta ref doesn't exist
+      const readme = {
+        raw: '# Netlify CMS\n\nThis tree is used by the Netlify CMS to store metadata information for specific files and branches.'
+      };
+
+      return this.uploadBlob(readme)
+      .then(item => this.request(`${this.repoURL}/git/trees`, {
+        method: 'POST',
+        body: JSON.stringify({ tree: [{ path: 'README.md', mode: '100644', type: 'blob', sha: item.sha }] })
+      }))
+      .then(tree => this.commit('First Commit', tree))
+      .then(response => this.createRef('meta', '_netlify_cms', response.sha))
+      .then(response => response.object);
+    });
+  }
+
+  storeMetadata(key, data) {
+    return this.checkMetadataRef()
+    .then((branchData) => {
+      const fileTree = {
+        [`${key}.json`]: {
+          path: `${key}.json`,
+          raw: JSON.stringify(data),
+          file: true
+        }
+      };
+
+      return this.uploadBlob(fileTree[`${key}.json`])
+      .then(item => this.updateTree(branchData.sha, '/', fileTree))
+      .then(changeTree => this.commit(`Updating “${key}” metadata`, changeTree))
+      .then(response => this.patchRef('meta', '_netlify_cms', response.sha));
+    });
+  }
+
+  retrieveMetadata(key, data) {
+    const cache = LocalForage.getItem(`gh.meta.${key}`);
+    return cache.then((cached) => {
+      if (cached && cached.expires > Date.now()) { return cached.data; }
+
+      return this.request(`${this.repoURL}/files/${key}.json?ref=refs/meta/_netlify_cms`, {
+        headers: { Accept: 'application/vnd.github.VERSION.raw' },
+        cache: 'no-store',
+      }).then((result) => {
+        LocalForage.setItem(`gh.meta.${key}`, {
+          expires: Date.now() + 300000, // In 5 minutes
+          data: result,
+        });
+        return result;
+      });
+    });
+  }
+
+  readFile(path, sha) {
+    const cache = sha ? LocalForage.getItem(`gh.${sha}`) : Promise.resolve(null);
+    return cache.then((cached) => {
+      if (cached) { return cached; }
+
+      return this.request(`${this.repoURL}/files/${path}`, {
+        headers: { Accept: 'application/vnd.github.VERSION.raw' },
+        params: { ref: this.branch },
+        cache: false,
+        raw: true
+      }).then((result) => {
+        if (sha) {
+          LocalForage.setItem(`gh.${sha}`, result);
+        }
+
+        return result;
+      });
+    });
+  }
+
+  listFiles(path) {
+    return this.request(`${this.repoURL}/files/${path}`, {
+      params: { ref: this.branch }
+    });
+  }
+
+  persistFiles(entry, mediaFiles, options) {
+    let filename, part, parts, subtree;
+    const fileTree = {};
+    const files = [];
+    mediaFiles.concat(entry).forEach((file) => {
+      if (file.uploaded) { return; }
+      files.push(this.uploadBlob(file));
+      parts = file.path.split('/').filter((part) => part);
+      filename = parts.pop();
+      subtree = fileTree;
+      while (part = parts.shift()) {
+        subtree[part] = subtree[part] || {};
+        subtree = subtree[part];
+      }
+      subtree[filename] = file;
+      file.file = true;
+    });
+    return Promise.all(files)
+      .then(() => this.getBranch())
+      .then(branchData => this.updateTree(branchData.commit.sha, '/', fileTree))
+      .then(changeTree => this.commit(options.commitMessage, changeTree))
+      .then((response) => {
+        if (options.mode && options.mode === BRANCH) {
+          const contentKey = options.collectionName ? `${options.collectionName}-${entry.slug}` : entry.slug;
+          return this.createBranch(`cms/${contentKey}`, response.sha)
+          .then(this.storeMetadata(contentKey, { status: 'draft' }))
+          .then(this.createPR(options.commitMessage, `cms/${contentKey}`));
+        } else {
+          return this.patchBranch(this.branch, response.sha);
+        }
+      });
+  }
+
+  createRef(type, name, sha) {
+    return this.request(`${this.repoURL}/git/refs`, {
+      method: 'POST',
+      body: JSON.stringify({ ref: `refs/${type}/${name}`, sha }),
+    });
+  }
+
+  createBranch(branchName, sha) {
+    return this.createRef('heads', branchName, sha);
+  }
+
+  patchRef(type, name, sha) {
+    return this.request(`${this.repoURL}/git/refs/${type}/${name}`, {
+      method: 'PATCH',
+      body: JSON.stringify({ sha })
+    });
+  }
+
+  patchBranch(branchName, sha) {
+    return this.patchRef('heads', branchName, sha);
+  }
+
+  getBranch() {
+    return this.request(`${this.repoURL}/branches/${this.branch}`);
+  }
+
+  createPR(title, head, base = 'master') {
+    const body = 'Automatically generated by Netlify CMS';
+    return this.request(`${this.repoURL}/pulls`, {
+      method: 'POST',
+      body: JSON.stringify({ title, body, head, base }),
+    });
+  }
+
+  getTree(sha) {
+    return sha ? this.request(`${this.repoURL}/git/trees/${sha}`) : Promise.resolve({ tree: [] });
+  }
+
+  toBase64(str) {
+    return Promise.resolve(
+      Base64.encode(str)
+    );
+  }
+
+  uploadBlob(item) {
+    const content = item instanceof MediaProxy ? item.toBase64() : this.toBase64(item.raw);
+
+    return content.then((contentBase64) => {
+      return this.request(`${this.repoURL}/git/blobs`, {
+        method: 'POST',
+        body: JSON.stringify({
+          content: contentBase64,
+          encoding: 'base64'
+        })
+      }).then((response) => {
+        item.sha = response.sha;
+        item.uploaded = true;
+        return item;
+      });
+    });
+  }
+
+  updateTree(sha, path, fileTree) {
+    return this.getTree(sha)
+      .then((tree) => {
+        var obj, filename, fileOrDir;
+        var updates = [];
+        var added = {};
+
+        for (var i = 0, len = tree.tree.length; i < len; i++) {
+          obj = tree.tree[i];
+          if (fileOrDir = fileTree[obj.path]) {
+            added[obj.path] = true;
+            if (fileOrDir.file) {
+              updates.push({ path: obj.path, mode: obj.mode, type: obj.type, sha: fileOrDir.sha });
+            } else {
+              updates.push(this.updateTree(obj.sha, obj.path, fileOrDir));
+            }
+          }
+        }
+        for (filename in fileTree) {
+          fileOrDir = fileTree[filename];
+          if (added[filename]) { continue; }
+          updates.push(
+            fileOrDir.file ?
+              { path: filename, mode: '100644', type: 'blob', sha: fileOrDir.sha } :
+              this.updateTree(null, filename, fileOrDir)
+          );
+        }
+        return Promise.all(updates)
+          .then((updates) => {
+            return this.request(`${this.repoURL}/git/trees`, {
+              method: 'POST',
+              body: JSON.stringify({ base_tree: sha, tree: updates })
+            });
+          }).then((response) => {
+            return { path: path, mode: '040000', type: 'tree', sha: response.sha, parentSha: sha };
+          });
+      });
+  }
+
+  commit(message, changeTree) {
+    const tree = changeTree.sha;
+    const parents = changeTree.parentSha ? [changeTree.parentSha] : [];
+    return this.request(`${this.repoURL}/git/commits`, {
+      method: 'POST',
+      body: JSON.stringify({ message, tree, parents })
+    });
+  }
+
+}

src/backends/netlify-git/AuthenticationPage.js

@@ -0,0 +1,61 @@
+import React from 'react';
+import Authenticator from '../../lib/netlify-auth';
+
+export default class AuthenticationPage extends React.Component {
+  static propTypes = {
+    onLogin: React.PropTypes.func.isRequired
+  };
+
+  constructor(props) {
+    super(props);
+    this.state = {};
+    this.handleLogin = this.handleLogin.bind(this);
+  }
+
+  handleLogin(e) {
+    e.preventDefault();
+    const {email, password} = this.state;
+    this.setState({authenticating: true});
+    fetch(`${AuthenticationPage.url}/token`, {
+      method: 'POST',
+      body: 'grant_type=client_credentials',
+      headers: {
+        'Content-Type': 'application/x-www-form-urlencoded',
+        'Authorization': 'Basic ' + btoa(`${email}:${password}`)
+      }
+    }).then((response) => {
+      console.log(response);
+      if (response.ok) {
+        return response.json().then((data) => {
+          this.props.onLogin(Object.assign({email}, data));
+        });
+      }
+      response.json().then((data) => {
+        this.setState({loginError: data.msg});
+      })
+    })
+  }
+
+  handleChange(key) {
+    return (e) => {
+      this.setState({[key]: e.target.value});
+    };
+  }
+
+  render() {
+    const { loginError } = this.state;
+
+    return <form onSubmit={this.handleLogin}>
+      {loginError && <p>{loginError}</p>}
+      <p>
+        <label>Your Email: <input type="email" onChange={this.handleChange('email')}/></label>
+      </p>
+      <p>
+        <label>Your Password: <input type="password" onChange={this.handleChange('password')}/></label>
+      </p>
+      <p>
+        <button>Login</button>
+      </p>
+    </form>;
+  }
+}

src/backends/netlify-git/implementation.js

@@ -0,0 +1,59 @@
+import semaphore from 'semaphore';
+import {createEntry} from '../../valueObjects/Entry';
+import AuthenticationPage from './AuthenticationPage';
+import API from './API';
+
+const MAX_CONCURRENT_DOWNLOADS = 10;
+
+export default class NetlifyGit {
+  constructor(config) {
+    this.config = config;
+    if (config.getIn(['backend', 'url']) == null) {
+      throw 'The netlify-git backend needs a "url" in the backend configuration.';
+    }
+    this.url = config.getIn(['backend', 'url']);
+    this.branch = config.getIn(['backend', 'branch']) || 'master';
+    AuthenticationPage.url = this.url;
+  }
+
+  authComponent() {
+    return AuthenticationPage;
+  }
+
+  setUser(user) {
+    this.api = new API(user.access_token, this.url, this.branch || 'master');
+  }
+
+  authenticate(state) {
+    return Promise.resolve(state);
+  }
+
+  entries(collection) {
+    return this.api.listFiles(collection.get('folder')).then((files) => {
+      const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
+      const promises = [];
+      files.map((file) => {
+        sem.take(() => {
+          promises.push(this.api.readFile(file.path, file.sha).then((data) => {
+            sem.leave();
+            return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data);
+          }));
+        });
+      });
+      return Promise.all(promises);
+    }).then((entries) => ({
+      pagination: {},
+      entries
+    }));
+  }
+
+  entry(collection, slug) {
+    return this.entries(collection).then((response) => (
+      response.entries.filter((entry) => entry.slug === slug)[0]
+    ));
+  }
+
+  persistEntry(entry, mediaFiles = [], options = {}) {
+    return this.api.persistFiles(entry, mediaFiles, options);
+  }
+}