diff --git a/package.json b/package.json index 05201017..c1db9538 100644 --- a/package.json +++ b/package.json @@ -77,6 +77,7 @@ "prismjs": "^1.5.1", "react-portal": "^2.2.1", "selection-position": "^1.0.0", + "semaphore": "^1.0.5", "slate": "^0.13.6" } } diff --git a/src/backends/backend.js b/src/backends/backend.js index cafcf668..0104866d 100644 --- a/src/backends/backend.js +++ b/src/backends/backend.js @@ -1,5 +1,6 @@ import TestRepoBackend from './test-repo/implementation'; import GitHubBackend from './github/implementation'; +import NetlifyGitBackend from './netlify-git/implementation'; import { resolveFormat } from '../formats/formats'; import { createEntry } from '../valueObjects/Entry'; import { SIMPLE, BRANCH } from './constants'; @@ -48,6 +49,7 @@ class Backend { entries(collection, page, perPage) { return this.implementation.entries(collection, page, perPage).then((response) => { + console.log("Got %s entries", response.entries.length); return { pagination: response.pagination, entries: response.entries.map(this.entryWithFormat(collection)) @@ -159,6 +161,8 @@ export function resolveBackend(config) { return new Backend(new TestRepoBackend(config), authStore); case 'github': return new Backend(new GitHubBackend(config), authStore); + case 'netlify-git': + return new Backend(new NetlifyGitBackend(config), authStore); default: throw `Backend not found: ${name}`; } diff --git a/src/backends/github/API.js b/src/backends/github/API.js index 7c4de494..a49a2585 100644 --- a/src/backends/github/API.js +++ b/src/backends/github/API.js @@ -35,9 +35,23 @@ export default class API { }); } + urlFor(path, options) { + const params = []; + if (options.params) { + for (const key in options.params) { + params.push(`${key}=${encodeURIComponent(options.params[key])}`); + } + } + if (params.length) { + path += `?${params.join('&')}`; + } + return API_ROOT + path; + } + request(path, options = {}) { const headers = this.requestHeaders(options.headers || {}); - return fetch(API_ROOT + path, { ...options, headers: headers }).then((response) => { + const url = this.urlFor(path, options); + return fetch(url, { ...options, headers: headers }).then((response) => { if (response.headers.get('Content-Type').match(/json/)) { return this.parseJsonResponse(response); } @@ -111,7 +125,7 @@ export default class API { return this.request(`${this.repoURL}/contents/${path}`, { headers: { Accept: 'application/vnd.github.VERSION.raw' }, - body: { ref: this.branch }, + params: { ref: this.branch }, cache: false }).then((result) => { if (sha) { @@ -125,7 +139,7 @@ export default class API { listFiles(path) { return this.request(`${this.repoURL}/contents/${path}`, { - body: { ref: this.branch } + params: { ref: this.branch } }); } diff --git a/src/backends/github/implementation.js b/src/backends/github/implementation.js index acabf3fd..9cb11f45 100644 --- a/src/backends/github/implementation.js +++ b/src/backends/github/implementation.js @@ -1,7 +1,10 @@ -import { createEntry } from '../../valueObjects/Entry'; +import semaphore from 'semaphore'; +import {createEntry} from '../../valueObjects/Entry'; import AuthenticationPage from './AuthenticationPage'; import API from './API'; +const MAX_CONCURRENT_DOWNLOADS = 10; + export default class GitHub { constructor(config) { this.config = config; @@ -9,6 +12,7 @@ export default class GitHub { throw 'The GitHub backend needs a "repo" in the backend configuration.'; } this.repo = config.getIn(['backend', 'repo']); + this.branch = config.getIn(['backend', 'branch']) || 'master'; } authComponent() { @@ -28,13 +32,22 @@ export default class GitHub { } entries(collection) { - return this.api.listFiles(collection.get('folder')).then((files) => ( - Promise.all(files.map((file) => ( - this.api.readFile(file.path, file.sha).then((data) => { - return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data); - }) - ))) - )).then((entries) => ({ + return this.api.listFiles(collection.get('folder')).then((files) => { + const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); + const promises = []; + files.map((file) => { + promises.push(new Promise((resolve, reject) => { + return sem.take(() => this.api.readFile(file.path, file.sha).then((data) => { + resolve(createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data)); + sem.leave(); + }).catch((err) => { + sem.leave(); + reject(err); + })); + })); + }); + return Promise.all(promises); + }).then((entries) => ({ pagination: {}, entries })); diff --git a/src/backends/netlify-git/API.js b/src/backends/netlify-git/API.js new file mode 100644 index 00000000..8c072a30 --- /dev/null +++ b/src/backends/netlify-git/API.js @@ -0,0 +1,284 @@ +import LocalForage from 'localforage'; +import MediaProxy from '../../valueObjects/MediaProxy'; +import { Base64 } from 'js-base64'; +import { BRANCH } from '../constants'; + +export default class API { + constructor(token, url, branch) { + this.token = token; + this.url = url; + this.branch = branch; + this.repoURL = ''; + } + + requestHeaders(headers = {}) { + return { + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json', + ...headers + }; + } + + parseJsonResponse(response) { + return response.json().then((json) => { + if (!response.ok) { + return Promise.reject(json); + } + + return json; + }); + } + + urlFor(path, options) { + const params = []; + if (options.params) { + for (const key in options.params) { + params.push(`${key}=${encodeURIComponent(options.params[key])}`); + } + } + if (params.length) { + path += `?${params.join('&')}`; + } + return this.url + path; + } + + request(path, options = {}) { + const headers = this.requestHeaders(options.headers || {}); + const url = this.urlFor(path, options); + return fetch(url, { ...options, headers: headers }).then((response) => { + if (response.headers.get('Content-Type').match(/json/) && !options.raw) { + return this.parseJsonResponse(response); + } + + return response.text(); + }); + } + + checkMetadataRef() { + return this.request(`${this.repoURL}/git/refs/meta/_netlify_cms?${Date.now()}`, { + cache: 'no-store', + }) + .then(response => response.object) + .catch(error => { + // Meta ref doesn't exist + const readme = { + raw: '# Netlify CMS\n\nThis tree is used by the Netlify CMS to store metadata information for specific files and branches.' + }; + + return this.uploadBlob(readme) + .then(item => this.request(`${this.repoURL}/git/trees`, { + method: 'POST', + body: JSON.stringify({ tree: [{ path: 'README.md', mode: '100644', type: 'blob', sha: item.sha }] }) + })) + .then(tree => this.commit('First Commit', tree)) + .then(response => this.createRef('meta', '_netlify_cms', response.sha)) + .then(response => response.object); + }); + } + + storeMetadata(key, data) { + return this.checkMetadataRef() + .then((branchData) => { + const fileTree = { + [`${key}.json`]: { + path: `${key}.json`, + raw: JSON.stringify(data), + file: true + } + }; + + return this.uploadBlob(fileTree[`${key}.json`]) + .then(item => this.updateTree(branchData.sha, '/', fileTree)) + .then(changeTree => this.commit(`Updating “${key}” metadata`, changeTree)) + .then(response => this.patchRef('meta', '_netlify_cms', response.sha)); + }); + } + + retrieveMetadata(key, data) { + const cache = LocalForage.getItem(`gh.meta.${key}`); + return cache.then((cached) => { + if (cached && cached.expires > Date.now()) { return cached.data; } + + return this.request(`${this.repoURL}/files/${key}.json?ref=refs/meta/_netlify_cms`, { + headers: { Accept: 'application/vnd.github.VERSION.raw' }, + cache: 'no-store', + }).then((result) => { + LocalForage.setItem(`gh.meta.${key}`, { + expires: Date.now() + 300000, // In 5 minutes + data: result, + }); + return result; + }); + }); + } + + readFile(path, sha) { + const cache = sha ? LocalForage.getItem(`gh.${sha}`) : Promise.resolve(null); + return cache.then((cached) => { + if (cached) { return cached; } + + return this.request(`${this.repoURL}/files/${path}`, { + headers: { Accept: 'application/vnd.github.VERSION.raw' }, + params: { ref: this.branch }, + cache: false, + raw: true + }).then((result) => { + if (sha) { + LocalForage.setItem(`gh.${sha}`, result); + } + + return result; + }); + }); + } + + listFiles(path) { + return this.request(`${this.repoURL}/files/${path}`, { + params: { ref: this.branch } + }); + } + + persistFiles(entry, mediaFiles, options) { + let filename, part, parts, subtree; + const fileTree = {}; + const files = []; + mediaFiles.concat(entry).forEach((file) => { + if (file.uploaded) { return; } + files.push(this.uploadBlob(file)); + parts = file.path.split('/').filter((part) => part); + filename = parts.pop(); + subtree = fileTree; + while (part = parts.shift()) { + subtree[part] = subtree[part] || {}; + subtree = subtree[part]; + } + subtree[filename] = file; + file.file = true; + }); + return Promise.all(files) + .then(() => this.getBranch()) + .then(branchData => this.updateTree(branchData.commit.sha, '/', fileTree)) + .then(changeTree => this.commit(options.commitMessage, changeTree)) + .then((response) => { + if (options.mode && options.mode === BRANCH) { + const contentKey = options.collectionName ? `${options.collectionName}-${entry.slug}` : entry.slug; + return this.createBranch(`cms/${contentKey}`, response.sha) + .then(this.storeMetadata(contentKey, { status: 'draft' })) + .then(this.createPR(options.commitMessage, `cms/${contentKey}`)); + } else { + return this.patchBranch(this.branch, response.sha); + } + }); + } + + createRef(type, name, sha) { + return this.request(`${this.repoURL}/git/refs`, { + method: 'POST', + body: JSON.stringify({ ref: `refs/${type}/${name}`, sha }), + }); + } + + createBranch(branchName, sha) { + return this.createRef('heads', branchName, sha); + } + + patchRef(type, name, sha) { + return this.request(`${this.repoURL}/git/refs/${type}/${name}`, { + method: 'PATCH', + body: JSON.stringify({ sha }) + }); + } + + patchBranch(branchName, sha) { + return this.patchRef('heads', branchName, sha); + } + + getBranch() { + return this.request(`${this.repoURL}/branches/${this.branch}`); + } + + createPR(title, head, base = 'master') { + const body = 'Automatically generated by Netlify CMS'; + return this.request(`${this.repoURL}/pulls`, { + method: 'POST', + body: JSON.stringify({ title, body, head, base }), + }); + } + + getTree(sha) { + return sha ? this.request(`${this.repoURL}/git/trees/${sha}`) : Promise.resolve({ tree: [] }); + } + + toBase64(str) { + return Promise.resolve( + Base64.encode(str) + ); + } + + uploadBlob(item) { + const content = item instanceof MediaProxy ? item.toBase64() : this.toBase64(item.raw); + + return content.then((contentBase64) => { + return this.request(`${this.repoURL}/git/blobs`, { + method: 'POST', + body: JSON.stringify({ + content: contentBase64, + encoding: 'base64' + }) + }).then((response) => { + item.sha = response.sha; + item.uploaded = true; + return item; + }); + }); + } + + updateTree(sha, path, fileTree) { + return this.getTree(sha) + .then((tree) => { + var obj, filename, fileOrDir; + var updates = []; + var added = {}; + + for (var i = 0, len = tree.tree.length; i < len; i++) { + obj = tree.tree[i]; + if (fileOrDir = fileTree[obj.path]) { + added[obj.path] = true; + if (fileOrDir.file) { + updates.push({ path: obj.path, mode: obj.mode, type: obj.type, sha: fileOrDir.sha }); + } else { + updates.push(this.updateTree(obj.sha, obj.path, fileOrDir)); + } + } + } + for (filename in fileTree) { + fileOrDir = fileTree[filename]; + if (added[filename]) { continue; } + updates.push( + fileOrDir.file ? + { path: filename, mode: '100644', type: 'blob', sha: fileOrDir.sha } : + this.updateTree(null, filename, fileOrDir) + ); + } + return Promise.all(updates) + .then((updates) => { + return this.request(`${this.repoURL}/git/trees`, { + method: 'POST', + body: JSON.stringify({ base_tree: sha, tree: updates }) + }); + }).then((response) => { + return { path: path, mode: '040000', type: 'tree', sha: response.sha, parentSha: sha }; + }); + }); + } + + commit(message, changeTree) { + const tree = changeTree.sha; + const parents = changeTree.parentSha ? [changeTree.parentSha] : []; + return this.request(`${this.repoURL}/git/commits`, { + method: 'POST', + body: JSON.stringify({ message, tree, parents }) + }); + } + +} diff --git a/src/backends/netlify-git/AuthenticationPage.js b/src/backends/netlify-git/AuthenticationPage.js new file mode 100644 index 00000000..4503a660 --- /dev/null +++ b/src/backends/netlify-git/AuthenticationPage.js @@ -0,0 +1,61 @@ +import React from 'react'; +import Authenticator from '../../lib/netlify-auth'; + +export default class AuthenticationPage extends React.Component { + static propTypes = { + onLogin: React.PropTypes.func.isRequired + }; + + constructor(props) { + super(props); + this.state = {}; + this.handleLogin = this.handleLogin.bind(this); + } + + handleLogin(e) { + e.preventDefault(); + const {email, password} = this.state; + this.setState({authenticating: true}); + fetch(`${AuthenticationPage.url}/token`, { + method: 'POST', + body: 'grant_type=client_credentials', + headers: { + 'Content-Type': 'application/x-www-form-urlencoded', + 'Authorization': 'Basic ' + btoa(`${email}:${password}`) + } + }).then((response) => { + console.log(response); + if (response.ok) { + return response.json().then((data) => { + this.props.onLogin(Object.assign({email}, data)); + }); + } + response.json().then((data) => { + this.setState({loginError: data.msg}); + }) + }) + } + + handleChange(key) { + return (e) => { + this.setState({[key]: e.target.value}); + }; + } + + render() { + const { loginError } = this.state; + + return
+ {loginError &&

{loginError}

} +

+ +

+

+ +

+

+ +

+
; + } +} diff --git a/src/backends/netlify-git/implementation.js b/src/backends/netlify-git/implementation.js new file mode 100644 index 00000000..cd52f5ff --- /dev/null +++ b/src/backends/netlify-git/implementation.js @@ -0,0 +1,62 @@ +import semaphore from 'semaphore'; +import {createEntry} from '../../valueObjects/Entry'; +import AuthenticationPage from './AuthenticationPage'; +import API from './API'; + +const MAX_CONCURRENT_DOWNLOADS = 10; + +export default class NetlifyGit { + constructor(config) { + this.config = config; + if (config.getIn(['backend', 'url']) == null) { + throw 'The netlify-git backend needs a "url" in the backend configuration.'; + } + this.url = config.getIn(['backend', 'url']); + this.branch = config.getIn(['backend', 'branch']) || 'master'; + AuthenticationPage.url = this.url; + } + + authComponent() { + return AuthenticationPage; + } + + setUser(user) { + this.api = new API(user.access_token, this.url, this.branch || 'master'); + } + + authenticate(state) { + return Promise.resolve(state); + } + + entries(collection) { + return this.api.listFiles(collection.get('folder')).then((files) => { + const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); + const promises = []; + files.map((file) => { + promises.push(new Promise((resolve, reject) => { + return sem.take(() => this.api.readFile(file.path, file.sha).then((data) => { + resolve(createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data)); + sem.leave(); + }).catch((err) => { + sem.leave(); + reject(err); + })); + })); + }); + return Promise.all(promises); + }).then((entries) => ({ + pagination: {}, + entries + })); + } + + entry(collection, slug) { + return this.entries(collection).then((response) => ( + response.entries.filter((entry) => entry.slug === slug)[0] + )); + } + + persistEntry(entry, mediaFiles = [], options = {}) { + return this.api.persistFiles(entry, mediaFiles, options); + } +}