diff --git a/dev-test/backends/azure/config.yml b/dev-test/backends/azure/config.yml new file mode 100644 index 00000000..1fed4943 --- /dev/null +++ b/dev-test/backends/azure/config.yml @@ -0,0 +1,65 @@ +backend: + name: azure + branch: master + repo: organization/project/repo # replace with actual path + tenant_id: tenantId # replace with your tenantId + app_id: appId # replace with your appId + +publish_mode: editorial_workflow +media_folder: static/media +public_folder: /media +collections: + - name: posts + label: Posts + label_singular: 'Post' + folder: content/posts + create: true + slug: '{{year}}-{{month}}-{{day}}-{{slug}}' + fields: + - label: Template + name: template + widget: hidden + default: post + - label: Title + name: title + widget: string + - label: 'Cover Image' + name: 'image' + widget: 'image' + required: false + - label: Publish Date + name: date + widget: datetime + - label: Description + name: description + widget: text + - label: Category + name: category + widget: string + - label: Body + name: body + widget: markdown + - label: Tags + name: tags + widget: list + - name: pages + label: Pages + label_singular: 'Page' + folder: content/pages + create: true + slug: '{{slug}}' + fields: + - label: Template + name: template + widget: hidden + default: page + - label: Title + name: title + widget: string + - label: Draft + name: draft + widget: boolean + default: true + - label: Body + name: body + widget: markdown diff --git a/dev-test/backends/azure/index.html b/dev-test/backends/azure/index.html new file mode 100644 index 00000000..103dbb27 --- /dev/null +++ b/dev-test/backends/azure/index.html @@ -0,0 +1,41 @@ + + + + + + Netlify CMS Development Test + + + + + + diff --git a/packages/netlify-cms-app/package.json b/packages/netlify-cms-app/package.json index 90f2cf85..76189a98 100644 --- a/packages/netlify-cms-app/package.json +++ b/packages/netlify-cms-app/package.json @@ -32,6 +32,7 @@ "immutable": "^3.7.6", "lodash": "^4.17.11", "moment": "^2.24.0", + "netlify-cms-backend-azure": "^1.0.0", "netlify-cms-backend-bitbucket": "^2.12.5", "netlify-cms-backend-git-gateway": "^2.11.6", "netlify-cms-backend-github": "^2.11.6", diff --git a/packages/netlify-cms-app/src/extensions.js b/packages/netlify-cms-app/src/extensions.js index 94e271e9..e7b484c3 100644 --- a/packages/netlify-cms-app/src/extensions.js +++ b/packages/netlify-cms-app/src/extensions.js @@ -2,6 +2,7 @@ import { NetlifyCmsCore as CMS } from 'netlify-cms-core'; // Backends +import { AzureBackend } from 'netlify-cms-backend-azure'; import { GitHubBackend } from 'netlify-cms-backend-github'; import { GitLabBackend } from 'netlify-cms-backend-gitlab'; import { GitGatewayBackend } from 'netlify-cms-backend-git-gateway'; @@ -35,6 +36,7 @@ import * as locales from 'netlify-cms-locales'; // Register all the things CMS.registerBackend('git-gateway', GitGatewayBackend); +CMS.registerBackend('azure', AzureBackend); CMS.registerBackend('github', GitHubBackend); CMS.registerBackend('gitlab', GitLabBackend); CMS.registerBackend('bitbucket', BitbucketBackend); diff --git a/packages/netlify-cms-backend-azure/README.md b/packages/netlify-cms-backend-azure/README.md new file mode 100644 index 00000000..3d236327 --- /dev/null +++ b/packages/netlify-cms-backend-azure/README.md @@ -0,0 +1,11 @@ +# Docs coming soon! + +Netlify CMS was recently converted from a single npm package to a "monorepo" of over 20 packages. +That's over 20 Readme's! We haven't created one for this package yet, but we will soon. + +In the meantime, you can: + +1. Check out the [main readme](https://github.com/netlify/netlify-cms/#readme) or the [documentation + site](https://www.netlifycms.org) for more info. +2. Reach out to the [community chat](https://netlifycms.org/chat/) if you need help. +3. Help out and [write the readme yourself](https://github.com/netlify/netlify-cms/edit/master/packages/netlify-cms-backend-azure/README.md)! diff --git a/packages/netlify-cms-backend-azure/package.json b/packages/netlify-cms-backend-azure/package.json new file mode 100644 index 00000000..d74b0c51 --- /dev/null +++ b/packages/netlify-cms-backend-azure/package.json @@ -0,0 +1,38 @@ +{ + "name": "netlify-cms-backend-azure", + "description": "Azure DevOps backend for Netlify CMS", + "version": "1.0.0", + "license": "MIT", + "repository": "https://github.com/netlify/netlify-cms/tree/master/packages/netlify-cms-backend-azure", + "bugs": "https://github.com/netlify/netlify-cms/issues", + "module": "dist/esm/index.js", + "main": "dist/netlify-cms-backend-azure.js", + "keywords": [ + "netlify", + "netlify-cms", + "backend", + "azure", + "devops" + ], + "sideEffects": false, + "scripts": { + "develop": "yarn build:esm --watch", + "build": "cross-env NODE_ENV=production webpack", + "build:esm": "cross-env NODE_ENV=esm babel src --out-dir dist/esm --ignore **/__tests__ --root-mode upward --extensions \".js,.jsx,.ts,.tsx\"" + }, + "dependencies": { + "js-base64": "^3.0.0", + "semaphore": "^1.1.0" + }, + "peerDependencies": { + "@emotion/core": "^10.0.9", + "@emotion/styled": "^10.0.9", + "immutable": "^3.7.6", + "lodash": "^4.17.11", + "netlify-cms-lib-auth": "^2.2.0", + "netlify-cms-lib-util": "^2.3.0", + "netlify-cms-ui-default": "^2.6.0", + "prop-types": "^15.7.2", + "react": "^16.8.4" + } +} diff --git a/packages/netlify-cms-backend-azure/src/API.ts b/packages/netlify-cms-backend-azure/src/API.ts new file mode 100644 index 00000000..abc2e4f3 --- /dev/null +++ b/packages/netlify-cms-backend-azure/src/API.ts @@ -0,0 +1,790 @@ +import { Base64 } from 'js-base64'; +import { partial, result, trim, trimStart } from 'lodash'; +import { + localForage, + APIError, + ApiRequest, + unsentRequest, + requestWithBackoff, + responseParser, + AssetProxy, + PersistOptions, + readFile, + DEFAULT_PR_BODY, + MERGE_COMMIT_MESSAGE, + generateContentKey, + parseContentKey, + labelToStatus, + isCMSLabel, + EditorialWorkflowError, + statusToLabel, + PreviewState, + readFileMetadata, + DataFile, + branchFromContentKey, +} from 'netlify-cms-lib-util'; +import { Map } from 'immutable'; +import { dirname, basename } from 'path'; + +export const API_NAME = 'Azure DevOps'; + +const API_VERSION = 'api-version'; + +type AzureUser = { + coreAttributes?: { + Avatar?: { value?: { value?: string } }; + DisplayName?: { value?: string }; + EmailAddress?: { value?: string }; + }; +}; + +type AzureGitItem = { + objectId: string; + gitObjectType: AzureObjectType; + path: string; +}; + +// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/pull%20requests/get%20pull%20request?view=azure-devops-rest-6.1#gitpullrequest +type AzureWebApiTagDefinition = { + active: boolean; + id: string; + name: string; + url: string; +}; + +type AzurePullRequest = { + title: string; + artifactId: string; + closedDate: string; + creationDate: string; + isDraft: string; + status: AzurePullRequestStatus; + lastMergeSourceCommit: AzureGitChangeItem; + mergeStatus: AzureAsyncPullRequestStatus; + pullRequestId: number; + labels: AzureWebApiTagDefinition[]; + sourceRefName: string; +}; + +type AzurePullRequestCommit = { commitId: string }; + +enum AzureCommitStatusState { + ERROR = 'error', + FAILED = 'failed', + NOT_APPLICABLE = 'notApplicable', + NOT_SET = 'notSet', + PENDING = 'pending', + SUCCEEDED = 'succeeded', +} + +type AzureCommitStatus = { + context: { genre?: string | null; name: string }; + state: AzureCommitStatusState; + targetUrl: string; +}; + +// This does not match Azure documentation, but it is what comes back from some calls +// PullRequest as an example is documented as returning PullRequest[], but it actually +// returns that inside of this value prop in the json +interface AzureArray { + value: T[]; +} + +enum AzureCommitChangeType { + ADD = 'add', + DELETE = 'delete', + RENAME = 'rename', + EDIT = 'edit', +} + +enum AzureItemContentType { + BASE64 = 'base64encoded', +} + +enum AzurePullRequestStatus { + ACTIVE = 'active', + COMPLETED = 'completed', + ABANDONED = 'abandoned', +} + +enum AzureAsyncPullRequestStatus { + CONFLICTS = 'conflicts', + FAILURE = 'failure', + QUEUED = 'queued', + REJECTED = 'rejectedByPolicy', + SUCCEEDED = 'succeeded', +} + +enum AzureObjectType { + BLOB = 'blob', + TREE = 'tree', +} + +// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/diffs/get?view=azure-devops-rest-6.1#gitcommitdiffs +interface AzureGitCommitDiffs { + changes: AzureGitChange[]; +} + +// https://docs.microsoft.com/en-us/rest/api/azure/devops/git/diffs/get?view=azure-devops-rest-6.1#gitchange +interface AzureGitChange { + changeId: number; + item: AzureGitChangeItem; + changeType: AzureCommitChangeType; + originalPath: string; + url: string; +} + +interface AzureGitChangeItem { + objectId: string; + originalObjectId: string; + gitObjectType: string; + commitId: string; + path: string; + isFolder: string; + url: string; +} + +type AzureRef = { + name: string; + objectId: string; +}; + +type AzureCommit = { + author: { + date: string; + email: string; + name: string; + }; +}; + +function delay(ms: number) { + return new Promise(resolve => setTimeout(resolve, ms)); +} + +const getChangeItem = (item: AzureCommitItem) => { + switch (item.action) { + case AzureCommitChangeType.ADD: + return { + changeType: AzureCommitChangeType.ADD, + item: { path: item.path }, + newContent: { + content: item.base64Content, + contentType: AzureItemContentType.BASE64, + }, + }; + case AzureCommitChangeType.EDIT: + return { + changeType: AzureCommitChangeType.EDIT, + item: { path: item.path }, + newContent: { + content: item.base64Content, + contentType: AzureItemContentType.BASE64, + }, + }; + case AzureCommitChangeType.DELETE: + return { + changeType: AzureCommitChangeType.DELETE, + item: { path: item.path }, + }; + case AzureCommitChangeType.RENAME: + return { + changeType: AzureCommitChangeType.RENAME, + item: { path: item.path }, + sourceServerItem: item.oldPath, + }; + default: + return {}; + } +}; + +type AzureCommitItem = { + action: AzureCommitChangeType; + base64Content?: string; + text?: string; + path: string; + oldPath?: string; +}; + +interface AzureApiConfig { + apiRoot: string; + repo: { org: string; project: string; repoName: string }; + branch: string; + squashMerges: boolean; + initialWorkflowStatus: string; + cmsLabelPrefix: string; + apiVersion: string; +} + +export default class API { + apiVersion: string; + token: string; + branch: string; + mergeStrategy: string; + endpointUrl: string; + initialWorkflowStatus: string; + cmsLabelPrefix: string; + + constructor(config: AzureApiConfig, token: string) { + const { repo } = config; + const apiRoot = trim(config.apiRoot, '/'); + this.endpointUrl = `${apiRoot}/${repo.org}/${repo.project}/_apis/git/repositories/${repo.repoName}`; + this.token = token; + this.branch = config.branch; + this.mergeStrategy = config.squashMerges ? 'squash' : 'noFastForward'; + this.initialWorkflowStatus = config.initialWorkflowStatus; + this.apiVersion = config.apiVersion; + this.cmsLabelPrefix = config.cmsLabelPrefix; + } + + withHeaders = (req: ApiRequest) => { + const withHeaders = unsentRequest.withHeaders( + { + Authorization: `Bearer ${this.token}`, + 'Content-Type': 'application/json; charset=utf-8', + }, + req, + ); + return withHeaders; + }; + + withAzureFeatures = (req: Map>) => { + if (req.hasIn(['params', API_VERSION])) { + return req; + } + const withParams = unsentRequest.withParams( + { + [API_VERSION]: `${this.apiVersion}`, + }, + req, + ); + + return withParams; + }; + + buildRequest = (req: ApiRequest) => { + const withHeaders = this.withHeaders(req); + const withAzureFeatures = this.withAzureFeatures(withHeaders); + if (withAzureFeatures.has('cache')) { + return withAzureFeatures; + } else { + const withNoCache = unsentRequest.withNoCache(withAzureFeatures); + return withNoCache; + } + }; + + request = (req: ApiRequest): Promise => { + try { + return requestWithBackoff(this, req); + } catch (err) { + throw new APIError(err.message, null, API_NAME); + } + }; + + responseToJSON = responseParser({ format: 'json', apiName: API_NAME }); + responseToBlob = responseParser({ format: 'blob', apiName: API_NAME }); + responseToText = responseParser({ format: 'text', apiName: API_NAME }); + + requestJSON = (req: ApiRequest) => this.request(req).then(this.responseToJSON) as Promise; + requestText = (req: ApiRequest) => this.request(req).then(this.responseToText) as Promise; + + toBase64 = (str: string) => Promise.resolve(Base64.encode(str)); + fromBase64 = (str: string) => Base64.decode(str); + + branchToRef = (branch: string): string => `refs/heads/${branch}`; + refToBranch = (ref: string): string => ref.substr('refs/heads/'.length); + + user = async () => { + const result = await this.requestJSON({ + url: 'https://app.vssps.visualstudio.com/_apis/profile/profiles/me', + params: { [API_VERSION]: '6.1-preview.2' }, + }); + + const name = result.coreAttributes?.DisplayName?.value; + const email = result.coreAttributes?.EmailAddress?.value; + const url = result.coreAttributes?.Avatar?.value?.value; + const user = { + name: name || email || '', + // eslint-disable-next-line @typescript-eslint/camelcase + avatar_url: `data:image/png;base64,${url}`, + email, + }; + return user; + }; + + async readFileMetadata( + path: string, + sha: string | null | undefined, + { branch = this.branch } = {}, + ) { + const fetchFileMetadata = async () => { + try { + const { value } = await this.requestJSON>({ + url: `${this.endpointUrl}/commits/`, + params: { + 'searchCriteria.itemPath': path, + 'searchCriteria.itemVersion.version': branch, + 'searchCriteria.$top': 1, + }, + }); + const [commit] = value; + + return { + author: commit.author.name || commit.author.email, + updatedOn: commit.author.date, + }; + } catch (error) { + return { author: '', updatedOn: '' }; + } + }; + + const fileMetadata = await readFileMetadata(sha, fetchFileMetadata, localForage); + return fileMetadata; + } + + readFile = ( + path: string, + sha?: string | null, + { parseText = true, branch = this.branch } = {}, + ) => { + const fetchContent = () => { + return this.request({ + url: `${this.endpointUrl}/items/`, + params: { version: branch, path }, + cache: 'no-store', + }).then(parseText ? this.responseToText : this.responseToBlob); + }; + + return readFile(sha, fetchContent, localForage, parseText); + }; + + listFiles = async (path: string, recursive: boolean, branch = this.branch) => { + try { + const { value: items } = await this.requestJSON>({ + url: `${this.endpointUrl}/items/`, + params: { + version: branch, + scopePath: path, + recursionLevel: recursive ? 'full' : 'oneLevel', + }, + }); + + const files = items + .filter(item => item.gitObjectType === AzureObjectType.BLOB) + .map(file => ({ + id: file.objectId, + path: trimStart(file.path, '/'), + name: basename(file.path), + })); + return files; + } catch (err) { + if (err && err.status === 404) { + console.log('This 404 was expected and handled appropriately.'); + return []; + } else { + throw err; + } + } + }; + + async getRef(branch: string = this.branch) { + const { value: refs } = await this.requestJSON>({ + url: `${this.endpointUrl}/refs`, + params: { + $top: '1', // There's only one ref, so keep the payload small + filter: 'heads/' + branch, + }, + }); + + return refs.find(b => b.name == this.branchToRef(branch))!; + } + + async deleteRef(ref: AzureRef): Promise { + const deleteBranchPayload = [ + { + name: ref.name, + oldObjectId: ref.objectId, + newObjectId: '0000000000000000000000000000000000000000', + }, + ]; + + await this.requestJSON({ + method: 'POST', + url: `${this.endpointUrl}/refs`, + body: JSON.stringify(deleteBranchPayload), + }); + } + + async uploadAndCommit( + items: AzureCommitItem[], + comment: string, + branch: string, + newBranch: boolean, + ) { + const ref = await this.getRef(newBranch ? this.branch : branch); + + const refUpdate = [ + { + name: this.branchToRef(branch), + oldObjectId: ref.objectId, + }, + ]; + + const changes = items.map(item => getChangeItem(item)); + const commits = [{ comment, changes }]; + const push = { + refUpdates: refUpdate, + commits, + }; + + return this.requestJSON({ + url: `${this.endpointUrl}/pushes`, + method: 'POST', + body: JSON.stringify(push), + }); + } + + async retrieveUnpublishedEntryData(contentKey: string) { + const { collection, slug } = parseContentKey(contentKey); + const branch = branchFromContentKey(contentKey); + const pullRequest = await this.getBranchPullRequest(branch); + const diffs = await this.getDifferences(pullRequest.sourceRefName); + const diffsWithIds = await Promise.all( + diffs.map(async d => { + const path = trimStart(d.item.path, '/'); + const newFile = d.changeType === AzureCommitChangeType.ADD; + const id = d.item.objectId; + return { id, path, newFile }; + }), + ); + const label = pullRequest.labels.find(l => isCMSLabel(l.name, this.cmsLabelPrefix)); + const labelName = label && label.name ? label.name : this.cmsLabelPrefix; + const status = labelToStatus(labelName, this.cmsLabelPrefix); + // Uses creationDate, as we do not have direct access to the updated date + const updatedAt = pullRequest.closedDate ? pullRequest.closedDate : pullRequest.creationDate; + return { + collection, + slug, + status, + diffs: diffsWithIds, + updatedAt, + }; + } + + async getPullRequestStatues(pullRequest: AzurePullRequest) { + const { value: commits } = await this.requestJSON>({ + url: `${this.endpointUrl}/pullrequests/${pullRequest.pullRequestId}/commits`, + params: { + $top: 1, + }, + }); + const { value: statuses } = await this.requestJSON>({ + url: `${this.endpointUrl}/commits/${commits[0].commitId}/statuses`, + params: { latestOnly: true }, + }); + return statuses; + } + + async getStatuses(collection: string, slug: string) { + const contentKey = generateContentKey(collection, slug); + const branch = branchFromContentKey(contentKey); + const pullRequest = await this.getBranchPullRequest(branch); + const statuses = await this.getPullRequestStatues(pullRequest); + return statuses.map(({ context, state, targetUrl }) => ({ + context: context.name, + state: state === AzureCommitStatusState.SUCCEEDED ? PreviewState.Success : PreviewState.Other, + // eslint-disable-next-line @typescript-eslint/camelcase + target_url: targetUrl, + })); + } + + async getCommitItems(files: { path: string; newPath?: string }[], branch: string) { + const items = await Promise.all( + files.map(async file => { + const [base64Content, fileExists] = await Promise.all([ + result(file, 'toBase64', partial(this.toBase64, (file as DataFile).raw)), + this.isFileExists(file.path, branch), + ]); + + const path = file.newPath || file.path; + const oldPath = file.path; + const renameOrEdit = + path !== oldPath ? AzureCommitChangeType.RENAME : AzureCommitChangeType.EDIT; + + const action = fileExists ? renameOrEdit : AzureCommitChangeType.ADD; + return { + action, + base64Content, + path, + oldPath, + } as AzureCommitItem; + }), + ); + + // move children + for (const item of items.filter(i => i.oldPath && i.action === AzureCommitChangeType.RENAME)) { + const sourceDir = dirname(item.oldPath as string); + const destDir = dirname(item.path); + const children = await this.listFiles(sourceDir, true, branch); + children + .filter(file => file.path !== item.oldPath) + .forEach(file => { + items.push({ + action: AzureCommitChangeType.RENAME, + path: file.path.replace(sourceDir, destDir), + oldPath: file.path, + }); + }); + } + + return items; + } + + async persistFiles(dataFiles: DataFile[], mediaFiles: AssetProxy[], options: PersistOptions) { + const files = [...dataFiles, ...mediaFiles]; + if (options.useWorkflow) { + const slug = dataFiles[0].slug; + return this.editorialWorkflowGit(files, slug, options); + } else { + const items = await this.getCommitItems(files, this.branch); + + return this.uploadAndCommit(items, options.commitMessage, this.branch, true); + } + } + + async deleteFiles(paths: string[], comment: string) { + const ref = await this.getRef(this.branch); + const refUpdate = { + name: ref.name, + oldObjectId: ref.objectId, + }; + + const changes = paths.map(path => + getChangeItem({ action: AzureCommitChangeType.DELETE, path }), + ); + const commits = [{ comment, changes }]; + const push = { + refUpdates: [refUpdate], + commits, + }; + + return this.requestJSON({ + url: `${this.endpointUrl}/pushes`, + method: 'POST', + body: JSON.stringify(push), + }); + } + + async getPullRequests(sourceBranch?: string) { + const { value: pullRequests } = await this.requestJSON>({ + url: `${this.endpointUrl}/pullrequests`, + params: { + 'searchCriteria.status': 'active', + 'searchCriteria.targetRefName': this.branchToRef(this.branch), + 'searchCriteria.includeLinks': false, + ...(sourceBranch ? { 'searchCriteria.sourceRefName': this.branchToRef(sourceBranch) } : {}), + }, + }); + + const filtered = pullRequests.filter(pr => { + return pr.labels.some(label => isCMSLabel(label.name, this.cmsLabelPrefix)); + }); + return filtered; + } + + async listUnpublishedBranches(): Promise { + const pullRequests = await this.getPullRequests(); + const branches = pullRequests.map(pr => this.refToBranch(pr.sourceRefName)); + return branches; + } + + async isFileExists(path: string, branch: string) { + try { + await this.requestText({ + url: `${this.endpointUrl}/items/`, + params: { version: branch, path }, + cache: 'no-store', + }); + return true; + } catch (error) { + if (error instanceof APIError && error.status === 404) { + return false; + } + throw error; + } + } + + async createPullRequest(branch: string, commitMessage: string, status: string) { + const pr = { + sourceRefName: this.branchToRef(branch), + targetRefName: this.branchToRef(this.branch), + title: commitMessage, + description: DEFAULT_PR_BODY, + labels: [ + { + name: statusToLabel(status, this.cmsLabelPrefix), + }, + ], + }; + + await this.requestJSON({ + method: 'POST', + url: `${this.endpointUrl}/pullrequests`, + params: { + supportsIterations: false, + }, + body: JSON.stringify(pr), + }); + } + + async getBranchPullRequest(branch: string) { + const pullRequests = await this.getPullRequests(branch); + + if (pullRequests.length <= 0) { + throw new EditorialWorkflowError('content is not under editorial workflow', true); + } + + return pullRequests[0]; + } + + async getDifferences(to: string) { + const result = await this.requestJSON({ + url: `${this.endpointUrl}/diffs/commits`, + params: { + baseVersion: this.branch, + targetVersion: this.refToBranch(to), + }, + }); + + return result.changes.filter( + d => + d.item.gitObjectType === AzureObjectType.BLOB && + Object.values(AzureCommitChangeType).includes(d.changeType), + ); + } + + async editorialWorkflowGit( + files: (DataFile | AssetProxy)[], + slug: string, + options: PersistOptions, + ) { + const contentKey = generateContentKey(options.collectionName as string, slug); + const branch = branchFromContentKey(contentKey); + const unpublished = options.unpublished || false; + + if (!unpublished) { + const items = await this.getCommitItems(files, this.branch); + + await this.uploadAndCommit(items, options.commitMessage, branch, true); + await this.createPullRequest( + branch, + options.commitMessage, + options.status || this.initialWorkflowStatus, + ); + } else { + const items = await this.getCommitItems(files, branch); + await this.uploadAndCommit(items, options.commitMessage, branch, false); + } + } + + async updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) { + const contentKey = generateContentKey(collection, slug); + const branch = branchFromContentKey(contentKey); + + const pullRequest = await this.getBranchPullRequest(branch); + + const nonCmsLabels = pullRequest.labels + .filter(label => !isCMSLabel(label.name, this.cmsLabelPrefix)) + .map(label => label.name); + + const labels = [...nonCmsLabels, statusToLabel(newStatus, this.cmsLabelPrefix)]; + await this.updatePullRequestLabels(pullRequest, labels); + } + + async deleteUnpublishedEntry(collectionName: string, slug: string) { + const contentKey = generateContentKey(collectionName, slug); + const branch = branchFromContentKey(contentKey); + const pullRequest = await this.getBranchPullRequest(branch); + await this.abandonPullRequest(pullRequest); + } + + async publishUnpublishedEntry(collectionName: string, slug: string) { + const contentKey = generateContentKey(collectionName, slug); + const branch = branchFromContentKey(contentKey); + const pullRequest = await this.getBranchPullRequest(branch); + await this.completePullRequest(pullRequest); + } + + async updatePullRequestLabels(pullRequest: AzurePullRequest, labels: string[]) { + const cmsLabels = pullRequest.labels.filter(l => isCMSLabel(l.name, this.cmsLabelPrefix)); + await Promise.all( + cmsLabels.map(l => { + return this.requestText({ + method: 'DELETE', + url: `${this.endpointUrl}/pullrequests/${encodeURIComponent( + pullRequest.pullRequestId, + )}/labels/${encodeURIComponent(l.id)}`, + }); + }), + ); + + await Promise.all( + labels.map(l => { + return this.requestText({ + method: 'POST', + url: `${this.endpointUrl}/pullrequests/${encodeURIComponent( + pullRequest.pullRequestId, + )}/labels`, + body: JSON.stringify({ name: l }), + }); + }), + ); + } + + async completePullRequest(pullRequest: AzurePullRequest) { + const pullRequestCompletion = { + status: AzurePullRequestStatus.COMPLETED, + lastMergeSourceCommit: pullRequest.lastMergeSourceCommit, + completionOptions: { + deleteSourceBranch: true, + mergeCommitMessage: MERGE_COMMIT_MESSAGE, + mergeStrategy: this.mergeStrategy, + }, + }; + + let response = await this.requestJSON({ + method: 'PATCH', + url: `${this.endpointUrl}/pullrequests/${encodeURIComponent(pullRequest.pullRequestId)}`, + body: JSON.stringify(pullRequestCompletion), + }); + + // We need to wait for Azure to complete the pull request to actually complete + // Sometimes this is instant, but frequently it is 1-3 seconds + const DELAY_MILLISECONDS = 500; + const MAX_ATTEMPTS = 10; + let attempt = 1; + while (response.mergeStatus === AzureAsyncPullRequestStatus.QUEUED && attempt <= MAX_ATTEMPTS) { + await delay(DELAY_MILLISECONDS); + response = await this.requestJSON({ + url: `${this.endpointUrl}/pullrequests/${encodeURIComponent(pullRequest.pullRequestId)}`, + }); + attempt = attempt + 1; + } + } + + async abandonPullRequest(pullRequest: AzurePullRequest) { + const pullRequestAbandon = { + status: AzurePullRequestStatus.ABANDONED, + }; + + await this.requestJSON({ + method: 'PATCH', + url: `${this.endpointUrl}/pullrequests/${encodeURIComponent(pullRequest.pullRequestId)}`, + body: JSON.stringify(pullRequestAbandon), + }); + + await this.deleteRef({ + name: pullRequest.sourceRefName, + objectId: pullRequest.lastMergeSourceCommit.commitId, + }); + } +} diff --git a/packages/netlify-cms-backend-azure/src/AuthenticationPage.js b/packages/netlify-cms-backend-azure/src/AuthenticationPage.js new file mode 100644 index 00000000..7ed93e08 --- /dev/null +++ b/packages/netlify-cms-backend-azure/src/AuthenticationPage.js @@ -0,0 +1,79 @@ +import React from 'react'; +import PropTypes from 'prop-types'; +import styled from '@emotion/styled'; +import { ImplicitAuthenticator } from 'netlify-cms-lib-auth'; +import { AuthenticationPage, Icon } from 'netlify-cms-ui-default'; + +const LoginButtonIcon = styled(Icon)` + margin-right: 18px; +`; + +export default class AzureAuthenticationPage extends React.Component { + static propTypes = { + onLogin: PropTypes.func.isRequired, + inProgress: PropTypes.bool, + base_url: PropTypes.string, + siteId: PropTypes.string, + authEndpoint: PropTypes.string, + config: PropTypes.object.isRequired, + clearHash: PropTypes.func, + t: PropTypes.func.isRequired, + }; + + state = {}; + + componentDidMount() { + this.auth = new ImplicitAuthenticator({ + base_url: `https://login.microsoftonline.com/${this.props.config.backend.tenant_id}`, + auth_endpoint: 'oauth2/authorize', + app_id: this.props.config.backend.app_id, + clearHash: this.props.clearHash, + }); + // Complete implicit authentication if we were redirected back to from the provider. + this.auth.completeAuth((err, data) => { + if (err) { + alert(err); + return; + } + this.props.onLogin(data); + }); + } + + handleLogin = e => { + e.preventDefault(); + this.auth.authenticate( + { + scope: 'vso.code_full,user.read', + resource: '499b84ac-1321-427f-aa17-267ca6975798', + prompt: 'select_account', + }, + (err, data) => { + if (err) { + this.setState({ loginError: err.toString() }); + return; + } + this.props.onLogin(data); + }, + ); + }; + + render() { + const { inProgress, config, t } = this.props; + + return ( + ( + + + {inProgress ? t('auth.loggingIn') : t('auth.loginWithAzure')} + + )} + t={t} + /> + ); + } +} diff --git a/packages/netlify-cms-backend-azure/src/implementation.ts b/packages/netlify-cms-backend-azure/src/implementation.ts new file mode 100644 index 00000000..b7c2792c --- /dev/null +++ b/packages/netlify-cms-backend-azure/src/implementation.ts @@ -0,0 +1,378 @@ +import { trimStart, trim } from 'lodash'; +import semaphore, { Semaphore } from 'semaphore'; +import AuthenticationPage from './AuthenticationPage'; +import API, { API_NAME } from './API'; +import { + Credentials, + Implementation, + ImplementationFile, + ImplementationMediaFile, + DisplayURL, + basename, + Entry, + AssetProxy, + PersistOptions, + getMediaDisplayURL, + generateContentKey, + getMediaAsBlob, + Config, + getPreviewStatus, + asyncLock, + AsyncLock, + runWithLock, + User, + unpublishedEntries, + UnpublishedEntryMediaFile, + entriesByFiles, + filterByExtension, + branchFromContentKey, + entriesByFolder, + contentKeyFromBranch, + getBlobSHA, +} from 'netlify-cms-lib-util'; + +const MAX_CONCURRENT_DOWNLOADS = 10; + +const parseAzureRepo = (config: Config) => { + const { repo } = config.backend; + + if (typeof repo !== 'string') { + throw new Error('The Azure backend needs a "repo" in the backend configuration.'); + } + + const parts = repo.split('/'); + if (parts.length !== 3) { + throw new Error('The Azure backend must be in a the format of {org}/{project}/{repo}'); + } + + const [org, project, repoName] = parts; + return { + org, + project, + repoName, + }; +}; + +export default class Azure implements Implementation { + lock: AsyncLock; + api?: API; + options: { + initialWorkflowStatus: string; + }; + repo: { + org: string; + project: string; + repoName: string; + }; + branch: string; + apiRoot: string; + apiVersion: string; + token: string | null; + squashMerges: boolean; + cmsLabelPrefix: string; + mediaFolder: string; + previewContext: string; + + _mediaDisplayURLSem?: Semaphore; + + constructor(config: Config, options = {}) { + this.options = { + initialWorkflowStatus: '', + ...options, + }; + + this.repo = parseAzureRepo(config); + this.branch = config.backend.branch || 'master'; + this.apiRoot = config.backend.api_root || 'https://dev.azure.com'; + this.apiVersion = config.backend.api_version || '6.1-preview'; + this.token = ''; + this.squashMerges = config.backend.squash_merges || false; + this.cmsLabelPrefix = config.backend.cms_label_prefix || ''; + this.mediaFolder = trim(config.media_folder, '/'); + this.previewContext = config.backend.preview_context || ''; + this.lock = asyncLock(); + } + + isGitBackend() { + return true; + } + + async status() { + const auth = + (await this.api!.user() + .then(user => !!user) + .catch(e => { + console.warn('Failed getting Azure user', e); + return false; + })) || false; + + return { auth: { status: auth }, api: { status: true, statusPage: '' } }; + } + + authComponent() { + return AuthenticationPage; + } + + restoreUser(user: User) { + return this.authenticate(user); + } + + async authenticate(state: Credentials) { + this.token = state.token as string; + this.api = new API( + { + apiRoot: this.apiRoot, + apiVersion: this.apiVersion, + repo: this.repo, + branch: this.branch, + squashMerges: this.squashMerges, + cmsLabelPrefix: this.cmsLabelPrefix, + initialWorkflowStatus: this.options.initialWorkflowStatus, + }, + this.token, + ); + + const user = await this.api.user(); + return { token: state.token as string, ...user }; + } + + /** + * Log the user out by forgetting their access token. + * TODO: *Actual* logout by redirecting to: + * https://login.microsoftonline.com/{tenantId}/oauth2/logout?client_id={clientId}&post_logout_redirect_uri={baseUrl} + */ + logout() { + this.token = null; + return; + } + + getToken() { + return Promise.resolve(this.token); + } + + async entriesByFolder(folder: string, extension: string, depth: number) { + const listFiles = async () => { + const files = await this.api!.listFiles(folder, depth > 1); + const filtered = files.filter(file => filterByExtension({ path: file.path }, extension)); + return filtered.map(file => ({ + id: file.id, + path: file.path, + })); + }; + + const entries = await entriesByFolder( + listFiles, + this.api!.readFile.bind(this.api!), + this.api!.readFileMetadata.bind(this.api), + API_NAME, + ); + return entries; + } + + entriesByFiles(files: ImplementationFile[]) { + return entriesByFiles( + files, + this.api!.readFile.bind(this.api!), + this.api!.readFileMetadata.bind(this.api), + API_NAME, + ); + } + + async getEntry(path: string) { + const data = (await this.api!.readFile(path)) as string; + return { + file: { path }, + data, + }; + } + + async getMedia() { + const files = await this.api!.listFiles(this.mediaFolder, false); + const mediaFiles = await Promise.all( + files.map(async ({ id, path, name }) => { + const blobUrl = await this.getMediaDisplayURL({ id, path }); + return { id, name, displayURL: blobUrl, path }; + }), + ); + return mediaFiles; + } + + getMediaDisplayURL(displayURL: DisplayURL) { + this._mediaDisplayURLSem = this._mediaDisplayURLSem || semaphore(MAX_CONCURRENT_DOWNLOADS); + return getMediaDisplayURL( + displayURL, + this.api!.readFile.bind(this.api!), + this._mediaDisplayURLSem, + ); + } + + async getMediaFile(path: string) { + const name = basename(path); + const blob = await getMediaAsBlob(path, null, this.api!.readFile.bind(this.api!)); + const fileObj = new File([blob], name); + const url = URL.createObjectURL(fileObj); + const id = await getBlobSHA(blob); + + return { + id, + displayURL: url, + path, + name, + size: fileObj.size, + file: fileObj, + url, + }; + } + + async persistEntry(entry: Entry, options: PersistOptions): Promise { + const mediaFiles: AssetProxy[] = entry.assets; + await this.api!.persistFiles(entry.dataFiles, mediaFiles, options); + } + + async persistMedia( + mediaFile: AssetProxy, + options: PersistOptions, + ): Promise { + const fileObj = mediaFile.fileObj as File; + + const [id] = await Promise.all([ + getBlobSHA(fileObj), + this.api!.persistFiles([], [mediaFile], options), + ]); + + const { path } = mediaFile; + const url = URL.createObjectURL(fileObj); + + return { + displayURL: url, + path: trimStart(path, '/'), + name: fileObj!.name, + size: fileObj!.size, + file: fileObj, + url, + id: id as string, + }; + } + + async deleteFiles(paths: string[], commitMessage: string) { + await this.api!.deleteFiles(paths, commitMessage); + } + + async loadMediaFile(branch: string, file: UnpublishedEntryMediaFile) { + const readFile = ( + path: string, + id: string | null | undefined, + { parseText }: { parseText: boolean }, + ) => this.api!.readFile(path, id, { branch, parseText }); + + const blob = await getMediaAsBlob(file.path, null, readFile); + const name = basename(file.path); + const fileObj = new File([blob], name); + return { + id: file.path, + displayURL: URL.createObjectURL(fileObj), + path: file.path, + name, + size: fileObj.size, + file: fileObj, + }; + } + + async loadEntryMediaFiles(branch: string, files: UnpublishedEntryMediaFile[]) { + const mediaFiles = await Promise.all(files.map(file => this.loadMediaFile(branch, file))); + + return mediaFiles; + } + + async unpublishedEntries() { + const listEntriesKeys = () => + this.api!.listUnpublishedBranches().then(branches => + branches.map(branch => contentKeyFromBranch(branch)), + ); + + const ids = await unpublishedEntries(listEntriesKeys); + return ids; + } + + async unpublishedEntry({ + id, + collection, + slug, + }: { + id?: string; + collection?: string; + slug?: string; + }) { + if (id) { + const data = await this.api!.retrieveUnpublishedEntryData(id); + return data; + } else if (collection && slug) { + const contentKey = generateContentKey(collection, slug); + const data = await this.api!.retrieveUnpublishedEntryData(contentKey); + return data; + } else { + throw new Error('Missing unpublished entry id or collection and slug'); + } + } + + getBranch(collection: string, slug: string) { + const contentKey = generateContentKey(collection, slug); + const branch = branchFromContentKey(contentKey); + return branch; + } + + async unpublishedEntryMediaFile(collection: string, slug: string, path: string, id: string) { + const branch = this.getBranch(collection, slug); + const mediaFile = await this.loadMediaFile(branch, { path, id }); + return mediaFile; + } + + async unpublishedEntryDataFile(collection: string, slug: string, path: string, id: string) { + const branch = this.getBranch(collection, slug); + const data = (await this.api!.readFile(path, id, { branch })) as string; + return data; + } + + updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) { + // updateUnpublishedEntryStatus is a transactional operation + return runWithLock( + this.lock, + () => this.api!.updateUnpublishedEntryStatus(collection, slug, newStatus), + 'Failed to acquire update entry status lock', + ); + } + + deleteUnpublishedEntry(collection: string, slug: string) { + // deleteUnpublishedEntry is a transactional operation + return runWithLock( + this.lock, + () => this.api!.deleteUnpublishedEntry(collection, slug), + 'Failed to acquire delete entry lock', + ); + } + + publishUnpublishedEntry(collection: string, slug: string) { + // publishUnpublishedEntry is a transactional operation + return runWithLock( + this.lock, + () => this.api!.publishUnpublishedEntry(collection, slug), + 'Failed to acquire publish entry lock', + ); + } + + async getDeployPreview(collection: string, slug: string) { + try { + const statuses = await this.api!.getStatuses(collection, slug); + const deployStatus = getPreviewStatus(statuses, this.previewContext); + + if (deployStatus) { + const { target_url: url, state } = deployStatus; + return { url, status: state }; + } else { + return null; + } + } catch (e) { + return null; + } + } +} diff --git a/packages/netlify-cms-backend-azure/src/index.ts b/packages/netlify-cms-backend-azure/src/index.ts new file mode 100644 index 00000000..3bcae7a6 --- /dev/null +++ b/packages/netlify-cms-backend-azure/src/index.ts @@ -0,0 +1,10 @@ +import AzureBackend from './implementation'; +import API from './API'; +import AuthenticationPage from './AuthenticationPage'; + +export const NetlifyCmsBackendAzure = { + AzureBackend, + API, + AuthenticationPage, +}; +export { AzureBackend, API, AuthenticationPage }; diff --git a/packages/netlify-cms-backend-azure/webpack.config.js b/packages/netlify-cms-backend-azure/webpack.config.js new file mode 100644 index 00000000..42edd361 --- /dev/null +++ b/packages/netlify-cms-backend-azure/webpack.config.js @@ -0,0 +1,3 @@ +const { getConfig } = require('../../scripts/webpack.js'); + +module.exports = getConfig(); diff --git a/packages/netlify-cms-core/index.d.ts b/packages/netlify-cms-core/index.d.ts index 496c5e9a..395dd2c3 100644 --- a/packages/netlify-cms-core/index.d.ts +++ b/packages/netlify-cms-core/index.d.ts @@ -3,7 +3,13 @@ declare module 'netlify-cms-core' { import React, { ComponentType } from 'react'; import { List, Map } from 'immutable'; - export type CmsBackendType = 'git-gateway' | 'github' | 'gitlab' | 'bitbucket' | 'test-repo'; + export type CmsBackendType = + | 'azure' + | 'git-gateway' + | 'github' + | 'gitlab' + | 'bitbucket' + | 'test-repo'; export type CmsMapWidgetType = 'Point' | 'LineString' | 'Polygon'; diff --git a/packages/netlify-cms-lib-auth/src/implicit-oauth.js b/packages/netlify-cms-lib-auth/src/implicit-oauth.js index 5bb1bf9a..1d394f38 100644 --- a/packages/netlify-cms-lib-auth/src/implicit-oauth.js +++ b/packages/netlify-cms-lib-auth/src/implicit-oauth.js @@ -42,7 +42,16 @@ export default class ImplicitAuthenticator { authURL.searchParams.set('response_type', 'token'); authURL.searchParams.set('scope', options.scope); + if (options.prompt != null && options.prompt != undefined) { + authURL.searchParams.set('prompt', options.prompt); + } + + if (options.resource != null && options.resource != undefined) { + authURL.searchParams.set('resource', options.resource); + } + const state = JSON.stringify({ auth_type: 'implicit', nonce: createNonce() }); + authURL.searchParams.set('state', state); document.location.assign(authURL.href); diff --git a/packages/netlify-cms-lib-util/src/API.ts b/packages/netlify-cms-lib-util/src/API.ts index f8ad4a8c..0f773101 100644 --- a/packages/netlify-cms-lib-util/src/API.ts +++ b/packages/netlify-cms-lib-util/src/API.ts @@ -15,7 +15,7 @@ interface API { export type ApiRequestObject = { url: string; params?: Record; - method?: 'POST' | 'PUT' | 'DELETE' | 'HEAD'; + method?: 'POST' | 'PUT' | 'DELETE' | 'HEAD' | 'PATCH'; headers?: Record; body?: string | FormData; cache?: 'no-store'; diff --git a/packages/netlify-cms-lib-util/src/implementation.ts b/packages/netlify-cms-lib-util/src/implementation.ts index 345e518c..350d53a4 100644 --- a/packages/netlify-cms-lib-util/src/implementation.ts +++ b/packages/netlify-cms-lib-util/src/implementation.ts @@ -106,6 +106,7 @@ export type Config = { auth_type?: string; app_id?: string; cms_label_prefix?: string; + api_version?: string; }; media_folder: string; base_url?: string; diff --git a/packages/netlify-cms-locales/src/en/index.js b/packages/netlify-cms-locales/src/en/index.js index e2ed293d..e13c78ac 100644 --- a/packages/netlify-cms-locales/src/en/index.js +++ b/packages/netlify-cms-locales/src/en/index.js @@ -3,6 +3,7 @@ const en = { login: 'Login', loggingIn: 'Logging in...', loginWithNetlifyIdentity: 'Login with Netlify Identity', + loginWithAzure: 'Login with Azure', loginWithBitbucket: 'Login with Bitbucket', loginWithGitHub: 'Login with GitHub', loginWithGitLab: 'Login with GitLab', diff --git a/packages/netlify-cms-locales/src/nl/index.js b/packages/netlify-cms-locales/src/nl/index.js index 562157ad..95b07b25 100644 --- a/packages/netlify-cms-locales/src/nl/index.js +++ b/packages/netlify-cms-locales/src/nl/index.js @@ -3,6 +3,7 @@ const nl = { login: 'Inloggen', loggingIn: 'Inloggen...', loginWithNetlifyIdentity: 'Inloggen met Netlify Identity', + loginWithAzure: 'Inloggen met Azure', loginWithBitbucket: 'Inloggen met Bitbucket', loginWithGitHub: 'Inloggen met GitHub', loginWithGitLab: 'Inloggen met GitLab', diff --git a/packages/netlify-cms-locales/src/tr/index.js b/packages/netlify-cms-locales/src/tr/index.js index e79c9e9f..2eceaaf9 100644 --- a/packages/netlify-cms-locales/src/tr/index.js +++ b/packages/netlify-cms-locales/src/tr/index.js @@ -3,6 +3,7 @@ const tr = { login: 'Giriş', loggingIn: 'Giriş yapılıyor..', loginWithNetlifyIdentity: 'Netlify Identity ile Giriş', + loginWithAzure: 'Azure ile Giriş', loginWithBitbucket: 'Bitbucket ile Giriş', loginWithGitHub: 'GitHub ile Giriş', loginWithGitLab: 'GitLab ile Giriş', diff --git a/packages/netlify-cms-ui-default/src/Icon/images/_index.js b/packages/netlify-cms-ui-default/src/Icon/images/_index.js index 450d656f..f774b2d1 100644 --- a/packages/netlify-cms-ui-default/src/Icon/images/_index.js +++ b/packages/netlify-cms-ui-default/src/Icon/images/_index.js @@ -1,6 +1,7 @@ import iconAdd from './add.svg'; import iconAddWith from './add-with.svg'; import iconArrow from './arrow.svg'; +import iconAzure from './azure.svg'; import iconBitbucket from './bitbucket.svg'; import iconBold from './bold.svg'; import iconCheck from './check.svg'; @@ -50,6 +51,7 @@ const images = { add: iconix, 'add-with': iconAddWith, arrow: iconArrow, + azure: iconAzure, bitbucket: iconBitbucket, bold: iconBold, check: iconCheck, diff --git a/packages/netlify-cms-ui-default/src/Icon/images/azure.svg b/packages/netlify-cms-ui-default/src/Icon/images/azure.svg new file mode 100644 index 00000000..0e8f1242 --- /dev/null +++ b/packages/netlify-cms-ui-default/src/Icon/images/azure.svg @@ -0,0 +1,9 @@ + + + + diff --git a/website/content/docs/azure-backend.md b/website/content/docs/azure-backend.md new file mode 100644 index 00000000..1814803c --- /dev/null +++ b/website/content/docs/azure-backend.md @@ -0,0 +1,32 @@ +--- +group: Accounts +weight: 20 +title: Azure +--- + +For repositories stored on Azure, the `azure` backend allows CMS users to log in directly with their Azure account. Note that all users must have write access to your content repository for this to work. + +In order to get Netlify-CMS working with Azure DevOps, you need a Tenant Id and an Application Id. + +1. If you do not have an Azure account, [create one here](https://azure.microsoft.com/en-us/free/?WT.mc_id=A261C142F) and make sure to have a credit card linked to the account. +2. If you do not have an Azure Active Directory Tenant Id, [set one up here](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-create-new-tenant). +3. [Register an application with Azure AD](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-register-app). Configure it as a Single tenant Web application and add a redirect URI (e.g. `http://localhost:8080/`) +4. Add the `Azure DevOps->user_impersonation` [permission](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-configure-app-access-web-apis#add-permissions-to-access-your-web-api) for the created application. +5. [Grant admin consent](https://docs.microsoft.com/en-us/azure/active-directory/develop/quickstart-configure-app-access-web-apis#admin-consent-button) for the application. +6. Under `Authentication->Implicit grant` enable [Access tokens](https://docs.microsoft.com/en-us/azure/active-directory/develop/access-tokens) for the application and click `Save`. +7. Verify your Azure DevOps organization is connected to the same directory as your tenant under: `https://dev.azure.com//_settings/organizationAad` +8. Add the following lines to your Netlify CMS `config.yml` file: + +```yaml +backend: + name: azure + repo: organization/project/repo # replace with actual path + tenant_id: tenantId # replace with your tenantId + app_id: appId # replace with your appId +``` + +### Limitations + +1. Pagination is not supported so some endpoints might return missing data + +2. Nested collection are partially supported as Azure doesn't allow [renaming and editing](https://docs.microsoft.com/en-us/rest/api/azure/devops/git/pushes/create?view=azure-devops-rest-6.1&source=docs#rename-a-file) in a single operation