884 lines
26 KiB
TypeScript

import {
localForage,
parseLinkHeader,
unsentRequest,
then,
APIError,
Cursor,
ApiRequest,
DataFile,
AssetProxy,
PersistOptions,
readFile,
CMS_BRANCH_PREFIX,
generateContentKey,
isCMSLabel,
EditorialWorkflowError,
labelToStatus,
statusToLabel,
DEFAULT_PR_BODY,
MERGE_COMMIT_MESSAGE,
responseParser,
PreviewState,
parseContentKey,
branchFromContentKey,
requestWithBackoff,
readFileMetadata,
FetchError,
throwOnConflictingBranches,
} from 'netlify-cms-lib-util';
import { Base64 } from 'js-base64';
import { Map } from 'immutable';
import { flow, partial, result, trimStart } from 'lodash';
import { dirname } from 'path';
export const API_NAME = 'GitLab';
export interface Config {
apiRoot?: string;
token?: string;
branch?: string;
repo?: string;
squashMerges: boolean;
initialWorkflowStatus: string;
cmsLabelPrefix: string;
}
export interface CommitAuthor {
name: string;
email: string;
}
enum CommitAction {
CREATE = 'create',
DELETE = 'delete',
MOVE = 'move',
UPDATE = 'update',
}
type CommitItem = {
base64Content?: string;
path: string;
oldPath?: string;
action: CommitAction;
};
interface CommitsParams {
commit_message: string;
branch: string;
author_name?: string;
author_email?: string;
actions?: {
action: string;
file_path: string;
previous_path?: string;
content?: string;
encoding?: string;
}[];
}
type GitLabCommitDiff = {
diff: string;
new_path: string;
old_path: string;
new_file: boolean;
renamed_file: boolean;
deleted_file: boolean;
};
enum GitLabCommitStatuses {
Pending = 'pending',
Running = 'running',
Success = 'success',
Failed = 'failed',
Canceled = 'canceled',
}
type GitLabCommitStatus = {
status: GitLabCommitStatuses;
name: string;
author: {
username: string;
name: string;
};
description: null;
sha: string;
ref: string;
target_url: string;
};
type GitLabMergeRebase = {
rebase_in_progress: boolean;
merge_error: string;
};
type GitLabMergeRequest = {
id: number;
iid: number;
title: string;
description: string;
state: string;
merged_by: {
name: string;
username: string;
};
merged_at: string;
created_at: string;
updated_at: string;
target_branch: string;
source_branch: string;
author: {
name: string;
username: string;
};
labels: string[];
sha: string;
};
type GitLabRepo = {
shared_with_groups: { group_access_level: number }[] | null;
permissions: {
project_access: { access_level: number } | null;
group_access: { access_level: number } | null;
};
};
type GitLabBranch = {
name: string;
developers_can_push: boolean;
developers_can_merge: boolean;
commit: {
id: string;
};
};
type GitLabCommitRef = {
type: string;
name: string;
};
type GitLabCommit = {
id: string;
short_id: string;
title: string;
author_name: string;
author_email: string;
authored_date: string;
committer_name: string;
committer_email: string;
committed_date: string;
created_at: string;
message: string;
};
export const getMaxAccess = (groups: { group_access_level: number }[]) => {
return groups.reduce((previous, current) => {
if (current.group_access_level > previous.group_access_level) {
return current;
}
return previous;
}, groups[0]);
};
export default class API {
apiRoot: string;
token: string | boolean;
branch: string;
useOpenAuthoring?: boolean;
repo: string;
repoURL: string;
commitAuthor?: CommitAuthor;
squashMerges: boolean;
initialWorkflowStatus: string;
cmsLabelPrefix: string;
constructor(config: Config) {
this.apiRoot = config.apiRoot || 'https://gitlab.com/api/v4';
this.token = config.token || false;
this.branch = config.branch || 'master';
this.repo = config.repo || '';
this.repoURL = `/projects/${encodeURIComponent(this.repo)}`;
this.squashMerges = config.squashMerges;
this.initialWorkflowStatus = config.initialWorkflowStatus;
this.cmsLabelPrefix = config.cmsLabelPrefix;
}
withAuthorizationHeaders = (req: ApiRequest) => {
const withHeaders = unsentRequest.withHeaders(
this.token ? { Authorization: `Bearer ${this.token}` } : {},
req,
);
return Promise.resolve(withHeaders);
};
buildRequest = async (req: ApiRequest) => {
const withRoot: ApiRequest = unsentRequest.withRoot(this.apiRoot)(req);
const withAuthorizationHeaders = await this.withAuthorizationHeaders(withRoot);
if (withAuthorizationHeaders.has('cache')) {
return withAuthorizationHeaders;
} else {
const withNoCache: ApiRequest = unsentRequest.withNoCache(withAuthorizationHeaders);
return withNoCache;
}
};
request = async (req: ApiRequest): Promise<Response> => {
try {
return requestWithBackoff(this, req);
} catch (err) {
throw new APIError(err.message, null, API_NAME);
}
};
responseToJSON = responseParser({ format: 'json', apiName: API_NAME });
responseToBlob = responseParser({ format: 'blob', apiName: API_NAME });
responseToText = responseParser({ format: 'text', apiName: API_NAME });
// eslint-disable-next-line @typescript-eslint/no-explicit-any
requestJSON = (req: ApiRequest) => this.request(req).then(this.responseToJSON) as Promise<any>;
requestText = (req: ApiRequest) => this.request(req).then(this.responseToText) as Promise<string>;
user = () => this.requestJSON('/user');
WRITE_ACCESS = 30;
MAINTAINER_ACCESS = 40;
hasWriteAccess = async () => {
const {
shared_with_groups: sharedWithGroups,
permissions,
}: GitLabRepo = await this.requestJSON(this.repoURL);
const { project_access: projectAccess, group_access: groupAccess } = permissions;
if (projectAccess && projectAccess.access_level >= this.WRITE_ACCESS) {
return true;
}
if (groupAccess && groupAccess.access_level >= this.WRITE_ACCESS) {
return true;
}
// check for group write permissions
if (sharedWithGroups && sharedWithGroups.length > 0) {
const maxAccess = getMaxAccess(sharedWithGroups);
// maintainer access
if (maxAccess.group_access_level >= this.MAINTAINER_ACCESS) {
return true;
}
// developer access
if (maxAccess.group_access_level >= this.WRITE_ACCESS) {
// check permissions to merge and push
try {
const branch = await this.getDefaultBranch();
if (branch.developers_can_merge && branch.developers_can_push) {
return true;
}
} catch (e) {
console.log('Failed getting default branch', e);
}
}
}
return false;
};
readFile = async (
path: string,
sha?: string | null,
{ parseText = true, branch = this.branch } = {},
): Promise<string | Blob> => {
const fetchContent = async () => {
const content = await this.request({
url: `${this.repoURL}/repository/files/${encodeURIComponent(path)}/raw`,
params: { ref: branch },
cache: 'no-store',
}).then<Blob | string>(parseText ? this.responseToText : this.responseToBlob);
return content;
};
const content = await readFile(sha, fetchContent, localForage, parseText);
return content;
};
async readFileMetadata(path: string, sha: string | null | undefined) {
const fetchFileMetadata = async () => {
try {
const result: GitLabCommit[] = await this.requestJSON({
url: `${this.repoURL}/repository/commits`,
// eslint-disable-next-line @typescript-eslint/camelcase
params: { path, ref_name: this.branch },
});
const commit = result[0];
return {
author: commit.author_name || commit.author_email,
updatedOn: commit.authored_date,
};
} catch (e) {
return { author: '', updatedOn: '' };
}
};
const fileMetadata = await readFileMetadata(sha, fetchFileMetadata, localForage);
return fileMetadata;
}
getCursorFromHeaders = (headers: Headers) => {
const page = parseInt(headers.get('X-Page') as string, 10);
const pageCount = parseInt(headers.get('X-Total-Pages') as string, 10);
const pageSize = parseInt(headers.get('X-Per-Page') as string, 10);
const count = parseInt(headers.get('X-Total') as string, 10);
const links = parseLinkHeader(headers.get('Link'));
const actions = Map(links)
.keySeq()
.flatMap(key =>
(key === 'prev' && page > 1) ||
(key === 'next' && page < pageCount) ||
(key === 'first' && page > 1) ||
(key === 'last' && page < pageCount)
? [key]
: [],
);
return Cursor.create({
actions,
meta: { page, count, pageSize, pageCount },
data: { links },
});
};
getCursor = ({ headers }: { headers: Headers }) => this.getCursorFromHeaders(headers);
// Gets a cursor without retrieving the entries by using a HEAD
// request
fetchCursor = (req: ApiRequest) =>
flow([unsentRequest.withMethod('HEAD'), this.request, then(this.getCursor)])(req);
fetchCursorAndEntries = (
req: ApiRequest,
): Promise<{
entries: { id: string; type: string; path: string; name: string }[];
cursor: Cursor;
}> =>
flow([
unsentRequest.withMethod('GET'),
this.request,
p =>
Promise.all([
p.then(this.getCursor),
p.then(this.responseToJSON).catch((e: FetchError) => {
if (e.status === 404) {
return [];
} else {
throw e;
}
}),
]),
then(([cursor, entries]: [Cursor, {}[]]) => ({ cursor, entries })),
])(req);
listFiles = async (path: string, recursive = false) => {
const { entries, cursor } = await this.fetchCursorAndEntries({
url: `${this.repoURL}/repository/tree`,
params: { path, ref: this.branch, recursive },
});
return {
files: entries.filter(({ type }) => type === 'blob'),
cursor,
};
};
traverseCursor = async (cursor: Cursor, action: string) => {
const link = cursor.data!.getIn(['links', action]);
const { entries, cursor: newCursor } = await this.fetchCursorAndEntries(link);
return {
entries: entries.filter(({ type }) => type === 'blob'),
cursor: newCursor,
};
};
listAllFiles = async (path: string, recursive = false, branch = this.branch) => {
const entries = [];
// eslint-disable-next-line prefer-const
let { cursor, entries: initialEntries } = await this.fetchCursorAndEntries({
url: `${this.repoURL}/repository/tree`,
// Get the maximum number of entries per page
// eslint-disable-next-line @typescript-eslint/camelcase
params: { path, ref: branch, per_page: 100, recursive },
});
entries.push(...initialEntries);
while (cursor && cursor.actions!.has('next')) {
const link = cursor.data!.getIn(['links', 'next']);
const { cursor: newCursor, entries: newEntries } = await this.fetchCursorAndEntries(link);
entries.push(...newEntries);
cursor = newCursor;
}
return entries.filter(({ type }) => type === 'blob');
};
toBase64 = (str: string) => Promise.resolve(Base64.encode(str));
fromBase64 = (str: string) => Base64.decode(str);
async getBranch(branchName: string) {
const branch: GitLabBranch = await this.requestJSON(
`${this.repoURL}/repository/branches/${encodeURIComponent(branchName)}`,
);
return branch;
}
async uploadAndCommit(
items: CommitItem[],
{ commitMessage = '', branch = this.branch, newBranch = false },
) {
const actions = items.map(item => ({
action: item.action,
// eslint-disable-next-line @typescript-eslint/camelcase
file_path: item.path,
// eslint-disable-next-line @typescript-eslint/camelcase
...(item.oldPath ? { previous_path: item.oldPath } : {}),
...(item.base64Content !== undefined
? { content: item.base64Content, encoding: 'base64' }
: {}),
}));
const commitParams: CommitsParams = {
branch,
// eslint-disable-next-line @typescript-eslint/camelcase
commit_message: commitMessage,
actions,
// eslint-disable-next-line @typescript-eslint/camelcase
...(newBranch ? { start_branch: this.branch } : {}),
};
if (this.commitAuthor) {
const { name, email } = this.commitAuthor;
// eslint-disable-next-line @typescript-eslint/camelcase
commitParams.author_name = name;
// eslint-disable-next-line @typescript-eslint/camelcase
commitParams.author_email = email;
}
try {
const result = await this.requestJSON({
url: `${this.repoURL}/repository/commits`,
method: 'POST',
headers: { 'Content-Type': 'application/json; charset=utf-8' },
body: JSON.stringify(commitParams),
});
return result;
} catch (error) {
const message = error.message || '';
if (newBranch && message.includes(`Could not update ${branch}`)) {
await throwOnConflictingBranches(branch, name => this.getBranch(name), API_NAME);
}
throw error;
}
}
async getCommitItems(files: { path: string; newPath?: string }[], branch: string) {
const items: CommitItem[] = await Promise.all(
files.map(async file => {
const [base64Content, fileExists] = await Promise.all([
result(file, 'toBase64', partial(this.toBase64, (file as DataFile).raw)),
this.isFileExists(file.path, branch),
]);
let action = CommitAction.CREATE;
let path = trimStart(file.path, '/');
let oldPath = undefined;
if (fileExists) {
oldPath = file.newPath && path;
action =
file.newPath && file.newPath !== oldPath ? CommitAction.MOVE : CommitAction.UPDATE;
path = file.newPath ? trimStart(file.newPath, '/') : path;
}
return {
action,
base64Content,
path,
oldPath,
};
}),
);
// move children
for (const item of items.filter(i => i.oldPath && i.action === CommitAction.MOVE)) {
const sourceDir = dirname(item.oldPath as string);
const destDir = dirname(item.path);
const children = await this.listAllFiles(sourceDir, true, branch);
children
.filter(f => f.path !== item.oldPath)
.forEach(file => {
items.push({
action: CommitAction.MOVE,
path: file.path.replace(sourceDir, destDir),
oldPath: file.path,
});
});
}
return items;
}
async persistFiles(dataFiles: DataFile[], mediaFiles: AssetProxy[], options: PersistOptions) {
const files = [...dataFiles, ...mediaFiles];
if (options.useWorkflow) {
const slug = dataFiles[0].slug;
return this.editorialWorkflowGit(files, slug, options);
} else {
const items = await this.getCommitItems(files, this.branch);
return this.uploadAndCommit(items, {
commitMessage: options.commitMessage,
});
}
}
deleteFiles = (paths: string[], commitMessage: string) => {
const branch = this.branch;
// eslint-disable-next-line @typescript-eslint/camelcase
const commitParams: CommitsParams = { commit_message: commitMessage, branch };
if (this.commitAuthor) {
const { name, email } = this.commitAuthor;
// eslint-disable-next-line @typescript-eslint/camelcase
commitParams.author_name = name;
// eslint-disable-next-line @typescript-eslint/camelcase
commitParams.author_email = email;
}
const items = paths.map(path => ({ path, action: CommitAction.DELETE }));
return this.uploadAndCommit(items, {
commitMessage,
});
};
async getMergeRequests(sourceBranch?: string) {
const mergeRequests: GitLabMergeRequest[] = await this.requestJSON({
url: `${this.repoURL}/merge_requests`,
params: {
state: 'opened',
labels: 'Any',
// eslint-disable-next-line @typescript-eslint/camelcase
target_branch: this.branch,
// eslint-disable-next-line @typescript-eslint/camelcase
...(sourceBranch ? { source_branch: sourceBranch } : {}),
},
});
return mergeRequests.filter(
mr =>
mr.source_branch.startsWith(CMS_BRANCH_PREFIX) &&
mr.labels.some(l => isCMSLabel(l, this.cmsLabelPrefix)),
);
}
async listUnpublishedBranches() {
console.log(
'%c Checking for Unpublished entries',
'line-height: 30px;text-align: center;font-weight: bold',
);
const mergeRequests = await this.getMergeRequests();
const branches = mergeRequests.map(mr => mr.source_branch);
return branches;
}
async getFileId(path: string, branch: string) {
const request = await this.request({
method: 'HEAD',
url: `${this.repoURL}/repository/files/${encodeURIComponent(path)}`,
params: { ref: branch },
});
const blobId = request.headers.get('X-Gitlab-Blob-Id') as string;
return blobId;
}
async isFileExists(path: string, branch: string) {
const fileExists = await this.requestText({
method: 'HEAD',
url: `${this.repoURL}/repository/files/${encodeURIComponent(path)}`,
params: { ref: branch },
})
.then(() => true)
.catch(error => {
if (error instanceof APIError && error.status === 404) {
return false;
}
throw error;
});
return fileExists;
}
async getBranchMergeRequest(branch: string) {
const mergeRequests = await this.getMergeRequests(branch);
if (mergeRequests.length <= 0) {
throw new EditorialWorkflowError('content is not under editorial workflow', true);
}
return mergeRequests[0];
}
async getDifferences(to: string, from = this.branch) {
if (to === from) {
return [];
}
const result: { diffs: GitLabCommitDiff[] } = await this.requestJSON({
url: `${this.repoURL}/repository/compare`,
params: {
from,
to,
},
});
if (result.diffs.length >= 1000) {
throw new APIError('Diff limit reached', null, API_NAME);
}
return result.diffs.map(d => {
let status = 'modified';
if (d.new_file) {
status = 'added';
} else if (d.deleted_file) {
status = 'deleted';
} else if (d.renamed_file) {
status = 'renamed';
}
return {
status,
oldPath: d.old_path,
newPath: d.new_path,
newFile: d.new_file,
path: d.new_path || d.old_path,
binary: d.diff.startsWith('Binary') || /.svg$/.test(d.new_path),
};
});
}
async retrieveUnpublishedEntryData(contentKey: string) {
const { collection, slug } = parseContentKey(contentKey);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
const diffs = await this.getDifferences(mergeRequest.sha);
const diffsWithIds = await Promise.all(
diffs.map(async d => {
const { path, newFile } = d;
const id = await this.getFileId(path, branch);
return { id, path, newFile };
}),
);
const label = mergeRequest.labels.find(l => isCMSLabel(l, this.cmsLabelPrefix)) as string;
const status = labelToStatus(label, this.cmsLabelPrefix);
const updatedAt = mergeRequest.updated_at;
return {
collection,
slug,
status,
diffs: diffsWithIds,
updatedAt,
};
}
async rebaseMergeRequest(mergeRequest: GitLabMergeRequest) {
let rebase: GitLabMergeRebase = await this.requestJSON({
method: 'PUT',
url: `${this.repoURL}/merge_requests/${mergeRequest.iid}/rebase?skip_ci=true`,
});
let i = 1;
while (rebase.rebase_in_progress) {
await new Promise(resolve => setTimeout(resolve, 1000));
rebase = await this.requestJSON({
url: `${this.repoURL}/merge_requests/${mergeRequest.iid}`,
params: {
// eslint-disable-next-line @typescript-eslint/camelcase
include_rebase_in_progress: true,
},
});
if (!rebase.rebase_in_progress || i > 30) {
break;
}
i++;
}
if (rebase.rebase_in_progress) {
throw new APIError('Timed out rebasing merge request', null, API_NAME);
} else if (rebase.merge_error) {
throw new APIError(`Rebase error: ${rebase.merge_error}`, null, API_NAME);
}
}
async createMergeRequest(branch: string, commitMessage: string, status: string) {
await this.requestJSON({
method: 'POST',
url: `${this.repoURL}/merge_requests`,
params: {
// eslint-disable-next-line @typescript-eslint/camelcase
source_branch: branch,
// eslint-disable-next-line @typescript-eslint/camelcase
target_branch: this.branch,
title: commitMessage,
description: DEFAULT_PR_BODY,
labels: statusToLabel(status, this.cmsLabelPrefix),
// eslint-disable-next-line @typescript-eslint/camelcase
remove_source_branch: true,
squash: this.squashMerges,
},
});
}
async editorialWorkflowGit(
files: (DataFile | AssetProxy)[],
slug: string,
options: PersistOptions,
) {
const contentKey = generateContentKey(options.collectionName as string, slug);
const branch = branchFromContentKey(contentKey);
const unpublished = options.unpublished || false;
if (!unpublished) {
const items = await this.getCommitItems(files, this.branch);
await this.uploadAndCommit(items, {
commitMessage: options.commitMessage,
branch,
newBranch: true,
});
await this.createMergeRequest(
branch,
options.commitMessage,
options.status || this.initialWorkflowStatus,
);
} else {
const mergeRequest = await this.getBranchMergeRequest(branch);
await this.rebaseMergeRequest(mergeRequest);
const [items, diffs] = await Promise.all([
this.getCommitItems(files, branch),
this.getDifferences(branch),
]);
// mark files for deletion
for (const diff of diffs.filter(d => d.binary)) {
if (!items.some(item => item.path === diff.path)) {
items.push({ action: CommitAction.DELETE, path: diff.newPath });
}
}
await this.uploadAndCommit(items, {
commitMessage: options.commitMessage,
branch,
});
}
}
async updateMergeRequestLabels(mergeRequest: GitLabMergeRequest, labels: string[]) {
await this.requestJSON({
method: 'PUT',
url: `${this.repoURL}/merge_requests/${mergeRequest.iid}`,
params: {
labels: labels.join(','),
},
});
}
async updateUnpublishedEntryStatus(collection: string, slug: string, newStatus: string) {
const contentKey = generateContentKey(collection, slug);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
const labels = [
...mergeRequest.labels.filter(label => !isCMSLabel(label, this.cmsLabelPrefix)),
statusToLabel(newStatus, this.cmsLabelPrefix),
];
await this.updateMergeRequestLabels(mergeRequest, labels);
}
async mergeMergeRequest(mergeRequest: GitLabMergeRequest) {
await this.requestJSON({
method: 'PUT',
url: `${this.repoURL}/merge_requests/${mergeRequest.iid}/merge`,
params: {
// eslint-disable-next-line @typescript-eslint/camelcase
merge_commit_message: MERGE_COMMIT_MESSAGE,
// eslint-disable-next-line @typescript-eslint/camelcase
squash_commit_message: MERGE_COMMIT_MESSAGE,
squash: this.squashMerges,
// eslint-disable-next-line @typescript-eslint/camelcase
should_remove_source_branch: true,
},
});
}
async publishUnpublishedEntry(collectionName: string, slug: string) {
const contentKey = generateContentKey(collectionName, slug);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
await this.mergeMergeRequest(mergeRequest);
}
async closeMergeRequest(mergeRequest: GitLabMergeRequest) {
await this.requestJSON({
method: 'PUT',
url: `${this.repoURL}/merge_requests/${mergeRequest.iid}`,
params: {
// eslint-disable-next-line @typescript-eslint/camelcase
state_event: 'close',
},
});
}
async getDefaultBranch() {
const branch: GitLabBranch = await this.getBranch(this.branch);
return branch;
}
async isShaExistsInBranch(branch: string, sha: string) {
const refs: GitLabCommitRef[] = await this.requestJSON({
url: `${this.repoURL}/repository/commits/${sha}/refs`,
params: {
type: 'branch',
},
});
return refs.some(r => r.name === branch);
}
async deleteBranch(branch: string) {
await this.request({
method: 'DELETE',
url: `${this.repoURL}/repository/branches/${encodeURIComponent(branch)}`,
});
}
async deleteUnpublishedEntry(collectionName: string, slug: string) {
const contentKey = generateContentKey(collectionName, slug);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
await this.closeMergeRequest(mergeRequest);
await this.deleteBranch(branch);
}
async getMergeRequestStatues(mergeRequest: GitLabMergeRequest, branch: string) {
const statuses: GitLabCommitStatus[] = await this.requestJSON({
url: `${this.repoURL}/repository/commits/${mergeRequest.sha}/statuses`,
params: {
ref: branch,
},
});
return statuses;
}
async getStatuses(collectionName: string, slug: string) {
const contentKey = generateContentKey(collectionName, slug);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
const statuses: GitLabCommitStatus[] = await this.getMergeRequestStatues(mergeRequest, branch);
// eslint-disable-next-line @typescript-eslint/camelcase
return statuses.map(({ name, status, target_url }) => ({
context: name,
state: status === GitLabCommitStatuses.Success ? PreviewState.Success : PreviewState.Other,
// eslint-disable-next-line @typescript-eslint/camelcase
target_url,
}));
}
async getUnpublishedEntrySha(collection: string, slug: string) {
const contentKey = generateContentKey(collection, slug);
const branch = branchFromContentKey(contentKey);
const mergeRequest = await this.getBranchMergeRequest(branch);
return mergeRequest.sha;
}
}