From 1523a4140a3d2f4cc01a1548514ae17bc1ad504e Mon Sep 17 00:00:00 2001 From: Erez Rokah Date: Tue, 28 Dec 2021 12:39:23 +0100 Subject: [PATCH] feat(backend-gitlab): initial GraphQL support (#6059) --- .../src/GitLabAPI.ts | 4 +- .../netlify-cms-backend-gitlab/package.json | 4 + .../netlify-cms-backend-gitlab/src/API.ts | 156 +++++++++++++++++- .../src/implementation.ts | 8 + .../netlify-cms-backend-gitlab/src/queries.ts | 73 ++++++++ .../src/implementation.ts | 10 +- website/content/docs/beta-features.md | 26 ++- 7 files changed, 274 insertions(+), 7 deletions(-) create mode 100644 packages/netlify-cms-backend-gitlab/src/queries.ts diff --git a/packages/netlify-cms-backend-git-gateway/src/GitLabAPI.ts b/packages/netlify-cms-backend-git-gateway/src/GitLabAPI.ts index f737e6fe..43292dab 100644 --- a/packages/netlify-cms-backend-git-gateway/src/GitLabAPI.ts +++ b/packages/netlify-cms-backend-git-gateway/src/GitLabAPI.ts @@ -1,10 +1,10 @@ import { API as GitlabAPI } from 'netlify-cms-backend-gitlab'; import { unsentRequest } from 'netlify-cms-lib-util'; -import type { Config as GitHubConfig, CommitAuthor } from 'netlify-cms-backend-gitlab/src/API'; +import type { Config as GitLabConfig, CommitAuthor } from 'netlify-cms-backend-gitlab/src/API'; import type { ApiRequest } from 'netlify-cms-lib-util'; -type Config = GitHubConfig & { tokenPromise: () => Promise; commitAuthor: CommitAuthor }; +type Config = GitLabConfig & { tokenPromise: () => Promise; commitAuthor: CommitAuthor }; export default class API extends GitlabAPI { tokenPromise: () => Promise; diff --git a/packages/netlify-cms-backend-gitlab/package.json b/packages/netlify-cms-backend-gitlab/package.json index d5acd37e..bb9ce23b 100644 --- a/packages/netlify-cms-backend-gitlab/package.json +++ b/packages/netlify-cms-backend-gitlab/package.json @@ -20,6 +20,10 @@ "build:esm": "cross-env NODE_ENV=esm babel src --out-dir dist/esm --ignore \"**/__tests__\" --root-mode upward --extensions \".js,.jsx,.ts,.tsx\"" }, "dependencies": { + "apollo-cache-inmemory": "^1.6.2", + "apollo-client": "^2.6.3", + "apollo-link-context": "^1.0.18", + "apollo-link-http": "^1.5.15", "js-base64": "^3.0.0", "semaphore": "^1.1.0" }, diff --git a/packages/netlify-cms-backend-gitlab/src/API.ts b/packages/netlify-cms-backend-gitlab/src/API.ts index 32b50894..8dc88f8d 100644 --- a/packages/netlify-cms-backend-gitlab/src/API.ts +++ b/packages/netlify-cms-backend-gitlab/src/API.ts @@ -1,3 +1,7 @@ +import { ApolloClient } from 'apollo-client'; +import { InMemoryCache } from 'apollo-cache-inmemory'; +import { createHttpLink } from 'apollo-link-http'; +import { setContext } from 'apollo-link-context'; import { localForage, parseLinkHeader, @@ -27,24 +31,32 @@ import { Map } from 'immutable'; import { flow, partial, result, trimStart } from 'lodash'; import { dirname } from 'path'; +const NO_CACHE = 'no-cache'; +import * as queries from './queries'; + +import type { ApolloQueryResult } from 'apollo-client'; +import type { NormalizedCacheObject } from 'apollo-cache-inmemory'; import type { ApiRequest, DataFile, AssetProxy, PersistOptions, FetchError, + ImplementationFile, } from 'netlify-cms-lib-util'; export const API_NAME = 'GitLab'; export interface Config { apiRoot?: string; + graphQLAPIRoot?: string; token?: string; branch?: string; repo?: string; squashMerges: boolean; initialWorkflowStatus: string; cmsLabelPrefix: string; + useGraphQL?: boolean; } export interface CommitAuthor { @@ -66,6 +78,8 @@ type CommitItem = { action: CommitAction; }; +type FileEntry = { id: string; type: string; path: string; name: string }; + interface CommitsParams { commit_message: string; branch: string; @@ -183,8 +197,16 @@ export function getMaxAccess(groups: { group_access_level: number }[]) { }, groups[0]); } +function batch(items: T[], maxPerBatch: number, action: (items: T[]) => void) { + for (let index = 0; index < items.length; index = index + maxPerBatch) { + const itemsSlice = items.slice(index, index + maxPerBatch); + action(itemsSlice); + } +} + export default class API { apiRoot: string; + graphQLAPIRoot: string; token: string | boolean; branch: string; useOpenAuthoring?: boolean; @@ -195,8 +217,11 @@ export default class API { initialWorkflowStatus: string; cmsLabelPrefix: string; + graphQLClient?: ApolloClient; + constructor(config: Config) { this.apiRoot = config.apiRoot || 'https://gitlab.com/api/v4'; + this.graphQLAPIRoot = config.graphQLAPIRoot || 'https://gitlab.com/api/graphql'; this.token = config.token || false; this.branch = config.branch || 'master'; this.repo = config.repo || ''; @@ -204,6 +229,40 @@ export default class API { this.squashMerges = config.squashMerges; this.initialWorkflowStatus = config.initialWorkflowStatus; this.cmsLabelPrefix = config.cmsLabelPrefix; + if (config.useGraphQL === true) { + this.graphQLClient = this.getApolloClient(); + } + } + + getApolloClient() { + const authLink = setContext((_, { headers }) => { + return { + headers: { + 'Content-Type': 'application/json; charset=utf-8', + ...headers, + authorization: this.token ? `token ${this.token}` : '', + }, + }; + }); + const httpLink = createHttpLink({ uri: this.graphQLAPIRoot }); + return new ApolloClient({ + link: authLink.concat(httpLink), + cache: new InMemoryCache(), + defaultOptions: { + watchQuery: { + fetchPolicy: NO_CACHE, + errorPolicy: 'ignore', + }, + query: { + fetchPolicy: NO_CACHE, + errorPolicy: 'all', + }, + }, + }); + } + + reset() { + return this.graphQLClient?.resetStore(); } withAuthorizationHeaders = (req: ApiRequest) => { @@ -352,7 +411,7 @@ export default class API { fetchCursorAndEntries = ( req: ApiRequest, ): Promise<{ - entries: { id: string; type: string; path: string; name: string }[]; + entries: FileEntry[]; cursor: Cursor; }> => flow([ @@ -392,7 +451,102 @@ export default class API { }; }; + listAllFilesGraphQL = async (path: string, recursive: boolean, branch: String) => { + const files: FileEntry[] = []; + let blobsPaths; + let cursor; + do { + blobsPaths = await this.graphQLClient!.query({ + query: queries.files, + variables: { repo: this.repo, branch, path, recursive, cursor }, + }); + files.push(...blobsPaths.data.project.repository.tree.blobs.nodes); + cursor = blobsPaths.data.project.repository.tree.blobs.pageInfo.endCursor; + } while (blobsPaths.data.project.repository.tree.blobs.pageInfo.hasNextPage); + + return files; + }; + + readFilesGraphQL = async (files: ImplementationFile[]) => { + const paths = files.map(({ path }) => path); + + type BlobResult = { + project: { repository: { blobs: { nodes: { id: string; data: string }[] } } }; + }; + + const blobPromises: Promise>[] = []; + batch(paths, 90, slice => { + blobPromises.push( + this.graphQLClient!.query({ + query: queries.blobs, + variables: { + repo: this.repo, + branch: this.branch, + paths: slice, + }, + fetchPolicy: 'cache-first', + }), + ); + }); + + type LastCommit = { + id: string; + authoredDate: string; + authorName: string; + author?: { + name: string; + username: string; + publicEmail: string; + }; + }; + + type CommitResult = { + project: { repository: { [tree: string]: { lastCommit: LastCommit } } }; + }; + + const commitPromises: Promise>[] = []; + batch(paths, 8, slice => { + commitPromises.push( + this.graphQLClient!.query({ + query: queries.lastCommits(slice), + variables: { + repo: this.repo, + branch: this.branch, + }, + fetchPolicy: 'cache-first', + }), + ); + }); + + const [blobsResults, commitsResults] = await Promise.all([ + (await Promise.all(blobPromises)).map(result => result.data.project.repository.blobs.nodes), + ( + await Promise.all(commitPromises) + ).map( + result => + Object.values(result.data.project.repository) + .map(({ lastCommit }) => lastCommit) + .filter(Boolean) as LastCommit[], + ), + ]); + + const blobs = blobsResults.flat().map(result => result.data) as string[]; + const metadata = commitsResults.flat().map(({ author, authoredDate, authorName }) => ({ + author: author ? author.name || author.username || author.publicEmail : authorName, + updatedOn: authoredDate, + })); + + const filesWithData = files.map((file, index) => ({ + file: { ...file, ...metadata[index] }, + data: blobs[index], + })); + return filesWithData; + }; + listAllFiles = async (path: string, recursive = false, branch = this.branch) => { + if (this.graphQLClient) { + return await this.listAllFilesGraphQL(path, recursive, branch); + } const entries = []; // eslint-disable-next-line prefer-const let { cursor, entries: initialEntries } = await this.fetchCursorAndEntries({ diff --git a/packages/netlify-cms-backend-gitlab/src/implementation.ts b/packages/netlify-cms-backend-gitlab/src/implementation.ts index b6b9127c..bf9df133 100644 --- a/packages/netlify-cms-backend-gitlab/src/implementation.ts +++ b/packages/netlify-cms-backend-gitlab/src/implementation.ts @@ -60,6 +60,8 @@ export default class GitLab implements Implementation { cmsLabelPrefix: string; mediaFolder: string; previewContext: string; + useGraphQL: boolean; + graphQLAPIRoot: string; _mediaDisplayURLSem?: Semaphore; @@ -88,6 +90,8 @@ export default class GitLab implements Implementation { this.cmsLabelPrefix = config.backend.cms_label_prefix || ''; this.mediaFolder = config.media_folder; this.previewContext = config.backend.preview_context || ''; + this.useGraphQL = config.backend.use_graphql || false; + this.graphQLAPIRoot = config.backend.graphql_api_root || 'https://gitlab.com/api/graphql'; this.lock = asyncLock(); } @@ -126,6 +130,8 @@ export default class GitLab implements Implementation { squashMerges: this.squashMerges, cmsLabelPrefix: this.cmsLabelPrefix, initialWorkflowStatus: this.options.initialWorkflowStatus, + useGraphQL: this.useGraphQL, + graphQLAPIRoot: this.graphQLAPIRoot, }); const user = await this.api.user(); const isCollab = await this.api.hasWriteAccess().catch((error: Error) => { @@ -212,7 +218,9 @@ export default class GitLab implements Implementation { getDifferences: (to, from) => this.api!.getDifferences(to, from), getFileId: path => this.api!.getFileId(path, this.branch), filterFile: file => this.filterFile(folder, file, extension, depth), + customFetch: this.useGraphQL ? files => this.api!.readFilesGraphQL(files) : undefined, }); + return files; } diff --git a/packages/netlify-cms-backend-gitlab/src/queries.ts b/packages/netlify-cms-backend-gitlab/src/queries.ts new file mode 100644 index 00000000..fb6e0201 --- /dev/null +++ b/packages/netlify-cms-backend-gitlab/src/queries.ts @@ -0,0 +1,73 @@ +import { gql } from 'graphql-tag'; +import { oneLine } from 'common-tags'; + +export const files = gql` + query files($repo: ID!, $branch: String!, $path: String!, $recursive: Boolean!, $cursor: String) { + project(fullPath: $repo) { + repository { + tree(ref: $branch, path: $path, recursive: $recursive) { + blobs(after: $cursor) { + nodes { + type + id: sha + path + name + } + pageInfo { + endCursor + hasNextPage + } + } + } + } + } + } +`; + +export const blobs = gql` + query blobs($repo: ID!, $branch: String!, $paths: [String!]!) { + project(fullPath: $repo) { + repository { + blobs(ref: $branch, paths: $paths) { + nodes { + id + data: rawBlob + } + } + } + } + } +`; + +export function lastCommits(paths: string[]) { + const tree = paths + .map( + (path, index) => oneLine` + tree${index}: tree(ref: $branch, path: "${path}") { + lastCommit { + authorName + authoredDate + author { + id + username + name + publicEmail + } + } + } + `, + ) + .join('\n'); + + const query = gql` + query lastCommits($repo: ID!, $branch: String!) { + project(fullPath: $repo) { + repository { + ${tree} + } + } + } +`; + + return query; +} diff --git a/packages/netlify-cms-lib-util/src/implementation.ts b/packages/netlify-cms-lib-util/src/implementation.ts index cb2e62b5..45542382 100644 --- a/packages/netlify-cms-lib-util/src/implementation.ts +++ b/packages/netlify-cms-lib-util/src/implementation.ts @@ -102,6 +102,7 @@ export type Config = { api_root?: string; squash_merges?: boolean; use_graphql?: boolean; + graphql_api_root?: string; preview_context?: string; identity_url?: string; gateway_url?: string; @@ -205,6 +206,8 @@ type ReadFile = ( type ReadFileMetadata = (path: string, id: string | null | undefined) => Promise; +type CustomFetchFunc = (files: ImplementationFile[]) => Promise; + async function fetchFiles( files: ImplementationFile[], readFile: ReadFile, @@ -461,6 +464,7 @@ type AllEntriesByFolderArgs = GetKeyArgs & isShaExistsInBranch: (branch: string, sha: string) => Promise; apiName: string; localForage: LocalForage; + customFetch?: CustomFetchFunc; }; export async function allEntriesByFolder({ @@ -478,6 +482,7 @@ export async function allEntriesByFolder({ getDifferences, getFileId, filterFile, + customFetch, }: AllEntriesByFolderArgs) { async function listAllFilesAndPersist() { const files = await listAllFiles(folder, extension, depth); @@ -561,5 +566,8 @@ export async function allEntriesByFolder({ } const files = await listFiles(); - return fetchFiles(files, readFile, readFileMetadata, apiName); + if (customFetch) { + return await customFetch(files); + } + return await fetchFiles(files, readFile, readFileMetadata, apiName); } diff --git a/website/content/docs/beta-features.md b/website/content/docs/beta-features.md index 256f89d2..741b3816 100644 --- a/website/content/docs/beta-features.md +++ b/website/content/docs/beta-features.md @@ -193,9 +193,9 @@ collections: Experimental support for GitHub's [GraphQL API](https://developer.github.com/v4/) is now available for the GitHub backend. -**Note: not currently compatible with Git Gateway.** +**Note: not compatible with Git Gateway.** -For many queries, GraphQL allows data to be retrieved using less individual API requests compared to a REST API. GitHub's GraphQL API still does not support all mutations necessary to completely replace their REST API, so this feature only calls the new GraphQL API where possible. +GraphQL allows to retrieve data using less individual API requests compared to a REST API. GitHub's GraphQL API still does not support all mutations necessary to completely replace their REST API, so this feature only calls the new GraphQL API where possible. You can use the GraphQL API for the GitHub backend by setting `backend.use_graphql` to `true` in your CMS config: @@ -208,6 +208,26 @@ backend: Learn more about the benefits of GraphQL in the [GraphQL docs](https://graphql.org). +## GitLab GraphQL API + +Experimental support for GitLab's [GraphQL API](https://docs.gitlab.com/ee/api/graphql/) is now available for the GitLab backend. + +**Note: not compatible with Git Gateway.** + +GraphQL allows to retrieve data using less individual API requests compared to a REST API. +The current implementation uses the GraphQL API in specific cases, where using the REST API can be slow and lead to exceeding GitLab's rate limits. As we receive feedback and extend the feature, we'll migrate more functionality to the GraphQL API. + +You can enable the GraphQL API for the GitLab backend by setting `backend.use_graphql` to `true` in your CMS config: + +```yml +backend: + name: gitlab + repo: owner/repo # replace this with your repo info + use_graphql: true + + # optional, defaults to 'https://gitlab.com/api/graphql'. Can be used to configured a self hosted GitLab instance. + graphql_api_root: https://my-self-hosted-gitlab.com/api/graphql +``` ## Open Authoring When using the [GitHub backend](/docs/github-backend), you can use Netlify CMS to accept contributions from GitHub users without giving them access to your repository. When they make changes in the CMS, the CMS forks your repository for them behind the scenes, and all the changes are made to the fork. When the contributor is ready to submit their changes, they can set their draft as ready for review in the CMS. This triggers a pull request to your repository, which you can merge using the GitHub UI. @@ -661,4 +681,4 @@ CMS.registerRemarkPlugin(plugin); CMS.registerRemarkPlugin({ settings: { bullet: '-' } }); ``` -Note that `netlify-widget-markdown` currently uses `remark@10`, so you should check a plugin's compatibility first. \ No newline at end of file +Note that `netlify-widget-markdown` currently uses `remark@10`, so you should check a plugin's compatibility first.