feat(backend-gitlab): initial GraphQL support (#6059)

packages/netlify-cms-backend-git-gateway/src/GitLabAPI.ts

@@ -1,10 +1,10 @@
 import { API as GitlabAPI } from 'netlify-cms-backend-gitlab';
 import { unsentRequest } from 'netlify-cms-lib-util';
 
-import type { Config as GitHubConfig, CommitAuthor } from 'netlify-cms-backend-gitlab/src/API';
+import type { Config as GitLabConfig, CommitAuthor } from 'netlify-cms-backend-gitlab/src/API';
 import type { ApiRequest } from 'netlify-cms-lib-util';
 
-type Config = GitHubConfig & { tokenPromise: () => Promise<string>; commitAuthor: CommitAuthor };
+type Config = GitLabConfig & { tokenPromise: () => Promise<string>; commitAuthor: CommitAuthor };
 
 export default class API extends GitlabAPI {
   tokenPromise: () => Promise<string>;

packages/netlify-cms-backend-gitlab/package.json

@@ -20,6 +20,10 @@
     "build:esm": "cross-env NODE_ENV=esm babel src --out-dir dist/esm --ignore \"**/__tests__\" --root-mode upward --extensions \".js,.jsx,.ts,.tsx\""
   },
   "dependencies": {
+    "apollo-cache-inmemory": "^1.6.2",
+    "apollo-client": "^2.6.3",
+    "apollo-link-context": "^1.0.18",
+    "apollo-link-http": "^1.5.15",
     "js-base64": "^3.0.0",
     "semaphore": "^1.1.0"
   },

packages/netlify-cms-backend-gitlab/src/API.ts

@@ -1,3 +1,7 @@
+import { ApolloClient } from 'apollo-client';
+import { InMemoryCache } from 'apollo-cache-inmemory';
+import { createHttpLink } from 'apollo-link-http';
+import { setContext } from 'apollo-link-context';
 import {
   localForage,
   parseLinkHeader,
@@ -27,24 +31,32 @@ import { Map } from 'immutable';
 import { flow, partial, result, trimStart } from 'lodash';
 import { dirname } from 'path';
 
+const NO_CACHE = 'no-cache';
+import * as queries from './queries';
+
+import type { ApolloQueryResult } from 'apollo-client';
+import type { NormalizedCacheObject } from 'apollo-cache-inmemory';
 import type {
   ApiRequest,
   DataFile,
   AssetProxy,
   PersistOptions,
   FetchError,
+  ImplementationFile,
 } from 'netlify-cms-lib-util';
 
 export const API_NAME = 'GitLab';
 
 export interface Config {
   apiRoot?: string;
+  graphQLAPIRoot?: string;
   token?: string;
   branch?: string;
   repo?: string;
   squashMerges: boolean;
   initialWorkflowStatus: string;
   cmsLabelPrefix: string;
+  useGraphQL?: boolean;
 }
 
 export interface CommitAuthor {
@@ -66,6 +78,8 @@ type CommitItem = {
   action: CommitAction;
 };
 
+type FileEntry = { id: string; type: string; path: string; name: string };
+
 interface CommitsParams {
   commit_message: string;
   branch: string;
@@ -183,8 +197,16 @@ export function getMaxAccess(groups: { group_access_level: number }[]) {
   }, groups[0]);
 }
 
+function batch<T>(items: T[], maxPerBatch: number, action: (items: T[]) => void) {
+  for (let index = 0; index < items.length; index = index + maxPerBatch) {
+    const itemsSlice = items.slice(index, index + maxPerBatch);
+    action(itemsSlice);
+  }
+}
+
 export default class API {
   apiRoot: string;
+  graphQLAPIRoot: string;
   token: string | boolean;
   branch: string;
   useOpenAuthoring?: boolean;
@@ -195,8 +217,11 @@ export default class API {
   initialWorkflowStatus: string;
   cmsLabelPrefix: string;
 
+  graphQLClient?: ApolloClient<NormalizedCacheObject>;
+
   constructor(config: Config) {
     this.apiRoot = config.apiRoot || 'https://gitlab.com/api/v4';
+    this.graphQLAPIRoot = config.graphQLAPIRoot || 'https://gitlab.com/api/graphql';
     this.token = config.token || false;
     this.branch = config.branch || 'master';
     this.repo = config.repo || '';
@@ -204,6 +229,40 @@ export default class API {
     this.squashMerges = config.squashMerges;
     this.initialWorkflowStatus = config.initialWorkflowStatus;
     this.cmsLabelPrefix = config.cmsLabelPrefix;
+    if (config.useGraphQL === true) {
+      this.graphQLClient = this.getApolloClient();
+    }
+  }
+
+  getApolloClient() {
+    const authLink = setContext((_, { headers }) => {
+      return {
+        headers: {
+          'Content-Type': 'application/json; charset=utf-8',
+          ...headers,
+          authorization: this.token ? `token ${this.token}` : '',
+        },
+      };
+    });
+    const httpLink = createHttpLink({ uri: this.graphQLAPIRoot });
+    return new ApolloClient({
+      link: authLink.concat(httpLink),
+      cache: new InMemoryCache(),
+      defaultOptions: {
+        watchQuery: {
+          fetchPolicy: NO_CACHE,
+          errorPolicy: 'ignore',
+        },
+        query: {
+          fetchPolicy: NO_CACHE,
+          errorPolicy: 'all',
+        },
+      },
+    });
+  }
+
+  reset() {
+    return this.graphQLClient?.resetStore();
   }
 
   withAuthorizationHeaders = (req: ApiRequest) => {
@@ -352,7 +411,7 @@ export default class API {
   fetchCursorAndEntries = (
     req: ApiRequest,
   ): Promise<{
-    entries: { id: string; type: string; path: string; name: string }[];
+    entries: FileEntry[];
     cursor: Cursor;
   }> =>
     flow([
@@ -392,7 +451,102 @@ export default class API {
     };
   };
 
+  listAllFilesGraphQL = async (path: string, recursive: boolean, branch: String) => {
+    const files: FileEntry[] = [];
+    let blobsPaths;
+    let cursor;
+    do {
+      blobsPaths = await this.graphQLClient!.query({
+        query: queries.files,
+        variables: { repo: this.repo, branch, path, recursive, cursor },
+      });
+      files.push(...blobsPaths.data.project.repository.tree.blobs.nodes);
+      cursor = blobsPaths.data.project.repository.tree.blobs.pageInfo.endCursor;
+    } while (blobsPaths.data.project.repository.tree.blobs.pageInfo.hasNextPage);
+
+    return files;
+  };
+
+  readFilesGraphQL = async (files: ImplementationFile[]) => {
+    const paths = files.map(({ path }) => path);
+
+    type BlobResult = {
+      project: { repository: { blobs: { nodes: { id: string; data: string }[] } } };
+    };
+
+    const blobPromises: Promise<ApolloQueryResult<BlobResult>>[] = [];
+    batch(paths, 90, slice => {
+      blobPromises.push(
+        this.graphQLClient!.query({
+          query: queries.blobs,
+          variables: {
+            repo: this.repo,
+            branch: this.branch,
+            paths: slice,
+          },
+          fetchPolicy: 'cache-first',
+        }),
+      );
+    });
+
+    type LastCommit = {
+      id: string;
+      authoredDate: string;
+      authorName: string;
+      author?: {
+        name: string;
+        username: string;
+        publicEmail: string;
+      };
+    };
+
+    type CommitResult = {
+      project: { repository: { [tree: string]: { lastCommit: LastCommit } } };
+    };
+
+    const commitPromises: Promise<ApolloQueryResult<CommitResult>>[] = [];
+    batch(paths, 8, slice => {
+      commitPromises.push(
+        this.graphQLClient!.query({
+          query: queries.lastCommits(slice),
+          variables: {
+            repo: this.repo,
+            branch: this.branch,
+          },
+          fetchPolicy: 'cache-first',
+        }),
+      );
+    });
+
+    const [blobsResults, commitsResults] = await Promise.all([
+      (await Promise.all(blobPromises)).map(result => result.data.project.repository.blobs.nodes),
+      (
+        await Promise.all(commitPromises)
+      ).map(
+        result =>
+          Object.values(result.data.project.repository)
+            .map(({ lastCommit }) => lastCommit)
+            .filter(Boolean) as LastCommit[],
+      ),
+    ]);
+
+    const blobs = blobsResults.flat().map(result => result.data) as string[];
+    const metadata = commitsResults.flat().map(({ author, authoredDate, authorName }) => ({
+      author: author ? author.name || author.username || author.publicEmail : authorName,
+      updatedOn: authoredDate,
+    }));
+
+    const filesWithData = files.map((file, index) => ({
+      file: { ...file, ...metadata[index] },
+      data: blobs[index],
+    }));
+    return filesWithData;
+  };
+
   listAllFiles = async (path: string, recursive = false, branch = this.branch) => {
+    if (this.graphQLClient) {
+      return await this.listAllFilesGraphQL(path, recursive, branch);
+    }
     const entries = [];
     // eslint-disable-next-line prefer-const
     let { cursor, entries: initialEntries } = await this.fetchCursorAndEntries({

packages/netlify-cms-backend-gitlab/src/implementation.ts

@@ -60,6 +60,8 @@ export default class GitLab implements Implementation {
   cmsLabelPrefix: string;
   mediaFolder: string;
   previewContext: string;
+  useGraphQL: boolean;
+  graphQLAPIRoot: string;
 
   _mediaDisplayURLSem?: Semaphore;
 
@@ -88,6 +90,8 @@ export default class GitLab implements Implementation {
     this.cmsLabelPrefix = config.backend.cms_label_prefix || '';
     this.mediaFolder = config.media_folder;
     this.previewContext = config.backend.preview_context || '';
+    this.useGraphQL = config.backend.use_graphql || false;
+    this.graphQLAPIRoot = config.backend.graphql_api_root || 'https://gitlab.com/api/graphql';
     this.lock = asyncLock();
   }
 
@@ -126,6 +130,8 @@ export default class GitLab implements Implementation {
       squashMerges: this.squashMerges,
       cmsLabelPrefix: this.cmsLabelPrefix,
       initialWorkflowStatus: this.options.initialWorkflowStatus,
+      useGraphQL: this.useGraphQL,
+      graphQLAPIRoot: this.graphQLAPIRoot,
     });
     const user = await this.api.user();
     const isCollab = await this.api.hasWriteAccess().catch((error: Error) => {
@@ -212,7 +218,9 @@ export default class GitLab implements Implementation {
       getDifferences: (to, from) => this.api!.getDifferences(to, from),
       getFileId: path => this.api!.getFileId(path, this.branch),
       filterFile: file => this.filterFile(folder, file, extension, depth),
+      customFetch: this.useGraphQL ? files => this.api!.readFilesGraphQL(files) : undefined,
     });
+
     return files;
   }
 

packages/netlify-cms-backend-gitlab/src/queries.ts

@@ -0,0 +1,73 @@
+import { gql } from 'graphql-tag';
+import { oneLine } from 'common-tags';
+
+export const files = gql`
+  query files($repo: ID!, $branch: String!, $path: String!, $recursive: Boolean!, $cursor: String) {
+    project(fullPath: $repo) {
+      repository {
+        tree(ref: $branch, path: $path, recursive: $recursive) {
+          blobs(after: $cursor) {
+            nodes {
+              type
+              id: sha
+              path
+              name
+            }
+            pageInfo {
+              endCursor
+              hasNextPage
+            }
+          }
+        }
+      }
+    }
+  }
+`;
+
+export const blobs = gql`
+  query blobs($repo: ID!, $branch: String!, $paths: [String!]!) {
+    project(fullPath: $repo) {
+      repository {
+        blobs(ref: $branch, paths: $paths) {
+          nodes {
+            id
+            data: rawBlob
+          }
+        }
+      }
+    }
+  }
+`;
+
+export function lastCommits(paths: string[]) {
+  const tree = paths
+    .map(
+      (path, index) => oneLine`
+    tree${index}: tree(ref: $branch, path: "${path}") {
+      lastCommit {
+        authorName
+        authoredDate
+        author {
+          id
+          username
+          name
+          publicEmail
+        }
+      }
+    }
+  `,
+    )
+    .join('\n');
+
+  const query = gql`
+  query lastCommits($repo: ID!, $branch: String!) {
+    project(fullPath: $repo) {
+      repository {
+        ${tree}
+      }
+    }
+  }
+`;
+
+  return query;
+}

packages/netlify-cms-lib-util/src/implementation.ts

@@ -102,6 +102,7 @@ export type Config = {
     api_root?: string;
     squash_merges?: boolean;
     use_graphql?: boolean;
+    graphql_api_root?: string;
     preview_context?: string;
     identity_url?: string;
     gateway_url?: string;
@@ -205,6 +206,8 @@ type ReadFile = (
 
 type ReadFileMetadata = (path: string, id: string | null | undefined) => Promise<FileMetadata>;
 
+type CustomFetchFunc = (files: ImplementationFile[]) => Promise<ImplementationEntry[]>;
+
 async function fetchFiles(
   files: ImplementationFile[],
   readFile: ReadFile,
@@ -461,6 +464,7 @@ type AllEntriesByFolderArgs = GetKeyArgs &
     isShaExistsInBranch: (branch: string, sha: string) => Promise<boolean>;
     apiName: string;
     localForage: LocalForage;
+    customFetch?: CustomFetchFunc;
   };
 
 export async function allEntriesByFolder({
@@ -478,6 +482,7 @@ export async function allEntriesByFolder({
   getDifferences,
   getFileId,
   filterFile,
+  customFetch,
 }: AllEntriesByFolderArgs) {
   async function listAllFilesAndPersist() {
     const files = await listAllFiles(folder, extension, depth);
@@ -561,5 +566,8 @@ export async function allEntriesByFolder({
   }
 
   const files = await listFiles();
-  return fetchFiles(files, readFile, readFileMetadata, apiName);
+  if (customFetch) {
+    return await customFetch(files);
+  }
+  return await fetchFiles(files, readFile, readFileMetadata, apiName);
 }