feat: add media lib virtualization (#3381)

2020-03-09 08:45:42 +01:00
parent 9c0f618148
commit 92e76011e7
10 changed files with 219 additions and 194 deletions
--- a/packages/netlify-cms-backend-git-gateway/src/implementation.ts
+++ b/packages/netlify-cms-backend-git-gateway/src/implementation.ts
@ -1,6 +1,6 @@
 import GoTrue from 'gotrue-js';
 import jwtDecode from 'jwt-decode';
-import { fromPairs, get, pick, intersection, unzip } from 'lodash';
+import { get, pick, intersection } from 'lodash';
 import ini from 'ini';
 import {
  APIError,
@ -21,9 +21,9 @@ import {
  UnpublishedEntryMediaFile,
  parsePointerFile,
  getLargeMediaPatternsFromGitAttributesFile,
-  PointerFile,
  getPointerFileForMediaFileObj,
  getLargeMediaFilteredMediaFiles,
+  DisplayURLObject,
 } from 'netlify-cms-lib-util';
 import { GitHubBackend } from 'netlify-cms-backend-github';
 import { GitLabBackend } from 'netlify-cms-backend-gitlab';
@ -75,12 +75,6 @@ interface NetlifyUser extends Credentials {
  user_metadata: { full_name: string; avatar_url: string };
 }

-interface GetMediaDisplayURLArgs {
-  path: string;
-  original: { id: string; path: string } | string;
-  largeMedia: PointerFile;
-}
-
 export default class GitGateway implements Implementation {
  config: Config;
  api?: GitHubAPI | GitLabAPI | BitBucketAPI;
@ -278,12 +272,8 @@ export default class GitGateway implements Implementation {
    const mediaFiles = await Promise.all(
      files.map(async file => {
        if (client.matchPath(file.path)) {
-          const { id, path } = file;
-          const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(
-            [{ ...file, id }],
-            branch,
-          );
-          const url = await client.getDownloadURL(largeMediaDisplayURLs[id]);
+          const { path, id } = file;
+          const url = await this.getLargeMediaDisplayURL({ path, id }, branch);
          return {
            id,
            name: basename(path),
@ -303,32 +293,7 @@ export default class GitGateway implements Implementation {
  }

  getMedia(mediaFolder = this.mediaFolder) {
-    return Promise.all([this.backend!.getMedia(mediaFolder), this.getLargeMediaClient()]).then(
-      async ([mediaFiles, largeMediaClient]) => {
-        if (!largeMediaClient.enabled) {
-          return mediaFiles.map(({ displayURL, ...rest }) => ({
-            ...rest,
-            displayURL: { original: displayURL },
-          }));
-        }
-        if (mediaFiles.length === 0) {
-          return [];
-        }
-        const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(mediaFiles);
-        return mediaFiles.map(({ id, displayURL, path, ...rest }) => {
-          return {
-            ...rest,
-            id,
-            path,
-            displayURL: {
-              path,
-              original: displayURL,
-              largeMedia: largeMediaDisplayURLs[id],
-            },
-          };
-        });
-      },
-    );
+    return this.backend!.getMedia(mediaFolder);
  }

  // this method memoizes this._getLargeMediaClient so that there can
@ -382,79 +347,54 @@ export default class GitGateway implements Implementation {
      },
    );
  }
-  async getLargeMediaDisplayURLs(
-    mediaFiles: { path: string; id: string | null }[],
+  async getLargeMediaDisplayURL(
+    { path, id }: { path: string; id: string | null },
    branch = this.branch,
  ) {
-    const client = await this.getLargeMediaClient();
    const readFile = (
      path: string,
      id: string | null | undefined,
      { parseText }: { parseText: boolean },
    ) => this.api!.readFile(path, id, { branch, parseText });

-    const filesPromise = entriesByFiles(mediaFiles, readFile, 'Git-Gateway');
+    const items = await entriesByFiles([{ path, id }], readFile, 'Git-Gateway');
+    const entry = items[0];
+    const pointerFile = parsePointerFile(entry.data);
+    if (!pointerFile.sha) {
+      console.warn(`Failed parsing pointer file ${path}`);
+      return path;
+    }

-    return filesPromise
-      .then(items =>
-        items.map(({ file: { id, path }, data }) => {
-          const parsedPointerFile = parsePointerFile(data);
-          if (!parsedPointerFile.sha) {
-            console.warn(`Failed parsing pointer file ${path}`);
-          }
-          return [
-            {
-              pointerId: id,
-              resourceId: parsedPointerFile.sha,
-            },
-            parsedPointerFile,
-          ];
-        }),
-      )
-      .then(unzip)
-      .then(([idMaps, files]) =>
-        Promise.all([
-          idMaps as { pointerId: string; resourceId: string }[],
-          client.getResourceDownloadURLArgs(files as PointerFile[]).then(r => fromPairs(r)),
-        ]),
-      )
-      .then(([idMaps, resourceMap]) =>
-        idMaps.map(({ pointerId, resourceId }) => [pointerId, resourceMap[resourceId]]),
-      )
-      .then(fromPairs);
+    const client = await this.getLargeMediaClient();
+    const url = await client.getDownloadURL(pointerFile);
+    return url;
  }

-  getMediaDisplayURL(displayURL: DisplayURL) {
-    const {
-      path,
-      original,
-      largeMedia: largeMediaDisplayURL,
-    } = (displayURL as unknown) as GetMediaDisplayURLArgs;
-    return this.getLargeMediaClient().then(client => {
-      if (client.enabled && client.matchPath(path)) {
-        return client.getDownloadURL(largeMediaDisplayURL);
-      }
-      if (typeof original === 'string') {
-        return original;
-      }
-      if (this.backend!.getMediaDisplayURL) {
-        return this.backend!.getMediaDisplayURL(original);
-      }
-      const err = new Error(
-        `getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
-      ) as Error & {
-        displayURL: DisplayURL;
-      };
-      err.displayURL = displayURL;
-      return Promise.reject(err);
-    });
+  async getMediaDisplayURL(displayURL: DisplayURL) {
+    const { path, id } = displayURL as DisplayURLObject;
+    const client = await this.getLargeMediaClient();
+    if (client.enabled && client.matchPath(path)) {
+      return this.getLargeMediaDisplayURL({ path, id });
+    }
+    if (typeof displayURL === 'string') {
+      return displayURL;
+    }
+    if (this.backend!.getMediaDisplayURL) {
+      return this.backend!.getMediaDisplayURL(displayURL);
+    }
+    const err = new Error(
+      `getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
+    ) as Error & {
+      displayURL: DisplayURL;
+    };
+    err.displayURL = displayURL;
+    return Promise.reject(err);
  }

  async getMediaFile(path: string) {
    const client = await this.getLargeMediaClient();
    if (client.enabled && client.matchPath(path)) {
-      const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs([{ path, id: null }]);
-      const url = await client.getDownloadURL(Object.values(largeMediaDisplayURLs)[0]);
+      const url = await this.getLargeMediaDisplayURL({ path, id: null });
      return {
        id: url,
        name: basename(path),
--- a/packages/netlify-cms-backend-git-gateway/src/netlify-lfs-client.ts
+++ b/packages/netlify-cms-backend-git-gateway/src/netlify-lfs-client.ts
@ -66,11 +66,6 @@ const getDownloadURL = (
      return Promise.resolve('');
    });

-const getResourceDownloadURLArgs = (_clientConfig: ClientConfig, objects: PointerFile[]) => {
-  const result = objects.map(({ sha }) => [sha, { sha }]) as [string, { sha: string }][];
-  return Promise.resolve(result);
-};
-
 const uploadOperation = (objects: PointerFile[]) => ({
  operation: 'upload',
  transfers: ['basic'],
@ -129,7 +124,6 @@ const configureFn = (config: ClientConfig, fn: Function) => (...args: unknown[])
 const clientFns: Record<string, Function> = {
  resourceExists,
  getResourceUploadURLs,
-  getResourceDownloadURLArgs,
  getDownloadURL,
  uploadResource,
  matchPath,
@ -138,7 +132,6 @@ const clientFns: Record<string, Function> = {
 export type Client = {
  resourceExists: (pointer: PointerFile) => Promise<boolean | undefined>;
  getResourceUploadURLs: (objects: PointerFile[]) => Promise<string>;
-  getResourceDownloadURLArgs: (objects: PointerFile[]) => Promise<[string, { sha: string }][]>;
  getDownloadURL: (pointer: PointerFile) => Promise<string>;
  uploadResource: (pointer: PointerFile, blob: Blob) => Promise<string>;
  matchPath: (path: string) => boolean;
--- a/packages/netlify-cms-core/package.json
+++ b/packages/netlify-cms-core/package.json
@ -56,7 +56,9 @@
    "react-sortable-hoc": "^1.0.0",
    "react-split-pane": "^0.1.85",
    "react-topbar-progress-indicator": "^2.0.0",
+    "react-virtualized-auto-sizer": "^1.0.2",
    "react-waypoint": "^8.1.0",
+    "react-window": "^1.8.5",
    "redux": "^4.0.1",
    "redux-notifications": "^4.0.1",
    "redux-optimist": "^1.0.0",
--- a/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCard.js
+++ b/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCard.js
@ -8,7 +8,7 @@ const IMAGE_HEIGHT = 160;

 const Card = styled.div`
  width: ${props => props.width};
-  height: 240px;
+  height: ${props => props.height};
  margin: ${props => props.margin};
  border: ${borders.textField};
  border-color: ${props => props.isSelected && colors.active};
@ -71,6 +71,7 @@ class MediaLibraryCard extends React.Component {
      onClick,
      draftText,
      width,
+      height,
      margin,
      isPrivate,
      type,
@ -83,6 +84,7 @@ class MediaLibraryCard extends React.Component {
        isSelected={isSelected}
        onClick={onClick}
        width={width}
+        height={height}
        margin={margin}
        tabIndex="-1"
        isPrivate={isPrivate}
@ -114,6 +116,7 @@ MediaLibraryCard.propTypes = {
  onClick: PropTypes.func.isRequired,
  draftText: PropTypes.string.isRequired,
  width: PropTypes.string.isRequired,
+  height: PropTypes.string.isRequired,
  margin: PropTypes.string.isRequired,
  isPrivate: PropTypes.bool,
  type: PropTypes.string,
--- a/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCardGrid.js
+++ b/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCardGrid.js
@ -5,6 +5,143 @@ import Waypoint from 'react-waypoint';
 import MediaLibraryCard from './MediaLibraryCard';
 import { Map } from 'immutable';
 import { colors } from 'netlify-cms-ui-default';
+import { FixedSizeGrid as Grid } from 'react-window';
+import AutoSizer from 'react-virtualized-auto-sizer';
+
+const CardWrapper = props => {
+  const {
+    rowIndex,
+    columnIndex,
+    style,
+    data: {
+      mediaItems,
+      isSelectedFile,
+      onAssetClick,
+      cardDraftText,
+      cardWidth,
+      cardHeight,
+      isPrivate,
+      displayURLs,
+      loadDisplayURL,
+      columnCount,
+      gutter,
+    },
+  } = props;
+  const index = rowIndex * columnCount + columnIndex;
+  if (index >= mediaItems.length) {
+    return null;
+  }
+  const file = mediaItems[index];
+
+  return (
+    <div
+      style={{
+        ...style,
+        left: style.left + gutter * columnIndex,
+        top: style.top + gutter,
+        width: style.width - gutter,
+        height: style.height - gutter,
+      }}
+    >
+      <MediaLibraryCard
+        key={file.key}
+        isSelected={isSelectedFile(file)}
+        text={file.name}
+        onClick={() => onAssetClick(file)}
+        isDraft={file.draft}
+        draftText={cardDraftText}
+        width={cardWidth}
+        height={cardHeight}
+        margin={'0px'}
+        isPrivate={isPrivate}
+        displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
+        loadDisplayURL={() => loadDisplayURL(file)}
+        type={file.type}
+        isViewableImage={file.isViewableImage}
+      />
+    </div>
+  );
+};
+
+const VirtualizedGrid = props => {
+  const { mediaItems, setScrollContainerRef } = props;
+
+  return (
+    <CardGridContainer ref={setScrollContainerRef}>
+      <AutoSizer>
+        {({ height, width }) => {
+          const cardWidth = parseInt(props.cardWidth, 10);
+          const cardHeight = parseInt(props.cardHeight, 10);
+          const gutter = parseInt(props.cardMargin, 10);
+          const columnWidth = cardWidth + gutter;
+          const rowHeight = cardHeight + gutter;
+          const columnCount = Math.floor(width / columnWidth);
+          const rowCount = Math.ceil(mediaItems.length / columnCount);
+          return (
+            <Grid
+              columnCount={columnCount}
+              columnWidth={columnWidth}
+              rowCount={rowCount}
+              rowHeight={rowHeight}
+              width={width}
+              height={height}
+              itemData={{ ...props, gutter, columnCount }}
+            >
+              {CardWrapper}
+            </Grid>
+          );
+        }}
+      </AutoSizer>
+    </CardGridContainer>
+  );
+};
+
+const PaginatedGrid = ({
+  setScrollContainerRef,
+  mediaItems,
+  isSelectedFile,
+  onAssetClick,
+  cardDraftText,
+  cardWidth,
+  cardHeight,
+  cardMargin,
+  isPrivate,
+  displayURLs,
+  loadDisplayURL,
+  canLoadMore,
+  onLoadMore,
+  isPaginating,
+  paginatingMessage,
+}) => {
+  return (
+    <CardGridContainer ref={setScrollContainerRef}>
+      <CardGrid>
+        {mediaItems.map(file => (
+          <MediaLibraryCard
+            key={file.key}
+            isSelected={isSelectedFile(file)}
+            text={file.name}
+            onClick={() => onAssetClick(file)}
+            isDraft={file.draft}
+            draftText={cardDraftText}
+            width={cardWidth}
+            height={cardHeight}
+            margin={cardMargin}
+            isPrivate={isPrivate}
+            displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
+            loadDisplayURL={() => loadDisplayURL(file)}
+            type={file.type}
+            isViewableImage={file.isViewableImage}
+          />
+        ))}
+        {!canLoadMore ? null : <Waypoint onEnter={onLoadMore} />}
+      </CardGrid>
+      {!isPaginating ? null : (
+        <PaginatingMessage isPrivate={isPrivate}>{paginatingMessage}</PaginatingMessage>
+      )}
+    </CardGridContainer>
+  );
+};

 const CardGridContainer = styled.div`
  overflow-y: auto;
@ -23,48 +160,13 @@ const PaginatingMessage = styled.h1`
  color: ${props => props.isPrivate && colors.textFieldBorder};
 `;

-const MediaLibraryCardGrid = ({
-  setScrollContainerRef,
-  mediaItems,
-  isSelectedFile,
-  onAssetClick,
-  canLoadMore,
-  onLoadMore,
-  isPaginating,
-  paginatingMessage,
-  cardDraftText,
-  cardWidth,
-  cardMargin,
-  isPrivate,
-  displayURLs,
-  loadDisplayURL,
-}) => (
-  <CardGridContainer ref={setScrollContainerRef}>
-    <CardGrid>
-      {mediaItems.map(file => (
-        <MediaLibraryCard
-          key={file.key}
-          isSelected={isSelectedFile(file)}
-          text={file.name}
-          onClick={() => onAssetClick(file)}
-          isDraft={file.draft}
-          draftText={cardDraftText}
-          width={cardWidth}
-          margin={cardMargin}
-          isPrivate={isPrivate}
-          displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
-          loadDisplayURL={() => loadDisplayURL(file)}
-          type={file.type}
-          isViewableImage={file.isViewableImage}
-        />
-      ))}
-      {!canLoadMore ? null : <Waypoint onEnter={onLoadMore} />}
-    </CardGrid>
-    {!isPaginating ? null : (
-      <PaginatingMessage isPrivate={isPrivate}>{paginatingMessage}</PaginatingMessage>
-    )}
-  </CardGridContainer>
-);
+const MediaLibraryCardGrid = props => {
+  const { canLoadMore, isPaginating } = props;
+  if (canLoadMore || isPaginating) {
+    return <PaginatedGrid {...props} />;
+  }
+  return <VirtualizedGrid {...props} />;
+};

 MediaLibraryCardGrid.propTypes = {
  setScrollContainerRef: PropTypes.func.isRequired,
--- a/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryModal.js
+++ b/packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryModal.js
@ -17,6 +17,7 @@ import { colors } from 'netlify-cms-ui-default';
 * widths per breakpoint.
 */
 const cardWidth = `280px`;
+const cardHeight = `240px`;
 const cardMargin = `10px`;

 /**
@ -172,6 +173,7 @@ const MediaLibraryModal = ({
        paginatingMessage={t('mediaLibrary.mediaLibraryModal.loading')}
        cardDraftText={t('mediaLibrary.mediaLibraryCard.draft')}
        cardWidth={cardWidth}
+        cardHeight={cardHeight}
        cardMargin={cardMargin}
        isPrivate={privateUpload}
        loadDisplayURL={loadDisplayURL}
--- a/packages/netlify-cms-core/src/components/MediaLibrary/tests/MediaLibraryCard.spec.js
+++ b/packages/netlify-cms-core/src/components/MediaLibrary/tests/MediaLibraryCard.spec.js
@ -10,6 +10,7 @@ describe('MediaLibraryCard', () => {
    onClick: jest.fn(),
    draftText: 'Draft',
    width: '100px',
+    height: '240px',
    margin: '10px',
    isViewableImage: true,
    loadDisplayURL: jest.fn(),
--- a/packages/netlify-cms-core/src/components/MediaLibrary/tests/snapshots/MediaLibraryCard.spec.js.snap
+++ b/packages/netlify-cms-core/src/components/MediaLibrary/tests/snapshots/MediaLibraryCard.spec.js.snap
@ -52,6 +52,7 @@ exports[`MediaLibraryCard should match snapshot for draft image 1`] = `

 <div
    class="emotion-8 emotion-9"
+    height="240px"
    tabindex="-1"
    width="100px"
  >
@ -122,6 +123,7 @@ exports[`MediaLibraryCard should match snapshot for non draft image 1`] = `

 <div
    class="emotion-6 emotion-7"
+    height="240px"
    tabindex="-1"
    width="100px"
  >
@ -188,6 +190,7 @@ exports[`MediaLibraryCard should match snapshot for non viewable image 1`] = `

 <div
    class="emotion-6 emotion-7"
+    height="240px"
    tabindex="-1"
    width="100px"
  >
--- a/packages/netlify-cms-lib-util/src/implementation.ts
+++ b/packages/netlify-cms-lib-util/src/implementation.ts
@ -4,10 +4,7 @@ import { AsyncLock } from './asyncLock';

 export type DisplayURLObject = { id: string; path: string };

-export type DisplayURL =
-  | DisplayURLObject
-  | string
-  | { original: DisplayURL; path?: string; largeMedia?: string };
+export type DisplayURL = DisplayURLObject | string;

 export interface ImplementationMediaFile {
  name: string;