Netlify Large Media integration (#2124)

packages/netlify-cms-backend-bitbucket/src/implementation.js

@@ -223,17 +223,18 @@ export default class Bitbucket {
 
     return this.api.listAllFiles(this.config.get('media_folder')).then(files =>
       files.map(({ id, name, path }) => {
-        const getBlobPromise = () =>
+        const getDisplayURL = () =>
           new Promise((resolve, reject) =>
             sem.take(() =>
               this.api
                 .readFile(path, id, { parseText: false })
+                .then(blob => URL.createObjectURL(blob))
                 .then(resolve, reject)
                 .finally(() => sem.leave()),
             ),
           );
 
-        return { id, name, getBlobPromise, path };
+        return { id, name, getDisplayURL, path };
       }),
     );
   }
@@ -245,8 +246,7 @@ export default class Bitbucket {
   async persistMedia(mediaFile, options = {}) {
     await this.api.persistFiles([mediaFile], options);
     const { value, path, fileObj } = mediaFile;
-    const getBlobPromise = () => Promise.resolve(fileObj);
+    return { name: value, size: fileObj.size, path: trimStart(path, '/k') };
-    return { name: value, size: fileObj.size, getBlobPromise, path: trimStart(path, '/k') };
   }
 
   deleteFile(path, commitMessage, options) {

packages/netlify-cms-backend-git-gateway/package.json

@@ -21,7 +21,9 @@
   },
   "dependencies": {
     "gotrue-js": "^0.9.22",
-    "jwt-decode": "^2.2.0"
+    "ini": "^1.3.5",
+    "jwt-decode": "^2.2.0",
+    "minimatch": "^3.0.4"
   },
   "devDependencies": {
     "cross-env": "^5.2.0",

packages/netlify-cms-backend-git-gateway/src/AuthenticationPage.js

@@ -127,7 +127,7 @@ export default class GitGatewayAuthenticationPage extends React.Component {
       return;
     }
 
-    AuthenticationPage.authClient
+    GitGatewayAuthenticationPage.authClient
       .login(this.state.email, this.state.password, true)
       .then(user => {
         this.props.onLogin(user);

packages/netlify-cms-backend-git-gateway/src/implementation.js

@@ -1,13 +1,20 @@
 import GoTrue from 'gotrue-js';
 import jwtDecode from 'jwt-decode';
-import { get, pick, intersection } from 'lodash';
+import { fromPairs, get, pick, intersection, unzip } from 'lodash';
-import { APIError, unsentRequest } from 'netlify-cms-lib-util';
+import ini from 'ini';
+import { APIError, getBlobSHA, unsentRequest } from 'netlify-cms-lib-util';
 import { GitHubBackend } from 'netlify-cms-backend-github';
 import { GitLabBackend } from 'netlify-cms-backend-gitlab';
 import { BitBucketBackend, API as BitBucketAPI } from 'netlify-cms-backend-bitbucket';
 import GitHubAPI from './GitHubAPI';
 import GitLabAPI from './GitLabAPI';
 import AuthenticationPage from './AuthenticationPage';
+import {
+  parsePointerFile,
+  createPointerFile,
+  getLargeMediaPatternsFromGitAttributesFile,
+  getClient,
+} from './netlify-lfs-client';
 
 const localHosts = {
   localhost: true,
@@ -17,6 +24,7 @@ const localHosts = {
 const defaults = {
   identity: '/.netlify/identity',
   gateway: '/.netlify/git',
+  largeMedia: '/.netlify/large-media',
 };
 
 function getEndpoint(endpoint, netlifySiteURL) {
@@ -58,7 +66,10 @@ export default class GitGateway {
       config.getIn(['backend', 'gateway_url'], defaults.gateway),
       netlifySiteURL,
     );
-
+    this.netlifyLargeMediaURL = getEndpoint(
+      config.getIn(['backend', 'large_media_url'], defaults.largeMedia),
+      netlifySiteURL,
+    );
     const backendTypeRegex = /\/(github|gitlab|bitbucket)\/?$/;
     const backendTypeMatches = this.gatewayUrl.match(backendTypeRegex);
     if (backendTypeMatches) {
@@ -196,14 +207,136 @@ export default class GitGateway {
   getEntry(collection, slug, path) {
     return this.backend.getEntry(collection, slug, path);
   }
+
   getMedia() {
-    return this.backend.getMedia();
+    return Promise.all([this.backend.getMedia(), this.getLargeMediaClient()]).then(
+      async ([mediaFiles, largeMediaClient]) => {
+        if (!largeMediaClient.enabled) {
+          return mediaFiles;
+        }
+        const largeMediaURLThunks = await this.getLargeMedia(mediaFiles);
+        return mediaFiles.map(({ id, url, getDisplayURL, ...rest }) => ({
+          ...rest,
+          id,
+          url,
+          urlIsPublicPath: false,
+          getDisplayURL: largeMediaURLThunks[id] ? largeMediaURLThunks[id] : getDisplayURL,
+        }));
+      },
+    );
+  }
+
+  // this method memoizes this._getLargeMediaClient so that there can
+  // only be one client at a time
+  getLargeMediaClient() {
+    if (this._largeMediaClientPromise) {
+      return this._largeMediaClientPromise;
+    }
+    this._largeMediaClientPromise = this._getLargeMediaClient();
+    return this._largeMediaClientPromise;
+  }
+  _getLargeMediaClient() {
+    const netlifyLargeMediaEnabledPromise = this.api
+      .readFile('.lfsconfig')
+      .then(ini.decode)
+      .then(({ lfs: { url } }) => new URL(url))
+      .then(lfsURL => ({ enabled: lfsURL.hostname.endsWith('netlify.com') }))
+      .catch(err => ({ enabled: false, err }));
+
+    const lfsPatternsPromise = this.api
+      .readFile('.gitattributes')
+      .then(getLargeMediaPatternsFromGitAttributesFile)
+      .then(patterns => ({ patterns }))
+      .catch(err => (err.message.includes('404') ? [] : { err }));
+
+    return Promise.all([netlifyLargeMediaEnabledPromise, lfsPatternsPromise]).then(
+      ([{ enabled: maybeEnabled }, { patterns, err: patternsErr }]) => {
+        const enabled = maybeEnabled && !patternsErr;
+
+        // We expect LFS patterns to exist when the .lfsconfig states
+        // that we're using Netlify Large Media
+        if (maybeEnabled && patternsErr) {
+          console.error(patternsErr);
+        }
+
+        return getClient({
+          enabled,
+          rootURL: this.netlifyLargeMediaURL,
+          makeAuthorizedRequest: this.requestFunction,
+          patterns,
+          transformImages: this.config.getIn(
+            ['backend', 'use_large_media_transforms_in_media_library'],
+            true,
+          )
+            ? { nf_resize: 'fit', w: 280, h: 160 }
+            : false,
+        });
+      },
+    );
+  }
+  getLargeMedia(mediaFiles) {
+    return this.getLargeMediaClient().then(client => {
+      const largeMediaItems = mediaFiles
+        .filter(({ path }) => client.matchPath(path))
+        .map(({ id, path }) => ({ path, sha: id }));
+      return this.backend
+        .fetchFiles(largeMediaItems)
+        .then(items =>
+          items.map(({ file: { sha }, data }) => {
+            const parsedPointerFile = parsePointerFile(data);
+            return [
+              {
+                pointerId: sha,
+                resourceId: parsedPointerFile.sha,
+              },
+              parsedPointerFile,
+            ];
+          }),
+        )
+        .then(unzip)
+        .then(async ([idMaps, files]) => [
+          idMaps,
+          await client.getResourceDownloadURLThunks(files).then(fromPairs),
+        ])
+        .then(([idMaps, resourceMap]) =>
+          idMaps.map(({ pointerId, resourceId }) => [pointerId, resourceMap[resourceId]]),
+        )
+        .then(fromPairs);
+    });
   }
   persistEntry(entry, mediaFiles, options) {
     return this.backend.persistEntry(entry, mediaFiles, options);
   }
   persistMedia(mediaFile, options) {
-    return this.backend.persistMedia(mediaFile, options);
+    const { fileObj, path, value } = mediaFile;
+    const { name, size } = fileObj;
+    return this.getLargeMediaClient().then(client => {
+      const fixedPath = path.startsWith('/') ? path.slice(1) : path;
+      if (!client.enabled || !client.matchPath(fixedPath)) {
+        return this.backend.persistMedia(mediaFile, options);
+      }
+
+      return getBlobSHA(fileObj).then(async sha => {
+        await client.uploadResource({ sha, size }, fileObj);
+        const pointerFileString = createPointerFile({ sha, size });
+        const pointerFileBlob = new Blob([pointerFileString]);
+        const pointerFile = new File([pointerFileBlob], name, { type: 'text/plain' });
+        const pointerFileSHA = await getBlobSHA(pointerFile);
+        const persistMediaArgument = {
+          fileObj: pointerFile,
+          size: pointerFileBlob.size,
+          path,
+          sha: pointerFileSHA,
+          raw: pointerFileString,
+          value,
+        };
+        const persistedMediaFile = await this.backend.persistMedia(persistMediaArgument, options);
+        return {
+          ...persistedMediaFile,
+          urlIsPublicPath: false,
+        };
+      });
+    });
   }
   deleteFile(path, commitMessage, options) {
     return this.backend.deleteFile(path, commitMessage, options);

packages/netlify-cms-backend-git-gateway/src/netlify-lfs-client.js

@@ -0,0 +1,189 @@
+import { filter, flow, fromPairs, map } from 'lodash/fp';
+import minimatch from 'minimatch';
+
+//
+// Pointer file parsing
+
+const splitIntoLines = str => str.split('\n');
+const splitIntoWords = str => str.split(/\s+/g);
+const isNonEmptyString = str => str !== '';
+const withoutEmptyLines = flow([map(str => str.trim()), filter(isNonEmptyString)]);
+export const parsePointerFile = flow([
+  splitIntoLines,
+  withoutEmptyLines,
+  map(splitIntoWords),
+  fromPairs,
+  ({ size, oid, ...rest }) => ({
+    size: parseInt(size),
+    sha: oid.split(':')[1],
+    ...rest,
+  }),
+]);
+
+export const createPointerFile = ({ size, sha }) => `\
+version https://git-lfs.github.com/spec/v1
+oid sha256:${sha}
+size ${size}
+`;
+
+//
+// .gitattributes file parsing
+
+const removeGitAttributesCommentsFromLine = line => line.split('#')[0];
+
+const parseGitPatternAttribute = attributeString => {
+  // There are three kinds of attribute settings:
+  // - a key=val pair sets an attribute to a specific value
+  // - a key without a value and a leading hyphen sets an attribute to false
+  // - a key without a value and no leading hyphen sets an attribute
+  //   to true
+  if (attributeString.includes('=')) {
+    return attributeString.split('=');
+  }
+  if (attributeString.startsWith('-')) {
+    return [attributeString.slice(1), false];
+  }
+  return [attributeString, true];
+};
+
+const parseGitPatternAttributes = flow([map(parseGitPatternAttribute), fromPairs]);
+
+const parseGitAttributesPatternLine = flow([
+  splitIntoWords,
+  ([pattern, ...attributes]) => [pattern, parseGitPatternAttributes(attributes)],
+]);
+
+const parseGitAttributesFileToPatternAttributePairs = flow([
+  splitIntoLines,
+  map(removeGitAttributesCommentsFromLine),
+  withoutEmptyLines,
+  map(parseGitAttributesPatternLine),
+]);
+
+export const getLargeMediaPatternsFromGitAttributesFile = flow([
+  parseGitAttributesFileToPatternAttributePairs,
+  filter(
+    // eslint-disable-next-line no-unused-vars
+    ([pattern, attributes]) =>
+      attributes.filter === 'lfs' && attributes.diff === 'lfs' && attributes.merge === 'lfs',
+  ),
+  map(([pattern]) => pattern),
+]);
+
+export const matchPath = ({ patterns }, path) =>
+  patterns.some(pattern => minimatch(path, pattern, { matchBase: true }));
+
+//
+// API interactions
+
+const defaultContentHeaders = {
+  Accept: 'application/vnd.git-lfs+json',
+  ['Content-Type']: 'application/vnd.git-lfs+json',
+};
+
+const resourceExists = async ({ rootURL, makeAuthorizedRequest }, { sha, size }) => {
+  const response = await makeAuthorizedRequest({
+    url: `${rootURL}/verify`,
+    method: 'POST',
+    headers: defaultContentHeaders,
+    body: JSON.stringify({ oid: sha, size }),
+  });
+  if (response.ok) {
+    return true;
+  }
+  if (response.status === 404) {
+    return false;
+  }
+
+  // TODO: what kind of error to throw here? APIError doesn't seem
+  // to fit
+};
+
+const getDownloadURLThunkFromSha = (
+  { rootURL, makeAuthorizedRequest, transformImages: t },
+  sha,
+) => () =>
+  makeAuthorizedRequest(
+    `${rootURL}/origin/${sha}${
+      t && Object.keys(t).length > 0 ? `?nf_resize=${t.nf_resize}&w=${t.w}&h=${t.h}` : ''
+    }`,
+  )
+    .then(res => (res.ok ? res : Promise.reject(res)))
+    .then(res => res.blob())
+    .then(blob => URL.createObjectURL(blob))
+    .catch(err => console.error(err) || Promise.resolve(''));
+
+// We allow users to get thunks which load the blobs instead of fully
+// resolved blob URLs so that media clients can download the blobs
+// lazily.  This behaves more similarly to the behavior of string
+// URLs, which only trigger an image download when the DOM element for
+// that image is created.
+const getResourceDownloadURLThunks = (clientConfig, objects) =>
+  Promise.resolve(objects.map(({ sha }) => [sha, getDownloadURLThunkFromSha(clientConfig, sha)]));
+
+const getResourceDownloadURLs = (clientConfig, objects) =>
+  getResourceDownloadURLThunks(clientConfig, objects)
+    .then(map(([sha, thunk]) => Promise.all([sha, thunk()])))
+    .then(Promise.all.bind(Promise));
+
+const uploadOperation = objects => ({
+  operation: 'upload',
+  transfers: ['basic'],
+  objects: objects.map(({ sha, ...rest }) => ({ ...rest, oid: sha })),
+});
+
+const getResourceUploadURLs = async ({ rootURL, makeAuthorizedRequest }, objects) => {
+  const response = await makeAuthorizedRequest({
+    url: `${rootURL}/objects/batch`,
+    method: 'POST',
+    headers: defaultContentHeaders,
+    body: JSON.stringify(uploadOperation(objects)),
+  });
+  return (await response.json()).objects.map(object => {
+    if (object.error) {
+      throw new Error(object.error.message);
+    }
+    return object.actions.upload.href;
+  });
+};
+
+const uploadBlob = (clientConfig, uploadURL, blob) =>
+  fetch(uploadURL, {
+    method: 'PUT',
+    body: blob,
+  });
+
+const uploadResource = async (clientConfig, { sha, size }, resource) => {
+  const existingFile = await resourceExists(clientConfig, { sha, size });
+  if (existingFile) {
+    return sha;
+  }
+  const [uploadURL] = await getResourceUploadURLs(clientConfig, [{ sha, size }]);
+  await uploadBlob(clientConfig, uploadURL, resource);
+  return sha;
+};
+
+//
+// Create Large Media client
+
+const configureFn = (config, fn) => (...args) => fn(config, ...args);
+const clientFns = {
+  resourceExists,
+  getResourceUploadURLs,
+  getResourceDownloadURLs,
+  getResourceDownloadURLThunks,
+  uploadResource,
+  matchPath,
+};
+export const getClient = clientConfig => {
+  return flow([
+    Object.keys,
+    map(key => [key, configureFn(clientConfig, clientFns[key])]),
+    fromPairs,
+    configuredFns => ({
+      ...configuredFns,
+      patterns: clientConfig.patterns,
+      enabled: clientConfig.enabled,
+    }),
+  ])(clientFns);
+};

packages/netlify-cms-backend-github/src/implementation.js

@@ -164,7 +164,7 @@ export default class GitHub {
         if (url.pathname.match(/.svg$/)) {
           url.search += (url.search.slice(1) === '' ? '?' : '&') + 'sanitize=true';
         }
-        return { id: sha, name, size, url: url.href, path };
+        return { id: sha, name, size, url: url.href, urlIsPublicPath: true, path };
       }),
     );
   }

packages/netlify-cms-backend-gitlab/src/implementation.js

@@ -147,7 +147,7 @@ export default class GitLab {
 
     return this.api.listAllFiles(this.config.get('media_folder')).then(files =>
       files.map(({ id, name, path }) => {
-        const getBlobPromise = () =>
+        const getDisplayURL = () =>
           new Promise((resolve, reject) =>
             sem.take(() =>
               this.api
@@ -159,12 +159,13 @@ export default class GitLab {
                   }
                   return blob;
                 })
+                .then(blob => URL.createObjectURL(blob))
                 .then(resolve, reject)
                 .finally(() => sem.leave()),
             ),
           );
 
-        return { id, name, getBlobPromise, path };
+        return { id, name, getDisplayURL, path };
       }),
     );
   }
@@ -176,8 +177,7 @@ export default class GitLab {
   async persistMedia(mediaFile, options = {}) {
     await this.api.persistFiles([mediaFile], options);
     const { value, path, fileObj } = mediaFile;
-    const getBlobPromise = () => Promise.resolve(fileObj);
+    return { name: value, size: fileObj.size, path: trimStart(path, '/') };
-    return { name: value, size: fileObj.size, getBlobPromise, path: trimStart(path, '/') };
   }
 
   deleteFile(path, commitMessage, options) {

packages/netlify-cms-core/src/actions/mediaLibrary.js

@@ -1,5 +1,6 @@
 import { Map } from 'immutable';
 import { actions as notifActions } from 'redux-notifications';
+import { getBlobSHA } from 'netlify-cms-lib-util';
 import { currentBackend } from 'src/backend';
 import { createAssetProxy } from 'ValueObjects/AssetProxy';
 import { selectIntegration } from 'Reducers';
@@ -119,7 +120,11 @@ export function loadMedia(opts = {}) {
           backend
             .getMedia()
             .then(files => dispatch(mediaLoaded(files)))
-            .catch(error => dispatch(error.status === 404 ? mediaLoaded() : mediaLoadFailed())),
+            .catch(
+              error =>
+                console.error(error) ||
+                dispatch(error.status === 404 ? mediaLoaded() : mediaLoadFailed()),
+            ),
         ),
       );
     }, delay);
@@ -153,13 +158,17 @@ export function persistMedia(file, opts = {}) {
     dispatch(mediaPersisting());
 
     try {
+      const id = await getBlobSHA(file);
+      const getDisplayURL = () => URL.createObjectURL(file);
       const assetProxy = await createAssetProxy(fileName, file, false, privateUpload);
       dispatch(addAsset(assetProxy));
       if (!integration) {
         const asset = await backend.persistMedia(state.config, assetProxy);
-        return dispatch(mediaPersisted(asset));
+        return dispatch(mediaPersisted({ id, getDisplayURL, ...asset }));
       }
-      return dispatch(mediaPersisted(assetProxy.asset, { privateUpload }));
+      return dispatch(
+        mediaPersisted({ id, getDisplayURL, ...assetProxy.asset }, { privateUpload }),
+      );
     } catch (error) {
       console.error(error);
       dispatch(
@@ -221,16 +230,26 @@ export function deleteMedia(file, opts = {}) {
 }
 
 export function loadMediaDisplayURL(file) {
-  return async dispatch => {
+  return async (dispatch, getState) => {
-    const { getBlobPromise, id } = file;
+    const { getDisplayURL, id, url, urlIsPublicPath } = file;
-
+    const { mediaLibrary: mediaLibraryState } = getState();
-    if (id && getBlobPromise) {
+    const displayURLPath = ['displayURLs', id];
+    const shouldLoadDisplayURL =
+      id &&
+      ((url && urlIsPublicPath) ||
+        (getDisplayURL &&
+          !mediaLibraryState.getIn([...displayURLPath, 'url']) &&
+          !mediaLibraryState.getIn([...displayURLPath, 'isFetching']) &&
+          !mediaLibraryState.getIn([...displayURLPath, 'err'])));
+    if (shouldLoadDisplayURL) {
       try {
         dispatch(mediaDisplayURLRequest(id));
-        const blob = await getBlobPromise();
+        const newURL = (urlIsPublicPath && url) || (await getDisplayURL());
-        const newURL = window.URL.createObjectURL(blob);
+        if (newURL) {
-        dispatch(mediaDisplayURLSuccess(id, newURL));
+          dispatch(mediaDisplayURLSuccess(id, newURL));
-        return newURL;
+          return newURL;
+        }
+        throw new Error('No display URL was returned!');
       } catch (err) {
         dispatch(mediaDisplayURLFailure(id, err));
       }

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibrary.js

@@ -3,7 +3,6 @@ import PropTypes from 'prop-types';
 import ImmutablePropTypes from 'react-immutable-proptypes';
 import { connect } from 'react-redux';
 import { orderBy, map } from 'lodash';
-import { Map } from 'immutable';
 import { translate } from 'react-polyglot';
 import fuzzy from 'fuzzy';
 import { resolvePath, fileExtension } from 'netlify-cms-lib-util';
@@ -26,11 +25,13 @@ const IMAGE_EXTENSIONS = [...IMAGE_EXTENSIONS_VIEWABLE];
 
 const fileShape = {
   key: PropTypes.string.isRequired,
+  id: PropTypes.string.isRequired,
   name: PropTypes.string.isRequired,
-  size: PropTypes.number.isRequired,
+  size: PropTypes.number,
   queryOrder: PropTypes.number,
-  url: PropTypes.string.isRequired,
+  url: PropTypes.string,
   urlIsPublicPath: PropTypes.bool,
+  getDisplayURL: PropTypes.func,
 };
 
 class MediaLibrary extends React.Component {
@@ -97,28 +98,9 @@ class MediaLibrary extends React.Component {
     }
   }
 
-  getDisplayURL = file => {
+  loadDisplayURL = file => {
-    const { isVisible, loadMediaDisplayURL, displayURLs } = this.props;
+    const { loadMediaDisplayURL } = this.props;
-
+    loadMediaDisplayURL(file);
-    if (!isVisible) {
-      return '';
-    }
-
-    if (file && file.url) {
-      return file.url;
-    }
-
-    const { url, isFetching } = displayURLs.get(file.id, Map()).toObject();
-
-    if (url && url !== '') {
-      return url;
-    }
-
-    if (!isFetching) {
-      loadMediaDisplayURL(file);
-    }
-
-    return '';
   };
 
   /**
@@ -137,7 +119,7 @@ class MediaLibrary extends React.Component {
   toTableData = files => {
     const tableData =
       files &&
-      files.map(({ key, name, id, size, queryOrder, url, urlIsPublicPath, getBlobPromise }) => {
+      files.map(({ key, name, id, size, queryOrder, url, urlIsPublicPath, getDisplayURL }) => {
         const ext = fileExtension(name).toLowerCase();
         return {
           key,
@@ -148,7 +130,7 @@ class MediaLibrary extends React.Component {
           queryOrder,
           url,
           urlIsPublicPath,
-          getBlobPromise,
+          getDisplayURL,
           isImage: IMAGE_EXTENSIONS.includes(ext),
           isViewableImage: IMAGE_EXTENSIONS_VIEWABLE.includes(ext),
         };
@@ -291,6 +273,7 @@ class MediaLibrary extends React.Component {
       hasNextPage,
       isPaginating,
       privateUpload,
+      displayURLs,
       t,
     } = this.props;
 
@@ -322,7 +305,8 @@ class MediaLibrary extends React.Component {
         setScrollContainerRef={ref => (this.scrollContainerRef = ref)}
         handleAssetClick={this.handleAssetClick}
         handleLoadMore={this.handleLoadMore}
-        getDisplayURL={this.getDisplayURL}
+        displayURLs={displayURLs}
+        loadDisplayURL={this.loadDisplayURL}
         t={t}
       />
     );

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCard.js

@@ -1,5 +1,6 @@
 import React from 'react';
 import PropTypes from 'prop-types';
+import ImmutablePropTypes from 'react-immutable-proptypes';
 import styled from 'react-emotion';
 import { colors, borders, lengths } from 'netlify-cms-ui-default';
 
@@ -43,32 +44,35 @@ const CardText = styled.p`
   line-height: 1.3 !important;
 `;
 
-const MediaLibraryCard = ({
+class MediaLibraryCard extends React.Component {
-  isSelected,
+  render() {
-  displayURL,
+    const { isSelected, displayURL, text, onClick, width, margin, isPrivate, type } = this.props;
-  text,
+    const url = displayURL.get('url');
-  onClick,
+    return (
-  width,
+      <Card
-  margin,
+        isSelected={isSelected}
-  isPrivate,
+        onClick={onClick}
-  type,
+        width={width}
-}) => (
+        margin={margin}
-  <Card
+        tabIndex="-1"
-    isSelected={isSelected}
+        isPrivate={isPrivate}
-    onClick={onClick}
+      >
-    width={width}
+        <div>{url ? <CardImage src={url} /> : <CardFileIcon>{type}</CardFileIcon>}</div>
-    margin={margin}
+        <CardText>{text}</CardText>
-    tabIndex="-1"
+      </Card>
-    isPrivate={isPrivate}
+    );
-  >
+  }
-    <div>{displayURL ? <CardImage src={displayURL} /> : <CardFileIcon>{type}</CardFileIcon>}</div>
+  UNSAFE_componentWillMount() {
-    <CardText>{text}</CardText>
+    const { displayURL, loadDisplayURL } = this.props;
-  </Card>
+    if (!displayURL || (!displayURL.url && !displayURL.isFetching && !displayURL.err)) {
-);
+      loadDisplayURL();
+    }
+  }
+}
 
 MediaLibraryCard.propTypes = {
   isSelected: PropTypes.bool,
-  displayURL: PropTypes.string,
+  displayURL: ImmutablePropTypes.map.isRequired,
   text: PropTypes.string.isRequired,
   onClick: PropTypes.func.isRequired,
   width: PropTypes.string.isRequired,

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCardGrid.js

@@ -3,6 +3,7 @@ import PropTypes from 'prop-types';
 import styled from 'react-emotion';
 import Waypoint from 'react-waypoint';
 import MediaLibraryCard from './MediaLibraryCard';
+import { Map } from 'immutable';
 import { colors } from 'netlify-cms-ui-default';
 
 const CardGridContainer = styled.div`
@@ -33,7 +34,8 @@ const MediaLibraryCardGrid = ({
   cardWidth,
   cardMargin,
   isPrivate,
-  getDisplayURL,
+  displayURLs,
+  loadDisplayURL,
 }) => (
   <CardGridContainer innerRef={setScrollContainerRef}>
     <CardGrid>
@@ -46,7 +48,8 @@ const MediaLibraryCardGrid = ({
           width={cardWidth}
           margin={cardMargin}
           isPrivate={isPrivate}
-          displayURL={file.isViewableImage && getDisplayURL(file)}
+          displayURL={displayURLs.get(file.id, Map())}
+          loadDisplayURL={() => loadDisplayURL(file)}
           type={file.type}
         />
       ))}
@@ -76,7 +79,7 @@ MediaLibraryCardGrid.propTypes = {
   paginatingMessage: PropTypes.string,
   cardWidth: PropTypes.string.isRequired,
   cardMargin: PropTypes.string.isRequired,
-  getDisplayURL: PropTypes.func.isRequired,
+  loadDisplayURL: PropTypes.func.isRequired,
   isPrivate: PropTypes.bool,
 };
 

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryModal.js

@@ -93,7 +93,8 @@ const MediaLibraryModal = ({
   setScrollContainerRef,
   handleAssetClick,
   handleLoadMore,
-  getDisplayURL,
+  loadDisplayURL,
+  displayURLs,
   t,
 }) => {
   const filteredFiles = forImage ? handleFilter(files) : files;
@@ -171,7 +172,8 @@ const MediaLibraryModal = ({
         cardWidth={cardWidth}
         cardMargin={cardMargin}
         isPrivate={privateUpload}
-        getDisplayURL={getDisplayURL}
+        loadDisplayURL={loadDisplayURL}
+        displayURLs={displayURLs}
       />
     </StyledModal>
   );
@@ -179,11 +181,13 @@ const MediaLibraryModal = ({
 
 const fileShape = {
   key: PropTypes.string.isRequired,
+  id: PropTypes.string.isRequired,
   name: PropTypes.string.isRequired,
-  size: PropTypes.number.isRequired,
+  size: PropTypes.number,
   queryOrder: PropTypes.number,
-  url: PropTypes.string.isRequired,
+  url: PropTypes.string,
   urlIsPublicPath: PropTypes.bool,
+  getDisplayURL: PropTypes.func.isRequired,
 };
 
 MediaLibraryModal.propTypes = {
@@ -213,7 +217,7 @@ MediaLibraryModal.propTypes = {
   setScrollContainerRef: PropTypes.func.isRequired,
   handleAssetClick: PropTypes.func.isRequired,
   handleLoadMore: PropTypes.func.isRequired,
-  getDisplayURL: PropTypes.func.isRequired,
+  loadDisplayURL: PropTypes.func.isRequired,
   t: PropTypes.func.isRequired,
 };
 

packages/netlify-cms-core/src/reducers/mediaLibrary.js

@@ -169,10 +169,14 @@ const mediaLibrary = (state = Map(defaultState), action) => {
 
     case MEDIA_DISPLAY_URL_FAILURE: {
       const displayURLPath = ['displayURLs', action.payload.key];
-      return state
+      return (
-        .setIn([...displayURLPath, 'isFetching'], false)
+        state
-        .setIn([...displayURLPath, 'err'], action.payload.err)
+          .setIn([...displayURLPath, 'isFetching'], false)
-        .deleteIn([...displayURLPath, 'url']);
+          // make sure that err is set so the CMS won't attempt to load
+          // the image again
+          .setIn([...displayURLPath, 'err'], action.payload.err || true)
+          .deleteIn([...displayURLPath, 'url'])
+      );
     }
     default:
       return state;

packages/netlify-cms-lib-util/package.json

@@ -16,7 +16,8 @@
     "build": "cross-env NODE_ENV=production webpack"
   },
   "dependencies": {
-    "localforage": "^1.4.2"
+    "localforage": "^1.4.2",
+    "js-sha256": "^0.9.0"
   },
   "devDependencies": {
     "cross-env": "^5.2.0",

packages/netlify-cms-lib-util/src/getBlobSHA.js

@@ -0,0 +1,12 @@
+import sha256 from 'js-sha256';
+
+export default blob =>
+  new Promise((resolve, reject) => {
+    const fr = new FileReader();
+    fr.onload = ({ target: { result } }) => resolve(sha256(result));
+    fr.onerror = err => {
+      fr.abort();
+      reject(err);
+    };
+    fr.readAsArrayBuffer(blob);
+  });

packages/netlify-cms-lib-util/src/index.js

@@ -7,3 +7,4 @@ export { filterPromises, resolvePromiseProperties, then } from './promise';
 export unsentRequest from './unsentRequest';
 export { filterByPropExtension, parseResponse, responseParser } from './backendUtil';
 export loadScript from './loadScript';
+export getBlobSHA from './getBlobSHA';

website/content/docs/netlify-large-media.md

@@ -0,0 +1,38 @@
+---
+title: Netlify Large Media
+group: media
+weight: 10
+---
+
+[Netlify Large Media](https://www.netlify.com/features/large-media/) is a [Git LFS](https://git-lfs.github.com/) implementation for repositories connected to Netlify sites. This means that you can use Git to work with large asset files like images, audio, and video, without bloating your repository. It does this by replacing the asset files in your repository with text pointer files, then uploading the assets to the Neltify Large Media storage service.
+
+If you have a Netlify site with Large Media enabled, Netlify CMS (version 2.5.0 and above) will handle Large Media asset files seamlessly, in the same way as files stored directly in the repository.
+
+## Requirements
+
+To use Netlify Large Media with Netlify CMS, you will need to do the following:
+
+- [Upgrade Netlify CMS](/docs/update-the-cms-version/) to version 2.5.0 or above
+- Configure Netlify CMS to use the [Git Gateway backend with Netlify Identity](/docs/authentication-backends/#git-gateway-with-netlify-identity).
+- Configure the Netlify site and connected repository to use Large Media, following the [Large Media docs on Netlify](https://www.netlify.com/docs/large-media/).
+
+When these are complete, you can use Netlify CMS as normal, and the configured asset files will automatically be handled by Netlify Large Media.
+
+## Image transformations
+
+All JPEG, PNG, and GIF files that are handled with Netlify Large Media also have access to Netlify's on-demand image transformation service. This service allows you to request an image to match the dimensions you specify in a query parameter added to the image URL.
+
+You can learn more about this feature in [Netlify's image transformation docs](https://www.netlify.com/docs/image-transformation/).
+
+### Transformation control for media gallery thumbnails
+
+In repositories enabled with Netlify Large Media, Netlify CMS will use the image transformation query parameters to load thumbnail-sized images for the media gallery view. This makes images in the media gallery load significantly faster.
+
+You can disable the automatic image transformations with the `use_large_media_transforms_in_media_library` configuration setting, nested under `backend` in the CMS `config.yml` file:
+
+```
+backend:
+  name: git-gateway
+  ## Set to false to prevent transforming images in media gallery view
+  use_large_media_transforms_in_media_library: false
+```

yarn.lock

@@ -6151,7 +6151,7 @@ inherits@2.0.1:
   resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.1.tgz#b17d08d326b4423e568eff719f91b0b1cbdf69f1"
   integrity sha1-sX0I0ya0Qj5Wjv9xn5GwscvfafE=
 
-ini@^1.3.2, ini@^1.3.4, ini@~1.3.0:
+ini@^1.3.2, ini@^1.3.4, ini@^1.3.5, ini@~1.3.0:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/ini/-/ini-1.3.5.tgz#eee25f56db1c9ec6085e0c22778083f596abf927"
   integrity sha512-RZY5huIKCMRWDUqZlEi72f/lmXKMvuszcMBduliQ3nnWbx9X/ZBQO7DijMEYS9EhHBb2qacRUMtC7svLwe0lcw==
@@ -7180,6 +7180,11 @@ js-levenshtein@^1.1.3:
   resolved "https://registry.yarnpkg.com/js-levenshtein/-/js-levenshtein-1.1.3.tgz#3ef627df48ec8cf24bacf05c0f184ff30ef413c5"
   integrity sha512-/812MXr9RBtMObviZ8gQBhHO8MOrGj8HlEE+4ccMTElNA/6I3u39u+bhny55Lk921yn44nSZFy9naNLElL5wgQ==
 
+js-sha256@^0.9.0:
+  version "0.9.0"
+  resolved "https://registry.yarnpkg.com/js-sha256/-/js-sha256-0.9.0.tgz#0b89ac166583e91ef9123644bd3c5334ce9d0966"
+  integrity sha512-sga3MHh9sgQN2+pJ9VYZ+1LPwXOxuBJBA5nrR5/ofPfuiJBE2hnjsaN8se8JznOmGLN2p49Pe5U/ttafcs/apA==
+
 js-tokens@^3.0.0, js-tokens@^3.0.2:
   version "3.0.2"
   resolved "https://registry.yarnpkg.com/js-tokens/-/js-tokens-3.0.2.tgz#9866df395102130e38f7f996bceb65443209c25b"