fix(gitlab): fetch media library images through API (#1433)

packages/netlify-cms-backend-bitbucket/src/API.js

@@ -52,7 +52,6 @@ export default class API {
   processFile = file => ({
     ...file,
     name: basename(file.path),
-    download_url: file.links.self.href,
 
     // BitBucket does not return file SHAs, but it does give us the
     // commit SHA. Since the commit SHA will change if any files do,

packages/netlify-cms-backend-bitbucket/src/implementation.js

@@ -209,7 +209,7 @@ export default class Bitbucket {
     const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
 
     return this.api.listAllFiles(this.config.get('media_folder')).then(files =>
-      files.map(({ id, name, download_url, path }) => {
+      files.map(({ id, name, path }) => {
         const getBlobPromise = () =>
           new Promise((resolve, reject) =>
             sem.take(() =>
@@ -220,7 +220,7 @@ export default class Bitbucket {
             ),
           );
 
-        return { id, name, getBlobPromise, url: download_url, path };
+        return { id, name, getBlobPromise, path };
       }),
     );
   }

packages/netlify-cms-backend-gitlab/src/API.js

@@ -1,6 +1,6 @@
 import { localForage, unsentRequest, then, APIError, Cursor } from 'netlify-cms-lib-util';
 import { Base64 } from 'js-base64';
-import { List, Map } from 'immutable';
+import { fromJS, List, Map } from 'immutable';
 import { flow, partial, result } from 'lodash';
 
 export default class API {
@@ -29,22 +29,48 @@ export default class API {
       p => p.catch(err => Promise.reject(new APIError(err.message, null, 'GitLab'))),
     ])(req);
 
-  parseResponse = async (res, { expectingOk = true, expectingFormat = false }) => {
-    const contentType = res.headers.get('Content-Type');
-    const isJSON = contentType === 'application/json';
+  catchFormatErrors = (format, formatter) => res => {
+    try {
+      return formatter(res);
+    } catch (err) {
+      throw new Error(
+        `Response cannot be parsed into the expected format (${format}): ${err.message}`,
+      );
+    }
+  };
+
+  responseFormats = fromJS({
+    json: async res => {
+      const contentType = res.headers.get('Content-Type');
+      if (contentType !== 'application/json' && contentType !== 'text/json') {
+        throw new Error(`${contentType} is not a valid JSON Content-Type`);
+      }
+      return res.json();
+    },
+    text: async res => res.text(),
+    blob: async res => res.blob(),
+  }).mapEntries(([format, formatter]) => [format, this.catchFormatErrors(format, formatter)]);
+
+  parseResponse = async (res, { expectingOk = true, expectingFormat = 'text' }) => {
     let body;
     try {
-      body = await (expectingFormat === 'json' || isJSON ? res.json() : res.text());
+      const formatter = this.responseFormats.get(expectingFormat, false);
+      if (!formatter) {
+        throw new Error(`${expectingFormat} is not a supported response format.`);
+      }
+      body = await formatter(res);
     } catch (err) {
       throw new APIError(err.message, res.status, 'GitLab');
     }
     if (expectingOk && !res.ok) {
+      const isJSON = expectingFormat === 'json';
       throw new APIError(isJSON && body.message ? body.message : body, res.status, 'GitLab');
     }
     return body;
   };
 
   responseToJSON = res => this.parseResponse(res, { expectingFormat: 'json' });
+  responseToBlob = res => this.parseResponse(res, { expectingFormat: 'blob' });
   responseToText = res => this.parseResponse(res, { expectingFormat: 'text' });
   requestJSON = req => this.request(req).then(this.responseToJSON);
   requestText = req => this.request(req).then(this.responseToText);
@@ -64,30 +90,23 @@ export default class API {
       return false;
     });
 
-  readFile = async (path, sha, ref = this.branch) => {
-    const cachedFile = sha ? await localForage.getItem(`gl.${sha}`) : null;
+  readFile = async (path, sha, { ref = this.branch, parseText = true } = {}) => {
+    const cacheKey = parseText ? `gl.${sha}` : `gl.${sha}.blob`;
+    const cachedFile = sha ? await localForage.getItem(cacheKey) : null;
     if (cachedFile) {
       return cachedFile;
     }
-    const result = await this.requestText({
+    const result = await this.request({
       url: `${this.repoURL}/repository/files/${encodeURIComponent(path)}/raw`,
       params: { ref },
       cache: 'no-store',
-    });
+    }).then(parseText ? this.responseToText : this.responseToBlob);
     if (sha) {
-      localForage.setItem(`gl.${sha}`, result);
+      localForage.setItem(cacheKey, result);
     }
     return result;
   };
 
-  fileDownloadURL = (path, ref = this.branch) =>
-    unsentRequest.toURL(
-      this.buildRequest({
-        url: `${this.repoURL}/repository/files/${encodeURIComponent(path)}/raw`,
-        params: { ref },
-      }),
-    );
-
   getCursorFromHeaders = headers => {
     // indices and page counts are assumed to be zero-based, but the
     // indices and page counts returned from GitLab are one-based

packages/netlify-cms-backend-gitlab/src/implementation.js

@@ -132,13 +132,21 @@ export default class GitLab {
   }
 
   getMedia() {
+    const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
+
     return this.api.listAllFiles(this.config.get('media_folder')).then(files =>
       files.map(({ id, name, path }) => {
-        const url = new URL(this.api.fileDownloadURL(path));
-        if (url.pathname.match(/.svg$/)) {
-          url.search += (url.search.slice(1) === '' ? '?' : '&') + 'sanitize=true';
-        }
-        return { id, name, url: url.href, path };
+        const getBlobPromise = () =>
+          new Promise((resolve, reject) =>
+            sem.take(() =>
+              this.api
+                .readFile(path, id, { parseText: false })
+                .then(resolve, reject)
+                .finally(() => sem.leave()),
+            ),
+          );
+
+        return { id, name, getBlobPromise, path };
       }),
     );
   }
@@ -150,8 +158,8 @@ export default class GitLab {
   async persistMedia(mediaFile, options = {}) {
     await this.api.persistFiles([mediaFile], options);
     const { value, path, fileObj } = mediaFile;
-    const url = this.api.fileDownloadURL(path);
-    return { name: value, size: fileObj.size, url, path: trimStart(path, '/') };
+    const getBlobPromise = () => Promise.resolve(fileObj);
+    return { name: value, size: fileObj.size, getBlobPromise, path: trimStart(path, '/') };
   }
 
   deleteFile(path, commitMessage, options) {

packages/netlify-cms-core/src/actions/mediaLibrary.js

@@ -21,6 +21,9 @@ export const MEDIA_PERSIST_FAILURE = 'MEDIA_PERSIST_FAILURE';
 export const MEDIA_DELETE_REQUEST = 'MEDIA_DELETE_REQUEST';
 export const MEDIA_DELETE_SUCCESS = 'MEDIA_DELETE_SUCCESS';
 export const MEDIA_DELETE_FAILURE = 'MEDIA_DELETE_FAILURE';
+export const MEDIA_DISPLAY_URL_REQUEST = 'MEDIA_DISPLAY_URL_REQUEST';
+export const MEDIA_DISPLAY_URL_SUCCESS = 'MEDIA_DISPLAY_URL_SUCCESS';
+export const MEDIA_DISPLAY_URL_FAILURE = 'MEDIA_DISPLAY_URL_FAILURE';
 
 export function openMediaLibrary(payload) {
   return { type: MEDIA_LIBRARY_OPEN, payload };
@@ -169,6 +172,24 @@ export function deleteMedia(file, opts = {}) {
   };
 }
 
+export function loadMediaDisplayURL(file) {
+  return async dispatch => {
+    const { getBlobPromise, id } = file;
+
+    if (id && getBlobPromise) {
+      try {
+        dispatch(mediaDisplayURLRequest(id));
+        const blob = await getBlobPromise();
+        const newURL = window.URL.createObjectURL(blob);
+        dispatch(mediaDisplayURLSuccess(id, newURL));
+        return newURL;
+      } catch (err) {
+        dispatch(mediaDisplayURLFailure(id, err));
+      }
+    }
+  };
+}
+
 export function mediaLoading(page) {
   return {
     type: MEDIA_LOAD_REQUEST,
@@ -221,3 +242,21 @@ export function mediaDeleteFailed(error, opts = {}) {
   const { privateUpload } = opts;
   return { type: MEDIA_DELETE_FAILURE, payload: { privateUpload } };
 }
+
+export function mediaDisplayURLRequest(key) {
+  return { type: MEDIA_DISPLAY_URL_REQUEST, payload: { key } };
+}
+
+export function mediaDisplayURLSuccess(key, url) {
+  return {
+    type: MEDIA_DISPLAY_URL_SUCCESS,
+    payload: { key, url },
+  };
+}
+
+export function mediaDisplayURLFailure(key, err) {
+  return {
+    type: MEDIA_DISPLAY_URL_FAILURE,
+    payload: { key, err },
+  };
+}

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibrary.js

@@ -1,6 +1,7 @@
 import React from 'react';
 import { connect } from 'react-redux';
 import { orderBy, map } from 'lodash';
+import { Map } from 'immutable';
 import fuzzy from 'fuzzy';
 import { resolvePath, fileExtension } from 'netlify-cms-lib-util';
 import {
@@ -8,6 +9,7 @@ import {
   persistMedia as persistMediaAction,
   deleteMedia as deleteMediaAction,
   insertMedia as insertMediaAction,
+  loadMediaDisplayURL as loadMediaDisplayURLAction,
   closeMediaLibrary as closeMediaLibraryAction,
 } from 'Actions/mediaLibrary';
 import MediaLibraryModal from './MediaLibraryModal';
@@ -53,6 +55,30 @@ class MediaLibrary extends React.Component {
     }
   }
 
+  getDisplayURL = file => {
+    const { isVisible, loadMediaDisplayURL, displayURLs } = this.props;
+
+    if (!isVisible) {
+      return '';
+    }
+
+    if (file && file.url) {
+      return file.url;
+    }
+
+    const { url, isFetching } = displayURLs.get(file.id, Map()).toObject();
+
+    if (url && url !== '') {
+      return url;
+    }
+
+    if (!isFetching) {
+      loadMediaDisplayURL(file);
+    }
+
+    return '';
+  };
+
   /**
    * Filter an array of file data to include only images.
    */
@@ -69,16 +95,18 @@ class MediaLibrary extends React.Component {
   toTableData = files => {
     const tableData =
       files &&
-      files.map(({ key, name, size, queryOrder, url, urlIsPublicPath }) => {
+      files.map(({ key, name, id, size, queryOrder, url, urlIsPublicPath, getBlobPromise }) => {
         const ext = fileExtension(name).toLowerCase();
         return {
           key,
+          id,
           name,
           type: ext.toUpperCase(),
           size,
           queryOrder,
           url,
           urlIsPublicPath,
+          getBlobPromise,
           isImage: IMAGE_EXTENSIONS.includes(ext),
           isViewableImage: IMAGE_EXTENSIONS_VIEWABLE.includes(ext),
         };
@@ -251,6 +279,7 @@ class MediaLibrary extends React.Component {
         setScrollContainerRef={ref => (this.scrollContainerRef = ref)}
         handleAssetClick={this.handleAssetClick}
         handleLoadMore={this.handleLoadMore}
+        getDisplayURL={this.getDisplayURL}
       />
     );
   }
@@ -265,6 +294,7 @@ const mapStateToProps = state => {
     isVisible: mediaLibrary.get('isVisible'),
     canInsert: mediaLibrary.get('canInsert'),
     files: mediaLibrary.get('files'),
+    displayURLs: mediaLibrary.get('displayURLs'),
     dynamicSearch: mediaLibrary.get('dynamicSearch'),
     dynamicSearchActive: mediaLibrary.get('dynamicSearchActive'),
     dynamicSearchQuery: mediaLibrary.get('dynamicSearchQuery'),
@@ -285,6 +315,7 @@ const mapDispatchToProps = {
   persistMedia: persistMediaAction,
   deleteMedia: deleteMediaAction,
   insertMedia: insertMediaAction,
+  loadMediaDisplayURL: loadMediaDisplayURLAction,
   closeMediaLibrary: closeMediaLibraryAction,
 };
 

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCard.js

@@ -36,7 +36,7 @@ const CardText = styled.p`
   line-height: 1.3 !important;
 `;
 
-const MediaLibraryCard = ({ isSelected, imageUrl, text, onClick, width, margin, isPrivate }) => (
+const MediaLibraryCard = ({ isSelected, displayURL, text, onClick, width, margin, isPrivate }) => (
   <Card
     isSelected={isSelected}
     onClick={onClick}
@@ -45,14 +45,14 @@ const MediaLibraryCard = ({ isSelected, imageUrl, text, onClick, width, margin,
     tabIndex="-1"
     isPrivate={isPrivate}
   >
-    <div>{imageUrl ? <CardImage src={imageUrl} /> : <CardImagePlaceholder />}</div>
+    <div>{displayURL ? <CardImage src={displayURL} /> : <CardImagePlaceholder />}</div>
     <CardText>{text}</CardText>
   </Card>
 );
 
 MediaLibraryCard.propTypes = {
   isSelected: PropTypes.bool,
-  imageUrl: PropTypes.string,
+  displayURL: PropTypes.string,
   text: PropTypes.string.isRequired,
   onClick: PropTypes.func.isRequired,
   width: PropTypes.string.isRequired,

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCardGrid.js

@@ -33,6 +33,7 @@ const MediaLibraryCardGrid = ({
   cardWidth,
   cardMargin,
   isPrivate,
+  getDisplayURL,
 }) => (
   <CardGridContainer innerRef={setScrollContainerRef}>
     <CardGrid>
@@ -40,12 +41,12 @@ const MediaLibraryCardGrid = ({
         <MediaLibraryCard
           key={file.key}
           isSelected={isSelectedFile(file)}
-          imageUrl={file.isViewableImage && file.url}
           text={file.name}
           onClick={() => onAssetClick(file)}
           width={cardWidth}
           margin={cardMargin}
           isPrivate={isPrivate}
+          displayURL={getDisplayURL(file)}
         />
       ))}
       {!canLoadMore ? null : <Waypoint onEnter={onLoadMore} />}
@@ -74,6 +75,7 @@ MediaLibraryCardGrid.propTypes = {
   paginatingMessage: PropTypes.string,
   cardWidth: PropTypes.string.isRequired,
   cardMargin: PropTypes.string.isRequired,
+  getDisplayURL: PropTypes.func.isRequired,
   isPrivate: PropTypes.bool,
 };
 

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryCardImage.js

@@ -0,0 +1,64 @@
+import React from 'react';
+import styled from 'react-emotion';
+
+const CardImage = styled.img`
+  width: 100%;
+  height: 160px;
+  object-fit: cover;
+  border-radius: 2px 2px 0 0;
+`;
+
+const CardImagePlaceholder = CardImage.withComponent(`div`);
+
+export default class MediaLibraryCardImage extends React.Component {
+  state = {
+    imageURL: '',
+    isFetching: false,
+  };
+
+  loadImage() {
+    const { image, getCachedImageURLByID, cacheImageURLByID } = this.props;
+    const { imageURL: existingImageURL, isFetching } = this.state;
+
+    if (existingImageURL !== '' || isFetching) {
+      return;
+    }
+
+    if (getCachedImageURLByID && image.key) {
+      const imageURL = getCachedImageURLByID(image.key);
+      if (imageURL) {
+        this.setState({ imageURL });
+        return;
+      }
+    }
+
+    if (image.url) {
+      this.setState({ imageURL: image.url });
+      if (image.key && cacheImageURLByID) {
+        cacheImageURLByID(image.key, image.url);
+      }
+      return;
+    }
+
+    if (image.getBlobPromise) {
+      this.setState({ isFetching: true });
+      image.getBlobPromise().then(blob => {
+        const imageURL = window.URL.createObjectURL(blob);
+        this.setState({ imageURL, isFetching: false });
+        if (image.key && cacheImageURLByID) {
+          cacheImageURLByID(image.key, imageURL);
+        }
+      });
+    }
+  }
+
+  render() {
+    const { imageURL, isFetching } = this.state;
+
+    if (imageURL === '' && !isFetching) {
+      this.loadImage();
+    }
+
+    return imageURL === '' ? <CardImagePlaceholder /> : <CardImage src={imageURL} />;
+  }
+}

packages/netlify-cms-core/src/components/MediaLibrary/MediaLibraryModal.js

@@ -92,6 +92,7 @@ const MediaLibraryModal = ({
   setScrollContainerRef,
   handleAssetClick,
   handleLoadMore,
+  getDisplayURL,
 }) => {
   const filteredFiles = forImage ? handleFilter(files) : files;
   const queriedFiles = !dynamicSearch && query ? handleQuery(query, filteredFiles) : filteredFiles;
@@ -156,6 +157,7 @@ const MediaLibraryModal = ({
         cardWidth={cardWidth}
         cardMargin={cardMargin}
         isPrivate={privateUpload}
+        getDisplayURL={getDisplayURL}
       />
     </StyledModal>
   );
@@ -197,6 +199,7 @@ MediaLibraryModal.propTypes = {
   setScrollContainerRef: PropTypes.func.isRequired,
   handleAssetClick: PropTypes.func.isRequired,
   handleLoadMore: PropTypes.func.isRequired,
+  getDisplayURL: PropTypes.func.isRequired,
 };
 
 export default MediaLibraryModal;

packages/netlify-cms-core/src/reducers/mediaLibrary.js

@@ -15,11 +15,18 @@ import {
   MEDIA_DELETE_REQUEST,
   MEDIA_DELETE_SUCCESS,
   MEDIA_DELETE_FAILURE,
+  MEDIA_DISPLAY_URL_REQUEST,
+  MEDIA_DISPLAY_URL_SUCCESS,
+  MEDIA_DISPLAY_URL_FAILURE,
 } from 'Actions/mediaLibrary';
 
-const mediaLibrary = (state = Map({ isVisible: false, controlMedia: Map() }), action) => {
+const mediaLibrary = (
+  state = Map({ isVisible: false, controlMedia: Map(), displayURLs: Map() }),
+  action,
+) => {
   const privateUploadChanged =
     state.get('privateUpload') !== get(action, ['payload', 'privateUpload']);
+  let displayURLPath;
   switch (action.type) {
     case MEDIA_LIBRARY_OPEN: {
       const { controlID, forImage, privateUpload } = action.payload || {};
@@ -108,13 +115,14 @@ const mediaLibrary = (state = Map({ isVisible: false, controlMedia: Map() }), ac
     case MEDIA_DELETE_REQUEST:
       return state.set('isDeleting', true);
     case MEDIA_DELETE_SUCCESS: {
-      const { key } = action.payload.file;
+      const { id, key } = action.payload.file;
       if (privateUploadChanged) {
         return state;
       }
       return state.withMutations(map => {
         const updatedFiles = map.get('files').filter(file => file.key !== key);
         map.set('files', updatedFiles);
+        map.deleteIn(['displayURLs', id]);
         map.set('isDeleting', false);
       });
     }
@@ -123,6 +131,23 @@ const mediaLibrary = (state = Map({ isVisible: false, controlMedia: Map() }), ac
         return state;
       }
       return state.set('isDeleting', false);
+
+    case MEDIA_DISPLAY_URL_REQUEST:
+      return state.setIn(['displayURLs', action.payload.key, 'isFetching'], true);
+
+    case MEDIA_DISPLAY_URL_SUCCESS:
+      displayURLPath = ['displayURLs', action.payload.key];
+      return state
+        .setIn([...displayURLPath, 'isFetching'], false)
+        .setIn([...displayURLPath, 'url'], action.payload.url);
+
+    case MEDIA_DISPLAY_URL_FAILURE:
+      displayURLPath = ['displayURLs', action.payload.key];
+      return state
+        .setIn([...displayURLPath, 'isFetching'], false)
+        .setIn([...displayURLPath, 'err'], action.payload.err)
+        .deleteIn([...displayURLPath, 'url']);
+
     default:
       return state;
   }