feat: add media lib virtualization (#3381)

This commit is contained in:
Erez Rokah
2020-03-09 08:45:42 +01:00
committed by GitHub
parent 9c0f618148
commit 92e76011e7
10 changed files with 219 additions and 194 deletions

View File

@ -1,6 +1,6 @@
import GoTrue from 'gotrue-js';
import jwtDecode from 'jwt-decode';
import { fromPairs, get, pick, intersection, unzip } from 'lodash';
import { get, pick, intersection } from 'lodash';
import ini from 'ini';
import {
APIError,
@ -21,9 +21,9 @@ import {
UnpublishedEntryMediaFile,
parsePointerFile,
getLargeMediaPatternsFromGitAttributesFile,
PointerFile,
getPointerFileForMediaFileObj,
getLargeMediaFilteredMediaFiles,
DisplayURLObject,
} from 'netlify-cms-lib-util';
import { GitHubBackend } from 'netlify-cms-backend-github';
import { GitLabBackend } from 'netlify-cms-backend-gitlab';
@ -75,12 +75,6 @@ interface NetlifyUser extends Credentials {
user_metadata: { full_name: string; avatar_url: string };
}
interface GetMediaDisplayURLArgs {
path: string;
original: { id: string; path: string } | string;
largeMedia: PointerFile;
}
export default class GitGateway implements Implementation {
config: Config;
api?: GitHubAPI | GitLabAPI | BitBucketAPI;
@ -278,12 +272,8 @@ export default class GitGateway implements Implementation {
const mediaFiles = await Promise.all(
files.map(async file => {
if (client.matchPath(file.path)) {
const { id, path } = file;
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(
[{ ...file, id }],
branch,
);
const url = await client.getDownloadURL(largeMediaDisplayURLs[id]);
const { path, id } = file;
const url = await this.getLargeMediaDisplayURL({ path, id }, branch);
return {
id,
name: basename(path),
@ -303,32 +293,7 @@ export default class GitGateway implements Implementation {
}
getMedia(mediaFolder = this.mediaFolder) {
return Promise.all([this.backend!.getMedia(mediaFolder), this.getLargeMediaClient()]).then(
async ([mediaFiles, largeMediaClient]) => {
if (!largeMediaClient.enabled) {
return mediaFiles.map(({ displayURL, ...rest }) => ({
...rest,
displayURL: { original: displayURL },
}));
}
if (mediaFiles.length === 0) {
return [];
}
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(mediaFiles);
return mediaFiles.map(({ id, displayURL, path, ...rest }) => {
return {
...rest,
id,
path,
displayURL: {
path,
original: displayURL,
largeMedia: largeMediaDisplayURLs[id],
},
};
});
},
);
return this.backend!.getMedia(mediaFolder);
}
// this method memoizes this._getLargeMediaClient so that there can
@ -382,79 +347,54 @@ export default class GitGateway implements Implementation {
},
);
}
async getLargeMediaDisplayURLs(
mediaFiles: { path: string; id: string | null }[],
async getLargeMediaDisplayURL(
{ path, id }: { path: string; id: string | null },
branch = this.branch,
) {
const client = await this.getLargeMediaClient();
const readFile = (
path: string,
id: string | null | undefined,
{ parseText }: { parseText: boolean },
) => this.api!.readFile(path, id, { branch, parseText });
const filesPromise = entriesByFiles(mediaFiles, readFile, 'Git-Gateway');
const items = await entriesByFiles([{ path, id }], readFile, 'Git-Gateway');
const entry = items[0];
const pointerFile = parsePointerFile(entry.data);
if (!pointerFile.sha) {
console.warn(`Failed parsing pointer file ${path}`);
return path;
}
return filesPromise
.then(items =>
items.map(({ file: { id, path }, data }) => {
const parsedPointerFile = parsePointerFile(data);
if (!parsedPointerFile.sha) {
console.warn(`Failed parsing pointer file ${path}`);
}
return [
{
pointerId: id,
resourceId: parsedPointerFile.sha,
},
parsedPointerFile,
];
}),
)
.then(unzip)
.then(([idMaps, files]) =>
Promise.all([
idMaps as { pointerId: string; resourceId: string }[],
client.getResourceDownloadURLArgs(files as PointerFile[]).then(r => fromPairs(r)),
]),
)
.then(([idMaps, resourceMap]) =>
idMaps.map(({ pointerId, resourceId }) => [pointerId, resourceMap[resourceId]]),
)
.then(fromPairs);
const client = await this.getLargeMediaClient();
const url = await client.getDownloadURL(pointerFile);
return url;
}
getMediaDisplayURL(displayURL: DisplayURL) {
const {
path,
original,
largeMedia: largeMediaDisplayURL,
} = (displayURL as unknown) as GetMediaDisplayURLArgs;
return this.getLargeMediaClient().then(client => {
if (client.enabled && client.matchPath(path)) {
return client.getDownloadURL(largeMediaDisplayURL);
}
if (typeof original === 'string') {
return original;
}
if (this.backend!.getMediaDisplayURL) {
return this.backend!.getMediaDisplayURL(original);
}
const err = new Error(
`getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
) as Error & {
displayURL: DisplayURL;
};
err.displayURL = displayURL;
return Promise.reject(err);
});
async getMediaDisplayURL(displayURL: DisplayURL) {
const { path, id } = displayURL as DisplayURLObject;
const client = await this.getLargeMediaClient();
if (client.enabled && client.matchPath(path)) {
return this.getLargeMediaDisplayURL({ path, id });
}
if (typeof displayURL === 'string') {
return displayURL;
}
if (this.backend!.getMediaDisplayURL) {
return this.backend!.getMediaDisplayURL(displayURL);
}
const err = new Error(
`getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
) as Error & {
displayURL: DisplayURL;
};
err.displayURL = displayURL;
return Promise.reject(err);
}
async getMediaFile(path: string) {
const client = await this.getLargeMediaClient();
if (client.enabled && client.matchPath(path)) {
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs([{ path, id: null }]);
const url = await client.getDownloadURL(Object.values(largeMediaDisplayURLs)[0]);
const url = await this.getLargeMediaDisplayURL({ path, id: null });
return {
id: url,
name: basename(path),

View File

@ -66,11 +66,6 @@ const getDownloadURL = (
return Promise.resolve('');
});
const getResourceDownloadURLArgs = (_clientConfig: ClientConfig, objects: PointerFile[]) => {
const result = objects.map(({ sha }) => [sha, { sha }]) as [string, { sha: string }][];
return Promise.resolve(result);
};
const uploadOperation = (objects: PointerFile[]) => ({
operation: 'upload',
transfers: ['basic'],
@ -129,7 +124,6 @@ const configureFn = (config: ClientConfig, fn: Function) => (...args: unknown[])
const clientFns: Record<string, Function> = {
resourceExists,
getResourceUploadURLs,
getResourceDownloadURLArgs,
getDownloadURL,
uploadResource,
matchPath,
@ -138,7 +132,6 @@ const clientFns: Record<string, Function> = {
export type Client = {
resourceExists: (pointer: PointerFile) => Promise<boolean | undefined>;
getResourceUploadURLs: (objects: PointerFile[]) => Promise<string>;
getResourceDownloadURLArgs: (objects: PointerFile[]) => Promise<[string, { sha: string }][]>;
getDownloadURL: (pointer: PointerFile) => Promise<string>;
uploadResource: (pointer: PointerFile, blob: Blob) => Promise<string>;
matchPath: (path: string) => boolean;

View File

@ -56,7 +56,9 @@
"react-sortable-hoc": "^1.0.0",
"react-split-pane": "^0.1.85",
"react-topbar-progress-indicator": "^2.0.0",
"react-virtualized-auto-sizer": "^1.0.2",
"react-waypoint": "^8.1.0",
"react-window": "^1.8.5",
"redux": "^4.0.1",
"redux-notifications": "^4.0.1",
"redux-optimist": "^1.0.0",

View File

@ -8,7 +8,7 @@ const IMAGE_HEIGHT = 160;
const Card = styled.div`
width: ${props => props.width};
height: 240px;
height: ${props => props.height};
margin: ${props => props.margin};
border: ${borders.textField};
border-color: ${props => props.isSelected && colors.active};
@ -71,6 +71,7 @@ class MediaLibraryCard extends React.Component {
onClick,
draftText,
width,
height,
margin,
isPrivate,
type,
@ -83,6 +84,7 @@ class MediaLibraryCard extends React.Component {
isSelected={isSelected}
onClick={onClick}
width={width}
height={height}
margin={margin}
tabIndex="-1"
isPrivate={isPrivate}
@ -114,6 +116,7 @@ MediaLibraryCard.propTypes = {
onClick: PropTypes.func.isRequired,
draftText: PropTypes.string.isRequired,
width: PropTypes.string.isRequired,
height: PropTypes.string.isRequired,
margin: PropTypes.string.isRequired,
isPrivate: PropTypes.bool,
type: PropTypes.string,

View File

@ -5,6 +5,143 @@ import Waypoint from 'react-waypoint';
import MediaLibraryCard from './MediaLibraryCard';
import { Map } from 'immutable';
import { colors } from 'netlify-cms-ui-default';
import { FixedSizeGrid as Grid } from 'react-window';
import AutoSizer from 'react-virtualized-auto-sizer';
const CardWrapper = props => {
const {
rowIndex,
columnIndex,
style,
data: {
mediaItems,
isSelectedFile,
onAssetClick,
cardDraftText,
cardWidth,
cardHeight,
isPrivate,
displayURLs,
loadDisplayURL,
columnCount,
gutter,
},
} = props;
const index = rowIndex * columnCount + columnIndex;
if (index >= mediaItems.length) {
return null;
}
const file = mediaItems[index];
return (
<div
style={{
...style,
left: style.left + gutter * columnIndex,
top: style.top + gutter,
width: style.width - gutter,
height: style.height - gutter,
}}
>
<MediaLibraryCard
key={file.key}
isSelected={isSelectedFile(file)}
text={file.name}
onClick={() => onAssetClick(file)}
isDraft={file.draft}
draftText={cardDraftText}
width={cardWidth}
height={cardHeight}
margin={'0px'}
isPrivate={isPrivate}
displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
loadDisplayURL={() => loadDisplayURL(file)}
type={file.type}
isViewableImage={file.isViewableImage}
/>
</div>
);
};
const VirtualizedGrid = props => {
const { mediaItems, setScrollContainerRef } = props;
return (
<CardGridContainer ref={setScrollContainerRef}>
<AutoSizer>
{({ height, width }) => {
const cardWidth = parseInt(props.cardWidth, 10);
const cardHeight = parseInt(props.cardHeight, 10);
const gutter = parseInt(props.cardMargin, 10);
const columnWidth = cardWidth + gutter;
const rowHeight = cardHeight + gutter;
const columnCount = Math.floor(width / columnWidth);
const rowCount = Math.ceil(mediaItems.length / columnCount);
return (
<Grid
columnCount={columnCount}
columnWidth={columnWidth}
rowCount={rowCount}
rowHeight={rowHeight}
width={width}
height={height}
itemData={{ ...props, gutter, columnCount }}
>
{CardWrapper}
</Grid>
);
}}
</AutoSizer>
</CardGridContainer>
);
};
const PaginatedGrid = ({
setScrollContainerRef,
mediaItems,
isSelectedFile,
onAssetClick,
cardDraftText,
cardWidth,
cardHeight,
cardMargin,
isPrivate,
displayURLs,
loadDisplayURL,
canLoadMore,
onLoadMore,
isPaginating,
paginatingMessage,
}) => {
return (
<CardGridContainer ref={setScrollContainerRef}>
<CardGrid>
{mediaItems.map(file => (
<MediaLibraryCard
key={file.key}
isSelected={isSelectedFile(file)}
text={file.name}
onClick={() => onAssetClick(file)}
isDraft={file.draft}
draftText={cardDraftText}
width={cardWidth}
height={cardHeight}
margin={cardMargin}
isPrivate={isPrivate}
displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
loadDisplayURL={() => loadDisplayURL(file)}
type={file.type}
isViewableImage={file.isViewableImage}
/>
))}
{!canLoadMore ? null : <Waypoint onEnter={onLoadMore} />}
</CardGrid>
{!isPaginating ? null : (
<PaginatingMessage isPrivate={isPrivate}>{paginatingMessage}</PaginatingMessage>
)}
</CardGridContainer>
);
};
const CardGridContainer = styled.div`
overflow-y: auto;
@ -23,48 +160,13 @@ const PaginatingMessage = styled.h1`
color: ${props => props.isPrivate && colors.textFieldBorder};
`;
const MediaLibraryCardGrid = ({
setScrollContainerRef,
mediaItems,
isSelectedFile,
onAssetClick,
canLoadMore,
onLoadMore,
isPaginating,
paginatingMessage,
cardDraftText,
cardWidth,
cardMargin,
isPrivate,
displayURLs,
loadDisplayURL,
}) => (
<CardGridContainer ref={setScrollContainerRef}>
<CardGrid>
{mediaItems.map(file => (
<MediaLibraryCard
key={file.key}
isSelected={isSelectedFile(file)}
text={file.name}
onClick={() => onAssetClick(file)}
isDraft={file.draft}
draftText={cardDraftText}
width={cardWidth}
margin={cardMargin}
isPrivate={isPrivate}
displayURL={displayURLs.get(file.id, file.url ? Map({ url: file.url }) : Map())}
loadDisplayURL={() => loadDisplayURL(file)}
type={file.type}
isViewableImage={file.isViewableImage}
/>
))}
{!canLoadMore ? null : <Waypoint onEnter={onLoadMore} />}
</CardGrid>
{!isPaginating ? null : (
<PaginatingMessage isPrivate={isPrivate}>{paginatingMessage}</PaginatingMessage>
)}
</CardGridContainer>
);
const MediaLibraryCardGrid = props => {
const { canLoadMore, isPaginating } = props;
if (canLoadMore || isPaginating) {
return <PaginatedGrid {...props} />;
}
return <VirtualizedGrid {...props} />;
};
MediaLibraryCardGrid.propTypes = {
setScrollContainerRef: PropTypes.func.isRequired,

View File

@ -17,6 +17,7 @@ import { colors } from 'netlify-cms-ui-default';
* widths per breakpoint.
*/
const cardWidth = `280px`;
const cardHeight = `240px`;
const cardMargin = `10px`;
/**
@ -172,6 +173,7 @@ const MediaLibraryModal = ({
paginatingMessage={t('mediaLibrary.mediaLibraryModal.loading')}
cardDraftText={t('mediaLibrary.mediaLibraryCard.draft')}
cardWidth={cardWidth}
cardHeight={cardHeight}
cardMargin={cardMargin}
isPrivate={privateUpload}
loadDisplayURL={loadDisplayURL}

View File

@ -10,6 +10,7 @@ describe('MediaLibraryCard', () => {
onClick: jest.fn(),
draftText: 'Draft',
width: '100px',
height: '240px',
margin: '10px',
isViewableImage: true,
loadDisplayURL: jest.fn(),

View File

@ -52,6 +52,7 @@ exports[`MediaLibraryCard should match snapshot for draft image 1`] = `
<div
class="emotion-8 emotion-9"
height="240px"
tabindex="-1"
width="100px"
>
@ -122,6 +123,7 @@ exports[`MediaLibraryCard should match snapshot for non draft image 1`] = `
<div
class="emotion-6 emotion-7"
height="240px"
tabindex="-1"
width="100px"
>
@ -188,6 +190,7 @@ exports[`MediaLibraryCard should match snapshot for non viewable image 1`] = `
<div
class="emotion-6 emotion-7"
height="240px"
tabindex="-1"
width="100px"
>

View File

@ -4,10 +4,7 @@ import { AsyncLock } from './asyncLock';
export type DisplayURLObject = { id: string; path: string };
export type DisplayURL =
| DisplayURLObject
| string
| { original: DisplayURL; path?: string; largeMedia?: string };
export type DisplayURL = DisplayURLObject | string;
export interface ImplementationMediaFile {
name: string;