gcg/static-cms
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

feat: add media lib virtualization (#3381)

Browse Source
This commit is contained in:
Erez Rokah
2020-03-09 08:45:42 +01:00
committed by GitHub
parent 9c0f618148
commit 92e76011e7
10 changed files with 219 additions and 194 deletions
Download Patch File Download Diff File Expand all files Collapse all files

134
packages/netlify-cms-backend-git-gateway/src/implementation.ts
View File

@ -1,6 +1,6 @@
import GoTrue from 'gotrue-js';
import jwtDecode from 'jwt-decode';
import { fromPairs, get, pick, intersection, unzip } from 'lodash';
import { get, pick, intersection } from 'lodash';
import ini from 'ini';
import {
APIError,
@ -21,9 +21,9 @@ import {
UnpublishedEntryMediaFile,
parsePointerFile,
getLargeMediaPatternsFromGitAttributesFile,
PointerFile,
getPointerFileForMediaFileObj,
getLargeMediaFilteredMediaFiles,
DisplayURLObject,
} from 'netlify-cms-lib-util';
import { GitHubBackend } from 'netlify-cms-backend-github';
import { GitLabBackend } from 'netlify-cms-backend-gitlab';
@ -75,12 +75,6 @@ interface NetlifyUser extends Credentials {
user_metadata: { full_name: string; avatar_url: string };
}
interface GetMediaDisplayURLArgs {
path: string;
original: { id: string; path: string } | string;
largeMedia: PointerFile;
}
export default class GitGateway implements Implementation {
config: Config;
api?: GitHubAPI | GitLabAPI | BitBucketAPI;
@ -278,12 +272,8 @@ export default class GitGateway implements Implementation {
const mediaFiles = await Promise.all(
files.map(async file => {
if (client.matchPath(file.path)) {
const { id, path } = file;
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(
[{ ...file, id }],
branch,
);
const url = await client.getDownloadURL(largeMediaDisplayURLs[id]);
const { path, id } = file;
const url = await this.getLargeMediaDisplayURL({ path, id }, branch);
return {
id,
name: basename(path),
@ -303,32 +293,7 @@ export default class GitGateway implements Implementation {
}
getMedia(mediaFolder = this.mediaFolder) {
return Promise.all([this.backend!.getMedia(mediaFolder), this.getLargeMediaClient()]).then(
async ([mediaFiles, largeMediaClient]) => {
if (!largeMediaClient.enabled) {
return mediaFiles.map(({ displayURL, ...rest }) => ({
...rest,
displayURL: { original: displayURL },
}));
}
if (mediaFiles.length === 0) {
return [];
}
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs(mediaFiles);
return mediaFiles.map(({ id, displayURL, path, ...rest }) => {
return {
...rest,
id,
path,
displayURL: {
path,
original: displayURL,
largeMedia: largeMediaDisplayURLs[id],
},
};
});
},
);
return this.backend!.getMedia(mediaFolder);
}
// this method memoizes this._getLargeMediaClient so that there can
@ -382,79 +347,54 @@ export default class GitGateway implements Implementation {
},
);
}
async getLargeMediaDisplayURLs(
mediaFiles: { path: string; id: string | null }[],
async getLargeMediaDisplayURL(
{ path, id }: { path: string; id: string | null },
branch = this.branch,
) {
const client = await this.getLargeMediaClient();
const readFile = (
path: string,
id: string | null | undefined,
{ parseText }: { parseText: boolean },
) => this.api!.readFile(path, id, { branch, parseText });
const filesPromise = entriesByFiles(mediaFiles, readFile, 'Git-Gateway');
const items = await entriesByFiles([{ path, id }], readFile, 'Git-Gateway');
const entry = items[0];
const pointerFile = parsePointerFile(entry.data);
if (!pointerFile.sha) {
console.warn(`Failed parsing pointer file ${path}`);
return path;
}
return filesPromise
.then(items =>
items.map(({ file: { id, path }, data }) => {
const parsedPointerFile = parsePointerFile(data);
if (!parsedPointerFile.sha) {
console.warn(`Failed parsing pointer file ${path}`);
}
return [
{
pointerId: id,
resourceId: parsedPointerFile.sha,
},
parsedPointerFile,
];
}),
)
.then(unzip)
.then(([idMaps, files]) =>
Promise.all([
idMaps as { pointerId: string; resourceId: string }[],
client.getResourceDownloadURLArgs(files as PointerFile[]).then(r => fromPairs(r)),
]),
)
.then(([idMaps, resourceMap]) =>
idMaps.map(({ pointerId, resourceId }) => [pointerId, resourceMap[resourceId]]),
)
.then(fromPairs);
const client = await this.getLargeMediaClient();
const url = await client.getDownloadURL(pointerFile);
return url;
}
getMediaDisplayURL(displayURL: DisplayURL) {
const {
path,
original,
largeMedia: largeMediaDisplayURL,
} = (displayURL as unknown) as GetMediaDisplayURLArgs;
return this.getLargeMediaClient().then(client => {
if (client.enabled && client.matchPath(path)) {
return client.getDownloadURL(largeMediaDisplayURL);
}
if (typeof original === 'string') {
return original;
}
if (this.backend!.getMediaDisplayURL) {
return this.backend!.getMediaDisplayURL(original);
}
const err = new Error(
`getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
) as Error & {
displayURL: DisplayURL;
};
err.displayURL = displayURL;
return Promise.reject(err);
});
async getMediaDisplayURL(displayURL: DisplayURL) {
const { path, id } = displayURL as DisplayURLObject;
const client = await this.getLargeMediaClient();
if (client.enabled && client.matchPath(path)) {
return this.getLargeMediaDisplayURL({ path, id });
}
if (typeof displayURL === 'string') {
return displayURL;
}
if (this.backend!.getMediaDisplayURL) {
return this.backend!.getMediaDisplayURL(displayURL);
}
const err = new Error(
`getMediaDisplayURL is not implemented by the ${this.backendType} backend, but the backend returned a displayURL which was not a string!`,
) as Error & {
displayURL: DisplayURL;
};
err.displayURL = displayURL;
return Promise.reject(err);
}
async getMediaFile(path: string) {
const client = await this.getLargeMediaClient();
if (client.enabled && client.matchPath(path)) {
const largeMediaDisplayURLs = await this.getLargeMediaDisplayURLs([{ path, id: null }]);
const url = await client.getDownloadURL(Object.values(largeMediaDisplayURLs)[0]);
const url = await this.getLargeMediaDisplayURL({ path, id: null });
return {
id: url,
name: basename(path),

7
packages/netlify-cms-backend-git-gateway/src/netlify-lfs-client.ts
View File

@ -66,11 +66,6 @@ const getDownloadURL = (
return Promise.resolve('');
});
const getResourceDownloadURLArgs = (_clientConfig: ClientConfig, objects: PointerFile[]) => {
const result = objects.map(({ sha }) => [sha, { sha }]) as [string, { sha: string }][];
return Promise.resolve(result);
};
const uploadOperation = (objects: PointerFile[]) => ({
operation: 'upload',
transfers: ['basic'],
@ -129,7 +124,6 @@ const configureFn = (config: ClientConfig, fn: Function) => (...args: unknown[])
const clientFns: Record<string, Function> = {
resourceExists,
getResourceUploadURLs,
getResourceDownloadURLArgs,
getDownloadURL,
uploadResource,
matchPath,
@ -138,7 +132,6 @@ const clientFns: Record<string, Function> = {
export type Client = {
resourceExists: (pointer: PointerFile) => Promise<boolean | undefined>;
getResourceUploadURLs: (objects: PointerFile[]) => Promise<string>;
getResourceDownloadURLArgs: (objects: PointerFile[]) => Promise<[string, { sha: string }][]>;
getDownloadURL: (pointer: PointerFile) => Promise<string>;
uploadResource: (pointer: PointerFile, blob: Blob) => Promise<string>;
matchPath: (path: string) => boolean;