import semaphore, { Semaphore } from 'semaphore'; import { unionBy, sortBy } from 'lodash'; import Cursor from './Cursor'; import { AsyncLock } from './asyncLock'; import { FileMetadata } from './API'; import { basename } from './path'; export type DisplayURLObject = { id: string; path: string }; export type DisplayURL = DisplayURLObject | string; export interface ImplementationMediaFile { name: string; id: string; size?: number; displayURL?: DisplayURL; path: string; draft?: boolean; url?: string; file?: File; } export interface UnpublishedEntryMediaFile { id: string; path: string; } export interface ImplementationEntry { data: string; file: { path: string; label?: string; id?: string | null; author?: string; updatedOn?: string }; } export interface UnpublishedEntryDiff { id: string; path: string; newFile: boolean; } export interface UnpublishedEntry { slug: string; collection: string; status: string; diffs: UnpublishedEntryDiff[]; updatedAt: string; } export interface Map { get: (key: string, defaultValue?: T) => T; getIn: (key: string[], defaultValue?: T) => T; setIn: (key: string[], value: T) => Map; set: (key: string, value: T) => Map; } export type DataFile = { path: string; slug: string; raw: string; newPath?: string; }; export type AssetProxy = { path: string; fileObj?: File; toBase64?: () => Promise; }; export type Entry = { dataFiles: DataFile[]; assets: AssetProxy[]; }; export type PersistOptions = { newEntry?: boolean; commitMessage: string; collectionName?: string; useWorkflow?: boolean; unpublished?: boolean; status?: string; }; export type DeleteOptions = {}; export type Credentials = { token: string | {}; refresh_token?: string }; export type User = Credentials & { backendName?: string; login?: string; name: string; useOpenAuthoring?: boolean; }; export type Config = { backend: { repo?: string | null; open_authoring?: boolean; branch?: string; api_root?: string; squash_merges?: boolean; use_graphql?: boolean; preview_context?: string; identity_url?: string; gateway_url?: string; large_media_url?: string; use_large_media_transforms_in_media_library?: boolean; proxy_url?: string; auth_type?: string; app_id?: string; cms_label_prefix?: string; api_version?: string; }; media_folder: string; base_url?: string; site_id?: string; }; export interface Implementation { authComponent: () => void; restoreUser: (user: User) => Promise; authenticate: (credentials: Credentials) => Promise; logout: () => Promise | void | null; getToken: () => Promise; getEntry: (path: string) => Promise; entriesByFolder: ( folder: string, extension: string, depth: number, ) => Promise; entriesByFiles: (files: ImplementationFile[]) => Promise; getMediaDisplayURL?: (displayURL: DisplayURL) => Promise; getMedia: (folder?: string) => Promise; getMediaFile: (path: string) => Promise; persistEntry: (entry: Entry, opts: PersistOptions) => Promise; persistMedia: (file: AssetProxy, opts: PersistOptions) => Promise; deleteFiles: (paths: string[], commitMessage: string) => Promise; unpublishedEntries: () => Promise; unpublishedEntry: (args: { id?: string; collection?: string; slug?: string; }) => Promise; unpublishedEntryDataFile: ( collection: string, slug: string, path: string, id: string, ) => Promise; unpublishedEntryMediaFile: ( collection: string, slug: string, path: string, id: string, ) => Promise; updateUnpublishedEntryStatus: ( collection: string, slug: string, newStatus: string, ) => Promise; publishUnpublishedEntry: (collection: string, slug: string) => Promise; deleteUnpublishedEntry: (collection: string, slug: string) => Promise; getDeployPreview: ( collectionName: string, slug: string, ) => Promise<{ url: string; status: string } | null>; allEntriesByFolder?: ( folder: string, extension: string, depth: number, ) => Promise; traverseCursor?: ( cursor: Cursor, action: string, ) => Promise<{ entries: ImplementationEntry[]; cursor: Cursor }>; isGitBackend?: () => boolean; status: () => Promise<{ auth: { status: boolean }; api: { status: boolean; statusPage: string }; }>; } const MAX_CONCURRENT_DOWNLOADS = 10; export type ImplementationFile = { id?: string | null | undefined; label?: string; path: string; }; type ReadFile = ( path: string, id: string | null | undefined, options: { parseText: boolean }, ) => Promise; type ReadFileMetadata = (path: string, id: string | null | undefined) => Promise; async function fetchFiles( files: ImplementationFile[], readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); const promises = [] as Promise[]; files.forEach(file => { promises.push( new Promise(resolve => sem.take(async () => { try { const [data, fileMetadata] = await Promise.all([ readFile(file.path, file.id, { parseText: true }), readFileMetadata(file.path, file.id), ]); resolve({ file: { ...file, ...fileMetadata }, data: data as string }); sem.leave(); } catch (error) { sem.leave(); console.error(`failed to load file from ${apiName}: ${file.path}`); resolve({ error: true }); } }), ), ); }); return Promise.all(promises).then(loadedEntries => loadedEntries.filter(loadedEntry => !(loadedEntry as { error: boolean }).error), ) as Promise; } export async function entriesByFolder( listFiles: () => Promise, readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { const files = await listFiles(); return fetchFiles(files, readFile, readFileMetadata, apiName); } export async function entriesByFiles( files: ImplementationFile[], readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { return fetchFiles(files, readFile, readFileMetadata, apiName); } export async function unpublishedEntries(listEntriesKeys: () => Promise) { try { const keys = await listEntriesKeys(); return keys; } catch (error) { if (error.message === 'Not Found') { return Promise.resolve([]); } throw error; } } export function blobToFileObj(name: string, blob: Blob) { const options = name.match(/.svg$/) ? { type: 'image/svg+xml' } : {}; return new File([blob], name, options); } export async function getMediaAsBlob(path: string, id: string | null, readFile: ReadFile) { let blob: Blob; if (path.match(/.svg$/)) { const text = (await readFile(path, id, { parseText: true })) as string; blob = new Blob([text], { type: 'image/svg+xml' }); } else { blob = (await readFile(path, id, { parseText: false })) as Blob; } return blob; } export async function getMediaDisplayURL( displayURL: DisplayURL, readFile: ReadFile, semaphore: Semaphore, ) { const { path, id } = displayURL as DisplayURLObject; return new Promise((resolve, reject) => semaphore.take(() => getMediaAsBlob(path, id, readFile) .then(blob => URL.createObjectURL(blob)) .then(resolve, reject) .finally(() => semaphore.leave()), ), ); } export async function runWithLock(lock: AsyncLock, func: Function, message: string) { try { const acquired = await lock.acquire(); if (!acquired) { console.warn(message); } const result = await func(); return result; } finally { lock.release(); } } const LOCAL_KEY = 'git.local'; type LocalTree = { head: string; files: { id: string; name: string; path: string }[]; }; type GetKeyArgs = { branch: string; folder: string; extension: string; depth: number; }; function getLocalKey({ branch, folder, extension, depth }: GetKeyArgs) { return `${LOCAL_KEY}.${branch}.${folder}.${extension}.${depth}`; } type PersistLocalTreeArgs = GetKeyArgs & { localForage: LocalForage; localTree: LocalTree; }; type GetLocalTreeArgs = GetKeyArgs & { localForage: LocalForage; }; export async function persistLocalTree({ localForage, localTree, branch, folder, extension, depth, }: PersistLocalTreeArgs) { await localForage.setItem( getLocalKey({ branch, folder, extension, depth }), localTree, ); } export async function getLocalTree({ localForage, branch, folder, extension, depth, }: GetLocalTreeArgs) { const localTree = await localForage.getItem( getLocalKey({ branch, folder, extension, depth }), ); return localTree; } type GetDiffFromLocalTreeMethods = { getDifferences: ( to: string, from: string, ) => Promise< { oldPath: string; newPath: string; status: string; }[] >; filterFile: (file: { path: string; name: string }) => boolean; getFileId: (path: string) => Promise; }; type GetDiffFromLocalTreeArgs = GetDiffFromLocalTreeMethods & { branch: { name: string; sha: string }; localTree: LocalTree; folder: string; extension: string; depth: number; }; async function getDiffFromLocalTree({ branch, localTree, folder, getDifferences, filterFile, getFileId, }: GetDiffFromLocalTreeArgs) { const diff = await getDifferences(branch.sha, localTree.head); const diffFiles = diff .filter(d => d.oldPath?.startsWith(folder) || d.newPath?.startsWith(folder)) .reduce((acc, d) => { if (d.status === 'renamed') { acc.push({ path: d.oldPath, name: basename(d.oldPath), deleted: true, }); acc.push({ path: d.newPath, name: basename(d.newPath), deleted: false, }); } else if (d.status === 'deleted') { acc.push({ path: d.oldPath, name: basename(d.oldPath), deleted: true, }); } else { acc.push({ path: d.newPath || d.oldPath, name: basename(d.newPath || d.oldPath), deleted: false, }); } return acc; }, [] as { path: string; name: string; deleted: boolean }[]) .filter(filterFile); const diffFilesWithIds = await Promise.all( diffFiles.map(async file => { if (!file.deleted) { const id = await getFileId(file.path); return { ...file, id }; } else { return { ...file, id: '' }; } }), ); return diffFilesWithIds; } type AllEntriesByFolderArgs = GetKeyArgs & GetDiffFromLocalTreeMethods & { listAllFiles: ( folder: string, extension: string, depth: number, ) => Promise; readFile: ReadFile; readFileMetadata: ReadFileMetadata; getDefaultBranch: () => Promise<{ name: string; sha: string }>; isShaExistsInBranch: (branch: string, sha: string) => Promise; apiName: string; localForage: LocalForage; }; export async function allEntriesByFolder({ listAllFiles, readFile, readFileMetadata, apiName, branch, localForage, folder, extension, depth, getDefaultBranch, isShaExistsInBranch, getDifferences, getFileId, filterFile, }: AllEntriesByFolderArgs) { async function listAllFilesAndPersist() { const files = await listAllFiles(folder, extension, depth); const branch = await getDefaultBranch(); await persistLocalTree({ localForage, localTree: { head: branch.sha, files: files.map(f => ({ id: f.id!, path: f.path, name: basename(f.path) })), }, branch: branch.name, depth, extension, folder, }); return files; } async function listFiles() { const localTree = await getLocalTree({ localForage, branch, folder, extension, depth }); if (localTree) { const branch = await getDefaultBranch(); // if the branch was forced pushed the local tree sha can be removed from the remote tree const localTreeInBranch = await isShaExistsInBranch(branch.name, localTree.head); if (!localTreeInBranch) { console.log( `Can't find local tree head '${localTree.head}' in branch '${branch.name}', rebuilding local tree`, ); return listAllFilesAndPersist(); } const diff = await getDiffFromLocalTree({ branch, localTree, folder, extension, depth, getDifferences, getFileId, filterFile, }).catch(e => { console.log('Failed getting diff from local tree:', e); return null; }); if (!diff) { console.log(`Diff is null, rebuilding local tree`); return listAllFilesAndPersist(); } if (diff.length === 0) { // return local copy return localTree.files; } else { const deleted = diff.reduce((acc, d) => { acc[d.path] = d.deleted; return acc; }, {} as Record); const newCopy = sortBy( unionBy( diff.filter(d => !deleted[d.path]), localTree.files.filter(f => !deleted[f.path]), file => file.path, ), file => file.path, ); await persistLocalTree({ localForage, localTree: { head: branch.sha, files: newCopy }, branch: branch.name, depth, extension, folder, }); return newCopy; } } else { return listAllFilesAndPersist(); } } const files = await listFiles(); return fetchFiles(files, readFile, readFileMetadata, apiName); }