import sortBy from 'lodash/sortBy'; import unionBy from 'lodash/unionBy'; import semaphore from 'semaphore'; import { basename } from './path'; import type { Semaphore } from 'semaphore'; import type { DisplayURL, DisplayURLObject, FileMetadata, ImplementationEntry, ImplementationFile, } from '@staticcms/core/interface'; import type { AsyncLock } from './asyncLock'; const MAX_CONCURRENT_DOWNLOADS = 10; type ReadFile = ( path: string, id: string | null | undefined, options: { parseText: boolean }, ) => Promise; type ReadFileMetadata = (path: string, id: string | null | undefined) => Promise; type CustomFetchFunc = (files: ImplementationFile[]) => Promise; async function fetchFiles( files: ImplementationFile[], readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); const promises = [] as Promise[]; files.forEach(file => { promises.push( new Promise(resolve => sem.take(async () => { try { const [data, fileMetadata] = await Promise.all([ readFile(file.path, file.id, { parseText: true }), readFileMetadata(file.path, file.id), ]); resolve({ file: { ...file, ...fileMetadata }, data: data as string }); sem.leave(); } catch (error) { sem.leave(); console.error(`failed to load file from ${apiName}: ${file.path}`); resolve({ error: true }); } }), ), ); }); return Promise.all(promises).then(loadedEntries => loadedEntries.filter(loadedEntry => !(loadedEntry as { error: boolean }).error), ) as Promise; } export async function entriesByFolder( listFiles: () => Promise, readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { const files = await listFiles(); return fetchFiles(files, readFile, readFileMetadata, apiName); } export async function entriesByFiles( files: ImplementationFile[], readFile: ReadFile, readFileMetadata: ReadFileMetadata, apiName: string, ) { return fetchFiles(files, readFile, readFileMetadata, apiName); } export function blobToFileObj(name: string, blob: Blob) { const options = name.match(/.svg$/) ? { type: 'image/svg+xml' } : {}; return new File([blob], name, options); } export async function getMediaAsBlob(path: string, id: string | null, readFile: ReadFile) { let blob: Blob; if (path.match(/.svg$/)) { const text = (await readFile(path, id, { parseText: true })) as string; blob = new Blob([text], { type: 'image/svg+xml' }); } else { blob = (await readFile(path, id, { parseText: false })) as Blob; } return blob; } export async function getMediaDisplayURL( displayURL: DisplayURL, readFile: ReadFile, semaphore: Semaphore, ) { const { path, id } = displayURL as DisplayURLObject; return new Promise((resolve, reject) => semaphore.take(() => getMediaAsBlob(path, id, readFile) .then(blob => URL.createObjectURL(blob)) .then(resolve, reject) .finally(() => semaphore.leave()), ), ); } export async function runWithLock(lock: AsyncLock, func: Function, message: string) { try { const acquired = await lock.acquire(); if (!acquired) { console.warn('[StaticCMS]', message); } const result = await func(); return result; } finally { lock.release(); } } const LOCAL_KEY = 'git.local'; type LocalTree = { head: string; files: { id: string; name: string; path: string }[]; }; type GetKeyArgs = { branch: string; folder: string; extension: string; depth: number; }; function getLocalKey({ branch, folder, extension, depth }: GetKeyArgs) { return `${LOCAL_KEY}.${branch}.${folder}.${extension}.${depth}`; } type PersistLocalTreeArgs = GetKeyArgs & { localForage: LocalForage; localTree: LocalTree; }; type GetLocalTreeArgs = GetKeyArgs & { localForage: LocalForage; }; export async function persistLocalTree({ localForage, localTree, branch, folder, extension, depth, }: PersistLocalTreeArgs) { await localForage.setItem( getLocalKey({ branch, folder, extension, depth }), localTree, ); } export async function getLocalTree({ localForage, branch, folder, extension, depth, }: GetLocalTreeArgs) { const localTree = await localForage.getItem( getLocalKey({ branch, folder, extension, depth }), ); return localTree; } type GetDiffFromLocalTreeMethods = { getDifferences: ( to: string, from: string, ) => Promise< { oldPath: string; newPath: string; status: string; }[] >; filterFile: (file: { path: string; name: string }) => boolean; getFileId: (path: string) => Promise; }; type GetDiffFromLocalTreeArgs = GetDiffFromLocalTreeMethods & { branch: { name: string; sha: string }; localTree: LocalTree; folder: string; extension: string; depth: number; }; async function getDiffFromLocalTree({ branch, localTree, folder, getDifferences, filterFile, getFileId, }: GetDiffFromLocalTreeArgs) { const diff = await getDifferences(branch.sha, localTree.head); const diffFiles = diff .filter(d => d.oldPath?.startsWith(folder) || d.newPath?.startsWith(folder)) .reduce((acc, d) => { if (d.status === 'renamed') { acc.push({ path: d.oldPath, name: basename(d.oldPath), deleted: true, }); acc.push({ path: d.newPath, name: basename(d.newPath), deleted: false, }); } else if (d.status === 'deleted') { acc.push({ path: d.oldPath, name: basename(d.oldPath), deleted: true, }); } else { acc.push({ path: d.newPath || d.oldPath, name: basename(d.newPath || d.oldPath), deleted: false, }); } return acc; }, [] as { path: string; name: string; deleted: boolean }[]) .filter(filterFile); const diffFilesWithIds = await Promise.all( diffFiles.map(async file => { if (!file.deleted) { const id = await getFileId(file.path); return { ...file, id }; } else { return { ...file, id: '' }; } }), ); return diffFilesWithIds; } type AllEntriesByFolderArgs = GetKeyArgs & GetDiffFromLocalTreeMethods & { listAllFiles: ( folder: string, extension: string, depth: number, ) => Promise; readFile: ReadFile; readFileMetadata: ReadFileMetadata; getDefaultBranch: () => Promise<{ name: string; sha: string }>; isShaExistsInBranch: (branch: string, sha: string) => Promise; apiName: string; localForage: LocalForage; customFetch?: CustomFetchFunc; }; export async function allEntriesByFolder({ listAllFiles, readFile, readFileMetadata, apiName, branch, localForage, folder, extension, depth, getDefaultBranch, isShaExistsInBranch, getDifferences, getFileId, filterFile, customFetch, }: AllEntriesByFolderArgs) { async function listAllFilesAndPersist() { const files = await listAllFiles(folder, extension, depth); const branch = await getDefaultBranch(); await persistLocalTree({ localForage, localTree: { head: branch.sha, files: files.map(f => ({ id: f.id!, path: f.path, name: basename(f.path) })), }, branch: branch.name, depth, extension, folder, }); return files; } async function listFiles() { const localTree = await getLocalTree({ localForage, branch, folder, extension, depth }); if (localTree) { const branch = await getDefaultBranch(); // if the branch was forced pushed the local tree sha can be removed from the remote tree const localTreeInBranch = await isShaExistsInBranch(branch.name, localTree.head); if (!localTreeInBranch) { console.info( `[StaticCMS] Can't find local tree head '${localTree.head}' in branch '${branch.name}', rebuilding local tree`, ); return listAllFilesAndPersist(); } const diff = await getDiffFromLocalTree({ branch, localTree, folder, extension, depth, getDifferences, getFileId, filterFile, }).catch(e => { console.info('[StaticCMS] Failed getting diff from local tree:', e); return null; }); if (!diff) { console.info(`[StaticCMS] Diff is null, rebuilding local tree`); return listAllFilesAndPersist(); } if (diff.length === 0) { // return local copy return localTree.files; } else { const deleted = diff.reduce((acc, d) => { acc[d.path] = d.deleted; return acc; }, {} as Record); const newCopy = sortBy( unionBy( diff.filter(d => !deleted[d.path]), localTree.files.filter(f => !deleted[f.path]), file => file.path, ), file => file.path, ); await persistLocalTree({ localForage, localTree: { head: branch.sha, files: newCopy }, branch: branch.name, depth, extension, folder, }); return newCopy; } } else { return listAllFilesAndPersist(); } } const files = await listFiles(); if (customFetch) { return await customFetch(files); } return await fetchFiles(files, readFile, readFileMetadata, apiName); }