gcg/static-cms
0
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Fix semaphore implementation for only doing 10 concurrent fetches at a time

Browse Source
This commit is contained in:
Mathias Biilmann Christensen 2016-09-04 20:55:05 +02:00
parent 2980ba8565
commit 964e158a5f
3 changed files with 15 additions and 8 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
src/backends/backend.js
View File

@ -49,6 +49,7 @@ class Backend {
entries(collection, page, perPage) { entries(collection, page, perPage) {
return this.implementation.entries(collection, page, perPage).then((response) => { return this.implementation.entries(collection, page, perPage).then((response) => {
console.log("Got %s entries", response.entries.length);
return { return {
pagination: response.pagination, pagination: response.pagination,
entries: response.entries.map(this.entryWithFormat(collection)) entries: response.entries.map(this.entryWithFormat(collection))

11
src/backends/github/implementation.js
View File

@ -36,12 +36,15 @@ export default class GitHub {
const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
const promises = []; const promises = [];
files.map((file) => { files.map((file) => {
sem.take(() => { promises.push(new Promise((resolve, reject) => {
promises.push(this.api.readFile(file.path, file.sha).then((data) => { return sem.take(() => this.api.readFile(file.path, file.sha).then((data) => {
resolve(createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data));
sem.leave(); sem.leave();
return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data); }).catch((err) => {
sem.leave();
reject(err);
})); }));
}); }));
}); });
return Promise.all(promises); return Promise.all(promises);
}).then((entries) => ({ }).then((entries) => ({

11
src/backends/netlify-git/implementation.js
View File

@ -33,12 +33,15 @@ export default class NetlifyGit {
const sem = semaphore(MAX_CONCURRENT_DOWNLOADS); const sem = semaphore(MAX_CONCURRENT_DOWNLOADS);
const promises = []; const promises = [];
files.map((file) => { files.map((file) => {
sem.take(() => { promises.push(new Promise((resolve, reject) => {
promises.push(this.api.readFile(file.path, file.sha).then((data) => { return sem.take(() => this.api.readFile(file.path, file.sha).then((data) => {
resolve(createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data));
sem.leave(); sem.leave();
return createEntry(file.path, file.path.split('/').pop().replace(/\.[^\.]+$/, ''), data); }).catch((err) => {
sem.leave();
reject(err);
})); }));
}); }));
}); });
return Promise.all(promises); return Promise.all(promises);
}).then((entries) => ({ }).then((entries) => ({