I had a question about some code that i had working earlier, but now decides it doesnt want to work at all. Basically, I have a promise that returns me an array of data. I'll explain whats happening at the bottom of the code.
function mkdir(path){
return new Promise(function(resolve,reject){
fs.mkdir(path, { recursive: true },function(err){
if(err){console.log(err)}
})
return resolve()
})
}
function writeFile(file,content){
return new Promise(function(resolve,reject){
fs.writeFile(file, content, function(err){
if(err){console.log(err)}
return resolve()
})
})
}
function rawData(socket){
var mintDataArray = []
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
getdat(i).then(function(r){
//mintDataArray.push(r)
for(o in r){
var dataurls = []
dataurls.push(r[o].discord_url,r[o].magic_eden_url,r[o].twitter_url)
//socket.emit('twitterFollowers',r[o])
const ProjectData = {
"mintDate": 0 || "",
"name":"",
"stock":0,
"links":[],
"mintTime": 0 || "",
"PricePlusDescription":0 || ""
}
if(r[o].mintDate != null){
ProjectData.mintDate = moment.utc(r[o].mintDate).format("MMMM Do")
}else{
ProjectData.mintDate = "No date specified yet"
}
ProjectData.name = r[o].name
ProjectData.stock = r[o].supply
ProjectData.links.push(dataurls)
ProjectData.PricePlusDescription = r[o].price
mintDataArray.push(ProjectData)
}
}).then(function(socket){
//CollectionSorter(mintDataArray)
for(i in mintDataArray){
var data = mintDataArray[i]
//console.log(data) // <----- This prints out the data that needs to be written to files.
var MintDateFolder = __dirname + "/UpcomingCollections/" +data.mintDate
if(!fs.existsSync(MintDateFolder)){
console.log('huh?')
mkdir(__dirname + '/UpcomingCollections/'+data.mintDate)
}
writeFile(MintDateFolder +"/"+data.name,JSON.stringify(data))
}
})
}
//socket.emit('twitterFollowers',mintDataArray)
})
}
So what the code is supposed to do, is check to see if that directory first exists in general. If it doesnt, then create the new directory. Then after that, its supposed to write files to it (not just that directory, but to other directories as well). It doesnt create the directory if it doesnt exists and it doesnt even write to it if i manually create the directory, however it does write to the other directories. I'm really not sure with this one because I had this working earlier, where it was creating the directory if it didn't exist. so i'm not sure what i messed up on.
I recently made mkdir and writefile functions and i thought they were the issue, because when i had this working I was just using fs.mkdir and fs.writefile. However, i went and tried again without those functions and i was still having the same troubles. I thought about making another promise to check if the directory existed but I already have quite a few nested promises.
read() function:
function read(i){
return new Promise(function(resolve,reject){
var r = https.request(options, function (res) {
var data = []
res.on('data', function (d) {
data.push(d)
}).on('end', function () {
var NFTTokenData = []
console.log(`STATUS: ${res.statusCode}`);
var info = Buffer.concat(data)
zlib.gunzip(info,function(err,buf){
var NFTData = []
var x = buf.toString()
var dat = JSON.parse(x)
var collectionList = dat.pageProps.__APOLLO_STATE__
for(keys in collectionList){
if(collectionList[keys].__typename.includes('Nft')){
collections.push(collectionList[keys])
resolve(collections)
}
}
})
})
})
r.end()
})
}
FINAL SOLUTION
function Project(fromApi) {
return {
mintDate: moment.utc(fromApi.mintDate).format("MMMM Do"),
name: fromApi.name,
imageLink:fromApi.project_image_link,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
console.log(dest)
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return new Promise(function(resolve,reject){
[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17].map(i =>
read(i).then(function(r){
var data = []
for(i in r){
if(r[i].__typename == "Nft"){
data.push(r[i])
}
}
data.map(item => {
var project = Project(item)
write(project,"./UpcomingCollections")
})
})
)
})
}
This is a modification of the answer down below!
You can do anything inside of .then – its only role is to sequence functions.
Let's talk about some other issues in your code -
function mkdir(path){
return new Promise(function(resolve,reject){
fs.mkdir(path, { recursive: true },function(err){
if(err){console.log(err)} // ❌ reject(err); do not log
})
return resolve() // ❌ resolve() outside of mkdir callback
})
}
function writeFile(file,content){
return new Promise(function(resolve,reject){
fs.writeFile(file, content, function(err){
if(err){console.log(err)} // ❌ reject(err); do not log
return resolve() // ⚠️ "return" not needed
})
})
}
Did you find it tedious to implement wrappers for each fs function in Node? You will be happy to know the fs/promises module already provides Promise-based APIs for each -
import { mkdir, writeFile } from "fs/promises" // ✅
// mkdir(path[, options])
// Returns: <Promise> Upon success, fulfills with undefined if recursive is false, or the first directory path created if recursive is true.
// writeFile(file, data[, options])
// Returns: <Promise> Fulfills with undefined upon success.
Next we'll look over the other issues in the main program -
function rawData(socket){
var mintDataArray = [] // ⚠️ state "outside" of Promise context
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
getdat(i).then(function(r){
for(o in r){ // ⚠️ global "o"; don't use for..in, use for..of instead
// ,,,
mintDataArray.push(Project) // ⚠️ attempting to send state "out" of Promise
}
// ⚠️ missing "return"
// implicitly returning "undefined"
}).then(function(socket){ // ⚠️ function parameter (socket) receives resolved Promise
for(i in mintDataArray){ // ⚠️ global "i", for..in again, accessing external state
var data = mintDataArray[i]
var MintDateFolder = __dirname + "/UpcomingCollections/" +data.mintDate // ⚠️ use "path" module
if(!fs.existsSync(MintDateFolder)){ // ⚠️ async methods until this point, why Sync here?
console.log('huh?')
mkdir(__dirname + '/UpcomingCollections/'+data.mintDate) // ⚠️ asynchronous
}
writeFile(MintDateFolder +"/"+data.name,JSON.stringify(data)) // ⚠️ attempts write before mkdir completes
}
})
}
socket.emit('twitterFollowers',mintDataArray) // ⚠️ accessing external state
})
}
read and transform data
It's quite a bit of work, but don't worry. By breaking big problems down into small ones, we can work smarter, not harder. We'll start by renaming getdat to read -
read(i).then(r => {
const mintData = [] // state *inside* promise context
for (const v of r) {
// for all v of r, add project data to mintData
mintData.push({
mintDate: v.mintDate,
name: v.name,
stock: v.supply,
links: [
v.discord_url,
v.magic_eden_url,
v.twitter_url
],
price: v.price
})
}
return mintData // "return" resolves the promise
}).then(...)
Already our .then function is getting big. There's quite a bit of code for extracting and constructing the project data, so make that its own function, Project -
function Project(fromApi) {
// add date logic, or more
return {
mintDate: fromApi.mintDate,
name: fromApi.name,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
read(i).then(r => {
const mintData = []
for (const v of r) {
mintData.push(Project(v)) // ✅
}
return mintData
}).then(...)
Which is the same thing as -
read(i)
.then(r => r.map(Project)) // ✨ no for loop needed!
.then(...)
write
Let's check back in with your code and see our progress -
function rawData(socket){
// var mintDataArray = [] // ✅ remove external state
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
read(i)
.then(r => r.map(Project)) // ✨
.then(function(socket) {
// ⚠️ remember, "socket" gets the resolved promise
// since `read` resolves an array of Projects, we should rename it
})
}
// ⚠️ we'll come back to this
// socket.emit('twitterFollowers',mintDataArray)
})
}
We continue with the .then(function(socket){ ... }) handler. There's a good amount of code here for creating the path, making a directory, and writing JSON to a file. Let's make that its own function called write -
import { stat, mkdir, writeFile } from "fs/promises"
import { join, resolve, dirname } from "path" // 🔍 unrelated to Promise "resolve"
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name)) // ✅ sanitary path handling
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true})) // create dir if not exists
.then(_ => writeFile(dest, JSON.stringify(project))) // write project JSON data
.then(_ => project) // return project
}
Our rawData function is cleaning up nicely, but we still have an outstanding issue of running async operations inside two separate loops -
function rawData(socket){
return new Promise(function(){
for (let i = 1; i < 13; i+= 1) {
read(i) //❓ how to resolve promise for each read?
.then(r => r.map(Project))
.then(projects => { // ✅ "projects", not "socket"
for (const p of projects) {
write(p, "./UpcomingCollections") //❓ how to resole promise for each write?
}
// ❓ what should we return?
})
}
// socket.emit('twitterFollowers',mintDataArray)
})
}
promises in a loop
Promise has another function we must become familiar with. Promise.all takes an array of promises and resolves only when all promises have resolved -
function myfunc(x) {
return Promise.resolve(x * 100)
}
Promise.all([myfunc(1), myfunc(2), myfunc(3)]).then(console.log)
// Promise{ [ 100, 200, 300 ] }
Promise.all([1,2,3].map(n => myfunc(n))).then(console.log)
// Promise{ [ 100, 200, 300 ] }
We can use Promise.all to clean up the two loops in rawData. And look at that, we can sequence the data directly into the socket -
function rawData(socket){
return Promise.all(
[1,2,3,4,5,6,7,8,9,10,11,12].map(i =>
read(i).then(r => r.map(Project))
)
)
.then(projects => {
Promise.all(projects.map(p => write(p, "./UpcomingCollections"))) // ✨
})
.then(projects => socket.emit("twitterFollowers", projects)) // ✨
}
all together now
We can see a lot of pain points are eliminated by using Promise-based code in an effective, idiomatic way. Until this point we have not addressed the issue of error handling, but now there's nothing left to say. Because we used Promises correctly, any errors will bubble up and the caller can .catch them and respond appropriately -
import { stat, mkdir, writeFile } from "fs/promises"
import { join, resolve, dirname } from "path"
function Project(fromApi) {
return {
mintDate: fromApi.mintDate,
name: fromApi.name,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return Promise.all(
[1,2,3,4,5,6,7,8,9,10,11,12].map(i =>
read(i).then(r => r.map(Project))
)
)
.then(projects => {
Promise.all(projects.map(p => write(p, "./UpcomingCollections")))
})
.then(projects => socket.emit("twitterFollowers", projects))
}
async..await
Modern JavaScript provides async..await syntax allowing us to blur the lines between synchronous and asynchronous code. This allows us to remove many .then calls, flattens our code, reduces cognitive load, and shares asynchronous values in the same scope -
async function rawData(socket) {
const r = await Promise.all([1,2,3,4,5,6,7,8,9,10,11,12].map(read))
const projects = await Promise.all(
r.flatMap(Project).map(p => write(p, "./UpcomingCollections"))
)
return socket.emit("twitterfollowers", projects)
}
Some of the complications of this program stems from the use of nested data. Promise.all is efficient and runs the promises in parallel, but keeping the data nested in the array makes it a bit harder to work with. To show that async..await truly blurs the line between sync and async, we will bring back the two for..of loops and call await in the loop. This results in a serial order processing but readability is terrific. Maybe your use-case isn't hyper demanding and so this style is completely adequate -
async function rawData(socket) {
for (const i of [1,2,3,4,5,6,7,8,9,10,11,12]) {
const result = await read(i)
for (const r of result) {
const project = Project(r)
await write(project, "./UpcomingCollections")
socket.emit("twitterFollowers", project)
}
}
}
Mulan's answer was correct, I just needed to change the for loop in my other function, to a .map. After that, everything worked perfectly. I had to do some modification to the rawData function as seen below.
function Project(fromApi) {
return {
mintDate: moment.utc(fromApi.mintDate).format("MMMM Do"),
name: fromApi.name,
imageLink:fromApi.project_image_link,
stock: fromApi.supply,
links: [
fromApi.discord_url,
fromApi.magic_eden_url,
fromApi.twitter_url
],
price: fromApi.price
}
}
function write(project, dir) {
const dest = resolve(dir, join(project.mintDate, project.name))
console.log(dest)
return stat(dirname(dest))
.catch(_ => mkdir(dirname(dest), {recursive: true}))
.then(_ => writeFile(dest, JSON.stringify(project)))
.then(_ => project)
}
function rawData(socket){
return new Promise(function(resolve,reject){
[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17].map(i =>
read(i).then(function(r){
var data = []
for(i in r){
if(r[i].__typename == "Nft"){
data.push(r[i])
}
}
data.map(item => {
var project = Project(item)
write(project,"./UpcomingCollections")
})
})
)
})
}
I am trying to make a network retry-er using RxJS repeatWhen operator. The idea is that when a new request receives to the scheduler, then I try the request directly and if it results in a network failure result, I add it to a pool to be retried on sometime later. So the entering point of my scheduler is the queue function which does the job like this:
queue({ operation, body }) {
const behaviourSubject = new BehaviorSubject();
const task = {
operation,
body,
behaviourSubject,
};
this.doTask(task).subscribe({
error: _ => this.notifier.next({ tasks: [task], remove: false }),
complete: () => console.log('Task successfully done on first try: ', task),
});
return behaviourSubject;
}
and this.notifier is a Subject which is used as the notifier of the worker. So the worker itself is like this:
const rawWorker = new Observable(subscriber => {
const doneTasks = [];
const jobs = [];
for (const task of this.tasks) {
jobs.push(
this.doTask(task).pipe(
tap(_ => {
doneTasks.push(task);
}),
catchError(error => { task.behaviourSubject.next(error); return of(error); }),
)
);
}
if (jobs.length > 0) {
forkJoin(...jobs).subscribe(_ => {
this.notifier.next({ tasks: doneTasks, remove: true });
subscriber.next(`One cycle of worker done. #${doneTasks.length} task(s) done and #${this.tasks.length} remaining.`);
// subscriber.complete();
if (this.tasks.length > 0) {
this.notifier.next();
}
})
} else {
subscriber.complete();
}
});
const scheduledWorker = rawWorker.pipe( // TODO: delay should be added to retry and repeat routines
retry(),
repeatWhen(_ => this.notifier.pipe(
filter(_ => this.tasks.length > 0),
)),
);
and also the notifier keeps track of all undone requests in an array like this:
this.notifierSubscription = this.notifier
.pipe(
filter(data => data && data.tasks)
)
.subscribe({
next: ({ tasks = [], remove = false }) => {
if (remove) {
console.log('removing tasks: ', tasks);
this.tasks = this.tasks.filter(task => !tasks.some(tsk => task === tsk));
} else {
console.log('inserting: ', tasks);
this.tasks.push.apply(
this.tasks,
tasks,
);
}
console.log('new tasks array: ', this.tasks);
}
});
As I know if a cycle of the worker is not completed then repeatWhen has nothing to do. For example if I remove the part:
else {
subscriber.complete();
}
from the worker, on the first try of the worker (an empty cycle) the Observable does not complete and repeatWhen won't do anything. But on the other hand as you see I have commented // subscriber.complete(); when there exist jobs but the repeat is occurring. And the worst part of the problem is that many different instances of the worker run in parallel which makes many duplicate requests.
I have spent a lot of time on this problem but do not have any clue to trace.
In the project, I have a loop going through a list of urls. It downloads file from every url and do some post process over the downloaded file.
After the all the process done (both download process and post process), I want to execute a callback function. Because post process includes some streaming task, it has close event. If the last item can be identified, I can pass the callback function to the close event. However, since the loop is async, I can't track which item is done at last.
For now, I use a 5 second timeout to make sure the callback is executed after the whole process. Obviously, this is not sustainable. What's a good way to handle this?
loop code:
exports.processArray = (items, process, callback) => {
var todo = items.concat();
setTimeout(function() {
process(todo.shift());
if(todo.length > 0) {
// execute download and post process each second
// however it doesn't guarantee one start after previous one done
setTimeout(arguments.callee, 1000);
} else {
setTimeout(() => {callback();}, 5000);
}
}, 1000);
};
processArray(
// First param, the array
urlList,
// Second param, download and post process
(url) => {
if(url.startsWith('http')) {
getDataReg(url, uid);
}
else if(url.startsWith('ftp')) {
getDataFtp(url, uid);
}
else {
console.log('not a valid resource');
}
},
// Third param, callback to be executed after all done
() => {
Request.get(`${config.demouri}bound=${request.query.boundary};uid=${uid}`, {
method: 'GET',
auth: auth
})
.on('response', (response) => {
console.log('response event emmits');
zipFiles(uid)
.then((path) => {
reply.file(path, { confine: false, filename: uid + '.zip', mode: 'inline'}).header('Content-Disposition');
});
});
}
);
Download and post process:
exports.getDataFtp = (url, uid) => {
console.log('get into ftp');
var usefulUrl = url.split('//')[1];
var spliter = usefulUrl.indexOf('/');
var host = usefulUrl.substring(0, spliter);
var dir = usefulUrl.substring(spliter+1, usefulUrl.length);
var client = new ftp();
var connection = {
host: host
};
var fileNameStart = dir.lastIndexOf('/') + 1;
var fileNameEnd = dir.length;
var fileName = dir.substring(fileNameStart, fileNameEnd);
console.log('filename: ', fileName);
client.on('ready', () => {
console.log('get into ftp ready');
client.get(dir, (err, stream) => {
if (err) {
console.log('get file err:', err);
return;
} else{
console.log('get into ftp get');
stream.pipe(fs.createWriteStream(datadir + `download/${uid}/${fileName}`));
stream.on('end', () => {
console.log('get into ftp close');
unzipData(datadir + `download/${uid}/`, fileName, uid);
client.end();
});
}
});
});
client.connect(connection);
};
exports.getDataReg = (url, uid) => {
console.log('get into http');
var fileNameStart = url.lastIndexOf('/') + 1;
var fileNameEnd = url.length;
var fileName = url.substring(fileNameStart, fileNameEnd);
var file = fs.createWriteStream(datadir + `download/${uid}/${fileName}`);
if (url.startsWith('https')) {
https.get(url, (response) => {
console.log('start piping file');
response.pipe(file);
file.on('finish', () => {
console.log('get into http finish');
unzipData(datadir + `download/${uid}/`, fileName, uid);
});
}).on('error', (err) => { // Handle errors
fs.unlink(datadir + `download/${uid}/${fileName}`);
console.log('download file err: ', err);
});
} else {
http.get(url, (response) => {
console.log('start piping file');
response.pipe(file);
file.on('finish', () => {
unzipData(datadir + `download/${uid}/`, fileName, uid);
});
}).on('error', (err) => {
fs.unlink(datadir + `download/${uid}/${fileName}`);
console.log('download file err: ', err);
});
}
};
function unzipData(path, fileName, uid) {
console.log('get into unzip');
console.log('creating: ', path + fileName);
fs.createReadStream(path + fileName)
.pipe(unzip.Extract({path: path}))
.on('close', () => {
console.log('get into unzip close');
var filelist = listFile(path);
filelist.forEach((filePath) => {
if (!filePath.endsWith('.zip')) {
var components = filePath.split('/');
var component = components[components.length-1];
mv(filePath, datadir + `processing/${uid}/${component}`, (err) => {
if(err) {
console.log('move file err: ');
} else {
console.log('move file done');
}
});
}
});
fs.unlink(path + fileName, (err) => {});
});
}
After the all the process done (both download process and post process), I want to execute a callback function.
The interesting thing about a series of asynchronous processes is that you can never know when exactly all processes will complete. So setting a timeout for the callback is quick&dirty way to do it, but it's not reliable for sure.
You can instead use a counter to solve this problem.
Let's say you have 10 operations to perform. At the beginning you set your counter to ten counter = 10 And after each process is completed, regardless how (it can either succeed or fail), you can decrement the counter by 1 like counter -= 1 and right after it you can check if the counter is 0, if so that means all processes are completed and we reached the end. You can now safely run your callback function, like if(counter === 0) callback();
If I were you, I would do something like this:
*Notice that the called process should return a promise, so that I can know when it finishes (again regardless how)
*If you need help about promises, this useful article might help you: https://howtonode.org/promises
*Oh and one more thing, you should avoid using arguments.callee, because it's deprecated. Here is why Why was the arguments.callee.caller property deprecated in JavaScript?
exports.processArray = (items, process, callback) => {
var todo = [].concat(items);
var counter = todo.length;
runProcess();
function runProcess() {
// Check if the counter already reached 0
if(checkCounter() === false) {
// Nope. Counter is still > 0, which means we got work to do.
var processPromise = process(todo.shift());
processPromise
.then(function() {
// success
})
.catch(function() {
// failure
})
.finally(function() {
// The previous process is done.
// Now we can go with the next one.
--counter;
runProcess();
})
}
};
function checkCounter() {
if(counter === 0) {
callback();
return true;
} else {
return false;
}
}
};
What you want to do is to make all your asynchronous processes converge into a single promise that you can use to execute the callback at the correct moment.
Lets start at the point each process is complete, which I assume is in the callback passed to the mv() function in unzipData(). You want to wrap each of these asynchronous actions in a Promise that resolves in the callback and you also want to use these promises later and for that you use the .map() method to collect the promises in an array (instead of .forEach()).
Here's the code:
var promises = filelist.map((filePath) => {
if (!filePath.endsWith('.zip')) {
var components = filePath.split('/');
var component = components[components.length-1];
return new Promise((resolve, reject) =>
mv(filePath, datadir + `processing/${uid}/${component}`, (err) => {
if(err) {
console.log('move file err: ');
reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
} else {
console.log('move file done');
resolve();
}
}));
}
return Promise.resolve();
});
(if the asynchronous action is not to be executed, a Promise that resolves immediately is returned instead)
Now, we can turn this list of Promises into a single Promise that resolves when all of the promises in the list has resolved:
var allPromise = Promise.all(promises);
Next, we need to look further up in the code. We can see that the code we've just been looking at is itself part of an event handler of an asynchronous action, i.e. fs.createReadStream(). You need to wrap that in a promise that gets resolved when the inner promises resolve and this is the promise that the unzipData() function shall return:
function unzipData(path, fileName, uid) {
console.log('get into unzip');
console.log('creating: ', path + fileName);
return new Promise((outerResolve) =>
fs.createReadStream(path + fileName)
.pipe(unzip.Extract({path: path}))
.on('close', () => {
console.log('get into unzip close');
var filelist = listFile(path);
// Code from previous examples
allPromise.then(outerResolve);
}));
}
Next, we look at the functions that use unzipData(): getDataReg() and getDataFtp(). They only perform one asynchronous action so all you need to do is to make them return a promise that resolves when the promise returned by unzipData() resolves.
Simplified example:
exports.getDataReg = (url, uid) => {
return new Promise((resolve, reject) => {
// ...
https.get(url, (response) => {
response.pipe(file);
file.on('finish', () => {
unzipData(datadir + `download/${uid}/`, fileName, uid)
.then(resolve);
});
}).on('error', (err) => { // Handle errors
fs.unlink(datadir + `download/${uid}/${fileName}`);
reject(); // Or resolve() if you want to ignore the error and not cause it to prevent the callback from executing later
});
// ...
});
}
Finally, we get to the processArray() function and here you need to do the same thing we did to begin with: map the processes into a list of promises. First, the process function passed needs to return the promises returned by getDataReg() and getDataFtp():
// Second param, download and post process
(url) => {
if(url.startsWith('http')) {
return getDataReg(url, uid);
}
else if(url.startsWith('ftp')) {
return getDataFtp(url, uid);
}
else {
console.log('not a valid resource');
}
return Promise.reject(); // or Promise.resolve() if you want invalid resources to be ignored and not prevent the callback from executing later
}
Now, your processArray() function can look like this:
exports.processArray = (items, process, callback) =>
Promise.all(items.map(process))
.then(callback)
.catch(() => console.log('Something went wrong somewhere'));
Your callback will get invoked when all asynchronous actions have completed, regardless of in which order they do. If any one of the promises rejects, the callback will never be executed so manage your promise rejections accordingly.
Here's a JSFiddle with the complete code: https://jsfiddle.net/upn4yqsw/
In general, since nodejs does not appear to have implemented Streams Standard to be Promise based, at least from what can gather; but rather, uses an event based or callback mechanism, you can use Promise constructor within function call, to return a fulfilled Promise object when a specific event has been dispatched
const doStuff = (...args) => new Promise((resolve, reject)) => {
/* define and do stream stuff */
doStreamStuff.on(/* "close", "end" */, => {
// do stuff
resolve(/* value */)
})
});
doStuff(/* args */)
.then(data => {})
.catch(err => {})
I'm trying to extend some existing code with additional promises, but they are a new topic for me at the moment and i'm obviously missing something. This is running as part of a build scrip for npm.
All i am currently trying to make happen is for the final then to be called after the pack operation has happened for each architecture. I have tried wrapping it in a
return new Promise
But at the moment i am not returning anything from that function so i'm not sure what i should include in the resolve call at the end. If i just call the resolve with a true nothing happens, and wrapping it in a promise seems to cause the function to not actually run, and no errors are caught anywhere?
I'm guessing i am going about this completely wrong, all i want to achieve is to run another function once the previous one has completed?
Here's the code as it stands with the additional .then that i can't get to be called.
function build(cfg) {
return new Promise((resolve, reject) => {
webpack(cfg, (err, stats) => {
if (err) return reject(err);
resolve(stats);
});
});
}
function startPack() {
console.log('start pack...');
build(electronCfg)
.then(() => build(cfg))
.then(() => del('release'))
.then(paths => {
if (shouldBuildAll) {
// build for all platforms
const archs = ['ia32', 'x64'];
const platforms = ['linux', 'win32', 'darwin'];
platforms.forEach(plat => {
archs.forEach(arch => {
pack(plat, arch, log(plat, arch));
});
});
} else {
// build for current platform only
pack(os.platform(), os.arch(), log(os.platform(), os.arch()));
}
})
.then(() => {
console.log('then!');
})
.catch(err => {
console.error(err);
});
}
function pack(plat, arch, cb) {
// there is no darwin ia32 electron
if (plat === 'darwin' && arch === 'ia32') return;
const iconObj = {
icon: DEFAULT_OPTS.icon + (() => {
let extension = '.png';
if (plat === 'darwin') {
extension = '.icns';
} else if (plat === 'win32') {
extension = '.ico';
}
return extension;
})()
};
const opts = Object.assign({}, DEFAULT_OPTS, iconObj, {
platform: plat,
arch,
prune: true,
'app-version': pkg.version || DEFAULT_OPTS.version,
out: `release/${plat}-${arch}`,
'osx-sign': true
});
packager(opts, cb);
}
You didn't say what log is, but if it's a plain logging function, then it looks like you're passing in undefined (the result from calling log(...)) as the cb argument to pack. Perhaps you meant:
pack(plat, arch, () => log(plat, arch));
In any case, this won't do anything to wait for packing to finish. I don't know why you're not seeing any console output, but if you're looking for this output to happen after all the packing has finished, then you need to wrap packager in a promise. Something like:
var pack = (plat, arch) => new Promise(resolve => {
// ...
packager(opts, resolve);
});
And then use Promise.all instead of forEach to do all the packaging (in parallel if that's OK):
.then(paths => {
if (!shouldBuildAll) {
return pack(os.platform(), os.arch());
}
return Promise.all(['linux', 'win32', 'darwin'].map(plat =>
Promise.all(['ia32', 'x64'].map(arch => pack(plat, arch))));
})
.then(() => console.log('then!'))
.catch(err => console.error(err));