I'm working with a module called findit, which recursive find files within a target folder and return the events "file", "error", "end":
const finder = findit("folder/");
finder.on("file", (file) => {});
finder.on("error", ((error) => {});
finder.on("end", () => {});
The end event will be called when the finder finds all files... but, inside the file event i'm making an async operation, that separate files only if they have something inside:
const finder = findit("folder/");
let neededFiles = [];
finder.on("file", (file) => {
// async operation here to store only the files that i want
// neededFiles = [...neededFiles, file];
});
finder.on("error", ((error) => {});
finder.on("end", () => {
console.log(neededFiles); // empty array
});
The neededFile will be empty because the async operation has not finished yet. My question is: What chances do i need to do to wait the async operation in the end event?
Thanks.
Since you've provided only a scratch of you app, I tried to build around that, and show how you could wrap that into a promise to handle the async part.
function find(folder){
return new Promise((resolve, reject) => {
const finder = findit(folder);
const files = [];
finder.on("file", (file) => {
//do stuff like
//files.push(valueOrPromise);
//or
//files.push( find(anotherPath) );
});
finder.on("error", reject);
finder.on("end", () => {
//this line finally "returns" the result.
//to this point you can modify files as you wish
resolve( Promise.all(files) );
//hint, if you're working with recursive stuff, you may have nested Arrays, so you should flatten the result
//resolve( Promise.all(files).then(values => values.reduce((a,b) => a.concat(b), [])) )
});
})
}
Usually people ask at this point: why do I need promises? Because they implement state management of async tasks; so why would you want to implement it yourself?
And why no use the 'end' inside the 'file'? some like:
finder.on("file", (file) => {
// async operation here to store only the files that i want
// neededFiles = [...neededFiles, file]; // GENERATE A PROMISE SO U CAN CHAIN THEN FUNCTION
neededFilesPromise.then(function(neededFiles){
finder.on("end", () => {
console.log(neededFiles); // empty array
});
}).catch(function() {
finder.on("error", ((error) => {});;
})
});
I would say this is the perfect use case for Promise.all(). So something like this. You might want to 'promisify' your file operations so it is even cleaner (if you are using something like bluebird to replace the native promises). I am not sure if your lib is return the content of the file or a filename
const fs = require('fs');
require('bluebird').promisifyAll(fs);
const finder = findit("folder/");
let neededFiles = [];
finder.on("file", (file) => {
// if file is the file content
neededFiles.push(/* result of your op above */);
// if file is a file name
neededFiles.push(fs.readFile(/* file name */));
});
finder.on("error", ((error) => {});
finder.on("end", () => {
Promise.all(neededFiles)
.then((nf) => {
console.log(nf); // you should now have something here...
});
});
Related
I am attempting to convert the following function to be async in React (using CompressorJS):
const UploadImages = ({ jobInfo, setLoading, getData, accountInfo }) => {
const [files, setFiles] = useState([]);
const onDrop = useCallback( uploadedFiles => {
uploadedFiles.forEach((file) => {
console.log(file)
new Compressor(file, {
quality: 0.5,
width: 500,
height: 500,
resize: "contain",function.
success(result) {
setFiles([...files, result]);
console.log(result)
},
error(err) {
console.log(err.message);
},
});
})
}, [files]);
return(<a bunch of stuff here>);
}
When using the snippet above, if uploadedFiles contains more than one file, there becomes an issue with compressing each photo and adding it to the state ([files, setFiles]). Only one of the images will be added and the others will be missed.
I have spent a number hours attempting to rework this function in different ways and the best solution I have come up with so far has been the following:
function compressFile(file) {
return new Promise((resolve, reject) => {
new Compressor(file, {
quality: 0.5,
width: 500,
height: 500,
resize: "contain",function.
success(result) {
resolve(result)
},
error(err) {
console.log(err.message);
reject(err)
},
});
});
}
function compressFiles(files) {
return new Promise((resolve, reject) => {
files.forEach((file) => {
console.log(file)
compressFile(file)
.then(compFile => {
console.log(compFile)
setFiles([...files, compFile]);
})
})
resolve()
})
}
const onDrop = useCallback( async acceptedFiles => {
message.loading('Compressing Images...');
compressFiles(acceptedFiles)
}, [files]);
Unfortunately, this still does not work and I was wondering if someone could explain where I am going wrong.
This should help:
const compressFiles = files => Promise.all(
files.map(file => compressFile(file))
)
.then(compressedFiles => {
console.log(compressedFiles);
setFiles([...files, ...compressedFiles]);
});
setFiles([...files, compFile]);
Every time this line runs, it is going to create an array starting with what's in files, and adding one new file. This will wipe out any other files that have been created since you started, because the files variable only stores what you started with, not any changes you've made. So you will need to use the function version of setFiles, to make sure you always have the latest version of the state:
setFiles(prev => [...prev, compFile]);
A couple other recommendations. First, i'd suggest that you don't set state multiple times in an async loop. This will result in the files being in an unpredictable order and will cause the component to render each step along the way. Instead, you can wait until all the compression is done, and then set state once at the end. To wait for an array of promises, you can use Promise.all. Secondly, since you've already made compressFile return a promise, you do not need to use the new Promise constructor again in compressFiles. Putting these together i recommend:
function compressFiles(files) {
const promises = files.map(file => compressFile(file));
Promise.all(promises).then(compressedFiles => {
setFiles(prev => [...prev, ...compressedFiles]);
});
}
Or using async/await:
async function compressFiles(files) {
const promises = files.map(file => compressFile(file));
const compressedFiles = await Promise.all(promises);
setFiles(prev => [...prev, ...compressedFiles]);
}
I'm working on a local Node.js app that needs to access the Google Chrome cookies. I've found the chrome-cookies-secure library that seems to do the job but I just can't figure out what's wrong with the code below.
const chrome = require('chrome-cookies-secure');
const domains = [
"google.com"
];
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
const final = resolveCookies();
console.log(final); //! This is going to return a Promise { <pending> } object
The idea is that I just want to store the cookies from all the domains in a list but no matter what I cannot resolve the Promise.
I didn't see any examples with the async call for this module but if I don't use it it's going to return me an empty list after the script execution.
My Node Version: v14.4.0
What am I doing wrong?
It looks like the implementation of getCookies is not correctly awaiting the asynchronous processes. You can see in the implementation that although getCookies itself is async, it calls getDerivedKey without awaiting it (and that function isn't async anyway).
Rather than trying to rely on this implementation, I'd suggest using Util.promisify to create a proper promise via the callback:
const util = require('util');
const chrome = require('chrome-cookies-secure');
const getCookies = util.promisify(chrome.getCookies);
// ...
const cookies = await getCookies(`https://${domain}/`);
Note that, as Reece Daniels pointed out in the comments, the getCookies implementation actually takes a profile parameter after the callback; if you need to use that parameter, you can't use the built-in promisify. You'd have to wrap it yourself instead, this could look like e.g.:
const getCookies = (url, format, profile) => new Promise((resolve, reject) => {
chrome.getCookies(url, format, (err, cookies) => {
if (err) {
reject(err);
} else {
resolve(cookies);
}
}, profile);
});
They already tried to fix the promise upstream, but the PR hasn't been merged in nearly nine months.
Note that once you have a working function to call you can also convert:
const resolveCookies = async () => {
let result = [];
for(domain of domains) {
await chrome.getCookies(`https://${domain}/`, (err, cookies) => {
result.push(cookies);
// console.log(cookies); //? This is going to correctly print the results
})
}
return result;
}
to simply:
const resolveCookies = () => Promise.all(domains.map((domain) => getCookies(`https://${domain}/`)));
An async function returns a Promise.
So your resolveCookies function will also return a Promise as you noticed.
You need to either chain the console.log with a .then e.g.
resolveCookies().then(console.log);
Or if you need to set it to a variable like final you need to await that Promise too. In that case you need an async IIFE:
(async () => {
const final = await resolveCookies();
console.log(final);
})();
try this.
const chrome = require('chrome-cookies-secure');
const domains = [
"www.google.com"
];
const resolveCookies = async() => {
let result = [];
for (domain of domains) {
const cookies = await getCookies(domain)
result.push(cookies)
}
return Promise.resolve(result);
}
const getCookies = async (domain) => {
chrome.getCookies(`https://${domain}/`, (err, cookies) => {
return Promise.resolve(cookies);
})
}
resolveCookies().then((resp) => {
console.log('FINAL ',resp)
}).catch((e) => {
console.log('ERROR ', e)
})
I am confused because I cannot seem to extract a value from an asynchronous operation. Basically I have to parse a csv file and the operation is asynchronous as shown below.
const csv = require('csv-parser')
const fs = require('fs')
const results = [];
fs.createReadStream('courses.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
console.log(results);
});
I am unable to completly extract and isolate the results variable from that stream. I have tried doing this by wrapping it with a promise but it shows pending. Here is what I am doing.
const getData = () => {
const prom = new Promise((res, rej) => {
fs.createReadStream('courses.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
res(results);
});
})
return prom;
}
async function GIVEMEMYVALUE() {
var result = await getData();
return result;
};
let data = GIVEMEMYVALUE();
console.log(data);
I have read other questions relating to promises but I still don't see what I am doing wrong. I can do whatever I want with the results variable inside the 'end' callback but cannot seem to extract it(for whatever reason I want to extract it.)
Is it wrong to want to extract that value outside the scope of the 'end' callback ?
Can everything I possibly want to do with the results be done inside the callback ?
I have already gone through How do I return the response from an asynchronous call? but don't quite get it as it doesn't mention anything about pending promises.
GIVEMEMYVALUE returns also an promise. However you could shorten your processes alot:
const getData = () =>
new Promise((res, rej) => {
fs.createReadStream("courses.csv")
.pipe(csv())
.on("data", data => results.push(data))
.on("end", () => {
res(results);
});
});
getData().then(data => {
console.log(data);
});
async/ await does not make your code work synchronous. As soon as you put async infront of your function, your function automatically returns an promise and acts like an promise.
I'm trying to download several files from a server. I ran into error memory leak with for, forEach and map, so I use this callback function and it works:
.then(files => {
const downloadFile = callback => {
if (files.length > 0) {
let file = files.shift();
client.download(`/${file}`, `./${file}`, () => {
console.log(`Downloaded: ${file}`);
downloadFile(callback);
});
}
else {
callback();
}
};
downloadFile(() => {
console.log('All done');
})
})
I'd like to convert it into a Promise function but I'm stuck, I've tried new Promise((resolve, reject)=>{}) and Promise.all() but it only returns the first file.
You can use map() with your files array to produce one promise for each file. Then you can can call Promise.all() to know when they have all been downloaded.
Since we don't know anything about your client.download() method, we are guessing that it's doing what it should. It's strange that it's callback doesn't take a parameter with the actual file data.
let promises = files.map(file => {
return new Promise((resolve, reject) => {
client.download(`/${file}`, `./${file}`, () => {
// should this call back have some data or error checking?
console.log(`Downloaded: ${file}`);
});
})
})
Promise.all(promises)
.then(() => console.log("all done"))
Since this is happening inside another then() you could just return Promise.all(promises) to handle it downstream.
I have a library that scans a directory for files on a remote server. It returns a Promise like this:
client.scanRemoteDirectory(path)
.then(files => {
console.log(files)
})
I'm trying to write a recursive method to scan directories and subdirectories too. But I'm running into some async issues. My function is like this:
const scanDir(path) {
// Scan the remote directory for files and sub-directories
return client.scanRemoteDirectory(path)
.then(files => {
for (const file of files) {
// If a sub-directory is found, scan it too
if (file.type === 'directory') {
return scanDir(file.path) // Recursive call
}
}
})
}
const scanDir('some/path')
.then(() => {
console.log('done')
})
This works however because of the return in front of the scanDir() recursive method call, this results in the method only scanning the first subdirectory in each directory and skipping the rest.
So for example if the structure is something like this:
/some/path
/some/path/dirA
/some/path/dirA/subdirA
/some/path/dirB
/some/path/dirB/subdirB
The above method will only scan:
/some/path
/some/path/dirA
/some/path/subdirA
It will skip dirB and it's children altogether since the method finds dirA first.
If I simply remove the return from the return scanDir(...) call, then it scans everything just fine. But then my final console.log('done') happens too soon because it's async.
So how do I solve this problem? What is the proper recursive Promise approach where I can still preserve async but also scan every subdirectory recursively?
You might want to use Promise.all in this situation to run your 'sub' promises in parallel, for example:
function scanDir(path) {
return client.scanRemoteDirectory(path)
.then(all => {
const files = all.where(file => file.type !== 'directory);
const dirs = all.where(file => file.type === 'directory);
return Promise.all(dirs.map(dir => scanDir(dir.path)) // Execute all 'sub' promises in parallel.
.then(subFiles => {
return files.concat(subFiles);
});
});
}
Alternatively you could use the reduce function to run your 'sub' promises in sequence:
function scanDir(path) {
return client.scanRemoteDirectory(path)
.then(all => {
const files = all.where(file => file.type !== 'directory);
const dirs = all.where(file => file.type === 'directory);
return dirs.reduce((prevPromise, dir) => { // Execute all 'sub' promises in sequence.
return prevPromise.then(output => {
return scanDir(dir.path)
.then(files => {
return output.concat(files);
});
});
}, Promise.resolve(files));
});
}
Async / await is definitely the easiest solution to read:
async function scanDir(path) {
const output = [];
const files = await client.scanRemoteDirectory(path);
for (const file of files) {
if (file.type !== 'directory') {
output.push(file);
continue;
}
const subFiles = await scanDir(file.path);
output = output.concat(subFiles);
}
return output;
}
I would make the then handler async so you can use await in the loop:
const scanDir(path) {
// Scan the remote directory for files and sub-directories
return client.scanRemoteDirectory(path)
.then(async files => {
for (const file of files) {
// If a sub-directory is found, scan it too
if (file.type === 'directory') {
await scanDir(file.path) // Recursive call
}
}
})
}