I am confused because I cannot seem to extract a value from an asynchronous operation. Basically I have to parse a csv file and the operation is asynchronous as shown below.
const csv = require('csv-parser')
const fs = require('fs')
const results = [];
fs.createReadStream('courses.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
console.log(results);
});
I am unable to completly extract and isolate the results variable from that stream. I have tried doing this by wrapping it with a promise but it shows pending. Here is what I am doing.
const getData = () => {
const prom = new Promise((res, rej) => {
fs.createReadStream('courses.csv')
.pipe(csv())
.on('data', (data) => results.push(data))
.on('end', () => {
res(results);
});
})
return prom;
}
async function GIVEMEMYVALUE() {
var result = await getData();
return result;
};
let data = GIVEMEMYVALUE();
console.log(data);
I have read other questions relating to promises but I still don't see what I am doing wrong. I can do whatever I want with the results variable inside the 'end' callback but cannot seem to extract it(for whatever reason I want to extract it.)
Is it wrong to want to extract that value outside the scope of the 'end' callback ?
Can everything I possibly want to do with the results be done inside the callback ?
I have already gone through How do I return the response from an asynchronous call? but don't quite get it as it doesn't mention anything about pending promises.
GIVEMEMYVALUE returns also an promise. However you could shorten your processes alot:
const getData = () =>
new Promise((res, rej) => {
fs.createReadStream("courses.csv")
.pipe(csv())
.on("data", data => results.push(data))
.on("end", () => {
res(results);
});
});
getData().then(data => {
console.log(data);
});
async/ await does not make your code work synchronous. As soon as you put async infront of your function, your function automatically returns an promise and acts like an promise.
Related
My code is nearly working well my problem is that only the first CSV data is logged and the lambda function ends.
I guess I need someway to wait for all stream pipes to end.
myCsvList.forEach((myElem) => {
const data = [];
// setup params
const csvFile = s3.getObject(params).createReadStream(); // works fine
csvFile
.pipe(csv())
.on('data', function(entry) {
data.push(entry);
})
.on('end', () => {
console.log(data); // after the log of the first element on myCsvList, the code finishes. It should log all csvFiles from myCsvList
});
});
I guess I need a promises or something?
you can change every item to Promise, then use Promise.all():
const getItem = (myElem) => {
const data = [];
// setup params
const csvFile = s3.getObject(params).createReadStream(); // works fine
return new Promise((resolve) => {
csvFile
.pipe(csv())
.on('data', function (entry) {
data.push(entry);
})
.on('end', () => {
resolve(data); // after the log of the first element on myCsvList, the code finishes. It should log all csvFiles from myCsvList
});
});
};
const start = () => {
// all promises have been finished
return Promise.all(myCsvList.map(myElem => getItem(myElem)));
}
myCsvList.map(myElem => getItem(myElem)) can get a promise list, when all promises have been resolved, it will trigger Promise.all().
I am trying to parse data from a .csv file, and save it to an array for later use.
I understand the concept of promises, but I have no idea what am I missing in my code that I cannot resolve the Promise and get the value (the string in the .csv file). It while I can view all the data inside the promise (.on('data')) from debugging mode, I just can't save it in order to use it later in my 'try&catch'.
const fs = require("fs");
const csv = require("csv-parser");
const { resolve } = require("path");
async function readCSV(filepath) {
return new Promise(async (resolve, reject) => {
await fs
.createReadStream(filepath)
.pipe(csv())
.on("data", (data) => {
results.push(data);
})
.on("error", (error) => reject(results))
.on("end", () => {
resolve(results);
});
});
}
const results = [];
const csvFilePath =
"/languages.csv";
try {
const languages = readCSV(csvFilePath).then((res) => {
return res;
});
console.log(languages);
} catch (e) {
console.log(e);
}
and the output on the console is:
>Promise {<pending>}
No debugger available, can not send 'variables'
** That's from the debugging mode when I pause inside the promise:
https://i.stack.imgur.com/H9nHi.png
You can't try catch a returned promise without the await keyword in an async function.
If you're returning a promise, you need to use the .catch method on the promise.
Also, when you're logging languages you're doing so before the promise resolves because you're not using the await keyword.
I'm sure the promise resolves. Instead, log res inside the .then method.
const fs = require("fs");
const csv = require("csv-parser");
const results = [];
function readCSV(filepath) {
return new Promise((resolve, reject) => {
fs
.createReadStream(filepath)
.pipe(csv())
.on("data", (data) => {
results.push(data);
})
.on("error", (error) => reject(results))
.on("end", () => {
resolve(results);
});
});
}
const csvFilePath = "./languages.csv";
(async () => {
const output = await readCSV(csvFilePath);
console.log(output)
})();
I have this function
function getJsonObjectFromURL(url, onData) {
let chunks = [];
return require('https').get(url, res => {
res.setEncoding('utf8')
.on('data', (chunk) => {
chunks.push(chunk);
})
.on('end', () => {
onData(JSON.parse(chunks.join('')));
});
}).on('error', function(e) {
console.log("Got an error: ", e);
});
}
Also I have this script that converts url's data to json array.
url = https://pu.vk.com/c824502/upload.php?act=do_add&mid=213468131&aid=-14&gid=156603484&hash=7ab9a7e723425f4a6ca08709cbd5ebd0&rhash=ba8f0ec6580a6eafce38349b12ed3789&swfupload=1&api=1&wallphoto=1
getJsonObjectFromURL(url, data => {
console.log(data.server, data.photo, data.hash);
});
It goes well when console.log. But when I want to make from this script variable, it gives me huge collection
var xx = getJsonObjectFromURL(url, data => {
return data.server;
});
console.log(xx);
Your function getJsonObjectFromURL() doesn't return the object returned by the URL. It returns the object responsible for the https request code, which is something you don't want.
I see that you are using ES6, so the best solution for you is to probably create an async function that returns a promise, which will give you great flexibility. Here is an improved version of your code:
const https = require('https');
async function getJsonObjectFromURL(url) {
return new Promise((resolve, reject) => {
const chunks = [];
try {
https.get(url, res => {
res.setEncoding('utf8')
.on('data', (chunk) => {
chunks.push(chunk);
})
.on('end', () => {
resolve(JSON.parse(chunks.join('')));
});
}).on('error', e => reject(e));
} catch (err) {
reject(err);
}
});
};
This code allows you to retrieve the remote contents of the HTTPS url synchronously or asynchronously.
Asynchronous Call
As you have already done in your code, you can use a lambda callback that handles the response when it is ready.
const url = 'https://pu.vk.com/c824502/upload.php?act=do_add&mid=213468131&aid=-14&gid=156603484&hash=7ab9a7e723425f4a6ca08709cbd5ebd0&rhash=ba8f0ec6580a6eafce38349b12ed3789&swfupload=1&api=1&wallphoto=1';
// here we use a lambda callback that handles the response
getJsonObjectFromURL(url)
.then(data => {
console.log(data.server, data.photo, data.hash);
})
.catch(err => console.error(err));
Synchronous Call
The synchronous call forces the function to wait for the result. This is how you can do it:
async function getSync() {
try {
// wait for the result
const data = await getJsonObjectFromURL(url);
console.log(data.server);
} catch(err) {
console.error(err);
}
}
getSync();
Please note that we can only use the await keyword when we are inside an async function. This is why I had to wrap the synchronous call with a function.
I'm trying to download several files from a server. I ran into error memory leak with for, forEach and map, so I use this callback function and it works:
.then(files => {
const downloadFile = callback => {
if (files.length > 0) {
let file = files.shift();
client.download(`/${file}`, `./${file}`, () => {
console.log(`Downloaded: ${file}`);
downloadFile(callback);
});
}
else {
callback();
}
};
downloadFile(() => {
console.log('All done');
})
})
I'd like to convert it into a Promise function but I'm stuck, I've tried new Promise((resolve, reject)=>{}) and Promise.all() but it only returns the first file.
You can use map() with your files array to produce one promise for each file. Then you can can call Promise.all() to know when they have all been downloaded.
Since we don't know anything about your client.download() method, we are guessing that it's doing what it should. It's strange that it's callback doesn't take a parameter with the actual file data.
let promises = files.map(file => {
return new Promise((resolve, reject) => {
client.download(`/${file}`, `./${file}`, () => {
// should this call back have some data or error checking?
console.log(`Downloaded: ${file}`);
});
})
})
Promise.all(promises)
.then(() => console.log("all done"))
Since this is happening inside another then() you could just return Promise.all(promises) to handle it downstream.
I'm working with a module called findit, which recursive find files within a target folder and return the events "file", "error", "end":
const finder = findit("folder/");
finder.on("file", (file) => {});
finder.on("error", ((error) => {});
finder.on("end", () => {});
The end event will be called when the finder finds all files... but, inside the file event i'm making an async operation, that separate files only if they have something inside:
const finder = findit("folder/");
let neededFiles = [];
finder.on("file", (file) => {
// async operation here to store only the files that i want
// neededFiles = [...neededFiles, file];
});
finder.on("error", ((error) => {});
finder.on("end", () => {
console.log(neededFiles); // empty array
});
The neededFile will be empty because the async operation has not finished yet. My question is: What chances do i need to do to wait the async operation in the end event?
Thanks.
Since you've provided only a scratch of you app, I tried to build around that, and show how you could wrap that into a promise to handle the async part.
function find(folder){
return new Promise((resolve, reject) => {
const finder = findit(folder);
const files = [];
finder.on("file", (file) => {
//do stuff like
//files.push(valueOrPromise);
//or
//files.push( find(anotherPath) );
});
finder.on("error", reject);
finder.on("end", () => {
//this line finally "returns" the result.
//to this point you can modify files as you wish
resolve( Promise.all(files) );
//hint, if you're working with recursive stuff, you may have nested Arrays, so you should flatten the result
//resolve( Promise.all(files).then(values => values.reduce((a,b) => a.concat(b), [])) )
});
})
}
Usually people ask at this point: why do I need promises? Because they implement state management of async tasks; so why would you want to implement it yourself?
And why no use the 'end' inside the 'file'? some like:
finder.on("file", (file) => {
// async operation here to store only the files that i want
// neededFiles = [...neededFiles, file]; // GENERATE A PROMISE SO U CAN CHAIN THEN FUNCTION
neededFilesPromise.then(function(neededFiles){
finder.on("end", () => {
console.log(neededFiles); // empty array
});
}).catch(function() {
finder.on("error", ((error) => {});;
})
});
I would say this is the perfect use case for Promise.all(). So something like this. You might want to 'promisify' your file operations so it is even cleaner (if you are using something like bluebird to replace the native promises). I am not sure if your lib is return the content of the file or a filename
const fs = require('fs');
require('bluebird').promisifyAll(fs);
const finder = findit("folder/");
let neededFiles = [];
finder.on("file", (file) => {
// if file is the file content
neededFiles.push(/* result of your op above */);
// if file is a file name
neededFiles.push(fs.readFile(/* file name */));
});
finder.on("error", ((error) => {});
finder.on("end", () => {
Promise.all(neededFiles)
.then((nf) => {
console.log(nf); // you should now have something here...
});
});