We Keep Coding

Extract matching row by comparing two CSV file in NodeJs

The scenario is I have two large CSV files csv1.csv and csv2.csv. In both the files, there is an email column and I have to read csv1.csv row by row and check if the email exists in csv2.csv and if matches write the row of csv2.csv in csv3.csv. I have tried read stream as well but it is not working as expected. Any guidance or help is appreciated.
Thanks to all in advance.
Following are the CSV files
csv1.csv
email,header1,header2
test1#example.com,test1,test1
test2#example.com,test2,test2
test3#example.com,test3,test3
test4#example.com,test4,test4
test5#example.com,test5,test5
csv2.csv
email,header1,header2
test4#example.com,test4,test4
test5#example.com,test5,test5
test6#example.com,test6,test6
test7#example.com,test7,test7
test8#example.com,test8,test8
Following is the code that I tried
const fs = require('fs');
const csv = require('fast-csv')
class CsvHelper {
static write(filestream, rows, options) {
return new Promise((res, rej) => {
csv.writeToStream(filestream, rows, options)
.on('error', err => rej(err))
.on('finish', () => res());
});
}
constructor(opts) {
this.headers = opts.headers;
this.path = opts.path;
this.writeOpts = {
headers: this.headers,
includeEndRowDelimeter: true
};
}
create(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, { ...this.writeOpts });
}
append(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, {
...this.writeOpts,
writeHeaders: false,
});
}
}
class Helper {
async matchCsv (outerRow) {
try {
const filePath2 = "csv2.csv";
const filePath3 = "csv3.csv";
let row = [];
const csvFile = new CsvHelper({
path: filePath3,
headers: ["Email", "Active"]
});
return new Promise((resolve, reject) => {
fs.createReadStream(filePath2)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async innerRow => {
if(outerRow["email"] === innerRow["email"]) {
console.log("====================");
console.log("match found");
console.log(innerRow);
console.log("====================");
row.push([innerRow["email"], "yes"]);
console.log("row: ", row);
}
})
.on("finish", async() => {
if (!fs.existsSync(filePath3)) {
await csvFile.create(row).then(() => {
resolve("Done from matchCsv");
})
} else {
await csvFile.append(row).then(() => {
resolve("Done from matchCsv");
})
}
})
});
} catch (err) {
throw(err);
}
}
async generateCsv () {
try {
const filePath1 = "csv1.csv";
return new Promise((resolve, reject) => {
fs.createReadStream(filePath1)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async outerRow => {
const result = await this.matchCsv(outerRow);
console.log("result: ", result);
})
.on("finish", () => {
resolve("Generated csv3.csv file.");
});
});
} catch (err) {
throw(err);
}
}
}
async function main() {
const helper = new Helper();
const result = await helper.generateCsv()
console.log(result);
}
main();

So the question is a little confusing, but I think I know what you want. Here's what I would do to check if the email exists. It will add all the rows to an array, cycle through them, then if the email address matches the email you're looking for, it will do something else... I think you said you wanted to write to a csv file again with the row, but that should be simple enough.
const csv = require('csv-parser');
const fs = require('fs');
const filepath = "./example_data.csv";
const emailAdd = "myemail#email.com";
var rowsArr = [];
fs.createReadStream(filepath)
.on('error', () => {
// handle error
})
.pipe(csv())
.on('data', (row) => {
rowsArr.push(row);
})
.on('end', () => {
for (var i = 0; i <= rowsArr.length; i++) {
if (rowsArr[i].emailAddress == emailAdd) {
//do something
}
}
})

My NodeJs code is returning undefined value

This is the code that i have written,
function getAllTasksToThatProjectType(typeId){
let storeTaskIds = [];
let projectTypeTasks = [];
let returnVar = new Array();
ProjectType.findOne({ _id: typeId })
.exec((error, projectType) => {
// if(error) return res.status(400).json({ error })
if(projectType){
projectType.tasks.map(tsk => storeTaskIds.push(tsk.taskId))
ProjectTask.find({ })
.exec((error, projectTask) => {
// if(error) return res.status(400).json({ error })
if(projectTask){
storeTaskIds.forEach(function(tID){
projectTask.forEach(function(tsk){
if(JSON.stringify(tsk._id) == JSON.stringify(tID) || JSON.stringify(tsk.parentId) == JSON.stringify(tID))
projectTypeTasks.push(tsk)
})
})
// Here would be the api return statement.
return projectTypeTasks;
}
})
}
})
}
const taskForPType = getAllTasksToThatProjectType(req.body.typeOfProject)
If i call the function like this, it will set the value of taskForPType to undefined.
I have to do this without a callback function. Any suggestions on how i could i achieve it.

It is returning undefined, because your function hasn't finished when you called. You could rewrite the function asynchronously like this:
async function getAllTasksToThatProjectType(typeId) {
const storeTaskIds = [];
const projectTypeTasks = [];
const returnVar = new Array();
return new Promise(function (resolv, reject) {
ProjectType.findOne({ _id: typeId }).exec((error, projectType) => {
if (error) {
reject(error);
}
if (projectType) {
projectType.tasks.map((tsk) => storeTaskIds.push(tsk.taskId));
ProjectTask.find({}).exec((error, projectTask) => {
if(error) {
reject(error)
}
if (projectTask) {
storeTaskIds.forEach(function (tID) {
projectTask.forEach(function (tsk) {
if (JSON.stringify(tsk._id) == JSON.stringify(tID) || JSON.stringify(tsk.parentId) == JSON.stringify(tID))
projectTypeTasks.push(tsk);
});
});
// Here would be the api return statement.
return resolve(projectTypeTasks);
}
});
}
});
});
}
const taskForPType = await getAllTasksToThatProjectType(req.body.typeOfProject);
For more info you could check more info about async calls:
How do I return the response from an asynchronous call?

How to wait for Firebase storage image upload, then run next function

I'm trying to write a function that uploads several images to Firebase, saves the URLs that get returned to an object, and then uploads that object to my Cloud Firestore. I don't really have a firm understanding of async/await or promises, so if anyone could help, it would be much appreciated.
Basically, I want uploadImages() to finish running and then run uploadData(), where saveIssue() is fired when a form is submitted.
Here's what I'm working with:
saveIssue() {
this.uploadImages();
this.uploadData();
},
uploadData() {
let self = this;
db.collection("issues")
.add(self.issue)
.then(docRef => {
self.$router.push({
name: "ReportPage",
params: { issueId: docRef.id }
});
})
.catch(error => {
console.error(error);
});
},
uploadImages() {
const storageRef = storage.ref();
let self = this;
this.imagePreviews.forEach(image => {
let imageName = uuidv1();
let fileExt = image.fileName.split(".").pop();
let uploadTask = storageRef
.child(`images/${imageName}.${fileExt}`)
.putString(image.base64String, "data_url");
uploadTask.on("state_changed", {
error: error => {
console.error(error);
},
complete: () => {
uploadTask.snapshot.ref.getDownloadURL().then(downloadURL => {
self.issue.images.push(downloadURL);
});
}
});
});
},

You have to use promises, as explained here. You can rewrite your code to support async wait, since firebase supports it out of the box, but for the beginning it can be like this:
async saveIssue() {
await this.uploadImages();
await this.uploadData();
},
uploadData() {
return new Promise((resolve, reject) => {
let self = this;
db.collection("issues")
.add(self.issue)
.then(docRef => {
self.$router.push({
name: "ReportPage",
params: {
issueId: docRef.id
}
});
resolve();
})
.catch(error => {
console.error(error);
reject(error);
});
})
},
uploadImages() {
return new Promise((resolve, reject) => {
const storageRef = storage.ref();
let self = this;
this.imagePreviews.forEach(image => {
let imageName = uuidv1();
let fileExt = image.fileName.split(".").pop();
let uploadTask = storageRef
.child(`images/${imageName}.${fileExt}`)
.putString(image.base64String, "data_url");
uploadTask.on("state_changed", {
error: error => {
console.error(error);
reject(error);
},
complete: () => {
uploadTask.snapshot.ref.getDownloadURL().then(downloadURL => {
self.issue.images.push(downloadURL);
resolve();
});
}
});
});
})
},

Okay, I found a solution based on #AlexBrohshtut's answer. I still had an issue where the first loop finishing would allow the await to continue, the code below seems to fix that. Happy to update if anyone has a more succinct answer! (I'm intermediate at best...)
saveIssue() {
Promise.all(
this.imagePreviews.map(async image => {
return await this.uploadImages(image);
})
).then(() => {
this.uploadData();
});
},
uploadData() {
let self = this;
db.collection("issues")
.add(self.issue)
.then(docRef => {
self.$router.push({
name: "ReportPage",
params: {
issueId: docRef.id
}
});
})
.catch(error => {
console.error(error);
});
},
uploadImages(image) {
const storageRef = storage.ref();
let self = this;
return new Promise((resolve, reject) => {
let imageName = uuidv1();
let fileExt = image.fileName.split(".").pop();
let uploadTask = storageRef
.child(`images/${imageName}.${fileExt}`)
.putString(image.base64String, "data_url");
uploadTask.on("state_changed", {
error: error => {
console.error(error);
reject(error);
},
complete: () => {
uploadTask.snapshot.ref.getDownloadURL().then(downloadURL => {
self.issue.images.push(downloadURL);
resolve();
});
}
});
});
}

How to use “.then()” with this.props

I have a Handle Submit function when we click on the button an API update forms appears. The wait time of the API a loader is displayed in this period. but when i try this code an error appears
TypeError: "_this.props.setDeviceInfoAction(...).then" is not a function
Here is the Handlesubmit function:
handleSubmit = (e) => {
e.preventDefault();
this.props.form.validateFieldsAndScroll((err, values) => {
if (!err) {
this.props.setDevicEtherConfigAction(values).then(() => {
this.props.showLoaderAction();
});
}
});
}
//********************* Action**************************//
export const setDeviceInfoAction = (values) => {
return {
type: SET_DEVICE_INFO,
payload: {
values
}
};
};
//********************* Sagas**************************//
function* setDeviceInfo({ payload }) {
try {
const data = yield call(setDeviceInfoRequest, payload.values);
if (data === 200) {
yield put(showAlertAction(...........................));
} else {
yield put(showAlertAction(...........................));
}
} catch (error) {}
}
export function* setDeviceInfoFork() {
yield takeEvery(SET_DEVICE_INFO, setDeviceInfo);
}
//***********************Service*******************************//
export const setDeviceInfoRequest = async (payload) => {
let request = ..........................
var AuthToken = .........................
return new Promise(function (resolve, reject) {
client........(request, {'x-authorization': AuthToken}, (err, response) => {
var status = 200
if (response === null) {
resolve(false);
} else {
resolve(status);
}
});
})
}

Node js pause while loop wait until functions inside get executed completely?

I am coding a post request which downloads all URL HTML,zips them and email it back. This all should happen in the backend. I am storing all the data in an array and extract the first element to start these operations.
I have while loop inside which I am calling some functions. Each function gets executed at a certain time.
I used async, await and promises to make sure they run one after the
other.
Coming to my problem.
My while loop starts getting executed again before all the
functions inside it are executed.
app.post('/?', async (req, res) => {
var urls = req.query.urls
var email = req.query.email;
var new_stack = [urls, email]
stack.push(new_stack)
res.send("Mail sent")
if (isFunctionRunning === false) { //initially it is false
console.log(isFunctionRunning, stack.length)
send_mails();
}
});
const getGoogleIndexHTML = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, res, body) => err ? reject(err) : resolve(body))
})
}
const some_function_to_download = async (url) => {
try {
const a = url.split(".")
let googleIndexHTML = await getGoogleIndexHTML(url)
await fs.writeFile(directory + '/' + a[1] + '.html', googleIndexHTML, (err) => {
if (err) throw err
})
console.log('File created.')
} catch (err) {
console.log(err)
}
}
const html_to_zip_file = async () => {
await zipper.zip(directory, function (error, zipped) {
if (!error) {
zipped.compress();
zipped.save('./package.zip', function (error) {
if (!error) {
console.log("Saved successfully !");
}
});
} else {
console.log(error)
}
})
}
const send_mails = async () => {
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
await html_to_zip_file() // takes 5 sec to zip
.then(result => {
transporter.sendMail(set_mail_options(to_email)) //2 sec to send mail
.then(result => {
console.log("Mail sent")
})
.catch(err => {
console.log(err)
})
})
.catch(err => {
console.log(err)
})
console.log("reached") // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false
}

You need to return transporter.sendMail in sendMail, fs.writeFile in someFunctionToDownload and zipper.zip in htmlToZipFile otherwise the await won't work as expected (I'm assuming that they actually do return promises, I'm only familiar with fs.writeFile)
Also: CamelCase is used in JS, not snake_case 🙃
And are you sure rimraf is synchronous?
const sendMails = async () => {
while (stack.length > 0) {
isFunctionRunning = true;
const [urls, toEmail] = stack.shift();
var collectionUrls = urls.split(",");
rimraf(directory, function() {
console.log("done");
});
await fs.mkdir(directory);
await Promise.All(collectionUrls.map(someFunctionToDownload)); // 5 sec per download
await htmlToZipFile() // takes 5 sec to zip
.then(result => transporter.sendMail(set_mail_options(toEmail))) //2 sec to send mail
.then(result => {
console.log("Mail sent");
})
.catch(err => {
console.log(err);
});
console.log("reached"); // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false;
};
const someFunctionToDownload = async url => {
const a = url.split(".");
const googleIndexHTML = await getGoogleIndexHTML(url);
return fs.writeFile(`${directory}/${a[1]}.html`, googleIndexHTML, err => {
if (err) throw err;
});
};
const htmlToZipFile = async () => {
return zipper.zip(directory, function(error, zipped) {
if (!error) {
zipped.compress();
zipped.save("./package.zip", function(error) {
if (!error) {
console.log("Saved successfully!");
}
});
} else {
console.log(error);
}
});
};

Try using the following
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
try {
const result = await html_to_zip_file() // takes 5 sec to zip
const sendMailResult = await transporter.sendMail(set_mail_options(to_email))
} catch(e)
{
console.log(e)
}
console.log("reached")
}
Since html_to_zip_file() and sendMail function are independent
we can use
const result = await Promise.all([html_to_zip_file(),transporter.sendMail(set_mail_options(to_email))]);