We Keep Coding

Extract matching row by comparing two CSV file in NodeJs

The scenario is I have two large CSV files csv1.csv and csv2.csv. In both the files, there is an email column and I have to read csv1.csv row by row and check if the email exists in csv2.csv and if matches write the row of csv2.csv in csv3.csv. I have tried read stream as well but it is not working as expected. Any guidance or help is appreciated.
Thanks to all in advance.
Following are the CSV files
csv1.csv
email,header1,header2
test1#example.com,test1,test1
test2#example.com,test2,test2
test3#example.com,test3,test3
test4#example.com,test4,test4
test5#example.com,test5,test5
csv2.csv
email,header1,header2
test4#example.com,test4,test4
test5#example.com,test5,test5
test6#example.com,test6,test6
test7#example.com,test7,test7
test8#example.com,test8,test8
Following is the code that I tried
const fs = require('fs');
const csv = require('fast-csv')
class CsvHelper {
static write(filestream, rows, options) {
return new Promise((res, rej) => {
csv.writeToStream(filestream, rows, options)
.on('error', err => rej(err))
.on('finish', () => res());
});
}
constructor(opts) {
this.headers = opts.headers;
this.path = opts.path;
this.writeOpts = {
headers: this.headers,
includeEndRowDelimeter: true
};
}
create(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, { ...this.writeOpts });
}
append(rows) {
return CsvHelper.write(fs.createWriteStream(this.path, { flags: 'a' }), rows, {
...this.writeOpts,
writeHeaders: false,
});
}
}
class Helper {
async matchCsv (outerRow) {
try {
const filePath2 = "csv2.csv";
const filePath3 = "csv3.csv";
let row = [];
const csvFile = new CsvHelper({
path: filePath3,
headers: ["Email", "Active"]
});
return new Promise((resolve, reject) => {
fs.createReadStream(filePath2)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async innerRow => {
if(outerRow["email"] === innerRow["email"]) {
console.log("====================");
console.log("match found");
console.log(innerRow);
console.log("====================");
row.push([innerRow["email"], "yes"]);
console.log("row: ", row);
}
})
.on("finish", async() => {
if (!fs.existsSync(filePath3)) {
await csvFile.create(row).then(() => {
resolve("Done from matchCsv");
})
} else {
await csvFile.append(row).then(() => {
resolve("Done from matchCsv");
})
}
})
});
} catch (err) {
throw(err);
}
}
async generateCsv () {
try {
const filePath1 = "csv1.csv";
return new Promise((resolve, reject) => {
fs.createReadStream(filePath1)
.on("error", err => {
reject(err);
})
.pipe(csv.parse({headers: true}))
.on("error", err => {
reject(err);
})
.on("data", async outerRow => {
const result = await this.matchCsv(outerRow);
console.log("result: ", result);
})
.on("finish", () => {
resolve("Generated csv3.csv file.");
});
});
} catch (err) {
throw(err);
}
}
}
async function main() {
const helper = new Helper();
const result = await helper.generateCsv()
console.log(result);
}
main();

So the question is a little confusing, but I think I know what you want. Here's what I would do to check if the email exists. It will add all the rows to an array, cycle through them, then if the email address matches the email you're looking for, it will do something else... I think you said you wanted to write to a csv file again with the row, but that should be simple enough.
const csv = require('csv-parser');
const fs = require('fs');
const filepath = "./example_data.csv";
const emailAdd = "myemail#email.com";
var rowsArr = [];
fs.createReadStream(filepath)
.on('error', () => {
// handle error
})
.pipe(csv())
.on('data', (row) => {
rowsArr.push(row);
})
.on('end', () => {
for (var i = 0; i <= rowsArr.length; i++) {
if (rowsArr[i].emailAddress == emailAdd) {
//do something
}
}
})

PWA Firebase Push Notifications "This site has been updated in the background "

I have been testing push notifications for a PWA(Progressive-Web-App) through Firebase but everytime i Recieve the notification it just says "Chrome Push Notification: This site has been updated in the background". I am testing it in localhost and tried testing similar answer but nothing seems to fix it.
And This is my serviceworker:
(function () {
'use strict';
// Update 'version' if you need to refresh the cache
var version = 'v1.0::CacheFirstSafe';
var offlineUrl = "Offline"; // <-- Offline/Index.cshtml
var urlsToCache = ['./', offlineUrl]; // <-- Add more URLs you would like to cache.
// Store core files in a cache (including a page to display when offline)
function updateStaticCache() {
return caches.open(version)
.then(function (cache) {
return cache.addAll(urlsToCache);
});
}
function addToCache(request, response) {
if (!response.ok && response.type !== 'opaque')
return;
var copy = response.clone();
caches.open(version)
.then(function (cache) {
cache.put(request, copy);
});
}
function serveOfflineImage(request) {
if (request.headers.get('Accept').indexOf('image') !== -1) {
return new Response('<svg role="img" aria-labelledby="offline-title" viewBox="0 0 400 300" xmlns="http://www.w3.org/2000/svg"><title id="offline-title">Offline</title><g fill="none" fill-rule="evenodd"><path fill="#D8D8D8" d="M0 0h400v300H0z"/><text fill="#9B9B9B" font-family="Helvetica Neue,Arial,Helvetica,sans-serif" font-size="72" font-weight="bold"><tspan x="93" y="172">offline</tspan></text></g></svg>', { headers: { 'Content-Type': 'image/svg+xml' } });
}
}
self.addEventListener('install', function (event) {
event.waitUntil(updateStaticCache());
});
self.addEventListener('activate', function (event) {
event.waitUntil(
caches.keys()
.then(function (keys) {
// Remove caches whose name is no longer valid
return Promise.all(keys
.filter(function (key) {
return key.indexOf(version) !== 0;
})
.map(function (key) {
return caches.delete(key);
})
);
})
);
});
self.addEventListener('fetch', function (event) {
var request = event.request;
// Always fetch non-GET requests from the network
if (request.method !== 'GET' || request.url.match(/\/browserLink/ig)) {
event.respondWith(
fetch(request)
.catch(function () {
return caches.match(offlineUrl);
})
);
return;
}
// For HTML requests, try the network first, fall back to the cache, finally the offline page
if (request.headers.get('Accept').indexOf('text/html') !== -1) {
event.respondWith(
fetch(request)
.then(function (response) {
// Stash a copy of this page in the cache
addToCache(request, response);
return response;
})
.catch(function () {
return caches.match(request)
.then(function (response) {
return response || caches.match(offlineUrl);
});
})
);
return;
}
// cache first for fingerprinted resources
if (request.url.match(/(\?|&)v=/ig)) {
event.respondWith(
caches.match(request)
.then(function (response) {
return response || fetch(request)
.then(function (response) {
addToCache(request, response);
return response || serveOfflineImage(request);
})
.catch(function () {
return serveOfflineImage(request);
});
})
);
return;
}
// network first for non-fingerprinted resources
event.respondWith(
fetch(request)
.then(function (response) {
// Stash a copy of this page in the cache
addToCache(request, response);
return response;
})
.catch(function () {
return caches.match(request)
.then(function (response) {
return response || serveOfflineImage(request);
})
.catch(function () {
return serveOfflineImage(request);
});
})
);
});
self.addEventListener('push', function (event) {
console.log('Received a push message', event);
var title = 'Yay a message.';
var body = 'We have received a push message.';
var icon = './Content/image/apple-icon-180x180.png';
var tag = 'simple-push-demo-notification-tag';
var data = {
doge: {
wow: 'such amaze notification data'
}
};
event.waitUntil(
self.registration.showNotification(title, {
body: body,
icon: icon,
tag: tag,
data: data
})
);
});
self.addEventListener('notificationclick', function (event) {
// here data you access from event using event.notification.data
console.log('On notification click: ', event.notification.tag);
});
})();

NodeJS + mongoose await for subquery

I have two models: ModelA and ModelB, I am trying to copy a tags = Array() column from ModelB into ModelA, like this:
function run() {
ModelA.find({}).limit(500).cursor()
.on('data', function(doc) {
let refID = doc.ref_id;
ModelB.findOne({_id: refID}).exec(function(err, modelb) {
if (err) {
console.log(`[Error]: Getting modelb ${refID}`);
}
if (modelb) {
if (modelb.tags.length > 0) {
doc.tags = modelb.tags;
}
}
});
doc.processed = true;
doc.save(function(err, hackAlert) {
if (err) {
console.log('[Error]: Saving ModelA: ' + err);
}
console.log(`Saved: ${doc._id}`);
});
})
.on('error', function(err){
console.log('[Error]');
})
.on('end', function(){
console.log('Done');
});
}
The script obviously saves all ModelA instances with processed = true but given the asynchronous nature of Node, ModelA documents end up without tags.
I'm new to this if you can't tell. I wanted to know what is the best, modern way to "await" for the ModelB.findOne() query to finish before saving ModelA ?

You can simply move the code to be called on the callback of your first query:
function run() {
ModelA.find({}).limit(500).cursor()
.on('data', function(doc) {
let refID = doc.ref_id;
ModelB.findOne({_id: refID}).exec(function(err, modelb) {
if (err) {
console.log(`[Error]: Getting modelb ${refID}`);
}
if (modelb) {
if (modelb.tags.length > 0) {
doc.tags = modelb.tags;
}
}
doc.processed = true;
doc.save(function(err, hackAlert) {
if (err) {
console.log('[Error]: Saving ModelA: ' + err);
}
console.log(`Saved: ${doc._id}`);
});
});
})
.on('error', function(err){
console.log('[Error]');
})
.on('end', function(){
console.log('Done');
});
}
For a more up-to-date approach using async/await and Promises:
async function run () {
ModelA.find({}).limit(500).cursor()
.on('data', async function (doc) {
let refID = doc.ref_id;
const modelb = await ModelB.findOne({_id: refID}).exec()
.catch(() => console.log(`[Error]: Getting modelb ${refID}`));
if (modelb) {
if (modelb.tags.length > 0) {
doc.tags = modelb.tags;
}
}
doc.processed = true;
await doc.save().exec()
.catch(err => console.log('[Error]: Saving ModelA: ' + err))
console.log(`Saved: ${doc._id}`);
})
.on('error', function (err){
console.log('[Error]');
})
.on('end', function () {
console.log('Done');
});
}

Node js pause while loop wait until functions inside get executed completely?

I am coding a post request which downloads all URL HTML,zips them and email it back. This all should happen in the backend. I am storing all the data in an array and extract the first element to start these operations.
I have while loop inside which I am calling some functions. Each function gets executed at a certain time.
I used async, await and promises to make sure they run one after the
other.
Coming to my problem.
My while loop starts getting executed again before all the
functions inside it are executed.
app.post('/?', async (req, res) => {
var urls = req.query.urls
var email = req.query.email;
var new_stack = [urls, email]
stack.push(new_stack)
res.send("Mail sent")
if (isFunctionRunning === false) { //initially it is false
console.log(isFunctionRunning, stack.length)
send_mails();
}
});
const getGoogleIndexHTML = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, res, body) => err ? reject(err) : resolve(body))
})
}
const some_function_to_download = async (url) => {
try {
const a = url.split(".")
let googleIndexHTML = await getGoogleIndexHTML(url)
await fs.writeFile(directory + '/' + a[1] + '.html', googleIndexHTML, (err) => {
if (err) throw err
})
console.log('File created.')
} catch (err) {
console.log(err)
}
}
const html_to_zip_file = async () => {
await zipper.zip(directory, function (error, zipped) {
if (!error) {
zipped.compress();
zipped.save('./package.zip', function (error) {
if (!error) {
console.log("Saved successfully !");
}
});
} else {
console.log(error)
}
})
}
const send_mails = async () => {
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
await html_to_zip_file() // takes 5 sec to zip
.then(result => {
transporter.sendMail(set_mail_options(to_email)) //2 sec to send mail
.then(result => {
console.log("Mail sent")
})
.catch(err => {
console.log(err)
})
})
.catch(err => {
console.log(err)
})
console.log("reached") // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false
}

You need to return transporter.sendMail in sendMail, fs.writeFile in someFunctionToDownload and zipper.zip in htmlToZipFile otherwise the await won't work as expected (I'm assuming that they actually do return promises, I'm only familiar with fs.writeFile)
Also: CamelCase is used in JS, not snake_case 🙃
And are you sure rimraf is synchronous?
const sendMails = async () => {
while (stack.length > 0) {
isFunctionRunning = true;
const [urls, toEmail] = stack.shift();
var collectionUrls = urls.split(",");
rimraf(directory, function() {
console.log("done");
});
await fs.mkdir(directory);
await Promise.All(collectionUrls.map(someFunctionToDownload)); // 5 sec per download
await htmlToZipFile() // takes 5 sec to zip
.then(result => transporter.sendMail(set_mail_options(toEmail))) //2 sec to send mail
.then(result => {
console.log("Mail sent");
})
.catch(err => {
console.log(err);
});
console.log("reached"); // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false;
};
const someFunctionToDownload = async url => {
const a = url.split(".");
const googleIndexHTML = await getGoogleIndexHTML(url);
return fs.writeFile(`${directory}/${a[1]}.html`, googleIndexHTML, err => {
if (err) throw err;
});
};
const htmlToZipFile = async () => {
return zipper.zip(directory, function(error, zipped) {
if (!error) {
zipped.compress();
zipped.save("./package.zip", function(error) {
if (!error) {
console.log("Saved successfully!");
}
});
} else {
console.log(error);
}
});
};

Try using the following
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
try {
const result = await html_to_zip_file() // takes 5 sec to zip
const sendMailResult = await transporter.sendMail(set_mail_options(to_email))
} catch(e)
{
console.log(e)
}
console.log("reached")
}
Since html_to_zip_file() and sendMail function are independent
we can use
const result = await Promise.all([html_to_zip_file(),transporter.sendMail(set_mail_options(to_email))]);

Use ldapjs with promise

I want to convert the following code to use promise. It is working and output a user's attributes within the active directory.
var client = ldap.createClient({
url: ldap_url
});
client.bind(ldap_username, ldap_password, function (err) {
client.search(ldap_dn_search, opts, function (err, search) {
search.on('searchEntry', function (entry) {
var user = entry.object;
// It is working!!!. It outputs all user attributes.
console.log(user);
});
});
});
The following is my attempt, butit doesn't output anything.
var Promise = require('promise');
var client_bind = Promise.denodeify(client.bind);
var client_search = Promise.denodeify(client.search);
client_bind(ldap_username, ldap_password)
.then(function(err){
client_search(ldap_dn_search, opts)
.then(function(search){
var search_on = Promise.denodeify(search.on);
search_on('searchEntry')
.then(function(entry){
var user = entry.object;
// It doesn't output anything !!!
console.log(user);
});
});
});

I had the same problem.
Search emits events, so we need something that processes them and passes further along the chain.
Here is piece of code, that works for me:
var ldap = require('ldapjs');
var promise = require('bluebird');
var client = ldap.createClient({url: app.settings['ldap']['server']});
var uid;
promise.promisifyAll(client);
function searchPromise(res, notfoundtext) {
return new Promise(function(resolve, reject) {
var found = false;
res.on('searchEntry', function(entry) {
found = true;
resolve(entry);
});
res.on('error', function(e) {
reject(e.message);
});
res.on('end', function() {
if (!found) {
reject(notfoundtext);
}
});
});
}
client.searchAsync(app.settings['ldap']['baseDn'], {filter: '(mail='+credentials.email+')', scope: 'sub'})
.then(function(res) {
return searchPromise(res, 'User isn\'t exists.');
})
.then(function (entry) {
uid = entry.object.uid;
return client.bindAsync(entry.object.dn, credentials.password);
})
.then(function() {
return client.searchAsync('cn='+app.settings['ldap']['group']+',cn=groups,'+app.settings['ldap']['baseDn'], {scope: 'sub', filter: '(memberUid='+uid+')'});
})
.then(function(res) {
return searchPromise(res, 'User is not in group ' + app.settings['ldap']['group']);
})
.then(function() {
console.log('All is ok');
})
.catch(function(message) {
console.log('Error:' + message);
});
Immediately after the search I add one more step that catches the events, processes them, and passes it further along the chain. This makes the function searchPromise.
Good luck coding )

Most likely those methods do require to be called on client as a context, so you will need to bind() them before passing them to Promise.denodeify:
var client_bind = Promise.denodeify(client.bind.bind(client));
var client_search = Promise.denodeify(client.search.bind(client));
Also, a proper use of promises would look like this:
client_bind(ldap_username, ldap_password).then(function() {
return client_search(ldap_dn_search, opts);
// ^^^^^^ always return something from the callback
}).then(function(search) { // flatten your chain
return Promise.denodeify(search.on).call(search, 'searchEntry');
// ^^^^^^ an alternative to `bind`
}).then(function(entry){
var user = entry.object;
console.log(user);
}).catch(function(err) { // always catch errors!
console.error(err);
});

Using Bluebird Promises, the easy way to do this is to create your client normally, and then run the promisifyAll() on the client.
var ldap = require('ldapjs');
var Promise = require('bluebird');
var client = ldap.createClient({
url: 'ldap://my-server:1234',
});
Promise.promisifyAll(client);
Now you can call client.addAsync() and client.searchAsync() and such.
client.bindAsync(secUserDn, secUserPassword)
.then(doSearch) // if it works, call doSearch
.catch(function (err) { // if bind fails, handle it
console.error('Error on bind', err)
});
function doSearch(data) {
client.searchAsync('CN=A Test,OU=Users,DC=website,DC=com', options)
.then(function (data) { // Handle the search result processing
console.log('I got a result');
})
.catch(function (err) { // Catch potential errors and handle them
console.error('Error on search', err);
});
}

i had the same issue here but i solved it by adding promise and resolve the response without using bluebird, this is an exemple of my code :
async getLdapUser(username: any): Promise<any> {
let myPromise = new Promise<boolean>((resolve, reject) => {
console.log('ssssssssss', username);
const adSuffix = 'OU=xxxx,OU=xxxxx,DC=xxxxxxx,DC=xxxxxx';
const password = 'xxxxxxxxxxxxx';
// Create client and bind to AD
const client = ldap.createClient({
url: 'ldap://1.1.1.1:389',
});
// promise.promisifyAll(client);
let resp = false;
// console.log(client);
client.bind('userTest', password,(err: any) => {
console.log('RESP', resp);
if (err) {
console.log('Error in new connetion ' + err);
} else {
/*if connection is success then go for any operation*/
console.log('Success');
const searchOptions: {} = {
scope: 'sub',
filter: '(sAMAccountName=' + username + ')',
attributes: ['sAMAccountName'],
};
client.search(adSuffix, searchOptions, (err: any, res: any) => {
assert.ifError(err);
res.on('searchEntry', (entry: any) => {
resp = true;
});
res.on('error', (error: any) => {
console.log('err');
reject(error.message);
});
await res.on('end', (result: any) => {
resolve(resp);
});
});
}
});
});
return myPromise;
}