We Keep Coding

Node js pause while loop wait until functions inside get executed completely?

I am coding a post request which downloads all URL HTML,zips them and email it back. This all should happen in the backend. I am storing all the data in an array and extract the first element to start these operations.
I have while loop inside which I am calling some functions. Each function gets executed at a certain time.
I used async, await and promises to make sure they run one after the
other.
Coming to my problem.
My while loop starts getting executed again before all the
functions inside it are executed.
app.post('/?', async (req, res) => {
var urls = req.query.urls
var email = req.query.email;
var new_stack = [urls, email]
stack.push(new_stack)
res.send("Mail sent")
if (isFunctionRunning === false) { //initially it is false
console.log(isFunctionRunning, stack.length)
send_mails();
}
});
const getGoogleIndexHTML = (url) => {
return new Promise((resolve, reject) => {
request(url, (err, res, body) => err ? reject(err) : resolve(body))
})
}
const some_function_to_download = async (url) => {
try {
const a = url.split(".")
let googleIndexHTML = await getGoogleIndexHTML(url)
await fs.writeFile(directory + '/' + a[1] + '.html', googleIndexHTML, (err) => {
if (err) throw err
})
console.log('File created.')
} catch (err) {
console.log(err)
}
}
const html_to_zip_file = async () => {
await zipper.zip(directory, function (error, zipped) {
if (!error) {
zipped.compress();
zipped.save('./package.zip', function (error) {
if (!error) {
console.log("Saved successfully !");
}
});
} else {
console.log(error)
}
})
}
const send_mails = async () => {
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
await html_to_zip_file() // takes 5 sec to zip
.then(result => {
transporter.sendMail(set_mail_options(to_email)) //2 sec to send mail
.then(result => {
console.log("Mail sent")
})
.catch(err => {
console.log(err)
})
})
.catch(err => {
console.log(err)
})
console.log("reached") // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false
}

You need to return transporter.sendMail in sendMail, fs.writeFile in someFunctionToDownload and zipper.zip in htmlToZipFile otherwise the await won't work as expected (I'm assuming that they actually do return promises, I'm only familiar with fs.writeFile)
Also: CamelCase is used in JS, not snake_case 🙃
And are you sure rimraf is synchronous?
const sendMails = async () => {
while (stack.length > 0) {
isFunctionRunning = true;
const [urls, toEmail] = stack.shift();
var collectionUrls = urls.split(",");
rimraf(directory, function() {
console.log("done");
});
await fs.mkdir(directory);
await Promise.All(collectionUrls.map(someFunctionToDownload)); // 5 sec per download
await htmlToZipFile() // takes 5 sec to zip
.then(result => transporter.sendMail(set_mail_options(toEmail))) //2 sec to send mail
.then(result => {
console.log("Mail sent");
})
.catch(err => {
console.log(err);
});
console.log("reached"); // this is reached before zip is done and mail sent. I want to prevent this
}
isFunctionRunning = false;
};
const someFunctionToDownload = async url => {
const a = url.split(".");
const googleIndexHTML = await getGoogleIndexHTML(url);
return fs.writeFile(`${directory}/${a[1]}.html`, googleIndexHTML, err => {
if (err) throw err;
});
};
const htmlToZipFile = async () => {
return zipper.zip(directory, function(error, zipped) {
if (!error) {
zipped.compress();
zipped.save("./package.zip", function(error) {
if (!error) {
console.log("Saved successfully!");
}
});
} else {
console.log(error);
}
});
};

Try using the following
while (stack.length > 0) {
isFunctionRunning = true
var a = stack.shift()
var urls = a[0]
var collection_urls = urls.split(",");
var to_email = a[1]
rimraf(directory, function () {
console.log("done");
});
fs.mkdirSync(directory);
for (url of collection_urls) {
await some_function_to_download(url); // 5 sec per download
}
try {
const result = await html_to_zip_file() // takes 5 sec to zip
const sendMailResult = await transporter.sendMail(set_mail_options(to_email))
} catch(e)
{
console.log(e)
}
console.log("reached")
}
Since html_to_zip_file() and sendMail function are independent
we can use
const result = await Promise.all([html_to_zip_file(),transporter.sendMail(set_mail_options(to_email))]);

Express dosn't get return of other function querying Mongodb [duplicate]

This question already has answers here:
How do I return the response from an asynchronous call?
(41 answers)
Closed 3 years ago.
I'm working in a simple API Key authentication, I just want to verify the given key against the user provied key.
I have a seperate file with the function querying the database, and returning true/false and the user object.
But in my route.js file, the return object is undefined even tough in my auth.js file it isn't.
I tried making the the function in router.get an async function using express-promise-router and making the function an await return var user = await auth.verify(req.params.uid, req.get("token")) but I don't realy know how async works.
router.js
[...]
router.get('/list/:uid', function(req, res) {
var user = auth.verify(req.params.uid, req.get("token"))
console.log("User: " + user) // <-- Undefined
if (user.status) {
res.send("Success")
} else {
res.status(403)
res.json({status: 403, error: "Unkown User / Token"})
}
})
[...]
auth.js
var db = require('./db')
var ObjectId = require('mongodb').ObjectId;
module.exports = {
verify: (uid, key) => {
try {
var collection = db.get().collection('users')
const obj_id = new ObjectId(uid)
const query = { _id: obj_id }
collection.find(query).limit(1).toArray(function(err, user) {
var status = 0;
var usr = {};
if (err) {throw err}else{status=1}
if (user.length <= 0) {throw "NotExistingExc"; status = 0}else{
usr = user[0];
if (key != usr.api) status = 0
}
var returnObj = {
status: status,
user: usr
} /* --> Is {
status: 1,
user: {
_id: d47a2b30b3d2770606942bf0,
name: 'Sh4dow',
groups: [ 0 ],
api: 'YWFiMDI1MGE4NjAyZTg0MWE3N2U0M2I1NzEzZGE1YjE='
}
}
*/
return returnObj;
})
} catch (e) {
console.error(e)
return {
status: 0,
user: {},
error: e
}
}
}
}
db.js (Idk if needed)
var MongoClient = require('mongodb').MongoClient
var state = {
db: null,
}
exports.connect = function(url, done) {
if (state.db) return done()
MongoClient.connect(url, { useNewUrlParser: true }, function(err, db) {
if (err) return done(err)
state.db = db
done()
})
}
exports.get = function() {
return state.db.db("database")
}
exports.close = function(done) {
if (state.db) {
state.db.close(function(err, result) {
state.db = null
state.mode = null
done(err)
})
}
}
I want to have the returnObjin auth.js in the router.get of my route.js file.

Make auth.verify return a Promise which we can then await for it inside router, You can just make the callback async no need for express-promise-router
router.get('/list/:uid', async function(req, res) {
try {
var user = await auth.verify(req.params.uid, req.get("token"))
console.log("User: " + user)
if (user.status) {
res.send("Success")
} else {
res.status(403).json({status: 403, error: "Unkown User / Token"})
}
} catch (e) {
console.error(e)
res.status(/* */).json(/* */)
}
})
auth
module.exports = {
verify: (uid, key) => new Promise((resolve, reject) => {
var collection = db.get().collection('users')
const obj_id = new ObjectId(uid)
const query = { _id: obj_id }
collection.find(query).limit(1).toArray(function(err, user) {
var status = 0;
var usr = {};
if (err) {
reject(err)
return
} else {
status = 1
}
if (user.length <= 0) {
reject(new Error("NotExistingExc"))
return
} else {
usr = user[0]
if (key != usr.api) status = 0
}
var returnObj = {
status: status,
user: usr
}
resolve(returnObj);
})
}
}

In short, the reason you get undefined is because the code in auth.js is asyncronous. But you're really close. The toArray method in MongoDB returns a promise, so you need to make sure you return that promise and then use it in the router correctly.
In auth.js, make sure verify returns a promise - just add return!
return collection.find(query).limit(1).toArray(...)
And then, change your usage of the verify to the async/await you originally tried:
router.get('/list/:uid', async function(req, res) {
var user = await auth.verify(req.params.uid, req.get("token"))
// More code here...
})

How to store the RFC822 message body body as String?

var inbox = require("inbox");
var client = inbox.createConnection(false, "imap.gmail.com", {
secureConnection: true,
auth:{
user: "myemail#gmail.com",
pass: "mypass"
}
});
client.connect();
client.on("connect", function(){
client.openMailbox("INBOX", function(error, info){
if(error) throw error;
client.listMessages(-10, function(err, messages){
var datas = [];
var data;
messages.forEach(function(message){
data = message.UID + ": " + message.title;
datas.push(data);
});
var messageStream = client.createMessageStream(22048);
console.log(messageStream);
client.close();
});
});
});
The above code returns
Stream {
domain: null,
_events: {},
_eventsCount: 0,
_maxListeners: undefined }
But by replacing
var messageStream = client.createMessageStream(22048);
console.log(messageStream);
with
client.createMessageStream(123).pipe(process.stdout, {end: false});
I get the entire RFC822 message body in terminal
How to store the RFC822 message body body as String?
I tried my best to do it but everytime it returns the same
how can i do it?

Since its a readable stream you can use NodeJS - Class: stream.Readable
You can use data event to get the message from stream and end event to know when there is no more data
Event: 'data'
const readable = getReadableStreamSomehow();
readable.on('data', (chunk) => {
console.log(`Received ${chunk.length} bytes of data.`);
});
Event: 'end'
const readable = getReadableStreamSomehow();
readable.on('data', (chunk) => {
console.log(`Received ${chunk.length} bytes of data.`);
});
readable.on('end', () => {
console.log('There will be no more data.');
});
P.S. createMessageStream accepts as argument a message UID.
In your case you might have to wrap this up with some promise base functions
Example
var inbox = require("inbox");
var client = inbox.createConnection(false, "imap.gmail.com", {
secureConnection: true,
auth: {
user: "myemail#gmail.com",
pass: "mypass"
}
});
client.connect();
client.on("connect", function() {
client.openMailbox("INBOX", function(error, info) {
if (error) throw error;
var emails = [];
var promises = [];
new Promise(function(resolve, reject) {
client.listMessages(-10, function(err, messages) {
if (err) {
reject(err)
return;
};
messages.forEach(function(message) {
var promise = new Promise(function(resolve, reject) {
var email = {
UID: messages.UID,
title: message.title,
body: ""
}
client.createMessageStream(message.UID)
.on('data', (chunk) => {
email.body += chunk.toString();
}).on('end', () => {
//return the email object
resolve(email);
}).on('error', (err) => {
reject(err);
});
})
promises.push(promise);
});
resolve();
});
}).then(() => {
return Promise.all(promises).then(function(emails) {
console.log(emails);
console.log("closing");
client.close();
})
})
});
});

How promise works with nested function calls

I have a piece of code which deals with user's data. There are bunch of nested function calls :
f1(){
f2(){
....
fn{
///
}
}
}
fn accesses a database which means it's asynchronous, so I wrote it somehow that it returns a promise and in fn-1 (the function which calls fn) , we use .then() to wait for this promise. But it looks like now I have to return a promise in fn-1 and so on. Is that true ?
var keyValueExists = function(key, value) {
var query = {};
query[key] = value;
return new Promise(function(resolve, reject) {
User.count(query, function(err, count) {
if (err) {
console.log(err);
console.log('Problem with `.find` function');
reject('Problem with `.find` function');
} else {
resolve(count !== 0);
}
});
});
};
var addUser = function(newUserInfo) {
var validationResult = Common._validateUserInfo(newUserInfo);
if (validationResult.isOK) {
keyValueExists('userName', newUserInfo.userName).then(function(userNameAlreadyExists) {
if (userNameAlreadyExists) {
validationResult = {
isOK: false,
reason: 'Username already exists',
infoWithBadInput: 'userName'
}
} else {
var newUserId = generateUserId();
//TODO: change it somehting more flexible. e.g. a predefined list of attributes to iterate over
var newUser = {
'userName': newUserInfo.userName,
'password': newUserInfo.password,
'userId': newUserId,
'lastModificationTime': Common.getCurrentFormanttedTime(),
'createdTime': Common.getCurrentFormanttedTime()
};
var user = new User(newUser);
user.save(function(err) {
if (err) {
console.log(err);
console.log('There is a problem saving the user info');
} else {
console.log('A new user added: ');
console.log(newUser);
}
});
}
return validationResult;
});
} else {
return validationResult;
}
};
addUser returns undefined ! It looks like that the caller of addUser doesn't wait for it !

This is what you are effectively doing in your addUser function
var addUser = function(newUserInfo) {
var validationResult = Common._validateUserInfo(newUserInfo);
if (validationResult.isOK) {
// ... do something asynchronously without returning anything
} else {
return validationResult;
}
}
So, yeah, if validationResult.isOK, adduser WILL return undefined
Here's some code loosely based on your code, but it runs standalone to demonstrate how you possibly should be doing things
var keyValueExists = function(key, value) {
// pseudo junk, this simulates any username starting with b as existing
return new Promise(function(resolve, reject) {
resolve(value.substr(0,1) == 'b'); // barny and betty are dupes, fred and wilma are not
});
}
var addUser = function (newUserInfo) {
// var validationResult = Common._validateUserInfo(newUserInfo);
var validationResult = {isOK: !!~(['fred', 'barny'].indexOf(newUserInfo.userName)), username: newUserInfo.userName}; // dummy code
if (validationResult.isOK) {
return keyValueExists('userName', newUserInfo.userName).then(function (userNameAlreadyExists) {
if (userNameAlreadyExists) {
validationResult = {
isOK: false,
reason: 'Username already exists',
infoWithBadInput: 'userName',
username: newUserInfo.userName
}
} else {
// create new user here
validationResult.userNumber = (Math.random() * 100000000) | 0;
}
return validationResult;
});
}
else {
// this function always needs to return a promise, even if it is resolved/rejected immediately
return Promise.reject(validationResult);
}
}
addUser({userName: 'fred'}).then(function (result) {
console.log(result);
}).catch(function(err) {
console.error(err);
});
addUser({userName: 'wilma'}).then(function (result) {
console.log(result);
}).catch(function(err) {
console.error(err);
});
addUser({userName: 'barny'}).then(function (result) {
console.log(result);
}).catch(function(err) {
console.error(err);
});
addUser({userName: 'betty'}).then(function (result) {
console.log(result);
}).catch(function(err) {
console.error(err);
});

Use ldapjs with promise

I want to convert the following code to use promise. It is working and output a user's attributes within the active directory.
var client = ldap.createClient({
url: ldap_url
});
client.bind(ldap_username, ldap_password, function (err) {
client.search(ldap_dn_search, opts, function (err, search) {
search.on('searchEntry', function (entry) {
var user = entry.object;
// It is working!!!. It outputs all user attributes.
console.log(user);
});
});
});
The following is my attempt, butit doesn't output anything.
var Promise = require('promise');
var client_bind = Promise.denodeify(client.bind);
var client_search = Promise.denodeify(client.search);
client_bind(ldap_username, ldap_password)
.then(function(err){
client_search(ldap_dn_search, opts)
.then(function(search){
var search_on = Promise.denodeify(search.on);
search_on('searchEntry')
.then(function(entry){
var user = entry.object;
// It doesn't output anything !!!
console.log(user);
});
});
});

I had the same problem.
Search emits events, so we need something that processes them and passes further along the chain.
Here is piece of code, that works for me:
var ldap = require('ldapjs');
var promise = require('bluebird');
var client = ldap.createClient({url: app.settings['ldap']['server']});
var uid;
promise.promisifyAll(client);
function searchPromise(res, notfoundtext) {
return new Promise(function(resolve, reject) {
var found = false;
res.on('searchEntry', function(entry) {
found = true;
resolve(entry);
});
res.on('error', function(e) {
reject(e.message);
});
res.on('end', function() {
if (!found) {
reject(notfoundtext);
}
});
});
}
client.searchAsync(app.settings['ldap']['baseDn'], {filter: '(mail='+credentials.email+')', scope: 'sub'})
.then(function(res) {
return searchPromise(res, 'User isn\'t exists.');
})
.then(function (entry) {
uid = entry.object.uid;
return client.bindAsync(entry.object.dn, credentials.password);
})
.then(function() {
return client.searchAsync('cn='+app.settings['ldap']['group']+',cn=groups,'+app.settings['ldap']['baseDn'], {scope: 'sub', filter: '(memberUid='+uid+')'});
})
.then(function(res) {
return searchPromise(res, 'User is not in group ' + app.settings['ldap']['group']);
})
.then(function() {
console.log('All is ok');
})
.catch(function(message) {
console.log('Error:' + message);
});
Immediately after the search I add one more step that catches the events, processes them, and passes it further along the chain. This makes the function searchPromise.
Good luck coding )

Most likely those methods do require to be called on client as a context, so you will need to bind() them before passing them to Promise.denodeify:
var client_bind = Promise.denodeify(client.bind.bind(client));
var client_search = Promise.denodeify(client.search.bind(client));
Also, a proper use of promises would look like this:
client_bind(ldap_username, ldap_password).then(function() {
return client_search(ldap_dn_search, opts);
// ^^^^^^ always return something from the callback
}).then(function(search) { // flatten your chain
return Promise.denodeify(search.on).call(search, 'searchEntry');
// ^^^^^^ an alternative to `bind`
}).then(function(entry){
var user = entry.object;
console.log(user);
}).catch(function(err) { // always catch errors!
console.error(err);
});

Using Bluebird Promises, the easy way to do this is to create your client normally, and then run the promisifyAll() on the client.
var ldap = require('ldapjs');
var Promise = require('bluebird');
var client = ldap.createClient({
url: 'ldap://my-server:1234',
});
Promise.promisifyAll(client);
Now you can call client.addAsync() and client.searchAsync() and such.
client.bindAsync(secUserDn, secUserPassword)
.then(doSearch) // if it works, call doSearch
.catch(function (err) { // if bind fails, handle it
console.error('Error on bind', err)
});
function doSearch(data) {
client.searchAsync('CN=A Test,OU=Users,DC=website,DC=com', options)
.then(function (data) { // Handle the search result processing
console.log('I got a result');
})
.catch(function (err) { // Catch potential errors and handle them
console.error('Error on search', err);
});
}

i had the same issue here but i solved it by adding promise and resolve the response without using bluebird, this is an exemple of my code :
async getLdapUser(username: any): Promise<any> {
let myPromise = new Promise<boolean>((resolve, reject) => {
console.log('ssssssssss', username);
const adSuffix = 'OU=xxxx,OU=xxxxx,DC=xxxxxxx,DC=xxxxxx';
const password = 'xxxxxxxxxxxxx';
// Create client and bind to AD
const client = ldap.createClient({
url: 'ldap://1.1.1.1:389',
});
// promise.promisifyAll(client);
let resp = false;
// console.log(client);
client.bind('userTest', password,(err: any) => {
console.log('RESP', resp);
if (err) {
console.log('Error in new connetion ' + err);
} else {
/*if connection is success then go for any operation*/
console.log('Success');
const searchOptions: {} = {
scope: 'sub',
filter: '(sAMAccountName=' + username + ')',
attributes: ['sAMAccountName'],
};
client.search(adSuffix, searchOptions, (err: any, res: any) => {
assert.ifError(err);
res.on('searchEntry', (entry: any) => {
resp = true;
});
res.on('error', (error: any) => {
console.log('err');
reject(error.message);
});
await res.on('end', (result: any) => {
resolve(resp);
});
});
}
});
});
return myPromise;
}