We Keep Coding

How to wait for a loop to finish running before sending back a response in NodeJs?

I'm trying to send back an array of objects after a GET request to a URL but I can't figure out how to structure the logic so the return array fully populates before sending it to the client.
Below is my server side code for responding to the request. I used passport.js previously in the code to create a login portal and user object. I'm trying to take the "connections" array from the user who is making the request and sending them back an array of objects with the connection's name and picture. I know that the code below is incorrect syntax wise but it's an overview of what I'm trying to accomplish. I've tried doing it the callback way but that just kept leading me in circles because I couldn't figure out the correct logic for it.
router.get('/data', function(req, res, next) {
var IdArr = req.user.connections;
var retArr = [];
function getUsers() {
for (i = 0; i < IdArr.length; i++) {
User.getUserById(IdArr[i], function(err, patient) {
retArr[i] = {name:patient.name, pic:patient.profileImage};
});
}
}
function getDataAndSend() {
function(getUsers(), function sendRes() { // I know this is incorrect syntax
console.log(retArr);
res.json(retArr);
});
}
getDataAndSend();
});

First of all, you should declare i, and when you do it should be with block-scope (let), so that the nested callback function will use the same variable.
You could check how many entries in retArr have been retrieved and call res.json once you know you have them all.
router.get('/data', function(req, res, next) {
var IdArr = req.user.connections;
var retArr = [];
for (let i = 0; i < IdArr.length; i++) {
User.getUserById(IdArr[i], function(err, patient) {
retArr[i] = {name:patient.name, pic:patient.profileImage};
if (Object.keys(retArr).length === IdArr.length) res.json(retArr);
});
}
});

I think the easiest way to deal with these things is with Promises. Any async function written in the callback way (e.g. this User.getUserById) can be turned into a function that returns a promise. You just wrap the call around a new Promise and resolve when you're done.
Say in your case.
function promiseGetById(id) {
return new Promise((resolve, reject) =>
User.getUserById(id, (err, pat) => resolve(pat))
);
}
Then something like
Promise.all(IdArr.map(id => promiseGetById(id))).then(arr =>
res.json(
arr.map(patient => ({ name: patient.name, pic: patient.profileImage }))
)
);
Alternatively if you don't like promises you can do it by having a counter or something which in each callback you increment and then in your callback you res.json when the counter is the correct value.

Change all your function logic to return promises and use async/await for code clarity.
const getUserById = (id) => {
return new Promise((resolve, reject) => {
User.getUserById(IdArr[i], function(err, patient) {
resolve({name:patient.name, pic:patient.profileImage});
});
});
}
const getAllUsers = async(idArr) => {
const retArr = [];
// for...of loop to await without continuing the loop
// but this will execute only sequentially
for (let id of idArr) {
const ret = await getUserById(id);
retArr.push(ret);
}
// for parallel execution, use Promise.all()
await Promise.all([...idArr.map(id => getUserById(id))]);
return retArr;
}
router.get('/data', async (req, res, next) => {
var IdArr = req.user.connections;
var retArr = await getAllUsers(IdArr);
console.log(retArr);
res.json(retArr);
});

You seem to be trying to extract way to much functions for an algorithm that isn't that complicated. The code below initialy extract the information it needs for the request. Then, it populates the array needed for the response and simply sends it.
router.get('/data', (req, res, next) => {
const idArray = req.user.connections;
let responseArray = [];
// loop to populate responseArray
for (let id of idArray) {
User.getUserById(id, (err, patient) => {
// add to the end of the array the patient informations
responseArray.push({
name: patient.name,
pic: patient.profileImage
});
});
}
// send back responseArray
res.json(responseArray);
});

how to get all keys and values in redis in javascript?

I am creating a node API using javascript. I have used redis as my key value store.
I created a redis-client in my app and am able to get values for perticular key.
I want to retrieve all keys along with their values.
So Far I have done this :
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
for(var i=0;i<keys.length;i++){
client.get(keys[i], function (error, value) {
if (err) return console.log(err);
var job = {};
job['jobId']=keys[i];
job['data']=value;
jobs.push(job);
});
}
console.log(jobs);
res.json({data:jobs});
}
});
});
but I always get blank array in response.
is there any way to do this in javascript?
Thanks

First of all, the issue in your question is that, inside the for loop, client.get is invoked with an asynchronous callback where the synchronous for loop will not wait for the asynchronous callback and hence the next line res.json({data:jobs}); is getting called immediately after the for loop before the asynchronous callbacks. At the time of the line res.json({data:jobs}); is getting invoked, the array jobs is still empty [] and getting returned with the response.
To mitigate this, you should use any promise modules like async, bluebird, ES6 Promise etc.
Modified code using async module,
app.get('/jobs', function (req, res) {
var jobs = [];
client.keys('*', function (err, keys) {
if (err) return console.log(err);
if(keys){
async.map(keys, function(key, cb) {
client.get(key, function (error, value) {
if (error) return cb(error);
var job = {};
job['jobId']=key;
job['data']=value;
cb(null, job);
});
}, function (error, results) {
if (error) return console.log(error);
console.log(results);
res.json({data:results});
});
}
});
});
But from the Redis documentation, it is observed that usage of
Keys are intended for debugging and special operations, such as
changing your keyspace layout and not advisable to production
environments.
Hence, I would suggest using another module called redisscan as below which uses SCAN instead of KEYS as suggested in the Redis documentation.
Something like,
var redisScan = require('redisscan');
var redis = require('redis').createClient();
redisScan({
redis: redis,
each_callback: function (type, key, subkey, value, cb) {
console.log(type, key, subkey, value);
cb();
},
done_callback: function (err) {
console.log("-=-=-=-=-=--=-=-=-");
redis.quit();
}
});

Combination of 2 requests:
import * as ioredis from 'ioredis';
const redis = new ioredis({
port: redisPort,
host: redisServer,
password: '',
db: 0
});
const keys = await redis.collection.keys('*');
const values = await redis.collection.mget(keys);
Order will be the same for both arrays.

This will get all keys but with no values:
const redis = require('redis');
const client = redis.createClient();
client.keys('*', (err, keys) => {
// ...
});
Now you need to get the values for those keys in a usual way. For example:
Promise.all(keys.map(key => client.getAsync(key))).then(values => {
// ...
});
or with async module or in any way you like.

You should never do this. First off, it is not recommended to use KEYS * in production. Second, this does not scale (cluster).
You can organise your cached entries into SETs and query for the items within the SET, then retrieve the references keys. This also makes invalidation easier.
Have a look at some data storage best practices.
https://redis.io/topics/data-types-intro
how to get all keys and values in redis in javascript?

You may find something useful in this link
https://github.com/NodeRedis/node_redis/tree/master/examples

Call async function while pushing the same function on array

I have this code:
var queue = [];
var allParserd = [];
_.each(webs, function (web) {
queue.push(function () {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
allParserd.push(parsed);
});
});
});
Promise.all(queue).then(function (data) {
console.log(allParserd);
});
Basically I need to fetch all my webs and be sure to give the result after that every parsing is done. the function parseWebsite return the correct data, but in this way is not called and allParsed return just as an empty array. I'm sure that I miss some things, I've started to use the promises just from some days.
If you need some more information just tell me.
P.s.
I want that all the functions to start at the same time; I don't want to wait for each one response for going forward.

Tagged with Bluebird so let's use it:
First, let's convert your callback API to promises:
Promise.promisifyAll(WebsitesUtils);
Now, let's use .map to map every item in webs to it being parsed parseWebsite:
Promise.map(webs, function(item){
return WebsitesUtils.parseWebsiteAsync(item); // note the suffix
}).then(function(results){
// all the results are here.
}).catch(function(err){
// handle any errors
});
As you can see - this is trivial to do with Bluebird.

Promise.all doesn't take a queue of functions to execute. It expects an array of promises which represent the results of the many concurrently running (still pending) requests.
The first step is to have a function that actually returns a promise, instead of only executing a callback. We can use
function parseWebsite(web) {
return new Promise(function(fulfill, reject) {
WebsitesUtils.parseWebsite(web, function (err, parsed) {
if (err)
reject(err);
else
fulfill(parsed);
});
});
}
or simply use promisification that does this generically:
var parseWebsite = Promise.promisify(WebsitesUtils.parseWebsite, WebsitesUtils);
Now we can go to construct our array of promises by calling that function for each site:
var promises = [];
_.each(webs, function (web) {
promises.push(parseWebsite(web));
});
or just
var promises = _.map(webs, parseWebsite);
so that in the end we can use Promise.all, and get back our allParsed array (which even is in the same order as webs was!):
Promise.all(promises).then(function(allParsed) {
console.log(allParsed);
});
Bluebird even provides a shortcut function so you don't need promises:
Promise.map(webs, parseWebsite).then(function(allParsed) {
console.log(allParsed);
});

Here's how might do it with async:
var async = require('async');
var webs = ...
async.map(webs, function(web, callback) {
WebsitesUtils.parseWebsite(web, callback);
}, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});
and if parseWebsite() isn't a prototype method dependent on WebsitesUtils then you could simplify it further:
async.map(webs, WebsitesUtils.parseWebsite, function(err, results) {
if (err) throw err; // TODO: handle errors better
// `results` contains all parsed results
});

Get node.js neDB data into a variable

I am able to insert and retrieve data from an neDB database in nodejs. But I cannot pass the data outside of the function that retrieves it.
I have read through the neDB documentation and I searched for and tried different combinations of callbacks and returns (see code below) without finding a solution.
I'm new to javascript so I do not know if I am misunderstanding how to use variables in general or if this issue is related to using neDB specifically or both.
Could someone please explain why "x" in my code does not ever contain the docs JSON results from the database? How can I make it work?
var fs = require('fs'),
Datastore = require('nedb')
, db = new Datastore({ filename: 'datastore', autoload: true });
//generate data to add to datafile
var document = { Shift: "Late"
, StartTime: "4:00PM"
, EndTime: "12:00AM"
};
// add the generated data to datafile
db.insert(document, function (err, newDoc) {
});
//test to ensure that this search returns data
db.find({ }, function (err, docs) {
console.log(JSON.stringify(docs)); // logs all of the data in docs
});
//attempt to get a variable "x" that has all
//of the data from the datafile
var x = function(err, callback){
db.find({ }, function (err, docs) {
callback(docs);
});
};
console.log(x); //logs "[Function]"
var x = db.find({ }, function (err, docs) {
return docs;
});
console.log(x); //logs "undefined"
var x = db.find({ }, function (err, docs) {
});
console.log(x); //logs "undefined"*

Callbacks are generally asynchronous in JavaScript meaning you can not use assignment operator, consequently you do not return anything from the callback function.
When you call an async function execution of you programme carries on, passing the 'var x = whatever' statement. The assignment to a variable, the result of whatever callback is received, you need to perform from within the callback itself... what you need is something in the lines of ...
var x = null;
db.find({ }, function (err, docs) {
x = docs;
do_something_when_you_get_your_result();
});
function do_something_when_you_get_your_result() {
console.log(x); // x have docs now
}
EDIT
Here is a nice blog post about asynchronous programming. And there is a lot more resources on this topic that you can pick up.
This is a popular library to help with async flow-control for node.
P.S.
Hope this helps. Please, by all means ask if you need something clarified :)

I ran into the same problem. In the end I used a combination between async-await and a promise with resolve to solve it.
In your example the following would work:
var x = new Promise((resolve,reject) {
db.find({ }, function (err, docs) {
resolve(docs);
});
});
console.log(x);

I had to learn a bit about async functions to get it right. For those who are looking for specific help about getting a return value from nedb, here's a snippet of what worked for me. I was using it in electron.
function findUser(searchParams,callBackFn)
{
db.find({}, function (err, docs))
{
//executes the callback
callBackFn(docs)
};
}
usage
findUser('John Doe',/*this is the callback -> */function(users){
for(i = 0; i < users.length; i++){
//the data will be here now
//eg users.phone will display the user's phone number
}})

How to aggregate array of json objects from forEach function?

I want to render array of stories from all projects in node.js. How to do that?
app.get('/stories', function(request, response) {
var projects_ids = [1,2];
project_ids.forEach(function(id) {
pivotal.getStories(id, function(err, project_stories) {
console.log("Stories: " + project_stories);
return JSON.stringify(project_stories);
});
});
response.send(array_of_stories);
});
In log I get:
Stories: [object Object]
Stories: [object Object]

Your current approach won't work since getStories are async (assumption based on the method signature of getStories). If you can, I would suggest that you create a method on pivotal that can get stories for multiple project ids, so your code would read:
app.get('/stories', function(req, res) {
var project_ids = [1, 2];
pivotal.getStories(project_ids, function(err, project_stories) {
res.send(project_stories);
}
});
If that is not an option, I would suggest that you look into a flow library like e.g. node-seq. Then you code could read something like this:
app.get('/stories', function(req, res) {
var project_ids = [1, 2];
Seq(project_ids)
.parEach(function(project_id) {
pivotal.getStories(project_id, this);
})
.seq(function() {
var aggregatedStories = [];
Hash.map(this.args, (function(arg) {
// Each parSeq aparently results in an array of objects
aggregatedStories.push(arg[0]);
}));
res.send(aggregatedStories);
});
});
The Hash.map function is from a node module called hashish
Edit: To elaborate a little more, parEach will execute the functions in parallel, and the following seq would execute after all the callbacks from the parallel executions have finished. node-seq puts the result from each parallel execution into the parameter array of the following seq, hence the somewhat cryptic Hash.map(this.args)

You want your server to send an array of project_stories objects as a response to the request? I'm not sure where your return statement returns to, so omitting that I would try something similar to the following to send an array of objects as the response to the request. Edit: As pointed out by others, .getStories is asynchronous. Maybe you could try out, caolan's async module. Implemented like this:
var async = require('async');
var array_of_stories = [];
var getStories = function(id, callback) {
pivotal.getStories(id, function(err, project_stories) {
array_of_stories.push(project_stories);
});
}
app.get('/stories', function(request, response) {
var projects_ids = [1,2];
async.forEach(project_ids, getStories, function(error) {
if(!error){
response.send(array_of_stories);
}
});
});