In my Express route, I am trying to return a list of elements that I am grabbing from MongoDB using Mongoose. I'm basically iterating through an array of items, and making MongoDB calls to get the parameter objects that each item has. However, I'm having trouble making sure that I get all the parameters before I send the response. I've tried using promises, other async library functions, etc, but none of them have seemed to work.
The current iteration of the code looks like this (I have tried a lot of different things):
exports.findAll = function(req, res){
Flow.find({}, function(err, items) {
console.log(items);
var payload = {}
var params = [];
for (var i=0; i < items.length; i++) {
var count2 = 0;
async.whilst(
function() {
return ((items[i]) && (count2 < items[i].params.length));
},
function(callback) {
Parameter.findById(items[i].params[count2], function(err, out) {
params.push(out);
count2++;
callback();
});
},
function(err) {
console.log(params);
var payload = {
"flows": items,
"params": params
};
res.send(payload);
console.log('success: flows found');
}
);
}
This code sends a payload with params not being completely full.
What would be a good way to deal with this? Honestly I just want these database calls to be synchronous, but I just can't figure out how to make this work.
This doesn't really seem necessary as you can actually use the $in operator with all the results from your first query:
Flow.find({},function(err,items) {
var ids = [];
// blocking? yes, but should be minor - do better if there are problems
for ( var i=0; i < items.length; i++ ) {
for ( var n=0; n < items[i].params.length; n++ ) {
ids.push( items[i].params[n] );
}
}
Parameter.find({ "_id": { "$in": ids } },function(err,params) {
res.send({ "flows": items, "params": params });
});
});
So there should be no reason to execute multiple queries inside an async loop, or loops as your code seems to be missing as the direct cause of the problem there.
Related
Im trying to make a webscraper(educational puposes), and I got really far, but this little issue is bugging me.
I made a request callback function, and im trying to get lines 75-78 to work. However to get this to work, I need PDF_LISTS and PDF_LINKS to initilaze to the right values.
I've already tried to make them global variables, and what not, for some reason that doesnt work. So my question is: How do I make a callback function that will call that for loop (75-78) and succesfully initilaze PDF_LISTS and PDF_LINKS to the correct values ?
(Dont worry I use this on educational content, with the prof's permission). First time posting here!
// URL_LINKS has the pdf links of the pages
PDF_LINKS = [];
// URL_LIST has the names of the pdf links
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a'); //jquery get all hyperlinks
$(links).each(function(i, link) {
var value = $(link).attr('href');
// creates objects to hold the file
if (value.substring(value.length - 3, value.length) == "pdf") {
PDF_LINKS[i] = $(link).attr('href');
PDF_LIST[i] = $(link).text();
}
})
});
}
// must decleare fillPDF variable or else you wont initilze teh variables
fillPDF() {
//HERE I WANT PDF_LINKS and PDF_LIST to be intialized to 33.....
}
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}
You may push your values into arrays using array's push method, avoiding array's element to be undefined.
You can put your final for loop into a function, and then use fillPDF();
You also need to call fillPDF's callback once the request is over.
PDF_LINKS = [];
PDF_LIST = [];
function fillPDF(callback) {
request(url, function(err, res, body) {
$ = cheerio.load(body);
links = $('a');
$(links).each(function(i, link) {
var value = $(link).attr('href');
if (value.slice(-3) == "pdf") {
PDF_LINKS.push(value);
PDF_LIST.push($(link).text());
}
})
callback();
});
}
function writePDF() {
for (j = 0; j < PDF_LIST.length; j++) {
request(PDF_LINKS[j]).pipe(fs.createWriteStream(PDF_LIST[j]));
}
}
fillPDF(writePDF);
I've been using request to iterate through several XML entries and return every article, date, and url to the console by using cheerio.js. The main issue is that the output will appear in a different order each time, as Request is an asynchronous function. I'm really inexperienced with javascript in general, and was wondering how I could retrieve consistent output (I've been reading about async and promises, I'm just unsure how to implement them).
Here is my code:
var count = 0;
for(var j = 0; j < arrNames.length; j++){
request('http://dblp.org/search/publ/api?q=' + arrNames[j], function(error, response, html){
if (!error && response.statusCode == 200){
var $ = cheerio.load(html, {xmlMode: true});
console.log($('query').text()+'\n');
$('title').each(function(i, element){
var title = $('title').eq(i).text();
var year = Number($('year').eq(i).text());
var url = $('ee').eq(i).text();
if (year >= arrTenures[count]){
console.log(title);
console.log(year);
console.log(url + '\n');
}
});
count++;
}
});
}
Though you've already found a solution, I thought I'd show you how you would do this using ES6 promises (a more modern approach for managing multiple asynchronous operations):
const rp = require('request-promise');
Promise.all(arrNames.map(item => {
return rp('http://dblp.org/search/publ/api?q=' + item).then(html => {
const $ = cheerio.load(html, {xmlMode: true});
return $('title').map(function(i, element){
const title = $(element).text();
const year = Number($('year').eq(i).text());
const url = $('ee').eq(i).text();
return {title, year, url};
}).get();
});
})).then(results => {
// results is an array of arrays, in order
console.log(results);
}).catch(err => {
console.log(err);
});
This offers several advantages:
Promise.all() puts the results in order for you.
rp() checks the 2xx status for you.
This can be chained/sequenced with other asynchronous operations more easily.
Error handling is simpler and low level errors are propagated out to the top level for you automatically.
This is throw-safe (if async errors are thrown, they are captured and propagated).
It looks like you're trying to capture the iteration number of each request, so use forEach and utilize its second parameter, which indicates the iteration index:
arrNames.forEach((q, requestIndex) => {
request('http://dblp.org/search/publ/api?q=' + q, (error, response, html) => {
if (error || response.statusCode == 200) return;
var $ = cheerio.load(html, {
xmlMode: true
});
console.log($('query').text() + '\n');
$('title').each(function(i, element) {
var title = $('title').eq(i).text();
var year = Number($('year').eq(i).text());
var url = $('ee').eq(i).text();
if (year >= arrTenures[requestIndex]) {
console.log(title);
console.log(year);
console.log(url + '\n');
}
});
});
});
As a side note, consistent indentation really improves code readability - you might consider a linter.
On your first try, you might have tried:
if (year >= arrTenures[j]) {
But noticed that didn't work. This is because of scoping issues
You can solve your problem by using an iterator like forEach(), or simply changing your for loop to use let:
for(let j = 0; j < arrNames.length; j++){
Now you can just use j in your check instead of count.
The real question though, is why are arrTenures and arrNames separate arrays? Their information clearly relates to each other, so relying on the array index to couple them seems like a bad idea. Instead, you should try and keep a single array of objects with all related information. For example:
[
{ name: 'some name', tenures: 2 },
{ name: 'another', tenures: 5 }
]
Sorry if this is a stupid question, but how would I go about adding items into a list? So what I've got is a loop that basically runs through and tries to convert all the urls to tinyurls from a web scraper . It still produces an empty list for images_short. I'm not very familiar with nodejs's syntax. Here's a snippet of code, I've put some data in the images_long list:
const TinyURL = require('tinyurl');
var images_long = ['https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-skateboarding-lucas-premiere-adv-primeknit-khaki-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=728297932403d74d2ac1afa5ecdfa97d', 'https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-nmd-r1-stlt-triple-black-first-look-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=918752eba81826e4398950efc69a5141'];
var images_short = [];
for (i = 0; i < 2; i++) {
TinyURL.shorten(images_long[i], function(res) {
images_short.push(res[i]);
});
}
I still get an empty list when I changed images_short.push(res[i]); to images_short.push(res);
res is a string, so just images_short.push(res); will do the trick. Also, you should iterate with respect to the length of the variable you're indexing, and you should var your indexing variable (i):
const TinyURL = require('tinyurl');
var images_long = [
'https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-skateboarding-lucas-premiere-adv-primeknit-khaki-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=728297932403d74d2ac1afa5ecdfa97d',
'https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-nmd-r1-stlt-triple-black-first-look-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=918752eba81826e4398950efc69a5141'];
var images_short = [];
for (var i = 0; i < images_long.length; i++) {
TinyURL.shorten(images_long[i], function(res) {
images_short.push(res);
});
}
The tinyurl library is async.
Is we use native map, the resulting callback wouldn't be returned if we try to console.log(images_short) until all the links in the array have been shortened.
We can however, use async and specificically use async.map to return the results like the example below.
const TinyURL = require('tinyurl');
const async = require('async');
var images_long = [
'https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-skateboarding-lucas-premiere-adv-primeknit-khaki-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=728297932403d74d2ac1afa5ecdfa97d',
'https://hypebeast.imgix.net/http%3A%2F%2Fhypebeast.com%2Fimage%2F2017%2F06%2Fadidas-nmd-r1-stlt-triple-black-first-look-0.jpg?fit=max&fm=pjpg&h=344&ixlib=php-1.1.0&q=90&w=516&s=918752eba81826e4398950efc69a5141'];
function shorten(item, cb) {
TinyURL.shorten(item, function(res) {
cb(null, res);
});
}
async.map(images_long, shorten, (err, results) => {
console.log(results);
});
we can assign images_short if you want to keep consistency.
I'm having trouble getting the following to work
var data = db.collection('mycollection', function(er, collection) {
return collection.find().toArray();
});
The resulting data is not an array. I don't get what's wrong. How do I set a varaible to the contents of find().toArray()?
I've tried logging the contents like this so I know that there must be data:
db.collection('mycollection', function(er, collection) {
collection.find().toArray(function(err, results) {
for (var i = 0; i < results.length; i++) {
console.log(results[i]);
}
});
});
Thanks! I'm very new to ajax programming and mongodb.
Just go it like in simple way:-
.find method always provide data in array format.
var query = db.collection.find({},function(err,data){
if(err) throw err;
console.log(data); // give in array.
})
thanks
I am trying to load data from the twitter api, getting user information and save that in a temporary array. That array will then be loaded on the page for viewing. The array is getting loaded by the API call, but it doesn't display.
I think I need to use an asynchronous thing like React or Angular, not sure. Would love some input!
function getUserIds (userId) {
T.get('statuses/retweeters/ids', { id: userId }, function (err, data, response) {
for(var i = 0; i < data.ids.length; i++){
ids.push(data.ids[i]);
}
getUserInfo();
});
}
function getUserInfo() {
for(var i = 0; i < ids.length; i++) {
T.get('users/lookup', { user_id: ids[i] }, function (err, data, response) {
names.push(data[0].screen_name);
pics.push(data[0].profile_image_url_https);
console.log(names);
});
}
res.render('display', {names: names, pics:pics});
}
The issue is that you are running ids.length async calls and those will finish some time in the future. You have to render your page only when they are all done. But, your for loop is synchronous so you are calling res.render() before any of them have finished. In addition, your T.get() calls may finish in any order (if that matters).
I would normally use promises for coordinating multiple asynchronous operations since it is a very, very good tool for that. But, if you aren't using promises, here's a simple technique to test when you have all your results back:
function getUserInfo() {
var names = [];
var pics = [];
for(var i = 0; i < ids.length; i++) {
T.get('users/lookup', { user_id: ids[i] }, function (err, data, response) {
if (err) {
// decide what to display if you get an API error
names.push("unknown due to API error");
} else {
names.push(data[0].screen_name);
pics.push(data[0].profile_image_url_https);
console.log(names);
}
if (names.length === ids.length) {
res.render('display', {names: names, pics:pics});
}
});
}
}
As I said above, this does not necessarily collect the results in order. If you need them in order, then you could do something like this:
function getUserInfo() {
var names = new Array(ids.length);
var pics = new Array(ids.length);
var doneCntr = 0;
ids.forEach(function(id, i) {
T.get('users/lookup', { user_id: id }, function (err, data, response) {
if (err) {
// decide what to display if you get an API error
names[i] = "unknown due to API error";
} else {
names[i] = data[0].screen_name;
pics[i] = data[0].profile_image_url_https;
}
++doneCntr;
if (doneCntr === ids.length) {
res.render('display', {names: names, pics: pics});
}
});
});
}
My preferred solution would to be to use Promise.all() and use a promisified version of T.get().