I have a mongodb query and I want to save the results to an array. I also define an empty array outside of query.
var dietaryResults = [];
for (var key in dietary){
Restaurant.find(
{ $text : { $search : dietary[key] } },
{ score : { $meta: "textScore" } }
).sort({ score : { $meta : 'textScore' } }).exec(function(err, results) {
for (var i in results){
dietaryResults.push(results[i]);
}
console.log(dietaryResults);
});
}
If I do console.log(dietaryResults) inside the query like above, I can see the results being pushed into the array. But if I put console.log(dietaryResults) outside of the array (which what I want), it prints an empty string. Can anyone explain this behavior and suggest a solution for this? Thanks.
When I use bluebird (which is a kind of Promise) it works.
var Promise = require('bluebird');
var mongoose = Promise.promisifyAll(require('mongoose'));
var promises = Restaurant.find({ $text : { $search : query } }, function(err, results) {
results.forEach(function(element) {
finalResults.push(element);
});
});
Promise.all(promises).then(function() {
console.log(finalResults);
}).error(console.error);
Restaurant.find is asynchronous.
Function insinde .exec part is being executed after loop ends.
Try
for (var key in dietary){
Restaurant.find(
{ $text : { $search : dietary[key] } },
{ score : { $meta: "textScore" } }
).sort({ score : { $meta : 'textScore' } }).exec(function(err, results) {
for (var i in results){
dietaryResults.push(results[i]);
}
console.log('added');
});
}
console.log('loop end');
You will be able to see, that 'loop end' log will be printed before 'added' log.
If you need all results in array, you should fill this array in callback. There is no way to get data synchronously.
For more information about callbacks and async functions check out this article: https://www.tutorialspoint.com/nodejs/nodejs_callbacks_concept.htm
I'd suggest to search for all required data at once and avoid searching inside loop to have single callback and do everything you need with results inside that callback.
Smth like this may work. If not you should search for some way to get all data in one request
Restaurant.find(
{ $text : { $search : { $in: Object.values(dietary)} } },
{ score : { $meta: "textScore" } }
).sort({ score : { $meta : 'textScore' } }).exec(function(err, results) {
for (var i in results){
dietaryResults.push(results[i]);
}
//do stuff here;
});
You need callback or promise.You can use outside of this **Restaurant.find( **
this:setTimeOut(function s(){console.log(dietaryResults)},1000)
Because you need to learn async functions.
As of javascript asynchronicity fashion, You can't be sure that the .exec callback will run (and end) before the for loop end. (you can have a better understanding on this topic by reading about the event-loop)
This way, dietaryResults can be empty (depend on the query speed) in any other place.
You can read more about using promises with mongoose and node in order to be sure dietaryResults will be full with the results,
and also have better understanding on this topic by reading about Node event loop.
For now, I'm not fully sure this will work - but that's is your direction:
var query = {...} // (enter your query here)
var dietaryResults = [];
Object.entries(diatery).map(async key => {
const results = await Restaurant.find(query).sort();
results.map(result => dietaryResults.push(result));
});
console.log(dietaryResults);
Related
I'm working on a map/reduce script to handle some automated billing processing. I run a search for invoices in the GetInput stage, group them by customer in the Map stage, and then create the payments in the Reduce stage. However, in the Summarize stage, only one key/value pair ever exists. So, I created a dummy test script to play with the functionality and figure it out, and kept running into the same problem.
Here's what I have for the testing script:
define(['N/search'], (search) => {
const getInputData = (inputContext) => {
let filt = [
["trandate","on","3/29/2022"]
, "and"
, ["mainline","is","T"]
];
let cols = [
search.createColumn({ name : "tranid" })
, search.createColumn({ name : "entity" })
, search.createColumn({ name : "total" })
];
let results;
try {
// custom search wrapper
results = getSearchResults("invoice", filt, cols);
}
catch (err) {
log.error({ title : "Error encountered retrieving invoices" , details : err });
}
return results;
}
const map = (mapContext) => {
try {
let data = JSON.parse(mapContext.value);
let output = { key : "" , value : data };
let rand = randomInt(0, 1); // custom random number generator
if (rand === 0) {
output.key = "FAILURE";
}
else {
output.key = "SUCCESS";
}
mapContext.write(output);
}
catch (err) {
log.error({ title : "Map Stage Error" , details : err });
}
}
const reduce = (reduceContext) => {
reduceContext.write({
key : reduceContext.key
, value : JSON.stringify(reduceContext.values)
});
}
const summarize = (summaryContext) => {
summaryContext.output.iterator().each((key, value) => {
log.audit({ title : `summary -- ${key} -- ${typeof value}` , details : value });
});
}
return {getInputData, map, reduce, summarize}
});
By all accounts, the summary logging should have two key entries to report, but only ever has one. In the test I've tried it both by marking values as either SUCCESS or FAILURE in the Map stage and then just passing it through the Reduce stage, and also by passing the values through the Map stage to be marked as SUCCESS or FAILURE in the Reduce stage and then passed on with the invoice record ID as the key. No matter what, the output iterator in the Summarize stage only ever reports back a single key. I've had this work correctly in one particular situation, but for the life of me I can't figure out what's different/wrong.
Any insights? Otherwise the only way I can think of to be able to propagate necessary data is to utilize the 'N/cache' module, which does work pretty well but feels like it should be unnecessary.
My understanding is that you need to return true; from the each() callback function, or it will stop processing.
const summarize = (summaryContext) => {
summaryContext.output.iterator().each((key, value) => {
log.audit({ title : `summary -- ${key} -- ${typeof value}` , details : value });
return true;
});
}
I handle errors in MP script by this way
summarize: function summarize(summarizeContext) {
function handleErrorIfAny(summary)
{
var mapSummary = summary.mapSummary;
var reduceSummary = summary.reduceSummary;
handleErrorInStage('map', mapSummary);
handleErrorInStage('reduce', reduceSummary);
}
function handleErrorInStage(stage, summary)
{
summary.errors.iterator().each(function(key, value){
nLog.error('Failure in key ' + key, value);
return true;
});
}
handleErrorIfAny(summarizeContext);
}
I am trying to load data from the twitter api, getting user information and save that in a temporary array. That array will then be loaded on the page for viewing. The array is getting loaded by the API call, but it doesn't display.
I think I need to use an asynchronous thing like React or Angular, not sure. Would love some input!
function getUserIds (userId) {
T.get('statuses/retweeters/ids', { id: userId }, function (err, data, response) {
for(var i = 0; i < data.ids.length; i++){
ids.push(data.ids[i]);
}
getUserInfo();
});
}
function getUserInfo() {
for(var i = 0; i < ids.length; i++) {
T.get('users/lookup', { user_id: ids[i] }, function (err, data, response) {
names.push(data[0].screen_name);
pics.push(data[0].profile_image_url_https);
console.log(names);
});
}
res.render('display', {names: names, pics:pics});
}
The issue is that you are running ids.length async calls and those will finish some time in the future. You have to render your page only when they are all done. But, your for loop is synchronous so you are calling res.render() before any of them have finished. In addition, your T.get() calls may finish in any order (if that matters).
I would normally use promises for coordinating multiple asynchronous operations since it is a very, very good tool for that. But, if you aren't using promises, here's a simple technique to test when you have all your results back:
function getUserInfo() {
var names = [];
var pics = [];
for(var i = 0; i < ids.length; i++) {
T.get('users/lookup', { user_id: ids[i] }, function (err, data, response) {
if (err) {
// decide what to display if you get an API error
names.push("unknown due to API error");
} else {
names.push(data[0].screen_name);
pics.push(data[0].profile_image_url_https);
console.log(names);
}
if (names.length === ids.length) {
res.render('display', {names: names, pics:pics});
}
});
}
}
As I said above, this does not necessarily collect the results in order. If you need them in order, then you could do something like this:
function getUserInfo() {
var names = new Array(ids.length);
var pics = new Array(ids.length);
var doneCntr = 0;
ids.forEach(function(id, i) {
T.get('users/lookup', { user_id: id }, function (err, data, response) {
if (err) {
// decide what to display if you get an API error
names[i] = "unknown due to API error";
} else {
names[i] = data[0].screen_name;
pics[i] = data[0].profile_image_url_https;
}
++doneCntr;
if (doneCntr === ids.length) {
res.render('display', {names: names, pics: pics});
}
});
});
}
My preferred solution would to be to use Promise.all() and use a promisified version of T.get().
Is any possible to increment var licznik in this block of code?
I try sth like this, But always receives 0. Could someone explain me what I'm doing wrong?
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik){
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik);
&licznik++;
});
}
});
})(licznik);
}
}
});
});
As the comment on your original question points out, I have no context for what this code is actually trying to accomplish. What I can tell you is that your callbacks supplied to connection.query are NOT fired on each iteration of the forEach. The whole reason connection.query takes a callback is because you don't know when the operation will complete. Node is designed to be asynchronous so all it does on each iteration of the forEach loop is begin the query. The callback supplied to the query could be invoked at any time which also means that a query that fired after another query could potentially fire its callback before the callback from the first query. It just depends on how long each query takes.
If you need licznik to be incremented on every iteration of the forEach then you need to increment it after your if statement.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
// .... omitted for brevity
}
licznik++; // <-- increment here, outside of the closure.
});
});
Again, I have zero clue what you're actually trying to do with that variable so this may not solve your real problem, but that's the way to get it to increment in that loop.
PS - You may not be understanding that you actually have two licznik variables here. You create a new one when you wrap all your logic in a closure function like you did. If you change the variable declared at the top of your closure function you'll see that it's not the same variable as the one outside the closure function.
rows.forEach(function(record) {
var licznik = 0;
var offer = manager.createOffer('76561198252553560');
inventory.forEach(function(item) {
if(licznik <= record.amount) {
if(item.market_hash_name == record.real_name) {
var asid = item.assetid;
(function(licznik2) { // <-- notice this is a new variable
connection.query('SELECT count(id) as wynik FROM used where asset_id = \'' + asid + '\'', function(err, wiersze) {
if (wiersze[0].wynik == 0) {
var employee = {
asset_id: asid,
trans_id: record.tid
};
connection.query('INSERT INTO used SET ?', employee, function(err, res) {
if (err) throw err;
offer.addMyItem(item);
console.log(licznik2);
licznik2++;
});
}
});
})(licznik);
}
}
});
});
I need to remove all documents from my mongo db, which dont exists in new array with objects.
So I have array with objects like :
var items = [
{product_id:15, pr_name: 'a', description : 'desc'},
{product_id:44, pr_name: 'b', description : 'desc2'}
{product_id:32, pr_name: 'c', description : 'desc3'}];
and I have array with db values which I get by calling Model.find({}).
So now I do it in a 'straight' way:
async.each(products, function (dbProduct, callback) { //cycle for products removing
var equals = false;
async.each(items, function(product, callback){
if (dbProduct.product_id === product.product_id){
product.description = dbProduct.description;// I need to save desc from db product to new product
equals = true;
}
callback();
});
if (!equals) {
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
}
});
But its blocks the whole app and its very slow, because I have around 5000 values in my items array and in database too. So its very huge cycle numbers.
Maybe there can be a faster way?
UPDATE1
Using code below, from TbWill4321 answer:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Its takes around 11 sec(blocks whole web-app) and takes 12 362 878 cycles for me.
So maybe somebody can advise me something?
The Async library does not execute synchronous code in an asynchronous fashion.
5000 items is not a huge number for JavaScript, as I've worked on Big Data set's with 5 million+ points and it doesn't take long. You can get better performance by structuring like this:
var removeIds = [];
// cycle for products removing
async.each(products, function (dbProduct, callback) {
for ( var i = 0; i < items.length; i++ ) {
if (dbProduct.product_id === product.product_id) {
// I need to save desc from db product to new product
product.description = dbProduct.description;
// Return early for performance
return callback();
}
}
// Mark product to remove.
removeIds.push( dbProduct._id );
log.warn("REMOVE PRODUCT " + dbProduct.product_id);
return callback();
}, function() {
Product.remove({ _id: { $in: removeIds } }, function (err) {
if (err) return updateDBCallback(err);
// Continue Here.
// TODO
});
});
Among the many problems you may have, off the top of my head you may want to start off by changing this bit:
Product.remove({ _id: dbProduct._id }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
Being within a .each() call, you'll make one call to the database for each element you want to delete. It's better to store all the ids in one array and then make a single query to delete all elements that have an _id that is in that array. Like this
Product.remove({ _id: {$in: myArrayWithIds} }, function (err) {
if (err) return updateDBCallback(err);
callback();
});
On another note, since async will execute synchronously, node.js does offer setImmediate() (docs here), that will execute the function from within the event loop. So basically you can "pause" execution of new elements and serve any incoming requests to simulate "non-blocking" processing.
I'm racking my brain trying to figure out how to sequence this / place callbacks to get what I need.
I have a loop that checks for the existence of files. I'd like it to callback when it's done, but the for loop and the callback finish before the "fs.open" finishes... typical asynchronous problem.
I am using node v0.10.29, express v3.14.0, and am looking at using the "async" library, but again, just can't figure out the logic that I need...
Here is what I have:
Input
function checkForAllFiles(callback)
{
var requiredFiles = [];
requiredFiles[requiredFiles.length] = "../Client/database/one.js";
requiredFiles[requiredFiles.length] = "../Client/database/two.dat";
requiredFiles[requiredFiles.length] = "../Client/database/three.xml";
requiredFiles[requiredFiles.length] = "../Client/database/four.dat";
requiredFiles[requiredFiles.length] = "../Client/database/five.dat";
requiredFiles[requiredFiles.length] = "../Client/database/six.xml";
var missingFiles = [];
for(var r=0; r<requiredFiles.length; r++)
{
fs.open(requiredFiles[r], 'r', function(err, fd){
if(err)
{
missingFiles[missingFiles.length] = err.path;
console.log("found missing file = ", err.path);
}
});
console.log("r = ", r);
}
console.log("sending callback: ", missingFiles);
callback(missingFiles);
}
Output
0
1
2
3
4
5
sending callback: []
found missing file: ../Client/database/three.xml
Desired Output
0
1
found missing file: ../Client/database/three.xml
2
3
4
5
sending callback: ["../Client/database/three.xml"]
I would use the reject method in the async module (which I see you've already found). What it will do is return an array in its callback that contains any elements that don't match a specified check function. For the check function, I'd recommend just using fs.exists instead of watching for an error on fs.open.
Using those functions you can actually reduce the whole check to one line. Something like this:
function checkForAllFiles(callback)
{
var requiredFiles = [];
requiredFiles[requiredFiles.length] = "../Client/database/one.js";
requiredFiles[requiredFiles.length] = "../Client/database/two.dat";
requiredFiles[requiredFiles.length] = "../Client/database/three.xml";
requiredFiles[requiredFiles.length] = "../Client/database/four.dat";
requiredFiles[requiredFiles.length] = "../Client/database/five.dat";
requiredFiles[requiredFiles.length] = "../Client/database/six.xml";
async.reject(requiredFiles, fs.exists, callback);
}
callback will get called with an array that contains just the files that don't exist.
Use the async library and the eachSeries method. Example:
async.eachSeries(array,
function(element, next) {
// do something with element
next();
}
);
It will sequentially go through the array and process each element. Calling next goes to the next element. Series makes sure it does it in the order of the array, otherwise the order of going through the array is not guaranteed. If you have other async functions called within it, just pass the next function around and call it when done with all the needed functions and the next array element will be processed.
Maybe something like this:
var missingFiles = []
async.eachSeries(requiredFiles, function(file, nextFile){
fs.open(file, 'r', function(err, fd){
if(err)
{
missingFiles[missingFiles.length] = err.path;
console.log("found missing file = ", err.path);
}
nextFile();
});
console.log("r = ", file);
});
console.log("sending callback: ", missingFiles);
callback(missingFiles);