I have a nodejs Mongodb code like this . I'm trying to push into a global array and return all my results but the array contains only one value . Any idea why ?
var s = [];
var p = function(){
return new Promise(function(resolve, reject) {
MongoClient.connect(url, function (err, db) {
db.listCollections().forEach(function(collname) {
db.collection(collname.name).find().sort( { duration_in_seconds: 1 }).limit(1).toArray(
function(err, docs) {
assert.equal(null, err);
s.push(docs);
resolve();
});
});
});
});
}
p().then(function() {
console.log(s);
});
You are resolving the promise when the first collections returns its document. You'd need to wait for all of them. Instead of wrapping everything in a large new Promise, promisify every asynchronous function on its own, and make the returned promise fulfill with the result value instead of putting it in a global variable:
function connect(url) {
return new Promise(function(resolve, reject) {
MongoClient.connect(url, function (err, db) {
if (err) reject(err);
else resolve(db);
});
});
}
function getArray(cursor) {
return new Promise(function(resolve, reject) {
cursor.toArray(function(err, docs) {
if (err) reject(err);
else resolve(docs);
});
});
}
Now you can write your p using these helpers, put the promises for each collection in an array, and await them with Promise.all, which yields a promise right for the array of all results:
function p() {
return connect(url).then(function(db) {
return getArray(db.listCollections()).then(function(collections) {
var promises = collections.map(function(collname) {
return getArray(db.collection(collname.name).find().sort({duration_in_seconds: 1 }).limit(1));
});
return Promise.all(promises);
});
});
}
p().then(function(s) {
console.log(s);
});
Related
Normally if I was going to run multiple mongoose queries I would use the built in promise to chain them all together. In this case, the user chooses which schemas to search. This could be one of them or both. The following example uses the post data to define which schemas to search and if one is false, it should continue through the promise chain. Right now the final promise is being called before the queries.
Example in my express controller:
app.post('/custom-search', function (req, res) {
var single = false
var multi = false
if(req.body.single){
var single = true
}
if(req.body.multi){
var multi = true
}
var promise = new Promise(function (resolve, reject) {
if(multi){
multiSchema.find({}, function (err, result) {
if(!err){
console.log(result);
resolve()
}
})
}else{
resolve()
}
}).then(function (value) {
if(single){
singleSchema.find({}, function (err, result) {
if(!err){
console.log(result);
resolve()
}
})
}else{
resolve()
}
}).then(function (value) {
console.log("done");
})
})
});
output:
>done
>[singleResults]
>[multiResults]
done should be printing last so that is the first problem.
Like we discussed, few things had to be clean up. First by actually using and returning the promise for it work properly, Second, creating a mini-promise within your first .then() to resolve and reject your single conditional statement. And third, handling/catching promises.
I wrote a pseudo version of your code to illustrate my point of view, hopefully it may be of a good use.
app.get('/custom-search', function (req, res) {
// Manipulating values to test responses
var single = false;
var multi = true;
var promise = new Promise(function (resolve, reject) {
if (multi) {
setTimeout(function () {
resolve('MULTI RESOLVED!');
}, 3000);
} else {
reject('MULTI REJECTED!');
}
})
.then(function (value) {
new Promise(function (resolve, reject) {
if (single) {
setTimeout(function () {
resolve('SINGLE RESOLVED!');
}, 3000);
} else {
reject('SINGLE REJECTED!');
}
})
.catch(function (error) {
console.log('SINGLE ERROR!', error);
})
.then(function (value) {
console.log('SINGLE DONE', value);
});
})
.catch(function (error) {
console.log('MULTI ERROR!', error);
});
return promise;
});
I have the question, below code:
The problem is:
How can I send each line from response promiseGetCitiesData to promiseGetInformationDataPerCity.
Can I do it in one async.each functions?
Now, I created multiple Promise functions. One general function, which one start the program - getDataAndCloseDb().
Also I used async.each to call promise function with array parameter - locationArray.
Now, I would like to send each line from json response to next promise function (create get url), and collect the general response.
const MongoClient = require("mongodb").MongoClient;
const request = require("request");
const async = require("async");
var locationsArray = [
'location1',
'location2',
'location3'
];
function promiseConnectToDatabase(urldb) {
return new Promise(function(resolve, reject) {
MongoClient.connect(urldb, (err, db) => {
if (err) {
console.log("MongoDb connection error.");
reject(err);
}
console.log("Connected to MongoDb.");
resolve(db);
});
});
}
function promiseGetCitiesData(location) {
return new Promise(function(resolve, reject) {
request({
url: `https://example.com/${location}`,
json: true
}, (error, response, body) => {
if (error) {
console.log("Error connection to url.");
reject();
}
console.log("location: " + location);
console.log({location: location, cities: body.result.cities});
resolve({location: location, cities: body.result.cities});
});
});
}
/*
Example response from promiseGetCitiesData:
Location: location1
{ location: 'location1',
cities:
[ 'information1',
'information2',
'information3',
'information4'' ] }
*/
function promiseGetInformationDataPerCity(location, cities) {
return new Promise(function(resolve, reject) {
request({
url: `https://example.com/${location}/${cities}`,
//f.e https://example.com/location1/information1 etc.
json: true
}, (error, response, information) => {
if (error) {
console.log("Error connection to url.");
reject();
}
console.log(information);
resolve(information);
});
});
}
function promiseSaveDataToDatabase(db, body) {
return new Promise(function(resolve, reject) {
db.collection("testlocation").insert(body, function(dbError) {
if (dbError) {
reject(dbError);
}
resolve()
});
});
}
function promiseDisconnectDatabase(db) {
return new Promise(function(resolve, reject) {
db.close((err) => {
if (err) {
console.log("MongoDb disconnect error.");
reject(err);
}
console.log("MongoDb disconnected.");
resolve();
});
});
}
function promiseProvideDataFromEach(locationsArray, db) {
return new Promise(function(resolve, reject) {
async.each(locationsArray, function(loc, locProcessedCb) {
promiseGetcitiesData(loc).then(function(resultscities) {
promiseGetInformationDataPerCity(loc, resultscities).then(function(resultDetails) {
promiseSaveDataToDatabase(db, resultDetails).then(function() {});
locProcessedCb();
});
});
}, function(err) {
if (err) {
locProcessedCb(err);
reject(err);
}
console.log("All locations have been processed.");
resolve();
});
});
}
function getDataAndCloseDb() {
return new Promise(function(resolve, reject) {
promiseConnectToDatabase("mongodb://127.0.0.1:27017/testApp").then(function(db) {
promiseProvideDataFromEach(locationsArray, db).then(function() {
promiseDisconnectDatabase(db).then(function() {});
});
});
});
}
getDataAndCloseDb();
I think this is a lot simpler than the code in the question makes it appear. In particular, new Promise(...) can be completely avoided by :
using require('async-request') instead of require('request').
allowing MongoDb methods to return Promise, as many of them will do if no callback is passed.
Also
by using the Promise.all(array.map(...)) pattern the need for require('async') disappears.
https://stackoverflow.com/a/28915678/3478010 - provides a great little reusable disposer utility, which is useful here.
Remember to return a promise/value from every .then() callback that is itself asynchronous and/or should deliver data.
With some guesswork, I think you want something like this :
const MongoClient = require('mongodb').MongoClient;
const request = require('async-request'); // just like `request()` but returns a promise
var locationsArray = [
'location1',
'location2',
'location3'
];
function promiseGetCitiesData(loc) {
return request({
url: `https://example.com/${loc}`,
json: true
}).then(body => body.result.cities);
}
function promiseGetInformationDataPerCity(loc, cities) {
return Promise.all(cities.map(city => {
return request({
'url': `https://example.com/${loc}/${city}`,
'json': true
}).then(cityInfo => ({ 'name':city, 'info':cityInfo }));
}));
}
function promiseProvideDataFromEach(locationsArray, db) {
return Promise.all(locationsArray.map(loc => {
return promiseGetCitiesData(loc)
.then(cities => promiseGetInformationDataPerCity(loc, cities)
.then(citiesWithCityInfo => ({ 'location':loc, 'cities':citiesWithCityInfo }));
}))
.then(resultDetails => db.collection('testlocation').insertMany(resultDetails));
}
// disposer utility - credit: https://stackoverflow.com/a/28915678/3478010
function withDb(work) {
var _db;
return MongoClient.connect("mongodb://127.0.0.1:27017/testApp")
.then((db) => {
_db = db; // keep reference
return work(db); // perform work on db
}).finally(() => {
if (_db)
_db.close();
});
}
withDb(db => promiseProvideDataFromEach(locationsArray, db))
.then(() => {
// connection released here
});
The guesswork centres mainly around what is to be inserted at db.collection('testlocation').insertMany(resultDetails). The code in the question gives no more than a clue. My attempt seems reasonable but may not be exactly what you want. Be prepared to make some changes in promiseProvideDataFromEach() and promiseGetInformationDataPerCity().
you can do something like this. Its a simpler code but I think you can map it to your current code.
const Promise = require('bluebird')
const cities = ['citya', 'cityb', 'cityc']
function resolveCities() {
return new Promise(function(resolve, reject) {
resolve(cities)
})
}
function logCity(city) {
console.log('city ', city)
}
return resolveCities()
.then(function(cities) {
return Promise.mapSeries(cities, function(city) {
logCity(city);
});
})
I am trying to create an array called pw_array, assign the contents of pw_subscribers to that array, then append each object in the pw_array with new key value pairs from the second promise. I am new to promises and having a lot of trouble making this work. Right now when I console.log(pw_customer) inside the second promise, inside the getCustomers function, it is returning what I want. But when I console.log(pw_array) later it is the original array.
var pw_array = [];
//first promise is working correctly
var getPaywhirlSubscribers = new Promise(function(resolve, reject) {
paywhirl.Subscriptions.getsubscribers({limit:100}, function(error, pw_subscribers) {
Promise.all(JSON.parse(pw_subscribers).forEach(function(pw_subscriber) {
pw_array.push(pw_subscriber);
}))
// console.log(pw_array);
return resolve(pw_array);
});
});
var getGatewayReferences = function(pw_array) {
return new Promise(function(resolve, reject) {
Promise.all(pw_array.forEach(function(pw_customer) {
paywhirl.Customers.getCustomer(pw_customer.customer_id, function(error, customer) {
pw_customer.phone = customer.phone;
pw_customer.address = customer.address;
pw_customer.gateway_reference = customer.gateway_reference;
// this console.log is returning what I want
// console.log(pw_customer);
});
}));
resolve(pw_array);
// console.log(pw_array);
});
};
and the promise chain...
getPaywhirlSubscribers.then(getGatewayReferences).then(function(pw_array) {
// this console.log is returning the original pw_array with pw_subscribers but not with the appended pw_customer keys
console.log(pw_array);
});
All of your code can be reduced to
var getPaywhirlSubscribers = function() {
return new Promise(function(res, rej) {
paywhirl.Subscriptions.getSubscribers({limit:100}, function(err, subs) {
if (err) {
rej(err);
} else {
res(JSON.parse(subs));
}
});
});
};
var gatewayRefs = function(promiseOfArray) {
return promiseOfArray.then(function(subs) {
return Promise.all(subs.map(function(sub) {
return new Promise(function(res, rej) {
paywhirl.Customers.getCustomer(sub.customer_id, function(err, customer) {
if (err) {
rej(err);
} else {
res(Object.assign({}, sub, customer);
}
});
});
});
});
};
gatewayRefs(getPaywhirlSubscribers()).then(function(arrayOfCustomers) {
// do something with the customer array
});
Note that you can make this even shorter/simpler if you use one of the many utilities available to automatically convert node.js-style error -first callback APIs into Promise-based ones. Search for 'promise denodeify'.
You could also conceivably pull some of the steps out into a .then chain to reduce nesting, but that's as much aesthetic as practical IMHO.
I got 4 promises here, and I thought that it would run the first one, then wait until its finished, THEN run the next one, wait till finished and THEN run the next one etc..
But what happens here is that it runs all of them all at once and does not wait for anything to finish.
This is my promise chain:
//
// Run the promises
//
findBanks
.then(findReceipts)
.then(findExpenses)
.then(sendResult)
.catch(err => {
console.error(err);
console.log("getbankAccountReport ERR: " + err);
res.json({error:true,err})
});
This is the output from my console.log
=====findAllBank=====
=====findAllReceipt=====
=====findAllExpense=====
=====RESOLVE findAllBank=====
=====sendResult=====
=====RESOLVE sendResult=====
=====RESOLVE findAllReceipt=====
=====RESOLVE findAllExpense=====
Am I not understanding promises correct or?
Anyway here is my nodejs controller:
exports.getBankAccountReport = function(req, res) {
//
// Find all bank accounts
//
var bankModel = require('../models/bankModel');
var bankTable = mongoose.model('bankModel');
var bankArray = [];
var findAllBank = new Promise(
(resolve, reject) => {
console.log("=====findAllBank=====")
bankTable.aggregate([
...lots of mongo stuff...
],function(err, data) {
if (!err) {
bankArray = data;
console.log("=====RESOLVE findAllBank=====")
resolve(data);
} else {
reject(new Error('findBank ERROR : ' + err));
}
});
});
//
// Find the RECEIPT for each bank account
//
var receiptModel = require('../models/receiptModel');
var receiptTable = mongoose.model('receiptModel');
var receiptArray = [];
var findAllReceipt = new Promise(
(resolve, reject) => {
console.log("=====findAllReceipt=====")
receiptTable.aggregate([
...lots of mongo stuff...
], function (err, data) {
if (!err) {
receiptArray = data;
console.log("=====RESOLVE findAllReceipt=====")
resolve(data);
} else {
reject(new Error('findReceipts ERROR : ' + err));
}
});
});
//
// Find the EXPENSE for each bank account
//
var expenseModel = require('../models/expenseModel');
var expenseTable = mongoose.model('expenseModel');
var expenseArray = [];
var findAllExpense = new Promise(
(resolve, reject) => {
console.log("=====findAllExpense=====")
expenseTable.aggregate([
...lots of mongo stuff...
], function (err, data) {
if (!err) {
expenseArray = data;
console.log("=====RESOLVE findAllExpense=====")
resolve(data);
} else {
reject(new Error('findExpense ERROR : ' + err));
}
});
});
var sendResult = function(data) {
var promise = new Promise(function(resolve, reject){
console.log("=====sendResult=====")
res.json({error:false,
"bank":bankArray,
"receipt":receiptArray,
"expense":expenseArray})
console.log("=====RESOLVE sendResult=====")
resolve();
});
return promise;
};
//
// Run the promises
//
findAllBank
.then(findAllReceipt)
.then(findAllExpense)
.then(sendResult)
.catch(err => {
console.error(err);
console.log("getbankAccountReport ERR: " + err);
res.json({error:true,err})
});
}
You need to wrap your Promises in functions
var findAllBank = function() {
return new Promise(
(resolve, reject) => {
console.log("=====findAllBank=====")
bankTable.aggregate([
...lots of mongo stuff...
],function(err, data) {
if (!err) {
bankArray = data;
console.log("=====RESOLVE findAllBank=====")
resolve(data);
} else {
reject(new Error('findBank ERROR : ' + err));
}
});
});
});
When resolved, the next function in the chain will be called with the data passed in the resolve() function.
Do no confuse the Promise and the function that builds it
When you create a new Promise(executor), you instanciate a new object that will have two methods (functions of an object), .then(resolveCB [, rejectCB]) and .catch(rejectCB).
The aime is to know, whenever a process is done, if it was successful or if it failed, and the continue accordingly.
var myFirstPromise = new Promise(function executor(resolve, reject) { resolve('resolved'); });
In other words, those methods are used to continue your process once the promise defined by executor is settled. It can either get fulfilled and call the resolveCB callback (usingthen), or rejected and call the rejectCBcallback (using both then and catch). A callback (resolveCB or rejectCB) is a function, not a Promise itself, even if the callback might return a Promise.
myFirstPromise
.then(function resolveCB(result) { console.log(result); }) //you can use a 2nd callback for rejection at this point
.catch(function rejectCB(err) { console.log(err); });
myFirstPromise
.then(
function resolveCB(result) { console.log(result); } // if resolved
, function rejectCB(err) { console.log(err); } // if rejected
)
.catch(function rejectCB(err) { console.log(err); }); // NEVER forget the last catch, just my 2cents :)
We saw the inputs of .then() and .catch() but what about their return value? The both of them will return a new Promise. That's why you can chain the .then()'s and .catch()'s.
myFirstPromise
.then(function resolveCB1(result) { console.log(result); })
.then(function resolveCB2(result) { console.log(result); }) // console.log is called but result is undefined
.catch(function rejectCB1(err) { console.log(err); });
myFirstPromise
.then(function resolveCB3(result) {
throw 'I threw an exception'; // an exception is thrown somewhere in the code
console.log(result);
})
.then(function resolveCB4(result) { console.log(result); })
.catch(function rejectCB2(err) { console.log(err); }); // a promise in the chain get rejected or error occured
In the previous example, we see that our second .then() is hit but result is undefined. The promise returned by first .then() fullfiled but no value as been passed by the executor to the resolve callback resolveCB2. In the second case, an exception occured in resolveCB3, it gets rejected so rejectCB2 is called. If we want our resolve callbacks to receive an argument, we have to notify the exector. To do so, the simplest way is for the callbacks to return a value:
myFirstPromise
.then(function resolveCB1(result) {
console.log(result);
result += ' again';
return result;
})
.then(function resolveCB2(result) {
console.log(result);
return result;
})
.catch(function rejectCB1(err) { console.log(err); });
Now that's said, you've got all the pieces together for understanding a Promise. Let's try to sum it up in a cleaner way:
var myFirstPromise = new Promise(function executor(resolve, reject) { resolve('resolved'); })
, resolveCB = function resolveCB(result) {
console.log(result);
result += ' again';
return result;
}
, resolveLastCB = function resolveLastCB(result) {
console.log(result);
result += ' and again';
return result;
}
, justLog = function justLog(result) {
console.log(result);
return result;
}
;
myFirstPromise
.then(resolveCB)
.then(resolveLastCB)
.then(justLog)
.catch(justLog);
You can now chan them nicely, it's cool and all
myFirstPromise
.then(resolveCB)
.then(resolveCB)
.then(resolveCB)
.then(resolveCB)
.then(resolveCB)
.then(resolveCB)
.then(resolveLastCB)
.then(justLog)
.catch(justLog);
But what if your Promise chain 'really' changes and you need to get rid of myFirstPromise and start with resolveCB instead ? It's just a function, it can be executed but doesn't have any .then() or .catch() method. It's not a Promise. You can't do resolveCB.then(resolveLastCB), it will thow an error resolveCB.then( is not a function or something similar. You might think this is a gross mistake, I didn't call resolveCB and resolveCB().then(resolveLastCB) should work? Unfortunalty for those who thought about that, it's still wrong. resolveCB returns a string, some characters, not a Promise.
In order to avoid this kind of maintenance issue, you should know that the resolve and reject callbacks can return a Promise instead of a value. To do so, we'll use what's called the factory pattern. In simple words, the factory pattern is about instanciating new object using a (static) function instead of using the constructor directly.
var myFirstPromiseFactory = function myFirstPromiseFactory() {
/*
return new Promise(function executor(resolve, reject) {
resolve('resolved');
});
if you just need to resolve a Promise, this is a quicker way
*/
return Promise.resolve('resolved');
}
, resolveFactory = function resolveFactory(result) {
return new Promise(function executor(resolve, reject) {
result = result || 'I started the chain';
console.log(result);
result += ' again';
return resolve(result); // you can avoid the return keyword if you want, I use it as a matter of readability
})
}
, resolveLastFactory = function resolveLastFactory(result) {
return new Promise(function executor(resolve, reject) {
console.log(result);
result += ' and again';
return resolve(result);
});
}
, justLogFactory = function justLogFactory(result) {
return new Promise(function executor(resolve, reject) {
console.log(result);
return resolve(result);
});
}
;
myFirstPromiseFactory() //!\\ notice I call the function so it returns a new Promise, previously we started directly with a Promise
.then(resolveFactory)
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
// Now you can switch easily, just call the first one
resolveFactory()
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
justLogFactory('I understand Javascript')
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
Factory functions might come handy when iterating over an array. It might be used to produce an array of promise given an input:
var myFirstPromiseFactory = function myFirstPromiseFactory() {
/*
return new Promise(function executor(resolve, reject) {
resolve('resolved');
});
if you just need to resolve a Promise, this is a quicker way
*/
return Promise.resolve('resolved');
}
, resolveFactory = function resolveFactory(result) {
return new Promise(function executor(resolve, reject) {
result = result || 'I started the chain';
console.log(result);
result += ' again';
return resolve(result); // you can avoid the return keyword if you want, I use it as a matter of readability
})
}
, resolveLastFactory = function resolveLastFactory(result) {
return new Promise(function executor(resolve, reject) {
console.log(result);
result += ' and again';
return resolve(result);
});
}
, justLogFactory = function justLogFactory(result) {
return new Promise(function executor(resolve, reject) {
console.log(result);
return resolve(result);
});
}
, concatValues = function concatValues(values) {
return Promise.resolve(values.join(' '));
}
, someInputs = [
'I am an input'
, 'I am a second input'
, 'I am a third input'
, 'I am yet an other input'
]
;
myFirstPromiseFactory() //!\\ notice I call the function so it returns a new Promise, previously we started directly with a Promise
.then(resolveFactory)
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
// Now you can switch easily, just call the first one
resolveFactory()
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
justLogFactory('I understand Javascript')
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
// Using a factory functions to create an array of promise usable with Promise.all()
var promiseArray = someInputs.map(function(input) {
return justLogFactory(input);
});
Promise.all(promiseArray)
.then(concatValues)
.then(resolveLastFactory)
.then(justLogFactory)
.catch(justLogFactory);
I am trying to get all records from a DynamoDB table using promises. The problem is that DynamoDB do not return all items in one call I have to make multiple calls. If LastEvaluatedKey is not null means that I need to make another call with that key to get the remaining records. In my code I am checking that and resolving only after LastEvaluatedKey is null. But the console.log("done") is not being executed.
Here is my code:
function query(params) {
return new Promise(function(resolve, reject) {
docClient.query(params, function(err, data) {
if (err) {
reject(err)
} else {
resolve(data);
}
});
})
}
function getAllRecords(params, combinedData) {
return new Promise(function(resolve, reject) {
query(params)
.then(function(data) {
if(!combinedData) {
combinedData = [];
}
combinedData.push(data.Items);
if(data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
getAllRecords(params, combinedData)
}
else {
resolve(combinedData);
}
})
})
}
getAllRecords(params)
.then(function() {
console.log('done')
})
.catch(function(error) {
console.log(error);
})
It's probably a misconception on how promises work from my part. If someone can give me an idea how to make this work. That would be great.
You've fallen prey to the explicit promise construction antipattern, in which you are manually constructing promises when you don't need to.
Generally the only time you need to use the Promise constructor is when you are converting non-Promise async code to Promise async code. You've already done that in your query() function, so you don't need to use the Promise constructor in your getAllRecords() function.
You should do this instead:
function getAllRecords(params) {
return query(params).then(function (data) {
var items = [data.Items];
if(data.LastEvaluatedKey) {
params.ExclusiveStartKey = data.LastEvaluatedKey;
return getAllRecords(params).then(function (theRest) {
return items.concat(theRest);
});
}
return items;
});
}