I've been digging into redis and working on a tiny webapp that uses redis as it's only data storage (I know it's not the intended purpose of redis but I am benefitting from learning the commands and just overall working with redis on Node. I'm using Node_Redis.
Here's what I want to accomplish (all in redis):
I am try to retrieve the users using their emails.
Here's the problem:
I have a Promise.all call which takes all emails (keys) and maps each to go a HGET command. When the Promise.all resolves I expect it to resolve with an array of user objects, but it ends up resolving to an array of booleans (i.e. [true, true, true]).
Here is the logic for /users
router.get("/", (req, res) => {
client.lrange("emails", 0, 1, (err, reply) => {
if (err) return console.log(err);
if (reply) {
Promise.all(
reply.map(email => {
return client.hgetall(email, (err, reply) => {
if (err) return console.log(err);
console.log("reply for hgetall", reply); // this prints a user object correct, but Promise.all still resolves to boolean array :(
return reply;
});
})
)
.then(replies => {
// replies = [true, true, true ...]
res.send(replies);
})
.catch(e => console.log(e));
} else {
res.send([reply, "reply is null"]);
}
});
});
I've actually used Promise.all plenty of times, and when I log the reply from redis, it shows the correct object too, so I'm pretty confused at this point. How can I get Promise.all to resolve to an array of User objects?
The problem is that client.hgetall doesn't return a promise. It's async function and you pass a callback to get a result. You should promisify this function to use it in Promise.all:
...
return new Promise((resolve, reject) => {
client.hgetall(email, (err, reply) => {
if (err) {
return reject(err);
}
resolve(reply);
});
});
You can do promisification manually (as in example above) or can use Bluebird or Q libraries with their promisify and promisifyAll methods.
Related
I'm using crypto.generateKeyPair inside a post endpoint in Express.
I need to insert the key generated inside my DB and then return within the endpoint the id row of the inserted row.
The endpoint code reads as:
app.post('/createKeys', (req, res) => {
crypto.generateKeyPair('rsa',,
(err, publicKey, privateKey) => {
if(!err) {
let id = myfunction(publicKey.toString('hex'),
privateKey.toString('hex'));
console.log(id)
} else {
res.status(500).send(err);
}
});
});
async function myfunction(publicKey, privateKey) {
await pool.query('INSERT INTO users (publickey, privatekey) VALUES ($1, $2) RETURNING id',
[publicKey, privateKey],
(error, results) => {
if (error) {
throw error;
}
resolve(results.rows[0]['id']);
});
};
However, inside the callback in crypto I get only a Promise, or undefined if I don't use async/await. How can I await myfunction result so I can send back to the user the id?
Several issues here:
await only does something useful when you are awaiting a promise.
pool.query() does not return a promise when you pass it a callback, so your await there is not doing anything useful.
resolve() is not a function that exists outside the context of creating a new promise with new Promise((resolve, reject) => { code here })
throw error inside an asynchronous callback will not do anything useful as there is no way to catch that exception and thus no way to implement any decent error handling. Don't write code that way. When you promisify the function (as shown below), you can then reject the promise and that will offer a way to propagate the error back to the caller.
Your choices for waiting for pool.query() with await here are:
Use the version of your database that natively supports promises and then don't pass a callback to pool.query() so that it returns a promise that tells you when it's complete.
Promisify your own function by wrapping pool.query() in a new promise and call resolve() and reject() appropriately.
Remember, do NOT mix plain callbacks and promise. Instead, promisify any asynchronous functions that use plain callbacks and then do all your logic flow with promises.
Here's a manually promisified version of your myfunction():
function myfunction(publicKey, privateKey) {
return new Promise((resolve, reject) => {
pool.query('INSERT INTO users (publickey, privatekey) VALUES ($1, $2) RETURNING id',
[publicKey, privateKey],
(error, results) => {
if (error) {
reject(error);
return;
}
resolve(results.rows[0]['id']);
});
});
}
crypto.generateKeyPairP = util.promisify(crypto.generateKeyPair);
app.post('/createKeys', async (req, res) => {
try {
const {publicKey, privateKey } = await crypto.generateKeyPairP('rsa');
const id = await myfunction(publicKey.toString('hex'), privateKey.toString('hex'));
console.log(id);
// send your response here, whatever you want it to be
res.send(id);
} catch(e) {
res.status(500).send(e);
}
});
Note that in this implementation, the resolve and reject functions come from the new Promise() - they don't just exist in this air.
But, there is likely a version of your database or an interface in your existing database module where pool.query() can return a promise directly using built-in promise support.
I have a project collection. Under the project schema have a property called tasks which is an array with the ids of the tasks. Now I am trying to write an api for singleProject, which should include the project details with a property taskDetails which will contain the detail of the tasks under the project. And for getting the task details I am using the ids which I stored in the task property under project. Below is the code I am using :
exports.getSingleProject = async (req, res) => {
req.project.logo = undefined;
await req.project.tasks.map((taskItem, index) => {
taskSchema.find({ _id: taskItem.toString() }).exec((err, task) => {
req.project.tasksDetails.push(task);
});
});
responseMessages(
res,
200,
true,
"Single project fetched successfully.",
req.project
);
};
but in the response getting taskDetails as an blank array. I guess I am using async await in wrong place. Need some help on this. Thanks in advance
The await keyword only works with a promise: an array is not a promise, but you can map to a promise and wait for all promises to resolve using Promise.all - see example below:
const tasks = await Promise.all(req.project.tasks.map(item => new Promise((accept, reject) => {
taskSchema.find({ _id: taskItem.toString() }).exec((err, task) => {
if(err) {
reject(err);
return;
}
accept(task);
});
});
The tasks variable would be an array of your tasks, which you can return in the response.
Introduction
Hey there,
I am trying to pass out data from the mongoose withTransaction callback. Right now, I am using the following code which implements callbacks:
const transactionSession = await mongoose.startSession()
await transactionSession.withTransaction(async (tSession) => {
try {
// MARK Transaction writes & reads removed for brevity
console.log("Successfully performed transaction!")
cb(null, "Any test data")
return Promise.resolve()
} catch (error) {
console.log("Transaction aborted due to error:", error)
cb(error)
return Promise.reject()
}
})
} catch (error) {
console.log(error)
return cb(error)
}
A more detailed snippet of the withTransaction helper in use can be found here.
A link to the official Mongoose documentation regarding the withTransaction helper can be found here.
At the moment, I am using a callback to pass out data from the withTransactioncallback:
cb(null, "Any test data")
However, the problem is that naturally the callback is executed first, before the Promise.resolve() is returned. This means, that (in my case) a success response is sent back to the client before any necessary database writes are committed:
// this is executed first - the callback will send back a response to the client
cb(null, "Any test data")
// only now, after the response already got sent to the client, the transaction is committed.
return Promise.resolve()
Why I think this is a problem:
Honestly, I am not sure. It just doesn't feel right to send back a success-response to the client, if there hasn't been any database write at that time. Does anybody know the appropriate way to deal with this specific use-case?
I thought about passing data out of the withTransaction helper using something like this:
const transactionResult = await transactionSession.withTransaction({...})
I've tried it, and the response is a CommandResult of MongoDB, which does not include any of the data I included in the resolved promise.
Summary
Is it a problem, if a success response is sent back to the client before the transaction is committed? If so, what is the appropriate way to pass out data from the withTransaction helper and thereby committing the transaction before sending back a response?
I would be thankful for any advice I get.
It looks like there is some confusion here as to how to correctly use Promises, on several levels.
Callback and Promise are being used incorrectly
If the function is supposed to accept a callback, don't return a Promise. If the function is supposed to return a Promise, use the callback given by the Promise:
const transactionSession = await mongoose.startSession()
await transactionSession.withTransaction( (tSession) => {
return new Promise( (resolve, reject) => {
//using Node-style callback
doSomethingAsync( (err, testData) => {
if(err) {
reject(err);
} else {
resolve(testData); //this is the equivalent of cb(null, "Any test data")
}
});
})
Let's look at this in more detail:
return new Promise( (resolve, reject) => { This creates a new Promise, and the Promise is giving you two callbacks to use. resolve is a callback to indicate success. You pass it the object you'd like to return. Note that I've removed the async keyword (more on this later).
For example:
const a = new Promise( (resolve, reject) => resolve(5) );
a.then( (result) => result == 5 ); //true
(err, testData) => { This function is used to map the Node-style cb(err, result) to the Promise's callbacks.
Try/catch are being used incorrectly.
Try/catch can only be used for synchronous statements. Let's compare a synchronous call, a Node-style (i.e. cb(err, result)) asynchronous callback, a Promise, and using await:
Synchronous:
try {
let a = doSomethingSync();
} catch(err) {
handle(err);
}
Async:
doSomethingAsync( (err, result) => {
if (err) {
handle(err);
} else {
let a = result;
}
});
Promise:
doSomethingPromisified()
.then( (result) => {
let a = result;
})
.catch( (err) => {
handle(err);
});
Await. Await can be used with any function that returns a Promise, and lets you handle the code as if it were synchronous:
try {
let a = await doSomethingPromisified();
} catch(err) {
handle(err);
}
Additional Info
Promise.resolve()
Promise.resolve() creates a new Promise and resolves that Promise with an undefined value. This is shorthand for:
new Promise( (resolve, reject) => resolve(undefined) );
The callback equivalent of this would be:
cb(err, undefined);
async
async goes with await. If you are using await in a function, that function must be declared to be async.
Just as await unwraps a Promise (resolve into a value, and reject into an exception), async wraps code into a Promise. A return value statement gets translated into Promise.resolve(value), and a thrown exception throw e gets translated into Promise.reject(e).
Consider the following code
async () => {
return doSomethingSync();
}
The code above is equivalent to this:
() => {
const p = new Promise(resolve, reject);
try {
const value = doSomethingSync();
p.resolve(value);
} catch(e) {
p.reject(e);
}
return p;
}
If you call either of the above functions without await, you will get back a Promise. If you await either of them, you will be returned a value, or an exception will be thrown.
So I created a function in my node server which takes in a query string, runs it on my db, and returns the results. I then wanted to use my function asynchronously using async await throughout my routes instead of having nested query within, nested query, within nested query etc.
So here is the code:
const runQuery = queryString => {
console.log("...runQuery")
db.query(queryString, (error, results, fields) => {
if (error) {
console.log("runQuery: FAILURE");
return error;
}
else {
console.log("runQuery: SUCCESS");
return(results);
}
})
}
register.post("/", async (req, res) => {
console.log(req.body);
const results = await runQuery("select * from test1");
res.send(results);
})
The database should have 3 entries, but unfortunately, it returns nothing. Meaning results is an empty variable when it is sent, meaning JS never properly waits for it to capture the db results. How can I use my function asynchronously, and how is this even feasible?
It seems your function "runQuery" does not return a promise, in fact, it's not returning anything. You are using "return" in the callback of the db.query function, not the function "runQuery" itself.
Since runQuery is performing an asynchronous operation, the result ought to be resolved via a promise (which is what the "await" in your request handler is looking for).
I'm not exactly sure but it seems you are using MySql, and I could not find anything on the npm page of the mysql package regarding the query being promisified, so we'll promisify it ourselves:
const runQuery = (queryString) => new Promise((resolve,reject) => {
console.log("...runQuery")
db.query(queryString, (error, results, fields) => {
if (error) {
console.error("runQuery: FAILURE");
reject(error);
} else {
console.log("runQuery: SUCCESS");
resolve(results);
}
})
})
register.post("/", async (req, res) => {
console.log(req.body);
try{
const results = await runQuery("select * from test1");
res.send(results);
}catch(e){
console.error(`ERROR THROWN BY runQuery:`,e);
res.status(500).send({
message: e.message || "INTERNAL SERVER ERROR"
})
}
})
Note that if an error occurs, our promisified function will reject the error and it will NOT be stored in the "results" variable in our request handler. Instead, an error will be thrown which needs to be handled. Therefore, it is always a good practice to put any async/await calls inside a try/catch.
I have a simple node module which connects to a database and has several functions to receive data, for example this function:
dbConnection.js:
import mysql from 'mysql';
const connection = mysql.createConnection({
host: 'localhost',
user: 'user',
password: 'password',
database: 'db'
});
export default {
getUsers(callback) {
connection.connect(() => {
connection.query('SELECT * FROM Users', (err, result) => {
if (!err){
callback(result);
}
});
});
}
};
The module would be called this way from a different node module:
app.js:
import dbCon from './dbConnection.js';
dbCon.getUsers(console.log);
I would like to use promises instead of callbacks in order to return the data.
So far I've read about nested promises in the following thread: Writing Clean Code With Nested Promises, but I couldn't find any solution that is simple enough for this use case.
What would be the correct way to return result using a promise?
Using the Promise class
I recommend to take a look at MDN's Promise docs which offer a good starting point for using Promises. Alternatively, I am sure there are many tutorials available online.:)
Note: Modern browsers already support ECMAScript 6 specification of Promises (see the MDN docs linked above) and I assume that you want to use the native implementation, without 3rd party libraries.
As for an actual example...
The basic principle works like this:
Your API is called
You create a new Promise object, this object takes a single function as constructor parameter
Your provided function is called by the underlying implementation and the function is given two functions - resolve and reject
Once you do your logic, you call one of these to either fullfill the Promise or reject it with an error
This might seem like a lot so here is an actual example.
exports.getUsers = function getUsers () {
// Return the Promise right away, unless you really need to
// do something before you create a new Promise, but usually
// this can go into the function below
return new Promise((resolve, reject) => {
// reject and resolve are functions provided by the Promise
// implementation. Call only one of them.
// Do your logic here - you can do WTF you want.:)
connection.query('SELECT * FROM Users', (err, result) => {
// PS. Fail fast! Handle errors first, then move to the
// important stuff (that's a good practice at least)
if (err) {
// Reject the Promise with an error
return reject(err)
}
// Resolve (or fulfill) the promise with data
return resolve(result)
})
})
}
// Usage:
exports.getUsers() // Returns a Promise!
.then(users => {
// Do stuff with users
})
.catch(err => {
// handle errors
})
Using the async/await language feature (Node.js >=7.6)
In Node.js 7.6, the v8 JavaScript compiler was upgraded with async/await support. You can now declare functions as being async, which means they automatically return a Promise which is resolved when the async function completes execution. Inside this function, you can use the await keyword to wait until another Promise resolves.
Here is an example:
exports.getUsers = async function getUsers() {
// We are in an async function - this will return Promise
// no matter what.
// We can interact with other functions which return a
// Promise very easily:
const result = await connection.query('select * from users')
// Interacting with callback-based APIs is a bit more
// complicated but still very easy:
const result2 = await new Promise((resolve, reject) => {
connection.query('select * from users', (err, res) => {
return void err ? reject(err) : resolve(res)
})
})
// Returning a value will cause the promise to be resolved
// with that value
return result
}
With bluebird you can use Promise.promisifyAll (and Promise.promisify) to add Promise ready methods to any object.
var Promise = require('bluebird');
// Somewhere around here, the following line is called
Promise.promisifyAll(connection);
exports.getUsersAsync = function () {
return connection.connectAsync()
.then(function () {
return connection.queryAsync('SELECT * FROM Users')
});
};
And use like this:
getUsersAsync().then(console.log);
or
// Spread because MySQL queries actually return two resulting arguments,
// which Bluebird resolves as an array.
getUsersAsync().spread(function(rows, fields) {
// Do whatever you want with either rows or fields.
});
Adding disposers
Bluebird supports a lot of features, one of them is disposers, it allows you to safely dispose of a connection after it ended with the help of Promise.using and Promise.prototype.disposer. Here's an example from my app:
function getConnection(host, user, password, port) {
// connection was already promisified at this point
// The object literal syntax is ES6, it's the equivalent of
// {host: host, user: user, ... }
var connection = mysql.createConnection({host, user, password, port});
return connection.connectAsync()
// connect callback doesn't have arguments. return connection.
.return(connection)
.disposer(function(connection, promise) {
//Disposer is used when Promise.using is finished.
connection.end();
});
}
Then use it like this:
exports.getUsersAsync = function () {
return Promise.using(getConnection()).then(function (connection) {
return connection.queryAsync('SELECT * FROM Users')
});
};
This will automatically end the connection once the promise resolves with the value (or rejects with an Error).
Node.js version 8.0.0+:
You don't have to use bluebird to promisify the node API methods anymore. Because, from version 8+ you can use native util.promisify:
const util = require('util');
const connectAsync = util.promisify(connection.connectAsync);
const queryAsync = util.promisify(connection.queryAsync);
exports.getUsersAsync = function () {
return connectAsync()
.then(function () {
return queryAsync('SELECT * FROM Users')
});
};
Now, don't have to use any 3rd party lib to do the promisify.
Assuming your database adapter API doesn't output Promises itself you can do something like:
exports.getUsers = function () {
var promise;
promise = new Promise();
connection.connect(function () {
connection.query('SELECT * FROM Users', function (err, result) {
if(!err){
promise.resolve(result);
} else {
promise.reject(err);
}
});
});
return promise.promise();
};
If the database API does support Promises you could do something like: (here you see the power of Promises, your callback fluff pretty much disappears)
exports.getUsers = function () {
return connection.connect().then(function () {
return connection.query('SELECT * FROM Users');
});
};
Using .then() to return a new (nested) promise.
Call with:
module.getUsers().done(function (result) { /* your code here */ });
I used a mockup API for my Promises, your API might be different. If you show me your API I can tailor it.
2019:
Use that native module const {promisify} = require('util'); to conver plain old callback pattern to promise pattern so you can get benfit from async/await code
const {promisify} = require('util');
const glob = promisify(require('glob'));
app.get('/', async function (req, res) {
const files = await glob('src/**/*-spec.js');
res.render('mocha-template-test', {files});
});
When setting up a promise you take two parameters, resolve and reject. In the case of success, call resolve with the result, in the case of failure call reject with the error.
Then you can write:
getUsers().then(callback)
callback will be called with the result of the promise returned from getUsers, i.e. result
Using the Q library for example:
function getUsers(param){
var d = Q.defer();
connection.connect(function () {
connection.query('SELECT * FROM Users', function (err, result) {
if(!err){
d.resolve(result);
}
});
});
return d.promise;
}
Below code works only for node -v > 8.x
I use this Promisified MySQL middleware for Node.js
read this article Create a MySQL Database Middleware with Node.js 8 and Async/Await
database.js
var mysql = require('mysql');
// node -v must > 8.x
var util = require('util');
// !!!!! for node version < 8.x only !!!!!
// npm install util.promisify
//require('util.promisify').shim();
// -v < 8.x has problem with async await so upgrade -v to v9.6.1 for this to work.
// connection pool https://github.com/mysqljs/mysql [1]
var pool = mysql.createPool({
connectionLimit : process.env.mysql_connection_pool_Limit, // default:10
host : process.env.mysql_host,
user : process.env.mysql_user,
password : process.env.mysql_password,
database : process.env.mysql_database
})
// Ping database to check for common exception errors.
pool.getConnection((err, connection) => {
if (err) {
if (err.code === 'PROTOCOL_CONNECTION_LOST') {
console.error('Database connection was closed.')
}
if (err.code === 'ER_CON_COUNT_ERROR') {
console.error('Database has too many connections.')
}
if (err.code === 'ECONNREFUSED') {
console.error('Database connection was refused.')
}
}
if (connection) connection.release()
return
})
// Promisify for Node.js async/await.
pool.query = util.promisify(pool.query)
module.exports = pool
You must upgrade node -v > 8.x
you must use async function to be able to use await.
example:
var pool = require('./database')
// node -v must > 8.x, --> async / await
router.get('/:template', async function(req, res, next)
{
...
try {
var _sql_rest_url = 'SELECT * FROM arcgis_viewer.rest_url WHERE id='+ _url_id;
var rows = await pool.query(_sql_rest_url)
_url = rows[0].rest_url // first record, property name is 'rest_url'
if (_center_lat == null) {_center_lat = rows[0].center_lat }
if (_center_long == null) {_center_long= rows[0].center_long }
if (_center_zoom == null) {_center_zoom= rows[0].center_zoom }
_place = rows[0].place
} catch(err) {
throw new Error(err)
}