I'm trying to couple ES7's async/await with knex.js transactions.
Although I can easily play around with non-transactional code, I'm struggling to get transactions working properly using the aforementioned async/await structure.
I'm using this module to simulate async/await
Here's what I currently have:
Non-transactional version:
works fine but is not transactional
app.js
// assume `db` is a knex instance
app.post("/user", async((req, res) => {
const data = {
idUser: 1,
name: "FooBar"
}
try {
const result = await(user.insert(db, data));
res.json(result);
} catch (err) {
res.status(500).json(err);
}
}));
user.js
insert: async (function(db, data) {
// there's no need for this extra call but I'm including it
// to see example of deeper call stacks if this is answered
const idUser = await(this.insertData(db, data));
return {
idUser: idUser
}
}),
insertData: async(function(db, data) {
// if any of the following 2 fails I should be rolling back
const id = await(this.setId(db, idCustomer, data));
const idCustomer = await(this.setData(db, id, data));
return {
idCustomer: idCustomer
}
}),
// DB Functions (wrapped in Promises)
setId: function(db, data) {
return new Promise(function (resolve, reject) {
db.insert(data)
.into("ids")
.then((result) => resolve(result)
.catch((err) => reject(err));
});
},
setData: function(db, id, data) {
data.id = id;
return new Promise(function (resolve, reject) {
db.insert(data)
.into("customers")
.then((result) => resolve(result)
.catch((err) => reject(err));
});
}
Attempt to make it transactional
user.js
// Start transaction from this call
insert: async (function(db, data) {
const trx = await(knex.transaction());
const idCustomer = await(user.insertData(trx, data));
return {
idCustomer: idCustomer
}
}),
it seems that await(knex.transaction()) returns this error:
[TypeError: container is not a function]
I couldn't find a solid answer for this anywhere (with rollbacks and commits) so here's my solution.
First you need to "Promisify" the knex.transaction function. There are libraries for this, but for a quick example I did this:
const promisify = (fn) => new Promise((resolve, reject) => fn(resolve));
This example creates a blog post and a comment, and rolls back both if there's an error with either.
const trx = await promisify(db.transaction);
try {
const postId = await trx('blog_posts')
.insert({ title, body })
.returning('id'); // returns an array of ids
const commentId = await trx('comments')
.insert({ post_id: postId[0], message })
.returning('id');
await trx.commit();
} catch (e) {
await trx.rollback();
}
Here is a way to write transactions in async / await.
It is working fine for MySQL.
const trx = await db.transaction();
try {
const catIds = await trx('catalogues').insert({name: 'Old Books'});
const bookIds = await trx('books').insert({catId: catIds[0], title: 'Canterbury Tales' });
await trx.commit();
} catch (error) {
await trx.rollback(error);
}
Async/await is based around promises, so it looks like you'd just need to wrap all the knex methods to return "promise compatible" objects.
Here is a description on how you can convert arbitrary functions to work with promises, so they can work with async/await:
Trying to understand how promisification works with BlueBird
Essentially you want to do this:
var transaction = knex.transaction;
knex.transaction = function(callback){ return knex.transaction(callback); }
This is because "async/await requires the either a function with a single callback argument, or a promise", whereas knex.transaction looks like this:
function transaction(container, config) {
return client.transaction(container, config);
}
Alternatively, you can create a new async function and use it like this:
async function transaction() {
return new Promise(function(resolve, reject){
knex.transaction(function(error, result){
if (error) {
reject(error);
} else {
resolve(result);
}
});
});
}
// Start transaction from this call
insert: async (function(db, data) {
const trx = await(transaction());
const idCustomer = await(person.insertData(trx, authUser, data));
return {
idCustomer: idCustomer
}
})
This may be useful too: Knex Transaction with Promises
(Also note, I'm not familiar with knex's API, so not sure what the params are passed to knex.transaction, the above ones are just for example).
For those who come in 2019.
After I updated Knex to version 0.16.5. sf77's answer doesn't work anymore due to the change in Knex's transaction function:
transaction(container, config) {
const trx = this.client.transaction(container, config);
trx.userParams = this.userParams;
return trx;
}
Solution
Keep sf77's promisify function:
const promisify = (fn) => new Promise((resolve, reject) => fn(resolve));
Update trx
from
const trx = await promisify(db.transaction);
to
const trx = await promisify(db.transaction.bind(db));
I think I have found a more elegant solution to the problem.
Borrowing from the knex Transaction docs, I will contrast their promise-style with the async/await-style that worked for me.
Promise Style
var Promise = require('bluebird');
// Using trx as a transaction object:
knex.transaction(function(trx) {
var books = [
{title: 'Canterbury Tales'},
{title: 'Moby Dick'},
{title: 'Hamlet'}
];
knex.insert({name: 'Old Books'}, 'id')
.into('catalogues')
.transacting(trx)
.then(function(ids) {
return Promise.map(books, function(book) {
book.catalogue_id = ids[0];
// Some validation could take place here.
return knex.insert(book).into('books').transacting(trx);
});
})
.then(trx.commit)
.catch(trx.rollback);
})
.then(function(inserts) {
console.log(inserts.length + ' new books saved.');
})
.catch(function(error) {
// If we get here, that means that neither the 'Old Books' catalogues insert,
// nor any of the books inserts will have taken place.
console.error(error);
});
async/await style
var Promise = require('bluebird'); // import Promise.map()
// assuming knex.transaction() is being called within an async function
const inserts = await knex.transaction(async function(trx) {
var books = [
{title: 'Canterbury Tales'},
{title: 'Moby Dick'},
{title: 'Hamlet'}
];
const ids = await knex.insert({name: 'Old Books'}, 'id')
.into('catalogues')
.transacting(trx);
const inserts = await Promise.map(books, function(book) {
book.catalogue_id = ids[0];
// Some validation could take place here.
return knex.insert(book).into('books').transacting(trx);
});
})
await trx.commit(inserts); // whatever gets passed to trx.commit() is what the knex.transaction() promise resolves to.
})
The docs state:
Throwing an error directly from the transaction handler function automatically rolls back the transaction, same as returning a rejected promise.
It seems that the transaction callback function is expected to return either nothing or a Promise. Declaring the callback as an async function means that it returns a Promise.
One advantage of this style is that you don't have to call the rollback manually. Returning a rejected Promise will trigger the rollback automatically.
Make sure to pass any results you want to use elsewhere to the final trx.commit() call.
I have tested this pattern in my own work and it works as expected.
Adding to sf77's excellent answer, I implemented this pattern in TypeScript for adding a new user where you need to do the following in 1 transaction:
creating a user record in the USER table
creating a login record in the LOGIN table
public async addUser(user: User, hash: string): Promise<User> {
//transform knex transaction such that can be used with async-await
const promisify = (fn: any) => new Promise((resolve, reject) => fn(resolve));
const trx: knex.Transaction = <knex.Transaction> await promisify(db.transaction);
try {
let users: User [] = await trx
.insert({
name: user.name,
email: user.email,
joined: new Date()})
.into(config.DB_TABLE_USER)
.returning("*")
await trx
.insert({
email: user.email,
hash
}).into(config.DB_TABLE_LOGIN)
.returning("email")
await trx.commit();
return Promise.resolve(users[0]);
}
catch(error) {
await trx.rollback;
return Promise.reject("Error adding user: " + error)
}
}
Related
I am using nodejs and mysql2. I am storing all my queries inside a class. When I console.log the results I am able to view the data correctly, however when I use a return statment to return the data, I am getting undefined. I believe I need to use promises, but am uncertain how to do so correctly. Here is what I have currently which is returning undefined,
viewManagerChoices() {
const sql = `SELECT CONCAT(first_name, ' ', last_name) AS manager, id FROM employee WHERE manager_id IS NULL`;
db.query(sql, (err, rows) => {
if (err) throw err;
const managers = rows.map(manager => ({ name: manager.manager, value: manager.id }));
managers.push({ name: 'None', value: null });
return managers;
});
};
This is my attempt at using promises which is returning as Promise {<pending>},
viewManagers() {
return new Promise((resolve, reject) => {
const sql = `SELECT CONCAT(first_name, ' ', last_name) AS manager FROM employee WHERE manager_id IS NULL`;
db.query(sql,
(error, results) => {
if (error) {
console.log('error', error);
reject(error);
}
const managers = [];
for (let i = 0; i < results.length; i++) {
managers.push({ name: results[i].manager, value: i+1 });
}
managers.push({ name: "None", value: null });
resolve(managers);
}
)
})
}
My class is called Query and I am calling these methods by doing,
const query = new Query();
query.viewManagerChoices();
query.viewManagers();
Your implementation for viewManagers is correct however promises don't make calls synchronous.
Either you need to use then callback or await the result in async context.
const query = new Query();
query.viewManagers().then((managers) => {
// do stuff here
}).catch((error) => console.error(error.message));
or
async someFunc() {
const query = new Query();
try{
const managers = await query.viewManagers();
}catch(error){
console.error(error.message);
}
}
Once you use promise you cannot just get a returned value without async/await or then flag. Once it's a promise the flow continues as the original.
For example:
// This is promise too because it has async flag.
// You cannot make it sync if you use promise in it
async myFunc(){
const const query = new Query();
const managers = await query.viewManagers();
return managers;
}
// It actually returns Promise<...>
// Now if you want to use myFunc in another function
// You need to do it the same way again
async anotherFunc(){
const something = await myFunc();
return something; // Returns promise
}
You can read more about promises here
I am writing a Firebase Cloud Function in node.JS that reads user data from Firebase Firestore.
I am unable to push the token values into the token array and return all tokens at the end of the function. My code is as follows:
function getTokens(subscribers) {
return new Promise(async function (resolve, reject) {
const tokenArray = [];
subscribers.forEach(async (subscriber) => {
await firestore
.collection('users')
.doc(subscriber)
.get()
.then((user) => {
console.log("Getting user data for user: ", user);
const tokens = user.data().tokens;
if (tokens) {
tokens.forEach((token) => {
console.log("Adding token to token array"); // data is available here but does not push into tokenArray
tokenArray.push(token);
});
}
}).catch((error) => { console.error(error); reject(error); });
});
console.log("Token Array -- Final: ", tokenArray);
resolve(tokenArray);
});
};
The OP code can be corrected and made more concise as follows:
async function getTokens(subscribers) {
const getUser = subscriber => firestore.collection('users').doc(subscriber).get();
const promises = subscribers.map(getUser);
const users = await Promise.all(promises);
return users.map(user => user.data().tokens).flat()
}
Some notes:
decorate the function as async, since it includes an await
don't create any extra promise with Promise.new(), Firestore's get() returns a promise
collect get promises in an array with map, and run them with Promise.all()
mapping the resulting users' data.tokens produces an array of arrays. Flatten it and you're done.
a catch that only throws, is just like having no catch
You cannot use async-await in a forEach loop. Try mapping an array of promises and then using Promise.all() as shown below:
function getTokens(subscribers) {
return new Promise(async function (resolve, reject) {
const subscribersDocs = await Promise.all(
subscribers.map((subscriber) => {
return firestore.collection("users").doc(subscriber).get();
})
);
const tokenArray = subscribersDocs.reduce((acc, curr) => {
const tokens = curr.data().tokens;
if (tokens) {
acc = acc.concat(tokens);
}
return acc;
}, []);
console.log("Token Array -- Final: ", tokenArray);
resolve(tokenArray);
});
}
Also checkout: Using async/await with a forEach loop
I just got rid of the promise chaining as it was very confusing and went ahead with async await. I'm still not able to get the required results. How do I get the result of finalResult() from the below code. It keeps returning promise pending. I tried doing the following
let sampleData = await finalResult()
Here how do I get the data from sampleData? I also tried attaching a then call to finalResult which didn't work either. Any help pls? All I need is the channelData in the code below
app.get("/graph/getChannelEvents", (req, res) => {
var idToken = req.query.idToken;
var teamId = req.query.teamId;
var channelData = req.query.channelData;
var tenantId = process.env.TENANT_ID;
if (!idToken) {
res.status(500).send("Could not find id token");
return;
}
request(
`https://login.microsoftonline.com/${tenantId}/oauth2/v2.0/token`,
{
method: "POST",
headers: {
"Content-Type": "application/x-www-form-urlencoded",
},
form: {
grant_type: "urn:ietf:params:oauth:grant-type:jwt-bearer",
client_id: process.env.APP_ID,
client_secret: process.env.APP_Secret,
scope: "Calendars.Read",
requested_token_use: "on_behalf_of",
assertion: idToken,
},
},
async (error, response, body) => {
const accessToken = JSON.parse(body)["access_token"];
var client = MicrosoftGraph.Client.init({
defaultVersion: "v1.0",
debugLogging: true,
authProvider: (done) => {
done(null, accessToken);
},
});
async function finalResult() {
let cdata = await client
.api(`/groups/${teamId}/events`)
.header("Prefer", 'outlook.timezone="Pacific Standard Time"')
.select("subject,onlinemeeting,start,end")
.get();
let channelData = await cdata.value.map(async (org) => {
var channelId = url
.parse(decodeURIComponent(org.onlineMeeting.joinUrl))
.pathname.split("/")[3];
var sessionData = await client
.api(`/teams/${teamId}/channels`)
.filter(`startswith(id, '${channelId}')`)
.select("displayName")
.get();
let myData = await sessionData.value.map(async (u) => {
return {
sessionName: org.subject,
channelName: u.displayName,
channelId: channelId,
startDate: org.start.dateTime.split("T")[0],
endDate: org.end.dateTime.split("T")[0],
startTime: org.start.dateTime.split("T")[1],
endTime: org.end.dateTime.split("T")[1],
};
});
});
console.log(channelData);
}
finalResult();
}
);
});
A good solution for this would be to make it asynchronously with a promise wrapping the whole shown code and returning that promise with a function.
After that you would want to make an array of promises to be able to call them all at once and wait for all the results before continuing. you use Promise.all() to call all the promises in the array and use it as you would use a single promise with a .then(). The received value from that then() would be an array containing all of the results from the promiseArray's promises.
var getStuff = () => {
//First and big promise starts here
return new Promise( (resolve, reject) => {
client.api(`/groups/${teamId}/events`)
.header("Prefer", 'outlook.timezone="Pacific Standard Time"')
.select("subject,onlinemeeting,start,end")
.get()
.then((result) => {
// Create an array for your promises
var promisesArray = [];
result.value.map((org) => {
// Fill your array with every promise you need, each one containing just one call to your api
promisesArray.push(new Promise((arrayResolve, arrayReject) => {
console.log("top: " + org.subject);
var channelId = url
.parse(decodeURIComponent(org.onlineMeeting.joinUrl))
.pathname.split("/")[3];
client
.api(`/teams/${teamId}/channels`)
.filter(`startswith(id, '${channelId}')`)
.select("displayName")
.get()
.then((result) => {
const cdata = result.value.map((u) => {
console.log("down " + org.subject);
return {
sessionName: org.subject,
channelName: u.displayName,
channelId: channelId,
startDate: org.start.dateTime.split("T")[0],
endDate: org.end.dateTime.split("T")[0],
startTime: org.start.dateTime.split("T")[1],
endTime: org.end.dateTime.split("T")[1],
};
});
})
.then((result) => {
// Resolve individual promises
arrayResolve(result);
}).catch((err) => {
reject(err);
})
}));
})
// This executes all the promises at the same time
Promise.all(promisesArray).then( (finalResult) => {
// Resolve big and initial promise
resolve(finalResult);
}).catch((err) => {
reject(err);
})
})
})
}
(NOTE: I just wrapped your existing code in a function and completed the braces and parenthesis needed. Then I added the promises that would make this work as intended but some code needs to be added to make it work, as i noticed you called some variables that weren't defined inside your shown code)
After this, you could just call this function and use a '.then()' to receive the value you want.
Hope this helps!
I am building a function in Firebase Cloud Functions, which can utilize Node.js modules.
I am still new to the use of .then() and I'm struggling to figure out a way to chain my 3 functions webhookSend(), emailSendgrid(), and removeSubmissionProcessor() that happen right after the 'count' is incremented (the if statement that checks temp_shouldSendWebhook). The whole idea of returning promises still confuses me a little, especially when it it involves external libraries.
const functions = require('firebase-functions');
const admin = require('firebase-admin');
admin.initializeApp();
const request = require('request');
const firebaseConfig = JSON.parse(process.env.FIREBASE_CONFIG);
const SENDGRID_API_KEY = firebaseConfig.sendgrid.key;
const sgMail = require('#sendgrid/mail');
sgMail.setApiKey(SENDGRID_API_KEY);
exports.submissionProcess = functions.database.ref('/submissions/processor/{submissionId}').onWrite((change, context) => {
var temp_metaSubmissionCount = 0; // omitted part of function correctly sets the count
var temp_shouldSendWebhook = true; // omitted part of function correctly sets the boolean
return admin.database().ref('/submissions/saved/'+'testuser'+'/'+'meta').child('count')
.set(temp_metaSubmissionCount + 1)
.then(() => {
// here is where im stuck
if (temp_shouldSendWebhook) {
webhookSend();
emailSendgrid();
removeSubmissionProcessor();
} else {
emailSendgrid();
removeSubmissionProcessor();
}
})
.catch(() => {
console.error("Error updating count")
});
});
function emailSendgrid() {
const user = 'test#example.com'
const name = 'Test name'
const msg = {
to: user,
from: 'hello#angularfirebase.com',
subject: 'New Follower',
// text: `Hey ${toName}. You have a new follower!!! `,
// html: `<strong>Hey ${toName}. You have a new follower!!!</strong>`,
// custom templates
templateId: 'your-template-id-1234',
substitutionWrappers: ['{{', '}}'],
substitutions: {
name: name
// and other custom properties here
}
};
return sgMail.send(msg)
}
function webhookSend() {
request.post(
{
url: 'URLHERE',
form: {test: "value"}
},
function (err, httpResponse, body) {
console.log('REQUEST RESPONSE', err, body);
}
);
}
function removeSubmissionProcessor() {
admin.database().ref('/submissions/processor').child('submissionkey').remove();
}
I want to be able to construct the 3 functions to be called one after another such that they will all execute.
In order to chain these functions, they each need to return a promise. When they do, you can call them sequentially like this:
return webhookSend()
.then(() => {
return emailSendgrid();
})
.then(() => {
return removeSubmissionProcessor();
});
Or in parallel like this:
return Promise.all([webhookSend, emailSendgrid, removeSubmissionProcessor]);
Now, to make your functions return promises:
emailSendgrid: It looks like this returns a promise (assuming sgMail.send(msg) returns a promise), so you shouldn't need to change this.
removeSubmissionProcessor: This calls a function that returns a promise, but doesn't return that promise. In other words it fires off an async call (admin.database....remove()) but doesn't wait for the response. If you add return before that call, this should work.
webhookSend calls a function that takes a callback, so you'll either need to use fetch (which is promise-based) instead of request, or you'll need to convert it to return a promise in order to chain it:
function webhookSend() {
return new Promise((resolve, reject) => {
request.post(
{
url: 'URLHERE',
form: {test: "value"}
},
function (err, httpResponse, body) {
console.log('REQUEST RESPONSE', err, body);
if (err) {
reject(err);
} else {
resolve(body);
}
}
);
});
}
Use async functions and then you can use .then() or await before every function calls
for reference read this
I'm trying to have a loop with some db calls, and once their all done ill send the result. - Using a promise, but if i have my promise after the callback it dosent work.
let notuser = [];
let promise = new Promise((resolve, reject) => {
users.forEach((x) => {
User.find({
/* query here */
}, function(err, results) {
if(err) throw err
if(results.length) {
notuser.push(x);
/* resolve(notuser) works here - but were not done yet*/
}
})
});
resolve(notuser); /*not giving me the array */
}).then((notuser) => {
return res.json(notuser)
})
how can i handle this ?
Below is a function called findManyUsers which does what you're looking for. Mongo find will return a promise to you, so just collect those promises in a loop and run them together with Promise.all(). So you can see it in action, I've added a mock User class with a promise-returning find method...
// User class pretends to be the mongo user. The find() method
// returns a promise to 'find" a user with a given id
class User {
static find(id) {
return new Promise(r => {
setTimeout(() => r({ id: `user-${id}` }), 500);
});
}
}
// return a promise to find all of the users with the given ids
async function findManyUsers(ids) {
let promises = ids.map(id => User.find(id));
return Promise.all(promises);
}
findManyUsers(['A', 'B', 'C']).then(result => console.log(result));
I suggest you take a look at async it's a great library for this sort of things and more, I really think you should get used to implement it.
I would solve your problem using the following
const async = require('async')
let notuser = [];
async.forEach(users, (user, callback)=>{
User.find({}, (err, results) => {
if (err) callback(err)
if(results.length) {
notUser.push(x)
callback(null)
}
})
}, (err) => {
err ? throw err : return(notuser)
})
However, if you don't want to use a 3rd party library, you are better off using promise.all and await for it to finish.
EDIT: Remember to install async using npm or yarn something similar to yarn add async -- npm install async
I used #danh solution for the basis of fixing in my scenario (so credit goes there), but thought my code may be relevant to someone else, looking to use standard mongoose without async. I want to gets a summary of how many reports for a certain status and return the last 5 for each, combined into one response.
const { Report } = require('../../models/report');
const Workspace = require('../../models/workspace');
// GET request to return page of items from users report
module.exports = (req, res, next) => {
const workspaceId = req.params.workspaceId || req.workspaceId;
let summary = [];
// returns a mongoose like promise
function addStatusSummary(status) {
let totalItems;
let $regex = `^${status}$`;
let query = {
$and: [{ workspace: workspaceId }, { status: { $regex, $options: 'i' } }],
};
return Report.find(query)
.countDocuments()
.then((numberOfItems) => {
totalItems = numberOfItems;
return Report.find(query)
.sort({ updatedAt: -1 })
.skip(0)
.limit(5);
})
.then((reports) => {
const items = reports.map((r) => r.displayForMember());
summary.push({
status,
items,
totalItems,
});
})
.catch((err) => {
if (!err.statusCode) {
err.statusCode = 500;
}
next(err);
});
}
Workspace.findById(workspaceId)
.then((workspace) => {
let promises = workspace.custom.statusList.map((status) =>
addStatusSummary(status)
);
return Promise.all(promises);
})
.then(() => {
res.status(200).json({
summary,
});
})
.catch((err) => {
if (!err.statusCode) {
err.statusCode = 500;
}
next(err);
});
};