I am new to async programming in node js and want to know how to translate my former (sync) programming to readable async programming in node js. Especially this example:
function getDefaultValue1() {
return getDefaultValue1FromDB();
};
function getDefaultValue2() {
return getDefaultValue2FromDB();
};
function doSomething(var optionalValue1, var optionalValue2){
try {
if(optionalValue1===null){
optionalValue1 = getDefaultValue1();
}
if(optionalValue2===null){
optionalValue2 = getDefaultValue2();
}
var resultList = [];
for(var i = 0; i < optionalValue2; i++) {
resultList.push(functionWithDbCall(optionalValue1, i));
}
return doSomeCalculations(resultList);
}catch(var e){
//log exception
return null
}
};
Note that all DB-Calls would be async in node js (getDefaultValue1FromDB(var callback), functionWithDbCall(var optionalValue1, var i, var callback)).
My thoughts are:
Adding callback-parameter to getDefaultValue => requires the same code in callback and after if. For loop exits before callbacks are called and callbacks need to handle how often they are called and to check to push the items in correct order. Changing a function to using an async method requires changing code everywhere it is called like changing getDefaultValue from returning a const int to a DB-call.
I am stuck here and all I found where advertising for some libraries or unreadable code worse then asm or regex (write once, read/touch never again). Did I miss something basic in my node js learning material or is this a design failure of the language?
So I hope somebody somebody can show me how to write this simple function readable with async node js.
I'd suggest using Bluebird:
var Promise = require('bluebird');
/**
* Lets assume getDefaultValue1FromDB() passes a callback fn;
* #return {Promise}
*/
function getDefaultValue1() {
return new Promise(function(resolve, reject){
return getDefaultValue1FromDB(function(err, value){
if (err) return reject(err);
return resolve(value);
});
});
}
/**
* Lets assume getDefaultValue2FromDB() passes a callback fn;
* #return {Promise}
*/
function getDefaultValue2() {
return new Promise(function(resolve, reject){
return getDefaultValue2FromDB(function(err, value){
if (err) return reject(err);
return resolve(value);
});
});
}
/**
* #param {Promise|int} optionalValue1
* #param {Prommise|int} optionalValue2
* #return {Promise}
*/
function doSomething(optionalValue1, optionalValue2){
if(optionalValue1 === null)
optionalValue1 = getDefaultValue1;
if(optionalValue2 === null)
optionalValue2 = getDefaultValue2;
return Promise
.all([optionalValue1, optionalValue2])
.spread(function(val1, val2){
var resultList = [];
for(var i = 0; i < val2; i++) {
resultList.push(functionWithDbCall(val1, i));
}
return doSomeCalculationsAsync(resultList);
}).catch(function(err){
//Log any errors here.
});
}
/**
* Calls the db and returns a promise with the result.
* #return {Promise} [description]
*/
function functionWithDbCall(val, i) {
Return new Promise();
}
/**
* #param {[Promise]} list Array of promises.
* #return {Promise}
*/
function doSomeCalculationsAsync(list) {
return Promise.all(list).then(doSomeCalculations);
}
Note that doSomething will return a promise, so you'll have to add a .then() to handle the results. Also asume that functionWithDbCall returns a Promise as well, you can then put all those together in a Promise.all() and then do the calculations.
You might want to use async/await from the ES7 proposal - a future version of JavaScript. See this answer on how to migrate from callbacks to promises to async/await.
To actually run future code today, you will need to compile it down to currently supported features with Babel. Plus it can do a lot more than async/await.
Related
I would like to learn how to return a value from one function and use it in another one.
I can show the value in a div or get it in an alert window but I need to use the value in the other function.
<script type="text/javascript">
SP.SOD.executeOrDelayUntilScriptLoaded(testCase,"sp.js");
function showUserInfo(){
var clientContext = SP.ClientContext.get_current();
var user = clientContext.get_web().get_currentUser();
clientContext.load(user);
clientContext.executeQueryAsync(function(){
return user.get_loginName();
},function(sender,args){alert(args.get_message());})
}
function testCase() {
var test = showUserInfo();
alert(test);
}
</script>
If you need to support Internet Explorer
To support Internet Explorer, you should use a callback function similar to what #Amos_MSFT suggested. Below you will find my solution, which is quite similar to the solution posted by #Amos_MSFW but with a few differences as well as comments.
// Execute the testCase() function only after sp.js has loaded and is ready.
SP.SOD.executeFunc('sp.js', 'SP.ClientContext', function() {
textCase();
});
function testCase() {
getLoginName(function(error, success) {
if (error) {
alert(error);
} else {
/*
* The line below is not really necessary, as the result of the
* getLoginName() function is already available in the success
* variable. You can already pass it to other functions, like
* this: alert(success);
*/
var loginName = success;
alert(loginName);
}
});
}
function getLoginName(callback) {
var clientContext = SP.ClientContext.get_current();
var user = clientContext.get_web().get_currentUser();
clientContext.load(user);
clientContext.executeQueryAsync(
function() {
/*
* The first argument of the callback() function is used for the
* error message, so we need to use null as the first argument
* when we want to return the login name as the success message.
*/
callback(null, user.get_loginName());
},
function(_sender, args) {
callback(args.get_message());
}
);
}
If you do not need to support Internet Explorer
I suggest you use a Promise if you don’t need to support Internet Explorer. A Promise is a special type of object that represents an operation that is yet to complete, and they make working with asynchronous operations easy and almost fun. I am not fluent enough to explain them in detail, so I advice you to read the article linked above if you are interested. What I will tell you, though, is that we can use a Promise to make sure the testCase() function stops and waits for the showUserInfo() function to finish, as in waits for the login name to be available.
I have not had the chance to test the code below, but it should work. If not, please let me know. I have also added a few comments in case you are not already familiar with Promises.
const showUserInfo = () => {
return new Promise((resolve, reject) => {
const clientContext = SP.ClientContext.get_current();
const currentUser = clientContext.get_web().get_currentUser();
clientContext.load(currentUser);
clientContext.executeQueryAsync(
() => {
/*
* We resolve the Promise when the query has executed successfully.
* Resolving a Promise means marking it as fullfilled, or complete, and
* returning the current user's login name.
*/
resolve(currentUser.get_loginName());
},
(_sender, args) => {
/*
* If something goes wrong, we reject the Promise. Rejecting means
* marking it as failed, while still returning something. In this
* case, we return the error message.
*/
reject(args.get_message());
}
);
});
}
const testCase = async () => {
/*
* We use the await keyword to halt the code and wait for the showUserInfo()
* function to finish. What really happens is that we wait for the Promise
* in the showUserInfo() function to be marked as settled, whether it is
* fullfilled or rejected, and then assign the result to the test constant.
*/
const test = await showUserInfo();
alert(test);
}
SP.SOD.executeOrDelayUntilScriptLoaded(testCase, 'sp.js');
Function executeQueryAsync is asynchronous. You can use the callback function to get the return value.
Sample code for your reference:
<script type="text/javascript">
SP.SOD.executeFunc('sp.js', 'SP.ClientContext', function(){
var testCase=function(callback){
var clientContext = SP.ClientContext.get_current();
var user = clientContext.get_web().get_currentUser();
clientContext.load(user);
clientContext.executeQueryAsync(function(){
callback(null,user.get_loginName()) ;
},function(sender,args){alert(args.get_message());})
}
testCase(function(error,name){
if (error) {
console.error(error);
return;
}
console.log(name);
})
});
</script>
Inside a promise, I need to call and process an indeterminate number of asynch API responses after individually calling them either inside another promise, or after said promise, but before another so the order of execution is respected.
var promiseA = function() {
return new Promise(function(resolve, reject) {
// 1. Establish objects needed from one API endpoint
// 2. Call API endpoint for each object and parse
// 3. Only then continue to next promise
}
}
var finalPromise = function() {
return new Promise(function(resolve, reject) {
//
}
}
promiseA()
.then(finalPromise)
So inside promiseA, I find out how many objects I'll need to poll individually from an API. Each request is of course asynchronous. I need to make these calls and process the response before the final promise is called.
I am struggling to determine a pattern for this with promises, where I can dynamically create these promises and only allow the final promise to execute after the indeterminate and asynchronous have executed and processed. I've worked with other languages where this is possible, but I'm struggling to see it here with Promises.
Any help is appreciated.
I have changed the answer to incorporate the comments below. Since, you mentioned ES6 promises I shall stick to that. There are two basic types of callbacks that we might care about.
DOM load or other one time event callbacks (window.onload and so on)
Async method callback (AJAX call, setTimout and so on)
Since,
1.DOM load or other one time event
var p = new Promise(function(res, rej) {
window.onload = res();
};
2.Plain callback: these are callbacks that don't conform to a convention. e.g. setTimeout
var p = new Promise(function(res, rej){
setTimeout(function() {
//your business/view logic
success? res():rej(); //if successful resolve else reject
}, 2000);
});
In each of the above case the promise (var p) can be wrapped to be returned by a function.
var myAsyncMethod = function () {
var p = new ... // as mentioned in 1 or 2
return p;
}
Then the usage:
myAsyncMethod()
.then(function(){/* success-handler */})
.catch(function(/* failure-handler */));
Specific to your question you may have many such methods:
function baseAJAXCall (url) {
new Promise(functoin(rej, res) {
$.get(url, function(err, data){
if(err) {
rej();
}
else {
resolve(data);
}
});
}
};
function callAPIEndpoint(url) {
return baseAJAXCall(url);
}
function finalPromiseHandler () {
//your final business/view logic
}
//USAGE
callAPIEndpoint('/my-first-call')
.then(function(data){
var promiseArray = data.map(function(item){
return baseAJAXCall(item.url);
});
return Promise.all(promiseArray);
})
.then(finalPromiseHandler)
.catch(function(){
console.log('.error-message.');
});
Ref:
How do I convert an existing callback API to promises?.
http://www.datchley.name/es6-promises/
Links from comments below.
---OLD ANSWER: PLEASE OVERLOOK---
I am familiar with this library : https://github.com/kriskowal/q. And, you can do this using using the q.all and q.allSettled constructs. May be that is what you are looking for.
Normally, the pattern is to create a function that returns a promise.
function someAsyncFuncName1(url) {
var def = q.defer();
//async function
$.get(url, function(err, data){ //suppose
if(err){
def.reject();
}
else {
def.resolve(data); //pass the data to the .then() handler.
}
});
return def.promise;
}
function someAsyncFuncName2() {
var def = q.defer();
//async function
setTimeout(function(){ //suppose
//do something
if(good) {
def.resolve();
} else {
def.reject();
}
}, 1000); //arbitrary timeout of 1 second
return def.promise;
}
USAGE:
q.all([someAsyncFuncName1('/api-1'), someAsyncFuncName2()])
.then(function() {
//final handler
});
On a similar line of thought one can use q.allSettled() if you want to wait for all promises to return.
Hope this helps.
---EOF OLD ANSWER---
First of all, if async functions used in PromiseA don't return promises, you need to promisify them. You can do that with Promise constructor, but it's much better to use libraries, such as bluebird with their promisify methods.
Let's imagine, that we have two functions getUserIdsAsync and getUserAsync. The first on returns a list of user ids, getUserAsync returns an user data by userId. And you need to get a list of users by their ids. The code of PromiseA could look so:
var promiseA = function() {
return getUserIdsAsync()
.then(userIds => {
let ops = users.map(uid => getUserAsync(uid));
return Promise.all(ops);
});
}
The following snippet shows a solution without using any external library like bluebird. It follows the code snippet in your question (which seems to be more complicate than needed).
You have to collect all api promisses in an array. Then you can call Promise.all() to get a Promise for the end of all api promisses. Then you can do some final stuff, like parsing the result of each promise and continue afterwards.
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
var apiEndpoint = function (name) {
return new Promise( (resolve, reject) => {
setTimeout(() => resolve('API ' + name + ' job done'), 1000);
});
}
var promiseA = function() {
return new Promise( (resolve, reject) => {
const promisses = [];
for (var i=1; i < getRandomInt(3,6); i++) {
// 1. Establish objects needed from one API endpoint
promisses.push(apiEndpoint('This is number ' + i));
}
Promise.all(promisses).then( results => {
// do final stuff
for (const s of results) {
// 2. Call API endpoint for each object and parse
console.log(s);
}
// continue ...
// 3. Only then continue to next promise
resolve('now it is finished');
}).catch( err => reject(err) );
});
}
var finalPromise = function() {
return new Promise( (resolve, reject) => {
console.log('finalPromise');
resolve();
});
}
promiseA()
.then( () => finalPromise())
.catch(err => console.log(err) );
Please hold in mind that this solution is not easy to read. Using external libraries or reducing promisses can improve readability. Maybe you should take a look to the async/await pattern to get a much more better (readable) solution.
Here is a solution with async/await:
function getRandomInt(min, max) {
return Math.floor(Math.random() * (max - min + 1)) + min;
}
const apiEndpoint = function (name) {
return new Promise( (resolve, reject) => {
setTimeout(() => resolve('API ' + name + ' job done'), 1000);
});
}
async function promiseParallel () {
const promisses = [];
for (let i = 1; i < getRandomInt(3,6); i++) {
promisses.push(apiEndpoint('This is number ' + i));
}
for (const p of promisses) {
const x = await p;
console.log(x);
}
return ('everything is done');
}
promiseParallel().then( result => {
console.log(result);
}).catch( err => console.log(err) );
If you want call the promisses sequentially you can replace with:
async function promiseSequ () {
for (let i = 1; i < getRandomInt(3,6); i++) {
const x = await apiEndpoint('This is number ' + i);
console.log(x);
}
return ('everything is done');
}
I'm trying to upload a file to a remote server, get the path for that file on that server, and then save it in a record to my own API. Over half the time, the code crashes because returnedNewFile is undefined (though I've checked console.log(newFile) in the previous .then() function and it's always a promise object. What could be causing this behavior? What could be causing returnedNewFile to sometimes be null/undefined? I'm using Django REST + django-rest-json-api + ember.js 2.6, and this code is in a component's javascript file. Thank you in advance!
store.findRecord('file', folderid).then(function(folder) {
var file = fileList.pop();
var newFile = fm.uploadFile(folder, file.name, file);
return newFile;
}).then(function(returnedNewFile) {
var name = returnedNewFile.get('name');
var path = returnedNewFile.get('path');
var doc = store.createRecord('document', {
name: name,
path: path,
});
console.log("doc: ", doc);
doc.save();
return doc;
}).then(function(doc) {
console.log("hi number three");
var grant = store.createRecord('grant', {
department: department,
number: number,
document: doc,
});
grant.save();
return grant;
}).then(null, function(error) {
console.log("Oops: " + error.message)
});
EDIT: Here's the code for uploadFile() (waterbutler is the name of the file server being used on the external server (link)):
uploadFile(folder, name, contents, options = {}) {
let url = folder.get('links').upload;
options.data = contents;
if (!options.query) {
options.query = {};
}
options.query.name = name;
options.query.kind = 'file';
let p = this._waterbutlerRequest('PUT', url, options);
return p.then(() => this._getNewFileInfo(folder, name));
}
And here's the code for _getNewFileInfo:
/**
* Get the `file` model for a newly created file.
*
* #method _getNewFileInfo
* #private
* #param {file} parentFolder Model for the new file's parent folder.
* #param {String} name Name of the new file.
* #return {Promise} Promise that resolves to the new file's model or
* rejects with an error message.
*/
_getNewFileInfo(parentFolder, name) {
let p = this._reloadModel(parentFolder.get('files'));
return p.then((files) => files.findBy('name', name));
}
Over half the time, the code crashes because returnedNewFile is
undefined (though I've checked console.log(newFile) in the previous
.then() function and it's always a promise object. What could be
causing this behavior?
Promises will resolve other Promise when calling .then(). So while you could return a string from promise 1's function, and you would get that same string in its .then() call, the same is not true if you returned another Promise inside its function. It will instead resolve that second promise to a concrete value before giving a .then result.
So if you called newFile.then(function(newFileResult) { ... then newFileResult would be the same as what you're now getting in the above code as the returnedNewFile argument. Why this library's promises sometimes come back with an undefined value is beyond me.
EDIT: Did not quite understand the comment, so an example. I have a function called getText(str) that asynchronously retrieves that string.
response1 = getText("abc").then(ignoredStr => {
return "def";
});
response2 = getText("abc").then(ignoredStr => {
return getText("def");
});
response1.then(function(str1) {
// "def"
});
response2.then(function(str2) {
// "def"
});
Under a certain understanding of promises, str2 would be a Promise instead of a string; but that's not the case. It sees it got back a Promise, not a "value", and waits for that before chaining the .then() call. Even if getText("def") came back in 3 seconds, it would wait 3 more seconds for the second call before giving us a .then() result.
How do I write a method that limits Q promise concurrency?
For instance, I have a method spawnProcess. It returns a Q promise.
I want no more than 5 process spawned at a time, but transparently to the calling code.
What I need to implement is a function with signature
function limitConcurrency(promiseFactory, limit)
that I can call like
spawnProcess = limitConcurrency(spawnProcess, 5);
// use spawnProcess as usual
I already started working on my version, but I wonder if anyone has a concise implementation that I can check against.
I have a library that does this for you https://github.com/ForbesLindesay/throat
You can use it via browserify or download the standalone build from brcdn (https://www.brcdn.org/?module=throat&version=latest) and add it as a script tag.
Then (assuming the Promise constructor is polyfilled or implemented in your environment) you can do:
//remove this line if using standalone build
var throat = require('throat');
function limitConcurrency(promiseFactory, limit) {
var fn = throat(promiseFactory, limit);
return function () {
return Q(fn.apply(this, arguments));
}
}
You could just call throat(promiseFactory, limit) directly but that would return a promise promise rather than a Q promise.
I also really like using it with array.map.
// only allow 3 parallel downloads
var downloadedItems = Q.all(items.map(throat(download, 3)));
This seems to be working for me.
I'm not sure if I could simplify it. The recursion in scheduleNextJob is necessary so the running < limit and limit++ always execute in the same tick.
Also available as a gist.
'use strict';
var Q = require('q');
/**
* Constructs a function that proxies to promiseFactory
* limiting the count of promises that can run simultaneously.
* #param promiseFactory function that returns promises.
* #param limit how many promises are allowed to be running at the same time.
* #returns function that returns a promise that eventually proxies to promiseFactory.
*/
function limitConcurrency(promiseFactory, limit) {
var running = 0,
semaphore;
function scheduleNextJob() {
if (running < limit) {
running++;
return Q();
}
if (!semaphore) {
semaphore = Q.defer();
}
return semaphore.promise
.finally(scheduleNextJob);
}
function processScheduledJobs() {
running--;
if (semaphore && running < limit) {
semaphore.resolve();
semaphore = null;
}
}
return function () {
var args = arguments;
function runJob() {
return promiseFactory.apply(this, args);
}
return scheduleNextJob()
.then(runJob)
.finally(processScheduledJobs);
};
}
module.exports = {
limitConcurrency: limitConcurrency
}
The Deferred promise implementation has gate function which works exactly that way:
spawnProcess = deferred.gate(spawnProcess, 5);
I wrote a little library to do this: https://github.com/suprememoocow/qlimit
It's extremely easy to use and is specifically designed to work with Q promises:
var qlimit = require('qlimit');
var limit = qlimit(2); // 2 being the maximum concurrency
// Using the same example as above
return Q.all(items.map(limit(function(item, index, collection) {
return performOperationOnItem(item);
}));
It can also be used to limit concurrency to a specific resource, like this:
var qlimit = require('qlimit');
var limit = qlimit(2); // 2 being the maximum concurrency
var fetchSomethingFromEasilyOverwhelmedBackendServer = limit(function(id) {
// Emulating the backend service
return Q.delay(1000)
.thenResolve({ hello: 'world' });
});
I think this is a really stupid question but I'm having a hard time wrapping my head around promises.
I'm using Q (for nodejs) to sync up a couple of async functions.
This works like a charm.
var first = function () {
var d = Q.defer();
fs.readdir(path,function(err,files){
if(err) console.log(err);
d.resolve(files);
});
return d.promise;
};
var second = function (files) {
var list = new Array;
files.forEach(function(value, index){
var d = Q.defer();
console.log('looking for item in db', value);
db.query(
'SELECT * FROM test WHERE local_name =? ', [value],{
local_name : String,
},
function(rows) {
if (typeof rows !== 'undefined' && rows.length > 0){
console.log('found item!', rows[0].local_name);
d.resolve(rows[0]);
} else {
var itemRequest = value;
getItemData(itemRequest);
}
}
);
list.push(d.promise);
});
return Q.all(list);
};
first()
.then(second)
.done(function(list){
res.send(list);
});
The problem I have is with this little function:
getItemData(itemRequest)
This function is filled with several of callbacks. The promise chain runs through the function just fine but ignores all the callbacks I use ( eg several XHR calls I make in the function).
A simplified version of the function looks like this (just to give you an idea):
function getItemData(itemRequest){
helper.xhrCall("call", function(response) {
var requestResponse = JSON.parse(response)
, requestInitialDetails = requestResponse.results[0];
downloadCache(requestInitialDetails,function(image) {
image = localImageDir+requestInitialDetails.image;
helper.xhrCall("call2", function(response) {
writeData(item,image,type, function(){
loadData(item);
});
});
} else {
writeData(item,image,type, function(){
loadData(item);
});
}
});
});
The xhr function I use looks like this:
xhrCall: function (url,callback) {
var request = require("request")
, colors = require('colors');
request({
url: url,
headers: {"Accept": "application/json"},
method: "GET"
}, function (error, response, body) {
if(!error){
callback(body);
}else{
console.log('Helper: XHR Error',error .red);
}
});
}
So my questions:
Can I leave the function unaltered and use the callbacks that are in place ánd the promise chain?
Or do I have to rewrite the function to use promises for the XHR?
And if so, How can I best write my promise chain? Should I reject the initial promise in the forEach?
Again, sorry if this is a really stupid question but I don't know what the right course of action is here.
Thanks!
[EDIT] Q.nfcall, I don't get it
So I've been looking into Q.nfcall which allows me to use node callbacks. Bu I just don't understand exacly how this works.
Could someone give a simple example how I would go about using it for a function with several async xhr calls?
I tried this but as you can see I don't really understand what I'm doing:
var second = Q.nfcall(second);
function second (files) {
[EDIT 2]
This is the final funcction in my getitemdata function callback chain. This function basically does the same as the function 'second' but I push the result directly and then return the promise. This works as stated, but without all the additional callback data, because it does not wait for the callbacks to return with any data.
function loadData(item) {
var d = Q.defer();
db.query(
'SELECT * FROM test WHERE local_name =? ', [item],{
local_name : String,
},
function(rows) {
if (typeof rows !== 'undefined' && rows.length > 0){
list.push(d.promise);
}
}
);
});
return Q.all(list);
};
Your answer is not really clear after your second edit.
First, on your orignal question, your getItemData has no influence on the promise chain.
You could change you the function's call signature and pass your deferred promise like so.
getItemData(itemRequest, d)
and pass this deferred promises all the way to your xhrCall and resolve there.
I would re-write your whole implementation and make sure all your functions return promises instead.
Many consider deferred promises as an anti-pattern. So I use use the Promise API defined in harmony (the next javascript)
After said that, I would re-implement your original code like so (I've not tested)
var Promise = Promise || require('es6-promise').Promise // a polyfill
;
function errHandler (err){
throw err
}
function makeQuery () {
var queryStr = 'SELECT * FROM test WHERE local_name =? '
, queryOpt = {local_name: String}
;
console.log('looking for item in db', value)
return new Promise(function(resolve, reject){
db.query(queryStr, [value], queryOpt, function(rows) {
if (typeof rows !== 'undefined' && rows.length > 0){
console.log('found item!', rows[0].local_name);
resolve(rows[0]);
} else {
// note that it returns a promise now.
getItemData(value).then(resolve).catch(errHandler)
}
})
})
}
function first () {
return new Promise(function(resolve, reject){
fs.readdir(path, function(err, files){
if (err) return reject(err)
resolve(files)
})
})
}
function second (files) {
return Promise.all(files.map(function(value){
return makeQuery(value)
});
}
first()
.then(second)
.then(res.send)
.catch(errHandler)
Note that there is no done method on the Promise API.
One down side of the new Promise API is error handling. Take a look at bluebird.
It is a robust promise library which is compatible with the new promise API and has many of the Q helper functions.
As far as I can tell, you need to return a promise from getItemData. Use Q.defer() as you do in second(), and resolve it when the callbacks complete with the data. You can then push that into list.
To save code, you can use Q.nfcall to immediately call a node-style-callback function, and return a promise instead. See the example in the API docs: https://github.com/kriskowal/q/wiki/API-Reference#qnfcallfunc-args