We Keep Coding

Javascript consecutive promises not being fired

I am using PhantomJS to login a website using node.js. After that I want to make an HTTP request to obtain a json with data. My code is as follows:
phantom.create()
.then(function (ph) {
this.ph = ph;
return ph.createPage();
})
.then(function (page) {
this.page = page;
return page.open('https://example.com');
})
.then(function (status) {
if ( status === "success" ) {
return this.page.evaluate(function() {
document.querySelector("input[name='j_username']").value = 'example#example.net';
document.querySelector("input[name='j_password']").value = 'example';
console.log('Submitting logging...');
document.querySelector("input[name='submit']").click();
});
} else {
return null;
}
})
.then(function() {
console.log("Logged in!");
this.page.property('onResourceRequested', function(requestData, networkRequest) {
//This event is fired 40 times, only one is interesting to me the one that has the text maps.json in it
if (requestData.url.indexOf('maps.json') !== -1) {
return {headers: requestData.headers, url: requestData.url};
}
});
});
.then(function (data) {
//This then block is only fired once, with the first call of the first url in previous then block and data is null
if (data) {
// This code block is never fired because this then is only called once with data=null
console.log(data);
request.get({
uri: data.url,
headers: data.headers
}, function(err, res, body){
if (!err) {
callback(null, res.headers);
} else {
console.log("Error getting data from URL: "+ err);
callback(true, null);
}
});
}
});
There must be something wrong with this subsequent promises because the this.page.property('onResourceRequested'... function in the penultimate then block is fired like 40 times (one per each url called inside the website after login) but the last then is only fired once (when first url is requested).
I want to obtain the data from one concrete url (the one that contains maps.json in the url) which is call number 32.
What am I doing wrong? How can I fire the last then only when my call is done?
EDIT:
Following #charlieetfl's advice I canged the code to the following, but still not working...:
phantom.create()
.then(function (ph) {
this.ph = ph;
return ph.createPage();
})
.then(function (page) {
this.page = page;
return page.open('https://example.com');
})
.then(function (status) {
if ( status === "success" ) {
return this.page.evaluate(function() {
document.querySelector("input[name='j_username']").value = 'example#example.net';
document.querySelector("input[name='j_password']").value = 'example';
console.log('Submitting logging...');
document.querySelector("input[name='submit']").click();
});
} else {
return null;
}
})
.then(function() {
console.log("Logged in!");
return new Promise(function(resolve, reject) {
this.page.property('onResourceRequested', function(requestData, networkRequest) {
//This event is fired 40 times, only one is interesting to me the one that has the text maps.json in it
if (requestData.url.indexOf('maps.json') !== -1) {
resolve({headers: requestData.headers, url: requestData.url});
}
});
});
});
.then(function (data) {
//This then block is only fired once, with the first call of the first url in previous then block and data is null
if (data) {
// This code block is never fired because this then is only called once with data=null
console.log(data);
request.get({
uri: data.url,
headers: data.headers
}, function(err, res, body){
if (!err) {
callback(null, res.headers);
} else {
console.log("Error getting data from URL: "+ err);
callback(true, null);
}
});
}
});
EDIT: 21/07/2017 10:52 UTC.
I changed the code to this new one:
phantom.create().then(function(ph) {
ph.createPage().then(function(page) {
page.open('https://example.com/').then(function(status) {
if ( status === "success" ) {
page.evaluate(function() {
document.querySelector("input[name='j_username']").value = 'example';
document.querySelector("input[name='j_password']").value = 'example';
console.log('Submitting logging...');
document.querySelector("input[name='submit']").click();
}).then(function(){
page.property('onResourceRequested', function(requestData) {
if (requestData.url.indexOf('geomaps.json') !== -1) {
console.log(JSON.stringify(datos));
request.get({
uri: requestData.url,
headers: requestData.headers
}, function(err, res, body){
if (!err) {
console.log(res.headers);
} else {
console.log("Error getting data from website: "+ err);
}
});
}
});
});
}
});
});
});
Same result. The console.log(JSON.stringify(datos)); is being fired but the request.get is never being fired.
I thing may have something to do with firing async funcions inside promises?
EDIT 21/07/2017 11:09 UTC
More tests. If I simplify the page.property('onResourceRequested' code block I see that the then() is called only once and before the requestData is received for each call...
I am a little bit confused and I don't know right now how to approach this...

I finally resolved the issue.
page.property('onResourceRequested' executes in the PhantomJS process. PhantomJS does not share any memory or variables with node. So using closures in javascript to share any variables outside of the function is not possible.
using page.on('onResourceRequested' instead fixed the issue!

Nodejs express and promises not doing what I expect

I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.

Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.

Node Js express async response with request and Redis call

So i'm having an issue with handling async actions in NodeJS while trying to send response to a request, with some async calls in the middle. (And to make this party even more complicated, i'm also want to use async.parallel )
Basically i'm trying to get value from Redis, and if he doesn't exist, get it from a provider (with request and response based using axios).
This is the code snippet :
this.getFixturesByTimeFrame = function (timeFrame, res) {
function getGamesData(timeFrame,finalCallback) {
var calls = [];
var readyList = [];
//Creating calls for parallel
timeFrame.forEach(function(startDay){
calls.push(function(callback) {
//Problematic async call
redisClient.get(startDay, function (error, exist) {
console.log('Got into the redis get!');
if (error){
console.log('Redis error : '+error);
}
if (exist) {
console.log('Date is in the cache! return it');
return exist;
} else {
//We need to fetch the data from the provider
console.log('Date is not in the cache, get it from the provider');
getFixturesDataFromProvider(startDay)
.then(organizeByLeagues)
.then(function (gamesForADay) {
redisClient.setex(startDay, 600, gamesForADay);
responsesList.add(gamesForADay);
callback(null, gamesForADay);
}).catch(function (response) {
if (response.status == 404) {
callback('Cant get games from provider');
}
});
}
});
}
)});
async.parallel(calls, function(err, responsesList) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err){
res.send(501);
} else {
console.log('Here is the final call, return the list here');
//Some data manipulation here - just some list ordering and for each loops
console.log('finished listing, send the list');
finalCallback(responsesList);
}
});
}
getGamesData(timeFrame, function (readyList) {
res.send(readyList);
});
};
function getFixturesDataFromProvider(date) {
var requestUrl = 'someURL/ + date;
return axios.get(requestUrl, config);
}
function organizeByLeagues(matchDay) {
if (matchDay.count == 0) {
console.log('No games in this day from the provider');
return [];
} else {
var organizedSet = [];
//Some list manipulations using for each
return organizedSet;
}
}
But the response is been sent before parallel has been starting doing his things...
i'm missing something with the callbacks and the async calls for sure but i'm not sure where...
Thanks

Async .eachLimit callback Error not called

I the following code in my node.js project.
async.eachLimit(dbresult, 1, function (record, callback) {
var json = JSON.stringify(record)
var form = new FormData()
form.append('data', json)
form.submit(cfg.server + '/external/api', function (err, res) {
if (err) {
callback(err)
}
if (res.statusCode === 200) {
connection.query('UPDATE selected_photos set synced = 1 WHERE selected_id = "' + record.selected_id + '"', function (err, result) {
if (err) {
console.log(err)
callback(err)
} else {
callback()
}
})
} else {
console.log(res.statusCode)
return callback(err)
}
})
}, function (err) {
// if any of the file processing produced an error, err would equal that error
if (err) {
// One of the iterations produced an error.
// All processing will now stop.
console.log('A share failed to process. Try rerunning the Offline Sync')
process.exit(0)
} else {
console.log('All files have been processed successfully')
process.exit(0)
}
})
}
res.statusCode = 302 So this should error out. But the the error callback is never triggered. How do it get it to trigger the error so that it stops eachLimit and the shows the
console.log('A share failed to process. Try rerunning the Offline Sync')

You have:
if (err) {
in first line of form submit handler. After that, you are sure that there was no error. So when you check response statusCode and try to call back with error, you are calling back with empty value.
That is why you do not get error when checking for it in your final callback function.
Try to create new Error('Status not OK: ' + res.statusCode) when calling back from your form submit handler.

Javascript unit testing, force error in sinon stub callback

I am trying to test one last bit of this function I cannot seem to hit correctly. Here is the function
CrowdControl.prototype.get = function(callback) {
var options = this.optionsFor('GET');
return q.Promise(function(resolve, reject) {
callback = callback || function callback(error, response, body) {
if (error) {
reject(error);
} else {
resolve(body);
}
};
callback();
request(options, callback);
});
};
And the part I can't seem to hit is
if (error) {
reject(error);
} else {
resolve(body);
}
I have the request set as a stub, and here is what I am trying
it("should call callback, which should reject if errors.", function() {
var testCallback = testHelpers.stub();
request.returns("Error");
crowdControl.get(testCallback);
//expect(testCallback).to.have.been.called;
expect(testCallback).to.have.been.calledWith("Error");
});
Seems like it is not working as I expected, I need to test the callback throwing an error. Thanks!