Node Js express async response with request and Redis call - javascript

So i'm having an issue with handling async actions in NodeJS while trying to send response to a request, with some async calls in the middle. (And to make this party even more complicated, i'm also want to use async.parallel )
Basically i'm trying to get value from Redis, and if he doesn't exist, get it from a provider (with request and response based using axios).
This is the code snippet :
this.getFixturesByTimeFrame = function (timeFrame, res) {
function getGamesData(timeFrame,finalCallback) {
var calls = [];
var readyList = [];
//Creating calls for parallel
timeFrame.forEach(function(startDay){
calls.push(function(callback) {
//Problematic async call
redisClient.get(startDay, function (error, exist) {
console.log('Got into the redis get!');
if (error){
console.log('Redis error : '+error);
}
if (exist) {
console.log('Date is in the cache! return it');
return exist;
} else {
//We need to fetch the data from the provider
console.log('Date is not in the cache, get it from the provider');
getFixturesDataFromProvider(startDay)
.then(organizeByLeagues)
.then(function (gamesForADay) {
redisClient.setex(startDay, 600, gamesForADay);
responsesList.add(gamesForADay);
callback(null, gamesForADay);
}).catch(function (response) {
if (response.status == 404) {
callback('Cant get games from provider');
}
});
}
});
}
)});
async.parallel(calls, function(err, responsesList) {
/* this code will run after all calls finished the job or
when any of the calls passes an error */
if (err){
res.send(501);
} else {
console.log('Here is the final call, return the list here');
//Some data manipulation here - just some list ordering and for each loops
console.log('finished listing, send the list');
finalCallback(responsesList);
}
});
}
getGamesData(timeFrame, function (readyList) {
res.send(readyList);
});
};
function getFixturesDataFromProvider(date) {
var requestUrl = 'someURL/ + date;
return axios.get(requestUrl, config);
}
function organizeByLeagues(matchDay) {
if (matchDay.count == 0) {
console.log('No games in this day from the provider');
return [];
} else {
var organizedSet = [];
//Some list manipulations using for each
return organizedSet;
}
}
But the response is been sent before parallel has been starting doing his things...
i'm missing something with the callbacks and the async calls for sure but i'm not sure where...
Thanks

Related

Node JS request blocking other requests

I have created Node JS app, i am explicitly downloading more than 100000 records from database. While the request is in proceeding, i try to login in the same application from another browser it wont respond unless the previous request is completed. Any idea ? Anything to do with event loop or threads ??
So here is my logic.
I make a get request to my API in Step 1, API calls database layer in Step 2. Database is returning only 21 records, i am explicitly looping 100000*21 to make heavy rendering just to test the load on json2csv. While doing so, the other requests to server will not respond until the last processing completes.
Step 1:
router.get('/report/downloadOverdueTrainings/:criteria', function (req, res, next) {
var overDueTrainings = [];
var reportManager = new ReportManager();
var result = reportManager.getOverdueTrainings(JSON.parse(req.params.criteria));
result.then(function (result) {
var fields = ['Over Due Trainings'];
for (var i = 0; i < 100000; i++) { //Testing purpose
for (var training of result) {
overDueTrainings.push({
'Over Due Trainings': training.OverDueTrainings
})
}
}
json2csv({
data: overDueTrainings,
fields: fields
}, function (err, csv) {
if (err)
throw err;
res.setHeader('Content-disposition', 'attachment; filename=OverdueTrainings.csv');
res.set('Content-Type', 'text/csv');
res.send(csv);
});
}).catch(function (err) {
next(err);
});
});
Step 2: Database Logic
var xtrDatabaseConnection =require('./xtrDatabaseConnection').XTRDatabaseConnection;
ReportData.prototype.getOverdueTrainings = async function (params) {
var connection = new xtrDatabaseConnection();
var sequelize = connection.getSequelize();
try {
var query = "CALL report_getOverdueTrainings(:p_CourseCode,:p_Revision,:p_RevisionNo,:p_UserGroup,:p_Username,:p_Status,:p_SortColoumnNo,:p_SortColoumnDirection,:p_callType,:p_StartIndex,:p_PageSize)";
var replacements = {
p_CourseCode: params.CourseCode,
p_Revision: params.Revision,
p_RevisionNo: (params.RevisionNo == '' || params.RevisionNo == null) ? 0 : params.RevisionNo,
p_UserGroup: params.UserGroup,
p_Username: params.Username,
p_Status: params.Status,
p_SortColoumnNo: params.SortColoumnNo,
p_SortColoumnDirection: params.SortColoumnDirection,
p_callType: params.callType,
p_StartIndex: params.startIndex,
p_PageSize: params.pageSize
};
//console.log(replacements);
return await connection.executeQuerySequelize(sequelize, query, Sequelize.QueryTypes.RAW, replacements);
} catch (e) {
throw (e);
} finally {
//To close connections
sequelize.connectionManager.close().then(() => console.log('Connection Closed'));
}
}
XTRDatabaseConnection.prototype.executeQuerySequelize = function (sequelize, query, queryType, replacements) {
return sequelize.authenticate()
.then(() => {
return sequelize.query(query, {
replacements: replacements,
type: queryType
}).
then(result => result)
.catch(err => {
throw (err);
});
})
.catch(err => {
xtrUtility.logErrorsToWinstonTransports('Unable to connect to the database or some error occurred while executing your query. ', err);
throw new AppError(err, "dbError", "Unable to connect to the database or some error occurred while executing your query.");
});
}
NodeJS is single-thread, single process. As long as a javascript function is running, nothing else can run. This is by design. The event loop will only kick in again once your functions have stopped executing.
This is blocking:
for (var i = 0; i < 100000; i++) { //Testing purpose
for (var training of result) {
overDueTrainings.push({
'Over Due Trainings': training.OverDueTrainings
})
}
}
until the execution is finished.
also
var result = reportManager.getOverdueTrainings(JSON.parse(req.params.criteria));
result.then(function (result) {
/****/
return await connection.executeQuerySequelize(sequelize, query, Sequelize.QueryTypes.RAW, replacements);
Node.js is working on single thread. So you need use it for small process with async calls. When you use async/await you are blocking the callback function.
So you need to use your database functions as promise. It will work async.
.query('CALL someFunction()')
.then(function(response){
//done
}).error(function(err){
//error
});
But there is a problem about it. You need to load 21 times your result. So in this case you need to use recursive function. When you finished recursive call you can send your data to client as async.

Node.js / Express.js can't get body response from API call via Request middleware

I'm trying to learn Node.js via Express.js framework. Currently I need to make an API call to get some data usefull for my app.
The API call is made with Request middleware, but when I'm out of the request my variable become undefined ... Let me show you :
var request = require('request');
var apiKey = "FOOFOO-FOOFOO-FOO-FOO-FOOFOO-FOOFOO";
var characters = [];
var gw2data;
var i = 0;
module.exports.account = function() {
request('https://api.guildwars2.com/v2/characters/?access_token=' + apiKey, function (error, response, body) {
gw2data = JSON.parse(body);
console.log('out request ' + gw2data); // {name1, name2 ...}
for (i; i < gw2data.length; i++) {
getCharacInfo(gw2data[i], i);
}
});
console.log('out request ' + characters); // undefined
return characters;
};
function getCharacInfo (name, position) {
request('https://api.guildwars2.com/v2/characters/' + name + '/?access_token=' + apiKey, function (error, response, body) {
if (!error && response.statusCode == 200) {
characters[position] = JSON.parse(body);
}
});
}
I don't understand why the gw2data variable become undefined when I go out of the request ... Someone can explain me ?
EDIT : I come to you because my problem has changed, I now need to make an API call loop in my first API call, same async problem I guess.
The previous code has evoluate with previous answers :
module.exports.account = function(cb) {
var request = require('request');
var apiKey = "FOOFOO-FOOFOO-FOO-FOO-FOOFOO-FOOFOO";
var characters = [];
var i = 0;
request('https://api.guildwars2.com/v2/characters/?access_token=' + apiKey, function(err, res, body) {
var numCompletedCalls = 1;
for (i; i < JSON.parse(body).length; i++) {
if (numCompletedCalls == JSON.parse(body).length) {
try {
console.log(characters); // {} empty array
return cb(null, characters);
} catch (e) {
return cb(e);
}
}else {
getCharacInfo(JSON.parse(body)[i]);
}
numCompletedCalls++;
}
});
};
function getCharacInfo (name) {
request('https://api.guildwars2.com/v2/characters/' + name + '/?access_token=' + apiKey, function (err, res, body) {
if (!err) {
console.log(characters); // {data ...}
characters.push(JSON.parse(body));
}
});
}
And the call of this function :
var express = require('express');
var router = express.Router();
router.get('/', function(req, res, next) {
var charactersInfos = require('../models/account');
charactersInfos.account(function(err, gw2data) {
if (err) {
console.error('request failed:', err.message);
} else {
res.render('index', {
persos : gw2data
});
}
});
});
The problem is, when I return the characters array, it's always an empty array, but when I check in my function getCharacInfo, the array contains the data ...
The reason gw2data is undefined in the second console.log is because your logging it too early. request is an asynchronous operation therefore it will return immediately...however, that doesn't mean the callback will.
So basically what your doing is logging gw2data before it's actually been set. When dealing with asynchronous operations the best approach is to generally make your own method asynchronous as well which can be accomplished in a couple of ways - the simplest being having your function accept a callback which expects the data in an asynchronous way e.g.
module.exports.account = function(cb) {
request(..., function(err, res, body) {
// return an error if `request` fails
if (err) return cb(err);
try {
return cb(null, JSON.parse(body));
} catch (e) {
// return an error if `JSON.parse` fails
return cb(e);
}
});
}
...
var myModule = require('mymodule');
myModule.account(function(err, g2wdata) {
if (err) {
console.error('request failed', err.message);
} else {
console.log('out request', g2wdata);
}
});
Based on your syntax I'm assuming you aren't working with ES6, worth looking at this (particularly if your just starting to learn). With built-in promises & also async-await support coming relatively soon this sort of stuff becomes relatively straightforward (at least at the calling end) e.g.
export default class MyModule {
account() {
// return a promise to the caller
return new Promise((resolve, reject) => {
// invoke request the same way but this time resolve/reject the promise
request(..., function(err, res, body) {
if (err) return reject(err);
try {
return resolve(JSON.parse(body));
} catch (e) {
return reject(e);
}
});
});
}
}
...
import myModule from 'mymodule';
// handle using promises
myModule.account()
.then(gw2data => console.log('out request', gw2data))
.catch(e => console.error('request failed', e));
// handle using ES7 async/await
// NOTE - self-executing function wrapper required for async support if using at top level,
// if using inside another function you can just mark that function as async instead
(async () => {
try {
const gw2data = await myModule.account();
console.log('out request', gw2data);
} catch (e) {
console.log('request failed', e);
}
})();
Finally, should you do decide to go down the Promise route, there are a couple of libraries out there that can polyfill Promise support into existing libraries. One example I can think of is Bluebird, which from experience I know works with request. Combine that with ES6 you get a pretty neat developer experience.
The request API is asynchronous, as is pretty much all I/O in Node.js. Check out the Promise API which is IMO the simplest way to write async code. There are a few Promise adapters for request, or you could use something like superagent with promise support.

Nodejs express and promises not doing what I expect

I am trying to build a login API using NodeJS, but my code is not doing what I expect it to. I am very new to js, promises and all so please simplify any answer if possible.
From what I can see in the output of my code, the first promise part does not wait until the function findUsers(...) is finished.
I have a routes file where I want to run a few functions sequentially:
Find if user exist in database
if(1 is true) Hash and salt the inputted password
... etc
The routes file now contains:
var loginM = require('../models/login');
var loginC = require('../controllers/login');
var Promise = require('promise');
module.exports = function(app) {
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
var rows = loginM.findUser(req.body, res);
if (rows.length > 0) {
console.log("Success");
resolve(rows);
} else {
console.log("Failed");
reject(reason);
}
});
promise.then(function(data) {
return new Promise(function (resolve, reject) {
loginC.doSomething(data);
if (success) {
console.log("Success 2");
resolve(data);
} else {
console.log("Failed 2");
reject(reason);
}
});
}, function (reason) {
console.log("error handler second");
});
});
}
And the findUser function contains pooling and a query and is in a models file:
var connection = require('../dbConnection');
var loginC = require('../controllers/login');
function Login() {
var me = this;
var pool = connection.getPool();
me.findUser = function(params, res) {
var username = params.username;
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
}
}
module.exports = new Login();
The output i get is:
Server listening on port 3000
Something is happening
error handler second
Connection
So what I want to know about this code is twofold:
Why is the first promise not waiting for findUser to return before proceeding with the if/else and what do I need to change for this to happen?
Why is error handler second outputed but not Failed?
I feel like there is something I am totally misunderstanding about promises.
I am grateful for any answer. Thanks.
Issues with the code
Ok, there are a lot of issues here so first things first.
connection.query('...', function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
This will not work because you are returning data to the caller, which is the database query that calls your callback with err and rows and doesn't care about the return value of your callback.
What you need to do is to call some other function or method when you have the rows or when you don't.
You are calling:
var rows = loginM.findUser(req.body, res);
and you expect to get the rows there, but you won't. What you'll get is undefined and you'll get it quicker than the database query is even started. It works like this:
me.findUser = function(params, res) {
// (1) you save the username in a variable
var username = params.username;
// (2) you pass a function to getConnection method
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
console.log("ERROR 1 ");
res.send({"code": 100, "status": "Error in connection database"});
return;
}
connection.query('select Id, Name, Password from Users ' +
'where Users.Name = ?', [username], function (err, rows) {
connection.release();
if (!err) {
return rows;
} else {
return false;
}
});
//connection.on('error', function (err) {
// res.send({"code": 100, "status": "Error in connection database"});
// return;
//});
});
// (3) you end a function and implicitly return undefined
}
The pool.getConnection method returns immediately after you pass a function, before the connection to the database is even made. Then, after some time, that function that you passed to that method may get called, but it will be long after you already returned undefined to the code that wanted a value in:
var rows = loginM.findUser(req.body, res);
Instead of returning values from callbacks you need to call some other functions or methods from them (like some callbacks that you need to call, or a method to resolve a promise).
Returning a value is a synchronous concept and will not work for asynchronous code.
How promises should be used
Now, if your function returned a promise:
me.findUser = function(params, res) {
var username = params.username;
return new Promise(function (res, rej) {
pool.getConnection(function (err, connection) {
console.log("Connection ");
if (err) {
rej('db error');
} else {
connection.query('...', [username], function (err, rows) {
connection.release();
if (!err) {
res(rows);
} else {
rej('other error');
}
});
});
});
}
then you'll be able to use it in some other part of your code in a way like this:
app.post('/login/', function(req, res, next) {
var promise = new Promise(function (resolve, reject) {
// rows is a promise now:
var rows = loginM.findUser(req.body, res);
rows.then(function (rowsValue) {
console.log("Success");
resolve(rowsValue);
}).catch(function (err) {
console.log("Failed");
reject(err);
});
});
// ...
Explanation
In summary, if you are running an asynchronous operation - like a database query - then you can't have the value immediately like this:
var value = query();
because the server would need to block waiting for the database before it could execute the assignment - and this is what happens in every language with synchronous, blocking I/O (that's why you need to have threads in those languages so that other things can be done while that thread is blocked).
In Node you can either use a callback function that you pass to the asynchronous function to get called when it has data:
query(function (error, data) {
if (error) {
// we have error
} else {
// we have data
}
});
otherCode();
Or you can get a promise:
var promise = query();
promise.then(function (data) {
// we have data
}).catch(function (error) {
// we have error
});
otherCode();
But in both cases otherCode() will be run immediately after registering your callback or promise handlers, before the query has any data - that is no blocking has to be done.
Summary
The whole idea is that in an asynchronous, non-blocking, single-threaded environment like Node.JS you never do more than one thing at a time - but you can wait for a lot of things. But you don't just wait for something and do nothing while you're waiting, you schedule other things, wait for more things, and eventually you get called back when it's ready.
Actually I wrote a short story on Medium to illustrate that concept: Nonblacking I/O on the planet Asynchronia256/16 - A short story loosely based on uncertain facts.

Javascript asynchronous callbacks and forEach [duplicate]

In my code below, 5 always prints before 4. I thought because the callback to postUsers was in a return statement from matchAgainstAD it would wait for the for loop and ad lookup to complete before returning. How can I do this in the most simple way?
var matchAgainstAD = function(stUsers) {
stUsers.forEach(function (element, i) {
var sAMAccountName = stUsers[i].guiLoginName;
// Find user by a sAMAccountName
var ad = new ActiveDirectory(config);
ad.findUser(sAMAccountName, function(err, user) {
if (err) {
console.log('ERROR: ' +JSON.stringify(err));
return;
}
if (!user) {
staleUsers.push(stUsers[i])
console.log(4)
}
// console.log(staleUsers);
});
})
return postUsers(staleUsers)
}
var postUsers = function(staleUsers) {
console.log(5);
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://localhost:8000/api/record/newRecord',
qs: staleUsers
}, function(err, res, body) {
// console.log(body);
})
}
matchAgainstAD();
This is very classic asynchronous problem in node.js. Your findUser() function has an asynchronous response which means the callback is called sometime later. Meanwhile, the rest of your loop continues to run so that all the requests are in-flight at the same time and then the responses start coming in sometime later. Thus, you can't call postUsers() after matchAgainstAd() returns because the inner async operations are not yet completed and thus staleUsers is not yet populated.
There are multiple approaches to solving this problem. In general, it is worth learning how to use promises for operations like this because they offer all sorts of very useful flow of control options when using asynchronous operations and node.js does pretty much all I/O operations as async. But, to best illustrate what's going on in this type of issue, I'll first show you a manually coded solution. In this manually coded solution, you keep track of how many operations are still remaining to complete and when all have completed, then and only then, do you call postUsers() with the accumulated data.
Manually Coded Solution
var matchAgainstAD = function (stUsers) {
var remaining = stUsers.length;
stUsers.forEach(function (element, i) {
var sAMAccountName = stUsers[i].guiLoginName;
function checkDone() {
if (remaining === 0) {
postUsers(staleUsers);
}
}
// Find user by a sAMAccountName
var ad = new ActiveDirectory(config);
ad.findUser(sAMAccountName, function (err, user) {
--remaining;
if (err) {
console.log('ERROR: ' + JSON.stringify(err));
checkDone();
return;
}
if (!user) {
staleUsers.push(stUsers[i])
}
checkDone();
});
});
}
var postUsers = function(staleUsers) {
request.post({
headers: {'content-type' : 'application/x-www-form-urlencoded'},
url: 'http://localhost:8000/api/record/newRecord',
qs: staleUsers
}, function(err, res, body) {
// console.log(body);
})
}
The core logic here is that you initialize a counter to the number of operations that will be carried out. Then, in your loop where each operation is happening, you decrement the remaining counter anytime one of the operations completes (calls it's completion callback). Then, after processing the result (in both the success and error code paths), you check if the remaining count has reached 0 indicating that all requests are now done. If so, then the staleUsers array is now fully populated and you can call postUsers(staleUsers) to process the accumulated result.
Solution Coded Using Bluebird Promises
The idea here is that we use the control flow logic and enhanced error handling of promises to manage the async control flow. This is done by "promisifying" each asynchronous interface we are using here. "promisifying" is the process of creating a small function wrapper around any async function that follows the node.js calling convention where the last argument to the function is a callback that takes at least two arguments, the first an error and the second a value. This can be automatically turned into a wrapper function that returns a promise, allow using to use promise logic flow with any normal async operation.
Here's how this would work using the bluebird promise library.
var Promise = require('bluebird');
var request = Promise.promisifyAll(request('require'));
var matchAgainstAD = function (stUsers) {
var staleUsers = [];
var ad = new ActiveDirectory(config);
// get promisified version of findUser
var findUser = Promise.promisify(ad.findUser, ad);
return Promise.map(stUsers, function(userToSearchFor) {
var sAMAccountName = userToSearchFor.guiLoginName;
return findUser(sAMAccountName).then(function(user) {
// if no user found, then consider it a staleUser
if (!user) {
staleusers.push(userToSearchFor);
}
}, function(err) {
// purposely skip any requests we get an error on
// having an error handler that does nothing will
// stop promise propagation of the error (it will be considered "handled")
});
}).then(function() {
if (staleUsers.length) {
return postUsers(staleUsers);
}
return 0;
});
}
var postUsers = function (staleUsers) {
return request.postAsync({
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
url: 'http://localhost:8000/api/record/newRecord',
qs: staleUsers
}).spread(function (res, body) {
// console.log(body);
return staleUsers.length;
})
}
matchAgainstAD(users).then(function(qtyStale) {
// success here
}, function(err) {
// error here
})
Standard ES6 Promises Version
And, here's a version that uses only the standard ES6 promises built-into node.js. The main difference here is that you have to code your own promisified versions of the async functions you want to use because you can't use the built-in promisify capabilities in Bluebird.
var request = request('require');
// make a promisified version of request.post
function requestPostPromise(options) {
return new Promise(function(resolve, reject) {
request.post(options, function(err, res, body) {
if (err) {
reject(err);
} else {
resolve([res, body]);
}
});
});
}
// make a function that gets a promisified version of ad.findUser
function getfindUserPromise(ad) {
return function(name) {
return new Promise(function(resolve, reject) {
ad.findUser(name, function(err, user) {
if (err) {
reject(err);
} else {
resolve(user);
}
});
});
}
}
var matchAgainstAD = function (stUsers) {
var staleUsers = [];
var promises = [];
var ad = new ActiveDirectory(config);
// get promisified version of findUser
var findUser = getFindUserPromise(ad);
stUsers.each(function(userToSearchFor) {
promises.push(findUser(userToSearchFor.guiLoginName).then(function(user) {
// if no user found, then consider it a staleUser
if (!user) {
staleusers.push(userToSearchFor);
}
}, function(err) {
// purposely skip any requests we get an error on
// have an error handler that does nothing will
// stop promise propagation of the error (it will be considered "handled")
}));
});
return Promise.all(promises).then(function() {
if (staleUsers.length) {
return postUsers(staleUsers);
}
return 0;
});
}
var postUsers = function (staleUsers) {
return requestPostPromise({
headers: {
'content-type': 'application/x-www-form-urlencoded'
},
url: 'http://localhost:8000/api/record/newRecord',
qs: staleUsers
}).then(function (err, results) {
var res = results[0];
var body = results[1];
// console.log(body);
return staleUsers.length;
})
}
matchAgainstAD(users).then(function(qtyStale) {
// success here
}, function(err) {
// error here
})
ad.findUser takes a callback that contains the console.log(4). That function is async, and will hit your callback when the IO operation has completed.
On the other hand, postUsers is called completely synchronously, so it will hit console.log(5) before ad.findUser enters your callback.
A simple way to fix this is to call postUsers from inside of your ad.findUser callback.
I would suggest looking into the promise pattern for JavaScript to manage dependencies between async operations. There are several popular libraries (Q and RSVSP.js being a couple of them).

Sequelize JS node updating multiple entities with new data from previous promise

I am new to sequelize and NodeJS promises in general. My app basically saves tweets from the Twitter API, but also needs to update some of the saved tweets' data in realtime, like the Retweet count or the Liked count.
But it seems like after fetching the new data, when trying to run all the update on my tweet instances, nothing happens. The promise doesn't go through.
To sum up : I find 100 saved tweets, chain on a callback that fetches their new data from Twitter, and then chain on updating every single 100 tweets with the new data. The later update doesn't go through.
var Sequelize = require('sequelize');
...
//Getting 100 tweets previously saved in DB
Sequelize.query("SELECT * FROM tweets WHERE ORDER BY id DESC LIMIT 100", { model: Model }).then(function(result) {
if(result.length == 0) {
callback(false);
} else {
var ids = [];
var realData = {};
for (var i in result) {
realData[result[i].dataValues.id_str] = result[i];
ids.push(result[i].dataValues.id_str);
}
//getting twitter data for 100 tweets previously saved in DB
twitVendor.get('statuses/lookup', {
id : ids.join(',')
}, function (err, tweets, response) {
if (typeof err == 'undefined') {
//to get a synchronous saving of all tweets
//this could be cleaned up with a Sequelize.Promise.push(...)
var i = 0;
var saving = false;
while (i < tweets.length) {
if (!saving) {
saving = true;
console.log('Updating tweet ', tweets[i].id_str);
//updating tweet with new data from twitter
Sequelize.query("UPDATE tweets SET retweet_count = "+tweets[i].retweet_count+", favorite_count = "+tweets[i].favorite_count+" WHERE id_str = '"+tweets[i].id_str+"'", {
model: Model
}).then(function(result) {
console.log('Updated tweet');
saving = false;
i++;
}).catch(function (err) {
console.log('Failed to update post ', err);
saving = false;
i++;
});
}
}
callback(true);
console.log("Updated tweets");
} else {
console.log("Failed :", err);
callback(false, err);
}
});
}
}).catch(function (err) {
console.log("Failed :", err);
callback(false, err);
})
EDIT : If you want to execute the above code, I'd recommend using this Twit to hit the Twitter API : https://github.com/ttezel/twit
To get credentials to hit the API, you will need to set up an app on Twitter though : https://apps.twitter.com/
EDIT 2 : I already tried to use transactions and pure Sequelize functions to make my queries, but the issue stood still.
Don't nest promises inside of promises. Instead, chain them by returning promises. If you are returning something that is not a promise, use Promise.resolve(value) to turn it into a promise. And certainly don't put promises inside of callbacks, or even mix them at all; instead create a promise that calls the action, and then in the callback resolve the promise.
Here's my attempt to rewrite what you're trying to do. You may need to wrap the first in a Promise.resolve to take advantage of returning the new promise:
Sequelize.query("SELECT * FROM tweets WHERE ORDER BY id DESC LIMIT 100"
, { model: Model }).then(function (results) {
if (results.length == 0) {
return Promise.reject(false); //reject to hit the catch of the promise. Change false to error.
}
var ids = [];
var realData = {};
for (var i in result) {
realData[result[i].dataValues.id_str] = result[i];
ids.push(result[i].dataValues.id_str);
}
return new Promise((resolve, reject) => {
twitVendor.get('status/lookup', {
id: ids.join(',')
}, function (err, tweets, response) {
if (err) {
reject(false); //reject to hit the catch of the promise. Change false to error message
}
resolve(tweets);
})
})
}).then(function (tweets) {
function updateTweet(tweet) {
return sequelize.query(...);
}
var updatesInParallel = tweets.map(updateTweet);
return Promise.all([updatesInParallel]);
}).then(function () {
callback(true);
}).catch(function (error) {
console.log("failed: ", error);
callback(false)
});

Categories