I have the code below, from a REST API, that inserts data in Mysql. I use Node and Express (this is, in fact, my 1st Node project, so please bear in mind I don't understand much about Node).
What I need is that response to client (browser, web application, Postman for testing or whatever access to the API) is returned only when the forEach loop and data insertion into DB terminates, so I get a JSON object with the list error messages, if any.
I've been hitting my head on the wall for half a day, and this is what I got so far.
var wait=require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
var errmsg = [];
router.route('/subscriber').post((req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
this.errmsg = [];
let asyncCall =
(async () => {
let rr = await new Promise (resolve => subscriberCollection.forEach(function (value, key){
var phoneNumber = value.phoneNumber;
var msg = "";
if (phoneNumber == ""){
msg = "ERROR","missing phoneNumber for subscriber index #" + key + ";Skipping";
console.log(msg);
errmsg[key] = msg
return;
}
var sql = "call insertSubscriber(?)";
console.log("INFO",`Inserting subscriber ${phoneNumber} index ${key}`);
connection.query(sql,[ phoneNumber ] ,function (err, data) {
if (err){
var msg = err.errno + " - " + err.sqlMessage;
console.log("ERROR" , msg);
errmsg[key] = msg;
}
});
}) //end forEach
); //end Promise
})();
asyncCall.then(console.log("ENDING!!") ); // THIS IS NOT WORKING
});
On the console, I get this:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ENDING!!
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
but what I need it to be is:
INFO Inserting 916311145 for index 0
INFO Inserting 916311146 for index 1
ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'
ERROR 1062 - Duplicate entry '916311146' for key 'phoneNumber_UNIQUE'
ENDING!!
Also, when all subscriber objects are saved on DB, I need to return a response to client, something like:
[{"key 0" : "ok"},{"key 1" : "ok"}, {"key 3": "ERROR 1062 - Duplicate entry '916311145' for key 'phoneNumber_UNIQUE'"}...]
and again, the response should only appear when all processing has finished.
How can I get this work?
Hmm try this:
var wait = require('wait.for');
var async = require('async');
var Promise = require('promise');
var Q = require('q');
router.route('/subscriber').post(async (req, res, callback) => {
const data = req.body;
var subscriberCollection = data;
const response = await Promise.all(
subscriberCollection.map(function (value, key) {
var phoneNumber = value.phoneNumber;
var msg = '';
const obj = {};
if (phoneNumber == '') {
msg = 'ERROR missing phoneNumber for subscriber index #' + key + ';Skipping';
console.log(msg);
obj[key] = msg;
Promise.resolve(obj);
return;
}
var sql = 'call insertSubscriber(?)';
console.log('INFO', `Inserting subscriber ${phoneNumber} index ${key}`);
return new Promise((resolve) => {
connection.query(sql, [phoneNumber], function (err, data) {
if (err) {
var msg = 'ERROR' + err.errno + ' - ' + err.sqlMessage;
console.log(msg);
obj[key] = msg;
resolve(obj);
return;
}
obj[key] = 'ok';
resolve(obj);
});
});
}) //end forEach
); //end Promise
console.log('ENDING!!');
res.send(response);
});
Related
I have a nodejs/express backend api that when I get hundreds of requests within seconds I start to get mixed results where data is crossed between requests and leads to unexpected and incorrected results. Can someone point me to where I am incorrectly defining variables that when I get one of the await functions it causes the other variables to be overwritten with the next requests data and causing my issues.
router.post('/autoscan3', async(req,res,next) => {
let authrule = "autoscan"
console.log(req.body.companyDomain + " " + req.body.user + " for folder: " + req.body.folderid)
let totalscans = 0;
let client = await getDB();
let authorizationResults = await checkAuthorization(client, req.body.companyDomain);
const featureSet = authorizationResults.features;
let company = authorizationResults.company;
const clientid = authorizationResults.client_id;
const clientsecret = authorizationResults.secret;
let tenantid = await getTenantID(req.body.companyDomain, client);
let token = await msgraphservices.getClientCredentialsAccessToken(tenantid, clientid, clientsecret)
let scanResults = []
let folderscans = 0;
try{
for(let a = 0; a < req.body.messages.length; a++){
let senderAddress = req.body.messages[a].senderAddress;
let emailBody = req.body.messages[a].emailBody;
let messageid = req.body.messages[a].messageid;
let receivedDateTime = req.body.messages[a].receivedDateTime;
let user = req.body.messages[a].recieverAddress;
let subject = req.body.messages[a].subject;
let attachments = req.body.messages[a].attachments;
let links = req.body.messages[a].emailBodyLinks;
let headers = req.body.messages[a].headers
let knownaddress
if (senderAddress.includes(req.body.companyDomain)) {
knownaddress = 10
} else {
knownaddress = await searchForSender(client, senderAddress, req.body.user, token, "");}
let assessmentResults = await assessment(
messageid,
emailBody,
senderAddress,
user,
subject,
receivedDateTime,
company,
attachments,
links,
req.body.companyDomain,
client,
headers,
knownaddress,
featureSet,
authrule
)
console.log('adding to folderscans')
folderscans++
try {
await msgraphservices.updateUserCategory2(messageid, req.body.user, req.body.folderid, assessmentResults.riskFactor,token)
}
catch(e) {
console.log(`error on category tag for ${messageid} with user ${req.body.user}`);
console.log(e);
}
}
console.log(`folder scans ${folderscans}`);
totalscans = totalscans + folderscans
return res.status(200).json({status:"success", totalscans:totalscans});
}
catch(e) {
console.log(`error while trying to loop for user ${req.body.user}`)
console.log(e)
logapierror(e.stack, req.body.user, req.body)
return res.status(200).json({status:'error', totalscans:totalscans, error:e});
}
});
It's very likely let client = await getDB();
If multiple requests use the same snapshot of the database but don't know about each other, it's very likely that they're overwriting each other's data.
This script is to get the title of the webpage where the URL of the website will be passed from an excel file, check to see if the title contains the keyword, and then store that domain in the new excel file.
There is no issue with the partial code, but the title comparison does not work as expected. Does anyone have an idea how to fix it?
here is my code
var request = require("request");
var cheerio = require("cheerio");
const xlsx = require("xlsx");
jsonData = [{ Domain: "blogger.com" }, { Domain: "stackoverflow.com" }];
function fetchTitle(url, onComplete = null) {
request(url, function (error, response, body) {
var output = url; // default to URL
if (!error && (response && response.statusCode) === 200) {
var $ = cheerio.load(body);
console.log(`URL = ${url}`);
var title = $("head > title").text().trim();
console.log(`Title = ${title}`);
output = `[${title}] (${url})`;
var keywords = ["Developers", "blog"];
var results = [];
var UrlArray = [];
for (var i = 0; i < keywords.length; i++) {
var match = title.match(new RegExp(keywords.join("|"), "g"));
results.push(keywords[i]);
}
if (match.length > 0) {
UrlArray.push({
Domain: url,
Keywords: results,
Title: output,
});
finalJsonData = JSON.stringify(UrlArray);
const ws = xlsx.utils.json_to_sheet(UrlArray);
const wb = xlsx.utils.book_new();
xlsx.utils.book_append_sheet(wb, ws, "Responses");
xlsx.writeFile(wb, "output.xlsx");
}
} else {
console.log(
`Error = ${error}, code = ${response && response.statusCode}`
);
}
console.log(`output = ${output} \n\n`);
if (onComplete) onComplete(output);
});
}
jsonData.forEach(function (table) {
var tableName = table.Domain;
var URL = "http://" + tableName;
fetchTitle(URL);
});
When I execute the script, I am able to get the title, but when I compare it with the keyword, it is not working as expected. Keywords are not being stored. You can see how the output looks after executing the script.
The script shows that both domains have keywords, but only blogger is stored in the spreadsheet, even then keywords aren't stored
you're overwriting the file on each loop,
keywords is an array, so it doesn't get saved, furthermore, keywords column will always contain all keywords, not the matching ones...
as requests are async, you need to track them all, and write results only when all requests are finished.
try this:
match case insensitive, and store only matching keywords for that site, not all (I also added "no match" for domains with no match)
store results outside the loop
move writing results into a separate function
add request counter and callback to track requests
write results when requests are done
the code:
var request = require("request");
var cheerio = require("cheerio");
const xlsx = require("xlsx");
const jsonData = [{ Domain: "blogger.com" }, { Domain: "stackoverflow.com" }];
var UrlArray = [];
function writeResults() {
const finalJsonData = JSON.stringify(UrlArray);
const ws = xlsx.utils.json_to_sheet(UrlArray);
const wb = xlsx.utils.book_new();
xlsx.utils.book_append_sheet(wb, ws, "Responses");
xlsx.writeFile(wb, "output.xlsx");
}
function fetchTitle(url, onComplete = null) {
request(url, function (error, response, body) {
var output = url; // default to URL
if (!error && (response && response.statusCode) === 200) {
var $ = cheerio.load(body);
console.log(`URL = ${url}`);
var title = $("head > title").text().trim();
console.log(`Title = ${title}`);
output = `[${title}] (${url})`;
var keywords = ["Developers", "blog"];
var results = [];
for (var i = 0; i < keywords.length; i++) {
let match = title.match(new RegExp(keywords[i], "gi"));
if (match && match.length > 0) {
results.push(keywords[i]);
}
}
UrlArray.push({
Domain: url,
Keywords: results.length > 0 ? results.join(', ') : 'no match',
Title: output,
});
} else {
console.log(
`Error = ${error}, code = ${response && response.statusCode}`
);
}
console.log(`output = ${output} \n\n`);
if (onComplete) onComplete(output);
});
}
let counter = 0;
jsonData.forEach(function (table) {
var tableName = table.Domain;
var URL = "http://" + tableName;
fetchTitle(URL, ()=>{
counter++;
if(counter === jsonData.length) {
console.log(`all ${counter} requests done`);
writeResults();
}
});
});
Been 2 days trying to find a solution to my problem.
I request data(json) from a website.
They return fine and json is valid but when i try to insert them to database almost 10% do not get inserted.
I dont know what to do, i even tried php with same results.
Any help world be appreciated thank you.
This is json ouptut after selecting the data attribute var result = obj.data; pastebin
var request = require("request");
var fs = require('fs');
var sleep = require('system-sleep');
var mysql = require('mysql');
var connection = mysql.createConnection({
host: 'localhost',
user: 'root',
password: '',
database: 'feeds',
timezone: 'Z'
});
request.post({
url: "xxx",
headers: {
"Content-Type": "application/json"
},
body: '{"xx": "true", "xxx": "true"}'
}, function(error, response, body) {
var obj = JSON.parse(body);
var result = obj.data;
console.log(result);
for (var i = 0; i < result.length; i++) {
var object = result[i];
for (property in object) {
var value = object[property];
var pid, pname;
if (property == "id") {
pid = value;
}
if (property == "name") {
pname = value;
}
if (property == "xxxxx") {}
if (property == "xxxxxxxx") {
connection.query('INSERT INTO tournaments(id, name) VALUES (' + pid + ', "' + pname + '")');
}
}
}
});
Welcome to SO.
There is probably an error that you are missing. You request the insertion but then let it go. connection.query allows a function as its second parameter, and that is a callback. Check out the following line
connection.query( 'SELECT * FROM some_table', ( err, rows ) => {
// do something with the results here
} );
For you it would look something like:
connection.query('INSERT INTO tournaments(id, name) VALUES (' + pid + ', "' + pname + '")', function(err, rows){
console.log('Successful write', rows);
console.error('Error! This did not write because: ', err);
})
You should console.log both err and rows, for each time this is called. Odds are you will now see why some of the results are not getting written into the DB, and they will show up as errors.
Hello all I have to need to add two field values {type:Number} of one collection from MongoDB using node js
and store the result in the same collection of MongoDB
1st node js query fetching the data value from MongoDB inside my controller.
2nd trying to add the fetched value.
3rd store the result in the same collection of MongoDB using node js.
1). Node js
var levelScoreQuiz = require('../models/levelscoreSchema.js');
try{
var queryObj = {};
var projection = '-id child.quiz_level.score_pre';
var projection2 = '-id child.quiz_level.score_curr';
var a = levelScoreQuiz.findOne(queryObj,projection);
var b = levelScoreQuiz.findOne(queryObj,projection2);
//console.log(a);
//console.log(b);
var add = a + b;
//console.log(add);
res.send(add);
var userObj = {
level_pre:req.params.add
};
var user = new levelScoreQuiz(userObj);
user.save(function(err, result){
if (err) {
console.log('Error While Saving the reuslt ' +err)}
else{
//console.log("User score saved successfully");
console.log("User Previous score saved successfully");
res.json(result);
}
});
}catch(err){
console.log('Error While Saving the reuslt ' +err);
return next(err);
}
2). MongoDB Schema
var userScore = new Schema({
child: {
quiz_level:{
current_level:{type:Number},
score_pre:{type:Number},
score_curr:{type:Number}
}
}
});
3). Result: it shows me object in my browser
"[object Object][object Object]"
var levelScoreQuiz = require('../models/levelscoreSchema.js');
try{
var queryObj = {};
var projection = {id: 0, 'child.quiz_level.score_pre': 1};
var projection2 = {id: 0, 'child.quiz_level.score_curr': 1};
var a = levelScoreQuiz.findOne(queryObj,projection);
var b = levelScoreQuiz.findOne(queryObj,projection2);
//console.log(a);
//console.log(b);
var add = a.child.quiz_level.score_pre +
b.child.quiz_level.score_curr;
//console.log(add);
res.send(add);
var userObj = {
child: {quiz_level: { score_pre: req.params.add}}
};
var user = new levelScoreQuiz(userObj);
user.save(function(err, result){
if (err) {
console.log('Error While Saving the reuslt ' +err)}
else{
//console.log("User score saved successfully");
console.log("User Previous score saved successfully");
res.json(result);
}
});
}catch(err){
console.log('Error While Saving the reuslt ' +err);
return next(err);
}
I am working on a project where I need to make multiple requests to a third party API and then send back an array of the data recieved to the client. Obviously the requests to the to the third party API are async, and if I put the res.json immediately after the request loop, the data will be empty. Do I need to wrap the request in a promise? Here is my code:
const historicalForecast = (req, res, next) => {
console.log(req.body);
// GET COORDS FROM GOOGLE API BY LOCATION INPUT BY USER
let googleUrl = `https://maps.googleapis.com/maps/api/geocode/json?address=${req.body.input}&key=${googleKey}`;
request(googleUrl, function(error, response, body){
if(error){
console.log(error);
next();
}
let data = JSON.parse(response.body);
//IF MORE THAN ONE RESULT FROM GEOLOCATION QUERY
//ADD DATES REQUESTED INTO RESPONSE AND
//SEND LIST OF LOCATIONS BACK SO USER CAN CHOOSE
if(data.results.length > 1){
response.body.startDate = req.body.startDate;
response.body.endDate = req.body.endDate;
res.json(JSON.parse(response.body));
//IF ONE RESULT, GET DATA IN BATCHES
}else if(data.results.length === 1) {
let coords = data.results[0].geometry.location;
const OneDay = 86400;
let timeFrame = Math.abs(req.body.startDate - req.body.endDate);
let numberOfDays = timeFrame/OneDay;
console.log(numberOfDays);
let results = [];
for(let i = 0; i < numberOfDays; i++){
let currentDay = Number(req.body.startDate) + (i*OneDay);
let urlWeather = `https://api.forecast.io/forecast/${weatherKey}/${coords.lat},${coords.lng},${currentDay}`;
request(urlWeather, function(error, response, body){
if(error){
console.log(error);
next();
}
results.push(JSON.parse(response.body));
res.send(results);
});
}
}
});
};
As per #jfriend00's suggestion I looked at:
Node.JS How to set a variable outside the current scope
And used one of the many options provided there. My solution is in the post above. Will be bookmarking that post for future reference.
I replaced all the code in the else if statement with:
let coords = data.results[0].geometry.location;
const OneDay = 86400;
let timeFrame = Math.abs(req.body.startDate - req.body.endDate);
let numberOfDays = timeFrame/OneDay;
const makeMultipleQueries = (url) => {
return new Promise(function(resolve, reject) {
request(url, function(error, response, body){
if(error){
reject(error);
}
resolve(response.body);
});
});
};
let promises = [];
for (let i = 0; i < numberOfDays; i++) {
let currentDay = Number(req.body.startDate) + (i*OneDay);
let url = `https://api.forecast.io/forecast/${weatherKey}/${coords.lat},${coords.lng},${currentDay}`;
promises.push(makeMultipleQueries(url));
}
Promise.all(promises).then(function(results) {
res.json(results);
}, function(err) {
next(err);
});