We Keep Coding

Why does my for loop mess up all the parameters?

I am trying to parse some data from several web pages using javascript. I wrote a small parser for this purpose. The algorithm looks like this:
Open first URL from my .csv file
Find the data I need on the page
Save URL and data to a json file
My code executes 1. and 2. perfectly but sometimes messes up with number 3. Output looks like this:
URL 1 + data from URL 1 (correct line)
URL 2 + data from URL 2 (correct line)
URL 3 + data from URL 3 (correct line)
URL 4 + data from URL 4 (correct line)
URL 6(wrong URL) + data from another URL
URL 5(wrong URL) + data from another URL
URL 7 + data from URL 7 (correct line)
URL 8 + data from URL 8 (correct line)
URL 9 + data from URL 9 (correct line)
I assume the problem is that some pages load way too long which messes up the whole process. But I still don't understand why it sometimes saves the wrong data.
Heres my code:
var request = require('request');
var cheerio = require('cheerio');
var cloudscraper = require('cloudscraper');
var fs = require('fs');
var path = require('path');
var csvjson = require('csvjson');
//First, we read .csv file with our URL list
function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
end = urlList.length;
logs = [];
//here we start the loop reading and saving data from each url
for (let p = 0; p < end; p += 1){
grabTheData(urlList, p)
}
}
//this code extracts the data from the page and saves it to a json file
function grabTheData(urlList, p){
setTimeout(function() {
url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
getTheList()

The reason this is happening is that there is a potential mismatch between the callback result and the url variable in grabTheData.
Now there is a very quick fix for this, simple change the scope of the url variable like so:
function grabTheData(urlList, p){
setTimeout(function() {
// Set scope of url variable to block
let url = url[p].ItemLink;
cloudscraper.get(url, function(err, res, body){
if (err) {
console.log(other.Time() + colors.yellow('Warn: ') + '- something went wrong with item ' + url);
callback();
} else {
var $ = cheerio.load(body);
/*
here are the lines which extract the data I need
dataIneed = ...;
*/
logs.push({
url, dataINeed
});
fs.writeFileSync('./logs.json', JSON.stringify(logs, null, 4));
}
});
//here I set a 2 seconds delay between each URL
}, 2000 * p);
}
This should keep your results in order.
Here's another (IMHO much better) option, using promises and avoiding the use of setTimeout to separate calls. This should avoid any potential race condition, since the Promise.all call will preserve order:
async function getTheList() {
urlList = fs.readFileSync(path.join(__dirname, 'data.csv'), { encoding : 'utf8'});
var options = {
delimiter : ';', // optional
quote : '"' // optional
};
urlList = csvjson.toObject(urlList, options);
let promiseList = urlList.map(urlEntry => grabTheDataUpdated(urlEntry.ItemLink));
let logs = await Promise.all(promiseList);
fs.writeFileSync('./new_logs.json', JSON.stringify(logs, null, 4));
}
// Promisified version of cloudscraper.get
function getCloudScraperData(url) {
return new Promise((resolve, reject) => {
cloudscraper.get(url, (err, res, body) => {
if (err) {
reject(err);
} else {
resolve ( { url, res, body });
}
})
})
}
function getDataINeed(url, body) {
// Use cheerio to process data..
// Return mock data for now.. replace with actual data processed by cheerio..
return `data from ${url}`;
}
async function grabTheDataUpdated(url) {
try {
let result = await getCloudScraperData(url);
let dataINeed = getDataINeed(result.url, result.body);
return { url, dataINeed };
} catch (error) {
return { url, dataINeed: "Error occurred: " + error.message };
}
}

Creating & sending JSON from client to server, and expected result

I'm trying to parse user input from my webpage and store it in a JSON object using inline JavaScript, make a POST request to my Node.js server, and access the contents of the request.
In my example.html, I have a function which does the following:
var xhttp = new XMLHttpRequest();
dataToSubmit = [];
// find some inputs
for ( /* each row of input */ ) {
dataToSubmit.push({
'item': itemName,
'quantity': quantity,
'price': itemPrice
});
}
xhttp.open("POST", "http://localhost:8080/menu", true);
xhttp.setRequestHeader('Content-Type', 'application/json');
xhttp.send(JSON.stringify(dataToSubmit));
EDIT :
After the POST request, I have a dispatcher.js file that processes the requests:
function(request, response) {
var qs = require('querystring');
var requestBody = '';
request.on('data', function(data) { requestBody += data; });
request.on('end', function() {
var qs = require('querystring');
var passed_data = qs.parse(requestBody);
if(request.url === "/menu") {
var menu_handler = require("./menu.js");
menu_handler.processOrder(passed_data);
}
}
I'm exporting processOrder() from my menu.js. The issue is that on the server-side, I have to do the following in order to access the object:
processOrder: function(data) {
for (var a in data) { <-------------- Having to do this seems incorrect
// a is a string, inputs is the expected object
var inputs = JSON.parse(a);
}
}
My question is: is the way I'm creating the JSON object incorrect, or is the way I'm accessing it on the server-side incorrect? My expectation is that, on the server-side, I should be able to do something like this:
processOrder: function(data) {
var inputs = JSON.parse(data);
for (var input in inputs) {
// access to input.item, input.quantity, input.price
}
}

Make dataToSubmit an object:
dataToSubmit = {};
For each input row, add a uniquely keyed property to your dataToSubmit, and assign it an object:
dataToSubmit['keyName' + index] = {}
Assign this new object properties like:
dataToSubmit['keyName' + index]['item'] = itemName;
dataToSubmit['keyName' + index]['quantity'] = quantity;
dataToSubmit['keyName' + index]['price'] = itemPrice;

The cause of me not being able to access the dataToSubmit variable as a JSON object was that I was doing parsing at a previous layer before the data reached the processOrder function. The solution was to make the following changes in my dispatcher.js file (which processes the requestBody before it makes its eventual way to menu.js):
function(request, response) {
var qs = require('querystring');
var requestBody = '';
request.on('data', function(data) { requestBody += data; });
request.on('end', function() {
var qs = require('querystring');
var passed_data;
if(request.headers['content-type'] === 'application/json') { <--- parse JSON data
passed_data = JSON.parse(requestBody);
else {
passed_data = qs.parse(requestBody);
}
if(request.url === "/menu") {
var menu_handler = require("./menu.js");
menu_handler.processOrder(passed_data);
}
}
Furthermore, when creating the JSON object, the following needed to be done in order to access the data as a JSON object rather than as an array:
dataToSubmit = {'content': []};
dataToSubmit['content'].push(
{
'item': itemName,
'quantity': quantity,
'price': itemPrice
}
);

Node.js http.request Results Back as Variable

All,
I am trying to figure out how to pass the results from an https.request in node.js code out to a variable. I have an https.request setup that correctly passes the correct information to a SOAP API and gets the correct response back. My ultimate goal is to get the output from the https.request into a variable that I can call using Express.
Here is are my code chunks.
HTML:
<div class="row">
<div class="col-md-12" class="pull-left">
<p> TEST </p>
<p>{{soapreply}}</p>
</div>
JS:
app.post('/cucmmapper/submit', function (req, res) {
// FORM - DATA COLLECTION
var cucmpub = req.body.cucmpub;
var cucmversion = req.body.cucmversion;
var username = req.body.username;
var password = req.body.password;
var authentication = username + ":" + password;
var soapreplyx = '';
// SOAP - BUILD CALL
var https = require("https");
var headers = {
'SoapAction': 'CUCM:DB ver=' + cucmversion + ' listCss',
'Authorization': 'Basic ' + new Buffer(authentication).toString('base64'),
'Content-Type': 'text/xml; charset=utf-8'
};
// SOAP - AXL CALL
var soapBody = new Buffer('<soapenv:Envelope xmlns:soapenv="http://schemas.xmlsoap.org/soap/envelope/" xmlns:ns="http://www.cisco.com/AXL/API/11.5">' +
'<soapenv:Header/>' +
'<soapenv:Body>' +
'<ns:listCss sequence="?">' +
'<searchCriteria>' +
'<name>%</name>' +
'</searchCriteria>' +
'<returnedTags uuid="?">' +
'<name>?</name>' +
'<description>?</description>' +
'<clause>?</clause>' +
'</returnedTags>' +
'</ns:listCss>' +
'</soapenv:Body>' +
'</soapenv:Envelope>');
// SOAP - OPTIONS
var options = {
host: cucmpub, // IP ADDRESS OF CUCM PUBLISHER
port: 8443, // DEFAULT CISCO SSL PORT
path: '/axl/', // AXL URL
method: 'POST', // AXL REQUIREMENT OF POST
headers: headers, // HEADER VAR
rejectUnauthorized: false // REQUIRED TO ACCEPT SELF-SIGNED CERTS
};
// SOAP - Doesn't seem to need this line, but it might be useful anyway for pooling?
options.agent = new https.Agent(options);
// SOAP - OPEN SESSION
var req = https.request(options, function (res) {
res.setEncoding('utf8');
res.on('data', function (d) {
soapreplyx = d;
console.log("Got Data: " + d);
});
});
// SOAP - SEND AXL CALL
req.write(soapBody);
res.render('cucmmapper-results.html'), {
'title': 'CUCM 2.1',
'soapreply': soapreplyx
};
req.end();
req.on('error', function (e) {
console.error(e);
});
});
}
The line "console.log("Got Data: " + d)" is getting the correct expected reply from the API, however, I can't figure out how to get that data into my variable "soapreplyx" which changes in Express to "soapreply".
Much appreciated for any help you might have!

You're not waiting for your request to respond before you call res.render(), so the value of soapreplyx is always '', its initial value. To correct this, add an 'end' event listener on the response object passed to your https.request() callback.
You're not appending the chunks of the response to your soapreplyx variable, you're reassigning its value with each successive chunk.
let soapRequest = https.request(options, soapResponse => {
soapResponse.on('data', chunk => {
soapreplyx += chunk
})
soapResponse.on('end', () => {
return res.render('cucmmapper-results.html', {
title: 'CUCM 2.1',
soapreply: soapreplyx
})
})
})
soapRequest.write(soapBody)
soapRequest.end()

Nodejs Module Exports TypeError

I am trying to determine what is wrong with my module setup that is throwing the following error:
[TypeError: transactionalEmails.request is not a function]`
It doesn't appear to be how the variable is being exported, but how I am creating the request function. Any help on what I'm missing or the mistake that I have made?
transactional-emails.js:
var transactionalEmails = function() {
var helper = require('sendgrid').mail;
request: function request(requester, receiver){
var fromEmail = new helper.Email(requester);
var toEmail = new helper.Email(receiver);
var subject = requester + ' has requested a message.';
var body = new helper.Content('text/plain', 'Hello Email');
var email = new helper.Mail(fromEmail, subject, toEmail, body);
var sg = require('sendgrid')(process.env.EMAIL_API_KEY);
var request = sg.emptyRequest({
method: 'POST',
path: '/v3/mail/send',
body: email.toJSON(),
});
return console.log(fromEmail + " Sending a request to " + toEmail);
/*sg.API(request, function(error, response) {
console.log(response.statusCode);
console.log(response.body);
console.log(response.headers);
});*/
}
};
module.exports = transactionalEmails;
Here is how I am calling the function:
var transactionalEmails = require('./transactional-emails');
transactionalEmails.request(req.user.email, req.body.receiverEmail);

You are exporting a function where you want to export an object.
var helper = require('sendgrid').mail;
var transactionalEmails = {
request: function request(requester, receiver){
...
}
};
module.exports = transactionalEmails;

Node.js - Can't send headers after they are sent

The closest issue I've found to mine is here. I believe I'm getting this error from how my .end() calls are set up. Here's the code we're working with:
app.get('/anihome',function(req,res){
var context = {};
function renderPage(context) {
res.render('anihome',context);
}
function addRequestToPage(text) {
context.data = text.toString('utf8');
context.info = JSON.parse(text);
return context;
}
function addAnimeToPage(text) {
context.anime = JSON.parse(text);
return context;
}
function addAnimeRequest(context) {
var options2 = {
host: 'anilist.co',
path: '/api/anime/20631?access_token=' + context.info.access_token,
method: 'GET'
};
https.request(options2, function(restRes) {
restRes.on('data',function(jsonResult) {
//context.anime = JSON.parse(jsonResult);
//console.log(JSON.parse(jsonResult));
console.log(context);
renderPage(context);
});
}).end();
}
function addHeaderRequest(context) {
var options = {
host: 'anilist.co',
path: '/api/auth/access_token?grant_type=client_credentials&client_id='
+ clientID + '&client_secret=' + secretKey,
method: 'POST'
};
https.request(options, function(restRes) {
restRes.on('data', function(jsonResult) {
context = addRequestToPage(jsonResult);
addAnimeRequest(context);
});
}).end();
}
addHeaderRequest(context);
});
I've tried setting up one of the .end()s with a callback, .end(addAnimeRequest(context));, which leaves me with a socket hang up error, so presumably something in my addAnimeRequest function is taking too long?
Is there a better way to make multiple requests to the same website with different options? I'm pretty new to Node.js.

The data event can be emitted more than once. You would need to add a listener for the end event and then pass in all of your data. Example:
https.request(options2, function(restRes) {
var buf = ''
restRes.on('data',function(jsonResult) {
//context.anime = JSON.parse(jsonResult);
//console.log(JSON.parse(jsonResult));
buf += jsonResult
});
restRes.on('end', function() {
// TODO JSON.parse can throw
var context = JSON.parse(buf)
renderPage(context)
})
}).end();